def process_remote(self, done):
time.sleep(0.05) # file upload
self.remote_task = OdmTaskMock(nonloc.remote_queue <= MAX_QUEUE, nonloc.remote_queue)
self.params['tasks'].append(self.remote_task)
if nonloc.should_fail:
if self.project_path.endswith("0006"):
raise exceptions.TaskFailedError("FAIL #6")
nonloc.remote_queue += 1
# Upload successful
done(error=None, partial=True)
# Async processing
def monitor():
try:
if nonloc.task_limit_reached and random.randint(0, 4) == 0:
nonloc.remote_queue -= 1
raise NodeTaskLimitReachedException("Random fail!")
if not nonloc.task_limit_reached and self.remote_task.queue_num > MAX_QUEUE:
nonloc.remote_queue -= 1
nonloc.task_limit_reached = True
raise NodeTaskLimitReachedException("Delayed task limit reached")
time.sleep(0.5)
nonloc.remote_queue -= 1
done()
except Exception as e:
done(e)
t = threading.Thread(target=monitor)
self.params['threads'].append(t)
t.start()
def monitor():
class nonloc:
status_callback_calls = 0
last_update = 0
def status_callback(info):
# If a task switches from RUNNING to QUEUED, then we need to
# stop the process and re-add the task to the queue.
if info.status == TaskStatus.QUEUED:
log.ODM_WARNING(
"LRE: %s (%s) turned from RUNNING to QUEUED. Re-adding to back of the queue."
% (self, task.uuid))
raise NodeTaskLimitReachedException(
"Delayed task limit reached")
elif info.status == TaskStatus.RUNNING:
# Print a status message once in a while
nonloc.status_callback_calls += 1
if nonloc.status_callback_calls > 30:
log.ODM_INFO("LRE: %s (%s) is still running" %
(self, task.uuid))
nonloc.status_callback_calls = 0
try:
def print_progress(percentage):
if (time.time() - nonloc.last_update >=
2) or int(percentage) == 100:
log.ODM_INFO("LRE: Download of %s at [%s%%]" %
(self, int(percentage)))
nonloc.last_update = time.time()
task.wait_for_completion(status_callback=status_callback)
log.ODM_INFO("LRE: Downloading assets for %s" % self)
task.download_assets(self.project_path,
progress_callback=print_progress)
log.ODM_INFO(
"LRE: Downloaded and extracted assets for %s" % self)
done()
except exceptions.TaskFailedError as e:
# Try to get output
try:
output_lines = task.output()
# Save to file
error_log_path = self.path("error.log")
with open(error_log_path, 'w') as f:
f.write('\n'.join(output_lines) + '\n')
msg = "(%s) failed with task output: %s\nFull log saved at %s" % (
task.uuid, "\n".join(
output_lines[-10:]), error_log_path)
done(exceptions.TaskFailedError(msg))
except:
log.ODM_WARNING(
"LRE: Could not retrieve task output for %s (%s)" %
(self, task.uuid))
done(e)
except Exception as e:
done(e)
def process_local(self):
# First task should be 0000 or 0001
if not nonloc.local_task_check: nonloc.local_task_check = self.project_path.endswith("0000") or self.project_path.endswith("0001")
if nonloc.should_fail:
if self.project_path.endswith("0006"):
raise exceptions.TaskFailedError("FAIL #6")
time.sleep(1)
def monitor():
class nonloc:
status_callback_calls = 0
def status_callback(info):
# If a task switches from RUNNING to QUEUED, then we need to
# stop the process and re-add the task to the queue.
if info.status == TaskStatus.QUEUED:
log.ODM_WARNING("LRE: %s (%s) turned from RUNNING to QUEUED. Re-adding to back of the queue." % (self, task.uuid))
raise NodeTaskLimitReachedException("Delayed task limit reached")
elif info.status == TaskStatus.RUNNING:
# Print a status message once in a while
nonloc.status_callback_calls += 1
if nonloc.status_callback_calls > 30:
log.ODM_DEBUG("LRE: %s (%s) is still running" % (self, task.uuid))
nonloc.status_callback_calls = 0
try:
def print_progress(percentage):
if percentage % 10 == 0:
log.ODM_DEBUG("LRE: Download of %s at [%s%%]" % (self, int(percentage)))
task.wait_for_completion(status_callback=status_callback)
log.ODM_DEBUG("LRE: Downloading assets for %s" % self)
task.download_assets(self.project_path, progress_callback=print_progress)
log.ODM_DEBUG("LRE: Downloaded and extracted assets for %s" % self)
done()
except exceptions.TaskFailedError as e:
# Try to get output
try:
msg = "(%s) failed with task output: %s" % (task.uuid, "\n".join(task.output()[-10:]))
done(exceptions.TaskFailedError(msg))
except:
log.ODM_WARNING("LRE: Could not retrieve task output for %s (%s)" % (self, task.uuid))
done(e)
except Exception as e:
done(e)