def process_remote(self, done): time.sleep(0.05) # file upload self.remote_task = OdmTaskMock(nonloc.remote_queue <= MAX_QUEUE, nonloc.remote_queue) self.params['tasks'].append(self.remote_task) if nonloc.should_fail: if self.project_path.endswith("0006"): raise exceptions.TaskFailedError("FAIL #6") nonloc.remote_queue += 1 # Upload successful done(error=None, partial=True) # Async processing def monitor(): try: if nonloc.task_limit_reached and random.randint(0, 4) == 0: nonloc.remote_queue -= 1 raise NodeTaskLimitReachedException("Random fail!") if not nonloc.task_limit_reached and self.remote_task.queue_num > MAX_QUEUE: nonloc.remote_queue -= 1 nonloc.task_limit_reached = True raise NodeTaskLimitReachedException("Delayed task limit reached") time.sleep(0.5) nonloc.remote_queue -= 1 done() except Exception as e: done(e) t = threading.Thread(target=monitor) self.params['threads'].append(t) t.start()
def monitor(): class nonloc: status_callback_calls = 0 last_update = 0 def status_callback(info): # If a task switches from RUNNING to QUEUED, then we need to # stop the process and re-add the task to the queue. if info.status == TaskStatus.QUEUED: log.ODM_WARNING( "LRE: %s (%s) turned from RUNNING to QUEUED. Re-adding to back of the queue." % (self, task.uuid)) raise NodeTaskLimitReachedException( "Delayed task limit reached") elif info.status == TaskStatus.RUNNING: # Print a status message once in a while nonloc.status_callback_calls += 1 if nonloc.status_callback_calls > 30: log.ODM_INFO("LRE: %s (%s) is still running" % (self, task.uuid)) nonloc.status_callback_calls = 0 try: def print_progress(percentage): if (time.time() - nonloc.last_update >= 2) or int(percentage) == 100: log.ODM_INFO("LRE: Download of %s at [%s%%]" % (self, int(percentage))) nonloc.last_update = time.time() task.wait_for_completion(status_callback=status_callback) log.ODM_INFO("LRE: Downloading assets for %s" % self) task.download_assets(self.project_path, progress_callback=print_progress) log.ODM_INFO( "LRE: Downloaded and extracted assets for %s" % self) done() except exceptions.TaskFailedError as e: # Try to get output try: output_lines = task.output() # Save to file error_log_path = self.path("error.log") with open(error_log_path, 'w') as f: f.write('\n'.join(output_lines) + '\n') msg = "(%s) failed with task output: %s\nFull log saved at %s" % ( task.uuid, "\n".join( output_lines[-10:]), error_log_path) done(exceptions.TaskFailedError(msg)) except: log.ODM_WARNING( "LRE: Could not retrieve task output for %s (%s)" % (self, task.uuid)) done(e) except Exception as e: done(e)
def process_local(self): # First task should be 0000 or 0001 if not nonloc.local_task_check: nonloc.local_task_check = self.project_path.endswith("0000") or self.project_path.endswith("0001") if nonloc.should_fail: if self.project_path.endswith("0006"): raise exceptions.TaskFailedError("FAIL #6") time.sleep(1)
def monitor(): class nonloc: status_callback_calls = 0 def status_callback(info): # If a task switches from RUNNING to QUEUED, then we need to # stop the process and re-add the task to the queue. if info.status == TaskStatus.QUEUED: log.ODM_WARNING("LRE: %s (%s) turned from RUNNING to QUEUED. Re-adding to back of the queue." % (self, task.uuid)) raise NodeTaskLimitReachedException("Delayed task limit reached") elif info.status == TaskStatus.RUNNING: # Print a status message once in a while nonloc.status_callback_calls += 1 if nonloc.status_callback_calls > 30: log.ODM_DEBUG("LRE: %s (%s) is still running" % (self, task.uuid)) nonloc.status_callback_calls = 0 try: def print_progress(percentage): if percentage % 10 == 0: log.ODM_DEBUG("LRE: Download of %s at [%s%%]" % (self, int(percentage))) task.wait_for_completion(status_callback=status_callback) log.ODM_DEBUG("LRE: Downloading assets for %s" % self) task.download_assets(self.project_path, progress_callback=print_progress) log.ODM_DEBUG("LRE: Downloaded and extracted assets for %s" % self) done() except exceptions.TaskFailedError as e: # Try to get output try: msg = "(%s) failed with task output: %s" % (task.uuid, "\n".join(task.output()[-10:])) done(exceptions.TaskFailedError(msg)) except: log.ODM_WARNING("LRE: Could not retrieve task output for %s (%s)" % (self, task.uuid)) done(e) except Exception as e: done(e)