def testSimple(self): """Tests simple waiting.""" self.worker.start() one = timer_worker.TimerItem(0.8) two = timer_worker.TimerItem(0.5) three = timer_worker.TimerItem(0.1) # T = 0, one = 0 begin = time.time() self.timer_queue.put(one) time.sleep(0.2) # T = 0.2, one = 0.2, two = 0 self.timer_queue.put(two) time.sleep(0.2) # T = 0.4, one = 0.4, two = 0.2 self.timer_queue.put(three) time.sleep(0.2) # T = 0.6, one = 0.6, two = 0.4, three = 0.1 ready! output_three = self.output_queue.get() time.sleep(0.1) # T = 0.7, one = 0.7, two = 0.5 ready! output_two = self.output_queue.get() # T = 0.8, one = 0.8 ready! output_one = self.output_queue.get() end = time.time() self.assertEquals(one.delay_seconds, output_one.delay_seconds) self.assertEquals(two.delay_seconds, output_two.delay_seconds) self.assertEquals(three.delay_seconds, output_three.delay_seconds) elapsed = end - begin self.assertTrue(1.0 > elapsed > 0.7)
def run(self): jobs = [] for i in xrange(3): job = EchoChildWorkflow(99, should_die=True, wait_seconds=i * 0.5) job.fire_and_forget = True result = yield job assert result is job assert not result.done assert not result.error jobs.append(job) yield timer_worker.TimerItem(0.5) assert jobs[0].done assert jobs[1].done is False assert jobs[2].done is False # for i, job in enumerate(jobs): # print '%d is done %r' % (i, job.done) # # assert str(job.error[1]) == 'Dying on 99' yield timer_worker.TimerItem(1.5) assert jobs[0].done assert jobs[1].done assert jobs[2].done raise workers.Return('All errors seen')
def run(self, tmp_dir, waitfor, heartbeat=None, start_time=None): assert 'url' in waitfor timeout = waitfor.get('timeout_secs', 10) if not start_time: start_time = time.time() class NotReadyError(Exception): pass try: url = waitfor['url'] r = requests.head(url, allow_redirects=True) if r.status_code != 200: yield heartbeat('Request for %s failed (%d)' % (url, r.status_code)) raise NotReadyError() yield heartbeat( 'Request for %s succeeded, continuing with tests...' % url) return except (requests.ConnectionError, NotReadyError): now = time.time() if now - start_time >= timeout: raise process_worker.TimeoutError() yield timer_worker.TimerItem(0.5) # wait 500ms between checks yield WaitForUrlWorkflowItem(tmp_dir, waitfor, heartbeat, start_time)
def run(self): output = yield workers.WaitAny([ EchoItem(10), EchoChild(42), EchoItem(2), EchoItem(25), ]) # At least one EchoItem will be done. We don't know exactly because # the jobs WorkflowItems in WaitAny are inserted into a dictionary # so their completion ordering is non-deterministic. assert len([x for x in output if x.done]) >= 1 # The EchoChild will not be ready yet. assert not output[1].done yield timer_worker.TimerItem(2) results = yield output # Now everything will be done. assert len([x for x in output if x.done]) >= 1 assert results[0].done and results[0].output_number == 10 assert results[1] == 42 assert results[2].done and results[2].output_number == 2 assert results[3].done and results[3].output_number == 25 raise workers.Return('Donezo')
def run(self, log_path, timeout_seconds=30): start_time = time.time() with open(log_path, 'a') as output_file: args = self.get_args() logging.info('item=%r Running subprocess: %r', self, args) try: process = subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=output_file, close_fds=True) except: logging.error('item=%r Failed to run subprocess: %r', self, args) raise while True: process.poll() if process.returncode is None: now = time.time() run_time = now - start_time if run_time > timeout_seconds: process.kill() raise TimeoutError( 'Sent SIGKILL to item=%r, pid=%s, run_time=%s' % (self, process.pid, run_time)) yield timer_worker.TimerItem(FLAGS.polltime) continue raise workers.Return(process.returncode)
def run(self, number, should_die=False, wait_seconds=0): if wait_seconds > 0: yield timer_worker.TimerItem(wait_seconds) if should_die: try: yield EchoChild(number, should_die=should_die) except Exception, e: raise e
def run(self, queue_name, local_queue_workflow, max_tasks=1, wait_seconds=0): queue_url = '%s/%s' % (FLAGS.queue_server_prefix, queue_name) outstanding = [] while True: next_count = max_tasks - len(outstanding) next_tasks = [] if next_count > 0: logging.info( 'Fetching %d tasks from queue_url=%r for workflow=%r', next_count, queue_url, local_queue_workflow) try: next_item = yield fetch_worker.FetchItem( queue_url + '/lease', post={'count': next_count}, username=FLAGS.release_client_id, password=FLAGS.release_client_secret) except Exception, e: logging.error( 'Could not fetch work from queue_url=%r. %s: %s', queue_url, e.__class__.__name__, e) else: if next_item.json: if next_item.json.get('error'): logging.error( 'Could not fetch work from queue_url=%r. %s', queue_url, next_item.json['error']) elif next_item.json['tasks']: next_tasks = next_item.json['tasks'] for index, task in enumerate(next_tasks): item = yield DoTaskWorkflow(queue_url, local_queue_workflow, task, wait_seconds=index * wait_seconds) outstanding.append(item) # Poll for new tasks frequently when we're currently handling # task load. Poll infrequently when there hasn't been anything # to do recently. poll_time = FLAGS.queue_idle_poll_seconds if outstanding: poll_time = FLAGS.queue_busy_poll_seconds yield timer_worker.TimerItem(poll_time) outstanding[:] = [x for x in outstanding if not x.done] logging.debug('%d items for %r still outstanding: %r', len(outstanding), local_queue_workflow, outstanding)
def run(self): jobs = [] for i in xrange(3): job = EchoChildWorkflow(99, should_die=True, wait_seconds=i * 0.5) job.fire_and_forget = True result = yield job assert result is job assert not result.done assert not result.error jobs.append(job) yield timer_worker.TimerItem(0.5) assert jobs[0].done assert jobs[1].done is False assert jobs[2].done is False yield timer_worker.TimerItem(1.5) assert jobs[0].done assert jobs[1].done assert jobs[2].done raise workers.Return('All errors seen')
def run(self): job = EchoChild(99, should_die=True) job.fire_and_forget = True result = yield job assert result is job assert not result.done assert not result.error result = yield EchoItem(25) assert result.done assert result.output_number == 25 yield timer_worker.TimerItem(2) assert job.done assert str(job.error[1]) == 'Dying on 99' raise workers.Return('No fire and forget error')
def run(self): output = yield workers.WaitAny([ EchoChild(42, should_die=True), EchoItem(10), EchoItem(33), ]) assert len([x for x in output if x.done]) == 2 assert not output[0].done assert output[1].done and output[1].output_number == 10 assert output[2].done and output[2].output_number == 33 yield timer_worker.TimerItem(2) try: yield output except Exception, e: raise workers.Return(str(e))
def run(self, queue_name, local_queue_workflow, max_tasks=1, wait_seconds=0): queue_url = '%s/%s' % (FLAGS.queue_server_prefix, queue_name) outstanding = [] while True: next_count = max_tasks - len(outstanding) next_tasks = [] if next_count > 0: logging.info( 'Fetching %d tasks from queue_url=%r for workflow=%r', next_count, queue_url, local_queue_workflow) try: next_item = yield fetch_worker.FetchItem( queue_url + '/lease', post={'count': next_count}, username=FLAGS.release_client_id, password=FLAGS.release_client_secret) except Exception, e: logging.error( 'Could not fetch work from queue_url=%r. %s: %s', queue_url, e.__class__.__name__, e) else: if next_item.json: if next_item.json.get('error'): logging.error( 'Could not fetch work from queue_url=%r. %s', queue_url, next_item.json['error']) elif next_item.json['tasks']: next_tasks = next_item.json['tasks'] for index, task in enumerate(next_tasks): item = yield DoTaskWorkflow(queue_url, local_queue_workflow, task, wait_seconds=index * wait_seconds) outstanding.append(item) yield timer_worker.TimerItem(FLAGS.queue_poll_seconds) outstanding[:] = [x for x in outstanding if not x.done]
def run(self, log_path, timeout_seconds=30): start_time = time.time() with open(log_path, 'a') as output_file: args = self.get_args() LOGGER.info('item=%r Running subprocess: %r', self, args) try: if sys.platform == 'win32': process = subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=output_file, creationflags=0x208) else: process = subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=output_file, close_fds=True) except: LOGGER.error('item=%r Failed to run subprocess: %r', self, args) raise while True: LOGGER.info('item=%r Polling pid=%r', self, process.pid) # NOTE: Use undocumented polling method to work around a # bug in subprocess for handling defunct zombie processes: # http://bugs.python.org/issue2475 process._internal_poll(_deadstate=127) if process.returncode is not None: LOGGER.info( 'item=%r Subprocess finished pid=%r, returncode=%r', self, process.pid, process.returncode) raise workers.Return(process.returncode) now = time.time() run_time = now - start_time if run_time > timeout_seconds: LOGGER.info('item=%r Subprocess timed out pid=%r', self, process.pid) process.kill() raise TimeoutError( 'Sent SIGKILL to item=%r, pid=%s, run_time=%s' % (self, process.pid, run_time)) yield timer_worker.TimerItem(FLAGS.polltime)
def run(self, queue_url, local_queue_workflow, task, wait_seconds=0): logging.info( 'Starting work item from queue_url=%r, ' 'task=%r, workflow=%r, wait_seconds=%r', queue_url, task, local_queue_workflow, wait_seconds) if wait_seconds > 0: yield timer_worker.TimerItem(wait_seconds) # Define a heartbeat closure that will return a workflow for # reporting status. This will auto-increment the index on each # call, so only the latest update will be saved. index = [0] task_id = task['task_id'] def heartbeat(message): next_index = index[0] index[0] = next_index + 1 return HeartbeatWorkflow(queue_url, task_id, message, next_index) payload = task['payload'] payload.update(heartbeat=heartbeat) error = False try: yield local_queue_workflow(**payload) except Exception, e: logging.exception( 'Exception while processing work from ' 'queue_url=%r, task=%r', queue_url, task) yield heartbeat('Task failed. %s: %s' % (e.__class__.__name__, str(e))) if (isinstance(e, GiveUpAfterAttemptsError) and task['lease_attempts'] >= e.max_attempts): logging.warning( 'Hit max attempts on task=%r, marking task as error', task) error = True else: # The task has legimiately failed. Do not mark the task as # finished. Let it retry in the queue again. return
def run(self): job1 = FireAndForgetEchoItem(10) result = yield job1 print result assert result is job1 assert not result.done result = yield EchoItem(25) assert result.done assert result.output_number == 25 job2 = EchoItem(30) job2.fire_and_forget = True result = yield job2 assert result is job2 assert not result.done job3 = FireAndForgetEchoItem(66) job3.fire_and_forget = False result = yield job3 assert result is job3 assert result.done assert result.output_number == 66 job4 = EchoChild(22) job4.fire_and_forget = True result = yield job4 assert result is job4 assert not result.done yield timer_worker.TimerItem(2) assert job1.done assert job1.output_number == 10 assert job2.done assert job2.output_number == 30 assert job4.done assert job4.result == 22 raise workers.Return('Okay')
def run(self): output = yield workers.WaitAny([ EchoItem(10), EchoChild(42), EchoItem(2), EchoItem(25), ]) assert len([x for x in output if x.done]) == 3 assert output[0].done and output[0].output_number == 10 assert not output[1].done assert output[2].done and output[2].output_number == 2 assert output[3].done and output[3].output_number == 25 yield timer_worker.TimerItem(2) results = yield output assert results[0].done and results[0].output_number == 10 assert results[1] == 42 assert results[2].done and results[2].output_number == 2 assert results[3].done and results[3].output_number == 25 raise workers.Return('Donezo')
def run(self, number, should_die=False, wait_seconds=0): if wait_seconds > 0: yield timer_worker.TimerItem(wait_seconds) item = yield EchoItem(number, should_die=should_die) raise workers.Return(item.output_number)