def run(self): output = yield workers.WaitAny([ EchoItem(10), EchoChild(42), EchoItem(2), EchoItem(25), ]) # At least one EchoItem will be done. We don't know exactly because # the jobs WorkflowItems in WaitAny are inserted into a dictionary # so their completion ordering is non-deterministic. assert len([x for x in output if x.done]) >= 1 # The EchoChild will not be ready yet. assert not output[1].done yield timer_worker.TimerItem(2) results = yield output # Now everything will be done. assert len([x for x in output if x.done]) >= 1 assert results[0].done and results[0].output_number == 10 assert results[1] == 42 assert results[2].done and results[2].output_number == 2 assert results[3].done and results[3].output_number == 25 raise workers.Return('Donezo')
def run(self): try: print 'up here' yield GeneratorExceptionReraiseParent() except Exception, e: assert str(e) == 'Another exception', str(e) raise workers.Return('good')
def run(self): jobs = [] for i in xrange(3): job = EchoChildWorkflow(99, should_die=True, wait_seconds=i * 0.5) job.fire_and_forget = True result = yield job assert result is job assert not result.done assert not result.error jobs.append(job) yield timer_worker.TimerItem(0.5) assert jobs[0].done assert jobs[1].done is False assert jobs[2].done is False # for i, job in enumerate(jobs): # print '%d is done %r' % (i, job.done) # # assert str(job.error[1]) == 'Dying on 99' yield timer_worker.TimerItem(1.5) assert jobs[0].done assert jobs[1].done assert jobs[2].done raise workers.Return('All errors seen')
def run(self, log_path, timeout_seconds=30): start_time = time.time() with open(log_path, 'a') as output_file: args = self.get_args() logging.info('item=%r Running subprocess: %r', self, args) try: process = subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=output_file, close_fds=True) except: logging.error('item=%r Failed to run subprocess: %r', self, args) raise while True: process.poll() if process.returncode is None: now = time.time() run_time = now - start_time if run_time > timeout_seconds: process.kill() raise TimeoutError( 'Sent SIGKILL to item=%r, pid=%s, run_time=%s' % (self, process.pid, run_time)) yield timer_worker.TimerItem(FLAGS.polltime) continue raise workers.Return(process.returncode)
class EchoChildWorkflow(workers.WorkflowItem): def run(self, number, should_die=False, wait_seconds=0): if wait_seconds > 0: yield timer_worker.TimerItem(wait_seconds) if should_die: try: yield EchoChild(number, should_die=should_die) except Exception, e: raise e raise workers.Return(number)
def run(self): output = yield [ FireAndForgetEchoItem(22), EchoItem(14), EchoChild(98), ] assert output[0].done assert output[1].done assert output[2] == 98 raise workers.Return('Waited for all of them')
def run(self, build_id, file_path): try: handle = StreamingSha1File(file_path, 'rb') upload = yield fetch_worker.FetchItem( FLAGS.release_server_prefix + '/upload', post={'build_id': build_id, 'file': handle}, timeout_seconds=120, username=FLAGS.release_client_id, password=FLAGS.release_client_secret) if upload.json and upload.json.get('error'): raise UploadFileError(upload.json.get('error')) sha1sum = handle.hexdigest() if not upload.json or upload.json.get('sha1sum') != sha1sum: raise UploadFileError('Bad response: %r' % upload) raise workers.Return(sha1sum) except IOError: raise workers.Return(None)
def run(self): output = yield workers.WaitAny([ FireAndForgetEchoItem(22), EchoItem(14), EchoChild(98), ]) assert output[0].done assert output[1].done assert not output[2].done # Yielding here will let the next pending WorkflowItem to run, # causing output #3 to finish. result = yield output assert result[2] == 98 raise workers.Return('All done')
def run(self): output = yield workers.WaitAny([ EchoChild(42, should_die=True), EchoItem(10), EchoItem(33), ]) assert len([x for x in output if x.done]) == 2 assert not output[0].done assert output[1].done and output[1].output_number == 10 assert output[2].done and output[2].output_number == 33 yield timer_worker.TimerItem(2) try: yield output except Exception, e: raise workers.Return(str(e))
def run(self): job = EchoChild(99, should_die=True) job.fire_and_forget = True result = yield job assert result is job assert not result.done assert not result.error result = yield EchoItem(25) assert result.done assert result.output_number == 25 yield timer_worker.TimerItem(2) assert job.done assert str(job.error[1]) == 'Dying on 99' raise workers.Return('No fire and forget error')
def run(self, build_id, run_name): call = yield fetch_worker.FetchItem( FLAGS.release_server_prefix + '/find_run', post={ 'build_id': build_id, 'run_name': run_name, }, username=FLAGS.release_client_id, password=FLAGS.release_client_secret) if call.json and call.json.get('error'): raise FindRunError(call.json.get('error')) if not call.json: raise FindRunError('Bad response: %r' % call) raise workers.Return(call.json)
def run(self, build_id, release_name, release_number): call = yield fetch_worker.FetchItem( FLAGS.release_server_prefix + '/runs_done', post={ 'build_id': build_id, 'release_name': release_name, 'release_number': release_number, }, username=FLAGS.release_client_id, password=FLAGS.release_client_secret) if call.json and call.json.get('error'): raise RunsDoneError(call.json.get('error')) if not call.json or not call.json.get('success'): raise RunsDoneError('Bad response: %r' % call) raise workers.Return(call.json['results_url'])
def run(self, build_id, release_name, url): call = yield fetch_worker.FetchItem( FLAGS.release_server_prefix + '/create_release', post={ 'build_id': build_id, 'release_name': release_name, 'url': url, }, username=FLAGS.release_client_id, password=FLAGS.release_client_secret) if call.json and call.json.get('error'): raise CreateReleaseError(call.json.get('error')) if not call.json or not call.json.get('release_number'): raise CreateReleaseError('Bad response: %r' % call) raise workers.Return(call.json['release_number'])
def run(self, log_path, timeout_seconds=30): start_time = time.time() with open(log_path, 'a') as output_file: args = self.get_args() LOGGER.info('item=%r Running subprocess: %r', self, args) try: if sys.platform == 'win32': process = subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=output_file, creationflags=0x208) else: process = subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=output_file, close_fds=True) except: LOGGER.error('item=%r Failed to run subprocess: %r', self, args) raise while True: LOGGER.info('item=%r Polling pid=%r', self, process.pid) # NOTE: Use undocumented polling method to work around a # bug in subprocess for handling defunct zombie processes: # http://bugs.python.org/issue2475 process._internal_poll(_deadstate=127) if process.returncode is not None: LOGGER.info( 'item=%r Subprocess finished pid=%r, returncode=%r', self, process.pid, process.returncode) raise workers.Return(process.returncode) now = time.time() run_time = now - start_time if run_time > timeout_seconds: LOGGER.info('item=%r Subprocess timed out pid=%r', self, process.pid) process.kill() raise TimeoutError( 'Sent SIGKILL to item=%r, pid=%s, run_time=%s' % (self, process.pid, run_time)) yield timer_worker.TimerItem(FLAGS.polltime)
def run(self): output = yield workers.WaitAny([ EchoItem(10), EchoChild(42), EchoItem(2), EchoItem(25), ]) assert len([x for x in output if x.done]) == 3 assert output[0].done and output[0].output_number == 10 assert not output[1].done assert output[2].done and output[2].output_number == 2 assert output[3].done and output[3].output_number == 25 yield timer_worker.TimerItem(2) results = yield output assert results[0].done and results[0].output_number == 10 assert results[1] == 42 assert results[2].done and results[2].output_number == 2 assert results[3].done and results[3].output_number == 25 raise workers.Return('Donezo')
def run(self): job1 = FireAndForgetEchoItem(10) result = yield job1 print result assert result is job1 assert not result.done result = yield EchoItem(25) assert result.done assert result.output_number == 25 job2 = EchoItem(30) job2.fire_and_forget = True result = yield job2 assert result is job2 assert not result.done job3 = FireAndForgetEchoItem(66) job3.fire_and_forget = False result = yield job3 assert result is job3 assert result.done assert result.output_number == 66 job4 = EchoChild(22) job4.fire_and_forget = True result = yield job4 assert result is job4 assert not result.done yield timer_worker.TimerItem(2) assert job1.done assert job1.output_number == 10 assert job2.done assert job2.output_number == 30 assert job4.done assert job4.result == 22 raise workers.Return('Okay')
def run(self): jobs = [] for i in xrange(3): job = EchoChildWorkflow(99, should_die=True, wait_seconds=i * 0.5) job.fire_and_forget = True result = yield job assert result is job assert not result.done assert not result.error jobs.append(job) yield timer_worker.TimerItem(0.5) assert jobs[0].done assert jobs[1].done is False assert jobs[2].done is False yield timer_worker.TimerItem(1.5) assert jobs[0].done assert jobs[1].done assert jobs[2].done raise workers.Return('All errors seen')
def run(self, number, should_die=False): item = yield EchoItem(number, should_die=should_die) raise workers.Return(item.output_number)
def run(self, number, should_die=False, wait_seconds=0): if wait_seconds > 0: yield timer_worker.TimerItem(wait_seconds) item = yield EchoItem(number, should_die=should_die) raise workers.Return(item.output_number)
yield GeneratorExceptionChild() except Exception, e: assert str(e) == 'My exception here 4', str(e) raise Exception('Another exception') class RootGeneratorExceptionWorkflow(workers.WorkflowItem): def run(self): try: print 'up here' yield GeneratorExceptionReraiseParent() except Exception, e: assert str(e) == 'Another exception', str(e) raise workers.Return('good') else: raise workers.Return('bad') class RootWaitAllWorkflow(workers.WorkflowItem): def run(self, child_count): wait_all = [EchoItem(i) for i in xrange(child_count)] output = yield wait_all raise workers.Return(sum(x.output_number for x in output)) class RootWaitAnyWorkflow(workers.WorkflowItem): def run(self): output = yield workers.WaitAny([ EchoItem(10), EchoChild(42), EchoItem(2),
def run(self, child_count): wait_all = [EchoItem(i) for i in xrange(child_count)] output = yield wait_all raise workers.Return(sum(x.output_number for x in output))