def run(self):
output = yield workers.WaitAny([
EchoItem(10),
EchoChild(42),
EchoItem(2),
EchoItem(25),
])
# At least one EchoItem will be done. We don't know exactly because
# the jobs WorkflowItems in WaitAny are inserted into a dictionary
# so their completion ordering is non-deterministic.
assert len([x for x in output if x.done]) >= 1
# The EchoChild will not be ready yet.
assert not output[1].done
yield timer_worker.TimerItem(2)
results = yield output
# Now everything will be done.
assert len([x for x in output if x.done]) >= 1
assert results[0].done and results[0].output_number == 10
assert results[1] == 42
assert results[2].done and results[2].output_number == 2
assert results[3].done and results[3].output_number == 25
raise workers.Return('Donezo')
def run(self):
try:
print 'up here'
yield GeneratorExceptionReraiseParent()
except Exception, e:
assert str(e) == 'Another exception', str(e)
raise workers.Return('good')
def run(self):
jobs = []
for i in xrange(3):
job = EchoChildWorkflow(99, should_die=True, wait_seconds=i * 0.5)
job.fire_and_forget = True
result = yield job
assert result is job
assert not result.done
assert not result.error
jobs.append(job)
yield timer_worker.TimerItem(0.5)
assert jobs[0].done
assert jobs[1].done is False
assert jobs[2].done is False
# for i, job in enumerate(jobs):
# print '%d is done %r' % (i, job.done)
# # assert str(job.error[1]) == 'Dying on 99'
yield timer_worker.TimerItem(1.5)
assert jobs[0].done
assert jobs[1].done
assert jobs[2].done
raise workers.Return('All errors seen')
def run(self, log_path, timeout_seconds=30):
start_time = time.time()
with open(log_path, 'a') as output_file:
args = self.get_args()
logging.info('item=%r Running subprocess: %r', self, args)
try:
process = subprocess.Popen(args,
stderr=subprocess.STDOUT,
stdout=output_file,
close_fds=True)
except:
logging.error('item=%r Failed to run subprocess: %r', self,
args)
raise
while True:
process.poll()
if process.returncode is None:
now = time.time()
run_time = now - start_time
if run_time > timeout_seconds:
process.kill()
raise TimeoutError(
'Sent SIGKILL to item=%r, pid=%s, run_time=%s' %
(self, process.pid, run_time))
yield timer_worker.TimerItem(FLAGS.polltime)
continue
raise workers.Return(process.returncode)
class EchoChildWorkflow(workers.WorkflowItem):
def run(self, number, should_die=False, wait_seconds=0):
if wait_seconds > 0:
yield timer_worker.TimerItem(wait_seconds)
if should_die:
try:
yield EchoChild(number, should_die=should_die)
except Exception, e:
raise e
raise workers.Return(number)
def run(self):
output = yield [
FireAndForgetEchoItem(22),
EchoItem(14),
EchoChild(98),
]
assert output[0].done
assert output[1].done
assert output[2] == 98
raise workers.Return('Waited for all of them')
def run(self, build_id, file_path):
try:
handle = StreamingSha1File(file_path, 'rb')
upload = yield fetch_worker.FetchItem(
FLAGS.release_server_prefix + '/upload',
post={'build_id': build_id, 'file': handle},
timeout_seconds=120,
username=FLAGS.release_client_id,
password=FLAGS.release_client_secret)
if upload.json and upload.json.get('error'):
raise UploadFileError(upload.json.get('error'))
sha1sum = handle.hexdigest()
if not upload.json or upload.json.get('sha1sum') != sha1sum:
raise UploadFileError('Bad response: %r' % upload)
raise workers.Return(sha1sum)
except IOError:
raise workers.Return(None)
def run(self):
output = yield workers.WaitAny([
FireAndForgetEchoItem(22),
EchoItem(14),
EchoChild(98),
])
assert output[0].done
assert output[1].done
assert not output[2].done
# Yielding here will let the next pending WorkflowItem to run,
# causing output #3 to finish.
result = yield output
assert result[2] == 98
raise workers.Return('All done')
def run(self):
output = yield workers.WaitAny([
EchoChild(42, should_die=True),
EchoItem(10),
EchoItem(33),
])
assert len([x for x in output if x.done]) == 2
assert not output[0].done
assert output[1].done and output[1].output_number == 10
assert output[2].done and output[2].output_number == 33
yield timer_worker.TimerItem(2)
try:
yield output
except Exception, e:
raise workers.Return(str(e))
def run(self):
job = EchoChild(99, should_die=True)
job.fire_and_forget = True
result = yield job
assert result is job
assert not result.done
assert not result.error
result = yield EchoItem(25)
assert result.done
assert result.output_number == 25
yield timer_worker.TimerItem(2)
assert job.done
assert str(job.error[1]) == 'Dying on 99'
raise workers.Return('No fire and forget error')
def run(self, build_id, run_name):
call = yield fetch_worker.FetchItem(
FLAGS.release_server_prefix + '/find_run',
post={
'build_id': build_id,
'run_name': run_name,
},
username=FLAGS.release_client_id,
password=FLAGS.release_client_secret)
if call.json and call.json.get('error'):
raise FindRunError(call.json.get('error'))
if not call.json:
raise FindRunError('Bad response: %r' % call)
raise workers.Return(call.json)
def run(self, build_id, release_name, release_number):
call = yield fetch_worker.FetchItem(
FLAGS.release_server_prefix + '/runs_done',
post={
'build_id': build_id,
'release_name': release_name,
'release_number': release_number,
},
username=FLAGS.release_client_id,
password=FLAGS.release_client_secret)
if call.json and call.json.get('error'):
raise RunsDoneError(call.json.get('error'))
if not call.json or not call.json.get('success'):
raise RunsDoneError('Bad response: %r' % call)
raise workers.Return(call.json['results_url'])
def run(self, build_id, release_name, url):
call = yield fetch_worker.FetchItem(
FLAGS.release_server_prefix + '/create_release',
post={
'build_id': build_id,
'release_name': release_name,
'url': url,
},
username=FLAGS.release_client_id,
password=FLAGS.release_client_secret)
if call.json and call.json.get('error'):
raise CreateReleaseError(call.json.get('error'))
if not call.json or not call.json.get('release_number'):
raise CreateReleaseError('Bad response: %r' % call)
raise workers.Return(call.json['release_number'])
def run(self, log_path, timeout_seconds=30):
start_time = time.time()
with open(log_path, 'a') as output_file:
args = self.get_args()
LOGGER.info('item=%r Running subprocess: %r', self, args)
try:
if sys.platform == 'win32':
process = subprocess.Popen(args,
stderr=subprocess.STDOUT,
stdout=output_file,
creationflags=0x208)
else:
process = subprocess.Popen(args,
stderr=subprocess.STDOUT,
stdout=output_file,
close_fds=True)
except:
LOGGER.error('item=%r Failed to run subprocess: %r', self,
args)
raise
while True:
LOGGER.info('item=%r Polling pid=%r', self, process.pid)
# NOTE: Use undocumented polling method to work around a
# bug in subprocess for handling defunct zombie processes:
# http://bugs.python.org/issue2475
process._internal_poll(_deadstate=127)
if process.returncode is not None:
LOGGER.info(
'item=%r Subprocess finished pid=%r, returncode=%r',
self, process.pid, process.returncode)
raise workers.Return(process.returncode)
now = time.time()
run_time = now - start_time
if run_time > timeout_seconds:
LOGGER.info('item=%r Subprocess timed out pid=%r', self,
process.pid)
process.kill()
raise TimeoutError(
'Sent SIGKILL to item=%r, pid=%s, run_time=%s' %
(self, process.pid, run_time))
yield timer_worker.TimerItem(FLAGS.polltime)
def run(self):
output = yield workers.WaitAny([
EchoItem(10),
EchoChild(42),
EchoItem(2),
EchoItem(25),
])
assert len([x for x in output if x.done]) == 3
assert output[0].done and output[0].output_number == 10
assert not output[1].done
assert output[2].done and output[2].output_number == 2
assert output[3].done and output[3].output_number == 25
yield timer_worker.TimerItem(2)
results = yield output
assert results[0].done and results[0].output_number == 10
assert results[1] == 42
assert results[2].done and results[2].output_number == 2
assert results[3].done and results[3].output_number == 25
raise workers.Return('Donezo')
def run(self):
job1 = FireAndForgetEchoItem(10)
result = yield job1
print result
assert result is job1
assert not result.done
result = yield EchoItem(25)
assert result.done
assert result.output_number == 25
job2 = EchoItem(30)
job2.fire_and_forget = True
result = yield job2
assert result is job2
assert not result.done
job3 = FireAndForgetEchoItem(66)
job3.fire_and_forget = False
result = yield job3
assert result is job3
assert result.done
assert result.output_number == 66
job4 = EchoChild(22)
job4.fire_and_forget = True
result = yield job4
assert result is job4
assert not result.done
yield timer_worker.TimerItem(2)
assert job1.done
assert job1.output_number == 10
assert job2.done
assert job2.output_number == 30
assert job4.done
assert job4.result == 22
raise workers.Return('Okay')
def run(self):
jobs = []
for i in xrange(3):
job = EchoChildWorkflow(99, should_die=True, wait_seconds=i * 0.5)
job.fire_and_forget = True
result = yield job
assert result is job
assert not result.done
assert not result.error
jobs.append(job)
yield timer_worker.TimerItem(0.5)
assert jobs[0].done
assert jobs[1].done is False
assert jobs[2].done is False
yield timer_worker.TimerItem(1.5)
assert jobs[0].done
assert jobs[1].done
assert jobs[2].done
raise workers.Return('All errors seen')
def run(self, number, should_die=False):
item = yield EchoItem(number, should_die=should_die)
raise workers.Return(item.output_number)
def run(self, number, should_die=False, wait_seconds=0):
if wait_seconds > 0:
yield timer_worker.TimerItem(wait_seconds)
item = yield EchoItem(number, should_die=should_die)
raise workers.Return(item.output_number)
yield GeneratorExceptionChild()
except Exception, e:
assert str(e) == 'My exception here 4', str(e)
raise Exception('Another exception')
class RootGeneratorExceptionWorkflow(workers.WorkflowItem):
def run(self):
try:
print 'up here'
yield GeneratorExceptionReraiseParent()
except Exception, e:
assert str(e) == 'Another exception', str(e)
raise workers.Return('good')
else:
raise workers.Return('bad')
class RootWaitAllWorkflow(workers.WorkflowItem):
def run(self, child_count):
wait_all = [EchoItem(i) for i in xrange(child_count)]
output = yield wait_all
raise workers.Return(sum(x.output_number for x in output))
class RootWaitAnyWorkflow(workers.WorkflowItem):
def run(self):
output = yield workers.WaitAny([
EchoItem(10),
EchoChild(42),
EchoItem(2),
def run(self, child_count):
wait_all = [EchoItem(i) for i in xrange(child_count)]
output = yield wait_all
raise workers.Return(sum(x.output_number for x in output))
