def testSimple(self):
"""Tests simple waiting."""
self.worker.start()
one = timer_worker.TimerItem(0.8)
two = timer_worker.TimerItem(0.5)
three = timer_worker.TimerItem(0.1)
# T = 0, one = 0
begin = time.time()
self.timer_queue.put(one)
time.sleep(0.2)
# T = 0.2, one = 0.2, two = 0
self.timer_queue.put(two)
time.sleep(0.2)
# T = 0.4, one = 0.4, two = 0.2
self.timer_queue.put(three)
time.sleep(0.2)
# T = 0.6, one = 0.6, two = 0.4, three = 0.1 ready!
output_three = self.output_queue.get()
time.sleep(0.1)
# T = 0.7, one = 0.7, two = 0.5 ready!
output_two = self.output_queue.get()
# T = 0.8, one = 0.8 ready!
output_one = self.output_queue.get()
end = time.time()
self.assertEquals(one.delay_seconds, output_one.delay_seconds)
self.assertEquals(two.delay_seconds, output_two.delay_seconds)
self.assertEquals(three.delay_seconds, output_three.delay_seconds)
elapsed = end - begin
self.assertTrue(1.0 > elapsed > 0.7)
def run(self):
jobs = []
for i in xrange(3):
job = EchoChildWorkflow(99, should_die=True, wait_seconds=i * 0.5)
job.fire_and_forget = True
result = yield job
assert result is job
assert not result.done
assert not result.error
jobs.append(job)
yield timer_worker.TimerItem(0.5)
assert jobs[0].done
assert jobs[1].done is False
assert jobs[2].done is False
# for i, job in enumerate(jobs):
# print '%d is done %r' % (i, job.done)
# # assert str(job.error[1]) == 'Dying on 99'
yield timer_worker.TimerItem(1.5)
assert jobs[0].done
assert jobs[1].done
assert jobs[2].done
raise workers.Return('All errors seen')
def run(self, tmp_dir, waitfor, heartbeat=None, start_time=None):
assert 'url' in waitfor
timeout = waitfor.get('timeout_secs', 10)
if not start_time:
start_time = time.time()
class NotReadyError(Exception):
pass
try:
url = waitfor['url']
r = requests.head(url, allow_redirects=True)
if r.status_code != 200:
yield heartbeat('Request for %s failed (%d)' %
(url, r.status_code))
raise NotReadyError()
yield heartbeat(
'Request for %s succeeded, continuing with tests...' % url)
return
except (requests.ConnectionError, NotReadyError):
now = time.time()
if now - start_time >= timeout:
raise process_worker.TimeoutError()
yield timer_worker.TimerItem(0.5) # wait 500ms between checks
yield WaitForUrlWorkflowItem(tmp_dir, waitfor, heartbeat,
start_time)
def run(self):
output = yield workers.WaitAny([
EchoItem(10),
EchoChild(42),
EchoItem(2),
EchoItem(25),
])
# At least one EchoItem will be done. We don't know exactly because
# the jobs WorkflowItems in WaitAny are inserted into a dictionary
# so their completion ordering is non-deterministic.
assert len([x for x in output if x.done]) >= 1
# The EchoChild will not be ready yet.
assert not output[1].done
yield timer_worker.TimerItem(2)
results = yield output
# Now everything will be done.
assert len([x for x in output if x.done]) >= 1
assert results[0].done and results[0].output_number == 10
assert results[1] == 42
assert results[2].done and results[2].output_number == 2
assert results[3].done and results[3].output_number == 25
raise workers.Return('Donezo')
def run(self, log_path, timeout_seconds=30):
start_time = time.time()
with open(log_path, 'a') as output_file:
args = self.get_args()
logging.info('item=%r Running subprocess: %r', self, args)
try:
process = subprocess.Popen(args,
stderr=subprocess.STDOUT,
stdout=output_file,
close_fds=True)
except:
logging.error('item=%r Failed to run subprocess: %r', self,
args)
raise
while True:
process.poll()
if process.returncode is None:
now = time.time()
run_time = now - start_time
if run_time > timeout_seconds:
process.kill()
raise TimeoutError(
'Sent SIGKILL to item=%r, pid=%s, run_time=%s' %
(self, process.pid, run_time))
yield timer_worker.TimerItem(FLAGS.polltime)
continue
raise workers.Return(process.returncode)
def run(self, number, should_die=False, wait_seconds=0):
if wait_seconds > 0:
yield timer_worker.TimerItem(wait_seconds)
if should_die:
try:
yield EchoChild(number, should_die=should_die)
except Exception, e:
raise e
def run(self,
queue_name,
local_queue_workflow,
max_tasks=1,
wait_seconds=0):
queue_url = '%s/%s' % (FLAGS.queue_server_prefix, queue_name)
outstanding = []
while True:
next_count = max_tasks - len(outstanding)
next_tasks = []
if next_count > 0:
logging.info(
'Fetching %d tasks from queue_url=%r for workflow=%r',
next_count, queue_url, local_queue_workflow)
try:
next_item = yield fetch_worker.FetchItem(
queue_url + '/lease',
post={'count': next_count},
username=FLAGS.release_client_id,
password=FLAGS.release_client_secret)
except Exception, e:
logging.error(
'Could not fetch work from queue_url=%r. %s: %s',
queue_url, e.__class__.__name__, e)
else:
if next_item.json:
if next_item.json.get('error'):
logging.error(
'Could not fetch work from queue_url=%r. %s',
queue_url, next_item.json['error'])
elif next_item.json['tasks']:
next_tasks = next_item.json['tasks']
for index, task in enumerate(next_tasks):
item = yield DoTaskWorkflow(queue_url,
local_queue_workflow,
task,
wait_seconds=index * wait_seconds)
outstanding.append(item)
# Poll for new tasks frequently when we're currently handling
# task load. Poll infrequently when there hasn't been anything
# to do recently.
poll_time = FLAGS.queue_idle_poll_seconds
if outstanding:
poll_time = FLAGS.queue_busy_poll_seconds
yield timer_worker.TimerItem(poll_time)
outstanding[:] = [x for x in outstanding if not x.done]
logging.debug('%d items for %r still outstanding: %r',
len(outstanding), local_queue_workflow, outstanding)
def run(self):
jobs = []
for i in xrange(3):
job = EchoChildWorkflow(99, should_die=True, wait_seconds=i * 0.5)
job.fire_and_forget = True
result = yield job
assert result is job
assert not result.done
assert not result.error
jobs.append(job)
yield timer_worker.TimerItem(0.5)
assert jobs[0].done
assert jobs[1].done is False
assert jobs[2].done is False
yield timer_worker.TimerItem(1.5)
assert jobs[0].done
assert jobs[1].done
assert jobs[2].done
raise workers.Return('All errors seen')
def run(self):
job = EchoChild(99, should_die=True)
job.fire_and_forget = True
result = yield job
assert result is job
assert not result.done
assert not result.error
result = yield EchoItem(25)
assert result.done
assert result.output_number == 25
yield timer_worker.TimerItem(2)
assert job.done
assert str(job.error[1]) == 'Dying on 99'
raise workers.Return('No fire and forget error')
def run(self):
output = yield workers.WaitAny([
EchoChild(42, should_die=True),
EchoItem(10),
EchoItem(33),
])
assert len([x for x in output if x.done]) == 2
assert not output[0].done
assert output[1].done and output[1].output_number == 10
assert output[2].done and output[2].output_number == 33
yield timer_worker.TimerItem(2)
try:
yield output
except Exception, e:
raise workers.Return(str(e))
def run(self,
queue_name,
local_queue_workflow,
max_tasks=1,
wait_seconds=0):
queue_url = '%s/%s' % (FLAGS.queue_server_prefix, queue_name)
outstanding = []
while True:
next_count = max_tasks - len(outstanding)
next_tasks = []
if next_count > 0:
logging.info(
'Fetching %d tasks from queue_url=%r for workflow=%r',
next_count, queue_url, local_queue_workflow)
try:
next_item = yield fetch_worker.FetchItem(
queue_url + '/lease',
post={'count': next_count},
username=FLAGS.release_client_id,
password=FLAGS.release_client_secret)
except Exception, e:
logging.error(
'Could not fetch work from queue_url=%r. %s: %s',
queue_url, e.__class__.__name__, e)
else:
if next_item.json:
if next_item.json.get('error'):
logging.error(
'Could not fetch work from queue_url=%r. %s',
queue_url, next_item.json['error'])
elif next_item.json['tasks']:
next_tasks = next_item.json['tasks']
for index, task in enumerate(next_tasks):
item = yield DoTaskWorkflow(queue_url,
local_queue_workflow,
task,
wait_seconds=index * wait_seconds)
outstanding.append(item)
yield timer_worker.TimerItem(FLAGS.queue_poll_seconds)
outstanding[:] = [x for x in outstanding if not x.done]
def run(self, log_path, timeout_seconds=30):
start_time = time.time()
with open(log_path, 'a') as output_file:
args = self.get_args()
LOGGER.info('item=%r Running subprocess: %r', self, args)
try:
if sys.platform == 'win32':
process = subprocess.Popen(args,
stderr=subprocess.STDOUT,
stdout=output_file,
creationflags=0x208)
else:
process = subprocess.Popen(args,
stderr=subprocess.STDOUT,
stdout=output_file,
close_fds=True)
except:
LOGGER.error('item=%r Failed to run subprocess: %r', self,
args)
raise
while True:
LOGGER.info('item=%r Polling pid=%r', self, process.pid)
# NOTE: Use undocumented polling method to work around a
# bug in subprocess for handling defunct zombie processes:
# http://bugs.python.org/issue2475
process._internal_poll(_deadstate=127)
if process.returncode is not None:
LOGGER.info(
'item=%r Subprocess finished pid=%r, returncode=%r',
self, process.pid, process.returncode)
raise workers.Return(process.returncode)
now = time.time()
run_time = now - start_time
if run_time > timeout_seconds:
LOGGER.info('item=%r Subprocess timed out pid=%r', self,
process.pid)
process.kill()
raise TimeoutError(
'Sent SIGKILL to item=%r, pid=%s, run_time=%s' %
(self, process.pid, run_time))
yield timer_worker.TimerItem(FLAGS.polltime)
def run(self, queue_url, local_queue_workflow, task, wait_seconds=0):
logging.info(
'Starting work item from queue_url=%r, '
'task=%r, workflow=%r, wait_seconds=%r', queue_url, task,
local_queue_workflow, wait_seconds)
if wait_seconds > 0:
yield timer_worker.TimerItem(wait_seconds)
# Define a heartbeat closure that will return a workflow for
# reporting status. This will auto-increment the index on each
# call, so only the latest update will be saved.
index = [0]
task_id = task['task_id']
def heartbeat(message):
next_index = index[0]
index[0] = next_index + 1
return HeartbeatWorkflow(queue_url, task_id, message, next_index)
payload = task['payload']
payload.update(heartbeat=heartbeat)
error = False
try:
yield local_queue_workflow(**payload)
except Exception, e:
logging.exception(
'Exception while processing work from '
'queue_url=%r, task=%r', queue_url, task)
yield heartbeat('Task failed. %s: %s' %
(e.__class__.__name__, str(e)))
if (isinstance(e, GiveUpAfterAttemptsError)
and task['lease_attempts'] >= e.max_attempts):
logging.warning(
'Hit max attempts on task=%r, marking task as error', task)
error = True
else:
# The task has legimiately failed. Do not mark the task as
# finished. Let it retry in the queue again.
return
def run(self):
job1 = FireAndForgetEchoItem(10)
result = yield job1
print result
assert result is job1
assert not result.done
result = yield EchoItem(25)
assert result.done
assert result.output_number == 25
job2 = EchoItem(30)
job2.fire_and_forget = True
result = yield job2
assert result is job2
assert not result.done
job3 = FireAndForgetEchoItem(66)
job3.fire_and_forget = False
result = yield job3
assert result is job3
assert result.done
assert result.output_number == 66
job4 = EchoChild(22)
job4.fire_and_forget = True
result = yield job4
assert result is job4
assert not result.done
yield timer_worker.TimerItem(2)
assert job1.done
assert job1.output_number == 10
assert job2.done
assert job2.output_number == 30
assert job4.done
assert job4.result == 22
raise workers.Return('Okay')
def run(self):
output = yield workers.WaitAny([
EchoItem(10),
EchoChild(42),
EchoItem(2),
EchoItem(25),
])
assert len([x for x in output if x.done]) == 3
assert output[0].done and output[0].output_number == 10
assert not output[1].done
assert output[2].done and output[2].output_number == 2
assert output[3].done and output[3].output_number == 25
yield timer_worker.TimerItem(2)
results = yield output
assert results[0].done and results[0].output_number == 10
assert results[1] == 42
assert results[2].done and results[2].output_number == 2
assert results[3].done and results[3].output_number == 25
raise workers.Return('Donezo')
def run(self, number, should_die=False, wait_seconds=0):
if wait_seconds > 0:
yield timer_worker.TimerItem(wait_seconds)
item = yield EchoItem(number, should_die=should_die)
raise workers.Return(item.output_number)
