def test_all_future_activities(self):
class AllFutureWorkflow(WorkflowDefinition):
@execute(version='1.1', execution_start_to_close_timeout=60)
def execute(self, arg1, arg2):
sum_future = BunchOfActivities.sum(arg1, arg2)
mul_future = BunchOfActivities.mul(arg1, arg2)
result = yield sum_future, mul_future
return_(result)
wf_worker = WorkflowWorker(
self.session, self.region, self.domain, self.task_list, AllFutureWorkflow)
act_worker = ActivityWorker(
self.session, self.region, self.domain, self.task_list, BunchOfActivities())
with workflow_starter(self.session, self.region, self.domain, self.task_list):
instance = AllFutureWorkflow.execute(arg1=1, arg2=2)
self.workflow_execution = instance.workflow_execution
wf_worker.run_once()
act_worker.run_once()
wf_worker.run_once()
act_worker.run_once()
wf_worker.run_once()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 17)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(self.serializer.loads(
hist[-1]['workflowExecutionCompletedEventAttributes']['result']), (3, 2))
def test_one_activity_options_overrides_priority(self):
class OneActivityWorkflow(WorkflowDefinition):
@execute(version='1.1', execution_start_to_close_timeout=60)
def execute(self, arg1, arg2):
with activity_options(task_priority=66):
arg_sum = yield BunchOfActivities.sum(arg1, arg2)
return_(arg_sum)
wf_worker = WorkflowWorker(
self.session, self.region, self.domain, self.task_list, OneActivityWorkflow)
act_worker = ActivityWorker(
self.session, self.region, self.domain, self.task_list, BunchOfActivities())
with workflow_starter(self.session, self.region, self.domain, self.task_list):
instance = OneActivityWorkflow.execute(arg1=1, arg2=2)
self.workflow_execution = instance.workflow_execution
wf_worker.run_once()
act_worker.run_once()
wf_worker.run_once()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 11)
self.assertEqual(hist[4]['activityTaskScheduledEventAttributes']['taskPriority'], '66')
def test_next_page_token_activities(self):
# process over a hundred events, so that we're clear we can work with nextPageToken
class NextPageTokenWorkflow(WorkflowDefinition):
@execute(version='1.1', execution_start_to_close_timeout=60)
def execute(self, repeat, arg1):
for i in range(repeat):
yield BunchOfActivities.sum(i, arg1)
return_(repeat)
wf_worker = WorkflowWorker(
self.session, self.region, self.domain, self.task_list, NextPageTokenWorkflow)
act_worker = ActivityWorker(
self.session, self.region, self.domain, self.task_list, BunchOfActivities())
with workflow_starter(self.session, self.region, self.domain, self.task_list):
instance = NextPageTokenWorkflow.execute(repeat=21, arg1=1)
self.workflow_execution = instance.workflow_execution
for i in range(21):
wf_worker.run_once()
act_worker.run_once()
wf_worker.run_once() # finish off
time.sleep(1)
hist, token = self.get_workflow_execution_history_with_token()
events = hist
hist = self.get_workflow_execution_history(next_page_token=token)
events.extend(hist)
self.assertEqual(len(events), 131)
self.assertEqual(events[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(self.serializer.loads(
events[-1]['workflowExecutionCompletedEventAttributes']['result']), 21)
def test_one_activity(self):
wf_worker = MultiprocessingWorkflowExecutor(WorkflowWorker(
self.session, self.region, self.domain, self.task_list, OneActivityWorkflow))
act_worker = MultiprocessingActivityExecutor(ActivityWorker(
self.session, self.region, self.domain, self.task_list, BunchOfActivities()))
with workflow_starter(self.session, self.region, self.domain, self.task_list):
instance = OneActivityWorkflow.execute(arg1=1, arg2=2)
self.workflow_execution = instance.workflow_execution
wf_worker.start()
act_worker.start()
time.sleep(20)
act_worker.stop()
wf_worker.stop()
act_worker.join()
wf_worker.join()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 11)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(self.serializer.loads(
hist[-1]['workflowExecutionCompletedEventAttributes']['result']), 3)
def test_one_activity_dynamic(self):
class OneActivityTimedWorkflow(WorkflowDefinition):
@execute(version='1.1', execution_start_to_close_timeout=60)
def execute(self, arg1, arg2):
# create an activity call dynamically
sum = flow_types.ActivityType('1.1', name='BunchOfActivities.sum')
arg_sum = yield sum(arg1, arg2)
return_(arg_sum)
wf_worker = WorkflowWorker(
self.session, self.region, self.domain, self.task_list, OneActivityTimedWorkflow)
act_worker = ActivityWorker(
self.session, self.region, self.domain, self.task_list, BunchOfActivities())
with workflow_starter(self.session, self.region, self.domain, self.task_list):
instance = OneActivityTimedWorkflow.execute(arg1=1, arg2=2)
self.workflow_execution = instance.workflow_execution
wf_worker.run_once()
act_worker.run_once()
wf_worker.run_once()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 11)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(self.serializer.loads(
hist[-1]['workflowExecutionCompletedEventAttributes']['result']), 3)
def test_two_workflows(self):
wf_worker = WorkflowWorker(self.session, self.region, self.domain,
self.task_list, MasterWorkflow,
ChildWorkflow)
act_worker = ActivityWorker(self.session, self.region, self.domain,
self.task_list, BunchOfActivities())
with workflow_starter(self.session, self.region, self.domain,
self.task_list):
instance = MasterWorkflow.execute(arg1=1, arg2=2)
self.workflow_execution = instance.workflow_execution
for i in range(3):
wf_worker.run_once()
act_worker.run_once()
for i in range(2):
wf_worker.run_once()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 14)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(
self.serializer.loads(
hist[-1]['workflowExecutionCompletedEventAttributes']
['result']), 3)
def test_two_workflows_priority(self):
wf_worker = WorkflowWorker(self.session, self.region, self.domain,
self.task_list, MasterWorkflowWithPriority,
ChildWorkflow)
act_worker = ActivityWorker(self.session, self.region, self.domain,
self.task_list, BunchOfActivities())
with workflow_starter(self.session, self.region, self.domain,
self.task_list):
instance = MasterWorkflowWithPriority.execute(arg1=1,
arg2=2,
priority=100)
self.workflow_execution = instance.workflow_execution
for i in range(3):
wf_worker.run_once()
act_worker.run_once()
for i in range(2):
wf_worker.run_once()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 14)
self.assertEqual(hist[4]['eventType'],
'StartChildWorkflowExecutionInitiated')
self.assertEqual(
hist[4]['startChildWorkflowExecutionInitiatedEventAttributes']
['taskPriority'], '100')
def test_one_activity(self):
class OneActivityWorkflow(WorkflowDefinition):
def __init__(self, workflow_execution):
super(OneActivityWorkflow, self).__init__(workflow_execution)
self.activities_client = BunchOfActivities()
@execute(version='1.1', execution_start_to_close_timeout=60)
def execute(self, arg1, arg2):
arg_sum = yield self.activities_client.sum(arg1, arg2)
return_(arg_sum)
wf_worker = GenericWorkflowWorker(
self.session, self.region, self.domain, self.task_list, WorkflowFinder(OneActivityWorkflow))
self._register_workflows(wf_worker)
act_worker = ThreadedActivityExecutor(ActivityWorker(
self.session, self.region, self.domain, self.task_list, BunchOfActivities()))
with workflow_starter(self.session, self.region, self.domain, self.task_list):
instance = OneActivityWorkflow.execute(arg1=1, arg2=2)
self.workflow_execution = instance.workflow_execution
wf_worker.run_once()
act_worker.start(1, 4)
act_worker.stop()
wf_worker.run_once()
act_worker.join()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 11)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(self.serializer.loads(
hist[-1]['workflowExecutionCompletedEventAttributes']['result']), 3)
def test_one_activity_timed(self):
class OneActivityTimedWorkflow(WorkflowDefinition):
@execute(version='1.1', execution_start_to_close_timeout=60)
def execute(self, arg1, arg2):
mytime = workflow_time.time()
yield BunchOfActivities.sum(arg1, arg2)
return_([mytime, workflow_time.time()])
wf_worker = WorkflowWorker(
self.session, self.region, self.domain, self.task_list, OneActivityTimedWorkflow)
act_worker = ActivityWorker(
self.session, self.region, self.domain, self.task_list, BunchOfActivities())
with workflow_starter(self.session, self.region, self.domain, self.task_list):
instance = OneActivityTimedWorkflow.execute(arg1=1, arg2=2)
self.workflow_execution = instance.workflow_execution
wf_worker.run_once()
act_worker.run_once()
wf_worker.run_once()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 11)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(self.serializer.loads(
hist[-1]['workflowExecutionCompletedEventAttributes']['result']), [
int(time.mktime(hist[2]['eventTimestamp'].timetuple())),
int(time.mktime(hist[8]['eventTimestamp'].timetuple()))])
def test_cancel_child_workflow(self):
class CancelChildWorkflowsParentWorkflow(WorkflowDefinition):
@execute(version='1.2', execution_start_to_close_timeout=60)
def execute(self, child_task_list):
with workflow_options(task_list=child_task_list):
instance = yield CancelChildWorkflowsChildWorkflow.execute()
yield instance.cancel()
try:
yield instance.workflow_result
except CancelledError:
return_('pass')
class CancelChildWorkflowsChildWorkflow(WorkflowDefinition):
@execute(version='1.3', execution_start_to_close_timeout=60)
def execute(self):
arg_sum = yield BunchOfActivities.sleep_activity(10)
return_(arg_sum)
child_task_list = self.task_list + '_child'
parent_wf = CancelChildWorkflowsParentWorkflow
child_worker = WorkflowWorker(self.session, self.region, self.domain, child_task_list,
CancelChildWorkflowsChildWorkflow)
wf_worker = WorkflowWorker(self.session, self.region, self.domain, self.task_list,
parent_wf)
act_worker = ActivityWorker(self.session, self.region, self.domain, child_task_list, BunchOfActivities())
self.start_workflow(parent_wf, child_task_list)
def activity_wrapper():
try:
act_worker.run_once()
except Exception:
pass
act_worker_thread = Thread(target=activity_wrapper)
act_worker_thread.start()
wf_worker.run_once() # start parent workflow
child_worker.run_once() # start child workflow
wf_worker.run_once() # cancel child
child_worker.run_once() # child intakes request and cancels
time.sleep(1)
wf_worker.run_once() # parent waits for cancellation to complete
act_worker_thread.join()
time.sleep(1)
parent_hist = self.get_workflow_execution_history()
self.assertEqual(parent_hist[-1]['eventType'], 'WorkflowExecutionCompleted')
child_cancel_events = self.get_events(parent_hist, 'ExternalWorkflowExecutionCancelRequested')
self.assertEqual(len(child_cancel_events), 1)
child_execution = child_cancel_events[0][
'externalWorkflowExecutionCancelRequestedEventAttributes']['workflowExecution']
workflow_id = child_execution['workflowId']
run_id = child_execution['runId']
child_hist = self.get_workflow_execution_history(workflow_id=workflow_id, run_id=run_id)
self.assertEqual(child_hist[-1]['eventType'], 'WorkflowExecutionCanceled')
def test_one_manual_one_automatic_activity(self):
swf_client = self.client
class OneManualOneAutomaticActivityWorkflow(WorkflowDefinition):
def __init__(self, workflow_execution):
super(OneManualOneAutomaticActivityWorkflow,
self).__init__(workflow_execution)
@execute(version='1.1', execution_start_to_close_timeout=60)
def execute(self, template):
(x, y) = yield ManualActivities.perform_task(template=template)
arg_sum = yield BunchOfActivities.sum(x, y)
return_(arg_sum)
wf_worker = WorkflowWorker(self.session, self.region, self.domain,
self.task_list,
OneManualOneAutomaticActivityWorkflow)
act_worker = ActivityWorker(self.session, self.region, self.domain,
self.task_list, BunchOfActivities(),
ManualActivities())
with workflow_starter(self.session, self.region, self.domain,
self.task_list):
instance = OneManualOneAutomaticActivityWorkflow.execute(
template='instructions.tmpl')
self.workflow_execution = instance.workflow_execution
def complete_this_activity():
activities_client = ManualActivityCompletionClient(swf_client)
with open('task_token.txt', 'r') as shared_file:
task_token = shared_file.read()
os.remove('task_token.txt')
activities_client.complete((3, 4), task_token)
wf_worker.run_once()
act_worker.run_once()
time.sleep(5)
activity_finisher = Thread(target=complete_this_activity)
activity_finisher.start()
activity_finisher.join()
wf_worker.run_once()
act_worker.run_once()
wf_worker.run_once()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 17)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(
self.serializer.loads(
hist[-1]['workflowExecutionCompletedEventAttributes']
['result']), 7)
def get_workers(self,
workflow_class,
threaded_act_worker=False,
activities=BunchOfActivities()):
if not isinstance(workflow_class, list):
workflow_class = [workflow_class]
wf_worker = WorkflowWorker(self.session, self.region, self.domain,
self.task_list, *workflow_class)
act_worker = ActivityWorker(self.session, self.region, self.domain,
self.task_list, activities)
if threaded_act_worker:
act_worker = ThreadedActivityExecutor(act_worker)
return wf_worker, act_worker
def test_try_except_with_timer(self):
class TryExceptFinallyWorkflow(WorkflowDefinition):
@execute(version='1.1', execution_start_to_close_timeout=60)
def execute(self, arg1, arg2):
@coroutine
def do_try_except():
arg_sum = 0
try:
arg_sum += yield BunchOfActivities.sum(arg1, arg2)
yield BunchOfActivities.throw()
except ActivityTaskFailedError as err:
if isinstance(err.cause, ValueError) \
and str(err.cause) == 'Hello-Error':
if err.event_id != 13 or err.activity_id != '2':
raise RuntimeError("Test Failed")
arg_sum += yield BunchOfActivities.sum(arg1, arg2)
yield workflow_time.sleep(1)
return_(arg_sum)
result = yield do_try_except()
return_(result)
wf_worker = WorkflowWorker(self.session, self.region, self.domain,
self.task_list, TryExceptFinallyWorkflow)
act_worker = ActivityWorker(self.session, self.region, self.domain,
self.task_list, BunchOfActivities())
with workflow_starter(self.session, self.region, self.domain,
self.task_list):
instance = TryExceptFinallyWorkflow.execute(arg1=1, arg2=2)
self.workflow_execution = instance.workflow_execution
for i in range(3):
wf_worker.run_once()
act_worker.run_once()
# Once for the timer
wf_worker.run_once()
# Once for the completion
wf_worker.run_once()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 28)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(
self.serializer.loads(
hist[-1]['workflowExecutionCompletedEventAttributes']
['result']), 6)
def test_activity_exception_retries(self):
class ActivityRetryOnExceptionWorkflow(WorkflowDefinition):
@execute(version='1.2', execution_start_to_close_timeout=60)
def execute(self):
try:
yield RetryingActivities.activity_raises_errors(RuntimeError)
except ActivityTaskFailedError as err:
assert isinstance(err.cause, RuntimeError)
try:
yield RetryingActivities.activity_raises_errors(AttributeError)
except ActivityTaskFailedError as err:
assert isinstance(err.cause, AttributeError)
pass
wf_worker = WorkflowWorker(
self.session, self.region, self.domain, self.task_list, ActivityRetryOnExceptionWorkflow)
act_worker = ActivityWorker(
self.session, self.region, self.domain, self.task_list, RetryingActivities())
with workflow_starter(self.session, self.region, self.domain, self.task_list):
instance = ActivityRetryOnExceptionWorkflow.execute()
self.workflow_execution = instance.workflow_execution
for i in range(2):
wf_worker.run_once()
act_worker.run_once()
# for the timer
wf_worker.run_once()
act_worker.run_once()
wf_worker.run_once()
# wf_worker.run_once()
# print 'wfrun'
time.sleep(1)
# check that we have a timer started and that the workflow length is the same to validate
# that retries happened only on one of the exceptions
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 28)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(hist[10]['eventType'], 'TimerStarted')
self.assertEqual(hist[10]['timerStartedEventAttributes']['startToFireTimeout'], "0")
def test_one_activity_default_task_list(self):
class OneActivityCustomTaskList(object):
@activity(version='1.1', task_list='abracadabra')
def sum(self, x, y):
return x + y
class OneActivityDefaultTaskListWorkflow(WorkflowDefinition):
@execute(version='1.1', execution_start_to_close_timeout=60)
def execute(self, arg1, arg2):
arg_sum = yield OneActivityCustomTaskList.sum(arg1, arg2)
return_(arg_sum)
wf_worker = WorkflowWorker(self.session, self.region, self.domain,
self.task_list,
OneActivityDefaultTaskListWorkflow)
act_worker = ThreadedActivityExecutor(
ActivityWorker(self.session, self.region, self.domain,
'abracadabra', OneActivityCustomTaskList()))
with workflow_starter(self.session, self.region, self.domain,
self.task_list):
instance = OneActivityDefaultTaskListWorkflow.execute(arg1=1,
arg2=2)
self.workflow_execution = instance.workflow_execution
wf_worker.run_once()
act_worker.start(1, 4)
act_worker.stop()
wf_worker.run_once()
act_worker.join()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 11)
self.assertEqual(
hist[4]['activityTaskScheduledEventAttributes']['taskList']
['name'], 'abracadabra')
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(
self.serializer.loads(
hist[-1]['workflowExecutionCompletedEventAttributes']
['result']), 3)
def test_two_workflows(self):
wf_worker = WorkflowWorker(self.session, self.region, self.domain,
self.task_list, OneMultiWorkflow,
TwoMultiWorkflow)
act_worker = ActivityWorker(self.session, self.region, self.domain,
self.task_list, BunchOfActivities())
with workflow_starter(self.session, self.region, self.domain,
self.task_list):
instance = OneMultiWorkflow.execute(arg1=1, arg2=2)
self.workflow_executions.append(instance.workflow_execution)
instance = TwoMultiWorkflow.execute(arg1=1, arg2=2)
self.workflow_executions.append(instance.workflow_execution)
for i in range(2):
wf_worker.run_once()
act_worker.run_once()
wf_worker.run_once()
wf_worker.run_once()
time.sleep(1)
def test_any_future_activities(self):
class SleepingActivities(object):
@activity(version='1.2',
schedule_to_start_timeout=60,
start_to_close_timeout=60)
def sleep(self, time_to_sleep):
time.sleep(time_to_sleep)
return time_to_sleep
class AnyFutureWorkflow(WorkflowDefinition):
@execute(version='1.1', execution_start_to_close_timeout=60)
def execute(self, arg1, arg2):
sleep1_future = SleepingActivities.sleep(arg1)
sleep2_future = SleepingActivities.sleep(arg2)
result = yield sleep1_future | sleep2_future
return_(result)
wf_worker = WorkflowWorker(self.session, self.region, self.domain,
self.task_list, AnyFutureWorkflow)
act_worker = ActivityWorker(self.session, self.region, self.domain,
self.task_list, SleepingActivities())
with workflow_starter(self.session, self.region, self.domain,
self.task_list):
instance = AnyFutureWorkflow.execute(arg1=5, arg2=1)
self.workflow_execution = instance.workflow_execution
wf_worker.run_once()
act_worker.run_once()
act_worker.run_once()
wf_worker.run_once()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 14)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertTrue(
self.serializer.loads(
hist[-1]['workflowExecutionCompletedEventAttributes']
['result']))
def test_one_activity_with_timer(self):
class OneActivityWithTimerWorkflow(WorkflowDefinition):
def __init__(self, workflow_execution):
super(OneActivityWithTimerWorkflow,
self).__init__(workflow_execution)
self.activities_client = BunchOfActivities()
@execute(version='1.1', execution_start_to_close_timeout=60)
def execute(self, arg1, arg2):
yield workflow_time.sleep(2)
arg_sum = yield self.activities_client.sum(arg1, arg2)
return_(arg_sum)
wf_worker = WorkflowWorker(self.session, self.region, self.domain,
self.task_list,
OneActivityWithTimerWorkflow)
act_worker = ActivityWorker(self.session, self.region, self.domain,
self.task_list, BunchOfActivities())
with workflow_starter(self.session, self.region, self.domain,
self.task_list):
instance = OneActivityWithTimerWorkflow.execute(arg1=1, arg2=2)
self.workflow_execution = instance.workflow_execution
wf_worker.run_once()
wf_worker.run_once()
act_worker.run_once()
wf_worker.run_once()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 16)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
# timer specific checks
self.assertEqual(hist[4]['eventType'], 'TimerStarted')
self.assertEqual(
hist[4]['timerStartedEventAttributes']['startToFireTimeout'], '2')
self.assertEqual(hist[5]['eventType'], 'TimerFired')
def test_activity_timeouts(self):
class ActivityRetryOnTimeoutWorkflow(WorkflowDefinition):
@execute(version='1.2', execution_start_to_close_timeout=60)
def execute(self, sleep):
try:
yield RetryingActivities.activity_timing_out(sleep)
except ActivityTaskTimedOutError:
pass
wf_worker = WorkflowWorker(
self.session, self.region, self.domain, self.task_list, ActivityRetryOnTimeoutWorkflow)
act_worker = ActivityWorker(
self.session, self.region, self.domain, self.task_list, RetryingActivities())
with workflow_starter(self.session, self.region, self.domain, self.task_list):
instance = ActivityRetryOnTimeoutWorkflow.execute(sleep=2)
self.workflow_execution = instance.workflow_execution
for i in range(2):
wf_worker.run_once()
try:
act_worker.run_once()
except swf_exceptions.UnknownResourceError:
# we expect the activity to have timed out already
pass
# for the timer
wf_worker.run_once()
time.sleep(1)
hist = self.get_workflow_execution_history()
self.assertEqual(len(hist), 22)
self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted')
self.assertEqual(hist[10]['eventType'], 'TimerStarted')
self.assertEqual(hist[10]['timerStartedEventAttributes']['startToFireTimeout'], "1")