def run(self, identity=None):
"""Poll for and run a decision task
This should be run in a loop. It will poll for up to 60 seconds. After
60 seconds, it will return without running any decision tasks. The user
should usually not need to interact with this class directly. Instead,
:class:`DeciderServer` can be used to run the loop.
:return: None
"""
decision_task = self.poll(identity=identity)
decisions = swf.Layer1Decisions()
try:
if 'events' not in decision_task:
logger.debug('LuigiSwfDecider().run(), poll timed out')
return
events = self._get_events(decision_task)
task_configs = self._get_task_configurations(
events, decision_task['workflowExecution']['runId'])
state = WfState()
state.read_wf_state(events, task_configs)
self._decide(state, decisions, task_configs)
self.complete(decisions=decisions)
except Exception as error:
tb = traceback.format_exc()
reason = ('Decider failed:\n' + str(error))[:255]
details = (tb + '\n' + str(error))[:32767]
logger.error('LuigiSwfDecider().run(), decider failed:\n%s',
details)
decisions.fail_workflow_execution(reason=reason, details=details)
self.complete(decisions=decisions)
raise
def run(self, identity=None):
"""Run the decider.
The decider defines which task needs to be launched and when based on
the list of events provided. It looks at the list of all the available
activities, and launch the ones that:
* are not been scheduled yet.
* have all the dependencies resolved.
If the decider is not able to find an uncompleted activity, the
workflow can safely mark its execution as complete.
Args:
identity (str): Identity of the worker making the request, which
is recorded in the DecisionTaskStarted event in the AWS
console. This enables diagnostic tracing when problems arise.
Return:
boolean: Always return true, so any loop on run can act as a long
running process.
"""
try:
poll = self.poll(identity=identity)
except Exception as error:
# Catch exceptions raised during poll() to avoid a Decider thread
# dying & the daemon unable to process subsequent workflows.
# AWS api limits on SWF calls are a common source of such
# exceptions.
# on_exception() can be overriden by the flow to send an alert
# when such an exception occurs.
if self.on_exception:
self.on_exception(self, error)
self.logger.error(error, exc_info=True)
return True
custom_decider = getattr(self.flow, 'decider', None)
if 'events' not in poll:
return True
history = self.get_history(poll)
activity_states = self.get_activity_states(history)
current_context = event.get_current_context(history)
current_context.set_workflow_execution_info(poll, self.domain)
decisions = swf.Layer1Decisions()
if not custom_decider:
self.create_decisions_from_flow(decisions, activity_states,
current_context)
else:
self.delegate_decisions(decisions, custom_decider, activity_states,
current_context)
self.complete(decisions=decisions)
return True
def run(self):
history = self.poll()
# Print history to familiarize yourself with its format.
print history
if 'events' in history:
# Get a list of non-decision events to see what event came in last.
workflow_events = [
e for e in history['events']
if not e['eventType'].startswith('Decision')
]
decisions = swf.Layer1Decisions()
# Record latest non-decision event.
last_event = workflow_events[-1]
last_event_type = last_event['eventType']
if last_event_type == 'WorkflowExecutionStarted':
# At the start, get the worker to fetch the first assignment.
decisions.schedule_activity_task('%s-%i' %
(ACTIVITY, time.time()),
ACTIVITY,
VERSION,
task_list=TASK_LIST)
elif last_event_type == 'ActivityTaskCompleted':
# Take decision based on the name of activity that has just completed.
# 1) Get activity's event id.
last_event_attrs = last_event[
'activityTaskCompletedEventAttributes']
completed_activity_id = last_event_attrs['scheduledEventId'] - 1
# 2) Extract its name.
activity_data = history['events'][completed_activity_id]
activity_attrs = activity_data[
'activityTaskScheduledEventAttributes']
activity_name = activity_attrs['activityType']['name']
# 3) Optionally, get the result from the activity.
result = last_event[
'activityTaskCompletedEventAttributes'].get('result')
# Take the decision.
if activity_name == ACTIVITY:
# only one activity, we're done
decisions.complete_workflow_execution()
self.complete(decisions=decisions)
return True
def run(self):
"""Run the decider.
The decider defines which task needs to be launched and when based on
the list of events provided. It looks at the list of all the available
activities, and launch the ones that:
* are not been scheduled yet.
* have all the dependencies resolved.
If the decider is not able to find an uncompleted activity, the
workflow can safely mark its execution as complete.
Return:
boolean: Always return true, so any loop on run can act as a long
running process.
"""
poll = self.poll()
custom_decider = getattr(self.flow, 'decider', None)
if 'events' not in poll:
return True
history = self.get_history(poll)
activity_states = self.get_activity_states(history)
current_context = event.get_current_context(history)
current_context.set_workflow_execution_info(poll, self.domain)
decisions = swf.Layer1Decisions()
if not custom_decider:
self.create_decisions_from_flow(decisions, activity_states,
current_context)
else:
self.delegate_decisions(decisions, custom_decider, activity_states,
current_context)
self.complete(decisions=decisions)
return True
def run(self):
history = self.poll()
# Print history to familiarize yourself with its format.
print history
print ""
if 'events' in history:
# Get a list of non-decision events to see what event came in last.
workflow_events = [
e for e in history['events']
if not e['eventType'].startswith('Decision')
]
decisions = swf.Layer1Decisions()
# Record latest non-decision event.
last_event = workflow_events[-1]
last_event_type = last_event['eventType']
if last_event_type == 'WorkflowExecutionStarted':
# At the start, get the worker to fetch the first assignment.
activity = self.activities[0].strip()
task_list = activity + 'TaskList'
input_data = last_event[
'workflowExecutionStartedEventAttributes']['input']
decisions.schedule_activity_task('%s-%s' %
(activity, time.ctime()),
activity,
self.version,
task_list=task_list,
input=input_data)
elif last_event_type == 'ActivityTaskCompleted':
# Take decision based on the name of activity that has just completed.
# Get activity's event id.
last_event_attrs = last_event[
'activityTaskCompletedEventAttributes']
completed_activity_id = last_event_attrs['scheduledEventId'] - 1
# Extract its name.
activity_data = history['events'][completed_activity_id]
activity_attrs = activity_data[
'activityTaskScheduledEventAttributes']
activity_name = activity_attrs['activityType']['name']
# Get the result from the activity.
result = last_event[
'activityTaskCompletedEventAttributes'].get('result')
# Take the decision.
activities_number = len(self.activities)
for i in range(0, activities_number):
activity = self.activities[i].strip()
if activity_name == activity:
if i < activities_number - 1:
next_activity = self.activities[i + 1].strip()
next_task_list = next_activity + 'TaskList'
decisions.schedule_activity_task(
'%s-%s' % (next_activity, time.ctime()),
next_activity,
self.version,
task_list=next_task_list,
input=result)
else:
# the last activity, we're done.
decisions.complete_workflow_execution()
break
self.complete(decisions=decisions)
return True
def _run(self, events, workflowExecution):
# Run the statemachine on the events
results = self.statemachine.eval(events)
# Now we can do 4 things:
# - Complete the workflow
# - Fail the workflow
# - Schedule more activities
# - Nothing
decisions = swf.Layer1Decisions()
if self.statemachine.is_succeeded:
self.sqs.send_message(
self.output_queue,
json.dumps({
'time': time.time(),
'type': 'WORKFLOW_COMPLETED',
'data': {
'workflow': workflowExecution
}
}))
decisions.complete_workflow_execution(result=None)
return decisions
elif self.statemachine.is_failed:
# FIXME: Improve error reporting
self.sqs.send_message(
self.output_queue,
json.dumps({
'time': time.time(),
'type': 'WORKFLOW_FAILED',
'data': {
'workflow': workflowExecution
}
}))
decisions.fail_workflow_execution(reason='State machine aborted')
return decisions
# We are still going, start any ready activity
for next_step in results:
activity = next_step.activity
# FIXME: We are assuming JSON activity input here
activity_input = (json.dumps(next_step.activity_input) if
next_step.activity_input is not None else None)
self.sqs.send_message(
self.output_queue,
json.dumps({
'time': time.time(),
'type': 'ACTIVITY_SCHEDULED',
'data': {
'workflow': workflowExecution,
'activity': {
'activityId': next_step.name,
'activityType': activity.name,
'activityVersion': activity.version
}
}
}))
decisions.schedule_activity_task(
activity_id=next_step.name,
activity_type_name=activity.name,
activity_type_version=activity.version,
task_list=activity.task_list,
control=None, # FIXME: Do we want to pass context data?
heartbeat_timeout=activity.heartbeat_timeout,
schedule_to_close_timeout=activity.schedule_to_close_timeout,
schedule_to_start_timeout=activity.schedule_to_start_timeout,
start_to_close_timeout=activity.start_to_close_timeout,
input=activity_input,
)
return decisions
def run(self):
history = self.poll()
# poll timed out (60s) w/ nothing to do
if not 'events' in history: return True
workflow_events = [
e for e in history['events']
if not e['eventType'].startswith('Decision')
]
decisions = swf.Layer1Decisions()
# Record latest non-decision event.
last_event = workflow_events[-1]
last_event_type = last_event['eventType']
if last_event_type == 'WorkflowExecutionStarted':
startInput = last_event[
'workflowExecutionStartedEventAttributes'].get('input')
print "Scheduling First workflow action, prefix: %s" % startInput
# Schedule the first activity.
activityId = '%s-%s' % (workers.CreateCombineList.name,
time.time())
input = {
'sourceBucket': 'telemetry-test-bucket',
'prefix': startInput,
'outputBucket': 'telemetry-test-bucket'
}
decisions.schedule_activity_task(
activityId,
workers.CreateCombineList.name,
workers.CreateCombineList.version,
workers.CreateCombineList.task_list,
input=json.dumps(input))
elif last_event_type == 'ActivityTaskFailed':
print json.dumps(last_event)
reason = last_event['activityTaskFailedEventAttributes']['reason']
if reason == 'no-files':
print "Nothing to do. Ending workflow."
decisions.complete_workflow_execution(
result='Nothing to do. Ending early.')
else:
# probably need something a bit more robust here w/ retry
# logic ...
decisions.fail_workflow_execution(reason=reason)
elif last_event_type == 'ActivityTaskCompleted':
# Take decision based on the name of activity that has just completed.
# 1) Get activity's event id.
last_event_attrs = last_event[
'activityTaskCompletedEventAttributes']
completed_activity_id = last_event_attrs['scheduledEventId'] - 1
# 2) Extract its name.
activity_data = history['events'][completed_activity_id]
activity_attrs = activity_data[
'activityTaskScheduledEventAttributes']
activity_name = activity_attrs['activityType']['name']
# 3) Optionally, get the result from the activity.
result = last_event['activityTaskCompletedEventAttributes'].get(
'result')
if activity_name == workers.CreateCombineList.name:
print 'Scheduling activity: %s ver: %s' % (
workers.CombineSourceObjects.name,
workers.CombineSourceObjects.version)
activityId = '%s-%s' % (workers.CombineSourceObjects.name,
time.time())
decisions.schedule_activity_task(
activityId,
workers.CombineSourceObjects.name,
workers.CombineSourceObjects.version,
workers.CombineSourceObjects.task_list,
input=result)
if activity_name == workers.CombineSourceObjects.name:
print "Completing Workflow"
decisions.complete_workflow_execution()
self.complete(decisions=decisions)
return True
def test_decide():
# Setup
state = fixture_state()
task_configs = fixture_task_configs()
decisions = swf.Layer1Decisions()
decider.LuigiSwfDecider.__init__ = lambda s: None
uut = decider.LuigiSwfDecider()
# Execute
uut._decide(state, decisions, task_configs)
# Test
actual = decisions._data
expected = [
{
'decisionType': 'ScheduleActivityTask',
'scheduleActivityTaskDecisionAttributes': {
'activityId':
'Task2',
'activityType': {
'name': 'Task2',
'version': 'version1'
},
'taskList': {
'name': 'default'
},
'scheduleToCloseTimeout':
'0',
'scheduleToStartTimeout':
'0',
'startToCloseTimeout':
'0',
'heartbeatTimeout':
'0',
'input':
json.dumps({
'class': ('aoeu', 'Task2'),
'params': {}
},
default=util.dthandler),
},
},
{
'decisionType': 'ScheduleActivityTask',
'scheduleActivityTaskDecisionAttributes': {
'activityId':
'Task4',
'activityType': {
'name': 'Task4',
'version': 'version1'
},
'taskList': {
'name': 'default'
},
'scheduleToCloseTimeout':
'0',
'scheduleToStartTimeout':
'0',
'startToCloseTimeout':
'0',
'heartbeatTimeout':
'0',
'input':
json.dumps({
'class': ('aoeu', 'Task4'),
'params': {}
},
default=util.dthandler),
},
},
]
assert normalize_decisions(actual) == normalize_decisions(expected)