def submit(job, protocol=None, priority=1):
"""Submit mass job to SWF with specific priority.
"""
client = boto3.client('swf', region_name=config.REGION)
handler = InputHandler(protocol)
res = client.start_workflow_execution(
domain=config.DOMAIN,
workflowId=job.title,
workflowType=config.WORKFLOW_TYPE_FOR_JOB,
taskList={'name': config.DECISION_TASK_LIST},
taskPriority=str(priority),
input=json.dumps({
'protocol': protocol,
'body': handler.save(
data=job,
job_title=job.title,
task_title=job.title
)
}),
executionStartToCloseTimeout=str(config.WORKFLOW_EXECUTION_START_TO_CLOSE_TIMEOUT),
tagList=[job.title],
taskStartToCloseTimeout=str(config.DECISION_TASK_START_TO_CLOSE_TIMEOUT),
childPolicy=config.WORKFLOW_CHILD_POLICY)
return job.title, res['runId']
def execute_action(self, action, priority):
"""Schedule action to SWF as activity task and wait. If action is not
completed, raise TaskWait.
"""
if self.handler.is_waiting():
raise TaskWait
elif self.handler.is_scheduled():
return
else:
handler = InputHandler(self.handler.protocol)
action_name = self.handler.get_next_activity_name()
ActivityTask.schedule(
self.decisions,
name=action_name,
input_data={
'protocol':
self.handler.protocol,
'body':
handler.save(data=action,
genealogy=self.handler.tag_list +
['Action%s' % action_name])
},
task_list=action['Action'].get('_role',
config.ACTIVITY_TASK_LIST),
priority=priority)
def __init__(self, events, activity_max_retry=0, workflow_max_retry=0):
self.events = []
self.activity_max_retry = activity_max_retry
self.workflow_max_retry = workflow_max_retry
self.activity_newbe_count = 0
self.workflow_newbe_count = 0
def get_start_event(events):
events = map(Event, events)
events = [e for e in events if not e.event_type.startswith('Decision')]
return events[0]
start_event = get_start_event(events)
self.input = json.loads(start_event.input)
self.tag_list = start_event.tag_list
self.priority = int(start_event.task_priority)
input_ = json.loads(start_event.input)
self.protocol = input_['protocol']
handler = InputHandler(self.protocol)
self.input = handler.load(input_['body'])
swf_event_groups = self.classify_events(
events, self.activity_max_retry, self.workflow_max_retry)
def to_event(swf_events):
if 'ActivityTask' in swf_events[0].event_type:
return ActivityTask(swf_events, self.activity_max_retry)
elif 'ChildWorkflowExecution' in swf_events[0].event_type:
return ChildWorkflowExecution(swf_events, self.workflow_max_retry)
self.events = [
to_event(swf_events) for swf_events in swf_event_groups.values()]
self.events = sorted(self.events, key=lambda a: a.created_time())
def run(self, task_list):
"""Poll activity task from SWF and process.
"""
task = self.poll(task_list)
if not task:
return
activity_input = json.loads(task['input'])
handler = InputHandler(activity_input['protocol'])
action = handler.load(activity_input['body'])
try:
result = self.execute(action)
except TaskError as err:
self.client.respond_activity_task_failed(
taskToken=task['taskToken'],
details=err.details,
reason=err.reason)
except:
_, error, _ = sys.exc_info()
self.client.respond_activity_task_failed(
taskToken=task['taskToken'],
details=json.dumps(traceback.format_exc()),
reason=repr(error))
else:
self.client.respond_activity_task_completed(
taskToken=task['taskToken'],
result=json.dumps(result))
def submit(job, protocol=None, priority=1, scheduler='swf', domain=None, region=None):
"""Submit mass job to SWF with specific priority.
"""
if scheduler != 'swf':
raise UnsupportedScheduler(scheduler)
from mass.scheduler.swf import config
import boto3
client = boto3.client(
'swf',
region_name=region or config.REGION,
config=Config(connect_timeout=config.CONNECT_TIMEOUT,
read_timeout=config.READ_TIMEOUT))
handler = InputHandler(protocol)
job_title = job['Job']['title']
res = client.start_workflow_execution(
domain=domain or config.DOMAIN,
workflowId=job_title,
workflowType=config.WORKFLOW_TYPE_FOR_JOB,
taskList={'name': config.DECISION_TASK_LIST},
taskPriority=str(priority),
input=json.dumps({
'protocol': protocol,
'body': handler.save(
data=job,
genealogy=[job_title]
)
}),
executionStartToCloseTimeout=str(config.WORKFLOW_EXECUTION_START_TO_CLOSE_TIMEOUT),
tagList=[job_title],
taskStartToCloseTimeout=str(config.DECISION_TASK_START_TO_CLOSE_TIMEOUT),
childPolicy=config.WORKFLOW_CHILD_POLICY)
return job_title, res['runId']
def run(self, task_list):
"""Poll activity task from SWF and process.
"""
task = self.poll(task_list)
if not task:
return
activity_input = json.loads(task['input'])
handler = InputHandler(activity_input['protocol'])
action = handler.load(activity_input['body'])
result = self.execute_action(action)
if result['status'] == 'completed':
self.client.respond_activity_task_completed(
taskToken=self.task_token,
result=json.dumps(result['result'])[:config.MAX_RESULT_SIZE])
else:
self.client.respond_activity_task_failed(
taskToken=self.task_token,
details=result['details'][:config.MAX_DETAIL_SIZE],
reason=result['reason'][:config.MAX_REASON_SIZE])
def run(self, task_list):
"""Poll activity task from SWF and process.
"""
task = self.poll(task_list)
if not task:
return
activity_input = json.loads(task['input'])
handler = InputHandler(activity_input['protocol'])
action = handler.load(activity_input['body'])
result = self.execute_action(action)
if result['status'] == 'completed':
self.client.respond_activity_task_completed(
taskToken=self.task_token,
result=json.dumps(result['result'])[:config.MAX_RESULT_SIZE])
else:
self.client.respond_activity_task_failed(
taskToken=self.task_token,
details=result['details'][:config.MAX_DETAIL_SIZE],
reason=result['reason'][:config.MAX_REASON_SIZE])
def execute_task(self, task, priority):
"""Schedule task to SWF as child workflow and wait. If the task is not
completed, raise TaskWait.
"""
if self.handler.is_waiting():
raise TaskWait
elif self.handler.is_scheduled():
return
else:
handler = InputHandler(self.handler.protocol)
ChildWorkflowExecution.start(
decisions=self.decisions,
name=self.handler.get_next_workflow_name(task['Task']['title']),
input_data={
'protocol': self.handler.protocol,
'body': handler.save(
data=task,
genealogy=self.handler.tag_list + [task['Task']['title']])
},
tag_list=self.handler.tag_list + [task['Task']['title']],
priority=priority)
def __init__(self, events, activity_max_retry=0, workflow_max_retry=0):
self.events = []
self.activity_max_retry = activity_max_retry
self.workflow_max_retry = workflow_max_retry
self.activity_newbe_count = 0
self.workflow_newbe_count = 0
def get_start_event(events):
events = map(Event, events)
events = [
e for e in events if not e.event_type.startswith('Decision')
]
return events[0]
start_event = get_start_event(events)
self.input = json.loads(start_event.input)
self.tag_list = start_event.tag_list
self.priority = int(start_event.task_priority)
input_ = json.loads(start_event.input)
self.protocol = input_['protocol']
handler = InputHandler(self.protocol)
self.input = handler.load(input_['body'])
swf_event_groups = self.classify_events(events,
self.activity_max_retry,
self.workflow_max_retry)
def to_event(swf_events):
if 'ActivityTask' in swf_events[0].event_type:
return ActivityTask(swf_events, self.activity_max_retry)
elif 'ChildWorkflowExecution' in swf_events[0].event_type:
return ChildWorkflowExecution(swf_events,
self.workflow_max_retry)
self.events = [
to_event(swf_events) for swf_events in swf_event_groups.values()
]
self.events = sorted(self.events, key=lambda a: a.created_time())
def execute_action(self, action, priority):
"""Schedule action to SWF as activity task and wait. If action is not
completed, raise TaskWait.
"""
if self.handler.is_waiting():
raise TaskWait
elif self.handler.is_scheduled():
return
else:
handler = InputHandler(self.handler.protocol)
action_name = self.handler.get_next_activity_name()
ActivityTask.schedule(
self.decisions,
name=action_name,
input_data={
'protocol': self.handler.protocol,
'body': handler.save(
data=action,
genealogy=self.handler.tag_list + ['Action%s' % action_name])
},
task_list=action['Action'].get('_role', config.ACTIVITY_TASK_LIST),
priority=priority
)
def execute_task(self, task, priority):
"""Schedule task to SWF as child workflow and wait. If the task is not
completed, raise TaskWait.
"""
if self.handler.is_waiting():
raise TaskWait
elif self.handler.is_scheduled():
return
else:
handler = InputHandler(self.handler.protocol)
ChildWorkflowExecution.start(
decisions=self.decisions,
name=self.handler.get_next_workflow_name(
task['Task']['title']),
input_data={
'protocol':
self.handler.protocol,
'body':
handler.save(data=task,
genealogy=self.handler.tag_list +
[task['Task']['title']])
},
tag_list=self.handler.tag_list + [task['Task']['title']],
priority=priority)
