def register(self):
"""Registers the workflow type and task types with SWF
It is necessary to do this each time a new task is added to a workflow.
It is safest to run this before each call to :meth:`execute` if you are
just launching a workflow from a cron. However, if you are launching
many workflows and calling :meth:`execute` many times, you may want to
consider calling this method only when necessary because it can
contribute to an SWF API throttling issue.
"""
tasks = get_task_configurations(self.workflow_task)
registerables = []
registerables.append(swf.Domain(name=self.domain))
task_dats = set((t['task_family'], t['task_list'])
for (t_id, t) in iteritems(tasks))
for task_dat in task_dats:
registerables.append(
swf.ActivityType(domain=self.domain,
version=self.version,
name=task_dat[0],
task_list=task_dat[1]))
wf_name = self.workflow_task.task_family
wf_task_list = getattr(self.workflow_task, 'swf_task_list', 'default')
registerables.append(
swf.WorkflowType(domain=self.domain,
version=self.version,
name=wf_name,
task_list=wf_task_list))
for swf_entity in registerables:
try:
swf_entity.register()
print_(swf_entity.name, 'registered successfully')
except (SWFDomainAlreadyExistsError, SWFTypeAlreadyExistsError):
print_(swf_entity.__class__.__name__, swf_entity.name,
'already exists')
def register(self):
"""Register the Workflow on SWF.
To work, SWF needs to have pre-registered the domain, the workflow,
and the different activities, this method takes care of this part.
"""
registerables = []
registerables.append(swf.Domain(name=self.domain))
registerables.append(
swf.WorkflowType(domain=self.domain,
name=self.task_list,
version=self.version,
task_list=self.task_list))
for current_activity in self.activities:
registerables.append(
swf.ActivityType(domain=self.domain,
name=current_activity.name,
version=self.version,
task_list=current_activity.task_list))
for swf_entity in registerables:
try:
swf_entity.register()
except (SWFDomainAlreadyExistsError, SWFTypeAlreadyExistsError):
print(swf_entity.__class__.__name__, swf_entity.name,
'already exists')
def execute(self, **kwargs):
"""Initiates a workflow execution on SWF and returns immediately
Run :meth:`register` first.
"""
wf_id = self.workflow_task.task_id
wf_input = json.dumps(
{
'wf_task': fullname(self.workflow_task),
'wf_params': get_luigi_params(self.workflow_task),
},
default=dthandler)
wf_type = swf.WorkflowType(domain=self.domain,
version=self.version,
name=self.workflow_task.task_family,
task_list='luigi')
timeout = getattr(self.workflow_task, 'swf_wf_start_to_close_timeout',
None)
if timeout is None:
timeout = 365 * days
if not isinstance(timeout, string_types) and \
not isinstance(timeout, text_type):
timeout = str(int(timeout))
logger.info('LuigiSwfExecutor().execute(), executing workflow %s',
wf_id)
execution = wf_type.start(workflow_id=wf_id,
input=wf_input,
execution_start_to_close_timeout=timeout,
**kwargs)
return execution
def execute_flow(flow, context, **kwargs):
"""Launch the workflow execution.
Args:
flow (module): Garcon flow module.
context (str): Initial context parsed from JSON.
"""
start_kwargs = dict(
workflow_id=flow.workflow_id(json.loads(context)),
input=context)
if hasattr(flow, 'timeout'):
start_kwargs.update(execution_start_to_close_timeout=str(flow.timeout))
execution = swf.WorkflowType(
name=flow.name,
domain=flow.domain,
version=flow.version,
task_list=flow.name).start(**start_kwargs)
return execution
# register.py
import boto.swf.layer2 as swf
import settings
from boto.swf.exceptions import SWFTypeAlreadyExistsError, SWFDomainAlreadyExistsError
DOMAIN = settings.DOMAIN
registerables = []
registerables.append(swf.Domain(name=DOMAIN))
# register workflow types, these all do the same thing
# but allow for different deciders
registerables.append(
swf.WorkflowType(domain=DOMAIN,
name='CombineObjects',
version='NoSourceModify',
task_list='WFHandsOff'))
registerables.append(
swf.WorkflowType(domain=DOMAIN,
name='CombineObjects',
version='CopySource',
task_list='WFCopySource'))
registerables.append(
swf.WorkflowType(domain=DOMAIN,
name='CombineObjects',
version='CopyAndDeleteSource',
task_list='WFCopyAndDelete'))
# workflow tasks
registerables.append(
swf.ActivityType(domain=DOMAIN,
from garcon import activity
from garcon import decider
from threading import Thread
import boto.swf.layer2 as swf
import time
import test_flow
deciderworker = decider.DeciderWorker(test_flow)
swf.WorkflowType(name=test_flow.name,
domain=test_flow.domain,
version='1.0',
task_list=test_flow.name).start()
Thread(target=activity.ActivityWorker(test_flow).run).start()
while (True):
deciderworker.run()
time.sleep(1)
import json
import boto.swf.layer2 as swf
import flow
initial_context = dict(foo='bar')
hello_world_flow = flow.Flow('dev', '1.0')
workflow_execution = swf.WorkflowType(
name=hello_world_flow.name,
domain=hello_world_flow.domain,
version=hello_world_flow.version,
task_list=hello_world_flow.name).start(input=json.dumps(initial_context))
print("Launching SWF for flow '{}' with WorkflowId '{}' and RunId '{}'".format(
workflow_execution.name, workflow_execution.workflowId,
workflow_execution.runId))
#! /usr/bin/env python
import ConfigParser
import boto.swf.layer2 as swf
cf = ConfigParser.ConfigParser()
cf.read('swf.conf')
domain = cf.get('SWF', 'DOMAIN')
activities = cf.get('SWF', 'ACTIVITIES').split(',')
version = cf.get('SWF', 'VERSION')
workflow = cf.get('SWF', 'WORKFLOW')
swf.Domain(name=domain).register()
for activity in activities:
activity = activity.strip()
task_list = activity+'TaskList'
swf.ActivityType(domain=domain, name=activity, version=version,task_list=task_list).register()
swf.WorkflowType(domain=domain, name=workflow, version=version, task_list='default_tasks').register()
#! /usr/bin/env python
import boto.swf.layer2 as swf
DOMAIN = 'HelloWorld'
ACTIVITY = 'HelloWorldActivity'
VERSION = '1.0'
TASK_LIST = 'HelloWorldTaskList'
WORKFLOW = 'HelloWorldWorkflow'
swf.Domain(name=DOMAIN).register()
swf.ActivityType(domain=DOMAIN,
name=ACTIVITY,
version=VERSION,
task_list=TASK_LIST).register()
swf.WorkflowType(domain=DOMAIN,
name=WORKFLOW,
version=VERSION,
task_list='default_tasks').register()
import boto.swf.layer2 as swf
import settings
wkf = swf.WorkflowType(domain=settings.DOMAIN,
name="CombineObjects",
version='NoSourceModify',
task_list='WFHandsOff')
print "Starting new Workflow"
wkf.start(
# what prefix to start combining for
#input="idle_daily/Firefox/nightly/27.0a1"
#input='idle_daily/Fennec/nightly/27.0a1/201309'
input='idle_daily/Fennec/nightly/27.0a1/201310')