def standalone(context, workflow, domain, workflow_id, execution_timeout, tags,
decision_tasks_timeout, input, nb_workers, heartbeat):
"""
This command spawn a decider and an activity worker to execute a workflow
with a single main process.
"""
if not workflow_id:
workflow_id = get_workflow(workflow).name
task_list = get_task_list(workflow_id)
logger.info('using task list {}'.format(task_list))
decider_proc = multiprocessing.Process(target=decider.command.start,
args=(
[workflow],
domain,
task_list,
))
decider_proc.start()
worker_proc = multiprocessing.Process(target=worker.command.start,
args=(
workflow,
domain,
task_list,
nb_workers,
heartbeat,
))
worker_proc.start()
print >> sys.stderr, 'starting workflow {}'.format(workflow)
ex = start_workflow.callback(
workflow,
domain,
workflow_id,
task_list,
execution_timeout,
tags,
decision_tasks_timeout,
input,
local=False,
)
while True:
time.sleep(2)
ex = helpers.get_workflow_execution(
domain,
ex.workflow_id,
ex.run_id,
)
if ex.status == ex.STATUS_CLOSED:
print >> sys.stderr, 'execution {} finished'.format(
ex.workflow_id, )
break
os.kill(worker_proc.pid, signal.SIGTERM)
worker_proc.join()
os.kill(decider_proc.pid, signal.SIGTERM)
decider_proc.join()
def activity_rerun(domain, workflow_id, run_id, input, scheduled_id,
activity_id):
# handle params
if not activity_id and not scheduled_id:
logger.error("Please supply --scheduled-id or --activity-id.")
sys.exit(1)
input_override = None
if input:
input_override = format.decode(input)
# find workflow execution
try:
wfe = helpers.get_workflow_execution(domain, workflow_id, run_id)
except (swf.exceptions.DoesNotExistError, IndexError):
logger.error("Couldn't find execution, exiting.")
sys.exit(1)
logger.info("Found execution: workflowId={} runId={}".format(
wfe.workflow_id, wfe.run_id))
# now rerun the specified activity
history = History(wfe.history())
history.parse()
task, args, kwargs, meta, params = helpers.find_activity(
history,
scheduled_id=scheduled_id,
activity_id=activity_id,
input=input_override,
)
kwargs["context"].update({
"workflow_id": wfe.workflow_id,
"run_id": wfe.run_id,
})
logger.debug("Found activity. Last execution:")
for line in json_dumps(params, pretty=True).split("\n"):
logger.debug(line)
if input_override:
logger.info("NB: input will be overriden with the passed one!")
logger.info("Will re-run: {}(*{}, **{}) [+meta={}]".format(
task, args, kwargs, meta))
# download binaries if needed
download_binaries(meta.get("binaries", {}))
# execute the activity task with the correct arguments
instance = ActivityTask(task, *args, **kwargs)
result = instance.execute()
if hasattr(instance, "post_execute"):
instance.post_execute()
logger.info("Result (JSON): {}".format(json_dumps(result, compact=False)))
def restart_workflow(domain, workflow_id, run_id):
ex = helpers.get_workflow_execution(domain, workflow_id, run_id)
history = ex.history()
ex.terminate()
new_ex = ex.workflow_type.start_execution(
ex.workflow_id,
task_list=ex.task_list,
execution_timeout=ex.execution_timeout,
input=history.events[0].input,
tag_list=ex.tag_list,
decision_tasks_timeout=ex.decision_tasks_timeout,
)
print '{workflow_id} {run_id}'.format(
workflow_id=new_ex.workflow_id,
run_id=new_ex.run_id,
)
def restart_workflow(domain, workflow_id, run_id):
ex = helpers.get_workflow_execution(domain, workflow_id, run_id)
history = ex.history()
ex.terminate(reason='workflow.restart')
new_ex = ex.workflow_type.start_execution(
ex.workflow_id,
task_list=ex.task_list,
execution_timeout=ex.execution_timeout,
input=history.events[0].input,
tag_list=ex.tag_list,
decision_tasks_timeout=ex.decision_tasks_timeout,
)
print('{workflow_id} {run_id}'.format(
workflow_id=new_ex.workflow_id,
run_id=new_ex.run_id,
))
def activity_rerun(domain,
workflow_id,
run_id,
input,
scheduled_id,
activity_id):
# handle params
if not activity_id and not scheduled_id:
logger.error("Please supply --scheduled-id or --activity-id.")
sys.exit(1)
input_override = None
if input:
input_override = format.decode(input)
# find workflow execution
try:
wfe = helpers.get_workflow_execution(domain, workflow_id, run_id)
except (swf.exceptions.DoesNotExistError, IndexError):
logger.error("Couldn't find execution, exiting.")
sys.exit(1)
logger.info("Found execution: workflowId={} runId={}".format(wfe.workflow_id, wfe.run_id))
# now rerun the specified activity
history = History(wfe.history())
history.parse()
task, args, kwargs, meta, params = helpers.find_activity(
history, scheduled_id=scheduled_id, activity_id=activity_id, input=input_override,
)
logger.debug("Found activity. Last execution:")
for line in json_dumps(params, pretty=True).split("\n"):
logger.debug(line)
if input_override:
logger.info("NB: input will be overriden with the passed one!")
logger.info("Will re-run: {}(*{}, **{}) [+meta={}]".format(task, args, kwargs, meta))
# download binaries if needed
download_binaries(meta.get("binaries", {}))
# execute the activity task with the correct arguments
instance = ActivityTask(task, *args, **kwargs)
result = instance.execute()
if hasattr(instance, 'post_execute'):
instance.post_execute()
logger.info("Result (JSON): {}".format(json_dumps(result, compact=False)))
def activity_rerun(domain, workflow_id, run_id, input, scheduled_id,
activity_id):
# handle params
if not activity_id and not scheduled_id:
logger.error("Please supply --scheduled-id or --activity-id.")
sys.exit(1)
input_override = None
if input:
input_override = json.loads(input)
# find workflow execution
try:
wfe = helpers.get_workflow_execution(domain, workflow_id, run_id)
except (swf.exceptions.DoesNotExistError, IndexError):
logger.error("Couldn't find execution, exiting.")
sys.exit(1)
logger.info("Found execution: workflowId={} runId={}".format(
wfe.workflow_id, wfe.run_id))
# now rerun the specified activity
history = History(wfe.history())
history.parse()
func, args, kwargs, params = helpers.find_activity(
history,
scheduled_id=scheduled_id,
activity_id=activity_id,
input=input_override,
)
logger.debug("Found activity. Last execution:")
for line in json_dumps(params, pretty=True).split("\n"):
logger.debug(line)
if input_override:
logger.info("NB: input will be overriden with the passed one!")
logger.info("Will re-run: {}(*{}, **{})".format(func.__name__, args,
kwargs))
# finally replay the function with the correct arguments
result = func(*args, **kwargs)
logger.info("Result (JSON): {}".format(json_dumps(result)))
def standalone(context,
workflow,
domain,
workflow_id,
execution_timeout,
tags,
decision_tasks_timeout,
input,
nb_workers,
heartbeat):
"""
This command spawn a decider and an activity worker to execute a workflow
with a single main process.
"""
if not workflow_id:
workflow_id = get_workflow(workflow).name
task_list = get_task_list(workflow_id)
logger.info('using task list {}'.format(task_list))
decider_proc = multiprocessing.Process(
target=decider.command.start,
args=(
[workflow],
domain,
task_list,
)
)
decider_proc.start()
worker_proc = multiprocessing.Process(
target=worker.command.start,
args=(
workflow,
domain,
task_list,
nb_workers,
heartbeat,
)
)
worker_proc.start()
print >> sys.stderr, 'starting workflow {}'.format(workflow)
ex = start_workflow.callback(
workflow,
domain,
workflow_id,
task_list,
execution_timeout,
tags,
decision_tasks_timeout,
input,
local=False,
)
while True:
time.sleep(2)
ex = helpers.get_workflow_execution(
domain,
ex.workflow_id,
ex.run_id,
)
if ex.status == ex.STATUS_CLOSED:
print >> sys.stderr, 'execution {} finished'.format(
ex.workflow_id,
)
break
os.kill(worker_proc.pid, signal.SIGTERM)
worker_proc.join()
os.kill(decider_proc.pid, signal.SIGTERM)
decider_proc.join()
def terminate_workflow(domain, workflow_id, run_id):
ex = helpers.get_workflow_execution(domain, workflow_id, run_id)
ex.terminate()
def standalone(context,
workflow,
domain,
workflow_id,
execution_timeout,
tags,
decision_tasks_timeout,
input,
input_file,
nb_workers,
nb_deciders,
heartbeat,
display_status,
repair,
force_activities,
):
"""
This command spawn a decider and an activity worker to execute a workflow
with a single main process.
"""
disable_boto_connection_pooling()
if force_activities and not repair:
raise ValueError(
"You should only use --force-activities with --repair."
)
if not workflow_id:
workflow_id = get_workflow(workflow).name
wf_input = None
if input or input_file:
wf_input = get_or_load_input(input_file, input)
if repair:
repair_run_id = None
if " " in repair:
repair, repair_run_id = repair.split(" ", 1)
# get the previous execution history, it will serve as "default history"
# for activities that succeeded in the previous execution
logger.info(
'retrieving history of previous execution: domain={} '
'workflow_id={} run_id={}'.format(domain, repair, repair_run_id)
)
previous_history = get_workflow_history(domain, repair, run_id=repair_run_id)
previous_history.parse()
# get the previous execution input if none passed
if not input and not input_file:
wf_input = previous_history.events[0].input
else:
previous_history = None
task_list = create_unique_task_list(workflow_id)
logger.info('using task list {}'.format(task_list))
decider_proc = multiprocessing.Process(
target=decider.command.start,
args=(
[workflow],
domain,
task_list,
),
kwargs={
'nb_processes': nb_deciders,
'repair_with': previous_history,
'force_activities': force_activities,
'is_standalone': True,
},
)
decider_proc.start()
worker_proc = multiprocessing.Process(
target=worker.command.start,
args=(
domain,
task_list,
),
kwargs={
'nb_processes': nb_workers,
'heartbeat': heartbeat,
},
)
worker_proc.start()
print('starting workflow {}'.format(workflow), file=sys.stderr)
ex = start_workflow.callback(
workflow,
domain,
workflow_id,
task_list,
execution_timeout,
tags,
decision_tasks_timeout,
json_dumps(wf_input),
None,
local=False,
)
while True:
time.sleep(2)
ex = helpers.get_workflow_execution(
domain,
ex.workflow_id,
ex.run_id,
)
if display_status:
print('status: {}'.format(ex.status), file=sys.stderr)
if ex.status == ex.STATUS_CLOSED:
print('execution {} finished'.format(ex.workflow_id), file=sys.stderr)
break
os.kill(worker_proc.pid, signal.SIGTERM)
worker_proc.join()
os.kill(decider_proc.pid, signal.SIGTERM)
decider_proc.join()
def terminate_workflow(domain, workflow_id, run_id):
ex = helpers.get_workflow_execution(domain, workflow_id, run_id)
ex.terminate()
def test_not_standalone(self):
decider_proc = multiprocessing.Process(
target=decider.command.start,
args=(
[
'tests.integration.workflow.ChainTestWorkflow',
'tests.integration.workflow.TestRunChild'
],
self.domain,
None,
),
kwargs={
'nb_processes': 1,
'repair_with': None,
'force_activities': False,
'is_standalone': False,
},
)
decider_proc.start()
worker_proc = multiprocessing.Process(
target=worker.command.start,
args=(
self.domain,
'quickstart',
),
kwargs={
'nb_processes': 1,
'heartbeat': 10,
},
)
worker_proc.start()
ex = start_workflow.callback(
'tests.integration.workflow.TestRunChild',
self.domain,
self.workflow_id,
task_list=None,
execution_timeout='10',
tags=None,
decision_tasks_timeout='10',
input='[]',
input_file=None,
local=False,
)
while True:
time.sleep(1)
ex = helpers.get_workflow_execution(
self.domain,
ex.workflow_id,
ex.run_id,
)
if ex.status == ex.STATUS_CLOSED:
break
expect(ex.status).to.equal(ex.STATUS_CLOSED)
expect(ex.close_status).to.equal(ex.CLOSE_STATUS_COMPLETED)
os.kill(worker_proc.pid, signal.SIGTERM)
worker_proc.join()
os.kill(decider_proc.pid, signal.SIGTERM)
decider_proc.join()
def standalone(
context,
workflow,
domain,
workflow_id,
execution_timeout,
tags,
decision_tasks_timeout,
input,
input_file,
nb_workers,
nb_deciders,
heartbeat,
display_status,
repair,
force_activities,
):
"""
This command spawn a decider and an activity worker to execute a workflow
with a single main process.
"""
disable_boto_connection_pooling()
if force_activities and not repair:
raise ValueError(
"You should only use --force-activities with --repair.")
workflow_class = get_workflow(workflow)
if not workflow_id:
workflow_id = workflow_class.name
wf_input = {}
if input or input_file:
wf_input = get_or_load_input(input_file, input)
if repair:
repair_run_id = None
if " " in repair:
repair, repair_run_id = repair.split(" ", 1)
# get the previous execution history, it will serve as "default history"
# for activities that succeeded in the previous execution
logger.info("retrieving history of previous execution: domain={} "
"workflow_id={} run_id={}".format(domain, repair,
repair_run_id))
workflow_execution = get_workflow_execution(domain,
repair,
run_id=repair_run_id)
previous_history = History(workflow_execution.history())
repair_run_id = workflow_execution.run_id
previous_history.parse()
# get the previous execution input if none passed
if not input and not input_file:
wf_input = previous_history.events[0].input
if not tags:
tags = workflow_execution.tag_list
else:
previous_history = None
repair_run_id = None
if not tags:
get_tag_list = getattr(workflow_class, "get_tag_list", None)
if get_tag_list:
tags = get_tag_list(workflow_class, *wf_input.get("args", ()),
**wf_input.get("kwargs", {}))
else:
tags = getattr(workflow_class, "tag_list", None)
if tags == Workflow.INHERIT_TAG_LIST:
tags = None
task_list = create_unique_task_list(workflow_id)
logger.info("using task list {}".format(task_list))
decider_proc = multiprocessing.Process(
target=decider.command.start,
args=(
[workflow],
domain,
task_list,
),
kwargs={
"nb_processes": nb_deciders,
"repair_with": previous_history,
"force_activities": force_activities,
"is_standalone": True,
"repair_workflow_id": repair or None,
"repair_run_id": repair_run_id,
},
)
decider_proc.start()
worker_proc = multiprocessing.Process(
target=worker.command.start,
args=(
domain,
task_list,
),
kwargs={
"nb_processes": nb_workers,
"heartbeat": heartbeat,
},
)
worker_proc.start()
print("starting workflow {}".format(workflow), file=sys.stderr)
ex = start_workflow.callback(
workflow,
domain,
workflow_id,
task_list,
execution_timeout,
tags,
decision_tasks_timeout,
format.input(wf_input),
None,
local=False,
)
while True:
time.sleep(2)
ex = helpers.get_workflow_execution(
domain,
ex.workflow_id,
ex.run_id,
)
if display_status:
print("status: {}".format(ex.status), file=sys.stderr)
if ex.status == ex.STATUS_CLOSED:
print("execution {} finished".format(ex.workflow_id),
file=sys.stderr)
break
os.kill(worker_proc.pid, signal.SIGTERM)
worker_proc.join()
os.kill(decider_proc.pid, signal.SIGTERM)
decider_proc.join()
def standalone(context,
workflow,
domain,
workflow_id,
execution_timeout,
tags,
decision_tasks_timeout,
input,
input_file,
nb_workers,
nb_deciders,
heartbeat,
display_status,
repair,
force_activities,
):
"""
This command spawn a decider and an activity worker to execute a workflow
with a single main process.
"""
disable_boto_connection_pooling()
if force_activities and not repair:
raise ValueError(
"You should only use --force-activities with --repair."
)
workflow_class = get_workflow(workflow)
if not workflow_id:
workflow_id = workflow_class.name
wf_input = {}
if input or input_file:
wf_input = get_or_load_input(input_file, input)
if repair:
repair_run_id = None
if " " in repair:
repair, repair_run_id = repair.split(" ", 1)
# get the previous execution history, it will serve as "default history"
# for activities that succeeded in the previous execution
logger.info(
'retrieving history of previous execution: domain={} '
'workflow_id={} run_id={}'.format(domain, repair, repair_run_id)
)
workflow_execution = get_workflow_execution(domain, repair, run_id=repair_run_id)
previous_history = History(workflow_execution.history())
repair_run_id = workflow_execution.run_id
previous_history.parse()
# get the previous execution input if none passed
if not input and not input_file:
wf_input = previous_history.events[0].input
if not tags:
tags = workflow_execution.tag_list
else:
previous_history = None
repair_run_id = None
if not tags:
get_tag_list = getattr(workflow_class, 'get_tag_list', None)
if get_tag_list:
tags = get_tag_list(workflow_class, *wf_input.get('args', ()), **wf_input.get('kwargs', {}))
else:
tags = getattr(workflow_class, 'tag_list', None)
if tags == Workflow.INHERIT_TAG_LIST:
tags = None
task_list = create_unique_task_list(workflow_id)
logger.info('using task list {}'.format(task_list))
decider_proc = multiprocessing.Process(
target=decider.command.start,
args=(
[workflow],
domain,
task_list,
),
kwargs={
'nb_processes': nb_deciders,
'repair_with': previous_history,
'force_activities': force_activities,
'is_standalone': True,
'repair_workflow_id': repair or None,
'repair_run_id': repair_run_id,
},
)
decider_proc.start()
worker_proc = multiprocessing.Process(
target=worker.command.start,
args=(
domain,
task_list,
),
kwargs={
'nb_processes': nb_workers,
'heartbeat': heartbeat,
},
)
worker_proc.start()
print('starting workflow {}'.format(workflow), file=sys.stderr)
ex = start_workflow.callback(
workflow,
domain,
workflow_id,
task_list,
execution_timeout,
tags,
decision_tasks_timeout,
format.input(wf_input),
None,
local=False,
)
while True:
time.sleep(2)
ex = helpers.get_workflow_execution(
domain,
ex.workflow_id,
ex.run_id,
)
if display_status:
print('status: {}'.format(ex.status), file=sys.stderr)
if ex.status == ex.STATUS_CLOSED:
print('execution {} finished'.format(ex.workflow_id), file=sys.stderr)
break
os.kill(worker_proc.pid, signal.SIGTERM)
worker_proc.join()
os.kill(decider_proc.pid, signal.SIGTERM)
decider_proc.join()
