def history(self):
# type: () -> Optional[History]
if not isinstance(self._history, History):
history = History(self._history)
history.parse()
return history
return self._history
def test_workflow_with_repair_and_force_activities():
workflow = ATestDefinitionWithInput
history = builder.History(workflow, input={'args': [4]})
# Now let's build the history to repair
previous_history = builder.History(workflow, input={'args': [4]})
decision_id = previous_history.last_id
(previous_history.add_activity_task(
increment,
decision_id=decision_id,
last_state='completed',
activity_id='activity-tests.data.activities.increment-1',
input={'args': 4},
result=57) # obviously wrong but helps see if things work
)
to_repair = History(previous_history)
to_repair.parse()
executor = Executor(DOMAIN,
workflow,
repair_with=to_repair,
force_activities="increment|something_else")
# The executor should not schedule anything, it should use previous history
decisions, _ = executor.replay(Response(history=history))
assert len(decisions) == 1
assert decisions[0]['decisionType'] == 'ScheduleActivityTask'
attrs = decisions[0]['scheduleActivityTaskDecisionAttributes']
assert not attrs['taskList']['name'].startswith("FAKE-")
check_task_scheduled_decision(decisions[0], increment)
def profile(workflow_execution, nb_tasks=None):
stats = WorkflowStats(History(workflow_execution.history()))
header = (
'Task',
'Last State',
'Scheduled',
'Time Scheduled',
'Start',
'Time Running',
'End',
'Percentage of total time',
)
values = ((task, last_state,
scheduled.strftime(TIME_FORMAT) if scheduled else None,
(start - scheduled).total_seconds() if scheduled else None,
start.strftime(TIME_FORMAT) if start else None,
(end - start).total_seconds() if start else None,
end.strftime(TIME_FORMAT) if end else None, percent)
for task, last_state, scheduled, start, end, timing, percent in (
row for row in stats.get_timings_with_percentage()
if row is not None))
rows = sorted(
values,
key=operator.itemgetter(5),
reverse=True,
)
if nb_tasks:
rows = rows[:nb_tasks]
return header, rows
def get_task(workflow_execution, task_id, details=False):
history = History(workflow_execution.history())
history.parse()
task = history.activities[task_id]
header = [
"type",
"id",
"name",
"version",
"state",
"timestamp",
"input",
"result",
"reason",
]
# TODO...
if details:
header.append("details")
# print >>sys.stderr, task
state = task["state"]
rows = [[
task["type"],
task["id"],
task["name"],
task["version"],
state,
task[state + "_timestamp"],
task["input"],
task.get("result"), # Absent for failed tasks
task.get("reason"),
]]
if details:
rows[0].append(task.get("details"))
return header, rows
def get_task(workflow_execution, task_id, details=False):
history = History(workflow_execution.history())
history.parse()
task = history._activities[task_id]
header = [
'type', 'id', 'name', 'version', 'state', 'timestamp', 'input',
'result', 'reason'
]
# TODO...
if details:
header.append('details')
# print >>sys.stderr, task
state = task['state']
rows = \
[
[
task['type'],
task['id'],
task['name'],
task['version'],
state,
task[state + '_timestamp'],
task['input'],
task.get('result'), # Absent for failed tasks
task.get('reason'),
]]
if details:
rows[0].append(task.get('details'))
return header, rows
def get_workflow_history(domain_name, workflow_id, run_id):
domain = swf.models.Domain(domain_name)
workflow_execution = (swf.querysets.WorkflowExecutionQuerySet(domain).get(
workflow_id=workflow_id,
run_id=run_id,
))
return History(workflow_execution.history())
def fake_history():
"""
Generates a SWF's History object like the SWF decider does, but from a fake
workflow execution history stored in a json file in tests/data/dumps/.
"""
with open("tests/data/dumps/workflow_execution_basic.json") as f:
basic_history_tree = json.loads(f.read())
basic_history = BasicHistory.from_event_list(basic_history_tree["events"])
return History(basic_history)
def status(workflow_execution, nb_tasks=None):
history = History(workflow_execution.history())
history.parse()
header = "Tasks", "Last State", "Last State Time", "Scheduled Time"
rows = [(task["name"], ) + get_timestamps(task)
for task in history.tasks[::-1]]
if nb_tasks:
rows = rows[:nb_tasks]
return header, rows
def status(workflow_execution, nb_tasks=None):
history = History(workflow_execution.history())
history.parse()
header = 'Tasks', 'Last State', 'Last State Time', 'Scheduled Time'
rows = [(task['name'], ) + get_timestamps(task)
for task in history._tasks[::-1]]
if nb_tasks:
rows = rows[:nb_tasks]
return header, rows
def get_workflow_history(domain_name, workflow_id, run_id=None):
"""
Get workflow history.
:param domain_name:
:type domain_name: str
:param workflow_id:
:type workflow_id: str
:param run_id:
:type run_id: str
:return:
:rtype: History
"""
workflow_execution = get_workflow_execution(domain_name, workflow_id, run_id=run_id)
return History(workflow_execution.history())
def test_last_state_times():
"""
This test checks an execution with a single activity tasks.
"""
history_builder = builder.History(ATestWorkflow)
last_state = 'completed'
activity_id = 'activity-tests.test_dataflow.increment-1'
history_builder.add_activity_task(
increment,
decision_id=0,
last_state=last_state,
activity_id=activity_id,
)
history = History(history_builder)
stats = WorkflowStats(history)
total_time = stats.total_time()
events = history.events
assert total_time == (events[-1].timestamp -
events[0].timestamp).total_seconds()
timings = stats.get_timings()[0]
assert timings[0] == activity_id
assert timings[1] == last_state
TIMING_SCHEDULED = 2
TIMING_STARTED = 3
TIMING_COMPLETED = 4
EV_SCHEDULED = -3
EV_STARTED = -2
EV_COMPLETED = -1
assert timings[TIMING_SCHEDULED] == events[EV_SCHEDULED].timestamp
assert timings[TIMING_STARTED] == events[EV_STARTED].timestamp
assert timings[TIMING_COMPLETED] == events[EV_COMPLETED].timestamp
TIMING_DURATION = 5
assert timings[TIMING_DURATION] == (
events[EV_COMPLETED].timestamp -
events[EV_STARTED].timestamp).total_seconds()
timings = stats.get_timings_with_percentage()[0]
TIMING_TOTAL_TIME = -2
TIMING_PERCENTAGE = -1
percentage = (timings[TIMING_TOTAL_TIME] / total_time) * 100.
assert percentage == timings[TIMING_PERCENTAGE]
def activity_rerun(domain, workflow_id, run_id, input, scheduled_id,
activity_id):
# handle params
if not activity_id and not scheduled_id:
logger.error("Please supply --scheduled-id or --activity-id.")
sys.exit(1)
input_override = None
if input:
input_override = format.decode(input)
# find workflow execution
try:
wfe = helpers.get_workflow_execution(domain, workflow_id, run_id)
except (swf.exceptions.DoesNotExistError, IndexError):
logger.error("Couldn't find execution, exiting.")
sys.exit(1)
logger.info("Found execution: workflowId={} runId={}".format(
wfe.workflow_id, wfe.run_id))
# now rerun the specified activity
history = History(wfe.history())
history.parse()
task, args, kwargs, meta, params = helpers.find_activity(
history,
scheduled_id=scheduled_id,
activity_id=activity_id,
input=input_override,
)
kwargs["context"].update({
"workflow_id": wfe.workflow_id,
"run_id": wfe.run_id,
})
logger.debug("Found activity. Last execution:")
for line in json_dumps(params, pretty=True).split("\n"):
logger.debug(line)
if input_override:
logger.info("NB: input will be overriden with the passed one!")
logger.info("Will re-run: {}(*{}, **{}) [+meta={}]".format(
task, args, kwargs, meta))
# download binaries if needed
download_binaries(meta.get("binaries", {}))
# execute the activity task with the correct arguments
instance = ActivityTask(task, *args, **kwargs)
result = instance.execute()
if hasattr(instance, "post_execute"):
instance.post_execute()
logger.info("Result (JSON): {}".format(json_dumps(result, compact=False)))
def get_task(workflow_execution, task_id):
history = History(workflow_execution.history())
history.parse()
task = history._activities[task_id]
header = ('type', 'id', 'name', 'version', 'state', 'timestamp', 'input',
'result')
state = task['state']
rows = [(
task['type'],
task['id'],
task['name'],
task['version'],
state,
task[state + '_timestamp'],
task['input'],
task['result'],
)]
return header, rows
def info(workflow_execution):
history = History(workflow_execution.history())
history.parse()
if history.tasks:
first_event = history.tasks[0]
first_timestamp = first_event[first_event["state"] + "_timestamp"]
last_event = history.tasks[-1]
last_timestamp = (last_event.get("timestamp")
or last_event[last_event["state"] + "_timestamp"])
workflow_input = first_event["input"]
else:
first_event = history.events[0]
first_timestamp = first_event.timestamp
last_event = history.events[-1]
last_timestamp = last_event.timestamp
workflow_input = first_event.input
execution_time = (last_timestamp - first_timestamp).total_seconds()
header = (
"domain",
"workflow_type.name",
"workflow_type.version",
"task_list",
"workflow_id",
"run_id",
"tag_list",
"execution_time",
"input",
)
ex = workflow_execution
rows = [(
ex.domain.name,
ex.workflow_type.name,
ex.workflow_type.version,
ex.task_list,
ex.workflow_id,
ex.run_id,
",".join(ex.tag_list),
execution_time,
workflow_input,
)]
return header, rows
def info(workflow_execution):
history = History(workflow_execution.history())
history.parse()
if history.tasks:
first_event = history.tasks[0]
first_timestamp = first_event[first_event['state'] + '_timestamp']
last_event = history.tasks[-1]
last_timestamp = last_event.get('timestamp') or last_event[
last_event['state'] + '_timestamp']
workflow_input = first_event['input']
else:
first_event = history.events[0]
first_timestamp = first_event.timestamp
last_event = history.events[-1]
last_timestamp = last_event.timestamp
workflow_input = first_event.input
execution_time = (last_timestamp - first_timestamp).total_seconds()
header = (
'domain',
'workflow_type.name',
'workflow_type.version',
'task_list',
'workflow_id',
'run_id',
'tag_list',
'execution_time',
'input',
)
ex = workflow_execution
rows = [(
ex.domain.name,
ex.workflow_type.name,
ex.workflow_type.version,
ex.task_list,
ex.workflow_id,
ex.run_id,
','.join(ex.tag_list),
execution_time,
workflow_input,
)]
return header, rows
def run(self, input=None):
if input is None:
input = {}
args = input.get('args', ())
kwargs = input.get('kwargs', {})
self.create_workflow()
self.initialize_history(input)
self.before_replay()
result = self.run_workflow(*args, **kwargs)
# Hack: self._history must be available to the callback as a
# simpleflow.history.History, not a swf.models.history.builder.History
self._history = History(self._history)
self._history.parse()
self.after_replay()
self.on_completed()
self.after_closed()
return result
def replay(self, history):
"""Executes the workflow from the start until it blocks.
"""
self.reset()
self._history = History(history)
self._history.parse()
workflow_started_event = history[0]
args = ()
kwargs = {}
input = workflow_started_event.input
if input is None:
input = {}
args = input.get('args', ())
kwargs = input.get('kwargs', {})
try:
result = self.run_workflow(*args, **kwargs)
except exceptions.ExecutionBlocked:
logger.info('{} open activities ({} decisions)'.format(
self._open_activity_count,
len(self._decisions),
))
return self._decisions, {}
except exceptions.TaskException, err:
reason = 'Workflow execution error in task {}: "{}"'.format(
err.task.name,
getattr(err.exception, 'reason', repr(err.exception)))
logger.exception(reason)
details = getattr(err.exception, 'details', None)
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
return [decision], {}
def activity_rerun(domain, workflow_id, run_id, input, scheduled_id,
activity_id):
# handle params
if not activity_id and not scheduled_id:
logger.error("Please supply --scheduled-id or --activity-id.")
sys.exit(1)
input_override = None
if input:
input_override = json.loads(input)
# find workflow execution
try:
wfe = helpers.get_workflow_execution(domain, workflow_id, run_id)
except (swf.exceptions.DoesNotExistError, IndexError):
logger.error("Couldn't find execution, exiting.")
sys.exit(1)
logger.info("Found execution: workflowId={} runId={}".format(
wfe.workflow_id, wfe.run_id))
# now rerun the specified activity
history = History(wfe.history())
history.parse()
func, args, kwargs, params = helpers.find_activity(
history,
scheduled_id=scheduled_id,
activity_id=activity_id,
input=input_override,
)
logger.debug("Found activity. Last execution:")
for line in json_dumps(params, pretty=True).split("\n"):
logger.debug(line)
if input_override:
logger.info("NB: input will be overriden with the passed one!")
logger.info("Will re-run: {}(*{}, **{})".format(func.__name__, args,
kwargs))
# finally replay the function with the correct arguments
result = func(*args, **kwargs)
logger.info("Result (JSON): {}".format(json_dumps(result)))
def test_workflow_with_repair_if_task_failed():
workflow = ATestDefinitionWithInput
history = builder.History(workflow, input={'args': [4]})
# Now let's build the history to repair
previous_history = builder.History(workflow, input={'args': [4]})
decision_id = previous_history.last_id
(previous_history.add_activity_task(
increment,
decision_id=decision_id,
last_state='failed',
activity_id='activity-tests.data.activities.increment-1',
input={'args': 4},
result=57) # obviously wrong but helps see if things work
)
to_repair = History(previous_history)
to_repair.parse()
executor = Executor(DOMAIN, workflow, repair_with=to_repair)
# The executor should not schedule anything, it should use previous history
decisions, _ = executor.replay(Response(history=history))
check_task_scheduled_decision(decisions[0], increment)
def replay(self, decision_response, decref_workflow=True):
"""Replay the workflow from the start until it blocks.
Called by the DeciderWorker.
:param decision_response: an object wrapping the PollForDecisionTask response
:type decision_response: swf.responses.Response
:param decref_workflow : Decref workflow once replay is done (to save memory)
:type decref_workflow : boolean
:returns: a list of decision and a context dict (obsolete, empty)
:rtype: ([swf.models.decision.base.Decision], dict)
"""
self.reset()
history = decision_response.history
self._history = History(history)
self._history.parse()
self.build_execution_context(decision_response)
self._execution = decision_response.execution
workflow_started_event = history[0]
input = workflow_started_event.input
if input is None:
input = {}
args = input.get('args', ())
kwargs = input.get('kwargs', {})
self.before_replay()
try:
self.propagate_signals()
result = self.run_workflow(*args, **kwargs)
except exceptions.ExecutionBlocked:
logger.info('{} open activities ({} decisions)'.format(
self._open_activity_count,
len(self._decisions),
))
self.after_replay()
if decref_workflow:
self.decref_workflow()
if self._append_timer:
self._add_start_timer_decision('_simpleflow_wake_up_timer')
return self._decisions, {}
except exceptions.TaskException as err:
reason = 'Workflow execution error in task {}: "{}"'.format(
err.task.name,
getattr(err.exception, 'reason', repr(err.exception)))
logger.exception(reason)
details = getattr(err.exception, 'details', None)
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
self.after_closed()
if decref_workflow:
self.decref_workflow()
return [decision], {}
except Exception as err:
reason = 'Cannot replay the workflow: {}({})'.format(
err.__class__.__name__,
err,
)
tb = traceback.format_exc()
details = 'Traceback:\n{}'.format(tb)
logger.exception(reason + '\n' + details)
self.on_failure(reason)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
self.after_closed()
if decref_workflow:
self.decref_workflow()
return [decision], {}
self.after_replay()
decision = swf.models.decision.WorkflowExecutionDecision()
decision.complete(result=swf.format.result(json_dumps(result)))
self.on_completed()
self.after_closed()
if decref_workflow:
self.decref_workflow()
return [decision], {}
def standalone(
context,
workflow,
domain,
workflow_id,
execution_timeout,
tags,
decision_tasks_timeout,
input,
input_file,
nb_workers,
nb_deciders,
heartbeat,
display_status,
repair,
force_activities,
):
"""
This command spawn a decider and an activity worker to execute a workflow
with a single main process.
"""
disable_boto_connection_pooling()
if force_activities and not repair:
raise ValueError(
"You should only use --force-activities with --repair.")
workflow_class = get_workflow(workflow)
if not workflow_id:
workflow_id = workflow_class.name
wf_input = {}
if input or input_file:
wf_input = get_or_load_input(input_file, input)
if repair:
repair_run_id = None
if " " in repair:
repair, repair_run_id = repair.split(" ", 1)
# get the previous execution history, it will serve as "default history"
# for activities that succeeded in the previous execution
logger.info('retrieving history of previous execution: domain={} '
'workflow_id={} run_id={}'.format(domain, repair,
repair_run_id))
workflow_execution = get_workflow_execution(domain,
repair,
run_id=repair_run_id)
previous_history = History(workflow_execution.history())
repair_run_id = workflow_execution.run_id
previous_history.parse()
# get the previous execution input if none passed
if not input and not input_file:
wf_input = previous_history.events[0].input
if not tags:
tags = workflow_execution.tag_list
else:
previous_history = None
repair_run_id = None
if not tags:
get_tag_list = getattr(workflow_class, 'get_tag_list', None)
if get_tag_list:
tags = get_tag_list(workflow_class, *wf_input.get('args', ()),
**wf_input.get('kwargs', {}))
else:
tags = getattr(workflow_class, 'tag_list', None)
if tags == Workflow.INHERIT_TAG_LIST:
tags = None
task_list = create_unique_task_list(workflow_id)
logger.info('using task list {}'.format(task_list))
decider_proc = multiprocessing.Process(
target=decider.command.start,
args=(
[workflow],
domain,
task_list,
),
kwargs={
'nb_processes': nb_deciders,
'repair_with': previous_history,
'force_activities': force_activities,
'is_standalone': True,
'repair_workflow_id': repair or None,
'repair_run_id': repair_run_id,
},
)
decider_proc.start()
worker_proc = multiprocessing.Process(
target=worker.command.start,
args=(
domain,
task_list,
),
kwargs={
'nb_processes': nb_workers,
'heartbeat': heartbeat,
},
)
worker_proc.start()
print('starting workflow {}'.format(workflow), file=sys.stderr)
ex = start_workflow.callback(
workflow,
domain,
workflow_id,
task_list,
execution_timeout,
tags,
decision_tasks_timeout,
format.input(wf_input),
None,
local=False,
)
while True:
time.sleep(2)
ex = helpers.get_workflow_execution(
domain,
ex.workflow_id,
ex.run_id,
)
if display_status:
print('status: {}'.format(ex.status), file=sys.stderr)
if ex.status == ex.STATUS_CLOSED:
print('execution {} finished'.format(ex.workflow_id),
file=sys.stderr)
break
os.kill(worker_proc.pid, signal.SIGTERM)
worker_proc.join()
os.kill(decider_proc.pid, signal.SIGTERM)
decider_proc.join()
def replay(self, decision_response):
"""Executes the workflow from the start until it blocks.
:param decision_response: an object wrapping the PollForDecisionTask response
:type decision_response: swf.responses.Response
:returns: a list of decision and a context dict
:rtype: ([swf.models.decision.base.Decision], dict)
"""
self.reset()
history = decision_response.history
self._history = History(history)
self._history.parse()
workflow_started_event = history[0]
input = workflow_started_event.input
if input is None:
input = {}
args = input.get('args', ())
kwargs = input.get('kwargs', {})
self.before_replay()
try:
result = self.run_workflow(*args, **kwargs)
except exceptions.ExecutionBlocked:
logger.info('{} open activities ({} decisions)'.format(
self._open_activity_count,
len(self._decisions),
))
self.after_replay()
return self._decisions, {}
except exceptions.TaskException as err:
reason = 'Workflow execution error in task {}: "{}"'.format(
err.task.name,
getattr(err.exception, 'reason', repr(err.exception)))
logger.exception(reason)
details = getattr(err.exception, 'details', None)
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
self.after_closed()
return [decision], {}
except Exception as err:
reason = 'Cannot replay the workflow: {}({})'.format(
err.__class__.__name__,
err,
)
tb = traceback.format_exc()
details = 'Traceback:\n{}'.format(tb)
logger.exception(reason + '\n' + details)
self.on_failure(reason)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=swf.format.reason(reason),
details=swf.format.details(details),
)
self.after_closed()
return [decision], {}
self.after_replay()
decision = swf.models.decision.WorkflowExecutionDecision()
decision.complete(result=swf.format.result(json.dumps(result)))
self.on_completed()
self.after_closed()
return [decision], {}
def replay(self, decision_response, decref_workflow=True):
# type: (swf.responses.Response, bool) -> DecisionsAndContext
"""Replay the workflow from the start until it blocks.
Called by the DeciderWorker.
:param decision_response: an object wrapping the PollForDecisionTask response
:param decref_workflow : Decref workflow once replay is done (to save memory)
:returns: a list of decision with an optional context
"""
self.reset()
# noinspection PyUnresolvedReferences
history = decision_response.history
self._history = History(history)
self._history.parse()
self.build_run_context(decision_response)
# noinspection PyUnresolvedReferences
self._execution = decision_response.execution
workflow_started_event = history[0]
input = workflow_started_event.input
if input is None:
input = {}
args = input.get('args', ())
kwargs = input.get('kwargs', {})
self.before_replay()
try:
if self._history.cancel_requested:
decisions = self.handle_cancel_requested()
if decisions is not None:
self.after_replay()
self.after_closed()
if decref_workflow:
self.decref_workflow()
return DecisionsAndContext(decisions)
self.propagate_signals()
result = self.run_workflow(*args, **kwargs)
except exceptions.ExecutionBlocked:
logger.info('{} open activities ({} decisions)'.format(
self._open_activity_count,
len(self._decisions_and_context.decisions),
))
self.after_replay()
if decref_workflow:
self.decref_workflow()
if self._append_timer:
self._add_start_timer_decision('_simpleflow_wake_up_timer')
if not self._decisions_and_context.execution_context:
self.maybe_clear_execution_context()
return self._decisions_and_context
except exceptions.TaskException as err:
def _extract_reason(err):
if hasattr(err.exception, 'reason'):
raw = err.exception.reason
# don't parse eventual json object here, since we will cast
# the result to a string anyway, better keep a json representation
return format.decode(raw, parse_json=False, use_proxy=False)
return repr(err.exception)
reason = 'Workflow execution error in task {}: "{}"'.format(
err.task.name,
_extract_reason(err))
logger.exception('%s', reason) # Don't let logger try to interpolate the message
details = getattr(err.exception, 'details', None)
self.on_failure(reason, details)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=reason,
details=details,
)
self.after_closed()
if decref_workflow:
self.decref_workflow()
return DecisionsAndContext([decision])
except Exception as err:
reason = 'Cannot replay the workflow: {}({})'.format(
err.__class__.__name__,
err,
)
tb = traceback.format_exc()
details = 'Traceback:\n{}'.format(tb)
logger.exception('%s', reason + '\n' + details) # Don't let logger try to interpolate the message
self.on_failure(reason)
decision = swf.models.decision.WorkflowExecutionDecision()
decision.fail(
reason=reason,
details=details,
)
self.after_closed()
if decref_workflow:
self.decref_workflow()
return DecisionsAndContext([decision])
self.after_replay()
decision = swf.models.decision.WorkflowExecutionDecision()
decision.complete(result=result)
self.on_completed()
self.after_closed()
if decref_workflow:
self.decref_workflow()
return DecisionsAndContext([decision])
def get_workflow_history(domain_name, workflow_id, run_id=None):
workflow_execution = get_workflow_execution(domain_name,
workflow_id,
run_id=run_id)
history = History(workflow_execution.history())
return history
