async def test():
tmpl = TEMPLATES['task_timeout']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
# The workflow is OK
await wflow
self.assertEqual(FutureState.get(wflow), FutureState.finished)
# The task has timed out
task = wflow._tasks_by_id.get('1')
with self.assertRaises(asyncio.CancelledError):
task.exception()
self.assertEqual(FutureState.get(task), FutureState.timeout)
async def test():
tmpl = TEMPLATES['task_timeout']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
# The workflow is OK
await wflow
self.assertEqual(FutureState.get(wflow), FutureState.finished)
# The task has timed out
task = wflow._tasks_by_id.get('1')
with self.assertRaises(asyncio.CancelledError):
task.exception()
self.assertEqual(FutureState.get(task), FutureState.timeout)
async def test():
tmpl = TEMPLATES['ok']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in tmpl['graph'].keys():
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
async def test():
tmpl = TEMPLATES['ok']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in tmpl['graph'].keys():
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
async def test():
tmpl = TEMPLATES['workflow_cancel']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
# Workflow is cancelled
with self.assertRaises(asyncio.CancelledError):
await wflow
self.assertEqual(FutureState.get(wflow), FutureState.cancelled)
# This task was cancelled
task = wflow._tasks_by_id.get('cancel')
with self.assertRaises(asyncio.CancelledError):
task.exception()
self.assertEqual(FutureState.get(task), FutureState.cancelled)
# These tasks were never started
for tid in ('2', '3', '4'):
task = wflow._tasks_by_id.get(tid)
self.assertIs(task, None)
async def test():
tmpl = TEMPLATES['workflow_cancel']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
# Workflow is cancelled
with self.assertRaises(asyncio.CancelledError):
await wflow
self.assertEqual(FutureState.get(wflow), FutureState.cancelled)
# This task was cancelled
task = wflow._tasks_by_id.get('cancel')
with self.assertRaises(asyncio.CancelledError):
task.exception()
self.assertEqual(FutureState.get(task), FutureState.cancelled)
# These tasks were never started
for tid in ('2', '3', '4'):
task = wflow._tasks_by_id.get(tid)
self.assertIs(task, None)
async def test():
tmpl = TEMPLATES['crash_test']
# Test crash at task __init__
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in ('1', '2'):
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# These tasks were never started
for tid in ('crash', 'wont_run'):
task = wflow._tasks_by_id.get(tid)
self.assertIs(task, None)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
# Test crash inside a task
tmpl['tasks'][0]['config'] = {'init_ok': None}
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in ('1', '2'):
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# This task crashed during execution
task = wflow._tasks_by_id.get('crash')
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.exception)
# This task was never started
task = wflow._tasks_by_id.get('wont_run')
self.assertIs(task, None)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
async def run_once(self, template, data):
"""
Starts a new execution of the workflow template regardless of the
overrun policy and already running workflows.
Note: it does NOT load the workflow template in the engine.
"""
if self._must_stop:
log.debug("The engine is stopping, cannot run a new workflow from"
"template id %s", template.uid)
return None
with await self._lock:
wflow = Workflow(template, loop=self._loop)
self._do_run(wflow, Event(data))
return wflow
async def test():
tmpl = TEMPLATES['crash_test']
# Test crash at task __init__
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in ('1', '2'):
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# These tasks were never started
for tid in ('crash', 'wont_run'):
task = wflow._tasks_by_id.get(tid)
self.assertIs(task, None)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
# Test crash inside a task
tmpl['tasks'][0]['config'] = {'init_ok': None}
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in ('1', '2'):
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# This task crashed during execution
task = wflow._tasks_by_id.get('crash')
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.exception)
# This task was never started
task = wflow._tasks_by_id.get('wont_run')
self.assertIs(task, None)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
async def rescue(self, template, report):
"""
Attaches a dangling workflow instance using its last known report.
"""
if self._must_stop:
log.debug("The engine is stopping, cannot rescue the workflow from"
"its report (execution id %s)", report['exec']['id'])
return None
with await self._lock:
wflow = Workflow(template, loop=self._loop)
self._add_wflow(wflow)
wflow.add_done_callback(self._remove_wflow)
wflow.fast_forward(report)
return wflow
async def execute(self, event):
"""
Entrypoint execution method.
"""
data = event.data
self._task = asyncio.Task.current_task()
# Send the HTTP request
log.info('Request %s to process template %s', self.nyuki_api,
self.template)
log.debug('Request details: url=%s, draft=%s, data=%s', self.url,
self.draft, data)
# Setup headers (set requester and exec-track to avoid workflow loops)
workflow = Workflow.current_workflow()
wf_instance = runtime.workflows[workflow.uid]
parent = wf_instance.exec.get('requester')
track = list(wf_instance.exec.get('track', []))
if parent:
track.append(parent)
headers = {
'Content-Type': 'application/json',
'Referer': URI.instance(workflow),
'X-Surycat-Exec-Track': ','.join(track)
}
# Handle blocking trigger_workflow using mqtt
if self.blocking:
topic = '{}/async/{}'.format(runtime.bus.name, str(uuid4())[:8])
headers['X-Surycat-Async-Topic'] = topic
self.async_future = asyncio.Future()
await runtime.bus.subscribe(topic, self.async_exec)
def _unsub(f):
asyncio.ensure_future(runtime.bus.unsubscribe(topic))
self._task.add_done_callback(_unsub)
async with ClientSession() as session:
params = {
'url':
self.url,
'headers':
headers,
'data':
json.dumps({
'id': self.template,
'draft': self.draft,
'inputs': data
})
}
async with session.put(**params) as response:
# Response validity
if response.status != 200:
msg = "Can't process workflow template {} on {}".format(
self.template, self.nyuki_api)
if response.status % 400 < 100:
reason = await response.json()
msg = "{}, reason: {}".format(msg, reason['error'])
raise RuntimeError(msg)
resp_body = await response.json()
instance = resp_body['exec']['id']
self._triggered['workflow_exec_id'] = instance
log.debug('Request sent successfully to {}', self.nyuki_api)
if not self.blocking:
self._triggered['status'] = Status.DONE.value
self._task.dispatch_progress(self.report())
# Block until task completed
if self.blocking:
self._triggered['status'] = Status.RUNNING.value
self._task.dispatch_progress(self.report())
log.info('Waiting for %s@%s to complete', instance, self.nyuki_api)
try:
await asyncio.wait_for(self.async_future, self.timeout)
self._triggered['status'] = Status.DONE.value
log.info('Instance %s@%s is done', instance, self.nyuki_api)
except asyncio.TimeoutError:
self._triggered['status'] = Status.TIMEOUT.value
log.info('Instance %s@%s has timeouted', instance,
self.nyuki_api)
self._task.dispatch_progress({'status': self._triggered['status']})
return data
async def execute(self, event):
"""
Entrypoint execution method.
"""
data = event.data
self._task = asyncio.Task.current_task()
# Send the HTTP request
log.info('Request %s to process template %s', self.nyuki_api, self.template)
log.debug(
'Request details: url=%s, draft=%s, data=%s',
self.url, self.draft, data
)
# Setup headers (set requester and exec-track to avoid workflow loops)
workflow = Workflow.current_workflow()
wf_instance = runtime.workflows[workflow.uid]
parent = wf_instance.exec.get('requester')
track = list(wf_instance.exec.get('track', []))
if parent:
track.append(parent)
headers = {
'Content-Type': 'application/json',
'Referer': URI.instance(workflow),
'X-Surycat-Exec-Track': ','.join(track)
}
# Handle blocking trigger_workflow using mqtt
if self.blocking:
topic = '{}/async/{}'.format(runtime.bus.name, str(uuid4())[:8])
headers['X-Surycat-Async-Topic'] = topic
self.async_future = asyncio.Future()
await runtime.bus.subscribe(topic, self.async_exec)
asyncio.get_event_loop().call_later(
self.timeout,
asyncio.ensure_future,
runtime.bus.unsubscribe(topic)
)
async with ClientSession() as session:
params = {
'url': self.url,
'headers': headers,
'data': json.dumps({
'id': self.template,
'draft': self.draft,
'inputs': data
})
}
async with session.put(**params) as response:
# Response validity
if response.status != 200:
msg = "Can't process workflow template {} on {}".format(
self.template, self.nyuki_api
)
if response.status % 400 < 100:
reason = await response.json()
msg = "{}, reason: {}".format(msg, reason['error'])
raise RuntimeError(msg)
resp_body = await response.json()
instance = resp_body['exec']['id']
self._triggered['workflow_exec_id'] = instance
log.debug('Request sent successfully to {}', self.nyuki_api)
if not self.blocking:
self._triggered['status'] = Status.DONE.value
self._task.dispatch_progress(self.report())
# Block until task completed
if self.blocking:
self._triggered['status'] = Status.RUNNING.value
self._task.dispatch_progress(self.report())
log.info('Waiting for %s@%s to complete', instance, self.nyuki_api)
try:
await asyncio.wait_for(self.async_future, self.timeout)
self._triggered['status'] = Status.DONE.value
log.info('Instance %s@%s is done', instance, self.nyuki_api)
except asyncio.TimeoutError:
self._triggered['status'] = Status.TIMEOUT.value
log.info('Instance %s@%s has timeouted', instance, self.nyuki_api)
self._task.dispatch_progress({'status': self._triggered['status']})
return data
async def execute(self, event):
Workflow.current_workflow().cancel()
await asyncio.sleep(1.0)
async def execute(self, event):
Workflow.current_workflow().cancel()
await asyncio.sleep(1.0)
async def execute(self, event):
"""
Entrypoint execution method.
"""
self.data = event.data
self.task = asyncio.Task.current_task()
is_draft = self.template.get('draft', False)
# Send the HTTP request
log.info('Triggering template %s%s on service %s', self.template['id'],
' (draft)' if is_draft else '', self.template['service'])
# Setup headers (set requester and exec-track to avoid workflow loops)
workflow = runtime.workflows[Workflow.current_workflow().uid]
parent = workflow.exec.get('requester')
track = list(workflow.exec.get('track', []))
if parent:
track.append(parent)
headers = {
'Content-Type': 'application/json',
'Referer': URI.instance(workflow.instance),
'X-Surycat-Exec-Track': ','.join(track)
}
# Handle blocking trigger_workflow using mqtt
if self.blocking:
topic = '{}/async/{}'.format(runtime.bus.name, self.uid[:8])
headers['X-Surycat-Async-Topic'] = topic
headers['X-Surycat-Async-Events'] = ','.join([
WorkflowExecState.END.value,
WorkflowExecState.ERROR.value,
])
self.async_future = asyncio.Future()
await runtime.bus.subscribe(topic, self.async_exec)
def _unsub(f):
asyncio.ensure_future(runtime.bus.unsubscribe(topic))
self.task.add_done_callback(_unsub)
async with ClientSession() as session:
# Compute data to send to sub-workflows
url = '{}/vars/{}{}'.format(
self._engine,
self.template['id'],
'/draft' if is_draft else '',
)
async with session.get(url) as response:
if response.status != 200:
raise RuntimeError("Can't load template info")
wf_vars = await response.json()
lightened_data = {
key: self.data[key]
for key in wf_vars if key in self.data
}
params = {
'url':
'{}/instances'.format(self._engine),
'headers':
headers,
'data':
json.dumps({
'id': self.template['id'],
'draft': is_draft,
'inputs': lightened_data,
})
}
async with session.put(**params) as response:
if response.status != 200:
log.critical(await response.text())
msg = "Can't process workflow template {} on {}".format(
self.template, self.nyuki_api)
if response.status % 400 < 100:
reason = await response.json()
msg = "{}, reason: {}".format(msg, reason['error'])
raise RuntimeError(msg)
resp_body = await response.json()
self.triggered_id = resp_body['id']
wf_id = '@'.join([self.triggered_id[:8], self.template['service']])
self.status = WorkflowStatus.RUNNING.value
log.info('Successfully started %s', wf_id)
self.task.dispatch_progress(self.report())
# Block until task completed
if self.blocking:
log.info('Waiting for workflow %s to complete', wf_id)
await self.async_future
self.status = WorkflowStatus.DONE.value
log.info('Workflow %s is done', wf_id)
self.task.dispatch_progress({'status': self.status})
return self.data