def test_workflow_schema(self):
tmpl = TEMPLATES['ok']
template = WorkflowTemplate.from_dict(tmpl)
self.assertEqual(template.schema, 1)
tmpl['schema'] = 5
template = WorkflowTemplate.from_dict(tmpl)
self.assertEqual(template.schema, 5)
def test_workflow_schema(self):
tmpl = TEMPLATES['ok']
template = WorkflowTemplate.from_dict(tmpl)
self.assertEqual(template.schema, 1)
tmpl['schema'] = 5
template = WorkflowTemplate.from_dict(tmpl)
self.assertEqual(template.schema, 5)
async def post(self, request, tid):
"""
Publish a draft into production
"""
try:
tmpl = await self.nyuki.storage.templates.get(tid, draft=True)
except AutoReconnect:
return Response(status=503)
if not tmpl:
return Response(status=404)
draft = {
**tmpl[0],
'draft': False
}
try:
# Set template ID from url
template = WorkflowTemplate.from_dict({**draft, 'draft': False})
except TemplateGraphError as exc:
return Response(status=400, body={
'error': str(exc)
})
errors = self.errors_from_validation(template)
if errors is not None:
return Response(status=400, body=errors)
await self.nyuki.engine.load(template)
# Update draft into a new template
await self.nyuki.storage.templates.publish_draft(tid)
return Response(draft)
async def put(self, request, tid):
"""
Create a new draft for this template id.
"""
try:
tmpl = await self.nyuki.storage.get_template(tid)
except AutoReconnect:
return Response(status=503)
if not tmpl:
return Response(status=404)
draft = await self.nyuki.storage.get_template(tid, draft=True)
if draft:
return Response(status=409,
body={'error': 'This draft already exists'})
request = await request.json()
try:
# Set template ID from url
template = WorkflowTemplate.from_dict({**request, 'id': tid})
except TemplateGraphError as exc:
return Response(status=400, body={'error': str(exc)})
try:
tmpl_dict = await self.upsert_draft(template, request)
except DuplicateKeyError as exc:
return Response(status=409, body={'error': exc})
tmpl_dict['errors'] = self.errors_from_validation(template)
return Response(tmpl_dict)
async def put(self, request):
"""
Create a workflow DAG from JSON
"""
request = await request.json()
if 'id' in request:
try:
draft = await self.nyuki.storage.templates.get(
request['id'], draft=True
)
except AutoReconnect:
return Response(status=503)
if draft:
return Response(status=409, body={
'error': 'draft already exists'
})
if self.nyuki.DEFAULT_POLICY is not None and 'policy' not in request:
request['policy'] = self.nyuki.DEFAULT_POLICY
try:
template = WorkflowTemplate.from_dict(request)
except TemplateGraphError as exc:
return Response(status=400, body={
'error': str(exc)
})
try:
metadata = await self.nyuki.storage.templates.get_metadata(template.uid)
except AutoReconnect:
return Response(status=503)
if not metadata:
if 'title' not in request:
return Response(status=400, body={
'error': "workflow 'title' key is mandatory"
})
metadata = {
'id': template.uid,
'title': request['title'],
'tags': request.get('tags', [])
}
await self.nyuki.storage.templates.insert_metadata(metadata)
else:
metadata = metadata[0]
try:
tmpl_dict = await self.update_draft(template, request)
except ConflictError as exc:
return Response(status=409, body={
'error': exc
})
return Response({
**tmpl_dict,
**metadata,
'errors': self.errors_from_validation(template)
})
async def post(self, request, tid):
"""
Publish a draft into production
"""
try:
tmpl = await self.nyuki.storage.templates.get(tid, draft=True)
except AutoReconnect:
return Response(status=503)
if not tmpl:
return Response(status=404)
draft = {**tmpl[0], 'draft': False}
try:
# Set template ID from url
template = WorkflowTemplate.from_dict({**draft, 'draft': False})
except TemplateGraphError as exc:
return Response(status=400, body={'error': str(exc)})
errors = self.errors_from_validation(template)
if errors is not None:
return Response(status=400, body=errors)
await self.nyuki.engine.load(template)
# Update draft into a new template
await self.nyuki.storage.templates.publish_draft(tid)
return Response(draft)
async def post(self, request, tid):
"""
Publish a draft into production
"""
try:
tmpl_dict = await self.nyuki.storage.get_template(tid, draft=True)
except AutoReconnect:
return Response(status=503)
if not tmpl_dict:
return Response(status=404)
try:
# Set template ID from url
template = WorkflowTemplate.from_dict(tmpl_dict)
except TemplateGraphError as exc:
return Response(status=400, body={'error': str(exc)})
errors = self.errors_from_validation(template)
if errors is not None:
return Response(errors, status=400)
# Update draft into a new template
await self.nyuki.storage.publish_draft(tid)
tmpl_dict['state'] = TemplateState.ACTIVE.value
return Response(tmpl_dict)
async def patch(self, request, tid):
"""
Modify the template's draft
"""
try:
tmpl = await self.nyuki.storage.get_template(tid, draft=True)
except AutoReconnect:
return Response(status=503)
if not tmpl:
return Response(status=404)
request = await request.json()
try:
# Set template ID from url
template = WorkflowTemplate.from_dict({**request, 'id': tid})
except TemplateGraphError as exc:
return Response(status=400, body={'error': str(exc)})
try:
tmpl_dict = await self.upsert_draft(template, request)
except DuplicateKeyError as exc:
return Response(status=409, body={'error': str(exc)})
tmpl_dict['errors'] = self.errors_from_validation(template)
return Response(tmpl_dict)
async def patch(self, request, tid):
"""
Modify the template's draft
"""
try:
tmpl = await self.nyuki.storage.templates.get(tid, draft=True)
except AutoReconnect:
return Response(status=503)
if not tmpl:
return Response(status=404)
request = await request.json()
try:
# Set template ID from url
template = WorkflowTemplate.from_dict({**request, 'id': tid})
except TemplateGraphError as exc:
return Response(status=400, body={'error': str(exc)})
try:
tmpl_dict = await self.update_draft(template, request)
except ConflictError as exc:
return Response(status=409, body={'error': str(exc)})
metadata = await self.nyuki.storage.templates.get_metadata(template.uid
)
metadata = metadata[0]
return Response({
**tmpl_dict,
**metadata, 'errors':
self.errors_from_validation(template)
})
async def put(self, request):
"""
Create a workflow DAG from JSON
"""
request = await request.json()
if 'id' in request:
del request['id']
if 'title' not in request:
return Response(status=400,
body={'error': "workflow 'title' is mandatory"})
if self.nyuki.DEFAULT_POLICY is not None and 'policy' not in request:
request['policy'] = self.nyuki.DEFAULT_POLICY
try:
template = WorkflowTemplate.from_dict(request)
except TemplateGraphError as exc:
return Response(status=400, body={'error': str(exc)})
try:
tmpl_dict = await self.upsert_draft(template, request)
except DuplicateKeyError as exc:
return Response(status=409, body={'error': exc})
tmpl_dict['errors'] = self.errors_from_validation(template)
return Response(tmpl_dict)
async def put(self, request):
"""
Create a workflow DAG from JSON
"""
request = await request.json()
if 'id' in request:
try:
draft = await self.nyuki.storage.templates.get(request['id'],
draft=True)
except AutoReconnect:
return Response(status=503)
if draft:
return Response(status=409,
body={'error': 'draft already exists'})
if self.nyuki.DEFAULT_POLICY is not None and 'policy' not in request:
request['policy'] = self.nyuki.DEFAULT_POLICY
try:
template = WorkflowTemplate.from_dict(request)
except TemplateGraphError as exc:
return Response(status=400, body={'error': str(exc)})
try:
metadata = await self.nyuki.storage.templates.get_metadata(
template.uid)
except AutoReconnect:
return Response(status=503)
if not metadata:
if 'title' not in request:
return Response(
status=400,
body={'error': "workflow 'title' key is mandatory"})
metadata = {
'id': template.uid,
'title': request['title'],
'tags': request.get('tags', [])
}
await self.nyuki.storage.templates.insert_metadata(metadata)
else:
metadata = metadata[0]
try:
tmpl_dict = await self.update_draft(template, request)
except ConflictError as exc:
return Response(status=409, body={'error': exc})
return Response({
**tmpl_dict,
**metadata, 'errors':
self.errors_from_validation(template)
})
async def test():
tmpl = TEMPLATES['task_timeout']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
# The workflow is OK
await wflow
self.assertEqual(FutureState.get(wflow), FutureState.finished)
# The task has timed out
task = wflow._tasks_by_id.get('1')
with self.assertRaises(asyncio.CancelledError):
task.exception()
self.assertEqual(FutureState.get(task), FutureState.timeout)
async def test():
tmpl = TEMPLATES['task_timeout']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
# The workflow is OK
await wflow
self.assertEqual(FutureState.get(wflow), FutureState.finished)
# The task has timed out
task = wflow._tasks_by_id.get('1')
with self.assertRaises(asyncio.CancelledError):
task.exception()
self.assertEqual(FutureState.get(task), FutureState.timeout)
async def test():
tmpl = TEMPLATES['crash_test']
# Test crash at task __init__
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in ('1', '2'):
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# These tasks were never started
for tid in ('crash', 'wont_run'):
task = wflow._tasks_by_id.get(tid)
self.assertIs(task, None)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
# Test crash inside a task
tmpl['tasks'][0]['config'] = {'init_ok': None}
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in ('1', '2'):
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# This task crashed during execution
task = wflow._tasks_by_id.get('crash')
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.exception)
# This task was never started
task = wflow._tasks_by_id.get('wont_run')
self.assertIs(task, None)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
async def test():
tmpl = TEMPLATES['ok']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in tmpl['graph'].keys():
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
async def test():
tmpl = TEMPLATES['crash_test']
# Test crash at task __init__
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in ('1', '2'):
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# These tasks were never started
for tid in ('crash', 'wont_run'):
task = wflow._tasks_by_id.get(tid)
self.assertIs(task, None)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
# Test crash inside a task
tmpl['tasks'][0]['config'] = {'init_ok': None}
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in ('1', '2'):
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# This task crashed during execution
task = wflow._tasks_by_id.get('crash')
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.exception)
# This task was never started
task = wflow._tasks_by_id.get('wont_run')
self.assertIs(task, None)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
async def test():
tmpl = TEMPLATES['ok']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
await wflow
# These tasks have finished
for tid in tmpl['graph'].keys():
task = wflow._tasks_by_id.get(tid)
self.assertTrue(task.done())
self.assertEqual(FutureState.get(task), FutureState.finished)
# The workflow finished properly
self.assertTrue(wflow.done())
self.assertEqual(FutureState.get(wflow), FutureState.finished)
async def reload_from_storage(self):
"""
Check mongo, retrieve and load all templates
"""
self.storage = MongoStorage(**self.mongo_config)
templates = await self.storage.templates.get_all(
latest=True,
with_metadata=False
)
for template in templates:
try:
await self.engine.load(WorkflowTemplate.from_dict(template))
except Exception as exc:
# Means a bad workflow is in database, report it
reporting.exception(exc)
async def test():
tmpl = TEMPLATES['workflow_cancel']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
# Workflow is cancelled
with self.assertRaises(asyncio.CancelledError):
await wflow
self.assertEqual(FutureState.get(wflow), FutureState.cancelled)
# This task was cancelled
task = wflow._tasks_by_id.get('cancel')
with self.assertRaises(asyncio.CancelledError):
task.exception()
self.assertEqual(FutureState.get(task), FutureState.cancelled)
# These tasks were never started
for tid in ('2', '3', '4'):
task = wflow._tasks_by_id.get(tid)
self.assertIs(task, None)
async def test():
tmpl = TEMPLATES['workflow_cancel']
wflow = Workflow(WorkflowTemplate.from_dict(tmpl))
wflow.run({'initial': 'data'})
# Workflow is cancelled
with self.assertRaises(asyncio.CancelledError):
await wflow
self.assertEqual(FutureState.get(wflow), FutureState.cancelled)
# This task was cancelled
task = wflow._tasks_by_id.get('cancel')
with self.assertRaises(asyncio.CancelledError):
task.exception()
self.assertEqual(FutureState.get(task), FutureState.cancelled)
# These tasks were never started
for tid in ('2', '3', '4'):
task = wflow._tasks_by_id.get(tid)
self.assertIs(task, None)
async def put(self, request, tid):
"""
Create a new draft for this template id
"""
try:
versions = await self.nyuki.storage.templates.get(tid)
except AutoReconnect:
return Response(status=503)
if not versions:
return Response(status=404)
for v in versions:
if v['draft'] is True:
return Response(status=409, body={
'error': 'This draft already exists'
})
request = await request.json()
try:
# Set template ID from url
template = WorkflowTemplate.from_dict({**request, 'id': tid})
except TemplateGraphError as exc:
return Response(status=400, body={
'error': str(exc)
})
try:
tmpl_dict = await self.update_draft(template, request)
except ConflictError as exc:
return Response(status=409, body={
'error': exc
})
metadata = await self.nyuki.storage.templates.get_metadata(template.uid)
metadata = metadata[0]
return Response({
**tmpl_dict,
**metadata,
'errors': self.errors_from_validation(template)
})
async def put(self, request, tid):
"""
Create a new draft for this template id
"""
try:
versions = await self.nyuki.storage.templates.get(tid)
except AutoReconnect:
return Response(status=503)
if not versions:
return Response(status=404)
for v in versions:
if v['draft'] is True:
return Response(status=409,
body={'error': 'This draft already exists'})
request = await request.json()
try:
# Set template ID from url
template = WorkflowTemplate.from_dict({**request, 'id': tid})
except TemplateGraphError as exc:
return Response(status=400, body={'error': str(exc)})
try:
tmpl_dict = await self.update_draft(template, request)
except ConflictError as exc:
return Response(status=409, body={'error': exc})
metadata = await self.nyuki.storage.templates.get_metadata(template.uid
)
metadata = metadata[0]
return Response({
**tmpl_dict,
**metadata, 'errors':
self.errors_from_validation(template)
})
async def patch(self, request, tid):
"""
Modify the template's draft
"""
try:
tmpl = await self.nyuki.storage.templates.get(tid, draft=True)
except AutoReconnect:
return Response(status=503)
if not tmpl:
return Response(status=404)
request = await request.json()
try:
# Set template ID from url
template = WorkflowTemplate.from_dict({**request, 'id': tid})
except TemplateGraphError as exc:
return Response(status=400, body={
'error': str(exc)
})
try:
tmpl_dict = await self.update_draft(template, request)
except ConflictError as exc:
return Response(status=409, body={
'error': str(exc)
})
metadata = await self.nyuki.storage.templates.get_metadata(template.uid)
metadata = metadata[0]
return Response({
**tmpl_dict,
**metadata,
'errors': self.errors_from_validation(template)
})
async def get(self, tmpl_id):
template = await self.storage.get_template(tmpl_id, draft=False)
return WorkflowTemplate.from_dict(template)
async def select(self, topic):
templates = await self.storage.get_for_topic(topic)
return [WorkflowTemplate.from_dict(template) for template in templates]
async def put(self, request):
"""
Start a workflow from payload:
{
"id": "template_id",
"draft": true/false
}
"""
async_topic = request.headers.get('X-Surycat-Async-Topic')
exec_track = request.headers.get('X-Surycat-Exec-Track')
requester = request.headers.get('Referer')
request = await request.json()
if 'id' not in request:
return Response(status=400, body={
'error': "Template's ID key 'id' is mandatory"
})
draft = request.get('draft', False)
try:
templates = await self.nyuki.storage.templates.get(
request['id'],
draft=draft,
with_metadata=True
)
except AutoReconnect:
return Response(status=503)
if not templates:
return Response(status=404, body={
'error': 'Could not find a suitable template to run'
})
wf_tmpl = WorkflowTemplate.from_dict(templates[0])
data = request.get('inputs', {})
if draft:
wflow = await self.nyuki.engine.run_once(wf_tmpl, data)
else:
wflow = await self.nyuki.engine.trigger(wf_tmpl.uid, data)
if wflow is None:
return Response(status=400, body={
'error': 'Could not start any workflow from this template'
})
# Prevent workflow loop
exec_track = exec_track.split(',') if exec_track else []
holder = self.nyuki.bus.name
for ancestor in exec_track:
try:
info = URI.parse(ancestor)
except InvalidWorkflowUri:
continue
if info.template_id == wf_tmpl.uid and info.holder == holder:
return Response(status=400, body={
'error': 'Loop detected between workflows'
})
# Keep full instance+template in nyuki's memory
wfinst = self.nyuki.new_workflow(
templates[0], wflow,
track=exec_track,
requester=requester
)
# Handle async workflow exec updates
if async_topic is not None:
self.register_async_handler(async_topic, wflow)
return Response(
json.dumps(
wfinst.report(),
default=serialize_wflow_exec
),
content_type='application/json'
)
async def put(self, request):
"""
Start a workflow from payload:
{
"id": "template_id",
"draft": true/false
"exec": {}
}
"""
async_topic = request.headers.get('X-Surycat-Async-Topic')
exec_track = request.headers.get('X-Surycat-Exec-Track')
requester = request.headers.get('Referer')
request = await request.json()
if 'id' not in request:
return Response(
status=400,
body={'error': "Template's ID key 'id' is mandatory"})
draft = request.get('draft', False)
data = request.get('inputs', {})
exec = request.get('exec')
if exec:
# Suspended/crashed instance
# The request's payload is the last known execution report
templates = [request]
if exec['id'] in self.nyuki.running_workflows:
return Response(
status=400,
body={'error': 'This workflow is already being rescued'})
else:
# Fetch the template from the storage
try:
templates = await self.nyuki.storage.templates.get(
request['id'], draft=draft, with_metadata=True)
except AutoReconnect:
return Response(status=503)
if not templates:
return Response(
status=404,
body={'error': 'Could not find a suitable template to run'})
wf_tmpl = WorkflowTemplate.from_dict(templates[0])
if exec:
wflow = await self.nyuki.engine.rescue(wf_tmpl, request)
elif draft:
wflow = await self.nyuki.engine.run_once(wf_tmpl, data)
else:
wflow = await self.nyuki.engine.trigger(wf_tmpl.uid, data)
if wflow is None:
return Response(
status=400,
body={
'error': 'Could not start any workflow from this template'
})
# Prevent workflow loop
exec_track = exec_track.split(',') if exec_track else []
holder = self.nyuki.bus.name
for ancestor in exec_track:
try:
info = URI.parse(ancestor)
except InvalidWorkflowUri:
continue
if info.template_id == wf_tmpl.uid and info.holder == holder:
return Response(
status=400,
body={'error': 'Loop detected between workflows'})
# Keep full instance+template in nyuki's memory
wfinst = self.nyuki.new_workflow(templates[0],
wflow,
track=exec_track,
requester=requester)
# Handle async workflow exec updates
if async_topic is not None:
self.register_async_handler(async_topic, wflow)
return Response(wfinst.report())
