def test_workflow_schema(self): tmpl = TEMPLATES['ok'] template = WorkflowTemplate.from_dict(tmpl) self.assertEqual(template.schema, 1) tmpl['schema'] = 5 template = WorkflowTemplate.from_dict(tmpl) self.assertEqual(template.schema, 5)
async def post(self, request, tid): """ Publish a draft into production """ try: tmpl = await self.nyuki.storage.templates.get(tid, draft=True) except AutoReconnect: return Response(status=503) if not tmpl: return Response(status=404) draft = { **tmpl[0], 'draft': False } try: # Set template ID from url template = WorkflowTemplate.from_dict({**draft, 'draft': False}) except TemplateGraphError as exc: return Response(status=400, body={ 'error': str(exc) }) errors = self.errors_from_validation(template) if errors is not None: return Response(status=400, body=errors) await self.nyuki.engine.load(template) # Update draft into a new template await self.nyuki.storage.templates.publish_draft(tid) return Response(draft)
async def put(self, request, tid): """ Create a new draft for this template id. """ try: tmpl = await self.nyuki.storage.get_template(tid) except AutoReconnect: return Response(status=503) if not tmpl: return Response(status=404) draft = await self.nyuki.storage.get_template(tid, draft=True) if draft: return Response(status=409, body={'error': 'This draft already exists'}) request = await request.json() try: # Set template ID from url template = WorkflowTemplate.from_dict({**request, 'id': tid}) except TemplateGraphError as exc: return Response(status=400, body={'error': str(exc)}) try: tmpl_dict = await self.upsert_draft(template, request) except DuplicateKeyError as exc: return Response(status=409, body={'error': exc}) tmpl_dict['errors'] = self.errors_from_validation(template) return Response(tmpl_dict)
async def put(self, request): """ Create a workflow DAG from JSON """ request = await request.json() if 'id' in request: try: draft = await self.nyuki.storage.templates.get( request['id'], draft=True ) except AutoReconnect: return Response(status=503) if draft: return Response(status=409, body={ 'error': 'draft already exists' }) if self.nyuki.DEFAULT_POLICY is not None and 'policy' not in request: request['policy'] = self.nyuki.DEFAULT_POLICY try: template = WorkflowTemplate.from_dict(request) except TemplateGraphError as exc: return Response(status=400, body={ 'error': str(exc) }) try: metadata = await self.nyuki.storage.templates.get_metadata(template.uid) except AutoReconnect: return Response(status=503) if not metadata: if 'title' not in request: return Response(status=400, body={ 'error': "workflow 'title' key is mandatory" }) metadata = { 'id': template.uid, 'title': request['title'], 'tags': request.get('tags', []) } await self.nyuki.storage.templates.insert_metadata(metadata) else: metadata = metadata[0] try: tmpl_dict = await self.update_draft(template, request) except ConflictError as exc: return Response(status=409, body={ 'error': exc }) return Response({ **tmpl_dict, **metadata, 'errors': self.errors_from_validation(template) })
async def post(self, request, tid): """ Publish a draft into production """ try: tmpl = await self.nyuki.storage.templates.get(tid, draft=True) except AutoReconnect: return Response(status=503) if not tmpl: return Response(status=404) draft = {**tmpl[0], 'draft': False} try: # Set template ID from url template = WorkflowTemplate.from_dict({**draft, 'draft': False}) except TemplateGraphError as exc: return Response(status=400, body={'error': str(exc)}) errors = self.errors_from_validation(template) if errors is not None: return Response(status=400, body=errors) await self.nyuki.engine.load(template) # Update draft into a new template await self.nyuki.storage.templates.publish_draft(tid) return Response(draft)
async def post(self, request, tid): """ Publish a draft into production """ try: tmpl_dict = await self.nyuki.storage.get_template(tid, draft=True) except AutoReconnect: return Response(status=503) if not tmpl_dict: return Response(status=404) try: # Set template ID from url template = WorkflowTemplate.from_dict(tmpl_dict) except TemplateGraphError as exc: return Response(status=400, body={'error': str(exc)}) errors = self.errors_from_validation(template) if errors is not None: return Response(errors, status=400) # Update draft into a new template await self.nyuki.storage.publish_draft(tid) tmpl_dict['state'] = TemplateState.ACTIVE.value return Response(tmpl_dict)
async def patch(self, request, tid): """ Modify the template's draft """ try: tmpl = await self.nyuki.storage.get_template(tid, draft=True) except AutoReconnect: return Response(status=503) if not tmpl: return Response(status=404) request = await request.json() try: # Set template ID from url template = WorkflowTemplate.from_dict({**request, 'id': tid}) except TemplateGraphError as exc: return Response(status=400, body={'error': str(exc)}) try: tmpl_dict = await self.upsert_draft(template, request) except DuplicateKeyError as exc: return Response(status=409, body={'error': str(exc)}) tmpl_dict['errors'] = self.errors_from_validation(template) return Response(tmpl_dict)
async def patch(self, request, tid): """ Modify the template's draft """ try: tmpl = await self.nyuki.storage.templates.get(tid, draft=True) except AutoReconnect: return Response(status=503) if not tmpl: return Response(status=404) request = await request.json() try: # Set template ID from url template = WorkflowTemplate.from_dict({**request, 'id': tid}) except TemplateGraphError as exc: return Response(status=400, body={'error': str(exc)}) try: tmpl_dict = await self.update_draft(template, request) except ConflictError as exc: return Response(status=409, body={'error': str(exc)}) metadata = await self.nyuki.storage.templates.get_metadata(template.uid ) metadata = metadata[0] return Response({ **tmpl_dict, **metadata, 'errors': self.errors_from_validation(template) })
async def put(self, request): """ Create a workflow DAG from JSON """ request = await request.json() if 'id' in request: del request['id'] if 'title' not in request: return Response(status=400, body={'error': "workflow 'title' is mandatory"}) if self.nyuki.DEFAULT_POLICY is not None and 'policy' not in request: request['policy'] = self.nyuki.DEFAULT_POLICY try: template = WorkflowTemplate.from_dict(request) except TemplateGraphError as exc: return Response(status=400, body={'error': str(exc)}) try: tmpl_dict = await self.upsert_draft(template, request) except DuplicateKeyError as exc: return Response(status=409, body={'error': exc}) tmpl_dict['errors'] = self.errors_from_validation(template) return Response(tmpl_dict)
async def put(self, request): """ Create a workflow DAG from JSON """ request = await request.json() if 'id' in request: try: draft = await self.nyuki.storage.templates.get(request['id'], draft=True) except AutoReconnect: return Response(status=503) if draft: return Response(status=409, body={'error': 'draft already exists'}) if self.nyuki.DEFAULT_POLICY is not None and 'policy' not in request: request['policy'] = self.nyuki.DEFAULT_POLICY try: template = WorkflowTemplate.from_dict(request) except TemplateGraphError as exc: return Response(status=400, body={'error': str(exc)}) try: metadata = await self.nyuki.storage.templates.get_metadata( template.uid) except AutoReconnect: return Response(status=503) if not metadata: if 'title' not in request: return Response( status=400, body={'error': "workflow 'title' key is mandatory"}) metadata = { 'id': template.uid, 'title': request['title'], 'tags': request.get('tags', []) } await self.nyuki.storage.templates.insert_metadata(metadata) else: metadata = metadata[0] try: tmpl_dict = await self.update_draft(template, request) except ConflictError as exc: return Response(status=409, body={'error': exc}) return Response({ **tmpl_dict, **metadata, 'errors': self.errors_from_validation(template) })
async def test(): tmpl = TEMPLATES['task_timeout'] wflow = Workflow(WorkflowTemplate.from_dict(tmpl)) wflow.run({'initial': 'data'}) # The workflow is OK await wflow self.assertEqual(FutureState.get(wflow), FutureState.finished) # The task has timed out task = wflow._tasks_by_id.get('1') with self.assertRaises(asyncio.CancelledError): task.exception() self.assertEqual(FutureState.get(task), FutureState.timeout)
async def test(): tmpl = TEMPLATES['crash_test'] # Test crash at task __init__ wflow = Workflow(WorkflowTemplate.from_dict(tmpl)) wflow.run({'initial': 'data'}) await wflow # These tasks have finished for tid in ('1', '2'): task = wflow._tasks_by_id.get(tid) self.assertTrue(task.done()) self.assertEqual(FutureState.get(task), FutureState.finished) # These tasks were never started for tid in ('crash', 'wont_run'): task = wflow._tasks_by_id.get(tid) self.assertIs(task, None) # The workflow finished properly self.assertTrue(wflow.done()) self.assertEqual(FutureState.get(wflow), FutureState.finished) # Test crash inside a task tmpl['tasks'][0]['config'] = {'init_ok': None} wflow = Workflow(WorkflowTemplate.from_dict(tmpl)) wflow.run({'initial': 'data'}) await wflow # These tasks have finished for tid in ('1', '2'): task = wflow._tasks_by_id.get(tid) self.assertTrue(task.done()) self.assertEqual(FutureState.get(task), FutureState.finished) # This task crashed during execution task = wflow._tasks_by_id.get('crash') self.assertTrue(task.done()) self.assertEqual(FutureState.get(task), FutureState.exception) # This task was never started task = wflow._tasks_by_id.get('wont_run') self.assertIs(task, None) # The workflow finished properly self.assertTrue(wflow.done()) self.assertEqual(FutureState.get(wflow), FutureState.finished)
async def test(): tmpl = TEMPLATES['ok'] wflow = Workflow(WorkflowTemplate.from_dict(tmpl)) wflow.run({'initial': 'data'}) await wflow # These tasks have finished for tid in tmpl['graph'].keys(): task = wflow._tasks_by_id.get(tid) self.assertTrue(task.done()) self.assertEqual(FutureState.get(task), FutureState.finished) # The workflow finished properly self.assertTrue(wflow.done()) self.assertEqual(FutureState.get(wflow), FutureState.finished)
async def reload_from_storage(self): """ Check mongo, retrieve and load all templates """ self.storage = MongoStorage(**self.mongo_config) templates = await self.storage.templates.get_all( latest=True, with_metadata=False ) for template in templates: try: await self.engine.load(WorkflowTemplate.from_dict(template)) except Exception as exc: # Means a bad workflow is in database, report it reporting.exception(exc)
async def test(): tmpl = TEMPLATES['workflow_cancel'] wflow = Workflow(WorkflowTemplate.from_dict(tmpl)) wflow.run({'initial': 'data'}) # Workflow is cancelled with self.assertRaises(asyncio.CancelledError): await wflow self.assertEqual(FutureState.get(wflow), FutureState.cancelled) # This task was cancelled task = wflow._tasks_by_id.get('cancel') with self.assertRaises(asyncio.CancelledError): task.exception() self.assertEqual(FutureState.get(task), FutureState.cancelled) # These tasks were never started for tid in ('2', '3', '4'): task = wflow._tasks_by_id.get(tid) self.assertIs(task, None)
async def put(self, request, tid): """ Create a new draft for this template id """ try: versions = await self.nyuki.storage.templates.get(tid) except AutoReconnect: return Response(status=503) if not versions: return Response(status=404) for v in versions: if v['draft'] is True: return Response(status=409, body={ 'error': 'This draft already exists' }) request = await request.json() try: # Set template ID from url template = WorkflowTemplate.from_dict({**request, 'id': tid}) except TemplateGraphError as exc: return Response(status=400, body={ 'error': str(exc) }) try: tmpl_dict = await self.update_draft(template, request) except ConflictError as exc: return Response(status=409, body={ 'error': exc }) metadata = await self.nyuki.storage.templates.get_metadata(template.uid) metadata = metadata[0] return Response({ **tmpl_dict, **metadata, 'errors': self.errors_from_validation(template) })
async def put(self, request, tid): """ Create a new draft for this template id """ try: versions = await self.nyuki.storage.templates.get(tid) except AutoReconnect: return Response(status=503) if not versions: return Response(status=404) for v in versions: if v['draft'] is True: return Response(status=409, body={'error': 'This draft already exists'}) request = await request.json() try: # Set template ID from url template = WorkflowTemplate.from_dict({**request, 'id': tid}) except TemplateGraphError as exc: return Response(status=400, body={'error': str(exc)}) try: tmpl_dict = await self.update_draft(template, request) except ConflictError as exc: return Response(status=409, body={'error': exc}) metadata = await self.nyuki.storage.templates.get_metadata(template.uid ) metadata = metadata[0] return Response({ **tmpl_dict, **metadata, 'errors': self.errors_from_validation(template) })
async def patch(self, request, tid): """ Modify the template's draft """ try: tmpl = await self.nyuki.storage.templates.get(tid, draft=True) except AutoReconnect: return Response(status=503) if not tmpl: return Response(status=404) request = await request.json() try: # Set template ID from url template = WorkflowTemplate.from_dict({**request, 'id': tid}) except TemplateGraphError as exc: return Response(status=400, body={ 'error': str(exc) }) try: tmpl_dict = await self.update_draft(template, request) except ConflictError as exc: return Response(status=409, body={ 'error': str(exc) }) metadata = await self.nyuki.storage.templates.get_metadata(template.uid) metadata = metadata[0] return Response({ **tmpl_dict, **metadata, 'errors': self.errors_from_validation(template) })
async def get(self, tmpl_id): template = await self.storage.get_template(tmpl_id, draft=False) return WorkflowTemplate.from_dict(template)
async def select(self, topic): templates = await self.storage.get_for_topic(topic) return [WorkflowTemplate.from_dict(template) for template in templates]
async def put(self, request): """ Start a workflow from payload: { "id": "template_id", "draft": true/false } """ async_topic = request.headers.get('X-Surycat-Async-Topic') exec_track = request.headers.get('X-Surycat-Exec-Track') requester = request.headers.get('Referer') request = await request.json() if 'id' not in request: return Response(status=400, body={ 'error': "Template's ID key 'id' is mandatory" }) draft = request.get('draft', False) try: templates = await self.nyuki.storage.templates.get( request['id'], draft=draft, with_metadata=True ) except AutoReconnect: return Response(status=503) if not templates: return Response(status=404, body={ 'error': 'Could not find a suitable template to run' }) wf_tmpl = WorkflowTemplate.from_dict(templates[0]) data = request.get('inputs', {}) if draft: wflow = await self.nyuki.engine.run_once(wf_tmpl, data) else: wflow = await self.nyuki.engine.trigger(wf_tmpl.uid, data) if wflow is None: return Response(status=400, body={ 'error': 'Could not start any workflow from this template' }) # Prevent workflow loop exec_track = exec_track.split(',') if exec_track else [] holder = self.nyuki.bus.name for ancestor in exec_track: try: info = URI.parse(ancestor) except InvalidWorkflowUri: continue if info.template_id == wf_tmpl.uid and info.holder == holder: return Response(status=400, body={ 'error': 'Loop detected between workflows' }) # Keep full instance+template in nyuki's memory wfinst = self.nyuki.new_workflow( templates[0], wflow, track=exec_track, requester=requester ) # Handle async workflow exec updates if async_topic is not None: self.register_async_handler(async_topic, wflow) return Response( json.dumps( wfinst.report(), default=serialize_wflow_exec ), content_type='application/json' )
async def put(self, request): """ Start a workflow from payload: { "id": "template_id", "draft": true/false "exec": {} } """ async_topic = request.headers.get('X-Surycat-Async-Topic') exec_track = request.headers.get('X-Surycat-Exec-Track') requester = request.headers.get('Referer') request = await request.json() if 'id' not in request: return Response( status=400, body={'error': "Template's ID key 'id' is mandatory"}) draft = request.get('draft', False) data = request.get('inputs', {}) exec = request.get('exec') if exec: # Suspended/crashed instance # The request's payload is the last known execution report templates = [request] if exec['id'] in self.nyuki.running_workflows: return Response( status=400, body={'error': 'This workflow is already being rescued'}) else: # Fetch the template from the storage try: templates = await self.nyuki.storage.templates.get( request['id'], draft=draft, with_metadata=True) except AutoReconnect: return Response(status=503) if not templates: return Response( status=404, body={'error': 'Could not find a suitable template to run'}) wf_tmpl = WorkflowTemplate.from_dict(templates[0]) if exec: wflow = await self.nyuki.engine.rescue(wf_tmpl, request) elif draft: wflow = await self.nyuki.engine.run_once(wf_tmpl, data) else: wflow = await self.nyuki.engine.trigger(wf_tmpl.uid, data) if wflow is None: return Response( status=400, body={ 'error': 'Could not start any workflow from this template' }) # Prevent workflow loop exec_track = exec_track.split(',') if exec_track else [] holder = self.nyuki.bus.name for ancestor in exec_track: try: info = URI.parse(ancestor) except InvalidWorkflowUri: continue if info.template_id == wf_tmpl.uid and info.holder == holder: return Response( status=400, body={'error': 'Loop detected between workflows'}) # Keep full instance+template in nyuki's memory wfinst = self.nyuki.new_workflow(templates[0], wflow, track=exec_track, requester=requester) # Handle async workflow exec updates if async_topic is not None: self.register_async_handler(async_topic, wflow) return Response(wfinst.report())