def setUp(self): case_database.initialize() self.app = flask_server.app.test_client(self) self.app.testing = True self.app.post('/login', data=dict(email='admin', password='******'), follow_redirects=True) self.c = controller.Controller() self.c.loadWorkflowsFromFile( path=path.join(config.testWorkflowsPath, 'simpleDataManipulationWorkflow.workflow')) self.id_tuple = ('simpleDataManipulationWorkflow', 'helloWorldWorkflow') self.workflow_name = construct_workflow_name_key(*self.id_tuple) self.testWorkflow = self.c.get_workflow(*self.id_tuple)
def test_display_workflow(self): workflow_filename = os.path.join(testWorkflowsPath, 'multiactionWorkflowTest.workflow') workflow_name = helpers.construct_workflow_name_key('multiactionWorkflowTest', 'multiactionWorkflow') flask_server.running_context.controller.loadWorkflowsFromFile(path=workflow_filename) workflow = flask_server.running_context.controller.get_workflow('multiactionWorkflowTest', 'multiactionWorkflow') steps_data = workflow.get_cytoscape_data() options_data = workflow.options.as_json() expected_response = {'status': 'success', 'steps': steps_data, 'options': options_data} response = self.app.get('/playbook/multiactionWorkflowTest/multiactionWorkflow/display', headers=self.headers) self.assertEqual(response.status_code, 200) response = json.loads(response.get_data(as_text=True)) self.assertDictEqual(response, expected_response)
def update_workflow_name(self, old_playbook, old_workflow, new_playbook, new_workflow): """Update the name of a workflow. Args: old_playbook (str): Name of the current playbook. old_workflow (str): Name of the current workflow. new_playbook (str): The new name of the playbook. new_workflow (str): The new name of the workflow. """ old_key = _WorkflowKey(old_playbook, old_workflow) new_key = _WorkflowKey(new_playbook, new_workflow) self.workflows[new_key] = self.workflows.pop(old_key) self.workflows[new_key].name = construct_workflow_name_key(new_playbook, new_workflow) self.workflows[new_key].reconstruct_ancestry([self.name]) logger.debug('updated workflow name {0} to {1}'.format(old_key, new_key))
def test_simple_workflow_execution(self): workflow_name = construct_workflow_name_key('basicWorkflowTest', 'helloWorldWorkflow') setup_subscriptions_for_step(workflow_name, ['start']) self.controller.execute_workflow('basicWorkflowTest', 'helloWorldWorkflow') shutdown_pool() steps = executed_steps('defaultController', workflow_name, self.start, datetime.utcnow()) self.assertEqual(len(steps), 1) step = steps[0] ancestry = step['ancestry'].split(',') self.assertEqual(ancestry[-1], "start") result = json.loads(step['data']) self.assertEqual(result['result'], "REPEATING: Hello World")
def test_copy_workflow_different_playbook(self): self.put_with_status_check('/playbooks/new_playbook', headers=self.headers, status_code=OBJECT_CREATED) data = {"playbook": "new_playbook"} self.post_with_status_check( '/playbooks/test/workflows/helloWorldWorkflow/copy', data=data, headers=self.headers, status_code=OBJECT_CREATED) self.assertEqual( len(flask_server.running_context.controller.workflows.keys()), 3) self.assertTrue( flask_server.running_context.controller.is_workflow_registered( 'test', 'helloWorldWorkflow')) self.assertTrue( flask_server.running_context.controller.is_workflow_registered( 'new_playbook', 'helloWorldWorkflow_Copy')) workflow_original = flask_server.running_context.controller.get_workflow( 'test', 'helloWorldWorkflow') workflow_copy = flask_server.running_context.controller.get_workflow( 'new_playbook', 'helloWorldWorkflow_Copy') new_workflow_name = helpers.construct_workflow_name_key( 'new_playbook', 'helloWorldWorkflow_Copy') self.assertEqual(workflow_copy.name, new_workflow_name) copy_workflow_json = workflow_copy.as_json() original_workflow_json = workflow_original.as_json() copy_workflow_json.pop('name', None) original_workflow_json.pop('name', None) self.assertDictEqual(copy_workflow_json, original_workflow_json) self.assertEqual(len(workflow_original.steps), len(workflow_copy.steps)) for step in workflow_copy.steps: self.assertTrue( new_workflow_name in workflow_copy.steps[step].ancestry) self.assertEqual(len(workflow_original.steps[step].conditionals), len(workflow_copy.steps[step].conditionals)) for nextstep in workflow_copy.steps[step].conditionals: self.assertTrue(new_workflow_name in nextstep.ancestry) for flag in nextstep.flags: self.assertTrue(new_workflow_name in flag.ancestry) for filter_element in flag.filters: self.assertTrue( new_workflow_name in filter_element.ancestry)
def test_ffkExecutionEventsCase(self): c = controller.Controller(name="testStepFFKEventsController") c.load_workflows_from_file(path=config.test_workflows_path + "basicWorkflowTest.workflow") workflow_name = construct_workflow_name_key('basicWorkflowTest', 'helloWorldWorkflow') filter_sub = Subscription(events=['Filter Error']) flag_sub = Subscription(events=['Flag Arguments Valid', 'Flag Arguments Invalid'], subscriptions={'length': filter_sub}) next_sub = Subscription(events=['Next Step Taken', 'Next Step Not Taken'], subscriptions={'regMatch': flag_sub}) step_sub = Subscription(events=['Function Execution Success', 'Input Validated', 'Conditionals Executed'], subscriptions={'1': next_sub}) subs = {'testStepFFKEventsController': Subscription(subscriptions= {workflow_name: Subscription(subscriptions= {'start': step_sub})})} global_subs = case_subscription.GlobalSubscriptions(step=['Function Execution Success', 'Input Validated', 'Conditionals Executed'], next_step=['Next Step Taken', 'Next Step Not Taken'], flag=['Flag Arguments Valid', 'Flag Arguments Invalid'], filter=['Filter Error']) case_subscription.set_subscriptions( {'testStepFFKEventsEvents': case_subscription.CaseSubscriptions(subscriptions=subs, global_subscriptions=global_subs)}) c.execute_workflow('basicWorkflowTest', 'helloWorldWorkflow') running_context.shutdown_threads() step_ffk_events_case = case_database.case_db.session.query(case_database.Case) \ .filter(case_database.Case.name == 'testStepFFKEventsEvents').first() step_ffk_event_history = step_ffk_events_case.events.all() self.assertEqual(len(step_ffk_event_history), 5, 'Incorrect length of event history. ' 'Expected {0}, got {1}'.format(5, len(step_ffk_event_history))) step_json = [step.as_json() for step in step_ffk_event_history if step.as_json()['message'] == 'STEP'] for step in step_json: if step['type'] == 'Function executed successfully': self.assertDictEqual(step['data'], {'result': 'REPEATING: Hello World'}) else: self.assertEqual(step['data'], '')
def __get_child_step_generator(self, tiered_step_str): params = tiered_step_str.split(':') if len(params) == 3: child_name, child_start, child_next = params[0].lstrip( '@'), params[1], params[2] child_name = construct_workflow_name_key(self.playbook_name, child_name) if (child_name in self.options.children and type( self.options.children[child_name]).__name__ == 'Workflow'): logger.debug( 'Executing child workflow {0} of workflow {1}'.format( child_name, self.ancestry)) callbacks.WorkflowExecutionStart.send( self.options.children[child_name]) child_step_generator = self.options.children[ child_name].__steps(start=child_start) return child_step_generator, child_next, child_name return None, None
def test_simple_workflow_execution(self): workflow_name = construct_workflow_name_key('basicWorkflowTest', 'helloWorldWorkflow') setup_subscriptions_for_step(workflow_name, ['start']) server.running_context.controller.execute_workflow( 'basicWorkflowTest', 'helloWorldWorkflow') with server.running_context.flask_app.app_context(): server.running_context.shutdown_threads() steps = executed_steps('defaultController', workflow_name, self.start, datetime.utcnow()) self.assertEqual(len(steps), 1) step = steps[0] ancestry = step['ancestry'].split(',') self.assertEqual(ancestry[-1], "start") self.assertEqual(step['data']['result'], "REPEATING: Hello World")
def load_workflows_from_file(self, path, name_override=None, playbook_override=None): """Loads multiple workloads from a file. Args: path (str): Path to the workflow. name_override (str, optional): Name that the workflow should be changed to. playbook_override (str, optional): Name that the playbook should be changed to. """ self.tree = ElementTree.ElementTree(file=path) playbook_name = playbook_override if playbook_override else os.path.splitext(os.path.basename(path))[0] for workflow in self.tree.iter(tag='workflow'): workflow_name = name_override if name_override else workflow.get('name') name = construct_workflow_name_key(playbook_name, workflow_name) key = _WorkflowKey(playbook_name, workflow_name) self.__add_workflow(key, name, workflow, playbook_name) self.add_child_workflows() self.add_workflow_scheduled_jobs()
def loadWorkflowsFromFile(self, path, name_override=None, playbook_override=None): self.tree = et.ElementTree(file=path) playbook_name = playbook_override if playbook_override else os.path.splitext( os.path.basename(path))[0] for workflow in self.tree.iter(tag='workflow'): workflow_name = name_override if name_override else workflow.get( 'name') name = construct_workflow_name_key(playbook_name, workflow_name) key = _WorkflowKey(playbook_name, workflow_name) self.workflows[key] = wf.Workflow(name=name, workflowConfig=workflow, parent_name=self.name, filename=playbook_name) self.addChildWorkflows() self.addWorkflowScheduledJobs()
def copy_workflow(self, old_playbook_name, new_playbook_name, old_workflow_name, new_workflow_name): """Duplicates a workflow into its current playbook, or a different playbook. Args: old_playbook_name (str): Playbook name under which the workflow is located. new_playbook_name (str): The new playbook name for the duplicated workflow. old_workflow_name (str): The name of the workflow to be copied. new_workflow_name (str): The new name of the duplicated workflow. """ workflow = self.get_workflow(old_playbook_name, old_workflow_name) workflow_copy = deepcopy(workflow) workflow_copy.playbook_name = new_playbook_name workflow_copy.name = construct_workflow_name_key(new_playbook_name, new_workflow_name) key = _WorkflowKey(new_playbook_name, new_workflow_name) self.workflows[key] = workflow_copy self.workflows[key].reconstruct_ancestry([self.name]) logger.info('Workflow copied from {0}-{1} to {2}-{3}'.format(old_playbook_name, old_workflow_name, new_playbook_name, new_workflow_name))
def test_ffkExecutionEvents(self): workflow_name = construct_workflow_name_key('basicWorkflowTest', 'helloWorldWorkflow') c = Controller(name="testStepFFKEventsController") c.load_workflows_from_file(path=config.test_workflows_path + "basicWorkflowTest.playbook") filter_sub = Subscription(events=['Filter Success', 'Filter Error']) flag_sub = Subscription(events=['Flag Success', 'Flag Error'], subscriptions={'length': filter_sub}) next_sub = Subscription( events=['Next Step Taken', 'Next Step Not Taken'], subscriptions={'regMatch': flag_sub}) step_sub = Subscription(events=[ "Function Execution Success", "Input Validated", "Conditionals Executed" ], subscriptions={'1': next_sub}) subs = { 'testStepFFKEventsController': Subscription( subscriptions={ workflow_name: Subscription( subscriptions={'start': step_sub}) }) } case_subscription.set_subscriptions({ 'testStepFFKEventsEvents': case_subscription.CaseSubscriptions(subscriptions=subs) }) c.execute_workflow('basicWorkflowTest', 'helloWorldWorkflow') shutdown_pool() step_ffk_events_case = case_database.case_db.session.query(case_database.Case) \ .filter(case_database.Case.name == 'testStepFFKEventsEvents').first() step_ffk_event_history = step_ffk_events_case.events.all() self.assertEqual( len(step_ffk_event_history), 6, 'Incorrect length of event history. ' 'Expected {0}, got {1}'.format(6, len(step_ffk_event_history)))
def load_workflow_from_file(self, path, workflow_name, name_override=None, playbook_override=None): """Loads a workflow from a file. Args: path (str): Path to the workflow. workflow_name (str): Name of the workflow to load. name_override (str, optional): Name that the workflow should be changed to. playbook_override (str, optional): Name that the playbook should be changed to. Returns: True on success, False otherwise. """ self.tree = cElementTree.ElementTree(file=path) playbook_name = playbook_override if playbook_override else os.path.splitext( os.path.basename(path))[0] for workflow in self.tree.iter(tag='workflow'): current_workflow_name = workflow.get('name') if current_workflow_name == workflow_name: if name_override: workflow_name = name_override name = construct_workflow_name_key(playbook_name, workflow_name) key = _WorkflowKey(playbook_name, workflow_name) self.workflows[key] = wf.Workflow(name=name, xml=workflow, parent_name=self.name, playbook_name=playbook_name) logger.info('Adding workflow {0} to controller'.format(name)) break else: logger.warning( 'Workflow {0} not found in playbook {0}. Cannot load.'.format( workflow_name, playbook_name)) return False self.add_child_workflows() self.add_workflow_scheduled_jobs() return True
def test_workflow_result_recording(self): print(server.workflowresults.results) flaskserver.running_context.controller.load_workflows_from_file(path=config.test_workflows_path + 'multiactionWorkflowTest.playbook') multiaction_key = construct_workflow_name_key('multiactionWorkflowTest', 'multiactionWorkflow') flaskserver.running_context.controller.execute_workflow('multiactionWorkflowTest', 'multiactionWorkflow') with flaskserver.running_context.flask_app.app_context(): flaskserver.running_context.shutdown_threads() print(server.workflowresults.results) self.assertEqual(len(server.workflowresults.results), 1) key = server.workflowresults.results.keys()[0] self.assertIn('status', server.workflowresults.results[key]) self.assertEqual(server.workflowresults.results[key]['status'], 'completed') self.assertIn('name', server.workflowresults.results[key]) self.assertEqual(server.workflowresults.results[key]['name'], multiaction_key) self.assertIn('completed_at', server.workflowresults.results[key]) self.assertIn('started_at', server.workflowresults.results[key]) self.assertIn('results', server.workflowresults.results[key]) self.assertEqual(len(server.workflowresults.results[key]['results']), 2) self.assertEqual(len(server.workflowresults.results[key]['results']), 2)
def _from_xml(self, xml_element, options_name='Default', workflow_name='', filename=''): ExecutionElement.__init__(self, name=options_name, parent_name=workflow_name) self.scheduler = { 'autorun': xml_element.find('.//scheduler').get('autorun'), 'type': xml_element.find('.//scheduler').get('type'), 'args': { option.tag: option.text for option in xml_element.findall('.//scheduler/*') } } self.enabled = bool(xml_element.find('.//enabled').text) self.children = { construct_workflow_name_key(filename, child.text): None for child in xml_element.findall('.//children/child') }
def test_execute_workflow(self): sync = Event() workflow_name = helpers.construct_workflow_name_key('test', 'helloWorldWorkflow') setup_subscriptions_for_step(workflow_name, ['start']) start = datetime.utcnow() def wait_for_completion(sender, **kwargs): sync.set() WorkflowShutdown.connect(wait_for_completion) self.post_with_status_check('/playbooks/test/workflows/helloWorldWorkflow/execute', 'success', headers=self.headers) sync.wait(timeout=10) steps = executed_steps('defaultController', workflow_name, start, datetime.utcnow()) self.assertEqual(len(steps), 1) step = steps[0] ancestry = step['ancestry'].split(',') self.assertEqual(ancestry[-1], "start") self.assertEqual(step['data']['result'], "REPEATING: Hello World")
def test_templated_workflow(self): workflow_name = construct_workflow_name_key('templatedWorkflowTest', 'templatedWorkflow') step_names = ['start', '1'] setup_subscriptions_for_step(workflow_name, step_names) self.controller.execute_workflow('templatedWorkflowTest', 'templatedWorkflow') shutdown_pool() steps = executed_steps('defaultController', workflow_name, self.start, datetime.utcnow()) self.assertEqual(len(steps), 2, 'Unexpected number of steps executed. ' 'Expected {0}, got {1}'.format(2, len(steps))) names = [step['ancestry'].split(',')[-1] for step in steps] orderless_list_compare(self, names, step_names) name_result = {'start': {'result': {'message': 'HELLO WORLD'}, 'status': 'Success'}, '1': {'status': 'Success', 'result': "REPEATING: {'message': 'HELLO WORLD'}"}} for step in steps: name = step['ancestry'].split(',')[-1] self.assertIn(name, name_result) result = json.loads(step['data']) self.assertDictEqual(result['result'], name_result[name])
def test_stepExecutionEvents(self): workflow_name = construct_workflow_name_key('basicWorkflowTest', 'helloWorldWorkflow') c = Controller(name="testStepExecutionEventsController") c.load_workflows_from_file(path=config.test_workflows_path + "basicWorkflowTest.playbook") subs = { 'testStepExecutionEventsController': Subscription( subscriptions={ workflow_name: Subscription( subscriptions={ 'start': Subscription(events=[ "Function Execution Success", "Input Validated", "Conditionals Executed" ]) }) }) } case_subscription.set_subscriptions({ 'testStepExecutionEvents': case_subscription.CaseSubscriptions(subscriptions=subs) }) c.execute_workflow('basicWorkflowTest', 'helloWorldWorkflow') shutdown_pool() step_execution_events_case = case_database.case_db.session.query(case_database.Case) \ .filter(case_database.Case.name == 'testStepExecutionEvents').first() step_execution_event_history = step_execution_events_case.events.all() self.assertEqual( len(step_execution_event_history), 3, 'Incorrect length of event history. ' 'Expected {0}, got {1}'.format(3, len(step_execution_event_history)))
def test_workflowExecutionEvents(self): workflow_name = construct_workflow_name_key('multiactionWorkflowTest', 'multiactionWorkflow') c = controller.Controller(name="testExecutionEventsController") c.loadWorkflowsFromFile(path=config.testWorkflowsPath + "multiactionWorkflowTest.workflow") subs = {'testExecutionEventsController': Subscription(subscriptions= {workflow_name: Subscription(events=["InstanceCreated", "StepExecutionSuccess", "NextStepFound", "WorkflowShutdown"])})} case_subscription.set_subscriptions( {'testExecutionEvents': case_subscription.CaseSubscriptions(subscriptions=subs)}) c.executeWorkflow('multiactionWorkflowTest', 'multiactionWorkflow') execution_events_case = case_database.case_db.session.query(case_database.Case) \ .filter(case_database.Case.name == 'testExecutionEvents').first() execution_event_history = execution_events_case.events.all() self.assertEqual(len(execution_event_history), 6, 'Incorrect length of event history. ' 'Expected {0}, got {1}'.format(6, len(execution_event_history)))
def test_error_workflow(self): workflow_name = construct_workflow_name_key('multistepError', 'multiactionErrorWorkflow') step_names = ['start', '1', 'error'] setup_subscriptions_for_step(workflow_name, step_names) self.controller.execute_workflow('multistepError', 'multiactionErrorWorkflow') shutdown_pool() steps = executed_steps('defaultController', workflow_name, self.start, datetime.utcnow()) self.assertEqual(len(steps), 2) names = [step['ancestry'].split(',')[-1] for step in steps] orderless_list_compare(self, names, ['start', 'error']) name_result = {'start': {"message": "HELLO WORLD"}, 'error': "REPEATING: Hello World"} for step in steps: name = step['ancestry'].split(',')[-1] self.assertIn(name, name_result) result = json.loads(step['data']) if type(name_result[name]) == dict: self.assertDictEqual(result['result'], name_result[name]) else: self.assertEqual(result['result'], name_result[name])
def test_Loop(self): self.c.loadWorkflowsFromFile(path=config.testWorkflowsPath + 'loopWorkflow.workflow') workflow_name = construct_workflow_name_key('loopWorkflow', 'loopWorkflow') step_names = ['start', '1'] setup_subscriptions_for_step(workflow_name, step_names) self.c.executeWorkflow('loopWorkflow', 'loopWorkflow') steps = executed_steps('defaultController', workflow_name, self.start, datetime.utcnow()) names = [step['ancestry'].split(',')[-1] for step in steps] expected_steps = ['start', 'start', 'start', 'start', '1'] self.assertListEqual(names, expected_steps) self.assertEqual(len(steps), 5) input_output = [('start', 1), ('start', 2), ('start', 3), ('start', 4), ('1', 'REPEATING: 5')] for step_name, output in input_output: for step in steps: name = step['ancestry'].split(',') if name == step_name: self.assertEqual(step['data']['result'], output)
def test_workflow_with_dataflow(self): workflow_name = construct_workflow_name_key('dataflowTest', 'dataflowWorkflow') step_names = ['start', '1', '2'] setup_subscriptions_for_step(workflow_name, step_names) self.controller.execute_workflow('dataflowTest', 'dataflowWorkflow') shutdown_pool() steps = executed_steps('defaultController', workflow_name, self.start, datetime.utcnow()) self.assertEqual(len(steps), 3) names = [step['ancestry'].split(',')[-1] for step in steps] orderless_list_compare(self, names, ['start', '1', '2']) name_result = {'start': 6, '1': 6, '2': 15} for step in steps: name = step['ancestry'].split(',')[-1] self.assertIn(name, name_result) result = json.loads(step['data']) if type(name_result[name]) == dict: self.assertDictEqual(result['result'], name_result[name]) else: self.assertEqual(result['result'], name_result[name])
def convert_ancestry(ancestry): if len(ancestry) >= 3: ancestry[1] = construct_workflow_name_key(ancestry[1], ancestry[2]) del ancestry[2] return ancestry
def __get_record(playbook, workflow): name = construct_workflow_name_key(playbook, workflow) return next((record for record in server.workflowresults.results if record['name'] == name), None)