def setUp(self):
case_database.initialize()
self.app = flask_server.app.test_client(self)
self.app.testing = True
self.app.post('/login',
data=dict(email='admin', password='******'),
follow_redirects=True)
self.c = controller.Controller()
self.c.loadWorkflowsFromFile(
path=path.join(config.testWorkflowsPath,
'simpleDataManipulationWorkflow.workflow'))
self.id_tuple = ('simpleDataManipulationWorkflow',
'helloWorldWorkflow')
self.workflow_name = construct_workflow_name_key(*self.id_tuple)
self.testWorkflow = self.c.get_workflow(*self.id_tuple)
def test_display_workflow(self):
workflow_filename = os.path.join(testWorkflowsPath, 'multiactionWorkflowTest.workflow')
workflow_name = helpers.construct_workflow_name_key('multiactionWorkflowTest', 'multiactionWorkflow')
flask_server.running_context.controller.loadWorkflowsFromFile(path=workflow_filename)
workflow = flask_server.running_context.controller.get_workflow('multiactionWorkflowTest',
'multiactionWorkflow')
steps_data = workflow.get_cytoscape_data()
options_data = workflow.options.as_json()
expected_response = {'status': 'success',
'steps': steps_data,
'options': options_data}
response = self.app.get('/playbook/multiactionWorkflowTest/multiactionWorkflow/display', headers=self.headers)
self.assertEqual(response.status_code, 200)
response = json.loads(response.get_data(as_text=True))
self.assertDictEqual(response, expected_response)
def update_workflow_name(self, old_playbook, old_workflow, new_playbook, new_workflow):
"""Update the name of a workflow.
Args:
old_playbook (str): Name of the current playbook.
old_workflow (str): Name of the current workflow.
new_playbook (str): The new name of the playbook.
new_workflow (str): The new name of the workflow.
"""
old_key = _WorkflowKey(old_playbook, old_workflow)
new_key = _WorkflowKey(new_playbook, new_workflow)
self.workflows[new_key] = self.workflows.pop(old_key)
self.workflows[new_key].name = construct_workflow_name_key(new_playbook, new_workflow)
self.workflows[new_key].reconstruct_ancestry([self.name])
logger.debug('updated workflow name {0} to {1}'.format(old_key, new_key))
def test_simple_workflow_execution(self):
workflow_name = construct_workflow_name_key('basicWorkflowTest', 'helloWorldWorkflow')
setup_subscriptions_for_step(workflow_name, ['start'])
self.controller.execute_workflow('basicWorkflowTest', 'helloWorldWorkflow')
shutdown_pool()
steps = executed_steps('defaultController', workflow_name, self.start, datetime.utcnow())
self.assertEqual(len(steps), 1)
step = steps[0]
ancestry = step['ancestry'].split(',')
self.assertEqual(ancestry[-1], "start")
result = json.loads(step['data'])
self.assertEqual(result['result'], "REPEATING: Hello World")
def test_copy_workflow_different_playbook(self):
self.put_with_status_check('/playbooks/new_playbook',
headers=self.headers,
status_code=OBJECT_CREATED)
data = {"playbook": "new_playbook"}
self.post_with_status_check(
'/playbooks/test/workflows/helloWorldWorkflow/copy',
data=data,
headers=self.headers,
status_code=OBJECT_CREATED)
self.assertEqual(
len(flask_server.running_context.controller.workflows.keys()), 3)
self.assertTrue(
flask_server.running_context.controller.is_workflow_registered(
'test', 'helloWorldWorkflow'))
self.assertTrue(
flask_server.running_context.controller.is_workflow_registered(
'new_playbook', 'helloWorldWorkflow_Copy'))
workflow_original = flask_server.running_context.controller.get_workflow(
'test', 'helloWorldWorkflow')
workflow_copy = flask_server.running_context.controller.get_workflow(
'new_playbook', 'helloWorldWorkflow_Copy')
new_workflow_name = helpers.construct_workflow_name_key(
'new_playbook', 'helloWorldWorkflow_Copy')
self.assertEqual(workflow_copy.name, new_workflow_name)
copy_workflow_json = workflow_copy.as_json()
original_workflow_json = workflow_original.as_json()
copy_workflow_json.pop('name', None)
original_workflow_json.pop('name', None)
self.assertDictEqual(copy_workflow_json, original_workflow_json)
self.assertEqual(len(workflow_original.steps),
len(workflow_copy.steps))
for step in workflow_copy.steps:
self.assertTrue(
new_workflow_name in workflow_copy.steps[step].ancestry)
self.assertEqual(len(workflow_original.steps[step].conditionals),
len(workflow_copy.steps[step].conditionals))
for nextstep in workflow_copy.steps[step].conditionals:
self.assertTrue(new_workflow_name in nextstep.ancestry)
for flag in nextstep.flags:
self.assertTrue(new_workflow_name in flag.ancestry)
for filter_element in flag.filters:
self.assertTrue(
new_workflow_name in filter_element.ancestry)
def test_ffkExecutionEventsCase(self):
c = controller.Controller(name="testStepFFKEventsController")
c.load_workflows_from_file(path=config.test_workflows_path + "basicWorkflowTest.workflow")
workflow_name = construct_workflow_name_key('basicWorkflowTest', 'helloWorldWorkflow')
filter_sub = Subscription(events=['Filter Error'])
flag_sub = Subscription(events=['Flag Arguments Valid',
'Flag Arguments Invalid'], subscriptions={'length': filter_sub})
next_sub = Subscription(events=['Next Step Taken',
'Next Step Not Taken'],
subscriptions={'regMatch': flag_sub})
step_sub = Subscription(events=['Function Execution Success',
'Input Validated',
'Conditionals Executed'], subscriptions={'1': next_sub})
subs = {'testStepFFKEventsController':
Subscription(subscriptions=
{workflow_name:
Subscription(subscriptions=
{'start': step_sub})})}
global_subs = case_subscription.GlobalSubscriptions(step=['Function Execution Success',
'Input Validated',
'Conditionals Executed'],
next_step=['Next Step Taken',
'Next Step Not Taken'],
flag=['Flag Arguments Valid',
'Flag Arguments Invalid'],
filter=['Filter Error'])
case_subscription.set_subscriptions(
{'testStepFFKEventsEvents': case_subscription.CaseSubscriptions(subscriptions=subs,
global_subscriptions=global_subs)})
c.execute_workflow('basicWorkflowTest', 'helloWorldWorkflow')
running_context.shutdown_threads()
step_ffk_events_case = case_database.case_db.session.query(case_database.Case) \
.filter(case_database.Case.name == 'testStepFFKEventsEvents').first()
step_ffk_event_history = step_ffk_events_case.events.all()
self.assertEqual(len(step_ffk_event_history), 5,
'Incorrect length of event history. '
'Expected {0}, got {1}'.format(5, len(step_ffk_event_history)))
step_json = [step.as_json() for step in step_ffk_event_history if step.as_json()['message'] == 'STEP']
for step in step_json:
if step['type'] == 'Function executed successfully':
self.assertDictEqual(step['data'], {'result': 'REPEATING: Hello World'})
else:
self.assertEqual(step['data'], '')
def __get_child_step_generator(self, tiered_step_str):
params = tiered_step_str.split(':')
if len(params) == 3:
child_name, child_start, child_next = params[0].lstrip(
'@'), params[1], params[2]
child_name = construct_workflow_name_key(self.playbook_name,
child_name)
if (child_name in self.options.children and type(
self.options.children[child_name]).__name__ == 'Workflow'):
logger.debug(
'Executing child workflow {0} of workflow {1}'.format(
child_name, self.ancestry))
callbacks.WorkflowExecutionStart.send(
self.options.children[child_name])
child_step_generator = self.options.children[
child_name].__steps(start=child_start)
return child_step_generator, child_next, child_name
return None, None
def test_simple_workflow_execution(self):
workflow_name = construct_workflow_name_key('basicWorkflowTest',
'helloWorldWorkflow')
setup_subscriptions_for_step(workflow_name, ['start'])
server.running_context.controller.execute_workflow(
'basicWorkflowTest', 'helloWorldWorkflow')
with server.running_context.flask_app.app_context():
server.running_context.shutdown_threads()
steps = executed_steps('defaultController', workflow_name, self.start,
datetime.utcnow())
self.assertEqual(len(steps), 1)
step = steps[0]
ancestry = step['ancestry'].split(',')
self.assertEqual(ancestry[-1], "start")
self.assertEqual(step['data']['result'], "REPEATING: Hello World")
def load_workflows_from_file(self, path, name_override=None, playbook_override=None):
"""Loads multiple workloads from a file.
Args:
path (str): Path to the workflow.
name_override (str, optional): Name that the workflow should be changed to.
playbook_override (str, optional): Name that the playbook should be changed to.
"""
self.tree = ElementTree.ElementTree(file=path)
playbook_name = playbook_override if playbook_override else os.path.splitext(os.path.basename(path))[0]
for workflow in self.tree.iter(tag='workflow'):
workflow_name = name_override if name_override else workflow.get('name')
name = construct_workflow_name_key(playbook_name, workflow_name)
key = _WorkflowKey(playbook_name, workflow_name)
self.__add_workflow(key, name, workflow, playbook_name)
self.add_child_workflows()
self.add_workflow_scheduled_jobs()
def loadWorkflowsFromFile(self,
path,
name_override=None,
playbook_override=None):
self.tree = et.ElementTree(file=path)
playbook_name = playbook_override if playbook_override else os.path.splitext(
os.path.basename(path))[0]
for workflow in self.tree.iter(tag='workflow'):
workflow_name = name_override if name_override else workflow.get(
'name')
name = construct_workflow_name_key(playbook_name, workflow_name)
key = _WorkflowKey(playbook_name, workflow_name)
self.workflows[key] = wf.Workflow(name=name,
workflowConfig=workflow,
parent_name=self.name,
filename=playbook_name)
self.addChildWorkflows()
self.addWorkflowScheduledJobs()
def copy_workflow(self, old_playbook_name, new_playbook_name, old_workflow_name, new_workflow_name):
"""Duplicates a workflow into its current playbook, or a different playbook.
Args:
old_playbook_name (str): Playbook name under which the workflow is located.
new_playbook_name (str): The new playbook name for the duplicated workflow.
old_workflow_name (str): The name of the workflow to be copied.
new_workflow_name (str): The new name of the duplicated workflow.
"""
workflow = self.get_workflow(old_playbook_name, old_workflow_name)
workflow_copy = deepcopy(workflow)
workflow_copy.playbook_name = new_playbook_name
workflow_copy.name = construct_workflow_name_key(new_playbook_name, new_workflow_name)
key = _WorkflowKey(new_playbook_name, new_workflow_name)
self.workflows[key] = workflow_copy
self.workflows[key].reconstruct_ancestry([self.name])
logger.info('Workflow copied from {0}-{1} to {2}-{3}'.format(old_playbook_name, old_workflow_name,
new_playbook_name, new_workflow_name))
def test_ffkExecutionEvents(self):
workflow_name = construct_workflow_name_key('basicWorkflowTest',
'helloWorldWorkflow')
c = Controller(name="testStepFFKEventsController")
c.load_workflows_from_file(path=config.test_workflows_path +
"basicWorkflowTest.playbook")
filter_sub = Subscription(events=['Filter Success', 'Filter Error'])
flag_sub = Subscription(events=['Flag Success', 'Flag Error'],
subscriptions={'length': filter_sub})
next_sub = Subscription(
events=['Next Step Taken', 'Next Step Not Taken'],
subscriptions={'regMatch': flag_sub})
step_sub = Subscription(events=[
"Function Execution Success", "Input Validated",
"Conditionals Executed"
],
subscriptions={'1': next_sub})
subs = {
'testStepFFKEventsController':
Subscription(
subscriptions={
workflow_name: Subscription(
subscriptions={'start': step_sub})
})
}
case_subscription.set_subscriptions({
'testStepFFKEventsEvents':
case_subscription.CaseSubscriptions(subscriptions=subs)
})
c.execute_workflow('basicWorkflowTest', 'helloWorldWorkflow')
shutdown_pool()
step_ffk_events_case = case_database.case_db.session.query(case_database.Case) \
.filter(case_database.Case.name == 'testStepFFKEventsEvents').first()
step_ffk_event_history = step_ffk_events_case.events.all()
self.assertEqual(
len(step_ffk_event_history), 6,
'Incorrect length of event history. '
'Expected {0}, got {1}'.format(6, len(step_ffk_event_history)))
def load_workflow_from_file(self,
path,
workflow_name,
name_override=None,
playbook_override=None):
"""Loads a workflow from a file.
Args:
path (str): Path to the workflow.
workflow_name (str): Name of the workflow to load.
name_override (str, optional): Name that the workflow should be changed to.
playbook_override (str, optional): Name that the playbook should be changed to.
Returns:
True on success, False otherwise.
"""
self.tree = cElementTree.ElementTree(file=path)
playbook_name = playbook_override if playbook_override else os.path.splitext(
os.path.basename(path))[0]
for workflow in self.tree.iter(tag='workflow'):
current_workflow_name = workflow.get('name')
if current_workflow_name == workflow_name:
if name_override:
workflow_name = name_override
name = construct_workflow_name_key(playbook_name,
workflow_name)
key = _WorkflowKey(playbook_name, workflow_name)
self.workflows[key] = wf.Workflow(name=name,
xml=workflow,
parent_name=self.name,
playbook_name=playbook_name)
logger.info('Adding workflow {0} to controller'.format(name))
break
else:
logger.warning(
'Workflow {0} not found in playbook {0}. Cannot load.'.format(
workflow_name, playbook_name))
return False
self.add_child_workflows()
self.add_workflow_scheduled_jobs()
return True
def test_workflow_result_recording(self):
print(server.workflowresults.results)
flaskserver.running_context.controller.load_workflows_from_file(path=config.test_workflows_path +
'multiactionWorkflowTest.playbook')
multiaction_key = construct_workflow_name_key('multiactionWorkflowTest', 'multiactionWorkflow')
flaskserver.running_context.controller.execute_workflow('multiactionWorkflowTest', 'multiactionWorkflow')
with flaskserver.running_context.flask_app.app_context():
flaskserver.running_context.shutdown_threads()
print(server.workflowresults.results)
self.assertEqual(len(server.workflowresults.results), 1)
key = server.workflowresults.results.keys()[0]
self.assertIn('status', server.workflowresults.results[key])
self.assertEqual(server.workflowresults.results[key]['status'], 'completed')
self.assertIn('name', server.workflowresults.results[key])
self.assertEqual(server.workflowresults.results[key]['name'], multiaction_key)
self.assertIn('completed_at', server.workflowresults.results[key])
self.assertIn('started_at', server.workflowresults.results[key])
self.assertIn('results', server.workflowresults.results[key])
self.assertEqual(len(server.workflowresults.results[key]['results']), 2)
self.assertEqual(len(server.workflowresults.results[key]['results']), 2)
def _from_xml(self,
xml_element,
options_name='Default',
workflow_name='',
filename=''):
ExecutionElement.__init__(self,
name=options_name,
parent_name=workflow_name)
self.scheduler = {
'autorun': xml_element.find('.//scheduler').get('autorun'),
'type': xml_element.find('.//scheduler').get('type'),
'args': {
option.tag: option.text
for option in xml_element.findall('.//scheduler/*')
}
}
self.enabled = bool(xml_element.find('.//enabled').text)
self.children = {
construct_workflow_name_key(filename, child.text): None
for child in xml_element.findall('.//children/child')
}
def test_execute_workflow(self):
sync = Event()
workflow_name = helpers.construct_workflow_name_key('test', 'helloWorldWorkflow')
setup_subscriptions_for_step(workflow_name, ['start'])
start = datetime.utcnow()
def wait_for_completion(sender, **kwargs):
sync.set()
WorkflowShutdown.connect(wait_for_completion)
self.post_with_status_check('/playbooks/test/workflows/helloWorldWorkflow/execute',
'success',
headers=self.headers)
sync.wait(timeout=10)
steps = executed_steps('defaultController', workflow_name, start, datetime.utcnow())
self.assertEqual(len(steps), 1)
step = steps[0]
ancestry = step['ancestry'].split(',')
self.assertEqual(ancestry[-1], "start")
self.assertEqual(step['data']['result'], "REPEATING: Hello World")
def test_templated_workflow(self):
workflow_name = construct_workflow_name_key('templatedWorkflowTest', 'templatedWorkflow')
step_names = ['start', '1']
setup_subscriptions_for_step(workflow_name, step_names)
self.controller.execute_workflow('templatedWorkflowTest', 'templatedWorkflow')
shutdown_pool()
steps = executed_steps('defaultController', workflow_name, self.start, datetime.utcnow())
self.assertEqual(len(steps), 2, 'Unexpected number of steps executed. '
'Expected {0}, got {1}'.format(2, len(steps)))
names = [step['ancestry'].split(',')[-1] for step in steps]
orderless_list_compare(self, names, step_names)
name_result = {'start': {'result': {'message': 'HELLO WORLD'}, 'status': 'Success'},
'1': {'status': 'Success', 'result': "REPEATING: {'message': 'HELLO WORLD'}"}}
for step in steps:
name = step['ancestry'].split(',')[-1]
self.assertIn(name, name_result)
result = json.loads(step['data'])
self.assertDictEqual(result['result'], name_result[name])
def test_stepExecutionEvents(self):
workflow_name = construct_workflow_name_key('basicWorkflowTest',
'helloWorldWorkflow')
c = Controller(name="testStepExecutionEventsController")
c.load_workflows_from_file(path=config.test_workflows_path +
"basicWorkflowTest.playbook")
subs = {
'testStepExecutionEventsController':
Subscription(
subscriptions={
workflow_name:
Subscription(
subscriptions={
'start':
Subscription(events=[
"Function Execution Success",
"Input Validated", "Conditionals Executed"
])
})
})
}
case_subscription.set_subscriptions({
'testStepExecutionEvents':
case_subscription.CaseSubscriptions(subscriptions=subs)
})
c.execute_workflow('basicWorkflowTest', 'helloWorldWorkflow')
shutdown_pool()
step_execution_events_case = case_database.case_db.session.query(case_database.Case) \
.filter(case_database.Case.name == 'testStepExecutionEvents').first()
step_execution_event_history = step_execution_events_case.events.all()
self.assertEqual(
len(step_execution_event_history), 3,
'Incorrect length of event history. '
'Expected {0}, got {1}'.format(3,
len(step_execution_event_history)))
def test_workflowExecutionEvents(self):
workflow_name = construct_workflow_name_key('multiactionWorkflowTest', 'multiactionWorkflow')
c = controller.Controller(name="testExecutionEventsController")
c.loadWorkflowsFromFile(path=config.testWorkflowsPath + "multiactionWorkflowTest.workflow")
subs = {'testExecutionEventsController':
Subscription(subscriptions=
{workflow_name:
Subscription(events=["InstanceCreated", "StepExecutionSuccess",
"NextStepFound", "WorkflowShutdown"])})}
case_subscription.set_subscriptions(
{'testExecutionEvents': case_subscription.CaseSubscriptions(subscriptions=subs)})
c.executeWorkflow('multiactionWorkflowTest', 'multiactionWorkflow')
execution_events_case = case_database.case_db.session.query(case_database.Case) \
.filter(case_database.Case.name == 'testExecutionEvents').first()
execution_event_history = execution_events_case.events.all()
self.assertEqual(len(execution_event_history), 6,
'Incorrect length of event history. '
'Expected {0}, got {1}'.format(6, len(execution_event_history)))
def test_error_workflow(self):
workflow_name = construct_workflow_name_key('multistepError', 'multiactionErrorWorkflow')
step_names = ['start', '1', 'error']
setup_subscriptions_for_step(workflow_name, step_names)
self.controller.execute_workflow('multistepError', 'multiactionErrorWorkflow')
shutdown_pool()
steps = executed_steps('defaultController', workflow_name, self.start, datetime.utcnow())
self.assertEqual(len(steps), 2)
names = [step['ancestry'].split(',')[-1] for step in steps]
orderless_list_compare(self, names, ['start', 'error'])
name_result = {'start': {"message": "HELLO WORLD"},
'error': "REPEATING: Hello World"}
for step in steps:
name = step['ancestry'].split(',')[-1]
self.assertIn(name, name_result)
result = json.loads(step['data'])
if type(name_result[name]) == dict:
self.assertDictEqual(result['result'], name_result[name])
else:
self.assertEqual(result['result'], name_result[name])
def test_Loop(self):
self.c.loadWorkflowsFromFile(path=config.testWorkflowsPath +
'loopWorkflow.workflow')
workflow_name = construct_workflow_name_key('loopWorkflow',
'loopWorkflow')
step_names = ['start', '1']
setup_subscriptions_for_step(workflow_name, step_names)
self.c.executeWorkflow('loopWorkflow', 'loopWorkflow')
steps = executed_steps('defaultController', workflow_name, self.start,
datetime.utcnow())
names = [step['ancestry'].split(',')[-1] for step in steps]
expected_steps = ['start', 'start', 'start', 'start', '1']
self.assertListEqual(names, expected_steps)
self.assertEqual(len(steps), 5)
input_output = [('start', 1), ('start', 2), ('start', 3), ('start', 4),
('1', 'REPEATING: 5')]
for step_name, output in input_output:
for step in steps:
name = step['ancestry'].split(',')
if name == step_name:
self.assertEqual(step['data']['result'], output)
def test_workflow_with_dataflow(self):
workflow_name = construct_workflow_name_key('dataflowTest', 'dataflowWorkflow')
step_names = ['start', '1', '2']
setup_subscriptions_for_step(workflow_name, step_names)
self.controller.execute_workflow('dataflowTest', 'dataflowWorkflow')
shutdown_pool()
steps = executed_steps('defaultController', workflow_name, self.start, datetime.utcnow())
self.assertEqual(len(steps), 3)
names = [step['ancestry'].split(',')[-1] for step in steps]
orderless_list_compare(self, names, ['start', '1', '2'])
name_result = {'start': 6,
'1': 6,
'2': 15}
for step in steps:
name = step['ancestry'].split(',')[-1]
self.assertIn(name, name_result)
result = json.loads(step['data'])
if type(name_result[name]) == dict:
self.assertDictEqual(result['result'], name_result[name])
else:
self.assertEqual(result['result'], name_result[name])
def convert_ancestry(ancestry):
if len(ancestry) >= 3:
ancestry[1] = construct_workflow_name_key(ancestry[1], ancestry[2])
del ancestry[2]
return ancestry
def __get_record(playbook, workflow):
name = construct_workflow_name_key(playbook, workflow)
return next((record for record in server.workflowresults.results
if record['name'] == name), None)
