def test_flow_action_registration(self):
step_id = "step_%s" % str(uuid.uuid4())
@Workflow.action(flow_name='test_flow_action_registration', action_name=step_id, on_success='second step',
on_error='error')
def test_action(context):
pass
flow = FlowRegistry.flow('test_flow_action_registration', False)
self.assertTrue(step_id in flow.__action_registry__)
def test_flow_action_registration(self):
step_id = "step_%s" % str(uuid.uuid4())
@Workflow.action(flow_name='test_flow_action_registration',
action_name=step_id,
on_success='second step',
on_error='error')
def test_action(context):
pass
flow = FlowRegistry.flow('test_flow_action_registration', False)
self.assertTrue(step_id in flow.__action_registry__)
def test_flow_routing_with_checked_exception(self):
flow_id = 'test_flow_routing_with_checked_exception'
first_step_id = 'step_001_%s' % str(uuid.uuid4())
second_step_id = 'step_002_%s' % str(uuid.uuid4())
error_handler_id = 'error_handler_%s' % str(uuid.uuid4())
@Workflow.action(flow_name=flow_id,
action_name=first_step_id,
on_success=second_step_id,
on_error=error_handler_id)
def flow_routing_with_checked_exception_first_action(context):
self.assertFalse(first_step_id in context)
self.assertFalse(second_step_id in context)
self.assertFalse(error_handler_id in context)
context[first_step_id] = True
@Workflow.action(flow_name=flow_id,
action_name=second_step_id,
on_success='end',
on_error=error_handler_id)
def flow_routing_with_checked_exception_second_action(context):
self.assertTrue(first_step_id in context)
self.assertFalse(second_step_id in context)
self.assertFalse(error_handler_id in context)
context[second_step_id] = True
raise Exception("excpected")
@Workflow.action(flow_name=flow_id,
action_name=error_handler_id,
on_success='end',
on_error='end')
def flow_routing_with_checked_exception_error_handler(context):
self.assertTrue(first_step_id in context)
self.assertTrue(second_step_id in context)
self.assertFalse(error_handler_id in context)
context[error_handler_id] = True
flow = FlowRegistry.flow(flow_id, False)
self.assertTrue(first_step_id in flow.__action_registry__)
self.assertTrue(second_step_id in flow.__action_registry__)
self.assertTrue(error_handler_id in flow.__action_registry__)
_context = flow.run(first_step_id)
self.assertTrue(first_step_id in _context)
self.assertTrue(second_step_id in _context)
self.assertTrue(error_handler_id in _context)
def test_flow_routing_with_checked_exception(self):
flow_id = 'test_flow_routing_with_checked_exception'
first_step_id = 'step_001_%s' % str(uuid.uuid4())
second_step_id = 'step_002_%s' % str(uuid.uuid4())
error_handler_id = 'error_handler_%s' % str(uuid.uuid4())
@Workflow.action(flow_name=flow_id,
action_name=first_step_id,
on_success=second_step_id,
on_error=error_handler_id)
def flow_routing_with_checked_exception_first_action(context):
self.assertFalse(first_step_id in context)
self.assertFalse(second_step_id in context)
self.assertFalse(error_handler_id in context)
context[first_step_id] = True
@Workflow.action(flow_name=flow_id,
action_name=second_step_id,
on_success='end',
on_error=error_handler_id)
def flow_routing_with_checked_exception_second_action(context):
self.assertTrue(first_step_id in context)
self.assertFalse(second_step_id in context)
self.assertFalse(error_handler_id in context)
context[second_step_id] = True
raise Exception("excpected")
@Workflow.action(flow_name=flow_id,
action_name=error_handler_id,
on_success='end',
on_error='end')
def flow_routing_with_checked_exception_error_handler(context):
self.assertTrue(first_step_id in context)
self.assertTrue(second_step_id in context)
self.assertFalse(error_handler_id in context)
context[error_handler_id] = True
flow = FlowRegistry.flow(flow_id, False)
self.assertTrue(first_step_id in flow.__action_registry__)
self.assertTrue(second_step_id in flow.__action_registry__)
self.assertTrue(error_handler_id in flow.__action_registry__)
_context = flow.run(first_step_id)
self.assertTrue(first_step_id in _context)
self.assertTrue(second_step_id in _context)
self.assertTrue(error_handler_id in _context)
def test_flow_routing_with_unchecked_exception(self):
flow_id = 'test_flow_routing_with_unchecked_exception'
first_step_id = 'step_001_%s' % str(uuid.uuid4())
second_step_id = 'step_002_%s' % str(uuid.uuid4())
error_handler_id = 'error_handler_%s' % str(uuid.uuid4())
@Workflow.action(flow_name=flow_id,
action_name=first_step_id,
on_success=second_step_id,
on_error=error_handler_id)
def first_action(context):
self.assertFalse(first_step_id in context)
self.assertFalse(second_step_id in context)
self.assertFalse(error_handler_id in context)
context[first_step_id] = True
raise FatalWorkflowError("this flow should fail")
@Workflow.action(flow_name=flow_id,
action_name=second_step_id,
on_success='end',
on_error=error_handler_id)
def second_action(context):
self.fail("this action should never be called")
@Workflow.action(flow_name=flow_id,
action_name=error_handler_id,
on_success='end',
on_error='end')
def error_handler(context):
self.fail(
"error handler should not be called for FatalWorkflowError")
flow = FlowRegistry.flow(flow_id, False)
self.assertTrue(first_step_id in flow.__action_registry__)
self.assertTrue(second_step_id in flow.__action_registry__)
self.assertTrue(error_handler_id in flow.__action_registry__)
_context = {}
self.assertRaises(FatalWorkflowError,
flow.run,
first_step_id,
context=_context)
self.assertTrue(first_step_id in _context)
self.assertFalse(second_step_id in _context)
self.assertFalse(error_handler_id in _context)
def test_event_dispatcher(self):
flow_id = 'test_event_dispatcher'
first_step_id = 'step_001_%s' % str(uuid.uuid4())
second_step_id = 'step_002_%s' % str(uuid.uuid4())
error_handler_id = 'error_handler_%s' % str(uuid.uuid4())
@Workflow.action(flow_name=flow_id,
action_name=first_step_id,
on_success=second_step_id,
on_error=error_handler_id)
def flow_routing_first_action(context):
pass
flow = FlowRegistry.flow(flow_id, False)
self.assertTrue(first_step_id in flow.__action_registry__)
listener = mock.MagicMock()
_context = flow.run(first_step_id, listeners=[listener])
listener.on_begin.assert_called_with(first_step_id)
listener.on_complete.assert_called_with(first_step_id)
def test_flow_routing_with_unchecked_exception(self):
flow_id = 'test_flow_routing_with_unchecked_exception'
first_step_id = 'step_001_%s' % str(uuid.uuid4())
second_step_id = 'step_002_%s' % str(uuid.uuid4())
error_handler_id = 'error_handler_%s' % str(uuid.uuid4())
@Workflow.action(flow_name=flow_id,
action_name=first_step_id,
on_success=second_step_id,
on_error=error_handler_id)
def first_action(context):
self.assertFalse(first_step_id in context)
self.assertFalse(second_step_id in context)
self.assertFalse(error_handler_id in context)
context[first_step_id] = True
raise FatalWorkflowError("this flow should fail")
@Workflow.action(flow_name=flow_id,
action_name=second_step_id,
on_success='end',
on_error=error_handler_id)
def second_action(context):
self.fail("this action should never be called")
@Workflow.action(flow_name=flow_id,
action_name=error_handler_id,
on_success='end',
on_error='end')
def error_handler(context):
self.fail("error handler should not be called for FatalWorkflowError")
flow = FlowRegistry.flow(flow_id, False)
self.assertTrue(first_step_id in flow.__action_registry__)
self.assertTrue(second_step_id in flow.__action_registry__)
self.assertTrue(error_handler_id in flow.__action_registry__)
_context = {}
self.assertRaises(FatalWorkflowError, flow.run, first_step_id, context=_context)
self.assertTrue(first_step_id in _context)
self.assertFalse(second_step_id in _context)
self.assertFalse(error_handler_id in _context)
def test_event_dispatcher(self):
flow_id = 'test_event_dispatcher'
first_step_id = 'step_001_%s' % str(uuid.uuid4())
second_step_id = 'step_002_%s' % str(uuid.uuid4())
error_handler_id = 'error_handler_%s' % str(uuid.uuid4())
@Workflow.action(flow_name=flow_id,
action_name=first_step_id,
on_success=second_step_id,
on_error=error_handler_id)
def flow_routing_first_action(context):
pass
flow = FlowRegistry.flow(flow_id, False)
self.assertTrue(first_step_id in flow.__action_registry__)
listener = mock.MagicMock()
_context = flow.run(first_step_id, listeners=[listener])
listener.on_begin.assert_called_with(first_step_id)
listener.on_complete.assert_called_with(first_step_id)
def test_flow_routing(self):
flow_id = 'test_flow_routing'
first_step_id = 'step_001_%s' % str(uuid.uuid4())
second_step_id = 'step_002_%s' % str(uuid.uuid4())
@Workflow.action(flow_name=flow_id,
action_name=first_step_id,
on_success=second_step_id,
on_error='error')
def flow_routing_first_action(context):
self.assertFalse(first_step_id in context)
self.assertFalse(second_step_id in context)
context[first_step_id] = True
@Workflow.action(flow_name=flow_id,
action_name=second_step_id,
on_success='end',
on_error='error')
def flow_routing_second_action(context):
self.assertTrue(first_step_id in context)
self.assertFalse(second_step_id in context)
context[second_step_id] = True
@Workflow.action(flow_name=flow_id,
action_name='error',
on_success='end',
on_error='end')
def flow_routing_error_handler(context):
raise FatalWorkflowError()
flow = FlowRegistry.flow(flow_id, False)
self.assertTrue(first_step_id in flow.__action_registry__)
self.assertTrue(second_step_id in flow.__action_registry__)
self.assertTrue('error' in flow.__action_registry__)
_context = flow.run(first_step_id)
self.assertTrue(first_step_id in _context)
self.assertTrue(second_step_id in _context)
def test_flow_routing(self):
flow_id = 'test_flow_routing'
first_step_id = 'step_001_%s' % str(uuid.uuid4())
second_step_id = 'step_002_%s' % str(uuid.uuid4())
@Workflow.action(flow_name=flow_id,
action_name=first_step_id,
on_success=second_step_id,
on_error='error')
def flow_routing_first_action(context):
self.assertFalse(first_step_id in context)
self.assertFalse(second_step_id in context)
context[first_step_id] = True
@Workflow.action(flow_name=flow_id,
action_name=second_step_id,
on_success='end',
on_error='error')
def flow_routing_second_action(context):
self.assertTrue(first_step_id in context)
self.assertFalse(second_step_id in context)
context[second_step_id] = True
@Workflow.action(flow_name=flow_id,
action_name='error',
on_success='end',
on_error='end')
def flow_routing_error_handler(context):
raise FatalWorkflowError()
flow = FlowRegistry.flow(flow_id, False)
self.assertTrue(first_step_id in flow.__action_registry__)
self.assertTrue(second_step_id in flow.__action_registry__)
self.assertTrue('error' in flow.__action_registry__)
_context = flow.run(first_step_id)
self.assertTrue(first_step_id in _context)
self.assertTrue(second_step_id in _context)
def test_error_notifier(self):
flow_id = 'test_event_dispatcher'
first_step_id = 'step_001_%s' % str(uuid.uuid4())
second_step_id = 'step_002_%s' % str(uuid.uuid4())
error_handler_id = 'error_handler_%s' % str(uuid.uuid4())
exception = FatalWorkflowError()
@Workflow.action(flow_name=flow_id,
action_name=first_step_id,
on_success=second_step_id,
on_error=error_handler_id)
def flow_routing_first_action(context):
raise exception
flow = FlowRegistry.flow(flow_id, False)
self.assertTrue(first_step_id in flow.__action_registry__)
listener = mock.MagicMock()
try:
_context = flow.run(first_step_id, listeners=[listener])
except FatalWorkflowError as ex:
pass
listener.on_begin.assert_called_with(first_step_id)
listener.on_error.assert_called_with(first_step_id, exception)
def test_error_notifier(self):
flow_id = 'test_event_dispatcher'
first_step_id = 'step_001_%s' % str(uuid.uuid4())
second_step_id = 'step_002_%s' % str(uuid.uuid4())
error_handler_id = 'error_handler_%s' % str(uuid.uuid4())
exception = FatalWorkflowError()
@Workflow.action(flow_name=flow_id,
action_name=first_step_id,
on_success=second_step_id,
on_error=error_handler_id)
def flow_routing_first_action(context):
raise exception
flow = FlowRegistry.flow(flow_id, False)
self.assertTrue(first_step_id in flow.__action_registry__)
listener = mock.MagicMock()
try:
_context = flow.run(first_step_id, listeners=[listener])
except FatalWorkflowError as ex:
pass
listener.on_begin.assert_called_with(first_step_id)
listener.on_error.assert_called_with(first_step_id, exception)
self.metrics = {}
def on_begin(self, action_name):
if 'error' not in action_name:
if os.path.isfile('resources/step'):
os.remove('resources/step')
def on_error(self, action_name, exception):
if action_name == "load_file_from_local_to_hdfs":
file = open('resources/step', 'w')
file.write(action_name)
file.close()
if __name__ == '__main__':
step = 'Load file descriptor for files on FTP'
# Checks if file with last failed step is exists
# and reads this step
if os.path.isfile('resources/step'):
file = open('resources/step', 'r')
step = file.read()
file.close()
flow = FlowRegistry.flow('Flow')
# Runs flow
_context = flow.run(
action=step,
listeners=[LoggingListener("Flow"),
WorkflowFailOverController()])
log = get_logger("SCD")
# Prepare paths
_pig_script = os.path.join(os.path.dirname(__file__), 'scd_processing.pig')
_scd_active_snapshot = '/tmp/scd.active/scd.active.csv'
_scd_updates = os.path.join(os.path.dirname(__file__), 'resources', 'scd.update.csv')
_hdfs_job_output = '/tmp/scd.updated'
_local_folder_to_monitor = LocalFS(os.path.join(os.path.dirname(__file__), 'resources'))
_hdfs_basedir = HDFS('/tmp/scd.active')
_hdfs_tmpdir = HDFS('/tmp/scd.tmp')
_hdfs_tmpdir.create_directory()
if _scd_updates and LocalFS(_scd_updates).exists():
# Checks if file with last failed step is exists
# and reads this step
step = 'Copying scd updates to raw area on HDFS'
if os.path.isfile('resources/step'):
file = open('resources/step', 'r')
step = file.read()
file.close()
flow = FlowRegistry.flow('Flow')
# Runs flow
_context = flow.run(action=step,
listeners=[LoggingListener("Flow"), WorkflowFailOverController()])
else:
log.info("Nothing to process")
def test_should_create_new_flow(self):
flow_name = "flow %s" % str(uuid.uuid4())
FlowRegistry.flow(flow_name, True)
self.assertIsNotNone(FlowRegistry.flow(flow_name, False), 'new flow was not created')
def test_should_create_new_flow(self):
flow_name = "flow %s" % str(uuid.uuid4())
FlowRegistry.flow(flow_name, True)
self.assertIsNotNone(FlowRegistry.flow(flow_name, False),
'new flow was not created')
