Exemple #1
0
    def test_flow_action_registration(self):
        step_id = "step_%s" % str(uuid.uuid4())

        @Workflow.action(flow_name='test_flow_action_registration', action_name=step_id, on_success='second step',
                         on_error='error')
        def test_action(context):
            pass

        flow = FlowRegistry.flow('test_flow_action_registration', False)
        self.assertTrue(step_id in flow.__action_registry__)
Exemple #2
0
    def test_flow_action_registration(self):
        step_id = "step_%s" % str(uuid.uuid4())

        @Workflow.action(flow_name='test_flow_action_registration',
                         action_name=step_id,
                         on_success='second step',
                         on_error='error')
        def test_action(context):
            pass

        flow = FlowRegistry.flow('test_flow_action_registration', False)
        self.assertTrue(step_id in flow.__action_registry__)
Exemple #3
0
    def test_flow_routing_with_checked_exception(self):
        flow_id = 'test_flow_routing_with_checked_exception'

        first_step_id = 'step_001_%s' % str(uuid.uuid4())
        second_step_id = 'step_002_%s' % str(uuid.uuid4())
        error_handler_id = 'error_handler_%s' % str(uuid.uuid4())

        @Workflow.action(flow_name=flow_id,
                         action_name=first_step_id,
                         on_success=second_step_id,
                         on_error=error_handler_id)
        def flow_routing_with_checked_exception_first_action(context):
            self.assertFalse(first_step_id in context)
            self.assertFalse(second_step_id in context)
            self.assertFalse(error_handler_id in context)
            context[first_step_id] = True

        @Workflow.action(flow_name=flow_id,
                         action_name=second_step_id,
                         on_success='end',
                         on_error=error_handler_id)
        def flow_routing_with_checked_exception_second_action(context):
            self.assertTrue(first_step_id in context)
            self.assertFalse(second_step_id in context)
            self.assertFalse(error_handler_id in context)
            context[second_step_id] = True
            raise Exception("excpected")

        @Workflow.action(flow_name=flow_id,
                         action_name=error_handler_id,
                         on_success='end',
                         on_error='end')
        def flow_routing_with_checked_exception_error_handler(context):
            self.assertTrue(first_step_id in context)
            self.assertTrue(second_step_id in context)
            self.assertFalse(error_handler_id in context)
            context[error_handler_id] = True

        flow = FlowRegistry.flow(flow_id, False)
        self.assertTrue(first_step_id in flow.__action_registry__)
        self.assertTrue(second_step_id in flow.__action_registry__)
        self.assertTrue(error_handler_id in flow.__action_registry__)
        _context = flow.run(first_step_id)

        self.assertTrue(first_step_id in _context)
        self.assertTrue(second_step_id in _context)
        self.assertTrue(error_handler_id in _context)
Exemple #4
0
    def test_flow_routing_with_checked_exception(self):
        flow_id = 'test_flow_routing_with_checked_exception'

        first_step_id = 'step_001_%s' % str(uuid.uuid4())
        second_step_id = 'step_002_%s' % str(uuid.uuid4())
        error_handler_id = 'error_handler_%s' % str(uuid.uuid4())

        @Workflow.action(flow_name=flow_id,
                         action_name=first_step_id,
                         on_success=second_step_id,
                         on_error=error_handler_id)
        def flow_routing_with_checked_exception_first_action(context):
            self.assertFalse(first_step_id in context)
            self.assertFalse(second_step_id in context)
            self.assertFalse(error_handler_id in context)
            context[first_step_id] = True

        @Workflow.action(flow_name=flow_id,
                         action_name=second_step_id,
                         on_success='end',
                         on_error=error_handler_id)
        def flow_routing_with_checked_exception_second_action(context):
            self.assertTrue(first_step_id in context)
            self.assertFalse(second_step_id in context)
            self.assertFalse(error_handler_id in context)
            context[second_step_id] = True
            raise Exception("excpected")

        @Workflow.action(flow_name=flow_id,
                         action_name=error_handler_id,
                         on_success='end',
                         on_error='end')
        def flow_routing_with_checked_exception_error_handler(context):
            self.assertTrue(first_step_id in context)
            self.assertTrue(second_step_id in context)
            self.assertFalse(error_handler_id in context)
            context[error_handler_id] = True

        flow = FlowRegistry.flow(flow_id, False)
        self.assertTrue(first_step_id in flow.__action_registry__)
        self.assertTrue(second_step_id in flow.__action_registry__)
        self.assertTrue(error_handler_id in flow.__action_registry__)
        _context = flow.run(first_step_id)

        self.assertTrue(first_step_id in _context)
        self.assertTrue(second_step_id in _context)
        self.assertTrue(error_handler_id in _context)
Exemple #5
0
    def test_flow_routing_with_unchecked_exception(self):
        flow_id = 'test_flow_routing_with_unchecked_exception'
        first_step_id = 'step_001_%s' % str(uuid.uuid4())
        second_step_id = 'step_002_%s' % str(uuid.uuid4())
        error_handler_id = 'error_handler_%s' % str(uuid.uuid4())

        @Workflow.action(flow_name=flow_id,
                         action_name=first_step_id,
                         on_success=second_step_id,
                         on_error=error_handler_id)
        def first_action(context):
            self.assertFalse(first_step_id in context)
            self.assertFalse(second_step_id in context)
            self.assertFalse(error_handler_id in context)
            context[first_step_id] = True
            raise FatalWorkflowError("this flow should fail")

        @Workflow.action(flow_name=flow_id,
                         action_name=second_step_id,
                         on_success='end',
                         on_error=error_handler_id)
        def second_action(context):
            self.fail("this action should never be called")

        @Workflow.action(flow_name=flow_id,
                         action_name=error_handler_id,
                         on_success='end',
                         on_error='end')
        def error_handler(context):
            self.fail(
                "error handler should not be called for FatalWorkflowError")

        flow = FlowRegistry.flow(flow_id, False)
        self.assertTrue(first_step_id in flow.__action_registry__)
        self.assertTrue(second_step_id in flow.__action_registry__)
        self.assertTrue(error_handler_id in flow.__action_registry__)

        _context = {}

        self.assertRaises(FatalWorkflowError,
                          flow.run,
                          first_step_id,
                          context=_context)
        self.assertTrue(first_step_id in _context)
        self.assertFalse(second_step_id in _context)
        self.assertFalse(error_handler_id in _context)
Exemple #6
0
    def test_event_dispatcher(self):
        flow_id = 'test_event_dispatcher'
        first_step_id = 'step_001_%s' % str(uuid.uuid4())
        second_step_id = 'step_002_%s' % str(uuid.uuid4())
        error_handler_id = 'error_handler_%s' % str(uuid.uuid4())

        @Workflow.action(flow_name=flow_id,
                         action_name=first_step_id,
                         on_success=second_step_id,
                         on_error=error_handler_id)
        def flow_routing_first_action(context):
            pass

        flow = FlowRegistry.flow(flow_id, False)
        self.assertTrue(first_step_id in flow.__action_registry__)
        listener = mock.MagicMock()
        _context = flow.run(first_step_id, listeners=[listener])
        listener.on_begin.assert_called_with(first_step_id)
        listener.on_complete.assert_called_with(first_step_id)
Exemple #7
0
    def test_flow_routing_with_unchecked_exception(self):
        flow_id = 'test_flow_routing_with_unchecked_exception'
        first_step_id = 'step_001_%s' % str(uuid.uuid4())
        second_step_id = 'step_002_%s' % str(uuid.uuid4())
        error_handler_id = 'error_handler_%s' % str(uuid.uuid4())


        @Workflow.action(flow_name=flow_id,
                         action_name=first_step_id,
                         on_success=second_step_id,
                         on_error=error_handler_id)
        def first_action(context):
            self.assertFalse(first_step_id in context)
            self.assertFalse(second_step_id in context)
            self.assertFalse(error_handler_id in context)
            context[first_step_id] = True
            raise FatalWorkflowError("this flow should fail")

        @Workflow.action(flow_name=flow_id,
                         action_name=second_step_id,
                         on_success='end',
                         on_error=error_handler_id)
        def second_action(context):
            self.fail("this action should never be called")

        @Workflow.action(flow_name=flow_id,
                         action_name=error_handler_id,
                         on_success='end',
                         on_error='end')
        def error_handler(context):
            self.fail("error handler should not be called for FatalWorkflowError")

        flow = FlowRegistry.flow(flow_id, False)
        self.assertTrue(first_step_id in flow.__action_registry__)
        self.assertTrue(second_step_id in flow.__action_registry__)
        self.assertTrue(error_handler_id in flow.__action_registry__)

        _context = {}

        self.assertRaises(FatalWorkflowError, flow.run, first_step_id, context=_context)
        self.assertTrue(first_step_id in _context)
        self.assertFalse(second_step_id in _context)
        self.assertFalse(error_handler_id in _context)
Exemple #8
0
    def test_event_dispatcher(self):
        flow_id = 'test_event_dispatcher'
        first_step_id = 'step_001_%s' % str(uuid.uuid4())
        second_step_id = 'step_002_%s' % str(uuid.uuid4())
        error_handler_id = 'error_handler_%s' % str(uuid.uuid4())

        @Workflow.action(flow_name=flow_id,
                         action_name=first_step_id,
                         on_success=second_step_id,
                         on_error=error_handler_id)
        def flow_routing_first_action(context):
            pass

        flow = FlowRegistry.flow(flow_id, False)
        self.assertTrue(first_step_id in flow.__action_registry__)
        listener = mock.MagicMock()
        _context = flow.run(first_step_id, listeners=[listener])
        listener.on_begin.assert_called_with(first_step_id)
        listener.on_complete.assert_called_with(first_step_id)
Exemple #9
0
    def test_flow_routing(self):
        flow_id = 'test_flow_routing'
        first_step_id = 'step_001_%s' % str(uuid.uuid4())
        second_step_id = 'step_002_%s' % str(uuid.uuid4())

        @Workflow.action(flow_name=flow_id,
                         action_name=first_step_id,
                         on_success=second_step_id,
                         on_error='error')
        def flow_routing_first_action(context):
            self.assertFalse(first_step_id in context)
            self.assertFalse(second_step_id in context)
            context[first_step_id] = True

        @Workflow.action(flow_name=flow_id,
                         action_name=second_step_id,
                         on_success='end',
                         on_error='error')
        def flow_routing_second_action(context):
            self.assertTrue(first_step_id in context)
            self.assertFalse(second_step_id in context)
            context[second_step_id] = True

        @Workflow.action(flow_name=flow_id,
                         action_name='error',
                         on_success='end',
                         on_error='end')
        def flow_routing_error_handler(context):
            raise FatalWorkflowError()

        flow = FlowRegistry.flow(flow_id, False)
        self.assertTrue(first_step_id in flow.__action_registry__)
        self.assertTrue(second_step_id in flow.__action_registry__)
        self.assertTrue('error' in flow.__action_registry__)
        _context = flow.run(first_step_id)

        self.assertTrue(first_step_id in _context)
        self.assertTrue(second_step_id in _context)
Exemple #10
0
    def test_flow_routing(self):
        flow_id = 'test_flow_routing'
        first_step_id = 'step_001_%s' % str(uuid.uuid4())
        second_step_id = 'step_002_%s' % str(uuid.uuid4())

        @Workflow.action(flow_name=flow_id,
                         action_name=first_step_id,
                         on_success=second_step_id,
                         on_error='error')
        def flow_routing_first_action(context):
            self.assertFalse(first_step_id in context)
            self.assertFalse(second_step_id in context)
            context[first_step_id] = True

        @Workflow.action(flow_name=flow_id,
                         action_name=second_step_id,
                         on_success='end',
                         on_error='error')
        def flow_routing_second_action(context):
            self.assertTrue(first_step_id in context)
            self.assertFalse(second_step_id in context)
            context[second_step_id] = True

        @Workflow.action(flow_name=flow_id,
                         action_name='error',
                         on_success='end',
                         on_error='end')
        def flow_routing_error_handler(context):
            raise FatalWorkflowError()

        flow = FlowRegistry.flow(flow_id, False)
        self.assertTrue(first_step_id in flow.__action_registry__)
        self.assertTrue(second_step_id in flow.__action_registry__)
        self.assertTrue('error' in flow.__action_registry__)
        _context = flow.run(first_step_id)

        self.assertTrue(first_step_id in _context)
        self.assertTrue(second_step_id in _context)
Exemple #11
0
    def test_error_notifier(self):
        flow_id = 'test_event_dispatcher'
        first_step_id = 'step_001_%s' % str(uuid.uuid4())
        second_step_id = 'step_002_%s' % str(uuid.uuid4())
        error_handler_id = 'error_handler_%s' % str(uuid.uuid4())

        exception = FatalWorkflowError()

        @Workflow.action(flow_name=flow_id,
                         action_name=first_step_id,
                         on_success=second_step_id,
                         on_error=error_handler_id)
        def flow_routing_first_action(context):
            raise exception

        flow = FlowRegistry.flow(flow_id, False)
        self.assertTrue(first_step_id in flow.__action_registry__)
        listener = mock.MagicMock()
        try:
            _context = flow.run(first_step_id, listeners=[listener])
        except FatalWorkflowError as ex:
            pass
        listener.on_begin.assert_called_with(first_step_id)
        listener.on_error.assert_called_with(first_step_id, exception)
Exemple #12
0
    def test_error_notifier(self):
        flow_id = 'test_event_dispatcher'
        first_step_id = 'step_001_%s' % str(uuid.uuid4())
        second_step_id = 'step_002_%s' % str(uuid.uuid4())
        error_handler_id = 'error_handler_%s' % str(uuid.uuid4())

        exception = FatalWorkflowError()

        @Workflow.action(flow_name=flow_id,
                         action_name=first_step_id,
                         on_success=second_step_id,
                         on_error=error_handler_id)
        def flow_routing_first_action(context):
            raise exception

        flow = FlowRegistry.flow(flow_id, False)
        self.assertTrue(first_step_id in flow.__action_registry__)
        listener = mock.MagicMock()
        try:
            _context = flow.run(first_step_id, listeners=[listener])
        except FatalWorkflowError as ex:
            pass
        listener.on_begin.assert_called_with(first_step_id)
        listener.on_error.assert_called_with(first_step_id, exception)
Exemple #13
0
        self.metrics = {}

    def on_begin(self, action_name):
        if 'error' not in action_name:
            if os.path.isfile('resources/step'):
                os.remove('resources/step')

    def on_error(self, action_name, exception):
        if action_name == "load_file_from_local_to_hdfs":
            file = open('resources/step', 'w')
            file.write(action_name)
            file.close()


if __name__ == '__main__':
    step = 'Load file descriptor for files on FTP'

    # Checks if file with last failed step is exists
    # and reads this step
    if os.path.isfile('resources/step'):
        file = open('resources/step', 'r')
        step = file.read()
        file.close()

    flow = FlowRegistry.flow('Flow')

    # Runs flow
    _context = flow.run(
        action=step,
        listeners=[LoggingListener("Flow"),
                   WorkflowFailOverController()])
Exemple #14
0
    log = get_logger("SCD")

    # Prepare paths
    _pig_script = os.path.join(os.path.dirname(__file__), 'scd_processing.pig')
    _scd_active_snapshot = '/tmp/scd.active/scd.active.csv'
    _scd_updates = os.path.join(os.path.dirname(__file__), 'resources', 'scd.update.csv')
    _hdfs_job_output = '/tmp/scd.updated'

    _local_folder_to_monitor = LocalFS(os.path.join(os.path.dirname(__file__), 'resources'))
    _hdfs_basedir = HDFS('/tmp/scd.active')
    _hdfs_tmpdir = HDFS('/tmp/scd.tmp')
    _hdfs_tmpdir.create_directory()

    if _scd_updates and LocalFS(_scd_updates).exists():

        # Checks if file with last failed step is exists
        # and reads this step
        step = 'Copying scd updates to raw area on HDFS'
        if os.path.isfile('resources/step'):
            file = open('resources/step', 'r')
            step = file.read()
            file.close()

        flow = FlowRegistry.flow('Flow')

        # Runs flow
        _context = flow.run(action=step,
                            listeners=[LoggingListener("Flow"), WorkflowFailOverController()])
    else:
        log.info("Nothing to process")
Exemple #15
0
 def test_should_create_new_flow(self):
     flow_name = "flow %s" % str(uuid.uuid4())
     FlowRegistry.flow(flow_name, True)
     self.assertIsNotNone(FlowRegistry.flow(flow_name, False), 'new flow was not created')
Exemple #16
0
 def test_should_create_new_flow(self):
     flow_name = "flow %s" % str(uuid.uuid4())
     FlowRegistry.flow(flow_name, True)
     self.assertIsNotNone(FlowRegistry.flow(flow_name, False),
                          'new flow was not created')