Exemple #1
0
def test_runner_read_start_queue_minion_already_running_success():
    config = {'juicer': {'servers': {'redis_url': "nonexisting.mock"}}}
    app_id = 1
    workflow_id = 1000
    workflow = {
        'workflow_id': workflow_id,
        'app_id': app_id,
        'type': 'execute',
        'workflow': {}
    }

    with mock.patch('redis.StrictRedis',
                    mock_strict_redis_client) as mocked_redis:
        with mock.patch('subprocess.Popen') as mocked_popen:
            server = JuicerServer(config, 'faked_minions.py')
            mocked_redis_conn = mocked_redis()
            state_control = StateControlRedis(mocked_redis_conn)

            # Publishes a message to process data
            state_control.push_start_queue(json.dumps(workflow))
            state_control.set_minion_status(app_id, JuicerServer.STARTED)

            # Start of testing
            server.read_job_start_queue(mocked_redis_conn)

            assert state_control.get_minion_status(
                app_id) == JuicerServer.STARTED

            assert not mocked_popen.called
            # Was command removed from the queue?
            assert mocked_redis_conn.lpop('start') is None
            assert json.loads(state_control.pop_app_queue(app_id)) == workflow

            assert state_control.get_workflow_status(
                workflow_id) == JuicerServer.STARTED
            assert json.loads(state_control.pop_app_output_queue(app_id)) == {
                'code': 0,
                'message': 'Minion is processing message execute'
            }
Exemple #2
0
def test_runner_read_start_queue_missing_details_failure():
    config = {'juicer': {'servers': {'redis_url': "nonexisting.mock"}}}
    app_id = 1
    workflow_id = 1000
    # incorrect key, should raise exception
    workflow = {
        'workflow_id': workflow_id,
        'xapp_id': app_id,
        'type': 'execute',
        'workflow': {}
    }

    with mock.patch('redis.StrictRedis',
                    mock_strict_redis_client) as mocked_redis:
        with mock.patch('subprocess.Popen') as mocked_popen:
            server = JuicerServer(config, 'faked_minions.py')
            mocked_redis_conn = mocked_redis()
            # Publishes a message to process data
            state_control = StateControlRedis(mocked_redis_conn)

            # Publishes a message to process data
            state_control.push_start_queue(json.dumps(workflow))
            state_control.set_minion_status(app_id, JuicerServer.STARTED)

            # Start of testing
            server.read_job_start_queue(mocked_redis_conn)

            assert state_control.get_minion_status(
                app_id) == JuicerServer.STARTED

            assert not mocked_popen.called
            # Was command removed from the queue?
            assert state_control.pop_job_start_queue(block=False) is None
            assert state_control.pop_app_queue(app_id, block=False) is None
            assert state_control.pop_app_output_queue(app_id,
                                                      block=False) is None

            assert mocked_redis_conn.hget(workflow_id, 'status') is None
Exemple #3
0
def test_runner_master_queue_client_shutdown_success():
    config = {'juicer': {'servers': {'redis_url': "nonexisting.mock"}}}
    app_id = 1
    ticket = {
        "app_id": app_id,
        'reason': JuicerServer.HELP_UNHANDLED_EXCEPTION
    }

    with mock.patch('redis.StrictRedis',
                    mock_strict_redis_client) as mocked_redis:
        with mock.patch('subprocess.Popen') as mocked_popen:
            with mock.patch('os.kill') as mocked_kill:
                server = JuicerServer(config, 'faked_minions.py')
                mocked_redis_conn = mocked_redis()
                # Publishes a message to process data
                state_control = StateControlRedis(mocked_redis_conn)

                # Configure minion
                status = {'app_id': app_id, 'pid': 9999}
                state_control.set_minion_status(app_id, json.dumps(status))
                error = OSError()
                error.errno = errno.ESRCH
                mocked_kill.side_effect = error
                # Publishes a message to master queue
                state_control.push_master_queue(json.dumps(ticket))

                # Start of testing
                server.read_minion_support_queue(mocked_redis_conn)

                d1 = json.loads(state_control.get_minion_status(app_id))
                d2 = {"pid": 1, "port": 36000}
                assert d1 == d2

                assert mocked_popen.called

                mocked_kill.assert_called_once_with(status['pid'],
                                                    signal.SIGKILL)
Exemple #4
0
class Minion:
    MSG_PROCESSED = 'message_processed'

    def __init__(self, redis_conn, workflow_id, app_id, config):
        self.redis_conn = redis_conn
        self.state_control = StateControlRedis(self.redis_conn)
        self.workflow_id = workflow_id
        self.app_id = app_id
        self.config = config

        # Errors and messages
        self.MNN000 = ('MNN000', _('Success.'))
        self.MNN001 = ('MNN001', _('Port output format not supported.'))
        self.MNN002 = ('MNN002', _('Success getting data from task.'))
        self.MNN003 = ('MNN003', _('State does not exists, processing app.'))
        self.MNN004 = ('MNN004', _('Invalid port.'))
        self.MNN005 = ('MNN005',
                       _('Unable to retrieve data because a previous error.'))
        self.MNN006 = ('MNN006',
                       _('Invalid Python code or incorrect encoding: {}'))
        self.MNN007 = ('MNN007', _('Job {} was canceled'))
        self.MNN008 = ('MNN008', _('App {} was terminated'))
        self.MNN009 = ('MNN009', _('Workflow specification is missing'))
        self.MNN010 = (
            'MNN010',
            _('Task completed, but not executed (not used in the workflow).'))

        # Used in the template file, declared here to gettext detect them
        self.msgs = [
            _('Task running'),
            _('Task completed'),
            _('Task running (cached data)')
        ]

    def process(self):
        raise NotImplementedError()

    def _generate_output(self, message, status=None, code=None):
        """
        Sends feedback about execution of this minion.
        """
        obj = {
            'message': message,
            'workflow_id': self.workflow_id,
            'app_id': self.app_id,
            'code': code,
            'date': datetime.datetime.now().isoformat(),
            'status': status if status is not None else 'OK'
        }

        m = json.dumps(obj)
        self.state_control.push_app_output_queue(self.app_id, m)

    def _perform_ping(self):
        status = {
            'status': 'READY',
            'pid': os.getpid(),
        }
        self.state_control.set_minion_status(self.app_id,
                                             json.dumps(status),
                                             ex=10,
                                             nx=False)

    @staticmethod
    def reload_code(q):
        wm = pyinotify.WatchManager()
        notifier = pyinotify.Notifier(wm, EventHandler())
        wm.add_watch(_watch_dir, pyinotify.ALL_EVENTS, rec=True)
        notifier.loop()

    def ping(self, q):
        """ Pings redis to inform master this minion is online """
        log.info('Start ping')
        while q.empty():
            self._perform_ping()
            time.sleep(5)