class BaseTaskRunner(object):

    def __init__(self, worker_id, server_port, *args, **kwargs):
        self.worker_id = worker_id
        self.json_rpc_client = JSONRPCWebClient('http://localhost:{}'.format(
            server_port))

    def setup(self):
        """
        Override this in subclass
        """
        pass

    @staticmethod
    def deserialize_task(serialized_task):
        return serialized_task

    def get_next_task(self):
        answer = self.json_rpc_client.call('get_next_task', self.worker_id)
        if answer:
            return self.deserialize_task(answer)

    @staticmethod
    def serialize_task_result(task_result):
        return task_result

    def send_task_results(self, task_result):
        self.json_rpc_client.notify(
            'store_results',
            self.worker_id,
            self.serialize_task_result(task_result)
        )

    def process_task(self, task, *args, **kwargs):
        raise NotImplemented

    def run_until_done(self):
        log.info("Worker waits for payload: %s" % self.worker_id)

        self.setup()

        task = self.get_next_task()
        while task:
            log.info("Worker received payload: %s #%s" %
                     (self.worker_id, task))
            try:
                task_result = self.process_task(task)
                self.send_task_results(task_result)
            except Exception as ex:
                log.info("Worker caught exception: %s, %s" %
                         (self.worker_id, ex))
            except:
                log.info("Worker caught undefined exception")

            task = self.get_next_task()

        log.info("Worker ending its run: %s" % (self.worker_id))
class BaseTaskRunner(object):

    def __init__(self, worker_id, server_port, *args, **kwargs):
        self.worker_id = worker_id
        self.json_rpc_client = JSONRPCWebClient('http://localhost:{}'.format(server_port))

    def setup(self):
        """
        Override this in subclass
        """
        pass

    @staticmethod
    def deserialize_task(serialized_task):
        return serialized_task

    def get_next_task(self):
        answer = self.json_rpc_client.call('get_next_task', self.worker_id)
        if answer:
            return self.deserialize_task(answer)

    @staticmethod
    def serialize_task_result(task_result):
        return task_result

    def send_task_results(self, task_result):
        self.json_rpc_client.notify(
            'store_results',
            self.worker_id,
            self.serialize_task_result(task_result)
        )

    def process_task(self, task, *args, **kwargs):
        raise NotImplemented

    def run_until_done(self):
        log.info("Worker waits for payload: %s" % self.worker_id)

        self.setup()

        task = self.get_next_task()
        while task:
            log.info("Worker received payload: %s #%s" % (self.worker_id, task))
            try:
                task_result = self.process_task(task)
                self.send_task_results(task_result)
            except Exception as ex:
                log.info("Worker caught exception: %s, %s" % (self.worker_id, ex))
            except:
                log.info("Worker caught undefined exception")

            task = self.get_next_task()

        log.info("Worker ending its run: %s" % (self.worker_id))
Пример #3
0
class JSONPRCWebClientTestSuite(TestCase):
    def setUp(self):
        super(JSONPRCWebClientTestSuite, self).setUp()

        server_app = JSONPRCWSGIApplication()
        server_app['echo'] = lambda a: a

        self.url = 'http://example.com/rpc'
        self.cl = WebClient(self.url)

    def test_client_call(self):

        cl = self.cl

        # mocking out response so we don't need a server
        cl._communicate = lambda request_json, expect_response: {
            'id': request_json.get('id'),
            'result': 'mocked response value'
        }

        assert 'mocked response value' == cl.call('method_name')

    def test_client_notify(self):

        cl = self.cl

        # mocking out response so we don't need a server
        cl._communicate = lambda request_json, expect_response: None

        # notifying is supposed to be ignorant of the answer.
        # Thus, notify returns with None as soon as the message is sent
        assert not cl.notify('method_name', 'a', 'b')

    def test_requests_is_called_correctly_for_notification(self):

        with mock.patch('requests.post',
                        return_value=ResponseMock(200)) as mocked_post:
            self.cl.notify('method_name', 'a', 'b')
            mocked_post.assert_called_once_with(
                self.url,
                data=json.dumps({
                    'method': 'method_name',
                    "jsonrpc": "2.0",
                    'params': ['a', 'b']
                }),
                headers={'Content-Type': 'application/json'},
            )

    def test_requests_is_called_correctly_for_call(self):

        content_type = 'application/json'
        body = '{"result":"result"}'

        with mock.patch('requests.post',
                        return_value=ResponseMock(
                            200, body, content_type)) as mocked_post:
            result = self.cl.call('method_name', 'a', 'b')

            self.assertEqual(mocked_post.call_count, 1)
            args, kw = mocked_post.call_args

            self.assertEqual(args[0], self.url)

            self.assertEqual(kw['headers'],
                             {'Content-Type': 'application/json'})

            data = json.loads(kw['data'])
            for key, value in {
                    'method': 'method_name',
                    "jsonrpc": "2.0",
                    'params': ['a', 'b']
            }.items():
                self.assertEqual(data[key], value)

            self.assertEqual(result, 'result')
Пример #4
0
class JSONPRCWebClientTestSuite(TestCase):

    def setUp(self):
        super(JSONPRCWebClientTestSuite, self).setUp()

        server_app = JSONPRCWSGIApplication()
        server_app['echo'] = lambda a: a

        self.url = 'http://example.com/rpc'
        self.cl = WebClient(self.url)

    def test_client_call(self):

        cl = self.cl

        # mocking out response so we don't need a server
        cl._communicate = lambda request_json, expect_response: {
            'id': request_json.get('id'),
            'result': 'mocked response value'
        }

        assert 'mocked response value' == cl.call('method_name')

    def test_client_notify(self):

        cl = self.cl

        # mocking out response so we don't need a server
        cl._communicate = lambda request_json, expect_response: None

        # notifying is supposed to be ignorant of the answer.
        # Thus, notify returns with None as soon as the message is sent
        assert not cl.notify('method_name', 'a', 'b')

    def test_requests_is_called_correctly_for_notification(self):

        with mock.patch('requests.post', return_value=ResponseMock(200)) as mocked_post:
            self.cl.notify('method_name', 'a', 'b')
            mocked_post.assert_called_once_with(
                self.url,
                data=json.dumps({
                    'method':'method_name',
                    "jsonrpc": "2.0",
                    'params':['a', 'b']
                }),
                headers={'Content-Type': 'application/json'},
            )

    def test_requests_is_called_correctly_for_call(self):

        content_type = 'application/json'
        body = '{"result":"result"}'

        with mock.patch('requests.post', return_value=ResponseMock(200, body, content_type)) as mocked_post:
            result = self.cl.call('method_name', 'a', 'b')

            self.assertEqual(
                mocked_post.call_count,
                1
            )
            args, kw = mocked_post.call_args

            self.assertEqual(
                args[0],
                self.url
            )

            self.assertEqual(
                kw['headers'],
                {'Content-Type': 'application/json'}
            )

            data = json.loads(kw['data'])
            for key, value in {
                'method':'method_name',
                "jsonrpc": "2.0",
                'params':['a', 'b']
            }.items():
                self.assertEqual(
                    data[key],
                    value
                )

            self.assertEqual(
                result,
                'result'
            )