コード例 #1
0
    def test_message(self):
        data = {'1': 2}
        msg = Message(**data)
        self.assertEquals(msg.serialize(), json.dumps(data))

        msg = Message.load_from_string(json.dumps(data))
        self.assertEquals(msg.serialize(), json.dumps(data))
コード例 #2
0
ファイル: test_message.py プロジェクト: EnTeQuAk/loads
    def test_message(self):
        data = {'1': 2}
        msg = Message(**data)
        self.assertEquals(msg.serialize(), json.dumps(data))

        msg = Message.load_from_string(json.dumps(data))
        self.assertEquals(msg.serialize(), json.dumps(data))
コード例 #3
0
ファイル: brokerctrl.py プロジェクト: jgsbennett/loads
    def stop_run(self, msg, data):
        run_id = data['run_id']
        agents = []

        for agent_id, (_run_id, when) in self._runs.items():
            if run_id != _run_id:
                continue
            agents.append(agent_id)

        if len(agents) == 0:
            # we don't have any agents running that test, let's
            # force the flags in the DB
            self.update_metadata(run_id,
                                 stopped=True,
                                 active=False,
                                 ended=time.time())
            return []

        # now we have a list of agents to stop
        stop_msg = json.dumps({'command': 'STOP'})

        for agent_id in agents:
            self.send_to_agent(agent_id, stop_msg)

        return agents
コード例 #4
0
    def _handle_recv_back(self, msg):
        # do the message and send the result
        if self.debug:
            #logger.debug('Message received from the broker')
            target = timed()(self._handle_commands)
        else:
            target = self._handle_commands

        duration = -1

        try:
            res = target(Message.load_from_string(msg[0]))
            if self.debug:
                duration, res = res

            res = json.dumps(res)
            # we're working with strings
            if isinstance(res, unicode):
                res = res.encode('utf8')

        except Exception, e:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            exc = traceback.format_tb(exc_traceback)
            exc.insert(0, str(e))
            res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
            logger.error(res)
コード例 #5
0
    def _handle_recv_back(self, msg):
        # do the message and send the result
        if self.debug:
            target = timed()(self._handle_commands)
        else:
            target = self._handle_commands

        duration = -1
        broker_id = msg[2]

        if len(msg) == 7:
            client_id = msg[4]
        else:
            client_id = None

        data = msg[-1]
        try:
            res = target(Message.load_from_string(data))
            if self.debug:
                duration, res = res

            res['hostname'] = get_hostname()
            res = json.dumps(res)
            # we're working with strings
            if isinstance(res, unicode):
                res = res.encode('utf8')

        except Exception, e:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            exc = traceback.format_tb(exc_traceback)
            exc.insert(0, str(e))
            res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
            logger.error(res)
コード例 #6
0
    def _handle_recv_back(self, msg):
        # let's remove the agent id and track the time it took
        agent_id = msg[0]
        msg = msg[1:]

        # grabbing the data to update the agents statuses if needed
        data = json.loads(msg[-1])
        if 'error' in data:
            result = data['error']
        else:
            result = data['result']

        if result.get('command') in ('_STATUS', 'STOP', 'QUIT'):
            statuses = result['status'].values()
            run_id = self.ctrl.update_status(agent_id, statuses)
            if run_id is not None:
                # if the tests are finished, publish this on the pubsub.
                self._publisher.send(
                    json.dumps({
                        'data_type': 'run-finished',
                        'run_id': run_id
                    }))
            return

        # other things are pass-through
        try:
            self._frontstream.send_multipart(msg)
        except Exception, e:
            logger.error('Could not send to front')
            logger.error(msg)
            # we don't want to die on error. we just log it
            exc_type, exc_value, exc_traceback = sys.exc_info()
            exc = traceback.format_tb(exc_traceback)
            exc.insert(0, str(e))
            logger.error('\n'.join(exc))
コード例 #7
0
ファイル: agent.py プロジェクト: EnTeQuAk/loads
    def _handle_recv_back(self, msg):
        # do the message and send the result
        if self.debug:
            #logger.debug('Message received from the broker')
            target = timed()(self._handle_commands)
        else:
            target = self._handle_commands

        duration = -1

        try:
            res = target(Message.load_from_string(msg[0]))
            if self.debug:
                duration, res = res

            res = json.dumps(res)
            # we're working with strings
            if isinstance(res, unicode):
                res = res.encode('utf8')

        except Exception, e:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            exc = traceback.format_tb(exc_traceback)
            exc.insert(0, str(e))
            res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
            logger.error(res)
コード例 #8
0
ファイル: agent.py プロジェクト: jgsbennett/loads
    def _handle_recv_back(self, msg):
        # do the message and send the result
        if self.debug:
            target = timed()(self._handle_commands)
        else:
            target = self._handle_commands

        duration = -1
        broker_id = msg[2]

        if len(msg) == 7:
            client_id = msg[4]
        else:
            client_id = None

        data = msg[-1]
        try:
            res = target(Message.load_from_string(data))
            if self.debug:
                duration, res = res

            res['hostname'] = get_hostname()
            res = json.dumps(res)
            # we're working with strings
            if isinstance(res, unicode):
                res = res.encode('utf8')

        except Exception, e:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            exc = traceback.format_tb(exc_traceback)
            exc.insert(0, str(e))
            res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
            logger.error(res)
コード例 #9
0
ファイル: brokerctrl.py プロジェクト: EnTeQuAk/loads
    def run_command(self, cmd, msg, data):
        cmd = cmd.lower()
        target = msg[:-1]

        # command for agents
        if cmd.startswith('agent_'):
            data['command'] = cmd[len('agent_'):].upper()
            msg = msg[:2] + [json.dumps(data)] + msg[2:]
            self.send_to_agent(str(data['agent_id']), msg)
            return    # returning None because it's async

        if not hasattr(self, cmd):
            raise AttributeError(cmd)

        # calling the command asynchronously
        def _call():
            try:
                res = getattr(self, cmd)(msg, data)
                res = {'result': res}
                self.broker.send_json(target, res)
            except Exception, e:
                logger.debug('Failed')
                exc_type, exc_value, exc_traceback = sys.exc_info()
                exc = traceback.format_tb(exc_traceback)
                exc.insert(0, str(e))
                self.broker.send_json(target, {'error': exc})
コード例 #10
0
ファイル: broker.py プロジェクト: EnTeQuAk/loads
    def _handle_recv_back(self, msg):
        # let's remove the agent id and track the time it took
        agent_id = msg[0]
        msg = msg[1:]

        # grabbing the data to update the agents statuses if needed
        data = json.loads(msg[-1])
        if 'error' in data:
            result = data['error']
        else:
            result = data['result']

        if result.get('command') in ('_STATUS', 'STOP', 'QUIT'):
            statuses = result['status'].values()
            run_id = self.ctrl.update_status(agent_id, statuses)
            if run_id is not None:
                # if the tests are finished, publish this on the pubsub.
                self._publisher.send(json.dumps({'data_type': 'run-finished',
                                                 'run_id': run_id}))
            return

        # other things are pass-through
        try:
            self._frontstream.send_multipart(msg)
        except Exception, e:
            logger.error('Could not send to front')
            logger.error(msg)
            # we don't want to die on error. we just log it
            exc_type, exc_value, exc_traceback = sys.exc_info()
            exc = traceback.format_tb(exc_traceback)
            exc.insert(0, str(e))
            logger.error('\n'.join(exc))
コード例 #11
0
 def send_json(self, target, data):
     assert isinstance(target, basestring), target
     msg = [target, '', json.dumps(data)]
     try:
         self._frontstream.send_multipart(msg)
     except ValueError:
         logger.error('Could not dump %s' % str(data))
         raise
コード例 #12
0
 def test_messages_are_relayed(self):
     runner = ExternalRunner()
     runner._test_result = mock.MagicMock()
     data = json.dumps({'data_type': 'foo', 'bar': 'barbaz', 'run_id': 1})
     runner._process_result([
         data,
     ])
     runner.test_result.foo.assertCalledWith(bar='barbaz')
コード例 #13
0
ファイル: broker.py プロジェクト: jgsbennett/loads
 def send_json(self, target, data):
     assert isinstance(target, basestring), target
     msg = [target, "", json.dumps(data)]
     try:
         self._frontstream.send_multipart(msg)
     except ValueError:
         logger.error("Could not dump %s" % str(data))
         raise
コード例 #14
0
ファイル: brokerctrl.py プロジェクト: jgsbennett/loads
    def clean(self):
        """This is called periodically to :

        - send a _STATUS command to all active agents to refresh their status
        - detect agents that have not responded for a while and discard them
          from the run and from the agents list
        """
        now = time.time()

        for agent_id, (run_id, when) in self._runs.items():
            # when was the last time we've got a response ?
            last_contact = self._agent_times.get(agent_id)

            # is the agent not responding since a while ?
            if (last_contact is not None
                    and now - last_contact > self.agent_timeout):
                # let's kill the agent...
                lag = now - last_contact
                logger.debug('No response from agent since %d s.' % lag)
                logger.debug('Killing agent %s' % str(agent_id))
                quit = json.dumps({'command': 'QUIT'})
                self.send_to_agent(agent_id, quit)

                # and remove it from the run
                run_id = self._terminate_run(agent_id)

                if run_id is not None:
                    logger.debug('publishing end of run')
                    # if the tests are finished, publish this on the pubsub.
                    msg = json.dumps({
                        'data_type': 'run-finished',
                        'run_id': run_id
                    })
                    self.broker._publisher.send(msg)
            else:
                # initialize the timer
                if last_contact is None:
                    self._agent_times[agent_id] = now

                # sending a _STATUS call to on each active agent
                status_msg = json.dumps({
                    'command': '_STATUS',
                    'run_id': run_id
                })
                self.send_to_agent(agent_id, status_msg)
コード例 #15
0
ファイル: _python.py プロジェクト: movermeyer/loads
    def _dump_queue(self, run_id, queue, filename, compress=True):
        # lines
        qsize = queue.qsize()
        if qsize == 0:
            return

        if run_id is None:
            run_id = 'unknown'

        with open(filename, 'ab+') as f:
            for i in range(qsize):
                line = queue.get()
                if 'run_id' not in line:
                    line['run_id'] = run_id
                line = self._compress_headers(run_id, line)
                if compress:
                    f.write(zlib.compress(json.dumps(line)) + ZLIB_END)
                else:
                    f.write(json.dumps(line) + '\n')
コード例 #16
0
ファイル: _python.py プロジェクト: jether2011/loads
    def _dump_queue(self, run_id, queue, filename, compress=True):
        # lines
        qsize = queue.qsize()
        if qsize == 0:
            return

        if run_id is None:
            run_id = 'unknown'

        with open(filename, 'ab+') as f:
            for i in range(qsize):
                line = queue.get()
                if 'run_id' not in line:
                    line['run_id'] = run_id
                line = self._compress_headers(run_id, line)
                if compress:
                    f.write(zlib.compress(json.dumps(line)) + ZLIB_END)
                else:
                    f.write(json.dumps(line) + '\n')
コード例 #17
0
ファイル: brokerctrl.py プロジェクト: jgsbennett/loads
    def clean(self):
        """This is called periodically to :

        - send a _STATUS command to all active agents to refresh their status
        - detect agents that have not responded for a while and discard them
          from the run and from the agents list
        """
        now = time.time()

        for agent_id, (run_id, when) in self._runs.items():
            # when was the last time we've got a response ?
            last_contact = self._agent_times.get(agent_id)

            # is the agent not responding since a while ?
            if (last_contact is not None and
               now - last_contact > self.agent_timeout):
                # let's kill the agent...
                lag = now - last_contact
                logger.debug('No response from agent since %d s.' % lag)
                logger.debug('Killing agent %s' % str(agent_id))
                quit = json.dumps({'command': 'QUIT'})
                self.send_to_agent(agent_id, quit)

                # and remove it from the run
                run_id = self._terminate_run(agent_id)

                if run_id is not None:
                    logger.debug('publishing end of run')
                    # if the tests are finished, publish this on the pubsub.
                    msg = json.dumps({'data_type': 'run-finished',
                                      'run_id': run_id})
                    self.broker._publisher.send(msg)
            else:
                # initialize the timer
                if last_contact is None:
                    self._agent_times[agent_id] = now

                # sending a _STATUS call to on each active agent
                status_msg = json.dumps({'command': '_STATUS',
                                         'run_id': run_id})
                self.send_to_agent(agent_id, status_msg)
コード例 #18
0
    def _handle_recv_back(self, msg):
        # let's remove the agent id and track the time it took
        agent_id = msg[0]
        if len(msg) == 7:
            client_id = msg[4]
        else:
            client_id = None

        # grabbing the data to update the agents statuses if needed
        try:
            data = json.loads(msg[-1])
        except ValueError:
            logger.error("Could not load the received message")
            logger.error(str(msg))
            return

        if 'error' in data:
            result = data['error']
        else:
            result = data['result']

        command = result.get('command')

        # results from commands sent by the broker
        if command in ('_STATUS', 'STOP', 'QUIT'):
            run_id = self.ctrl.update_status(agent_id, result)

            if run_id is not None:
                # if the tests are finished, publish this on the pubsub.
                self._publisher.send(
                    json.dumps({
                        'data_type': 'run-finished',
                        'run_id': run_id
                    }))
            return

        # other things are pass-through (asked by a client)
        if client_id is None:
            return

        try:
            self._frontstream.send_multipart([client_id, '', msg[-1]])
        except Exception, e:
            logger.error('Could not send to front')
            logger.error(msg)
            # we don't want to die on error. we just log it
            exc_type, exc_value, exc_traceback = sys.exc_info()
            exc = traceback.format_tb(exc_traceback)
            exc.insert(0, str(e))
            logger.error('\n'.join(exc))
コード例 #19
0
ファイル: _python.py プロジェクト: EnTeQuAk/loads
    def _dump_queue(self, run_id, queue, filename):
        # lines
        qsize = queue.qsize()
        if qsize == 0:
            return

        if run_id is None:
            run_id = "unknown"

        with open(filename, "ab+") as f:
            for i in range(qsize):
                line = queue.get()
                if "run_id" not in line:
                    line["run_id"] = run_id
                line = self._compress_headers(run_id, line)
                f.write(zlib.compress(json.dumps(line)) + ZLIB_END)
コード例 #20
0
ファイル: broker.py プロジェクト: jgsbennett/loads
    def _handle_recv_back(self, msg):
        # let's remove the agent id and track the time it took
        agent_id = msg[0]
        if len(msg) == 7:
            client_id = msg[4]
        else:
            client_id = None

        # grabbing the data to update the agents statuses if needed
        try:
            data = json.loads(msg[-1])
        except ValueError:
            logger.error("Could not load the received message")
            logger.error(str(msg))
            return

        if "error" in data:
            result = data["error"]
        else:
            result = data["result"]

        command = result.get("command")

        # results from commands sent by the broker
        if command in ("_STATUS", "STOP", "QUIT"):
            run_id = self.ctrl.update_status(agent_id, result)

            if run_id is not None:
                # if the tests are finished, publish this on the pubsub.
                self._publisher.send(json.dumps({"data_type": "run-finished", "run_id": run_id}))
            return

        # other things are pass-through (asked by a client)
        if client_id is None:
            return

        try:
            self._frontstream.send_multipart([client_id, "", msg[-1]])
        except Exception, e:
            logger.error("Could not send to front")
            logger.error(msg)
            # we don't want to die on error. we just log it
            exc_type, exc_value, exc_traceback = sys.exc_info()
            exc = traceback.format_tb(exc_traceback)
            exc.insert(0, str(e))
            logger.error("\n".join(exc))
コード例 #21
0
ファイル: _redis.py プロジェクト: kayoung1222/KorStock
    def add(self, data):
        run_id = data['run_id']
        data_type = data['data_type'] = data.get('data_type', 'unknown')
        size = data.get('size', 1)

        pipeline = self._redis.pipeline()
        pipeline.sadd('runs', run_id)

        # adding counts
        counter = 'count:%s:%s' % (run_id, data_type)
        counters = 'counters:%s' % run_id
        if not self._redis.sismember(counters, counter):
            pipeline.sadd(counters, counter)

        pipeline.incrby('count:%s:%s' % (run_id, data_type), size)

        # adding urls
        if 'url' in data:
            url = data['url']
            urls = 'urls:%s' % run_id
            if not self._redis.sismember(urls, url):
                pipeline.sadd(urls, url)
            pipeline.incrby('url:%s:%s' % (run_id, url), 1)

        # adding data
        dumped = json.dumps(data)
        pipeline.lpush('data:%s' % run_id, dumped)

        # adding errors
        if data_type == 'addError':
            pipeline.lpush('errors:%s' % run_id, dumped)

        # adding group by
        md5 = hashlib.md5(dumped).hexdigest()
        pipeline.incrby('bcount:%s:%s' % (run_id, md5), size)
        pipeline.set('bvalue:%s:%s' % (run_id, md5), dumped)
        bcounters = 'bcounters:%s' % run_id
        if not self._redis.sismember(bcounters, md5):
            pipeline.sadd(bcounters, md5)

        pipeline.execute()
コード例 #22
0
ファイル: brokerctrl.py プロジェクト: jgsbennett/loads
    def run(self, msg, data):
        target = msg[0]

        # create a unique id for this run
        run_id = str(uuid4())

        # get some agents
        try:
            agents = self.reserve_agents(data['agents'], run_id)
        except NotEnoughWorkersError:
            self.broker.send_json(target, {'error': 'Not enough agents'})
            return

        # make sure the DB is prepared
        self._db.prepare_run()

        # send to every agent with the run_id and the receiver endpoint
        data['run_id'] = run_id
        data['args']['zmq_receiver'] = self.broker.endpoints['receiver']

        # replace CTRL_RUN by RUN
        data['command'] = 'RUN'

        # rebuild the ZMQ message to pass to agents
        msg = json.dumps(data)

        # notice when the test was started
        data['args']['started'] = time.time()
        data['args']['active'] = True

        # save the tests metadata in the db
        self.save_metadata(run_id, data['args'])
        self.flush_db()

        for agent_id in agents:
            self.send_to_agent(agent_id, msg)

        # tell the client which agents where selected.
        res = {'result': {'agents': agents, 'run_id': run_id}}
        self.broker.send_json(target, res)
コード例 #23
0
ファイル: brokerctrl.py プロジェクト: jgsbennett/loads
    def run(self, msg, data):
        target = msg[0]

        # create a unique id for this run
        run_id = str(uuid4())

        # get some agents
        try:
            agents = self.reserve_agents(data['agents'], run_id)
        except NotEnoughWorkersError:
            self.broker.send_json(target, {'error': 'Not enough agents'})
            return

        # make sure the DB is prepared
        self._db.prepare_run()

        # send to every agent with the run_id and the receiver endpoint
        data['run_id'] = run_id
        data['args']['zmq_receiver'] = self.broker.endpoints['receiver']

        # replace CTRL_RUN by RUN
        data['command'] = 'RUN'

        # rebuild the ZMQ message to pass to agents
        msg = json.dumps(data)

        # notice when the test was started
        data['args']['started'] = time.time()
        data['args']['active'] = True

        # save the tests metadata in the db
        self.save_metadata(run_id, data['args'])
        self.flush_db()

        for agent_id in agents:
            self.send_to_agent(agent_id, msg)

        # tell the client which agents where selected.
        res = {'result': {'agents': agents, 'run_id': run_id}}
        self.broker.send_json(target, res)
コード例 #24
0
ファイル: brokerctrl.py プロジェクト: jether2011/loads
    def run_command(self, cmd, msg, data):
        cmd = cmd.lower()
        target = msg[0]

        # command for agents
        if cmd.startswith('agent_'):
            command = cmd[len('agent_'):].upper()

            # when a STATUS call is made, we make it
            # an indirect call
            if command == 'STATUS':
                command = '_STATUS'

            data['command'] = command
            agent_id = str(data['agent_id'])
            self.send_to_agent(agent_id, json.dumps(data), target=target)

            if command == '_STATUS':
                logger.debug('current cache %s' % str(self._cached_status))
                return self._cached_status[agent_id]

            return

        if not hasattr(self, cmd):
            raise AttributeError(cmd)

        # calling the command asynchronously
        def _call():
            try:
                res = getattr(self, cmd)(msg, data)
                res = {'result': res}
                self.broker.send_json(target, res)
            except Exception, e:
                logger.debug('Failed')
                exc_type, exc_value, exc_traceback = sys.exc_info()
                exc = traceback.format_tb(exc_traceback)
                exc.insert(0, str(e))
                self.broker.send_json(target, {'error': exc})
コード例 #25
0
    def run_command(self, cmd, msg, data):
        cmd = cmd.lower()
        target = msg[0]

        # command for agents
        if cmd.startswith('agent_'):
            command = cmd[len('agent_'):].upper()

            # when a STATUS call is made, we make it
            # an indirect call
            if command == 'STATUS':
                command = '_STATUS'

            data['command'] = command
            agent_id = str(data['agent_id'])
            self.send_to_agent(agent_id, json.dumps(data), target=target)

            if command == '_STATUS':
                logger.debug('current cache %s' % str(self._cached_status))
                return self._cached_status[agent_id]

            return

        if not hasattr(self, cmd):
            raise AttributeError(cmd)

        # calling the command asynchronously
        def _call():
            try:
                res = getattr(self, cmd)(msg, data)
                res = {'result': res}
                self.broker.send_json(target, res)
            except Exception, e:
                logger.debug('Failed')
                exc_type, exc_value, exc_traceback = sys.exc_info()
                exc = traceback.format_tb(exc_traceback)
                exc.insert(0, str(e))
                self.broker.send_json(target, {'error': exc})
コード例 #26
0
ファイル: brokerctrl.py プロジェクト: EnTeQuAk/loads
    def stop_run(self, msg, data):
        run_id = data['run_id']
        agents = []

        for agent_id, (_run_id, when) in self._runs.items():
            if run_id != _run_id:
                continue
            agents.append(agent_id)

        if len(agents) == 0:
            # we don't have any agents running that test, let's
            # force the flags in the DB
            self.update_metadata(run_id, stopped=True, active=False,
                                 ended=time.time())
            return []

        # now we have a list of agents to stop
        stop_msg = msg[:-1] + [json.dumps({'command': 'STOP'})]

        for agent_id in agents:
            self.send_to_agent(agent_id, stop_msg)

        return agents
コード例 #27
0
 def test_messages_are_relayed(self):
     runner = ExternalRunner()
     runner._test_result = mock.MagicMock()
     data = json.dumps({'data_type': 'foo', 'bar': 'barbaz', 'run_id': 1})
     runner._process_result([data, ])
     runner.test_result.foo.assertCalledWith(bar='barbaz')
コード例 #28
0
ファイル: broker.py プロジェクト: EnTeQuAk/loads
 def send_json(self, target, data):
     try:
         self._frontstream.send_multipart(target + [json.dumps(data)])
     except ValueError:
         logger.error('Could not dump %s' % str(data))
         raise
コード例 #29
0
ファイル: message.py プロジェクト: EnTeQuAk/loads
 def serialize(self):
     return json.dumps(self.data)
コード例 #30
0
ファイル: _redis.py プロジェクト: kayoung1222/KorStock
 def save_metadata(self, run_id, metadata):
     key = 'metadata:%s' % run_id
     self._redis.set(key, json.dumps(metadata))
コード例 #31
0
 def serialize(self):
     return json.dumps(self.data)
コード例 #32
0
 def register(self):
     # telling the broker we are ready
     data = {'pid': self.pid, 'hostname': get_hostname()}
     self._reg.send_multipart(['REGISTER', json.dumps(data)])
コード例 #33
0
 def send_json(self, target, data):
     try:
         self._frontstream.send_multipart(target + [json.dumps(data)])
     except ValueError:
         logger.error('Could not dump %s' % str(data))
         raise
コード例 #34
0
ファイル: agent.py プロジェクト: jgsbennett/loads
 def register(self):
     # telling the broker we are ready
     data = {'pid': self.pid, 'hostname': get_hostname()}
     self._reg.send_multipart(['REGISTER', json.dumps(data)])