def test_message(self):
data = {'1': 2}
msg = Message(**data)
self.assertEquals(msg.serialize(), json.dumps(data))
msg = Message.load_from_string(json.dumps(data))
self.assertEquals(msg.serialize(), json.dumps(data))
def test_message(self):
data = {'1': 2}
msg = Message(**data)
self.assertEquals(msg.serialize(), json.dumps(data))
msg = Message.load_from_string(json.dumps(data))
self.assertEquals(msg.serialize(), json.dumps(data))
def stop_run(self, msg, data):
run_id = data['run_id']
agents = []
for agent_id, (_run_id, when) in self._runs.items():
if run_id != _run_id:
continue
agents.append(agent_id)
if len(agents) == 0:
# we don't have any agents running that test, let's
# force the flags in the DB
self.update_metadata(run_id,
stopped=True,
active=False,
ended=time.time())
return []
# now we have a list of agents to stop
stop_msg = json.dumps({'command': 'STOP'})
for agent_id in agents:
self.send_to_agent(agent_id, stop_msg)
return agents
def _handle_recv_back(self, msg):
# do the message and send the result
if self.debug:
#logger.debug('Message received from the broker')
target = timed()(self._handle_commands)
else:
target = self._handle_commands
duration = -1
try:
res = target(Message.load_from_string(msg[0]))
if self.debug:
duration, res = res
res = json.dumps(res)
# we're working with strings
if isinstance(res, unicode):
res = res.encode('utf8')
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
logger.error(res)
def _handle_recv_back(self, msg):
# do the message and send the result
if self.debug:
target = timed()(self._handle_commands)
else:
target = self._handle_commands
duration = -1
broker_id = msg[2]
if len(msg) == 7:
client_id = msg[4]
else:
client_id = None
data = msg[-1]
try:
res = target(Message.load_from_string(data))
if self.debug:
duration, res = res
res['hostname'] = get_hostname()
res = json.dumps(res)
# we're working with strings
if isinstance(res, unicode):
res = res.encode('utf8')
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
logger.error(res)
def _handle_recv_back(self, msg):
# let's remove the agent id and track the time it took
agent_id = msg[0]
msg = msg[1:]
# grabbing the data to update the agents statuses if needed
data = json.loads(msg[-1])
if 'error' in data:
result = data['error']
else:
result = data['result']
if result.get('command') in ('_STATUS', 'STOP', 'QUIT'):
statuses = result['status'].values()
run_id = self.ctrl.update_status(agent_id, statuses)
if run_id is not None:
# if the tests are finished, publish this on the pubsub.
self._publisher.send(
json.dumps({
'data_type': 'run-finished',
'run_id': run_id
}))
return
# other things are pass-through
try:
self._frontstream.send_multipart(msg)
except Exception, e:
logger.error('Could not send to front')
logger.error(msg)
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def _handle_recv_back(self, msg):
# do the message and send the result
if self.debug:
#logger.debug('Message received from the broker')
target = timed()(self._handle_commands)
else:
target = self._handle_commands
duration = -1
try:
res = target(Message.load_from_string(msg[0]))
if self.debug:
duration, res = res
res = json.dumps(res)
# we're working with strings
if isinstance(res, unicode):
res = res.encode('utf8')
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
logger.error(res)
def _handle_recv_back(self, msg):
# do the message and send the result
if self.debug:
target = timed()(self._handle_commands)
else:
target = self._handle_commands
duration = -1
broker_id = msg[2]
if len(msg) == 7:
client_id = msg[4]
else:
client_id = None
data = msg[-1]
try:
res = target(Message.load_from_string(data))
if self.debug:
duration, res = res
res['hostname'] = get_hostname()
res = json.dumps(res)
# we're working with strings
if isinstance(res, unicode):
res = res.encode('utf8')
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
logger.error(res)
def run_command(self, cmd, msg, data):
cmd = cmd.lower()
target = msg[:-1]
# command for agents
if cmd.startswith('agent_'):
data['command'] = cmd[len('agent_'):].upper()
msg = msg[:2] + [json.dumps(data)] + msg[2:]
self.send_to_agent(str(data['agent_id']), msg)
return # returning None because it's async
if not hasattr(self, cmd):
raise AttributeError(cmd)
# calling the command asynchronously
def _call():
try:
res = getattr(self, cmd)(msg, data)
res = {'result': res}
self.broker.send_json(target, res)
except Exception, e:
logger.debug('Failed')
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
self.broker.send_json(target, {'error': exc})
def _handle_recv_back(self, msg):
# let's remove the agent id and track the time it took
agent_id = msg[0]
msg = msg[1:]
# grabbing the data to update the agents statuses if needed
data = json.loads(msg[-1])
if 'error' in data:
result = data['error']
else:
result = data['result']
if result.get('command') in ('_STATUS', 'STOP', 'QUIT'):
statuses = result['status'].values()
run_id = self.ctrl.update_status(agent_id, statuses)
if run_id is not None:
# if the tests are finished, publish this on the pubsub.
self._publisher.send(json.dumps({'data_type': 'run-finished',
'run_id': run_id}))
return
# other things are pass-through
try:
self._frontstream.send_multipart(msg)
except Exception, e:
logger.error('Could not send to front')
logger.error(msg)
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def send_json(self, target, data):
assert isinstance(target, basestring), target
msg = [target, '', json.dumps(data)]
try:
self._frontstream.send_multipart(msg)
except ValueError:
logger.error('Could not dump %s' % str(data))
raise
def test_messages_are_relayed(self):
runner = ExternalRunner()
runner._test_result = mock.MagicMock()
data = json.dumps({'data_type': 'foo', 'bar': 'barbaz', 'run_id': 1})
runner._process_result([
data,
])
runner.test_result.foo.assertCalledWith(bar='barbaz')
def send_json(self, target, data):
assert isinstance(target, basestring), target
msg = [target, "", json.dumps(data)]
try:
self._frontstream.send_multipart(msg)
except ValueError:
logger.error("Could not dump %s" % str(data))
raise
def clean(self):
"""This is called periodically to :
- send a _STATUS command to all active agents to refresh their status
- detect agents that have not responded for a while and discard them
from the run and from the agents list
"""
now = time.time()
for agent_id, (run_id, when) in self._runs.items():
# when was the last time we've got a response ?
last_contact = self._agent_times.get(agent_id)
# is the agent not responding since a while ?
if (last_contact is not None
and now - last_contact > self.agent_timeout):
# let's kill the agent...
lag = now - last_contact
logger.debug('No response from agent since %d s.' % lag)
logger.debug('Killing agent %s' % str(agent_id))
quit = json.dumps({'command': 'QUIT'})
self.send_to_agent(agent_id, quit)
# and remove it from the run
run_id = self._terminate_run(agent_id)
if run_id is not None:
logger.debug('publishing end of run')
# if the tests are finished, publish this on the pubsub.
msg = json.dumps({
'data_type': 'run-finished',
'run_id': run_id
})
self.broker._publisher.send(msg)
else:
# initialize the timer
if last_contact is None:
self._agent_times[agent_id] = now
# sending a _STATUS call to on each active agent
status_msg = json.dumps({
'command': '_STATUS',
'run_id': run_id
})
self.send_to_agent(agent_id, status_msg)
def _dump_queue(self, run_id, queue, filename, compress=True):
# lines
qsize = queue.qsize()
if qsize == 0:
return
if run_id is None:
run_id = 'unknown'
with open(filename, 'ab+') as f:
for i in range(qsize):
line = queue.get()
if 'run_id' not in line:
line['run_id'] = run_id
line = self._compress_headers(run_id, line)
if compress:
f.write(zlib.compress(json.dumps(line)) + ZLIB_END)
else:
f.write(json.dumps(line) + '\n')
def _dump_queue(self, run_id, queue, filename, compress=True):
# lines
qsize = queue.qsize()
if qsize == 0:
return
if run_id is None:
run_id = 'unknown'
with open(filename, 'ab+') as f:
for i in range(qsize):
line = queue.get()
if 'run_id' not in line:
line['run_id'] = run_id
line = self._compress_headers(run_id, line)
if compress:
f.write(zlib.compress(json.dumps(line)) + ZLIB_END)
else:
f.write(json.dumps(line) + '\n')
def clean(self):
"""This is called periodically to :
- send a _STATUS command to all active agents to refresh their status
- detect agents that have not responded for a while and discard them
from the run and from the agents list
"""
now = time.time()
for agent_id, (run_id, when) in self._runs.items():
# when was the last time we've got a response ?
last_contact = self._agent_times.get(agent_id)
# is the agent not responding since a while ?
if (last_contact is not None and
now - last_contact > self.agent_timeout):
# let's kill the agent...
lag = now - last_contact
logger.debug('No response from agent since %d s.' % lag)
logger.debug('Killing agent %s' % str(agent_id))
quit = json.dumps({'command': 'QUIT'})
self.send_to_agent(agent_id, quit)
# and remove it from the run
run_id = self._terminate_run(agent_id)
if run_id is not None:
logger.debug('publishing end of run')
# if the tests are finished, publish this on the pubsub.
msg = json.dumps({'data_type': 'run-finished',
'run_id': run_id})
self.broker._publisher.send(msg)
else:
# initialize the timer
if last_contact is None:
self._agent_times[agent_id] = now
# sending a _STATUS call to on each active agent
status_msg = json.dumps({'command': '_STATUS',
'run_id': run_id})
self.send_to_agent(agent_id, status_msg)
def _handle_recv_back(self, msg):
# let's remove the agent id and track the time it took
agent_id = msg[0]
if len(msg) == 7:
client_id = msg[4]
else:
client_id = None
# grabbing the data to update the agents statuses if needed
try:
data = json.loads(msg[-1])
except ValueError:
logger.error("Could not load the received message")
logger.error(str(msg))
return
if 'error' in data:
result = data['error']
else:
result = data['result']
command = result.get('command')
# results from commands sent by the broker
if command in ('_STATUS', 'STOP', 'QUIT'):
run_id = self.ctrl.update_status(agent_id, result)
if run_id is not None:
# if the tests are finished, publish this on the pubsub.
self._publisher.send(
json.dumps({
'data_type': 'run-finished',
'run_id': run_id
}))
return
# other things are pass-through (asked by a client)
if client_id is None:
return
try:
self._frontstream.send_multipart([client_id, '', msg[-1]])
except Exception, e:
logger.error('Could not send to front')
logger.error(msg)
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def _dump_queue(self, run_id, queue, filename):
# lines
qsize = queue.qsize()
if qsize == 0:
return
if run_id is None:
run_id = "unknown"
with open(filename, "ab+") as f:
for i in range(qsize):
line = queue.get()
if "run_id" not in line:
line["run_id"] = run_id
line = self._compress_headers(run_id, line)
f.write(zlib.compress(json.dumps(line)) + ZLIB_END)
def _handle_recv_back(self, msg):
# let's remove the agent id and track the time it took
agent_id = msg[0]
if len(msg) == 7:
client_id = msg[4]
else:
client_id = None
# grabbing the data to update the agents statuses if needed
try:
data = json.loads(msg[-1])
except ValueError:
logger.error("Could not load the received message")
logger.error(str(msg))
return
if "error" in data:
result = data["error"]
else:
result = data["result"]
command = result.get("command")
# results from commands sent by the broker
if command in ("_STATUS", "STOP", "QUIT"):
run_id = self.ctrl.update_status(agent_id, result)
if run_id is not None:
# if the tests are finished, publish this on the pubsub.
self._publisher.send(json.dumps({"data_type": "run-finished", "run_id": run_id}))
return
# other things are pass-through (asked by a client)
if client_id is None:
return
try:
self._frontstream.send_multipart([client_id, "", msg[-1]])
except Exception, e:
logger.error("Could not send to front")
logger.error(msg)
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error("\n".join(exc))
def add(self, data):
run_id = data['run_id']
data_type = data['data_type'] = data.get('data_type', 'unknown')
size = data.get('size', 1)
pipeline = self._redis.pipeline()
pipeline.sadd('runs', run_id)
# adding counts
counter = 'count:%s:%s' % (run_id, data_type)
counters = 'counters:%s' % run_id
if not self._redis.sismember(counters, counter):
pipeline.sadd(counters, counter)
pipeline.incrby('count:%s:%s' % (run_id, data_type), size)
# adding urls
if 'url' in data:
url = data['url']
urls = 'urls:%s' % run_id
if not self._redis.sismember(urls, url):
pipeline.sadd(urls, url)
pipeline.incrby('url:%s:%s' % (run_id, url), 1)
# adding data
dumped = json.dumps(data)
pipeline.lpush('data:%s' % run_id, dumped)
# adding errors
if data_type == 'addError':
pipeline.lpush('errors:%s' % run_id, dumped)
# adding group by
md5 = hashlib.md5(dumped).hexdigest()
pipeline.incrby('bcount:%s:%s' % (run_id, md5), size)
pipeline.set('bvalue:%s:%s' % (run_id, md5), dumped)
bcounters = 'bcounters:%s' % run_id
if not self._redis.sismember(bcounters, md5):
pipeline.sadd(bcounters, md5)
pipeline.execute()
def run(self, msg, data):
target = msg[0]
# create a unique id for this run
run_id = str(uuid4())
# get some agents
try:
agents = self.reserve_agents(data['agents'], run_id)
except NotEnoughWorkersError:
self.broker.send_json(target, {'error': 'Not enough agents'})
return
# make sure the DB is prepared
self._db.prepare_run()
# send to every agent with the run_id and the receiver endpoint
data['run_id'] = run_id
data['args']['zmq_receiver'] = self.broker.endpoints['receiver']
# replace CTRL_RUN by RUN
data['command'] = 'RUN'
# rebuild the ZMQ message to pass to agents
msg = json.dumps(data)
# notice when the test was started
data['args']['started'] = time.time()
data['args']['active'] = True
# save the tests metadata in the db
self.save_metadata(run_id, data['args'])
self.flush_db()
for agent_id in agents:
self.send_to_agent(agent_id, msg)
# tell the client which agents where selected.
res = {'result': {'agents': agents, 'run_id': run_id}}
self.broker.send_json(target, res)
def run(self, msg, data):
target = msg[0]
# create a unique id for this run
run_id = str(uuid4())
# get some agents
try:
agents = self.reserve_agents(data['agents'], run_id)
except NotEnoughWorkersError:
self.broker.send_json(target, {'error': 'Not enough agents'})
return
# make sure the DB is prepared
self._db.prepare_run()
# send to every agent with the run_id and the receiver endpoint
data['run_id'] = run_id
data['args']['zmq_receiver'] = self.broker.endpoints['receiver']
# replace CTRL_RUN by RUN
data['command'] = 'RUN'
# rebuild the ZMQ message to pass to agents
msg = json.dumps(data)
# notice when the test was started
data['args']['started'] = time.time()
data['args']['active'] = True
# save the tests metadata in the db
self.save_metadata(run_id, data['args'])
self.flush_db()
for agent_id in agents:
self.send_to_agent(agent_id, msg)
# tell the client which agents where selected.
res = {'result': {'agents': agents, 'run_id': run_id}}
self.broker.send_json(target, res)
def run_command(self, cmd, msg, data):
cmd = cmd.lower()
target = msg[0]
# command for agents
if cmd.startswith('agent_'):
command = cmd[len('agent_'):].upper()
# when a STATUS call is made, we make it
# an indirect call
if command == 'STATUS':
command = '_STATUS'
data['command'] = command
agent_id = str(data['agent_id'])
self.send_to_agent(agent_id, json.dumps(data), target=target)
if command == '_STATUS':
logger.debug('current cache %s' % str(self._cached_status))
return self._cached_status[agent_id]
return
if not hasattr(self, cmd):
raise AttributeError(cmd)
# calling the command asynchronously
def _call():
try:
res = getattr(self, cmd)(msg, data)
res = {'result': res}
self.broker.send_json(target, res)
except Exception, e:
logger.debug('Failed')
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
self.broker.send_json(target, {'error': exc})
def run_command(self, cmd, msg, data):
cmd = cmd.lower()
target = msg[0]
# command for agents
if cmd.startswith('agent_'):
command = cmd[len('agent_'):].upper()
# when a STATUS call is made, we make it
# an indirect call
if command == 'STATUS':
command = '_STATUS'
data['command'] = command
agent_id = str(data['agent_id'])
self.send_to_agent(agent_id, json.dumps(data), target=target)
if command == '_STATUS':
logger.debug('current cache %s' % str(self._cached_status))
return self._cached_status[agent_id]
return
if not hasattr(self, cmd):
raise AttributeError(cmd)
# calling the command asynchronously
def _call():
try:
res = getattr(self, cmd)(msg, data)
res = {'result': res}
self.broker.send_json(target, res)
except Exception, e:
logger.debug('Failed')
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
self.broker.send_json(target, {'error': exc})
def stop_run(self, msg, data):
run_id = data['run_id']
agents = []
for agent_id, (_run_id, when) in self._runs.items():
if run_id != _run_id:
continue
agents.append(agent_id)
if len(agents) == 0:
# we don't have any agents running that test, let's
# force the flags in the DB
self.update_metadata(run_id, stopped=True, active=False,
ended=time.time())
return []
# now we have a list of agents to stop
stop_msg = msg[:-1] + [json.dumps({'command': 'STOP'})]
for agent_id in agents:
self.send_to_agent(agent_id, stop_msg)
return agents
def test_messages_are_relayed(self):
runner = ExternalRunner()
runner._test_result = mock.MagicMock()
data = json.dumps({'data_type': 'foo', 'bar': 'barbaz', 'run_id': 1})
runner._process_result([data, ])
runner.test_result.foo.assertCalledWith(bar='barbaz')
def send_json(self, target, data):
try:
self._frontstream.send_multipart(target + [json.dumps(data)])
except ValueError:
logger.error('Could not dump %s' % str(data))
raise
def serialize(self):
return json.dumps(self.data)
def save_metadata(self, run_id, metadata):
key = 'metadata:%s' % run_id
self._redis.set(key, json.dumps(metadata))
def serialize(self):
return json.dumps(self.data)
def register(self):
# telling the broker we are ready
data = {'pid': self.pid, 'hostname': get_hostname()}
self._reg.send_multipart(['REGISTER', json.dumps(data)])
def send_json(self, target, data):
try:
self._frontstream.send_multipart(target + [json.dumps(data)])
except ValueError:
logger.error('Could not dump %s' % str(data))
raise
def register(self):
# telling the broker we are ready
data = {'pid': self.pid, 'hostname': get_hostname()}
self._reg.send_multipart(['REGISTER', json.dumps(data)])