def _handle_recv_back(self, msg):
# do the message and send the result
if self.debug:
logger.debug('Message received')
target = timed()(self.target)
else:
target = self.target
duration = -1
# results are sent with a PID:OK: or a PID:ERROR prefix
try:
with self.timer.run_message():
res = target(Message.load_from_string(msg[0]))
# did we timout ?
if self.timer.timed_out:
# let's dump the last
for line in self.timer.last_dump:
logger.error(line)
if self.debug:
duration, res = res
# we're working with strings
if isinstance(res, unicode):
res = res.encode('utf8')
res = '%d:OK:%s' % (self.pid, res)
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = '%d:ERROR:%s' % (self.pid, '\n'.join(exc))
logger.error(res)
def test_ended(self, run_id):
# we want to ping all observers that things are done
# for a given test.
# get the list of observers
args = self.get_metadata(run_id)
observers = _compute_observers(args.get('observer'))
if observers == []:
return
# rebuild the test result instance
test_result = LazyTestResult(args=args)
test_result.args = args
data = self.get_data(run_id)
if len(data) > 0:
started = datetime.datetime.utcfromtimestamp(data[0]['started'])
test_result.startTestRun(when=started)
test_result.set_counts(self.get_counts(run_id))
# for each observer we call it with the test results
for observer in observers:
try:
observer(test_result, args)
except Exception:
# the observer code failed. We want to log it
logger.error('%r failed' % observer)
def _handle_recv_back(self, msg):
# do the message and send the result
if self.debug:
#logger.debug('Message received from the broker')
target = timed()(self._handle_commands)
else:
target = self._handle_commands
duration = -1
try:
res = target(Message.load_from_string(msg[0]))
if self.debug:
duration, res = res
res = json.dumps(res)
# we're working with strings
if isinstance(res, unicode):
res = res.encode('utf8')
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
logger.error(res)
def _handle_recv_back(self, msg):
# do the message and send the result
if self.debug:
target = timed()(self._handle_commands)
else:
target = self._handle_commands
duration = -1
broker_id = msg[2]
if len(msg) == 7:
client_id = msg[4]
else:
client_id = None
data = msg[-1]
try:
res = target(Message.load_from_string(data))
if self.debug:
duration, res = res
res['hostname'] = get_hostname()
res = json.dumps(res)
# we're working with strings
if isinstance(res, unicode):
res = res.encode('utf8')
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
logger.error(res)
def _handle_recv_back(self, msg):
# do the message and send the result
if self.debug:
#logger.debug('Message received from the broker')
target = timed()(self._handle_commands)
else:
target = self._handle_commands
duration = -1
try:
res = target(Message.load_from_string(msg[0]))
if self.debug:
duration, res = res
res = json.dumps(res)
# we're working with strings
if isinstance(res, unicode):
res = res.encode('utf8')
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
logger.error(res)
def test_ended(self, run_id):
# first of all, we want to mark it done in the DB
self.update_metadata(run_id, stopped=True, active=False,
ended=time.time())
# we want to ping all observers that things are done
# for a given test.
# get the list of observers
args = self._db.get_metadata(run_id)
observers = _compute_observers(args.get('observer'))
if observers == []:
return
# rebuild the test result instance
test_result = RemoteTestResult(args=args)
test_result.args = args
data = list(self._db.get_data(run_id, size=1))
if len(data) > 0:
started = datetime.datetime.utcfromtimestamp(data[0]['started'])
test_result.startTestRun(when=started)
test_result.set_counts(self._db.get_counts(run_id))
# for each observer we call it with the test results
for observer in observers:
try:
observer(test_result, args)
except Exception:
# the observer code failed. We want to log it
logger.error('%r failed' % observer)
def _handle_recv_back(self, msg):
# do the message and send the result
if self.debug:
target = timed()(self._handle_commands)
else:
target = self._handle_commands
duration = -1
broker_id = msg[2]
if len(msg) == 7:
client_id = msg[4]
else:
client_id = None
data = msg[-1]
try:
res = target(Message.load_from_string(data))
if self.debug:
duration, res = res
res['hostname'] = get_hostname()
res = json.dumps(res)
# we're working with strings
if isinstance(res, unicode):
res = res.encode('utf8')
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
logger.error(res)
def send_json(self, target, data):
assert isinstance(target, basestring), target
msg = [target, '', json.dumps(data)]
try:
self._frontstream.send_multipart(msg)
except ValueError:
logger.error('Could not dump %s' % str(data))
raise
def send_json(self, target, data):
assert isinstance(target, basestring), target
msg = [target, "", json.dumps(data)]
try:
self._frontstream.send_multipart(msg)
except ValueError:
logger.error("Could not dump %s" % str(data))
raise
def test_ended(self, run_id):
# first of all, we want to mark it done in the DB
logger.debug('test %s ended marking the metadata' % run_id)
self.update_metadata(run_id,
stopped=True,
active=False,
ended=time.time())
# we want to ping all observers that things are done
# for a given test.
# get the list of observers
args = self._db.get_metadata(run_id)
observers = _compute_observers(args.get('observer'))
if observers == []:
self._db.summarize_run(run_id)
return
logger.debug('test %s ended calling the observers' % run_id)
# if we are using the web dashboard - we're just providing a link
if self.broker.web_root is not None:
test_result = '%s/run/%s' % (self.broker.web_root, run_id)
else:
# rebuild the test result instance
test_result = RemoteTestResult(args=args)
test_result.args = args
if 'started' in args:
started = args['started']
started = datetime.datetime.utcfromtimestamp(started)
test_result.startTestRun(when=started)
test_result.set_counts(self._db.get_counts(run_id))
# for each observer we call it with the test results
for observer in observers:
options = {}
prefix = 'observer_%s_' % observer.name
for name, value in args.items():
if name.startswith(prefix):
options[name[len(prefix):]] = value
# get the options
try:
observer(args=args, **options)(test_result)
except Exception:
# the observer code failed. We want to log it
logger.error('%r failed' % observer)
self._db.summarize_run(run_id)
def _handle_recv_back(self, msg):
# let's remove the agent id and track the time it took
agent_id = msg[0]
if len(msg) == 7:
client_id = msg[4]
else:
client_id = None
# grabbing the data to update the agents statuses if needed
try:
data = json.loads(msg[-1])
except ValueError:
logger.error("Could not load the received message")
logger.error(str(msg))
return
if 'error' in data:
result = data['error']
else:
result = data['result']
command = result.get('command')
# results from commands sent by the broker
if command in ('_STATUS', 'STOP', 'QUIT'):
run_id = self.ctrl.update_status(agent_id, result)
if run_id is not None:
# if the tests are finished, publish this on the pubsub.
self._publisher.send(
json.dumps({
'data_type': 'run-finished',
'run_id': run_id
}))
return
# other things are pass-through (asked by a client)
if client_id is None:
return
try:
self._frontstream.send_multipart([client_id, '', msg[-1]])
except Exception, e:
logger.error('Could not send to front')
logger.error(msg)
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def test_ended(self, run_id):
# first of all, we want to mark it done in the DB
logger.debug('test %s ended marking the metadata' % run_id)
self.update_metadata(run_id, stopped=True, active=False,
ended=time.time())
# we want to ping all observers that things are done
# for a given test.
# get the list of observers
args = self._db.get_metadata(run_id)
observers = _compute_observers(args.get('observer'))
if observers == []:
self._db.summarize_run(run_id)
return
logger.debug('test %s ended calling the observers' % run_id)
# if we are using the web dashboard - we're just providing a link
if self.broker.web_root is not None:
test_result = '%s/run/%s' % (self.broker.web_root, run_id)
else:
# rebuild the test result instance
test_result = RemoteTestResult(args=args)
test_result.args = args
if 'started' in args:
started = args['started']
started = datetime.datetime.utcfromtimestamp(started)
test_result.startTestRun(when=started)
test_result.set_counts(self._db.get_counts(run_id))
# for each observer we call it with the test results
for observer in observers:
options = {}
prefix = 'observer_%s_' % observer.name
for name, value in args.items():
if name.startswith(prefix):
options[name[len(prefix):]] = value
# get the options
try:
observer(args=args, **options)(test_result)
except Exception:
# the observer code failed. We want to log it
logger.error('%r failed' % observer)
self._db.summarize_run(run_id)
def send_to_agent(self, agent_id, msg):
msg = list(msg)
# start the timer
self._agent_times[agent_id] = time.time(), None
# now we can send to the right guy
msg.insert(0, agent_id)
try:
self.broker._backstream.send_multipart(msg)
except Exception, e:
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def send_to_agent(self, agent_id, msg):
msg = list(msg)
# now we can send to the right guy
msg.insert(0, agent_id)
try:
self.broker._backstream.send_multipart(msg)
except Exception, e:
logger.debug('Failed to send %s' % str(msg))
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
logger.debug('Removing agent')
self._remove_agent(agent_id)
def send_to_agent(self, agent_id, msg):
msg = list(msg)
# start the timer
self._agent_times[agent_id] = time.time(), None
# now we can send to the right guy
msg.insert(0, agent_id)
try:
self.broker._backstream.send_multipart(msg)
except Exception, e:
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def _handle_recv_front(self, msg, tentative=0):
# front => back
# if the last part of the message is 'PING', we just PONG back
# this is used as a health check
if msg[-1] == 'PING':
self._frontstream.send_multipart(msg[:-1] + [str(os.getpid())])
return
#logger.debug('front -> back [choosing a worker]')
if tentative == 3:
logger.debug('No workers')
self._frontstream.send_multipart(msg[:-1] +
['%d:ERROR:No worker' % os.getpid()])
return
# we want to decide who's going to do the work
found_worker = False
while not found_worker and len(self._workers) > 0:
worker_id = random.choice(self._workers)
if not self._check_worker(worker_id):
self._remove_worker(worker_id)
else:
found_worker = True
if not found_worker:
logger.debug('No worker, will try later')
later = time.time() + 0.5 + (tentative * 0.2)
self.loop.add_timeout(later, lambda: self._handle_recv_front(msg,
tentative + 1))
return
# start the timer
self._worker_times[worker_id] = time.time(), None
# now we can send to the right guy
msg.insert(0, worker_id)
#logger.debug('front -> back [%s]' % worker_id)
try:
self._backstream.send_multipart(msg)
except Exception, e:
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def _send_to_worker(self, worker_id, msg):
msg = list(msg)
# start the timer
self._worker_times[worker_id] = time.time(), None
# now we can send to the right guy
msg.insert(0, worker_id)
#logger.debug('front -> back [%s]' % worker_id)
try:
self._backstream.send_multipart(msg)
except Exception, e:
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def _handle_recv_back(self, msg):
# let's remove the agent id and track the time it took
agent_id = msg[0]
if len(msg) == 7:
client_id = msg[4]
else:
client_id = None
# grabbing the data to update the agents statuses if needed
try:
data = json.loads(msg[-1])
except ValueError:
logger.error("Could not load the received message")
logger.error(str(msg))
return
if "error" in data:
result = data["error"]
else:
result = data["result"]
command = result.get("command")
# results from commands sent by the broker
if command in ("_STATUS", "STOP", "QUIT"):
run_id = self.ctrl.update_status(agent_id, result)
if run_id is not None:
# if the tests are finished, publish this on the pubsub.
self._publisher.send(json.dumps({"data_type": "run-finished", "run_id": run_id}))
return
# other things are pass-through (asked by a client)
if client_id is None:
return
try:
self._frontstream.send_multipart([client_id, "", msg[-1]])
except Exception, e:
logger.error("Could not send to front")
logger.error(msg)
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error("\n".join(exc))
def send_to_agent(self, agent_id, msg, target=None):
# now we can send to the right guy
data = [agent_id, '', self.broker.pid, '']
if target is not None:
data += [target, '']
data.append(msg)
try:
self.broker._backend.send_multipart(data)
except Exception, e:
logger.debug('Failed to send %s' % str(msg))
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
logger.debug('Removing agent')
self._remove_agent(agent_id, '\n'.join(exc))
def send_to_agent(self, agent_id, msg, target=None):
# now we can send to the right guy
data = [agent_id, '', self.broker.pid, '']
if target is not None:
data += [target, '']
data.append(msg)
try:
self.broker._backend.send_multipart(data)
except Exception, e:
logger.debug('Failed to send %s' % str(msg))
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
logger.debug('Removing agent')
self._remove_agent(agent_id, '\n'.join(exc))
def _handle_recv_back(self, msg):
# let's remove the agent id and track the time it took
agent_id = msg[0]
msg = msg[1:]
# grabbing the data to update the agents statuses if needed
data = json.loads(msg[-1])
if 'error' in data:
result = data['error']
else:
result = data['result']
if result.get('command') in ('_STATUS', 'STOP', 'QUIT'):
statuses = result['status'].values()
run_id = self.ctrl.update_status(agent_id, statuses)
if run_id is not None:
# if the tests are finished, publish this on the pubsub.
self._publisher.send(json.dumps({'data_type': 'run-finished',
'run_id': run_id}))
return
# other things are pass-through
try:
self._frontstream.send_multipart(msg)
except Exception, e:
logger.error('Could not send to front')
logger.error(msg)
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def _handle_recv_back(self, msg):
# let's remove the agent id and track the time it took
agent_id = msg[0]
msg = msg[1:]
# grabbing the data to update the agents statuses if needed
data = json.loads(msg[-1])
if 'error' in data:
result = data['error']
else:
result = data['result']
if result.get('command') in ('_STATUS', 'STOP', 'QUIT'):
statuses = result['status'].values()
run_id = self.ctrl.update_status(agent_id, statuses)
if run_id is not None:
# if the tests are finished, publish this on the pubsub.
self._publisher.send(
json.dumps({
'data_type': 'run-finished',
'run_id': run_id
}))
return
# other things are pass-through
try:
self._frontstream.send_multipart(msg)
except Exception, e:
logger.error('Could not send to front')
logger.error(msg)
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def _handle_recv_back(self, msg):
# back => front
#logger.debug('front <- back [%s]' % msg[0])
# let's remove the worker id and track the time it took
worker_id = msg[0]
msg = msg[1:]
now = time.time()
if worker_id in self._worker_times:
start, stop = self._worker_times[worker_id]
self._worker_times[worker_id] = start, now
else:
self._worker_times[worker_id] = now, now
try:
self._frontstream.send_multipart(msg)
except Exception, e:
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def send_json(self, target, data):
try:
self._frontstream.send_multipart(target + [json.dumps(data)])
except ValueError:
logger.error('Could not dump %s' % str(data))
raise
def send_json(self, target, data):
try:
self._frontstream.send_multipart(target + [json.dumps(data)])
except ValueError:
logger.error('Could not dump %s' % str(data))
raise