def start_servers(): global _RUNNING if _RUNNING: return start_process('loads.transport.broker') for x in range(3): start_process('loads.transport.agent') start_process('loads.examples.echo_server') # wait for the echo server to be started tries = 0 while True: try: requests.get('http://0.0.0.0:9000') break except requests.ConnectionError: time.sleep(.3) tries += 1 if tries > 3: raise # wait for the broker to be up with 3 slaves. client = Client() while len(client.list()) != 3: time.sleep(.1) # control that the broker is responsive client.ping() for wid in client.list(): assert client.status(wid) == {} client.close() _RUNNING = True
class Controller(object): def __init__(self, db='redis', dboptions=None, broker=None): if dboptions is None: self.dboptions = {} else: self.dboptions = dboptions self.broker = broker self.backend = db self.db = None self.client = None self._init() def reconnect(self): self._init() def _init(self): # close any previous connector if self.client is not None: try: self.close() except zmq.ZMQError: pass self.db = get_database(self.backend, **self.dboptions) self.client = Client(self.broker, timeout_max_overflow=20.) def health_check(self): client = Client(self.broker, timeout_max_overflow=20.) ping = client.ping() total_agents = len(ping['agents']) if total_agents == 0: msg = 'No agents currently registered.' return False, msg, 0 runs = client.list_runs().items() busy_agents = sum([len(agents) for run_id, agents in runs]) avail = total_agents - busy_agents if avail == 0: # no agents are available. msg = 'All agents are busy.' return False, msg, 0 args = {'fqn': 'loads.examples.test_blog.TestWebSite.test_health', 'hits': '1', 'agents': avail, 'users': '1', 'detach': True} client.run(args) return True, 'Health check launched', avail def close(self): self.client.close() self.db.close() def stop(self, run_id): self.client.stop_run(run_id) self.get_broker_info() def agent_status(self, agent_id): return self.client.status(agent_id) def ping_db(self): return self.db.ping() def get_broker_info(self): return self.client.ping() def get_runs(self, **filters): if filters == {}: return self.db.get_runs() runs = [] for run_id in self.db.get_runs(): metadata = self.db.get_metadata(run_id) for key, value in filters.items(): if key not in metadata: continue else: if metadata[key] == value: runs.append(run_id) break return runs def get_run_info(self, run_id, data=True): result = {} # we need to batch XXX if data: data = self.db.get_data(run_id, size=100) result['data'] = data errors = {} for line in self.db.get_errors(run_id, size=100): error, tb, tb2 = line['exc_info'] hashed = md5(error).hexdigest() if hashed in errors: old_count, tb = errors[hashed] errors[hashed] = old_count + 1, tb else: errors[hashed] = 1, tb + '\n' + tb2 errors = errors.items() errors.sort() result['errors'] = errors counts = self.db.get_counts(run_id) custom = {} for key, value in list(counts.items()): if key in _COUNTS: continue custom[key] = value del counts[key] result['custom'] = custom metadata = self.db.get_metadata(run_id) started = metadata.get('started') ended = metadata.get('ended', time.time()) active = metadata.get('active', False) # aproximative = should be set by the broker if started is not None: if active: elapsed = time.time() - started else: elapsed = ended - started hits = counts.get('add_hit', 0) if hits == 0: rps = 0 else: rps = hits / elapsed def _stamp2time(stamp): if not isinstance(stamp, datetime): stamp = datetime.fromtimestamp(int(stamp)) return stamp.strftime('%Y-%m-%d %H:%M:%S UTC') started = datetime.fromtimestamp(int(started)) metadata['started'] = _stamp2time(started) counts['rps'] = int(rps) counts['elapsed'] = seconds_to_time(elapsed) ended = started + timedelta(seconds=elapsed) counts['finished'] = finished(ended) metadata['ended'] = _stamp2time(ended) counts['success'] = (counts.get('addError', 0) == 0 and counts.get('addFailure', 0) == 0) metadata['style'] = counts['success'] and 'green' or 'red' else: metadata['started'] = metadata['ended'] = 'N/A' counts['rps'] = 0 counts['elapsed'] = 0 counts['finished'] = 'N/A' counts['success'] = False metadata['style'] = 'red' if metadata.get('active', False): metadata['active_label'] = 'Running' else: metadata['active_label'] = 'Ended' result['counts'] = counts result['metadata'] = metadata return result