def get_worker_markers(): markers = [] points = utils.get_points_per_worker() # Worker start points for worker_no, worker_points in enumerate(points): coords = utils.get_start_coords(worker_no) markers.append({ 'icon': icons.dots.green, 'lat': coords[0], 'lng': coords[1], 'infobox': 'Worker %d' % worker_no, 'type': 'custom', 'subtype': 'worker', 'key': 'start-position-%d' % worker_no, 'disappear_time': -1 }) # Circles for i, point in enumerate(worker_points): markers.append({ 'lat': point[0], 'lng': point[1], 'infobox': 'Worker %d point %d' % (worker_no, i), 'subtype': 'point', }) return markers
def __init__(self, status_bar, loop): self.logger = logging.getLogger('overseer') self.workers = {} self.count = config.GRID[0] * config.GRID[1] self.logger.info('Generating points...') self.points = utils.get_points_per_worker(gen_alts=True) self.cell_ids = [{} for _ in range(self.count)] self.logger.info('Done') self.start_date = datetime.now() self.status_bar = status_bar self.things_count = [] self.killed = False self.loop = loop self.db_processor = DatabaseProcessor() self.cell_ids_executor = ThreadPoolExecutor(config.COMPUTE_THREADS) self.network_executor = ThreadPoolExecutor(config.NETWORK_THREADS) self.logger.info('Overseer initialized')
def spawn_workers(workers, status_bar=True): points = utils.get_points_per_worker() count = len(points) start_date = datetime.now() for worker_no in range(count): start_worker(worker_no, points[worker_no]) lenghts = [len(p) for p in points] points_stats = { 'max': max(lenghts), 'min': min(lenghts), 'avg': sum(lenghts) / float(len(lenghts)), } last_cleaned_cache = time.time() last_workers_checked = time.time() workers_check = [ (worker, worker.total_seen) for worker in workers.values() if worker.running ] while True: now = time.time() # Clean cache if now - last_cleaned_cache > (15 * 60): # clean cache db.CACHE.clean_expired() last_cleaned_cache = now # Check up on workers if now - last_workers_checked > (5 * 60): # Kill those not doing anything for worker, total_seen in workers_check: if not worker.running: continue if worker.total_seen <= total_seen: worker.kill() # Prepare new list workers_check = [ (worker, worker.total_seen) for worker in workers.values() ] last_workers_checked = now if status_bar: if sys.platform == 'win32': _ = os.system('cls') else: _ = os.system('clear') print(get_status_message(workers, count, start_date, points_stats)) time.sleep(0.5)
def spawn_workers(workers, status_bar=True): points = utils.get_points_per_worker() start_time = datetime.now() count = config.GRID[0] * config.GRID[1] for worker_no in range(count): start_worker(worker_no, points[worker_no]) lenghts = [len(p) for p in points] points_stats = { 'max': max(lenghts), 'min': min(lenghts), 'avg': sum(lenghts) / float(len(lenghts)), } while True: if status_bar: sf = open('../status', 'w') sf.write(get_status_message(workers, count, start_time, points_stats)) sf.close() time.sleep(0.5)
def spawn_workers(workers, status_bar=True): points = utils.get_points_per_worker() start_time = datetime.now() count = config.GRID[0] * config.GRID[1] for worker_no in range(count): start_worker(worker_no, points[worker_no]) lenghts = [len(p) for p in points] points_stats = { 'max': max(lenghts), 'min': min(lenghts), 'avg': sum(lenghts) / float(len(lenghts)), } while True: if status_bar: if sys.platform == 'win32': _ = os.system('cls') else: _ = os.system('clear') print get_status_message(workers, count, start_time, points_stats) time.sleep(0.5)
def spawn_workers(workers, status_bar=True): points = utils.get_points_per_worker() start_date = datetime.now() count = config.GRID[0] * config.GRID[1] for worker_no in range(count): start_worker(worker_no, points[worker_no]) lenghts = [len(p) for p in points] points_stats = { 'max': max(lenghts), 'min': min(lenghts), 'avg': sum(lenghts) / float(len(lenghts)), } last_cleaned_cache = time.time() last_workers_checked = time.time() workers_check = [(worker, worker.total_seen) for worker in workers.values() if worker.running] while True: now = time.time() # Clean cache if now - last_cleaned_cache > (15 * 60): # clean cache db.SIGHTING_CACHE.clean_expired() last_cleaned_cache = now # Check up on workers if now - last_workers_checked > (5 * 60): # Kill those not doing anything for worker, total_seen in workers_check: if not worker.running: continue if worker.total_seen <= total_seen: worker.kill() # Prepare new list workers_check = [(worker, worker.total_seen) for worker in workers.values()] last_workers_checked = now if status_bar: if sys.platform == 'win32': _ = os.system('cls') else: _ = os.system('clear') print(get_status_message(workers, count, start_date, points_stats)) time.sleep(0.5)
def get_worker_markers(): markers = [] points = utils.get_points_per_worker() # Worker start points for worker_no, worker_points in enumerate(points): coords = utils.get_start_coords(worker_no) markers.append({ 'lat': coords[0], 'lon': coords[1], 'type': 'worker', 'worker_no': worker_no, }) # Circles for i, point in enumerate(worker_points): markers.append({ 'lat': point[0], 'lon': point[1], 'type': 'worker_point', 'worker_no': worker_no, 'point_no': i, }) return markers