def get_worker_markers():
markers = []
points = utils.get_points_per_worker()
# Worker start points
for worker_no, worker_points in enumerate(points):
coords = utils.get_start_coords(worker_no)
markers.append({
'icon': icons.dots.green,
'lat': coords[0],
'lng': coords[1],
'infobox': 'Worker %d' % worker_no,
'type': 'custom',
'subtype': 'worker',
'key': 'start-position-%d' % worker_no,
'disappear_time': -1
})
# Circles
for i, point in enumerate(worker_points):
markers.append({
'lat': point[0],
'lng': point[1],
'infobox': 'Worker %d point %d' % (worker_no, i),
'subtype': 'point',
})
return markers
def get_worker_markers():
markers = []
points = utils.get_points_per_worker()
# Worker start points
for worker_no, worker_points in enumerate(points):
coords = utils.get_start_coords(worker_no)
markers.append({
'icon': icons.dots.green,
'lat': coords[0],
'lng': coords[1],
'infobox': 'Worker %d' % worker_no,
'type': 'custom',
'subtype': 'worker',
'key': 'start-position-%d' % worker_no,
'disappear_time': -1
})
# Circles
for i, point in enumerate(worker_points):
markers.append({
'lat': point[0],
'lng': point[1],
'infobox': 'Worker %d point %d' % (worker_no, i),
'subtype': 'point',
})
return markers
def __init__(self, status_bar, loop):
self.logger = logging.getLogger('overseer')
self.workers = {}
self.count = config.GRID[0] * config.GRID[1]
self.logger.info('Generating points...')
self.points = utils.get_points_per_worker(gen_alts=True)
self.cell_ids = [{} for _ in range(self.count)]
self.logger.info('Done')
self.start_date = datetime.now()
self.status_bar = status_bar
self.things_count = []
self.killed = False
self.loop = loop
self.db_processor = DatabaseProcessor()
self.cell_ids_executor = ThreadPoolExecutor(config.COMPUTE_THREADS)
self.network_executor = ThreadPoolExecutor(config.NETWORK_THREADS)
self.logger.info('Overseer initialized')
def spawn_workers(workers, status_bar=True):
points = utils.get_points_per_worker()
count = len(points)
start_date = datetime.now()
for worker_no in range(count):
start_worker(worker_no, points[worker_no])
lenghts = [len(p) for p in points]
points_stats = {
'max': max(lenghts),
'min': min(lenghts),
'avg': sum(lenghts) / float(len(lenghts)),
}
last_cleaned_cache = time.time()
last_workers_checked = time.time()
workers_check = [
(worker, worker.total_seen) for worker in workers.values()
if worker.running
]
while True:
now = time.time()
# Clean cache
if now - last_cleaned_cache > (15 * 60): # clean cache
db.CACHE.clean_expired()
last_cleaned_cache = now
# Check up on workers
if now - last_workers_checked > (5 * 60):
# Kill those not doing anything
for worker, total_seen in workers_check:
if not worker.running:
continue
if worker.total_seen <= total_seen:
worker.kill()
# Prepare new list
workers_check = [
(worker, worker.total_seen) for worker in workers.values()
]
last_workers_checked = now
if status_bar:
if sys.platform == 'win32':
_ = os.system('cls')
else:
_ = os.system('clear')
print(get_status_message(workers, count, start_date, points_stats))
time.sleep(0.5)
def spawn_workers(workers, status_bar=True):
points = utils.get_points_per_worker()
start_time = datetime.now()
count = config.GRID[0] * config.GRID[1]
for worker_no in range(count):
start_worker(worker_no, points[worker_no])
lenghts = [len(p) for p in points]
points_stats = {
'max': max(lenghts),
'min': min(lenghts),
'avg': sum(lenghts) / float(len(lenghts)),
}
while True:
if status_bar:
sf = open('../status', 'w')
sf.write(get_status_message(workers, count, start_time, points_stats))
sf.close()
time.sleep(0.5)
def spawn_workers(workers, status_bar=True):
points = utils.get_points_per_worker()
start_time = datetime.now()
count = config.GRID[0] * config.GRID[1]
for worker_no in range(count):
start_worker(worker_no, points[worker_no])
lenghts = [len(p) for p in points]
points_stats = {
'max': max(lenghts),
'min': min(lenghts),
'avg': sum(lenghts) / float(len(lenghts)),
}
while True:
if status_bar:
if sys.platform == 'win32':
_ = os.system('cls')
else:
_ = os.system('clear')
print get_status_message(workers, count, start_time, points_stats)
time.sleep(0.5)
def spawn_workers(workers, status_bar=True):
points = utils.get_points_per_worker()
start_time = datetime.now()
count = config.GRID[0] * config.GRID[1]
for worker_no in range(count):
start_worker(worker_no, points[worker_no])
lenghts = [len(p) for p in points]
points_stats = {
'max': max(lenghts),
'min': min(lenghts),
'avg': sum(lenghts) / float(len(lenghts)),
}
while True:
if status_bar:
if sys.platform == 'win32':
_ = os.system('cls')
else:
_ = os.system('clear')
print get_status_message(workers, count, start_time, points_stats)
time.sleep(0.5)
def spawn_workers(workers, status_bar=True):
points = utils.get_points_per_worker()
start_date = datetime.now()
count = config.GRID[0] * config.GRID[1]
for worker_no in range(count):
start_worker(worker_no, points[worker_no])
lenghts = [len(p) for p in points]
points_stats = {
'max': max(lenghts),
'min': min(lenghts),
'avg': sum(lenghts) / float(len(lenghts)),
}
last_cleaned_cache = time.time()
last_workers_checked = time.time()
workers_check = [(worker, worker.total_seen)
for worker in workers.values() if worker.running]
while True:
now = time.time()
# Clean cache
if now - last_cleaned_cache > (15 * 60): # clean cache
db.SIGHTING_CACHE.clean_expired()
last_cleaned_cache = now
# Check up on workers
if now - last_workers_checked > (5 * 60):
# Kill those not doing anything
for worker, total_seen in workers_check:
if not worker.running:
continue
if worker.total_seen <= total_seen:
worker.kill()
# Prepare new list
workers_check = [(worker, worker.total_seen)
for worker in workers.values()]
last_workers_checked = now
if status_bar:
if sys.platform == 'win32':
_ = os.system('cls')
else:
_ = os.system('clear')
print(get_status_message(workers, count, start_date, points_stats))
time.sleep(0.5)
def get_worker_markers():
markers = []
points = utils.get_points_per_worker()
# Worker start points
for worker_no, worker_points in enumerate(points):
coords = utils.get_start_coords(worker_no)
markers.append({
'lat': coords[0],
'lon': coords[1],
'type': 'worker',
'worker_no': worker_no,
})
# Circles
for i, point in enumerate(worker_points):
markers.append({
'lat': point[0],
'lon': point[1],
'type': 'worker_point',
'worker_no': worker_no,
'point_no': i,
})
return markers
def get_worker_markers():
markers = []
points = utils.get_points_per_worker()
# Worker start points
for worker_no, worker_points in enumerate(points):
coords = utils.get_start_coords(worker_no)
markers.append({
'lat': coords[0],
'lon': coords[1],
'type': 'worker',
'worker_no': worker_no,
})
# Circles
for i, point in enumerate(worker_points):
markers.append({
'lat': point[0],
'lon': point[1],
'type': 'worker_point',
'worker_no': worker_no,
'point_no': i,
})
return markers