def __call__(self, coords):
timing = {}
with time_block(timing, 'total'):
with time_block(timing, 'fetch'):
toi, is_toi_cached = self.tiles_of_interest()
with time_block(timing, 'intersect'):
coord_ints = convert_to_coord_ints(coords)
intersected_coord_ints, intersect_metrics = \
explode_and_intersect(coord_ints, toi)
coords = map(coord_unmarshall_int, intersected_coord_ints)
intersect_metrics['cached'] = is_toi_cached
return coords, intersect_metrics, timing
def __call__(self, coords):
timing = {}
with time_block(timing, 'total'):
with time_block(timing, 'fetch'):
toi, is_toi_cached = self.tiles_of_interest()
with time_block(timing, 'intersect'):
coord_ints = convert_to_coord_ints(coords)
intersected_coord_ints, intersect_metrics = \
explode_and_intersect(coord_ints, toi)
coords = map(coord_unmarshall_int, intersected_coord_ints)
intersect_metrics['cached'] = is_toi_cached
return coords, intersect_metrics, timing
def __call__(self, coords):
timing = dict(
fetch=0,
intersect=0,
)
with time_block(timing, 'intersect'):
all_coord_ints = set()
for coord in coords:
while coord.zoom >= self.zoom_stop_inclusive:
coord_int = coord_marshall_int(coord)
if coord_int in all_coord_ints:
# as an optimization, assume that if the coord is
# already in the set, then all its parents will be too
break
all_coord_ints.add(coord_int)
coord = coord.zoomBy(-1).container()
coords = imap(coord_unmarshall_int, all_coord_ints)
metrics = dict(
total=len(all_coord_ints),
hits=len(all_coord_ints),
misses=0,
n_toi=0,
cached=False,
)
return coords, metrics, timing
def __call__(self, coords):
timing = dict(
fetch=0,
intersect=0,
)
with time_block(timing, 'intersect'):
all_coord_ints = set()
for coord in coords:
while coord.zoom >= self.zoom_stop_inclusive:
coord_int = coord_marshall_int(coord)
if coord_int in all_coord_ints:
# as an optimization, assume that if the coord is
# already in the set, then all its parents will be too
break
all_coord_ints.add(coord_int)
coord = coord.zoomBy(-1).container()
coords = imap(coord_unmarshall_int, all_coord_ints)
metrics = dict(
total=len(all_coord_ints),
hits=len(all_coord_ints),
misses=0,
n_toi=0,
cached=False,
)
return coords, metrics, timing
def __call__(self):
self.rawr_proc_logger.lifecycle('Processing started')
import atexit
atexit.register(self._atexit_log)
while True:
timing = {}
try:
# NOTE: it's ok if reading from the queue takes a long time
with time_block(timing, 'queue_read'):
msg_handle = self.rawr_queue.read()
except Exception as e:
self.log_exception(e, 'queue read')
continue
if not msg_handle:
# this gets triggered when no messages are returned
continue
try:
coords = self.msg_marshaller.unmarshall(msg_handle.payload)
except Exception as e:
self.log_exception(e, 'unmarshall payload')
continue
# split coordinates into group by zoom and higher and low zoom
# the message payload is either coordinates that are at group by
# zoom and higher, or all below the group by zoom
is_low_zoom = False
did_rawr_tile_gen = False
for coord in coords:
if coord.zoom < self.group_by_zoom:
is_low_zoom = True
else:
assert not is_low_zoom, \
'Mix of low/high zoom coords in payload'
# check if we need to generate the rawr tile
# proceed directly to enqueueing the coordinates if not
if not is_low_zoom:
did_rawr_tile_gen = True
try:
parent = common_parent(coords, self.group_by_zoom)
except Exception as e:
self.log_exception(e, 'find parent')
continue
try:
rawr_tile_coord = convert_coord_object(parent)
except Exception as e:
self.log_exception(e, 'convert coord', parent)
continue
try:
rawr_gen_timing = {}
with time_block(rawr_gen_timing, 'total'):
# grab connection
with self.conn_ctx() as conn:
# commit transaction
with conn as conn:
# cleanup cursor resources
with conn.cursor() as cur:
table_reader = TableReader(cur)
rawr_gen_specific_timing = self.rawr_gen(
table_reader, rawr_tile_coord)
rawr_gen_timing.update(rawr_gen_specific_timing)
timing['rawr_gen'] = rawr_gen_timing
except Exception as e:
self.log_exception(e, 'rawr tile gen', parent)
continue
try:
with time_block(timing, 'queue_write'):
n_enqueued, n_inflight = \
self.queue_writer.enqueue_batch(coords)
except Exception as e:
self.log_exception(e, 'queue write', parent)
continue
try:
with time_block(timing, 'queue_done'):
self.rawr_queue.done(msg_handle)
except Exception as e:
self.log_exception(e, 'queue done', parent)
continue
try:
self.rawr_proc_logger.processed(n_enqueued, n_inflight,
did_rawr_tile_gen, timing,
parent)
except Exception as e:
self.log_exception(e, 'log', parent)
continue
try:
self.stats_handler(n_enqueued, n_inflight, did_rawr_tile_gen,
timing)
except Exception as e:
self.log_exception(e, 'stats', parent)
def __call__(self):
self.rawr_proc_logger.lifecycle('Processing started')
import atexit
atexit.register(self._atexit_log)
while True:
timing = {}
try:
# NOTE: it's ok if reading from the queue takes a long time
with time_block(timing, 'queue_read'):
msg_handle = self.rawr_queue.read()
except Exception as e:
self.log_exception(e, 'queue read')
continue
if not msg_handle:
# this gets triggered when no messages are returned
continue
try:
coords = self.msg_marshaller.unmarshall(msg_handle.payload)
except Exception as e:
self.log_exception(e, 'unmarshall payload')
continue
# split coordinates into group by zoom and higher and low zoom
# the message payload is either coordinates that are at group by
# zoom and higher, or all below the group by zoom
is_low_zoom = False
did_rawr_tile_gen = False
for coord in coords:
if coord.zoom < self.group_by_zoom:
is_low_zoom = True
else:
assert not is_low_zoom, \
'Mix of low/high zoom coords in payload'
# check if we need to generate the rawr tile
# proceed directly to enqueueing the coordinates if not
if not is_low_zoom:
did_rawr_tile_gen = True
try:
parent = common_parent(coords, self.group_by_zoom)
except Exception as e:
self.log_exception(e, 'find parent')
continue
try:
rawr_tile_coord = convert_coord_object(parent)
except Exception as e:
self.log_exception(e, 'convert coord', parent)
continue
try:
rawr_gen_timing = {}
with time_block(rawr_gen_timing, 'total'):
# grab connection
with self.conn_ctx() as conn:
# commit transaction
with conn as conn:
# cleanup cursor resources
with conn.cursor() as cur:
table_reader = TableReader(cur)
rawr_gen_specific_timing = self.rawr_gen(
table_reader, rawr_tile_coord)
rawr_gen_timing.update(rawr_gen_specific_timing)
timing['rawr_gen'] = rawr_gen_timing
except Exception as e:
self.log_exception(e, 'rawr tile gen', parent)
continue
try:
with time_block(timing, 'queue_write'):
n_enqueued, n_inflight = \
self.queue_writer.enqueue_batch(coords)
except Exception as e:
self.log_exception(e, 'queue write', parent)
continue
try:
with time_block(timing, 'queue_done'):
self.rawr_queue.done(msg_handle)
except Exception as e:
self.log_exception(e, 'queue done', parent)
continue
try:
self.rawr_proc_logger.processed(
n_enqueued, n_inflight, did_rawr_tile_gen, timing, parent)
except Exception as e:
self.log_exception(e, 'log', parent)
continue
try:
self.stats_handler(
n_enqueued, n_inflight, did_rawr_tile_gen, timing)
except Exception as e:
self.log_exception(e, 'stats', parent)