def _flush(self):
if len(self._pending_batch) > 0:
pipe = self._r.pipeline()
pipe.lpush(self._current_key, *self._pending_batch)
pipe.expire(self._current_key, self._ttl)
pipe.execute()
logger.info(
"Stored LabelBatch for Key: {k}".format(k=self._current_key))
self._pending_batch = []
def listen(self):
self._message = self._bus.receive_subscription_message(self.topic_name, self.subscription)
if self._message is not None and self._message.body is not None:
SB_IO_COUNTER.labels(direction='inbound',
kind=self._kind).inc()
try:
envelope = self._deserialize()
logger.info(
"Ingested Envelope with correlation id {cid}".format(
cid=envelope.correlation_id))
return envelope
except EOFError, ef:
logger.error("Poison message received: %s", ef)
self._bus.delete_subscription_message(self._message)
return None
def _workflow(self):
self._pano_event_envelope = self._subscriber.listen()
if self._pano_event_envelope is not None:
if self._pano_event_envelope.pano_upload is not None:
PANO_INGEST_COUNTER.labels(direction='inbound').inc()
try:
logger.info(
"Ingested PanoEventEnvelope with correlation id {cid}".format(
cid=self._pano_event_envelope.correlation_id))
return self.pano_service.download(self._pano_event_envelope.pano_upload)
except EOFError, ef:
logger.error("Poison message received: %s", ef)
self._subscriber.acknowledge()
else:
logger.warn("Irrelevant envelope with correlation id {cid}".format(
cid=self._pano_event_envelope.correlation_id))
self._subscriber.acknowledge()
def consume_n(self, n=100):
count = 0
while count < n:
envelope = self._client.listen()
if envelope is not None:
count += 1
batch = envelope.label_detected_batch
if batch is not None:
try:
self._redis_storage.push(batch)
except AzureException, aze:
logger.error(
"Unable to store AisleCompletionEventEntity %s",
aze)
else:
logger.info("Stored LabelDetectedBatch")
self._client.acknowledge()
envelope = None
else:
self._client.acknowledge()
else:
return
TF_SUCCESS_COUNT.inc()
return self.coordinates
except AzureException, ae:
# TODO more specific errors for transient exceptions
TF_FAILURE_COUNT.inc()
logger.error("Azure Exception ", exc_info=True)
except tf.OpError, oe:
TF_FAILURE_COUNT.inc()
logger.error("Tensorflow Exception", exc_info=True)
except WorkflowException, we:
TF_FAILURE_COUNT.inc()
logger.error("Workflow Exception {0}".format(we), exc_info=True)
except Exception, e:
TF_FAILURE_COUNT.inc()
logger.info("pano not processed", exc_info=True)
return None
def return_panos(byte_array):
# TODO need to write a iterator to deliver a pano
return byte_array
def load_graph(trained_model_env):
''' This method loads the graph '''
dir_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'tf_files', '{env}/')
dir_path = dir_path.format(env=trained_model_env.lower())
pb_graphs = [dir_path + TF.config('OOS_PB_FILE')]