class ExtractionChecker(ConfReader):
"""ExtractionChecker class
"""
def __init__(self,
global_conf,
prefix=DEFAULT_EXTR_CHECK_PREFIX,
pid=None):
"""ExtractionChecker constructor
:param global_conf_in: configuration file or dictionary
:type global_conf_in: str, dict
:param prefix: prefix in configuration
:type prefix: str
:param pid: process id
:type pid: int
"""
self.list_extr_prefix = []
self.pid = pid
self.dict_sha1_infos = dict()
super(ExtractionChecker, self).__init__(global_conf, prefix)
self.last_push = time.time()
self.nb_imgs_check = 0
self.nb_imgs_unproc = 0
self.nb_imgs_unproc_lastprint = 0
self.featurizer_type = self.get_required_param("featurizer_type")
self.detector_type = self.get_required_param("detector_type")
self.input_type = self.get_required_param("input_type")
# Max delay
self.max_delay = int(
self.get_param("max_delay", default=DEFAULT_MAX_DELAY))
self.min_len_check = int(
self.get_param("min_len_check", default=DEFAULT_MIN_LENGTH_CHECK))
self.list_extr_prefix = [
self.featurizer_type, "feat", self.detector_type, self.input_type
]
self.extr_prefix = "_".join(self.list_extr_prefix)
self.batch_check_column = None
self.check_columns = []
# changed to: get column family from indexer in set_check_columns
# Need to be build from extraction type and detection input + "_processed"
#self.extr_family_column = self.get_param("extr_family_column", default="ext")
# self.extr_prefix_base_column_name = self.extr_family_column + ":" + self.extr_prefix
# self.extr_check_column = self.extr_prefix_base_column_name + "_processed"
# # Need to be build from extraction type and extraction input + "_batchid"
# self.batch_check_column = self.extr_prefix_base_column_name + "_updateid"
# self.check_columns = [self.extr_check_column, self.batch_check_column]
self.set_pp()
# Initialize indexer
self.indexer = HBaseIndexerMinimal(
self.global_conf, prefix=self.get_required_param("indexer_prefix"))
self.indexer.pp = "CheckerHBase"
print(self.get_required_param("indexer_prefix"),
self.indexer.get_dictcf_sha1_table())
self.set_check_columns()
print(self.check_columns)
# Initialize ingester, that could now be Kafka or Kinesis. Should we have a default?
ingester_type = self.get_required_param("image_ingestion_type")
try:
if ingester_type == "kafka":
self.ingester = KafkaIngester(
self.global_conf,
prefix=self.get_required_param("check_ingester_prefix"))
elif ingester_type == "kinesis":
self.ingester = KinesisIngester(
self.global_conf,
prefix=self.get_required_param("check_ingester_prefix"))
else:
raise ValueError(
"Unknown 'ingester_type': {}".format(ingester_type))
except Exception as inst:
# print "Could not initialize checker, sleeping for {}s.".format(self.max_delay)
# time.sleep(self.max_delay)
# raise(inst)
#print("Could not initialize 'updates_out_topic' ({}). Will write only to HBase.".format(inst))
print("[{}: ERROR] Could not start ingester.".format(
self.pp, inst))
raise inst
# Initialize producer
# TODO: also check for 'update_ingestion_type' as producer_type?
producer_type = self.get_param("update_ingestion_type",
DEFAULT_UPDATE_INGESTION_TYPE)
# TODO: create a producer if 'update_ingestion_type' is Kinesis or Kafka
# if producer_type != "hbase":
# self.updates_out_topic = self.ingester.get_required_param("producer_updates_out_topic")
if producer_type == "kafka":
self.pusher = KafkaPusher(
self.global_conf,
prefix=self.get_required_param("check_ingester_prefix"))
elif producer_type == "kinesis":
self.pusher = KinesisPusher(
self.global_conf,
prefix=self.get_required_param("check_ingester_prefix"))
elif producer_type == "hbase":
self.pusher = None
print("[{}: log] Will write updates only to HBase.".format(
self.pp))
else:
raise ValueError(
"Unknown 'producer_type': {}".format(producer_type))
#self.ingester.pp = self.get_param("pp", "ImageIngester")
# Only if daemon mode, as we may have multiple ingesters
# But for Kinesis the `shard_infos_filename` may not be re-used...
#if self.pid:
# self.ingester.pp += str(self.pid)
def set_check_columns(self):
"""Set columns to be checked in indexer
"""
# changed to: get column family from indexer
# TODO: get the suffixes as global variables maybe from common.defaults
extr_prefix_base_column_name = self.indexer.extrcf + ":" + self.extr_prefix
extr_check_column = extr_prefix_base_column_name + "_processed"
# Need to be build from extraction type and extraction input + "_batchid"
self.batch_check_column = extr_prefix_base_column_name + "_updateid"
self.check_columns = [extr_check_column, self.batch_check_column]
#print(self.check_columns)
def set_pp(self, pp=""):
"""Set pretty name
"""
self.pp = "ExtractionChecker"
self.pp += "-".join(self.list_extr_prefix)
if self.pid:
self.pp += "." + str(self.pid)
def store_img_infos(self, msg):
"""Store information about the images of ``msg`` in ``self.dict_sha1_infos``
:param msg: message
:type msg: dict
"""
strk = str(msg['sha1']).upper()
self.dict_sha1_infos[strk] = dict()
for key in msg:
# dumps json of 'img_info'
# We actually need that only for DIG...
if key == "img_info":
self.dict_sha1_infos[strk][key] = json.dumps(msg[key])
else:
# discard 'img_buffer' (if it exists?...), and 'sha1'
# if k != "img_buffer" and k != "sha1":
# self.dict_sha1_infos[strk][k] = msg[k]
# discard 'sha1'
if key != "sha1":
self.dict_sha1_infos[strk][key] = msg[key]
def cleanup_dict_infos(self, list_del_sha1s):
"""Remove images ``list_del_sha1s`` from ``self.dict_sha1_infos``
:param list_del_sha1s: list of images sha1 to remove
:type list_del_sha1s: list
"""
for sha1 in list_del_sha1s:
try:
del self.dict_sha1_infos[str(sha1)]
except:
# could happen when cleaning up duplicates or image processed by another process
pass
def get_dict_push(self, list_get_sha1s, daemon=False):
"""Get dictionary to be pushed to HBase for images in ``list_get_sha1s``
:param list_get_sha1s: list of images
:type list_get_sha1s: list
:param daemon: whether the checker is running in daemon mode
:type daemon: bool
:return: (dict_push, update_id)
:rtype: tuple
"""
#TODO: is this needed for every get_dict_push call?
self.set_check_columns()
# TODO: also pass current update_id, and move the creation of update id out of this method
# this method should actually be used to 'claim' an image as soon as we can.
dict_push = dict()
# append processid to 'update_id' for safe use with multiple consumers, even after restart
# /!\ beware, it should not contain underscores
tmp_update_id, _ = self.indexer.get_next_update_id(
today=None, extr_type=self.extr_prefix)
update_id = tmp_update_id + '-' + self.ingester.pp + '-' + str(
time.time())
for sha1 in list_get_sha1s:
dict_push[str(sha1)] = dict()
try:
tmp_dict = self.dict_sha1_infos[str(sha1)]
except:
# This would mean the image has been marked as part of another batch by another process,
# and thus deleted in a previous 'get_unprocessed_rows' call
# This is also only relevant if we run on Daemon mode...
# TODO: for transition we won't really have any info to push except the update_id...
if daemon:
del dict_push[str(sha1)]
continue
# build column names properly i.e. appending 'info:'
for key in tmp_dict:
# changed to: use column_family from indexer
# But the use of 'key' here also means we rely on the input to define column name...
#dict_push[str(sha1)]['info:' + key] = tmp_dict[key]
dict_push[str(sha1)][self.indexer.imginfocf + ':' +
key] = tmp_dict[key]
dict_push[str(sha1)][self.batch_check_column] = update_id
return dict_push, update_id
def get_unprocessed_rows(self, list_check_sha1s):
"""Get the subset of the list of sha1s ``list_check_sha1s`` that have not been processed yet
:param list_check_sha1s: list of images sha1 to check
:type list_check_sha1s: list
:return: set of unprocessed images
:rtype: set
"""
# TODO: also pass current update_id and only delete if != from current update...
unprocessed_rows = set(list_check_sha1s)
if list_check_sha1s:
# Check if the selected sha1 rows in HBase table 'sha1infos' have those check_column
# This call will only return rows that DO have those check_column
fam = self.indexer.get_dictcf_sha1_table()
try:
sha1s_rows = self.indexer.get_columns_from_sha1_rows(
list_check_sha1s, self.check_columns, families=fam)
except Exception as inst:
print("[{}.get_unprocessed_rows: log] fam: {}".format(
self.pp, fam))
raise inst
#families=self.tablesha1_col_families)
if sha1s_rows:
# TODO: only delete if really previously processed, i.e. if != from current update...
found_sha1_rows = set([str(row[0]) for row in sha1s_rows])
# Clean up 'dict_sha1_infos' deleting found_sha1_rows
self.cleanup_dict_infos(found_sha1_rows)
set_list_check_sha1s = set(list_check_sha1s)
# TODO: but we should not re-add them, so we should discard them from unprocessed_rows
unprocessed_rows = set_list_check_sha1s - found_sha1_rows
return unprocessed_rows
def run(self, daemon=False):
"""Run extraction checker
:param daemon: whether we are running in daemon mode
:type daemon: bool
:raises Exception: if check fails
"""
# import inspect
# if not inspect.isgeneratorfunction(self.ingester.get_msg_json()):
# msg = "[{}: Warning] Ingester {} function `get_msg_json` is not a generator"
# print(msg.format(self.pp, type(self.ingester)))
try:
list_sha1s_to_process = []
list_check_sha1s = []
# TODO: create update_id here
if self.verbose > 1:
msg = "[{}: log] Start run main loop"
msg.format(self.pp)
while True:
try:
# Accumulate images infos
#while len(list_check_sha1s) < self.indexer.batch_update_size:
#while len(list_check_sha1s) < self.min_len_check:
for msg in self.ingester.get_msg_json():
try:
# Fix if input was JSON dumped twice?
if not isinstance(msg, dict):
msg = json.loads(msg)
# msg could now contain keys 'sha1' or 'list_sha1s'
if 'sha1' in msg:
list_check_sha1s.append(
str(msg['sha1']).upper())
# Store other fields to be able to push them too
self.store_img_infos(msg)
elif 'list_sha1s' in msg:
for sha1 in msg['list_sha1s']:
list_check_sha1s.append(str(sha1).upper())
# We won't have any additional infos no?
# But we should still build a dict for each sample for consistency...
tmp_dict = dict()
tmp_dict['sha1'] = str(sha1).upper()
# will basically push a dict with just the sha1 to self.dict_sha1_infos, so self.get_dict_push
# works properly later on...
self.store_img_infos(tmp_dict)
else:
raise ValueError(
'Unknown keys in msg: {}'.format(
msg.keys()))
# This is dangerous, as it assumes the self.ingester.get_msg_json() generator
# would restart from the next point... Is this the case for Kafka?
prev_len = len(list_check_sha1s)
list_check_sha1s = list(set(list_check_sha1s))
if len(list_check_sha1s) < prev_len:
msg = "[{}: log] Removed {} duplicate from `list_check_sha1s`"
print(
msg.format(
self.pp,
prev_len - len(list_check_sha1s)))
if len(list_check_sha1s
) >= self.indexer.batch_update_size:
break
except Exception as inst:
pr_msg = "[{}: ERROR] Could not process message: {}. {}"
print(pr_msg.format(self.pp, msg, inst))
except Exception as inst:
pr_msg = "[{}: at {} ERROR] Caught {} {} in consumer loop"
now_str = datetime.now().strftime('%Y-%m-%d:%H.%M.%S')
print(pr_msg.format(self.pp, now_str, type(inst), inst))
if msg is not None:
print(msg)
sys.stdout.flush()
if self.verbose > 3:
msg = "[{}: log] Gathered {} images to check so far"
msg = msg.format(self.pp, len(list_check_sha1s))
msg2 = ""
if len(list_check_sha1s) > 0:
msg2 = " (first: {}, last: {})"
msg2 = msg2.format(list_check_sha1s[0],
list_check_sha1s[-1])
print(msg + msg2)
# To be able to push one (non empty) update every max_delay
#if not list_check_sha1s and (time.time() - self.last_push) < self.max_delay:
if len(list_check_sha1s) < self.indexer.batch_update_size and (
time.time() - self.last_push) < self.max_delay:
time.sleep(1)
continue
self.nb_imgs_check += len(list_check_sha1s)
push_delay = (time.time() - self.last_push) > max(
int(self.max_delay / 60), 10)
if push_delay and self.nb_imgs_unproc_lastprint != self.nb_imgs_unproc:
msg = "[{}: log] Pushed {} unprocessed images so far"
print(
msg.format(self.pp, self.nb_imgs_unproc,
self.nb_imgs_check))
self.nb_imgs_unproc_lastprint = self.nb_imgs_unproc
if list_check_sha1s:
# Check which images have not been processed (or pushed in an update) yet
# This seems slow
start_check = time.time()
unprocessed_rows = self.get_unprocessed_rows(
list_check_sha1s)
msg = "[{}: log] Found {}/{} unprocessed images in {:.2f}s"
print(
msg.format(self.pp, len(unprocessed_rows),
len(list_check_sha1s),
time.time() - start_check))
if len(unprocessed_rows) != len(
list_check_sha1s) and self.verbose > 5:
already_processed = list(
set(list_check_sha1s) - set(unprocessed_rows))
msg = "[{}: log] Images ".format(self.pp)
for ap in already_processed:
msg += "{} ".format(ap)
msg += "were already processed."
print(msg)
#unprocessed_rows = self.get_unprocessed_rows(list_check_sha1s)
# TODO: we should mark those images as being 'owned' by the update we are constructing
# (only important if we are running multiple threads i.e. daemon is True)
# otherwise another update running at the same time could also claim it (in another ad)
# could be handle when adding data to the searcher but duplicates in extraction process...
# Push sha1s to be processed
for sha1 in unprocessed_rows:
list_sha1s_to_process.append(sha1)
# Remove potential duplicates
list_sha1s_to_process = list(set(list_sha1s_to_process))
list_check_sha1s = []
if list_sha1s_to_process:
# Push them to HBase by batch of 'batch_update_size'
push_delay = (time.time() -
self.last_push) > self.max_delay
full_batch = len(list_sha1s_to_process
) >= self.indexer.batch_update_size
if full_batch or (push_delay and list_sha1s_to_process):
# Trim here to push exactly a batch of 'batch_update_size'
list_push = list_sha1s_to_process[:min(
self.indexer.
batch_update_size, len(list_sha1s_to_process))]
# TODO: this should be done before,
# to 'claim' the images as soon as we plan to process them for this update
# Gather corresponding sha1 infos
dict_push, update_id = self.get_dict_push(
list_push, daemon=daemon)
if dict_push:
self.nb_imgs_unproc += len(dict_push.keys())
msg = "[{}: at {}] Pushing update {} of {} images."
now_str = datetime.now().strftime(
'%Y-%m-%d:%H.%M.%S')
print(
msg.format(self.pp, now_str, update_id,
len(dict_push.keys())))
sys.stdout.flush()
# Push images
fam = self.indexer.get_dictcf_sha1_table()
if self.verbose > 5:
msg = "[{}] Pushing images for update {} with fam {}"
print(msg.format(self.pp, update_id, fam))
sha1s_table = self.indexer.table_sha1infos_name
self.indexer.push_dict_rows(dict_push,
sha1s_table,
families=fam)
# Build HBase updates dict
dict_updates_db = dict()
now_str = datetime.now().strftime(
'%Y-%m-%d:%H.%M.%S')
list_sha1s_col = self.indexer.get_col_listsha1s()
dict_updates_db[update_id] = {
list_sha1s_col: ','.join(dict_push.keys()),
self.indexer.get_col_upcreate(): now_str
}
# Push it
fam = self.indexer.get_dictcf_update_table()
if self.verbose > 5:
msg = "[{}] Pushing update {} info with fam {}"
print(msg.format(self.pp, update_id, fam))
self.indexer.push_dict_rows(
dict_updates_db,
self.indexer.table_updateinfos_name,
families=fam)
# Build pusher updates dict
if self.pusher is not None:
dict_updates_kafka = dict()
dict_updates_kafka[update_id] = ','.join(
dict_push.keys())
# Push it
#self.ingester.producer.send(self.updates_out_topic, json.dumps(dict_updates_kafka))
#self.pusher.send(self.updates_out_topic, dict_updates_kafka)
self.pusher.send(dict_updates_kafka)
# Gather any remaining sha1s and clean up infos
if len(list_sha1s_to_process
) > self.indexer.batch_update_size:
list_sha1s_to_process = list_sha1s_to_process[
self.indexer.batch_update_size:]
else:
list_sha1s_to_process = []
# if duplicates wrt list_push, remove them. Can this still happen?
list_sha1s_to_process = [
sh1 for sh1 in list_sha1s_to_process
if sh1 not in list_push
]
self.cleanup_dict_infos(list_push)
else:
msg = "[{}: at {}] Nothing to push for update {}"
print(
msg.format(
self.pp,
datetime.now().strftime(
'%Y-%m-%d:%H.%M.%S'), update_id))
sys.stdout.flush()
self.last_push = time.time()
# TODO: we should create a new update_id here,
# and let it claim the potential remaining images in 'list_sha1s_to_process'
# sanity check that len(list_sha1s_to_process) == len(self.dict_sha1_infos) ?
else:
if self.verbose > 3:
msg = "[{}: at {}] Gathered {} images so far..."
now_str = datetime.now().strftime(
'%Y-%m-%d:%H.%M.%S')
print(
msg.format(self.pp, now_str,
len(list_sha1s_to_process)))
except Exception as inst:
exc_type, exc_obj, exc_tb = sys.exc_info()
fulltb = traceback.format_tb(exc_tb)
raise type(inst)(" {} ({})".format(inst, ''.join(fulltb)))
class ExtractionChecker(ConfReader):
"""ExtractionChecker class
"""
def __init__(self, global_conf, prefix=DEFAULT_EXTR_CHECK_PREFIX, pid=None):
"""ExtractionChecker constructor
:param global_conf_in: configuration file or dictionary
:type global_conf_in: str, dict
:param prefix: prefix in configuration
:type prefix: str
:param pid: process id
:type pid: int
"""
self.list_extr_prefix = []
self.pid = pid
self.dict_sha1_infos = dict()
super(ExtractionChecker, self).__init__(global_conf, prefix)
self.last_push = time.time()
self.nb_imgs_check = 0
self.nb_imgs_unproc = 0
self.nb_imgs_unproc_lastprint = 0
self.featurizer_type = self.get_required_param("featurizer_type")
self.detector_type = self.get_required_param("detector_type")
self.input_type = self.get_required_param("input_type")
# Max delay
self.max_delay = int(self.get_param("max_delay", default=3600))
self.list_extr_prefix = [self.featurizer_type, "feat", self.detector_type, self.input_type]
self.extr_prefix = "_".join(self.list_extr_prefix)
self.batch_check_column = None
self.check_columns = []
# changed to: get column family from indexer in set_check_columns
# Need to be build from extraction type and detection input + "_processed"
#self.extr_family_column = self.get_param("extr_family_column", default="ext")
# self.extr_prefix_base_column_name = self.extr_family_column + ":" + self.extr_prefix
# self.extr_check_column = self.extr_prefix_base_column_name + "_processed"
# # Need to be build from extraction type and extraction input + "_batchid"
# self.batch_check_column = self.extr_prefix_base_column_name + "_updateid"
# self.check_columns = [self.extr_check_column, self.batch_check_column]
self.set_pp()
# Initialize indexer
self.indexer = HBaseIndexerMinimal(self.global_conf,
prefix=self.get_required_param("indexer_prefix"))
self.indexer.pp = "CheckerHBase"
print(self.get_required_param("indexer_prefix"), self.indexer.get_dictcf_sha1_table())
self.set_check_columns()
print(self.check_columns)
# Initialize ingester
try:
self.ingester = GenericKafkaProcessor(self.global_conf,
prefix=self.get_required_param("check_ingester_prefix"))
except Exception as inst:
# print "Could not initialize checker, sleeping for {}s.".format(self.max_delay)
# time.sleep(self.max_delay)
# raise(inst)
#print("Could not initialize 'updates_out_topic' ({}). Will write only to HBase.".format(inst))
print("[{}: ERROR] Could not start ingester.".format(self.pp, inst))
raise inst
# This will not be set for HBase processing, but checker would keep dying here...
self.updates_out_topic = None
try:
self.updates_out_topic = self.ingester.get_required_param("producer_updates_out_topic")
except Exception as inst:
# print "Could not initialize checker, sleeping for {}s.".format(self.max_delay)
# time.sleep(self.max_delay)
# raise(inst)
#print("Could not initialize 'updates_out_topic' ({}). Will write only to HBase.".format(inst))
print("{}. Will write only to HBase.".format(inst))
self.ingester.pp = "ec"
if self.pid:
self.ingester.pp += str(self.pid)
def set_check_columns(self):
"""Set columns to be checked in indexer
"""
# changed to: get column family from indexer
extr_prefix_base_column_name = self.indexer.extrcf + ":" + self.extr_prefix
extr_check_column = extr_prefix_base_column_name + "_processed"
# Need to be build from extraction type and extraction input + "_batchid"
self.batch_check_column = extr_prefix_base_column_name + "_updateid"
self.check_columns = [extr_check_column, self.batch_check_column]
#print(self.check_columns)
def set_pp(self, pp=""):
"""Set pretty name
"""
self.pp = "ExtractionChecker"
self.pp += "-".join(self.list_extr_prefix)
if self.pid:
self.pp += "." + str(self.pid)
def store_img_infos(self, msg):
"""Store information about the images of ``msg`` in ``self.dict_sha1_infos``
:param msg: Kafka record
:type msg: collections.namedtuple
"""
# msg is technically a ConsumerRecord that is a collections.namedtuple, see:
# https://github.com/dpkp/kafka-python/blob/master/kafka/consumer/fetcher.py#L30
strk = str(msg['sha1'])
self.dict_sha1_infos[strk] = dict()
for key in msg:
# dumps json of 'img_info'
# We actually need that only for DIG...
if key == "img_info":
self.dict_sha1_infos[strk][key] = json.dumps(msg[key])
else:
# discard 'img_buffer' (if it exists?...), and 'sha1'
# if k != "img_buffer" and k != "sha1":
# self.dict_sha1_infos[strk][k] = msg[k]
# discard 'sha1'
if key != "sha1":
self.dict_sha1_infos[strk][key] = msg[key]
def cleanup_dict_infos(self, list_del_sha1s):
"""Remove images ``list_del_sha1s`` from ``self.dict_sha1_infos``
:param list_del_sha1s: list of images sha1 to remove
:type list_del_sha1s: list
"""
for sha1 in list_del_sha1s:
try:
del self.dict_sha1_infos[str(sha1)]
except:
# could happen when cleaning up duplicates or image processed by another process
pass
def get_dict_push(self, list_get_sha1s, daemon=False):
"""Get dictionary to be pushed to HBase for images in ``list_get_sha1s``
:param list_get_sha1s: list of images
:type list_get_sha1s: list
:param daemon: whether the checker is running in daemon mode
:type daemon: bool
:return: (dict_push, update_id)
:rtype: tuple
"""
#TODO: is this needed for every get_dict_push call?
self.set_check_columns()
# TODO: also pass current update_id, and move the creation of update id out of this method
# this method should actually be used to 'claim' an image as soon as we can.
dict_push = dict()
# append processid to 'update_id' for safe use with multiple consumers, even after restart
# /!\ beware, it should not contain underscores
tmp_update_id, _ = self.indexer.get_next_update_id(today=None, extr_type=self.extr_prefix)
update_id = tmp_update_id + '-' + self.ingester.pp + '-' + str(time.time())
for sha1 in list_get_sha1s:
dict_push[str(sha1)] = dict()
try:
tmp_dict = self.dict_sha1_infos[str(sha1)]
except:
# This would mean the image has been marked as part of another batch by another process,
# and thus deleted in a previous 'get_unprocessed_rows' call
# This is also only relevant if we run on Daemon mode...
# TODO: for transition we won't really have any info to push except the update_id...
if daemon:
del dict_push[str(sha1)]
continue
# build column names properly i.e. appending 'info:'
for key in tmp_dict:
# changed to: use column_family from indexer
# But the use of 'key' here also means we rely on the input to define column name...
#dict_push[str(sha1)]['info:' + key] = tmp_dict[key]
dict_push[str(sha1)][self.indexer.imginfocf + ':' + key] = tmp_dict[key]
dict_push[str(sha1)][self.batch_check_column] = update_id
return dict_push, update_id
def get_unprocessed_rows(self, list_check_sha1s):
"""Get the subset of the list of sha1s ``list_check_sha1s`` that have not been processed yet
:param list_check_sha1s: list of images sha1 to check
:type list_check_sha1s: list
:return: set of unprocessed images
:rtype: set
"""
# TODO: also pass current update_id and only delete if != from current update...
unprocessed_rows = set(list_check_sha1s)
if list_check_sha1s:
# Check if the selected sha1 rows in HBase table 'sha1infos' have those check_column
# This call will only return rows that DO have those check_column
fam = self.indexer.get_dictcf_sha1_table()
try:
sha1s_rows = self.indexer.get_columns_from_sha1_rows(list_check_sha1s, self.check_columns,
families=fam)
except Exception as inst:
print("[{}.get_unprocessed_rows: log] fam: {}".format(self.pp, fam))
raise inst
#families=self.tablesha1_col_families)
if sha1s_rows:
# TODO: only delete if really previously processed, i.e. if != from current update...
found_sha1_rows = set([str(row[0]) for row in sha1s_rows])
# Clean up 'dict_sha1_infos' deleting found_sha1_rows
self.cleanup_dict_infos(found_sha1_rows)
set_list_check_sha1s = set(list_check_sha1s)
# TODO: but we should not re-add them, so we should discard them from unprocessed_rows
unprocessed_rows = set_list_check_sha1s - found_sha1_rows
return unprocessed_rows
def run(self, daemon=False):
"""Run extraction checker
:param daemon: whether we are running in daemon mode
:type daemon: bool
:raises Exception: if check fails
"""
i = 0
try:
list_sha1s_to_process = []
# TODO: create update_id here
while True:
list_check_sha1s = []
try:
# Accumulate images infos
for msg_json in self.ingester.consumer:
msg = json.loads(msg_json.value)
# i += 1
# print((i, len(list_check_sha1s), msg))
# msg could now contain keys 'sha1' or 'list_sha1s'
# should we check that we can't have both or other keys?...
if 'sha1' in msg:
list_check_sha1s.append(str(msg['sha1']))
# Store other fields to be able to push them too
self.store_img_infos(msg)
elif 'list_sha1s' in msg:
for sha1 in msg['list_sha1s']:
list_check_sha1s.append(str(sha1))
# We won't have any additional infos no?
# But we should still build a dict for each sample for consistency...
tmp_dict = dict()
tmp_dict['sha1'] = str(sha1)
# will basically push an empty dict to self.dict_sha1_infos, so self.get_dict_push
# works properly later on...
self.store_img_infos(tmp_dict)
else:
print('Unknown keys in msg: {}'.format(msg.keys()))
if len(list_check_sha1s) >= self.indexer.batch_update_size:
break
except Exception as inst:
# trying to use 'consumer_timeout_ms' to raise timeout and get last samples
msg = "[{}: warning] At {}, caught {} {} in consumer loop"
now_str = datetime.now().strftime('%Y-%m-%d:%H.%M.%S')
print(msg.format(self.pp, now_str, type(inst), inst))
sys.stdout.flush()
if not list_check_sha1s:
# TODO: should we fallback to scanning Hbase table here?
continue
# Check which images have not been processed (or pushed in an update) yet
unprocessed_rows = self.get_unprocessed_rows(list_check_sha1s)
self.nb_imgs_check += len(list_check_sha1s)
push_delay = (time.time() - self.last_push) > self.max_delay / 60
if push_delay and self.nb_imgs_unproc_lastprint != self.nb_imgs_unproc:
msg = "[{}: log] Found {}/{} unprocessed images"
print(msg.format(self.pp, self.nb_imgs_unproc, self.nb_imgs_check))
self.nb_imgs_unproc_lastprint = self.nb_imgs_unproc
# TODO: we should mark those images as being 'owned' by the update we are constructing
# (only important if we are running multiple threads i.e. daemon is True)
# otherwise another update running at the same time could also claim it (in another ad)
# could be handle when adding data to the searcher but duplicates in extraction process...
# Push sha1s to be processed
for sha1 in unprocessed_rows:
list_sha1s_to_process.append(sha1)
# Remove potential duplicates
list_sha1s_to_process = list(set(list_sha1s_to_process))
if list_sha1s_to_process:
# Push them to HBase by batch of 'batch_update_size'
push_delay = (time.time() - self.last_push) > self.max_delay
full_batch = len(list_sha1s_to_process) >= self.indexer.batch_update_size
if full_batch or (push_delay and list_sha1s_to_process):
# Trim here to push exactly a batch of 'batch_update_size'
list_push = list_sha1s_to_process[:min(self.indexer.batch_update_size,
len(list_sha1s_to_process))]
# TODO: this should be done before,
# to 'claim' the images as soon as we plan to process them for this update
# Gather corresponding sha1 infos
dict_push, update_id = self.get_dict_push(list_push, daemon=daemon)
if dict_push:
self.nb_imgs_unproc += len(dict_push.keys())
msg = "[{}: at {}] Pushing update {} of {} images."
now_str = datetime.now().strftime('%Y-%m-%d:%H.%M.%S')
print(msg.format(self.pp, now_str, update_id, len(dict_push.keys())))
sys.stdout.flush()
# Push images
fam = self.indexer.get_dictcf_sha1_table()
if self.verbose > 4:
msg = "[{}] Pushing images for update {} with fam {}"
print(msg.format(self.pp, update_id, fam))
sha1s_table = self.indexer.table_sha1infos_name
self.indexer.push_dict_rows(dict_push, sha1s_table, families=fam)
# Build HBase updates dict
dict_updates_db = dict()
now_str = datetime.now().strftime('%Y-%m-%d:%H.%M.%S')
list_sha1s_col = self.indexer.get_col_listsha1s()
dict_updates_db[update_id] = {list_sha1s_col: ','.join(dict_push.keys()),
self.indexer.get_col_upcreate(): now_str}
# Push it
fam = self.indexer.get_dictcf_update_table()
if self.verbose > 4:
msg = "[{}] Pushing update {} info with fam {}"
print(msg.format(self.pp, update_id, fam))
self.indexer.push_dict_rows(dict_updates_db, self.indexer.table_updateinfos_name,
families=fam)
# Build HBase updates dict
if self.updates_out_topic is not None:
dict_updates_kafka = dict()
dict_updates_kafka[update_id] = ','.join(dict_push.keys())
# Push it
self.ingester.producer.send(self.updates_out_topic, json.dumps(dict_updates_kafka))
# Gather any remaining sha1s and clean up infos
if len(list_sha1s_to_process) > self.indexer.batch_update_size:
list_sha1s_to_process = list_sha1s_to_process[self.indexer.batch_update_size:]
else:
list_sha1s_to_process = []
# if duplicates wrt list_push, remove them. Can this still happen?
list_sha1s_to_process = [sh1 for sh1 in list_sha1s_to_process if sh1 not in list_push]
self.cleanup_dict_infos(list_push)
else:
msg = "[{}: at {}] Nothing to push for update {}"
print(msg.format(self.pp, datetime.now().strftime('%Y-%m-%d:%H.%M.%S'), update_id))
sys.stdout.flush()
self.last_push = time.time()
# TODO: we should create a new update_id here,
# and let it claim the potential remaining images in 'list_sha1s_to_process'
# sanity check that len(list_sha1s_to_process) == len(self.dict_sha1_infos) ?
except Exception as inst:
exc_type, exc_obj, exc_tb = sys.exc_info()
fulltb = traceback.format_tb(exc_tb)
raise type(inst)(" {} ({})".format(inst, ''.join(fulltb)))
class ExtractionChecker(ConfReader):
def __init__(self, global_conf, prefix=default_extr_check_prefix, pid=None):
self.list_extr_prefix = []
self.pid = pid
self.dict_sha1_infos = dict()
super(ExtractionChecker, self).__init__(global_conf, prefix)
self.featurizer_type = self.get_required_param("featurizer_type")
self.detector_type = self.get_required_param("detector_type")
self.input_type = self.get_required_param("input_type")
# Need to be build from extraction type and detection input + "_processed"
self.extr_family_column = "ext"
tmp_extr_family_column = self.get_param("extr_family_column")
if tmp_extr_family_column:
self.extr_family_column = tmp_extr_family_column
# Max delay
self.max_delay = 3600
# self.max_delay = 600
max_delay = self.get_param("max_delay")
if max_delay:
self.max_delay = int(max_delay)
self.last_push = time.time()
self.nb_imgs_check = 0
self.nb_imgs_unproc = 0
self.nb_imgs_unproc_lastprint = 0
# Beware, the self.extr_family_column should be added to the indexer families parameter in get_create_table...
# TODO: should we add the 'ad' column family too here by default
self.tablesha1_col_families = {'info': dict(), self.extr_family_column: dict()}
self.list_extr_prefix = [self.featurizer_type, "feat", self.detector_type, self.input_type]
self.extr_prefix = "_".join(self.list_extr_prefix)
self.extr_prefix_base_column_name = self.extr_family_column + ":" + self.extr_prefix
self.extr_check_column = self.extr_prefix_base_column_name + "_processed"
# Need to be build from extraction type and extraction input + "_batchid"
self.batch_check_column = self.extr_prefix_base_column_name + "_updateid"
self.check_columns = [self.extr_check_column, self.batch_check_column]
self.set_pp()
# Initialize indexer and ingester
self.indexer = HBaseIndexerMinimal(self.global_conf,
prefix=self.get_required_param("indexer_prefix"))
self.ingester = GenericKafkaProcessor(self.global_conf,
prefix=self.get_required_param("check_ingester_prefix"))
# This will not be set for HBase processing, but checker would keep dying here...
try:
self.updates_out_topic = self.ingester.get_required_param("producer_updates_out_topic")
except Exception as inst:
print "Could not initialize checker, sleeping for {}s.".format(self.max_delay)
time.sleep(self.max_delay)
raise(inst)
self.ingester.pp = "ec"
if self.pid:
self.ingester.pp += str(self.pid)
def set_pp(self):
self.pp = "ExtractionChecker."
self.pp += "-".join(self.list_extr_prefix)
if self.pid:
self.pp += "." + str(self.pid)
def store_img_infos(self, msg):
strk = str(msg['sha1'])
self.dict_sha1_infos[strk] = dict()
for k in msg:
# dumps json of 'img_info'
if k == "img_info":
self.dict_sha1_infos[strk][k] = json.dumps(msg[k])
else:
# discard 'img_buffer' (if it exists?...), and 'sha1'
# if k != "img_buffer" and k != "sha1":
# self.dict_sha1_infos[strk][k] = msg[k]
# discard 'sha1'
if k != "sha1":
self.dict_sha1_infos[strk][k] = msg[k]
def cleanup_dict_infos(self, list_del_sha1s):
for sha1 in list_del_sha1s:
try:
del self.dict_sha1_infos[str(sha1)]
except:
# could happen when cleaning up duplicates or image processed by another process
pass
def get_dict_push(self, list_get_sha1s):
# TODO: also pass current update_id, and move the creation of update id out of this method
# this method should actually be used to 'claim' an image as soon as we can.
dict_push = dict()
# append unique processid to 'update_id' to make it safe to use with multiple consumers, even after a restart.
# /!\ beware, it should not contain underscores
tmp_update_id, _ = self.indexer.get_next_update_id(today=None, extr_type=self.extr_prefix)
update_id = tmp_update_id + '-' + self.ingester.pp + '-' + str(time.time())
for sha1 in list_get_sha1s:
dict_push[str(sha1)] = dict()
try:
tmp_dict = self.dict_sha1_infos[str(sha1)]
except:
# This would mean the image has been marked as part of another batch by another process,
# and thus deleted in a previous 'get_unprocessed_rows' call
del dict_push[str(sha1)]
continue
# build column names properly i.e. appending 'info:'
for k in tmp_dict:
dict_push[str(sha1)]['info:' + k] = tmp_dict[k]
dict_push[str(sha1)][self.batch_check_column] = update_id
return dict_push, update_id
def get_unprocessed_rows(self, list_sha1s_to_check):
# TODO: also pass current update_id and only delete if != from current update...
unprocessed_rows = set(list_sha1s_to_check)
if list_sha1s_to_check:
# Check if the selected sha1 rows in HBase table 'sha1infos' have those check_column
# This call will only return rows that DO have those check_column
sha1s_rows = self.indexer.get_columns_from_sha1_rows(list_sha1s_to_check, self.check_columns,
families=self.tablesha1_col_families)
if sha1s_rows:
# TODO: only delete if really previously processed, i.e. if != from current update...
found_sha1_rows = set([str(row[0]) for row in sha1s_rows])
# Clean up 'dict_sha1_infos' deleting found_sha1_rows
self.cleanup_dict_infos(found_sha1_rows)
set_list_sha1s_to_check = set(list_sha1s_to_check)
# TODO: but we should not re-add them, so we should discard them from unprocessed_rows
unprocessed_rows = set_list_sha1s_to_check - found_sha1_rows
return unprocessed_rows
def run(self):
try:
list_sha1s_to_process = []
# TODO: create update_id here
while True:
list_sha1s_to_check = []
try:
# Accumulate images infos
for msg_json in self.ingester.consumer:
msg = json.loads(msg_json.value)
list_sha1s_to_check.append(str(msg['sha1']))
# Store other fields to be able to push them too
self.store_img_infos(msg)
if len(list_sha1s_to_check) >= self.indexer.batch_update_size:
break
except Exception as inst:
# trying to use 'consumer_timeout_ms' to raise timeout and get last samples
warn_msg = "[{}: warning] At {}, caught {} {} in consumer loop"
print warn_msg.format(self.pp, datetime.now().strftime('%Y-%m-%d:%H.%M.%S'), type(inst), inst)
sys.stdout.flush()
# Check which images have not been processed (or pushed in an update) yet
unprocessed_rows = self.get_unprocessed_rows(list_sha1s_to_check)
self.nb_imgs_check += len(list_sha1s_to_check)
if (
time.time() - self.last_push) > self.max_delay / 60 and self.nb_imgs_unproc_lastprint != self.nb_imgs_unproc:
msg_log = "[{}: log] Found {}/{} unprocessed images"
print msg_log.format(self.pp, self.nb_imgs_unproc, self.nb_imgs_check)
self.nb_imgs_unproc_lastprint = self.nb_imgs_unproc
# TODO: we should mark those images as being 'owned' by the update we are constructing
# (only reallyimportant if we are running multiple threads...)
# otherwise another update running at the same time could also claim it (if it appears in another ad)
# this can be handle when adding data to the searcher but induces duplicates in extraction process...
# Push sha1s to be processed
for sha1 in unprocessed_rows:
list_sha1s_to_process.append(sha1)
# Remove potential duplicates
list_sha1s_to_process = list(set(list_sha1s_to_process))
if list_sha1s_to_process:
# Push them to HBase by batch of 'batch_update_size'
if len(list_sha1s_to_process) >= self.indexer.batch_update_size or (
(time.time() - self.last_push) > self.max_delay and len(list_sha1s_to_process) > 0):
# Trim here to push exactly a batch of 'batch_update_size'
list_push = list_sha1s_to_process[:min(self.indexer.batch_update_size, len(list_sha1s_to_process))]
# TODO: this should be done before, to 'claim' the images as soon as we plan to process them for this update
# Gather corresponding sha1 infos
dict_push, update_id = self.get_dict_push(list_push)
if dict_push:
self.nb_imgs_unproc += len(dict_push.keys())
push_msg = "[{}: at {}] Pushing update {} of {} images."
print push_msg.format(self.pp, datetime.now().strftime('%Y-%m-%d:%H.%M.%S'), update_id,
len(dict_push.keys()))
sys.stdout.flush()
# Push images
self.indexer.push_dict_rows(dict_push, self.indexer.table_sha1infos_name,
families=self.tablesha1_col_families)
# Build updates dict
dict_updates_db = dict()
dict_updates_kafka = dict()
dict_updates_db[update_id] = {self.indexer.column_list_sha1s: ','.join(dict_push.keys()),
'info:' + update_str_created: datetime.now().strftime('%Y-%m-%d:%H.%M.%S')}
dict_updates_kafka[update_id] = ','.join(dict_push.keys())
# Push them
self.indexer.push_dict_rows(dict_updates_db, self.indexer.table_updateinfos_name)
self.ingester.producer.send(self.updates_out_topic, json.dumps(dict_updates_kafka))
# Gather any remaining sha1s and clean up infos
if len(list_sha1s_to_process) > self.indexer.batch_update_size:
list_sha1s_to_process = list_sha1s_to_process[self.indexer.batch_update_size:]
else:
list_sha1s_to_process = []
# if duplicates wrt list_push, remove them. Can this still happen?
list_sha1s_to_process = [sha1 for sha1 in list_sha1s_to_process if sha1 not in list_push]
self.cleanup_dict_infos(list_push)
else:
no_push_msg = "[{}: at {}] Nothing to push for update {}"
print no_push_msg.format(self.pp, datetime.now().strftime('%Y-%m-%d:%H.%M.%S'), update_id)
sys.stdout.flush()
self.last_push = time.time()
# TODO: we should create a new update_id here, and let it claim the potential remaining images in 'list_sha1s_to_process'
# sanity check that len(list_sha1s_to_process) == len(self.dict_sha1_infos) ?
except Exception as inst:
exc_type, exc_obj, exc_tb = sys.exc_info()
fulltb = traceback.format_tb(exc_tb)
raise type(inst)(" {} ({})".format(inst, ''.join(fulltb)))