def get_retriever(cls, retriever_pk):
if retriever_pk not in cls._visual_retriever:
dr = Retriever.objects.get(pk=retriever_pk)
cls._retriever_object[retriever_pk] = dr
if dr.algorithm == Retriever.EXACT and dr.approximator_shasum and dr.approximator_shasum.strip():
approximator, da = Approximators.get_approximator_by_shasum(dr.approximator_shasum)
da.ensure()
approximator.load()
cls._visual_retriever[retriever_pk] = retriever.BaseRetriever(name=dr.name, approximator=approximator)
elif dr.algorithm == Retriever.EXACT:
cls._visual_retriever[retriever_pk] = retriever.BaseRetriever(name=dr.name)
elif dr.algorithm == Retriever.FAISS and dr.approximator_shasum is None:
di = Indexers.get_indexer_by_shasum(dr.indexer_shasum)
cls._visual_retriever[retriever_pk] = retriever.FaissFlatRetriever(name=dr.name,
components=di.arguments[
'components'])
elif dr.algorithm == Retriever.FAISS:
approximator, da = Approximators.get_approximator_by_shasum(dr.approximator_shasum)
da.ensure()
approximator.load()
cls._visual_retriever[retriever_pk] = retriever.FaissApproximateRetriever(name=dr.name,
approximator=approximator)
elif dr.algorithm == Retriever.LOPQ:
approximator, da = Approximators.get_approximator_by_shasum(dr.approximator_shasum)
da.ensure()
approximator.load()
cls._visual_retriever[retriever_pk] = retriever.LOPQRetriever(name=dr.name,
approximator=approximator)
else:
raise ValueError("{} not valid retriever algorithm".format(dr.algorithm))
return cls._visual_retriever[retriever_pk], cls._retriever_object[retriever_pk]
def get_retriever(cls, args):
selector = args['retriever_selector']
if str(selector) in cls._selector_to_dr:
dr = cls._selector_to_dr[str(selector)]
else:
dr = Retriever.objects.get(**selector)
cls._selector_to_dr[str(selector)] = dr
retriever_pk = dr.pk
if retriever_pk not in cls._visual_retriever:
cls._retriever_object[retriever_pk] = dr
if dr.algorithm == Retriever.EXACT and dr.approximator_shasum and dr.approximator_shasum.strip(
):
approximator, da = Approximators.get_trained_model({
"trainedmodel_selector": {
"shasum": dr.approximator_shasum
}
})
da.ensure()
approximator.load()
cls._visual_retriever[
retriever_pk] = retriever.SimpleRetriever(
name=dr.name, approximator=approximator)
elif dr.algorithm == Retriever.EXACT:
cls._visual_retriever[
retriever_pk] = retriever.SimpleRetriever(name=dr.name)
elif dr.algorithm == Retriever.FAISS and dr.approximator_shasum is None:
_, di = Indexers.get_trained_model(
{"trainedmodel_selector": {
"shasum": dr.indexer_shasum
}})
cls._visual_retriever[
retriever_pk] = retriever.FaissFlatRetriever(
name=dr.name, components=di.arguments['components'])
elif dr.algorithm == Retriever.FAISS:
approximator, da = Approximators.get_trained_model({
"trainedmodel_selector": {
"shasum": dr.approximator_shasum
}
})
da.ensure()
approximator.load()
cls._visual_retriever[
retriever_pk] = retriever.FaissApproximateRetriever(
name=dr.name, approximator=approximator)
elif dr.algorithm == Retriever.LOPQ:
approximator, da = Approximators.get_trained_model({
"trainedmodel_selector": {
"shasum": dr.approximator_shasum
}
})
da.ensure()
approximator.load()
cls._visual_retriever[retriever_pk] = retriever.LOPQRetriever(
name=dr.name, approximator=approximator)
else:
raise ValueError("{} not valid retriever algorithm".format(
dr.algorithm))
return cls._visual_retriever[retriever_pk], cls._retriever_object[
retriever_pk]
def get_retriever(cls, retriever_pk):
if retriever_pk not in cls._visual_retriever:
dr = Retriever.objects.get(pk=retriever_pk)
cls._retriever_object[retriever_pk] = dr
if dr.algorithm == Retriever.EXACT and dr.approximator_shasum and dr.approximator_shasum.strip(
):
approximator, da = Approximators.get_approximator_by_shasum(
dr.approximator_shasum)
da.ensure()
approximator.load()
cls._visual_retriever[retriever_pk] = retriever.BaseRetriever(
name=dr.name, approximator=approximator)
elif dr.algorithm == Retriever.EXACT:
cls._visual_retriever[retriever_pk] = retriever.BaseRetriever(
name=dr.name)
elif dr.algorithm == Retriever.LOPQ:
approximator, da = Approximators.get_approximator_by_shasum(
dr.approximator_shasum)
da.ensure()
approximator.load()
cls._visual_retriever[retriever_pk] = retriever.LOPQRetriever(
name=dr.name, approximator=approximator)
else:
raise ValueError, "{} not valid retriever algorithm".format(
dr.algorithm)
return cls._visual_retriever[retriever_pk], cls._retriever_object[
retriever_pk]
def perform_retriever_creation(cluster_task_id, test=False):
start = TEvent.objects.get(pk=cluster_task_id)
if start.started:
return 0 # to handle celery bug with ACK in SOLO mode
else:
start.started = True
start.save()
dc = Retriever.objects.get(pk=start.arguments['retriever_pk'])
dc.create_directory()
dc.arguments['fnames'] = [ i.npy_path() for i in IndexEntries.objects.filter(**dc.source_filters) ]
dc.arguments['proto_filename'] = dc.proto_filename()
c = retriever.LOPQRetriever(name=dc.name,args=dc.arguments)
c.cluster()
cluster_codes = []
for e in c.entries:
cc = LOPQCodes()
cc.retriever_id = dc.pk
cc.video_id = e['video_primary_key']
if 'detection_primary_key' in e:
cc.detection_id = e['detection_primary_key']
cc.frame_id = Region.objects.get(pk=cc.detection_id).frame_id
else:
cc.frame_id = e['frame_primary_key']
cc.clusters = dc
cc.coarse = e['coarse']
cc.fine = e['fine']
cc.coarse_text = " ".join(map(str, e['coarse']))
cc.fine_text = " ".join(map(str, e['fine']))
cc.searcher_index = e['index']
cluster_codes.append(cc)
LOPQCodes.objects.bulk_create(cluster_codes)
c.save()
dc.completed = True
dc.save()
mark_as_completed(start)
def get_retriever(self,retriever_pk):
if retriever_pk not in RetrieverTask._visual_retriever:
dr = Retriever.objects.get(pk=retriever_pk)
RetrieverTask._retriever_object[retriever_pk] = dr
if dr.algorithm == Retriever.EXACT:
RetrieverTask._visual_retriever[retriever_pk] = retriever.BaseRetriever(name=dr.name)
elif dr.algorithm == Retriever.LOPQ:
dr.arguments['proto_filename'] = dr.proto_filename()
RetrieverTask._visual_retriever[retriever_pk] = retriever.LOPQRetriever(name=dr.name,args=dr.arguments)
RetrieverTask._visual_retriever[retriever_pk].load()
else:
raise ValueError,"{} not valid retriever algorithm".format(dr.algorithm)
return RetrieverTask._visual_retriever[retriever_pk], RetrieverTask._retriever_object[retriever_pk]
def perform_retriever_creation(cluster_task_id, test=False):
start = TEvent.objects.get(pk=cluster_task_id)
if start.started:
return 0 # to handle celery bug with ACK in SOLO mode
else:
start.started = True
start.save()
dc = Retriever.objects.get(pk=start.arguments['retriever_pk'])
dc.create_directory()
c = retriever.LOPQRetriever(name=dc.name,args=dc.arguments, proto_filename=dc.proto_filename())
for i in IndexEntries.objects.filter(**dc.source_filters):
c.load_index(np.load(i.npy_path()),i.entries_path())
c.cluster()
c.save()
dc.last_built = timezone.now()
dc.completed = True
dc.save()
mark_as_completed(start)
def perform_retriever_creation(cluster_task_id, test=False):
start = TEvent.objects.get(pk=cluster_task_id)
if start.started:
return 0 # to handle celery bug with ACK in SOLO mode
else:
start.started = True
start.save()
dc = Retriever.objects.get(pk=start.arguments['retriever_pk'])
dc.create_directory()
fnames = []
for i in IndexEntries.objects.filter(**dc.source_filters):
fnames.append("{}/{}/indexes/{}".format(settings.MEDIA_ROOT,
i.video.pk,
i.features_file_name))
c = retriever.LOPQRetriever(name=dc.name, args=dc.arguments)
c.cluster()
cluster_codes = []
for e in c.entries:
cc = ClusterCodes()
cc.video_id = e['video_primary_key']
if 'detection_primary_key' in e:
cc.detection_id = e['detection_primary_key']
cc.frame_id = Region.objects.get(pk=cc.detection_id).frame_id
else:
cc.frame_id = e['frame_primary_key']
cc.clusters = dc
cc.coarse = e['coarse']
cc.fine = e['fine']
cc.coarse_text = " ".join(map(str, e['coarse']))
cc.fine_text = " ".join(map(str, e['fine']))
cc.searcher_index = e['index']
cluster_codes.append(cc)
ClusterCodes.objects.bulk_create(cluster_codes)
c.save()
dc.completed = True
dc.save()
mark_as_completed(start)
