def get_retriever(cls, retriever_pk): if retriever_pk not in cls._visual_retriever: dr = Retriever.objects.get(pk=retriever_pk) cls._retriever_object[retriever_pk] = dr if dr.algorithm == Retriever.EXACT and dr.approximator_shasum and dr.approximator_shasum.strip(): approximator, da = Approximators.get_approximator_by_shasum(dr.approximator_shasum) da.ensure() approximator.load() cls._visual_retriever[retriever_pk] = retriever.BaseRetriever(name=dr.name, approximator=approximator) elif dr.algorithm == Retriever.EXACT: cls._visual_retriever[retriever_pk] = retriever.BaseRetriever(name=dr.name) elif dr.algorithm == Retriever.FAISS and dr.approximator_shasum is None: di = Indexers.get_indexer_by_shasum(dr.indexer_shasum) cls._visual_retriever[retriever_pk] = retriever.FaissFlatRetriever(name=dr.name, components=di.arguments[ 'components']) elif dr.algorithm == Retriever.FAISS: approximator, da = Approximators.get_approximator_by_shasum(dr.approximator_shasum) da.ensure() approximator.load() cls._visual_retriever[retriever_pk] = retriever.FaissApproximateRetriever(name=dr.name, approximator=approximator) elif dr.algorithm == Retriever.LOPQ: approximator, da = Approximators.get_approximator_by_shasum(dr.approximator_shasum) da.ensure() approximator.load() cls._visual_retriever[retriever_pk] = retriever.LOPQRetriever(name=dr.name, approximator=approximator) else: raise ValueError("{} not valid retriever algorithm".format(dr.algorithm)) return cls._visual_retriever[retriever_pk], cls._retriever_object[retriever_pk]
def get_retriever(cls, args): selector = args['retriever_selector'] if str(selector) in cls._selector_to_dr: dr = cls._selector_to_dr[str(selector)] else: dr = Retriever.objects.get(**selector) cls._selector_to_dr[str(selector)] = dr retriever_pk = dr.pk if retriever_pk not in cls._visual_retriever: cls._retriever_object[retriever_pk] = dr if dr.algorithm == Retriever.EXACT and dr.approximator_shasum and dr.approximator_shasum.strip( ): approximator, da = Approximators.get_trained_model({ "trainedmodel_selector": { "shasum": dr.approximator_shasum } }) da.ensure() approximator.load() cls._visual_retriever[ retriever_pk] = retriever.SimpleRetriever( name=dr.name, approximator=approximator) elif dr.algorithm == Retriever.EXACT: cls._visual_retriever[ retriever_pk] = retriever.SimpleRetriever(name=dr.name) elif dr.algorithm == Retriever.FAISS and dr.approximator_shasum is None: _, di = Indexers.get_trained_model( {"trainedmodel_selector": { "shasum": dr.indexer_shasum }}) cls._visual_retriever[ retriever_pk] = retriever.FaissFlatRetriever( name=dr.name, components=di.arguments['components']) elif dr.algorithm == Retriever.FAISS: approximator, da = Approximators.get_trained_model({ "trainedmodel_selector": { "shasum": dr.approximator_shasum } }) da.ensure() approximator.load() cls._visual_retriever[ retriever_pk] = retriever.FaissApproximateRetriever( name=dr.name, approximator=approximator) elif dr.algorithm == Retriever.LOPQ: approximator, da = Approximators.get_trained_model({ "trainedmodel_selector": { "shasum": dr.approximator_shasum } }) da.ensure() approximator.load() cls._visual_retriever[retriever_pk] = retriever.LOPQRetriever( name=dr.name, approximator=approximator) else: raise ValueError("{} not valid retriever algorithm".format( dr.algorithm)) return cls._visual_retriever[retriever_pk], cls._retriever_object[ retriever_pk]
def get_retriever(cls, retriever_pk): if retriever_pk not in cls._visual_retriever: dr = Retriever.objects.get(pk=retriever_pk) cls._retriever_object[retriever_pk] = dr if dr.algorithm == Retriever.EXACT and dr.approximator_shasum and dr.approximator_shasum.strip( ): approximator, da = Approximators.get_approximator_by_shasum( dr.approximator_shasum) da.ensure() approximator.load() cls._visual_retriever[retriever_pk] = retriever.BaseRetriever( name=dr.name, approximator=approximator) elif dr.algorithm == Retriever.EXACT: cls._visual_retriever[retriever_pk] = retriever.BaseRetriever( name=dr.name) elif dr.algorithm == Retriever.LOPQ: approximator, da = Approximators.get_approximator_by_shasum( dr.approximator_shasum) da.ensure() approximator.load() cls._visual_retriever[retriever_pk] = retriever.LOPQRetriever( name=dr.name, approximator=approximator) else: raise ValueError, "{} not valid retriever algorithm".format( dr.algorithm) return cls._visual_retriever[retriever_pk], cls._retriever_object[ retriever_pk]
def perform_retriever_creation(cluster_task_id, test=False): start = TEvent.objects.get(pk=cluster_task_id) if start.started: return 0 # to handle celery bug with ACK in SOLO mode else: start.started = True start.save() dc = Retriever.objects.get(pk=start.arguments['retriever_pk']) dc.create_directory() dc.arguments['fnames'] = [ i.npy_path() for i in IndexEntries.objects.filter(**dc.source_filters) ] dc.arguments['proto_filename'] = dc.proto_filename() c = retriever.LOPQRetriever(name=dc.name,args=dc.arguments) c.cluster() cluster_codes = [] for e in c.entries: cc = LOPQCodes() cc.retriever_id = dc.pk cc.video_id = e['video_primary_key'] if 'detection_primary_key' in e: cc.detection_id = e['detection_primary_key'] cc.frame_id = Region.objects.get(pk=cc.detection_id).frame_id else: cc.frame_id = e['frame_primary_key'] cc.clusters = dc cc.coarse = e['coarse'] cc.fine = e['fine'] cc.coarse_text = " ".join(map(str, e['coarse'])) cc.fine_text = " ".join(map(str, e['fine'])) cc.searcher_index = e['index'] cluster_codes.append(cc) LOPQCodes.objects.bulk_create(cluster_codes) c.save() dc.completed = True dc.save() mark_as_completed(start)
def get_retriever(self,retriever_pk): if retriever_pk not in RetrieverTask._visual_retriever: dr = Retriever.objects.get(pk=retriever_pk) RetrieverTask._retriever_object[retriever_pk] = dr if dr.algorithm == Retriever.EXACT: RetrieverTask._visual_retriever[retriever_pk] = retriever.BaseRetriever(name=dr.name) elif dr.algorithm == Retriever.LOPQ: dr.arguments['proto_filename'] = dr.proto_filename() RetrieverTask._visual_retriever[retriever_pk] = retriever.LOPQRetriever(name=dr.name,args=dr.arguments) RetrieverTask._visual_retriever[retriever_pk].load() else: raise ValueError,"{} not valid retriever algorithm".format(dr.algorithm) return RetrieverTask._visual_retriever[retriever_pk], RetrieverTask._retriever_object[retriever_pk]
def perform_retriever_creation(cluster_task_id, test=False): start = TEvent.objects.get(pk=cluster_task_id) if start.started: return 0 # to handle celery bug with ACK in SOLO mode else: start.started = True start.save() dc = Retriever.objects.get(pk=start.arguments['retriever_pk']) dc.create_directory() c = retriever.LOPQRetriever(name=dc.name,args=dc.arguments, proto_filename=dc.proto_filename()) for i in IndexEntries.objects.filter(**dc.source_filters): c.load_index(np.load(i.npy_path()),i.entries_path()) c.cluster() c.save() dc.last_built = timezone.now() dc.completed = True dc.save() mark_as_completed(start)
def perform_retriever_creation(cluster_task_id, test=False): start = TEvent.objects.get(pk=cluster_task_id) if start.started: return 0 # to handle celery bug with ACK in SOLO mode else: start.started = True start.save() dc = Retriever.objects.get(pk=start.arguments['retriever_pk']) dc.create_directory() fnames = [] for i in IndexEntries.objects.filter(**dc.source_filters): fnames.append("{}/{}/indexes/{}".format(settings.MEDIA_ROOT, i.video.pk, i.features_file_name)) c = retriever.LOPQRetriever(name=dc.name, args=dc.arguments) c.cluster() cluster_codes = [] for e in c.entries: cc = ClusterCodes() cc.video_id = e['video_primary_key'] if 'detection_primary_key' in e: cc.detection_id = e['detection_primary_key'] cc.frame_id = Region.objects.get(pk=cc.detection_id).frame_id else: cc.frame_id = e['frame_primary_key'] cc.clusters = dc cc.coarse = e['coarse'] cc.fine = e['fine'] cc.coarse_text = " ".join(map(str, e['coarse'])) cc.fine_text = " ".join(map(str, e['fine'])) cc.searcher_index = e['index'] cluster_codes.append(cc) ClusterCodes.objects.bulk_create(cluster_codes) c.save() dc.completed = True dc.save() mark_as_completed(start)