def perform_retrieval(task_id): dt = get_and_check_task(task_id) if dt is None: return 0 args = dt.arguments target = args.get('target', 'query') # by default target is query if target == 'query': vector = np.load( io.BytesIO(redis_client.get("query_vector_{}".format( dt.parent_id)))) Retrievers.retrieve(dt, args.get('retriever_pk', 20), vector, args.get('count', 20)) elif target == 'query_region_index_vectors': qr_pk_vector = redis_client.hgetall("query_region_vectors_{}".format( dt.parent_id)) for query_region_pk, vector in qr_pk_vector.items(): vector = np.load(io.BytesIO(vector)) Retrievers.retrieve(dt, args.get('retriever_pk', 20), vector, args.get('count', 20), region_pk=query_region_pk) else: raise NotImplementedError(target) mark_as_completed(dt) return 0
def perform_retrieval(task_id): start = models.TEvent.objects.get(pk=task_id) if start.started: return 0 # to handle celery bug with ACK in SOLO mode elif start.queue.startswith( settings.GLOBAL_RETRIEVER) and global_model_retriever.defer(start): logging.info("rerouting...") return 0 else: start.started = True start.save() args = start.arguments target = args.get('target', 'query') # by default target is query if target == 'query': vector = np.load(io.BytesIO(redis_client.get(start.parent_id))) Retrievers.retrieve(start, args.get('retriever_pk', 20), vector, args.get('count', 20)) elif target == 'query_region_index_vectors': queryset, target = task_shared.build_queryset(args=args) for dr in queryset: vector = np.load(io.BytesIO(dr.vector)) Retrievers.retrieve(start, args.get('retriever_pk', 20), vector, args.get('count', 20), region=dr.query_region) else: raise NotImplementedError(target) mark_as_completed(start) return 0
def get_from_cache(path): """ :param path: :return: """ if not path.startswith('/'): path = "/{}".format(path) if cacheable(path): body = redis_client.get(path) return body return None
def download_and_get_query_path(start): local_path = "{}/queries/{}_{}.png".format(settings.MEDIA_ROOT, start.pk, start.parent_process.uuid) if not os.path.isfile(local_path): source_path = "/queries/{}.png".format(start.parent_process.uuid) image_data = redis_client.get(source_path) if image_data: with open(local_path, 'w') as fh: fh.write(str(image_data)) else: ensure(source_path,safe=True) shutil.copy("{}{}".format(settings.MEDIA_ROOT,source_path),local_path) return local_path
def perform_retrieval(task_id): dt = get_and_check_task(task_id) if dt is None: return 0 args = dt.arguments target = args.get('target', 'query') # by default target is query if target == 'query': vector = np.load(io.BytesIO(redis_client.get(dt.parent_id))) Retrievers.retrieve(dt, args.get('retriever_pk', 20), vector, args.get('count', 20)) elif target == 'query_region_index_vectors': queryset, target = task_shared.build_queryset(args=args) for dr in queryset: vector = np.load(io.BytesIO(dr.vector)) Retrievers.retrieve(dt, args.get('retriever_pk', 20), vector, args.get('count', 20), region=dr.query_region) else: raise NotImplementedError(target) mark_as_completed(dt) return 0