def perform_retrieval(task_id):
dt = get_and_check_task(task_id)
if dt is None:
return 0
args = dt.arguments
target = args.get('target', 'query') # by default target is query
if target == 'query':
vector = np.load(
io.BytesIO(redis_client.get("query_vector_{}".format(
dt.parent_id))))
Retrievers.retrieve(dt, args.get('retriever_pk', 20), vector,
args.get('count', 20))
elif target == 'query_region_index_vectors':
qr_pk_vector = redis_client.hgetall("query_region_vectors_{}".format(
dt.parent_id))
for query_region_pk, vector in qr_pk_vector.items():
vector = np.load(io.BytesIO(vector))
Retrievers.retrieve(dt,
args.get('retriever_pk', 20),
vector,
args.get('count', 20),
region_pk=query_region_pk)
else:
raise NotImplementedError(target)
mark_as_completed(dt)
return 0
def perform_retrieval(task_id):
start = models.TEvent.objects.get(pk=task_id)
if start.started:
return 0 # to handle celery bug with ACK in SOLO mode
elif start.queue.startswith(
settings.GLOBAL_RETRIEVER) and global_model_retriever.defer(start):
logging.info("rerouting...")
return 0
else:
start.started = True
start.save()
args = start.arguments
target = args.get('target', 'query') # by default target is query
if target == 'query':
vector = np.load(io.BytesIO(redis_client.get(start.parent_id)))
Retrievers.retrieve(start, args.get('retriever_pk', 20), vector,
args.get('count', 20))
elif target == 'query_region_index_vectors':
queryset, target = task_shared.build_queryset(args=args)
for dr in queryset:
vector = np.load(io.BytesIO(dr.vector))
Retrievers.retrieve(start,
args.get('retriever_pk', 20),
vector,
args.get('count', 20),
region=dr.query_region)
else:
raise NotImplementedError(target)
mark_as_completed(start)
return 0
def get_from_cache(path):
"""
:param path:
:return:
"""
if not path.startswith('/'):
path = "/{}".format(path)
if cacheable(path):
body = redis_client.get(path)
return body
return None
def download_and_get_query_path(start):
local_path = "{}/queries/{}_{}.png".format(settings.MEDIA_ROOT, start.pk, start.parent_process.uuid)
if not os.path.isfile(local_path):
source_path = "/queries/{}.png".format(start.parent_process.uuid)
image_data = redis_client.get(source_path)
if image_data:
with open(local_path, 'w') as fh:
fh.write(str(image_data))
else:
ensure(source_path,safe=True)
shutil.copy("{}{}".format(settings.MEDIA_ROOT,source_path),local_path)
return local_path
def perform_retrieval(task_id):
dt = get_and_check_task(task_id)
if dt is None:
return 0
args = dt.arguments
target = args.get('target', 'query') # by default target is query
if target == 'query':
vector = np.load(io.BytesIO(redis_client.get(dt.parent_id)))
Retrievers.retrieve(dt, args.get('retriever_pk', 20), vector, args.get('count', 20))
elif target == 'query_region_index_vectors':
queryset, target = task_shared.build_queryset(args=args)
for dr in queryset:
vector = np.load(io.BytesIO(dr.vector))
Retrievers.retrieve(dt, args.get('retriever_pk', 20), vector, args.get('count', 20),
region=dr.query_region)
else:
raise NotImplementedError(target)
mark_as_completed(dt)
return 0