def handle_perform_indexing(start): json_args = start.arguments target = json_args.get('target', 'frames') if 'index' in json_args: index_name = json_args['index'] visual_index, di = indexing.Indexers.get_index_by_name(index_name) else: visual_index, di = indexing.Indexers.get_index_by_pk( json_args['indexer_pk']) sync = True if target == 'query': local_path = task_shared.download_and_get_query_path(start) vector = visual_index.apply(local_path) # TODO: figure out a better way to store numpy arrays. s = io.BytesIO() np.save(s, vector) redis_client.set("query_vector_{}".format(start.pk), s.getvalue()) sync = False elif target == 'query_regions': queryset, target = task_shared.build_queryset(args=start.arguments) region_paths = task_shared.download_and_get_query_region_path( start, queryset) for i, dr in enumerate(queryset): local_path = region_paths[i] vector = visual_index.apply(local_path) s = io.BytesIO() np.save(s, vector) redis_client.hset("query_region_vectors_{}".format(start.pk), dr.pk, s.getvalue()) sync = False elif target == 'regions': # For regions simply download/ensure files exists. queryset, target = task_shared.build_queryset(args=start.arguments, video_id=start.video_id) task_shared.ensure_files(queryset, target) indexing.Indexers.index_queryset(di, visual_index, start, target, queryset) elif target == 'frames': queryset, target = task_shared.build_queryset(args=start.arguments, video_id=start.video_id) if visual_index.cloud_fs_support and settings.ENABLE_CLOUDFS: # if NFS is disabled and index supports cloud file systems natively (e.g. like Tensorflow) indexing.Indexers.index_queryset(di, visual_index, start, target, queryset, cloud_paths=True) else: # Otherwise download and ensure that the files exist task_shared.ensure_files(queryset, target) indexing.Indexers.index_queryset(di, visual_index, start, target, queryset) return sync
def cache_path(path, payload=None, expire_in_seconds=600): if not path.startswith('/'): path = "/{}".format(path) if cacheable(path): if payload is None: with open('{}{}'.format(settings.MEDIA_ROOT, path), 'rb') as body: redis_client.set(path, body.read(), ex=expire_in_seconds, nx=True) else: redis_client.set(path, payload, ex=expire_in_seconds, nx=True) return True else: return False
def cache_path(path, expire_in_seconds=120): """ :param path: :return: """ if not path.startswith('/'): path = "/{}".format(path) if cacheable(path): with open('{}{}'.format(settings.MEDIA_ROOT, path), 'rb') as body: redis_client.set(path, body.read(), ex=expire_in_seconds, nx=True) return True else: return False
def handle_perform_indexing(start): json_args = start.arguments target = json_args.get('target', 'frames') if 'index' in json_args: index_name = json_args['index'] visual_index, di = indexing.Indexers.get_index_by_name(index_name) else: visual_index, di = indexing.Indexers.get_index_by_pk(json_args['indexer_pk']) sync = True if target == 'query': local_path = task_shared.download_and_get_query_path(start) vector = visual_index.apply(local_path) # TODO: figure out a better way to store numpy arrays. s = io.BytesIO() np.save(s, vector) redis_client.set(start.pk, s.getvalue()) sync = False elif target == 'query_regions': queryset, target = task_shared.build_queryset(args=start.arguments) region_paths = task_shared.download_and_get_query_region_path(start, queryset) for i, dr in enumerate(queryset): local_path = region_paths[i] vector = visual_index.apply(local_path) s = io.BytesIO() np.save(s, vector) # can be replaced by Redis instead of using DB redis_client.hset(start.pk, dr.pk, s.getvalue()) _ = models.QueryRegionIndexVector.objects.create(vector=s.getvalue(), event=start, query_region=dr) sync = False elif target == 'regions': # For regions simply download/ensure files exists. queryset, target = task_shared.build_queryset(args=start.arguments, video_id=start.video_id) task_shared.ensure_files(queryset, target) indexing.Indexers.index_queryset(di, visual_index, start, target, queryset) elif target == 'frames': queryset, target = task_shared.build_queryset(args=start.arguments, video_id=start.video_id) if visual_index.cloud_fs_support and settings.ENABLE_CLOUDFS and (not settings.KUBE_MODE): # TODO Re-enable this in Kube Mode when issues with GCS are resolved. # if NFS is disabled and index supports cloud file systems natively (e.g. like Tensorflow) indexing.Indexers.index_queryset(di, visual_index, start, target, queryset, cloud_paths=True) else: # Otherwise download and ensure that the files exist task_shared.ensure_files(queryset, target) indexing.Indexers.index_queryset(di, visual_index, start, target, queryset) return sync