def video_path(gif_path):
# Get the gif file
try:
LOGGER.info(("Successfully load gif: {}".format(gif_path)))
return FileResponse(gif_path)
except Exception as e:
LOGGER.error("upload image error: {}".format(e))
return {'status': False, 'msg': e}, 400
async def count_video(table_name: str = None):
# Returns the total number of images in the system
try:
num = do_count(table_name, MILVUS_CLI, MYSQL_CLI)
LOGGER.info("Successfully count the number:{} of images!".format(num))
return num
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
async def drop_tables(table_name: str = None):
# Delete the collection of Milvus and MySQL
try:
status = do_drop(table_name, MILVUS_CLI, MYSQL_CLI)
LOGGER.info("Successfully drop tables in Milvus and MySQL!")
return {'status': True, 'msg': status}
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
async def count_images(table_name: str = None):
# Returns the total number of questions in the system
try:
num = do_count(table_name, MILVUS_CLI)
LOGGER.info("Successfully count the number of questions!")
return num
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
async def do_get_question_api(question: str, table_name: str = None):
try:
questions, distances = do_search(table_name, question, MODEL,
MILVUS_CLI, MYSQL_CLI)
#res = dict(zip(questions, distances))
# res = sorted(res.items(), key=lambda item: item[1])
LOGGER.info("Successfully searched similar images!")
return {'status': True, 'msg': questions}, 200
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
async def load_images(item: Item):
# Insert all the image under the file path to Milvus/MySQL
try:
total_num = do_load(item.Table, item.File, MODEL, MILVUS_CLI,
MYSQL_CLI)
LOGGER.info(
"Successfully loaded data, total objects: {}".format(total_num))
return "Successfully loaded data!"
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
async def load_video(item: Item):
# Insert all the video under the file path to Milvus/MySQL
try:
total_num = do_load(item.Table, item.File, MODEL, FRAME, MILVUS_CLI,
MYSQL_CLI)
LOGGER.info(
"Successfully loaded data, total count: {}".format(total_num))
return {'status': True, 'msg': "Successfully loaded data!"}
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
async def do_search_api(table_name: str = None, query_sentence: str = None):
try:
ids, title, text, distances = search_in_milvus(table_name,
query_sentence,
MILVUS_CLI, MYSQL_CLI)
res = []
for p, d in zip(title, text):
dicts = {'title': p, 'content': d}
res += [dicts]
LOGGER.info("Successfully searched similar text!")
return res
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
def search_in_milvus(table_name, query_sentence,milvus_cli, mysql_cli):
if not table_name:
table_name = DEFAULT_TABLE
try:
query_data = [query_sentence]
vectors = bc.encode(query_data)
query_list = normaliz_vec(vectors.tolist())
LOGGER.info("Successfully insert query list")
results = milvus_cli.search_vectors(table_name,query_list,TOP_K)
vids = [str(x.id) for x in results[0]]
print("-----------------", vids)
ids,title,text= mysql_cli.search_by_milvus_ids(vids, table_name)
distances = [x.distance for x in results[0]]
return ids,title, text, distances
except Exception as e:
LOGGER.error(" Error with search : {}".format(e))
sys.exit(1)
def get_index_params(index_type):
if index_type == 'FLAT':
index_param = {"index_type": index_type}
elif index_type == 'RNSG':
params = {
"search_length": SEARCH_LENGTH,
"out_degree": OUT_DEGREE,
"candidate_pool_size": CANDIDATE_POOL,
"knng": KNNG
}
index_param = {
"index_type": index_type,
"metric_type": METRIC_TYPE,
"params": params
}
elif index_type == 'HNSW':
params = {"M": HNSW_M, "efConstruction": EFCONSTRUCTION}
index_param = {
"index_type": index_type,
"metric_type": METRIC_TYPE,
"params": params
}
elif index_type == 'ANNOY':
params = {"n_tress": N_TREE}
index_param = {
"index_type": index_type,
"metric_type": METRIC_TYPE,
"params": params
}
elif index_type == 'IVF_PQ':
params = {"nlist": NLIST, "m": PQ_M}
index_param = {
"index_type": index_type,
"metric_type": METRIC_TYPE,
"params": params
}
else:
params = {"nlist": NLIST}
index_param = {
"index_type": index_type,
"metric_type": METRIC_TYPE,
"params": params
}
LOGGER.info(index_param)
return index_param
async def search_images(image: UploadFile = File(...), table_name: str = None):
# Search the upload image in Milvus/MySQL
try:
# Save the upload image to server.
content = await image.read()
print('read pic succ')
img_path = os.path.join(UPLOAD_PATH, image.filename)
with open(img_path, "wb+") as f:
f.write(content)
paths, distances = do_search(table_name, img_path, MODEL, MILVUS_CLI,
MYSQL_CLI)
res = dict(zip(paths, distances))
res = sorted(res.items(), key=lambda item: item[1])
LOGGER.info("Successfully searched similar images!")
return res
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
async def do_search_api(table_name: str = None, query_sentence: str = None):
try:
ids, results_classes, seq_genes, distances = search_in_milvus(
table_name, query_sentence, MILVUS_CLI, MYSQL_CLI)
res = []
for i, c, s, d in zip(ids, results_classes, seq_genes, distances):
dicts = {
'milvus_id': i,
'seq_class': c,
'seq_gene': s,
'IP distance': d
}
res += [dicts]
LOGGER.info("Successfully searched similar sequence!")
return res
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
async def load_text(file: UploadFile = File(...), table_name: str = None):
try:
text = await file.read()
fname = file.filename
dirs = "data"
if not os.path.exists(dirs):
os.makedirs(dirs)
fname_path = os.path.join(os.getcwd(), os.path.join(dirs, fname))
with open(fname_path, 'wb') as f:
f.write(text)
except Exception as e:
return {'status': False, 'msg': 'Failed to load data.'}
# Insert all the image under the file path to Milvus/MySQL
try:
total_num = import_data(table_name, fname_path, MILVUS_CLI, MYSQL_CLI)
LOGGER.info(
"Successfully loaded data, total count: {}".format(total_num))
return "Successfully loaded data!"
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
def search_in_milvus(table_name, query_sentence, milvus_cli, mysql_cli):
if not table_name:
table_name = DEFAULT_TABLE
try:
kmers = build_kmers(query_sentence,KMER_K)
query_data = [" ".join(kmers)]
query_list = encode_seq(query_data)
LOGGER.info("Searching...")
results = milvus_cli.search_vectors(table_name,query_list,TOP_K)
vids = [str(x.id) for x in results[0]]
print("-----------------", vids)
ids, results_classes = mysql_cli.search_by_milvus_ids(vids, table_name)
distances = [x.distance for x in results[0]]
df_class = pd.read_table(SEQ_CLASS_PATH)
class_dict = dict()
for i in range(len(df_class)):
class_dict[df_class['class'][i]] = df_class['gene_family'][i]
seq_genes = [class_dict[int(x)] for x in results_classes]
return ids, results_classes, seq_genes, distances
except Exception as e:
LOGGER.error(" Error with search : {}".format(e))
sys.exit(1)
async def search_images(request: Request,
image: UploadFile = File(...),
table_name: str = None):
# Search the upload image in Milvus/MySQL
try:
# Save the upload image to server.
content = await image.read()
img_path = os.path.join(UPLOAD_PATH, image.filename)
with open(img_path, "wb+") as f:
f.write(content)
host = request.headers['host']
paths, distances = do_search(host, table_name, img_path, MODEL,
MILVUS_CLI, MYSQL_CLI)
res = {}
for p, d in zip(paths, distances):
if not p in res or res[p] > d:
res[p] = d
res = sorted(res.items(), key=lambda item: item[1])
LOGGER.info("Successfully searched similar images!")
return res
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
async def do_load_api(file: UploadFile = File(...), table_name: str = None):
try:
text = await file.read()
fname = file.filename
dirs = "QA_data"
if not os.path.exists(dirs):
os.makedirs(dirs)
fname_path = os.path.join(os.getcwd(), os.path.join(dirs, fname))
with open(fname_path, 'wb') as f:
f.write(text)
except Exception as e:
return {'status': False, 'msg': 'Failed to load data.'}
try:
total_num = do_load(table_name, fname_path, MODEL, MILVUS_CLI,
MYSQL_CLI)
LOGGER.info(
"Successfully loaded data, total count: {}".format(total_num))
return {
'status': True,
'msg': "Successfully loaded data: {}".format(total_num)
}, 200
except Exception as e:
LOGGER.error(e)
return {'status': False, 'msg': e}, 400
app = FastAPI()
app.add_middleware(CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"])
MODEL = Resnet50()
MILVUS_CLI = MilvusHelper()
MYSQL_CLI = MySQLHelper()
FRAME = FrameExtract()
# Mkdir '/tmp/search-images'
if not os.path.exists(UPLOAD_PATH):
os.makedirs(UPLOAD_PATH)
LOGGER.info("mkdir the path:{} ".format(UPLOAD_PATH))
@app.get('/data')
def video_path(gif_path):
# Get the gif file
try:
LOGGER.info(("Successfully load gif: {}".format(gif_path)))
return FileResponse(gif_path)
except Exception as e:
LOGGER.error("upload image error: {}".format(e))
return {'status': False, 'msg': e}, 400
@app.get('/progress')
def get_progress():
def create_index(client, collection_name, index_type):
index_param = get_index_params(index_type)
time1 = time.time()
client.create_index(collection_name, index_param)
LOGGER.info("create index total cost time: {}".format(time.time() - time1))