def do_insert_images_api():
args = reqparse.RequestParser(). \
add_argument('Id', type=str). \
add_argument('Image', type=str). \
add_argument('Size', type=int). \
add_argument('Table', type=str). \
parse_args()
file_id = request.files.get('FileId', "")
file_image = request.files.get('FileImage', "")
size = args['Size']
table_name = args['Table']
print(args['Id'])
if file_id:
ids = str(file_id.read().decode("utf-8")).strip().split(",")
ids = ids[:-1]
else:
ids = args['Id'].split(",")
if file_image:
image = str(file_image.read().decode("utf-8")).strip().split(",")
image = image[:-1]
else:
image = args['Image'].split(",")
try:
index_client, conn, cursor = init_conn()
status, info = do_insert(index_client, conn, cursor, img_to_vec, ids,
image, size, table_name)
return "{0},{1}".format(status, info)
except Exception as e:
write_log(e, 1)
return "Error with {}".format(e), 400
def count_collection(client, table_name):
try:
status, num = client.count_collection(collection_name=table_name)
return num
except Exception as e:
print("Milvus ERROR:", e)
write_log(e,1)
def do_insert(index_client, conn, cursor, img_to_vec, ids_image, img, size, table_name):
if not table_name:
table_name = DEFAULT_TABLE
if not size:
size = 200
print("table_name:", table_name, ", num of orgin ids:", len(ids_image),", num of orgin img:", len(img))
if len(ids_image)!= len(img):
return "The number of pictures is not consistent with the ID number, please check!", None
init_table(index_client, conn, cursor, table_name)
img_list, ids_img, info = get_img_ids(conn, cursor, ids_image, img, table_name)
print("num of the insert images:", len(img_list))
if not img_list:
return None, "All the image id exists!"
try:
i = 0
while i+size<len(ids_img):
insert_img_list = img_list[i:i+size]
insert_ids_img = ids_img[i:i+size]
i = i+size
print("doing insert, size:", size, "the num of insert vectors:", len(insert_img_list))
status = insert_img(index_client, conn, cursor, img_to_vec, insert_img_list, insert_ids_img, table_name)
else:
insert_img_list = img_list[i:len(ids_image)]
insert_ids_img = ids_img[i:len(ids_image)]
print("doing insert, size:", size, ",the num of insert vectors:", len(insert_img_list))
status = insert_img(index_client, conn, cursor, img_to_vec, insert_img_list, insert_ids_img, table_name)
return status, info
except Exception as e:
# log.error(e)
write_log(e, 1)
return None, "Error with {}".format(e)
def insert_vectors(client, table_name, vectors):
try:
status, ids = client.insert(collection_name=table_name, records=vectors)
return status, ids
except Exception as e:
print("Milvus ERROR:", e)
write_log(e,1)
def delete_collection(client, table_name):
try:
status = client.drop_collection(collection_name=table_name)
return status
except Exception as e:
print("Milvus ERROR:", e)
write_log(e,1)
def has_table(client, table_name):
try:
status, ok = client.has_collection(collection_name=table_name)
return status, ok
except Exception as e:
print("Milvus ERROR:", e)
write_log(e,1)
def delete_vectors(client, table_name, ids):
try:
status = client.delete_by_id(table_name, ids)
return status
except Exception as e:
print("Milvus ERROR:", e)
write_log(e,1)
def get_vector_by_ids(client, table_name, ids):
try:
status, vector = client.get_vector_by_id(collection_name=table_name, vector_id=ids)
return status, vector
except Exception as e:
print("Milvus ERROR:", e)
write_log(e,1)
def delete_table(conn, cursor, table_name):
sql = "drop table if exists " + table_name + ";"
try:
cursor.execute(sql)
print("MYSQL delete table.")
except:
print("MYSQL ERROR:", sql, e)
write_log(e, 1)
def create_table_mysql(conn, cursor, table_name):
sql = "create table if not exists " + table_name + "(milvus_id bigint, images_id varchar(30), index ix_milvus (milvus_id), index ix_images (images_id));"
try:
cursor.execute(sql)
print("MYSQL create table.")
except Exception as e:
print("MYSQL ERROR:", sql, e)
write_log(e, 1)
def milvus_client():
try:
milvus = Milvus()
milvus.connect(host=MILVUS_HOST, port=MILVUS_PORT)
return milvus
except Exception as e:
print("Milvus ERROR:", e)
write_log(e,1)
def create_index(client, table_name):
param = {'nlist': 16384}
try:
status = client.create_index(table_name, IndexType.IVF_FLAT, param)
return status
except Exception as e:
print("Milvus ERROR:", e)
write_log(e,1)
def search_vectors(client, table_name, vectors, top_k=TOP_K):
try:
search_param = {'nprobe': 16}
print("_______topk:", top_k)
status, res = client.search(collection_name=table_name, query_records=vectors, top_k=top_k, params=search_param)
return status, res
except Exception as e:
print("Milvus ERROR:", e)
write_log(e,1)
def do_count(index_client, conn, cursor, table_name):
if not table_name:
table_name = DEFAULT_TABLE
write_log("doing count, table_name:" + table_name)
print("doing count, table_name:", table_name)
num_milvus = count_collection(index_client, table_name)
num_mysql = count_table(conn, cursor, table_name)
return num_milvus, num_mysql
def load_data_to_mysql(conn, cursor, table_name, file_name):
sql = "load data local infile '" + file_name + "' into table " + table_name + " fields terminated by ',';"
try:
cursor.execute(sql)
conn.commit()
print("MYSQL load table.")
except Exception as e:
print("MYSQL ERROR:", sql, e)
write_log(e, 1)
def delete_all_data(conn, cursor, table_name):
sql = 'delete from ' + table_name + ';'
try:
cursor.execute(sql)
conn.commit()
print("MYSQL delete all data.")
except:
print("MYSQL ERROR:", sql, e)
write_log(e, 1)
def count_table(conn, cursor, table_name):
sql = "select count(milvus_id) from " + table_name + ";"
try:
cursor.execute(sql)
results = cursor.fetchall()
print("MYSQL count table.")
return results[0][0]
except Exception as e:
print("MYSQL ERROR:", sql, e)
write_log(e, 1)
def do_search_images_api():
args = reqparse.RequestParser(). \
add_argument('Id', type=str). \
add_argument('Table', type=str). \
add_argument('Image', type=str). \
add_argument('FileId', type=str). \
add_argument('FileImage', type=str). \
parse_args()
file_id = request.files.get('FileId', "")
file_image = request.files.get('FileImage', "")
table_name = args['Table']
if file_id:
ids = str(file_id.read().decode("utf-8")).strip().split(",")
ids = ids[:-1]
else:
ids = args['Id'].split(",")
if file_image:
image = str(file_image.read().decode("utf-8")).strip().split(",")
image = image[:-1]
else:
image = args['Image'].split(",")
try:
index_client, conn, cursor = init_conn()
result, distance = do_search(index_client, conn, cursor, img_to_vec,
image, table_name)
print("----------------search_results:", result, distance)
# Python 字典类型转换为 JSON 对象
result_dic = {"code": 0, "msg": "success"}
data = []
for i in range(len(ids)):
id_dis = []
for j in range(len(result[i])):
id_sim = {"id": result[i][j], "similarity": distance[i][j]}
id_dis.append(id_sim)
re_sim = {"id": ids[i], "similarImages": id_dis}
data.append(re_sim)
result_dic["data"] = data
# with open("results_0630.txt","w") as f:
# f.write(str(ids).replace('[','').replace(']','').replace('\'','').replace('‘','')+'\n')
# f.write("\n")
# for i in result:
# f.write(str(i).replace('[','').replace(']','').replace('\'','').replace('‘','')+'\n')
# return "{0},{1}".format(ids, result), 200
return result_dic, 200
except Exception as e:
write_log(e, 1)
return "Error with {}".format(e), 400
def delete_data(conn, cursor, image_id, table_name):
str_ids = [str(_id) for _id in image_id]
str_ids = str(str_ids).replace('[', '').replace(']', '')
sql = "delete from " + table_name + " where images_id in (" + str_ids + ");"
try:
cursor.execute(sql)
conn.commit()
print("MYSQL delete data.")
except Exception as e:
print("MYSQL ERROR:", sql, e)
write_log(e, 1)
def do_delete_table_api():
args = reqparse.RequestParser(). \
add_argument('Table', type=str). \
parse_args()
try:
index_client, conn, cursor = init_conn()
table_name = args['Table']
status = do_delete_table(index_client, conn, cursor, table_name)
return "{}".format(status)
except Exception as e:
write_log(e, 1)
return "Error with {}".format(e), 400
def connect_mysql():
try:
# conn = pymysql.connect(host="127.0.0.1",user="******",port=3306,password="******",database="mysql", local_infile=True)
conn = pymysql.connect(host=MYSQL_HOST,
user=MYSQL_USER,
port=MYSQL_PORT,
password=MYSQL_PWD,
database=MYSQL_DB,
local_infile=True)
return conn
except Exception as e:
print("MYSQL ERROR: connect failed", e)
write_log(e, 1)
def do_count_images_api():
args = reqparse.RequestParser(). \
add_argument('Table', type=str). \
parse_args()
table_name = args['Table']
try:
index_client, conn, cursor = init_conn()
rows_milvus, rows_mysql = do_count(index_client, conn, cursor,
table_name)
return "{0},{1}".format(rows_milvus, rows_mysql), 200
except Exception as e:
write_log(e, 1)
return "Error with {}".format(e), 400
def search_by_milvus_ids(conn, cursor, ids, table_name):
str_ids = str(ids)
str_ids = str(str_ids).replace('[', '').replace(']', '')
sql = "select images_id from " + table_name + " where milvus_id in (" + str_ids + ") order by field (milvus_id," + str_ids + ");"
try:
cursor.execute(sql)
results = cursor.fetchall()
results = [res[0] for res in results]
print("MYSQL search by milvus id.")
return results
except Exception as e:
print("MYSQL ERROR:", sql, e)
write_log(e, 1)
def search_by_image_id(conn, cursor, image_id, table_name):
sql = "select milvus_id from " + table_name + " where images_id = '" + image_id + "';"
try:
cursor.execute(sql)
results = cursor.fetchall()
print("MYSQL search by image id.")
if len(results):
results = [res[0] for res in results]
return results
else:
return None
except Exception as e:
print("MYSQL ERROR:", sql, e)
write_log(e, 1)
def create_table(client, table_name=None, dimension=VECTOR_DIMENSION,
index_file_size=512, metric_type=METRIC_TYPE):
table_param = {
'collection_name': table_name,
'dimension': dimension,
'index_file_size':index_file_size,
'metric_type': metric_type
}
try:
status = client.create_collection(table_param)
return status
except Exception as e:
print("Milvus ERROR:", e)
write_log(e,1)
def do_delete_images_api():
args = reqparse.RequestParser(). \
add_argument('Table', type=str). \
add_argument('Id', type=str). \
parse_args()
table_name = args['Table']
file_id = request.files.get('FileId', "")
if file_id:
ids = str(file_id.read().decode("utf-8")).strip().split(",")
ids = ids[:-1]
else:
ids = args['Id'].split(",")
try:
index_client, conn, cursor = init_conn()
status, info = do_delete_images(index_client, conn, cursor, ids,
table_name)
return "{0},{1}".format(status, info), 200
except Exception as e:
write_log(e, 1)
return "Error with {}".format(e), 400
from indexer.logs import write_log
write_log("Start app service.")
# import logging
# logging.basicConfig(filename='app.log', filemode='w', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)
# import tensorflow as tf
# config = tf.ConfigProto()
# config.gpu_options.allow_growth = True
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
# sess = tf.Session(config=config)
