def do_insert_images_api(): args = reqparse.RequestParser(). \ add_argument('Id', type=str). \ add_argument('Image', type=str). \ add_argument('Size', type=int). \ add_argument('Table', type=str). \ parse_args() file_id = request.files.get('FileId', "") file_image = request.files.get('FileImage', "") size = args['Size'] table_name = args['Table'] print(args['Id']) if file_id: ids = str(file_id.read().decode("utf-8")).strip().split(",") ids = ids[:-1] else: ids = args['Id'].split(",") if file_image: image = str(file_image.read().decode("utf-8")).strip().split(",") image = image[:-1] else: image = args['Image'].split(",") try: index_client, conn, cursor = init_conn() status, info = do_insert(index_client, conn, cursor, img_to_vec, ids, image, size, table_name) return "{0},{1}".format(status, info) except Exception as e: write_log(e, 1) return "Error with {}".format(e), 400
def count_collection(client, table_name): try: status, num = client.count_collection(collection_name=table_name) return num except Exception as e: print("Milvus ERROR:", e) write_log(e,1)
def do_insert(index_client, conn, cursor, img_to_vec, ids_image, img, size, table_name): if not table_name: table_name = DEFAULT_TABLE if not size: size = 200 print("table_name:", table_name, ", num of orgin ids:", len(ids_image),", num of orgin img:", len(img)) if len(ids_image)!= len(img): return "The number of pictures is not consistent with the ID number, please check!", None init_table(index_client, conn, cursor, table_name) img_list, ids_img, info = get_img_ids(conn, cursor, ids_image, img, table_name) print("num of the insert images:", len(img_list)) if not img_list: return None, "All the image id exists!" try: i = 0 while i+size<len(ids_img): insert_img_list = img_list[i:i+size] insert_ids_img = ids_img[i:i+size] i = i+size print("doing insert, size:", size, "the num of insert vectors:", len(insert_img_list)) status = insert_img(index_client, conn, cursor, img_to_vec, insert_img_list, insert_ids_img, table_name) else: insert_img_list = img_list[i:len(ids_image)] insert_ids_img = ids_img[i:len(ids_image)] print("doing insert, size:", size, ",the num of insert vectors:", len(insert_img_list)) status = insert_img(index_client, conn, cursor, img_to_vec, insert_img_list, insert_ids_img, table_name) return status, info except Exception as e: # log.error(e) write_log(e, 1) return None, "Error with {}".format(e)
def insert_vectors(client, table_name, vectors): try: status, ids = client.insert(collection_name=table_name, records=vectors) return status, ids except Exception as e: print("Milvus ERROR:", e) write_log(e,1)
def delete_collection(client, table_name): try: status = client.drop_collection(collection_name=table_name) return status except Exception as e: print("Milvus ERROR:", e) write_log(e,1)
def has_table(client, table_name): try: status, ok = client.has_collection(collection_name=table_name) return status, ok except Exception as e: print("Milvus ERROR:", e) write_log(e,1)
def delete_vectors(client, table_name, ids): try: status = client.delete_by_id(table_name, ids) return status except Exception as e: print("Milvus ERROR:", e) write_log(e,1)
def get_vector_by_ids(client, table_name, ids): try: status, vector = client.get_vector_by_id(collection_name=table_name, vector_id=ids) return status, vector except Exception as e: print("Milvus ERROR:", e) write_log(e,1)
def delete_table(conn, cursor, table_name): sql = "drop table if exists " + table_name + ";" try: cursor.execute(sql) print("MYSQL delete table.") except: print("MYSQL ERROR:", sql, e) write_log(e, 1)
def create_table_mysql(conn, cursor, table_name): sql = "create table if not exists " + table_name + "(milvus_id bigint, images_id varchar(30), index ix_milvus (milvus_id), index ix_images (images_id));" try: cursor.execute(sql) print("MYSQL create table.") except Exception as e: print("MYSQL ERROR:", sql, e) write_log(e, 1)
def milvus_client(): try: milvus = Milvus() milvus.connect(host=MILVUS_HOST, port=MILVUS_PORT) return milvus except Exception as e: print("Milvus ERROR:", e) write_log(e,1)
def create_index(client, table_name): param = {'nlist': 16384} try: status = client.create_index(table_name, IndexType.IVF_FLAT, param) return status except Exception as e: print("Milvus ERROR:", e) write_log(e,1)
def search_vectors(client, table_name, vectors, top_k=TOP_K): try: search_param = {'nprobe': 16} print("_______topk:", top_k) status, res = client.search(collection_name=table_name, query_records=vectors, top_k=top_k, params=search_param) return status, res except Exception as e: print("Milvus ERROR:", e) write_log(e,1)
def do_count(index_client, conn, cursor, table_name): if not table_name: table_name = DEFAULT_TABLE write_log("doing count, table_name:" + table_name) print("doing count, table_name:", table_name) num_milvus = count_collection(index_client, table_name) num_mysql = count_table(conn, cursor, table_name) return num_milvus, num_mysql
def load_data_to_mysql(conn, cursor, table_name, file_name): sql = "load data local infile '" + file_name + "' into table " + table_name + " fields terminated by ',';" try: cursor.execute(sql) conn.commit() print("MYSQL load table.") except Exception as e: print("MYSQL ERROR:", sql, e) write_log(e, 1)
def delete_all_data(conn, cursor, table_name): sql = 'delete from ' + table_name + ';' try: cursor.execute(sql) conn.commit() print("MYSQL delete all data.") except: print("MYSQL ERROR:", sql, e) write_log(e, 1)
def count_table(conn, cursor, table_name): sql = "select count(milvus_id) from " + table_name + ";" try: cursor.execute(sql) results = cursor.fetchall() print("MYSQL count table.") return results[0][0] except Exception as e: print("MYSQL ERROR:", sql, e) write_log(e, 1)
def do_search_images_api(): args = reqparse.RequestParser(). \ add_argument('Id', type=str). \ add_argument('Table', type=str). \ add_argument('Image', type=str). \ add_argument('FileId', type=str). \ add_argument('FileImage', type=str). \ parse_args() file_id = request.files.get('FileId', "") file_image = request.files.get('FileImage', "") table_name = args['Table'] if file_id: ids = str(file_id.read().decode("utf-8")).strip().split(",") ids = ids[:-1] else: ids = args['Id'].split(",") if file_image: image = str(file_image.read().decode("utf-8")).strip().split(",") image = image[:-1] else: image = args['Image'].split(",") try: index_client, conn, cursor = init_conn() result, distance = do_search(index_client, conn, cursor, img_to_vec, image, table_name) print("----------------search_results:", result, distance) # Python 字典类型转换为 JSON 对象 result_dic = {"code": 0, "msg": "success"} data = [] for i in range(len(ids)): id_dis = [] for j in range(len(result[i])): id_sim = {"id": result[i][j], "similarity": distance[i][j]} id_dis.append(id_sim) re_sim = {"id": ids[i], "similarImages": id_dis} data.append(re_sim) result_dic["data"] = data # with open("results_0630.txt","w") as f: # f.write(str(ids).replace('[','').replace(']','').replace('\'','').replace('‘','')+'\n') # f.write("\n") # for i in result: # f.write(str(i).replace('[','').replace(']','').replace('\'','').replace('‘','')+'\n') # return "{0},{1}".format(ids, result), 200 return result_dic, 200 except Exception as e: write_log(e, 1) return "Error with {}".format(e), 400
def delete_data(conn, cursor, image_id, table_name): str_ids = [str(_id) for _id in image_id] str_ids = str(str_ids).replace('[', '').replace(']', '') sql = "delete from " + table_name + " where images_id in (" + str_ids + ");" try: cursor.execute(sql) conn.commit() print("MYSQL delete data.") except Exception as e: print("MYSQL ERROR:", sql, e) write_log(e, 1)
def do_delete_table_api(): args = reqparse.RequestParser(). \ add_argument('Table', type=str). \ parse_args() try: index_client, conn, cursor = init_conn() table_name = args['Table'] status = do_delete_table(index_client, conn, cursor, table_name) return "{}".format(status) except Exception as e: write_log(e, 1) return "Error with {}".format(e), 400
def connect_mysql(): try: # conn = pymysql.connect(host="127.0.0.1",user="******",port=3306,password="******",database="mysql", local_infile=True) conn = pymysql.connect(host=MYSQL_HOST, user=MYSQL_USER, port=MYSQL_PORT, password=MYSQL_PWD, database=MYSQL_DB, local_infile=True) return conn except Exception as e: print("MYSQL ERROR: connect failed", e) write_log(e, 1)
def do_count_images_api(): args = reqparse.RequestParser(). \ add_argument('Table', type=str). \ parse_args() table_name = args['Table'] try: index_client, conn, cursor = init_conn() rows_milvus, rows_mysql = do_count(index_client, conn, cursor, table_name) return "{0},{1}".format(rows_milvus, rows_mysql), 200 except Exception as e: write_log(e, 1) return "Error with {}".format(e), 400
def search_by_milvus_ids(conn, cursor, ids, table_name): str_ids = str(ids) str_ids = str(str_ids).replace('[', '').replace(']', '') sql = "select images_id from " + table_name + " where milvus_id in (" + str_ids + ") order by field (milvus_id," + str_ids + ");" try: cursor.execute(sql) results = cursor.fetchall() results = [res[0] for res in results] print("MYSQL search by milvus id.") return results except Exception as e: print("MYSQL ERROR:", sql, e) write_log(e, 1)
def search_by_image_id(conn, cursor, image_id, table_name): sql = "select milvus_id from " + table_name + " where images_id = '" + image_id + "';" try: cursor.execute(sql) results = cursor.fetchall() print("MYSQL search by image id.") if len(results): results = [res[0] for res in results] return results else: return None except Exception as e: print("MYSQL ERROR:", sql, e) write_log(e, 1)
def create_table(client, table_name=None, dimension=VECTOR_DIMENSION, index_file_size=512, metric_type=METRIC_TYPE): table_param = { 'collection_name': table_name, 'dimension': dimension, 'index_file_size':index_file_size, 'metric_type': metric_type } try: status = client.create_collection(table_param) return status except Exception as e: print("Milvus ERROR:", e) write_log(e,1)
def do_delete_images_api(): args = reqparse.RequestParser(). \ add_argument('Table', type=str). \ add_argument('Id', type=str). \ parse_args() table_name = args['Table'] file_id = request.files.get('FileId', "") if file_id: ids = str(file_id.read().decode("utf-8")).strip().split(",") ids = ids[:-1] else: ids = args['Id'].split(",") try: index_client, conn, cursor = init_conn() status, info = do_delete_images(index_client, conn, cursor, ids, table_name) return "{0},{1}".format(status, info), 200 except Exception as e: write_log(e, 1) return "Error with {}".format(e), 400
from indexer.logs import write_log write_log("Start app service.") # import logging # logging.basicConfig(filename='app.log', filemode='w', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO) # import tensorflow as tf # config = tf.ConfigProto() # config.gpu_options.allow_growth = True # config.gpu_options.per_process_gpu_memory_fraction = 0.5 # sess = tf.Session(config=config)