def do_load(table_name, database_path):
if not table_name:
table_name = DEFAULT_TABLE
cache = Cache(default_cache_dir)
try:
vectors, names = feature_extract(table_name, database_path)
print("start connetc to milvus")
index_client = milvus_client()
status, ok = has_table(index_client, table_name)
if not ok:
print("create table.")
create_table(index_client, table_name=table_name)
print("insert into:", table_name)
# status, ids = insert_vectors(index_client, table_name, vectors)
total_ids = []
ids_lens = 0
while ids_lens<len(vectors) :
try:
status, ids = insert_vectors(index_client, table_name, vectors[ids_lens:ids_lens+100000])
except:
status, ids = insert_vectors(index_client, table_name, vectors[ids_lens:len(vectors)])
ids_lens += 100000
total_ids += ids
print("ids:",len(ids))
create_index(index_client, table_name)
for i in range(len(names)):
cache[total_ids[i]] = names[i]
print("FP finished")
return "FP finished"
except Exception as e:
logging.error(e)
return "Error with {}".format(e)
def do_train(table_name, data_loader, model, args):
if not table_name:
table_name = DEFAULT_TABLE
cache = Cache(default_cache_dir)
try:
vectors, names = feature_extract(data_loader, model, args)
vectors = vectors / vectors.norm()
print(vectors.shape)
vectors = vectors.tolist()
index_client = milvus_client()
# delete_table(index_client, table_name=table_name)
# time.sleep(1)
status, ok = has_table(index_client, table_name)
if not ok:
print("create table.")
create_table(index_client, table_name=table_name)
print("insert into:", table_name)
status, ids = insert_vectors(index_client, table_name, vectors)
create_index(index_client, table_name)
for i in range(len(names)):
# cache[names[i]] = ids[i]
cache[ids[i]] = names[i]
print("Train finished")
return "Train finished"
except Exception as e:
logging.error(e)
return "Error with {}".format(e)
def do_train(table_name, database_path):
if not table_name:
table_name = DEFAULT_TABLE
cache = Cache(default_cache_dir)
try:
vectors, names = feature_extract(database_path, VGGNet())
print("start connetc to milvus")
index_client = milvus_client()
# delete_table(index_client, table_name=table_name)
# time.sleep(1)
status, ok = has_table(index_client, table_name)
if not ok:
print("create table.")
create_table(index_client, table_name=table_name)
print("insert into:", table_name)
status, ids = insert_vectors(index_client, table_name, vectors)
create_index(index_client, table_name)
for i in range(len(names)):
# cache[names[i]] = ids[i]
cache[ids[i]] = names[i]
print("Train finished")
return "Train finished"
except Exception as e:
logging.error(e)
return "Error with {}".format(e)
def insert_data(index_client, conn, cursor, dataset_path, milvus_table=MILVUS_TABLE):
vectors = np.load(dataset_path + '/img.npy')
vectors = vectors.tolist()
ids = [i for i in range(len(vectors))]
status, _ = insert_vectors(index_client, milvus_table, vectors, ids)
print("milvus insert status:", status)
print("load data to mysql:")
Image_path=dataset_path+'/images'
load_images_to_mysql(conn,cursor,milvus_table,ids,Image_path)
def insert_img(index_client, conn, cursor, img_to_vec, insert_img_list, insert_ids_img, table_name):
vectors_img = img_to_vec(insert_img_list)
# print(len(insert_img_list),len(insert_ids_img))
status, ids_milvus = insert_vectors(index_client, table_name, vectors_img)
file_name = str(uuid.uuid1()) + ".csv"
get_ids_file(ids_milvus, insert_ids_img, file_name)
print("load data to mysql:", file_name)
load_data_to_mysql(conn, cursor, table_name, file_name)
if os.path.exists(file_name):
os.remove(file_name)
return status
def insert_data_to_milvus(ids, img, voc):
index_client = milvus_client()
status, ok = has_table(index_client, IMG_TABLE)
if not ok:
print("create table.")
create_table(index_client, table_name=IMG_TABLE)
create_table(index_client, table_name=VOC_TABLE)
vectors_img = img_to_vec(img)
vectors_voc = voc_to_vec(voc)
status = {}
if not vectors_img:
status = {'status': 'faile', 'message': 'there is no file data'}
return status
try:
insert_vectors(index_client, IMG_TABLE, [vectors_img], [ids])
insert_vectors(index_client, VOC_TABLE, [vectors_voc], [ids])
status = {'status': 'success'}
return status
except Exception as e:
logging.error(e)
return "Fail with error {}".format(e)
def insert_data(index_client,
conn,
cursor,
dataset_path,
milvus_table=MILVUS_TABLE):
vectors = np.load(dataset_path + '/h_item.npy')
vectors = vectors.tolist()
ids = [i for i in range(len(vectors))]
status, _ = insert_vectors(index_client, milvus_table, vectors, ids)
print("milvus insert status:", status)
print("load data to mysql:")
load_movies_to_mysql(conn, cursor, milvus_table,
dataset_path + '/movie_id.dat')
def do_insert_logo(image_encoder, index_client, conn, cursor, table_name,
filename, name, info):
if not table_name:
table_name = LOGO_TABLE
if table_name not in index_client.list_collections():
print("create table.")
create_table_mysql(conn, cursor, table_name)
create_table(index_client, table_name, dimension=2048)
vector = image_encoder.execute(filename)
ids = insert_vectors(index_client, table_name, [vector])
insert_data_to_pg(conn, cursor, table_name, ids[0], name, info, filename)
return "insert successfully!"
def do_train(table_name, database_path):
detector = Detector()
if not table_name:
table_name = DEFAULT_TABLE
cache = Cache(default_cache_dir)
try:
result_images, object_num = run(detector, database_path)
#print("after detect:", object_num)
vectors, obj_images = get_object_vector(cache, image_encoder,
database_path + "/object")
#print("after detect:", len(vectors), obj_images)
index_client = milvus_client()
status, ok = has_table(index_client, table_name)
if not ok:
print("create table.")
create_table(index_client, table_name=table_name)
print("insert into:", table_name)
# vectors = normaliz_vec(vectors)
status, ids = insert_vectors(index_client, table_name, vectors)
#print(status,ids)
create_index(index_client, table_name)
shutil.rmtree(database_path + "/object")
imgs = os.listdir(database_path)
imgs.sort()
#print("-----imgs", imgs)
k = 0
ids = list(reversed(ids))
#print("ids", ids)
for num in object_num:
for i in range(num):
a = ids.pop()
#print("real;;;;;;;;;",a, imgs[k])
cache[a] = imgs[k]
k += 1
return print("train finished")
except Exception as e:
logging.error(e)
return "Error with {}".format(e)
def curd(vectors, img_name, mycol, partition=None, table_name=DEFAULT_TABLE):
try:
client = milvus_client()
if not table_name:
table_name = DEFAULT_TABLE
status, ok = has_table(client, table_name)
if not ok:
print('开始创建table')
create_table(client, table_name)
if partition:
status, ok = has_partition(client, table_name, partition)
if not ok:
create_partition(client, table_name, partition)
status, id = insert_vectors(client, table_name, vectors, partition)
# 存入缓存 以便后续进行反查
# redis.hset(REDIS_NAME, id[0], img_name)
mycol.insert_one({'id': id[0], 'img': img_name, 'table': table_name})
create_index(client, table_name)
print('OK 了')
return True, ''
except Exception as e:
logging.error(e)
return False, e
