def do_train(table_name, database_path):
if not table_name:
table_name = DEFAULT_TABLE
cache = Cache(default_cache_dir)
try:
vectors, names = feature_extract(database_path, VGGNet())
print("start connetc to milvus")
index_client = milvus_client()
# delete_table(index_client, table_name=table_name)
# time.sleep(1)
status, ok = has_table(index_client, table_name)
if not ok:
print("create table.")
create_table(index_client, table_name=table_name)
print("insert into:", table_name)
status, ids = insert_vectors(index_client, table_name, vectors)
create_index(index_client, table_name)
for i in range(len(names)):
# cache[names[i]] = ids[i]
cache[ids[i]] = names[i]
print("Train finished")
return "Train finished"
except Exception as e:
logging.error(e)
return "Error with {}".format(e)
def do_train(table_name, data_loader, model, args):
if not table_name:
table_name = DEFAULT_TABLE
cache = Cache(default_cache_dir)
try:
vectors, names = feature_extract(data_loader, model, args)
vectors = vectors / vectors.norm()
print(vectors.shape)
vectors = vectors.tolist()
index_client = milvus_client()
# delete_table(index_client, table_name=table_name)
# time.sleep(1)
status, ok = has_table(index_client, table_name)
if not ok:
print("create table.")
create_table(index_client, table_name=table_name)
print("insert into:", table_name)
status, ids = insert_vectors(index_client, table_name, vectors)
create_index(index_client, table_name)
for i in range(len(names)):
# cache[names[i]] = ids[i]
cache[ids[i]] = names[i]
print("Train finished")
return "Train finished"
except Exception as e:
logging.error(e)
return "Error with {}".format(e)
def do_load(table_name, database_path):
if not table_name:
table_name = DEFAULT_TABLE
cache = Cache(default_cache_dir)
try:
vectors, names = feature_extract(table_name, database_path)
print("start connetc to milvus")
index_client = milvus_client()
status, ok = has_table(index_client, table_name)
if not ok:
print("create table.")
create_table(index_client, table_name=table_name)
print("insert into:", table_name)
# status, ids = insert_vectors(index_client, table_name, vectors)
total_ids = []
ids_lens = 0
while ids_lens<len(vectors) :
try:
status, ids = insert_vectors(index_client, table_name, vectors[ids_lens:ids_lens+100000])
except:
status, ids = insert_vectors(index_client, table_name, vectors[ids_lens:len(vectors)])
ids_lens += 100000
total_ids += ids
print("ids:",len(ids))
create_index(index_client, table_name)
for i in range(len(names)):
cache[total_ids[i]] = names[i]
print("FP finished")
return "FP finished"
except Exception as e:
logging.error(e)
return "Error with {}".format(e)