Пример #1
0
def delete_imgs(table_name, ids, mycol):
    print(table_name, ids, '内部')
    if not table_name:
        return False
    try:
        client = milvus_client()
        status, ok = has_table(client, table_name)
        if not ok:
            return False, '表不存在'
        if ok:
            list = []
            for id in ids:
                item = mycol.find_one({'img': id})
                if item:
                    list.append(int(item['id']))
            if list:
                dstatus = delete_entity(client, table_name, list)
                for id in ids:
                    mycol.delete_one({"img": id})
                if dstatus.code == 0:
                    return True, '删除成功'
            else:
                return True, '该图片未进行索引或已被删除'
    except Exception as e:
        logging.error(e)
        return False
Пример #2
0
def do_train(table_name, data_loader, model, args):
    if not table_name:
        table_name = DEFAULT_TABLE
    cache = Cache(default_cache_dir)
    try:
        vectors, names = feature_extract(data_loader, model, args)
        vectors = vectors / vectors.norm()
        print(vectors.shape)
        vectors = vectors.tolist()
        index_client = milvus_client()
        # delete_table(index_client, table_name=table_name)
        # time.sleep(1)
        status, ok = has_table(index_client, table_name)
        if not ok:
            print("create table.")
            create_table(index_client, table_name=table_name)
        print("insert into:", table_name)
        status, ids = insert_vectors(index_client, table_name, vectors)
        create_index(index_client, table_name)
        for i in range(len(names)):
            # cache[names[i]] = ids[i]
            cache[ids[i]] = names[i]
        print("Train finished")
        return "Train finished"
    except Exception as e:
        logging.error(e)
        return "Error with {}".format(e)
Пример #3
0
def do_train(table_name, database_path):
    if not table_name:
        table_name = DEFAULT_TABLE
    cache = Cache(default_cache_dir)
    try:
        vectors, names = feature_extract(database_path, VGGNet())
        print("start connetc to milvus")
        index_client = milvus_client()
        # delete_table(index_client, table_name=table_name)
        # time.sleep(1)
        status, ok = has_table(index_client, table_name)
        if not ok:
            print("create table.")
            create_table(index_client, table_name=table_name)
        print("insert into:", table_name)
        status, ids = insert_vectors(index_client, table_name, vectors)
        create_index(index_client, table_name)
        for i in range(len(names)):
            # cache[names[i]] = ids[i]
            cache[ids[i]] = names[i]
        print("Train finished")
        return "Train finished"
    except Exception as e:
        logging.error(e)
        return "Error with {}".format(e)
def init_table(index_client, conn, cursor, milvus_table=MILVUS_TABLE):
    status, ok = has_table(index_client, milvus_table)
    if not ok:
        print("create table.")
        create_table(index_client, milvus_table)
        create_index(index_client, milvus_table)
        create_table_mysql(conn, cursor, milvus_table)
def do_load(table_name, database_path):
    if not table_name:
        table_name = DEFAULT_TABLE
    cache = Cache(default_cache_dir)
    try:
        vectors, names = feature_extract(table_name, database_path)
        print("start connetc to milvus")
        index_client = milvus_client()
        status, ok = has_table(index_client, table_name)
        if not ok:
            print("create table.")
            create_table(index_client, table_name=table_name)
        print("insert into:", table_name)

        # status, ids = insert_vectors(index_client, table_name, vectors)
        total_ids = []
        ids_lens = 0
        while ids_lens<len(vectors) :
            try:
                status, ids = insert_vectors(index_client, table_name, vectors[ids_lens:ids_lens+100000])
            except:
                status, ids = insert_vectors(index_client, table_name, vectors[ids_lens:len(vectors)])
            ids_lens += 100000
            total_ids += ids
            print("ids:",len(ids))

        create_index(index_client, table_name)
        for i in range(len(names)):
            cache[total_ids[i]] = names[i]
        print("FP finished")
        return "FP finished"
    except Exception as e:
        logging.error(e)
        return "Error with {}".format(e)
Пример #6
0
def collection_info(table_name):
    client = milvus_client()
    tablestatus, ok = has_table(client, table_name)
    if not ok:
        return False, '', ''
    status, info = get_collection_info(client, table_name)
    return True, status, info
Пример #7
0
def get_partitions_list(table_name):
    client = milvus_client()
    tablestatus, ok = has_table(client, table_name)
    if not ok:
        return ok, '', ''
    status, info = get_list_partitions(client, table_name)
    return ok, status, info
Пример #8
0
def init_table(index_client, conn, cursor, table_name):
    status, ok = has_table(index_client, table_name)
    print("has_table:", status, ok)
    if not ok:
        print("create table.")
        create_table(index_client, table_name)
        create_index(index_client, table_name)
        create_table_mysql(conn, cursor, table_name)
Пример #9
0
def create_collection(table_name):
    print(table_name, '1111')
    client = milvus_client()
    tablestatus, ok = has_table(client, table_name)
    print(tablestatus, ok)
    if ok:
        return False, ''
    status = create_table(client, table_name)
    return True, status
Пример #10
0
def create_p(table_name, partition_name):
    client = milvus_client()
    try:
        status, ok = has_table(client, table_name)
        if not ok:
            create_table(client, table_name)

        statusp = create_partition(client, table_name, partition_name)
        return statusp
    except Exception as e:
        logging.error(e)
        return '发生错误'.format(e)
Пример #11
0
def count_tab(table_name):
    if not table_name:
        return False
    try:
        client = milvus_client()
        status, ok = has_table(client, table_name)
        if not ok:
            return False, ''
        val = count_table(client, table_name)
        return True, val
    except Exception as e:
        logging.error(e)
        return '发生错误'.format(e)
Пример #12
0
def delete_collection(table_name):
    if not table_name:
        return False
    try:
        client = milvus_client()
        status, ok = has_table(client, table_name)
        if not ok:
            return False
        if ok:
            status = delete_table(client, table_name)
            if status.code == 0:
                return True
    except Exception as e:
        logging.error(e)
        return '错误'.format(e)
def insert_data_to_milvus(ids, img, voc):
    index_client = milvus_client()
    status, ok = has_table(index_client, IMG_TABLE)
    if not ok:
        print("create table.")
        create_table(index_client, table_name=IMG_TABLE)
        create_table(index_client, table_name=VOC_TABLE)

    vectors_img = img_to_vec(img)
    vectors_voc = voc_to_vec(voc)
    status = {}
    if not vectors_img:
        status = {'status': 'faile', 'message': 'there is no file data'}
        return status
    try:
        insert_vectors(index_client, IMG_TABLE, [vectors_img], [ids])
        insert_vectors(index_client, VOC_TABLE, [vectors_voc], [ids])
        status = {'status': 'success'}
        return status
    except Exception as e:
        logging.error(e)
        return "Fail with error {}".format(e)
Пример #14
0
def do_train(table_name, database_path):
    detector = Detector()
    if not table_name:
        table_name = DEFAULT_TABLE
    cache = Cache(default_cache_dir)
    try:
        result_images, object_num = run(detector, database_path)
        #print("after detect:", object_num)
        vectors, obj_images = get_object_vector(cache, image_encoder,
                                                database_path + "/object")
        #print("after detect:", len(vectors), obj_images)
        index_client = milvus_client()
        status, ok = has_table(index_client, table_name)
        if not ok:
            print("create table.")
            create_table(index_client, table_name=table_name)
        print("insert into:", table_name)
        # vectors = normaliz_vec(vectors)
        status, ids = insert_vectors(index_client, table_name, vectors)
        #print(status,ids)
        create_index(index_client, table_name)
        shutil.rmtree(database_path + "/object")
        imgs = os.listdir(database_path)
        imgs.sort()
        #print("-----imgs", imgs)
        k = 0
        ids = list(reversed(ids))
        #print("ids", ids)
        for num in object_num:
            for i in range(num):
                a = ids.pop()
                #print("real;;;;;;;;;",a, imgs[k])
                cache[a] = imgs[k]
            k += 1
        return print("train finished")
    except Exception as e:
        logging.error(e)
        return "Error with {}".format(e)
Пример #15
0
def curd(vectors, img_name, mycol, partition=None, table_name=DEFAULT_TABLE):
    try:
        client = milvus_client()
        if not table_name:
            table_name = DEFAULT_TABLE
        status, ok = has_table(client, table_name)
        if not ok:
            print('开始创建table')
            create_table(client, table_name)
        if partition:
            status, ok = has_partition(client, table_name, partition)
            if not ok:
                create_partition(client, table_name, partition)
        status, id = insert_vectors(client, table_name, vectors, partition)
        # 存入缓存 以便后续进行反查
        # redis.hset(REDIS_NAME, id[0], img_name)
        mycol.insert_one({'id': id[0], 'img': img_name, 'table': table_name})
        create_index(client, table_name)
        print('OK 了')
        return True, ''
    except Exception as e:
        logging.error(e)
        return False, e