def test_search_during_createIndex(self, args):
loops = 100000
table = gen_unique_str()
query_vecs = [vectors[0], vectors[1]]
uri = "tcp://%s:%s" % (args["ip"], args["port"])
id_0 = 0
id_1 = 0
milvus_instance = Milvus()
milvus_instance.connect(uri=uri)
milvus_instance.create_table({
'table_name': table,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.L2
})
for i in range(10):
status, ids = milvus_instance.add_vectors(table, vectors)
# logging.getLogger().info(ids)
if i == 0:
id_0 = ids[0]
id_1 = ids[1]
def create_index(milvus_instance):
logging.getLogger().info("In create index")
status = milvus_instance.create_index(table, index_params)
logging.getLogger().info(status)
status, result = milvus_instance.describe_index(table)
logging.getLogger().info(result)
def add_vectors(milvus_instance):
logging.getLogger().info("In add vectors")
status, ids = milvus_instance.add_vectors(table, vectors)
logging.getLogger().info(status)
def search(milvus_instance):
logging.getLogger().info("In search vectors")
for i in range(loops):
status, result = milvus_instance.search_vectors(
table, top_k, nprobe, query_vecs)
logging.getLogger().info(status)
assert result[0][0].id == id_0
assert result[1][0].id == id_1
milvus_instance = Milvus()
milvus_instance.connect(uri=uri)
p_search = Process(target=search, args=(milvus_instance, ))
p_search.start()
milvus_instance = Milvus()
milvus_instance.connect(uri=uri)
p_create = Process(target=add_vectors, args=(milvus_instance, ))
p_create.start()
p_create.join()
def milvus_test(usr_features, mov_features, ids):
_HOST = '127.0.0.1'
_PORT = '19530' # default value
milvus = Milvus()
param = {'host': _HOST, 'port': _PORT}
status = milvus.connect(**param)
if status.OK():
print("\nServer connected.")
else:
print("\nServer connect fail.")
sys.exit(1)
table_name = 'paddle_demo1'
status, ok = milvus.has_table(table_name)
if not ok:
param = {
'table_name': table_name,
'dimension': 200,
'index_file_size': 1024, # optional
'metric_type': MetricType.IP # optional
}
milvus.create_table(param)
insert_vectors = normaliz_data([usr_features.tolist()])
status, ids = milvus.insert(table_name=table_name,
records=insert_vectors,
ids=ids)
time.sleep(1)
status, result = milvus.count_table(table_name)
print("rows in table paddle_demo1:", result)
status, table = milvus.describe_table(table_name)
search_vectors = normaliz_data([mov_features.tolist()])
param = {
'table_name': table_name,
'query_records': search_vectors,
'top_k': 1,
'nprobe': 16
}
status, results = milvus.search_vectors(**param)
print("Searched ids:", results[0][0].id)
print("Score:", float(results[0][0].distance) * 5)
status = milvus.drop_table(table_name)
def create_table(_table_name):
milvus = Milvus()
milvus.connect(host="localhost", port="19530")
if milvus.has_table(_table_name):
print(f"Table {_table_name} found, now going to delete it")
status = milvus.delete_table(_table_name)
assert status.OK(), "delete table {} failed".format(_table_name)
time.sleep(5)
if milvus.has_table(_table_name):
raise Exception("Delete table error")
print("delete table {} successfully!".format(_table_name))
# wait for table deleted
status = milvus.create_table(param)
if not status.OK():
print("Create table {} failed".format(_table_name))
# in main process, milvus must be closed before subprocess start
milvus.disconnect()
time.sleep(1)
def multi_thread_opr(table_name, utid):
print("[{}] | T{} | Running .....".format(datetime.datetime.now(),
utid))
client0 = Milvus(handler="HTTP")
table_param = {'table_name': table_name, 'dimension': 64}
vectors = [[random.random() for _ in range(64)] for _ in range(10000)]
client0.connect()
client0.create_table(table_param)
# print("[{}] | T{} | O{} | Start insert data .....".format(datetime.datetime.now(), utid, i))
client0.insert(table_name, vectors)
# print("[{}] | T{} | O{} | Stop insert data .....".format(datetime.datetime.now(), utid, i))
client0.disconnect()
def milvus_test(usr_features, IS_INFER, mov_features=None, ids=None):
_HOST = '127.0.0.1'
_PORT = '19530' # default value
table_name = 'recommender_demo'
milvus = Milvus()
param = {'host': _HOST, 'port': _PORT}
status = milvus.connect(**param)
if status.OK():
print("Server connected.")
else:
print("Server connect fail.")
sys.exit(1)
if IS_INFER:
status = milvus.drop_table(table_name)
time.sleep(3)
status, ok = milvus.has_table(table_name)
if not ok:
if mov_features is None:
print("Insert vectors is none!")
sys.exit(1)
param = {
'table_name': table_name,
'dimension': 200,
'index_file_size': 1024, # optional
'metric_type': MetricType.IP # optional
}
print(milvus.create_table(param))
insert_vectors = normaliz_data(mov_features)
status, ids = milvus.insert(table_name=table_name,
records=insert_vectors,
ids=ids)
time.sleep(1)
status, result = milvus.count_table(table_name)
print("rows in table recommender_demo:", result)
search_vectors = normaliz_data(usr_features)
param = {
'table_name': table_name,
'query_records': search_vectors,
'top_k': 5,
'nprobe': 16
}
time1 = time.time()
status, results = milvus.search_vectors(**param)
time2 = time.time()
print("Top\t", "Ids\t", "Title\t", "Score")
for i, re in enumerate(results[0]):
title = paddle.dataset.movielens.movie_info()[int(re.id)].title
print(i, "\t", re.id, "\t", title, "\t", float(re.distance) * 5)
def test_search_multi_table_IP(search, args):
'''
target: test search multi tables of IP
method: add vectors into 10 tables, and search
expected: search status ok, the length of result
'''
num = 10
top_k = 10
nprobe = 1
tables = []
idx = []
for i in range(num):
table = gen_unique_str("test_add_multitable_%d" % i)
uri = "tcp://%s:%s" % (args["ip"], args["port"])
param = {
'table_name': table,
'dimension': dim,
'index_file_size': 10,
'metric_type': MetricType.L2
}
# create table
milvus = Milvus()
milvus.connect(uri=uri)
milvus.create_table(param)
status, ids = milvus.add_vectors(table, vectors)
assert status.OK()
assert len(ids) == len(vectors)
tables.append(table)
idx.append(ids[0])
idx.append(ids[10])
idx.append(ids[20])
time.sleep(6)
query_vecs = [vectors[0], vectors[10], vectors[20]]
# start query from random table
for i in range(num):
table = tables[i]
status, result = milvus.search_vectors(table, top_k, nprobe,
query_vecs)
assert status.OK()
assert len(result) == len(query_vecs)
for j in range(len(query_vecs)):
assert len(result[j]) == top_k
for j in range(len(query_vecs)):
assert check_result(result[j], idx[3 * i + j])
def test_not_connect(self):
client = Milvus()
with pytest.raises(NotConnectError):
client.create_table({})
with pytest.raises(NotConnectError):
client.has_table("a")
with pytest.raises(NotConnectError):
client.describe_table("a")
with pytest.raises(NotConnectError):
client.drop_table("a")
with pytest.raises(NotConnectError):
client.create_index("a")
with pytest.raises(NotConnectError):
client.insert("a", [], None)
with pytest.raises(NotConnectError):
client.count_table("a")
with pytest.raises(NotConnectError):
client.show_tables()
with pytest.raises(NotConnectError):
client.search("a", 1, 2, [])
with pytest.raises(NotConnectError):
client.search_in_files("a", [], [], 2, 1)
with pytest.raises(NotConnectError):
client._cmd("")
with pytest.raises(NotConnectError):
client.preload_table("a")
with pytest.raises(NotConnectError):
client.describe_index("a")
with pytest.raises(NotConnectError):
client.drop_index("")
def _test_add_vector_with_multiprocessing(self, args):
'''
target: test add vectors, with multi processes
method: 10 processed add vectors concurrently
expected: status ok and result length is equal to the length off added vectors
'''
table = gen_unique_str("test_add_vector_with_multiprocessing")
uri = "tcp://%s:%s" % (args["ip"], args["port"])
param = {
'table_name': table,
'dimension': dim,
'index_file_size': index_file_size
}
# create table
milvus = Milvus()
milvus.connect(uri=uri)
milvus.create_table(param)
vector = gen_single_vector(dim)
process_num = 4
loop_num = 10
processes = []
# with dependent connection
def add(milvus):
i = 0
while i < loop_num:
status, ids = milvus.add_vectors(table, vector)
i = i + 1
for i in range(process_num):
milvus = Milvus()
milvus.connect(uri=uri)
p = Process(target=add, args=(milvus, ))
processes.append(p)
p.start()
time.sleep(0.2)
for p in processes:
p.join()
time.sleep(3)
status, count = milvus.get_table_row_count(table)
assert count == process_num * loop_num
def _test_search_concurrent_multiprocessing(self, args):
'''
target: test concurrent search with multiprocessess
method: search with 10 processes, each process uses dependent connection
expected: status ok and the returned vectors should be query_records
'''
nb = 100
top_k = 10
process_num = 4
processes = []
table = gen_unique_str("test_search_concurrent_multiprocessing")
uri = "tcp://%s:%s" % (args["ip"], args["port"])
param = {
'table_name': table,
'dimension': dim,
'index_type': IndexType.FLAT,
'store_raw_vector': False
}
# create table
milvus = Milvus()
milvus.connect(uri=uri)
milvus.create_table(param)
vectors, ids = self.init_data(milvus, table, nb=nb)
query_vecs = vectors[nb // 2:nb]
def search(milvus):
status, result = milvus.search_vectors(table, top_k, query_vecs)
assert len(result) == len(query_vecs)
for i in range(len(query_vecs)):
assert result[i][0].id in ids
assert result[i][0].distance == 0.0
for i in range(process_num):
milvus = Milvus()
milvus.connect(uri=uri)
p = Process(target=search, args=(milvus, ))
processes.append(p)
p.start()
time.sleep(0.2)
for p in processes:
p.join()
def _create_table(_table_param):
milvus = Milvus()
milvus.connect(**server_config)
status, ok = milvus.has_table(_table_name)
if ok:
print("Table {} found, now going to delete it".format(_table_name))
status = milvus.delete_table(_table_name)
if not status.OK():
raise Exception("Delete table error")
print("delete table {} successfully!".format(_table_name))
time.sleep(5)
status, ok = milvus.has_table(_table_name)
if ok:
raise Exception("Delete table error")
status = milvus.create_table(param)
if not status.OK():
print("Create table {} failed".format(_table_name))
milvus.disconnect()
def main():
milvus = Milvus()
# Print client version
print('# Client version: {}'.format(milvus.client_version()))
# Connect milvus server
# Please change HOST and PORT to the correct one
param = {'host': _HOST, 'port': _PORT}
cnn_status = milvus.connect(**param)
print('# Connect Status: {}'.format(cnn_status))
# Check if connected
# is_connected = milvus.connected
print('# Is connected: {}'.format(milvus.connected))
# Print milvus server version
print('# Server version: {}'.format(milvus.server_version()))
# Describe table
table_name = 'table01'
res_status, table = milvus.describe_table(table_name)
print('# Describe table status: {}'.format(res_status))
print('# Describe table:{}'.format(table))
# Create table
# Check if `table01` exists, if not, create a table `table01`
dimension = 256
if not table:
param = {
'table_name': table_name,
'dimension': dimension,
'index_type': IndexType.IDMAP,
'store_raw_vector': False
}
res_status = milvus.create_table(Prepare.table_schema(**param))
print('# Create table status: {}'.format(res_status))
# Show tables and their description
status, tables = milvus.show_tables()
pprint(tables)
# Add vectors
# Prepare vector with 256 dimension
vectors = Prepare.records([[random.random() for _ in range(dimension)]
for _ in range(20)])
# Insert vectors into table 'table01'
status, ids = milvus.add_vectors(table_name=table_name, records=vectors)
print('# Add vector status: {}'.format(status))
pprint(ids)
# Search vectors
# When adding vectors for the first time, server will take at least 5s to
# persist vector data, so you have to wait for 6s after adding vectors for
# the first time.
print('# Waiting for 6s...')
time.sleep(6)
q_records = Prepare.records([[random.random() for _ in range(dimension)]
for _ in range(2)])
param = {
'table_name': table_name,
'query_records': q_records,
'top_k': 10,
}
status, results = milvus.search_vectors(**param)
print('# Search vectors status: {}'.format(status))
pprint(results)
# Get table row count
status, result = milvus.get_table_row_count(table_name)
print('# Status: {}'.format(status))
print('# Count: {}'.format(result))
# Disconnect
status = milvus.disconnect()
print('# Disconnect Status: {}'.format(status))
# Connect Milvus server.
# You may need to change HOST and PORT accordingly.
milvus.connect(host='192.168.1.233', port='19530')
# milvus.connect(host='127.0.0.1', port='19531')
# table_name = '500_501'
table_name = '{}_{}'.format(random.randint(800, 805), random.randint(800, 802))
# Table name is defined
# table_name = 'yyyyy'
# table_name = 'nnnns'
# table_name = 'test_search_K2YcLOHW'
# Create table: table name, vector dimension and index type
milvus.create_table(
dict(table_name=table_name, dimension=256, index_type=IndexType.FLAT))
# Insert 20 256-dim-vectors into demo_table
vectors = [[random.random() for _ in range(256)] for _ in range(200)]
milvus.add_vectors(table_name=table_name, records=vectors)
vectors = [[random.random() for _ in range(256)] for _ in range(10)]
param = {
'table_name': table_name,
'query_records': vectors,
'top_k': 5,
}
status, results = milvus.search_vectors(**param)
print(results)
print(status)
class MilvusClient(object):
def __init__(self, table_name=None, host=None, port=None):
self._milvus = Milvus()
self._table_name = table_name
try:
if not host:
self._milvus.connect(host=SERVER_HOST_DEFAULT,
port=SERVER_PORT_DEFAULT)
else:
self._milvus.connect(host=host, port=port)
except Exception as e:
raise e
def __str__(self):
return 'Milvus table %s' % self._table_name
def check_status(self, status):
if not status.OK():
logger.error(status.message)
raise Exception("Status not ok")
def create_table(self, table_name, dimension, index_file_size,
metric_type):
if not self._table_name:
self._table_name = table_name
if metric_type == "l2":
metric_type = MetricType.L2
elif metric_type == "ip":
metric_type = MetricType.IP
else:
logger.error("Not supported metric_type: %s" % metric_type)
self._metric_type = metric_type
create_param = {
'table_name': table_name,
'dimension': dimension,
'index_file_size': index_file_size,
"metric_type": metric_type
}
status = self._milvus.create_table(create_param)
self.check_status(status)
@time_wrapper
def insert(self, X, ids):
if self._metric_type == MetricType.IP:
logger.info("Set normalize for metric_type: Inner Product")
X = sklearn.preprocessing.normalize(X, axis=1, norm='l2')
X = X.astype(numpy.float32)
status, result = self._milvus.add_vectors(self._table_name,
X.tolist(),
ids=ids)
self.check_status(status)
return status, result
@time_wrapper
def create_index(self, index_type, nlist):
if index_type == "flat":
index_type = IndexType.FLAT
elif index_type == "ivf_flat":
index_type = IndexType.IVFLAT
elif index_type == "ivf_sq8":
index_type = IndexType.IVF_SQ8
elif index_type == "ivf_sq8h":
index_type = IndexType.IVF_SQ8H
elif index_type == "nsg":
index_type = IndexType.NSG
elif index_type == "ivf_pq":
index_type = IndexType.IVF_PQ
index_params = {
"index_type": index_type,
"nlist": nlist,
}
logger.info("Building index start, table_name: %s, index_params: %s" %
(self._table_name, json.dumps(index_params)))
status = self._milvus.create_index(self._table_name,
index=index_params,
timeout=6 * 3600)
self.check_status(status)
def describe_index(self):
return self._milvus.describe_index(self._table_name)
def drop_index(self):
logger.info("Drop index: %s" % self._table_name)
return self._milvus.drop_index(self._table_name)
@time_wrapper
def query(self, X, top_k, nprobe):
if self._metric_type == MetricType.IP:
logger.info("Set normalize for metric_type: Inner Product")
X = sklearn.preprocessing.normalize(X, axis=1, norm='l2')
X = X.astype(numpy.float32)
status, results = self._milvus.search_vectors(self._table_name, top_k,
nprobe, X.tolist())
self.check_status(status)
ids = []
for result in results:
tmp_ids = []
for item in result:
tmp_ids.append(item.id)
ids.append(tmp_ids)
return ids
def count(self):
return self._milvus.get_table_row_count(self._table_name)[1]
def delete(self, table_name):
logger.info("Start delete table: %s" % table_name)
return self._milvus.delete_table(table_name)
def describe(self):
return self._milvus.describe_table(self._table_name)
def exists_table(self, table_name):
return self._milvus.has_table(table_name)
def get_server_version(self):
status, res = self._milvus.server_version()
self.check_status(status)
return res
@time_wrapper
def preload_table(self):
return self._milvus.preload_table(self._table_name)
class MilvusClient(object):
def __init__(self, table_name=None, ip=None, port=None, timeout=60):
self._milvus = Milvus()
self._table_name = table_name
try:
i = 1
start_time = time.time()
if not ip:
self._milvus.connect(host=SERVER_HOST_DEFAULT,
port=SERVER_PORT_DEFAULT)
else:
# retry connect for remote server
while time.time() < start_time + timeout:
try:
self._milvus.connect(host=ip, port=port)
if self._milvus.connected() is True:
logger.debug(
"Try connect times: %d, %s" %
(i, round(time.time() - start_time, 2)))
break
except Exception as e:
logger.debug("Milvus connect failed")
i = i + 1
except Exception as e:
raise e
def __str__(self):
return 'Milvus table %s' % self._table_name
def check_status(self, status):
if not status.OK():
logger.error(status.message)
# raise Exception("Status not ok")
def create_table(self, table_name, dimension, index_file_size,
metric_type):
if not self._table_name:
self._table_name = table_name
if metric_type == "l2":
metric_type = MetricType.L2
elif metric_type == "ip":
metric_type = MetricType.IP
elif metric_type == "jaccard":
metric_type = MetricType.JACCARD
elif metric_type == "hamming":
metric_type = MetricType.HAMMING
else:
logger.error("Not supported metric_type: %s" % metric_type)
create_param = {
'table_name': table_name,
'dimension': dimension,
'index_file_size': index_file_size,
"metric_type": metric_type
}
status = self._milvus.create_table(create_param)
self.check_status(status)
@time_wrapper
def insert(self, X, ids=None):
status, result = self._milvus.add_vectors(self._table_name, X, ids)
self.check_status(status)
return status, result
@time_wrapper
def create_index(self, index_type, nlist):
index_params = {
"index_type": INDEX_MAP[index_type],
"nlist": nlist,
}
logger.info("Building index start, table_name: %s, index_params: %s" %
(self._table_name, json.dumps(index_params)))
status = self._milvus.create_index(self._table_name,
index=index_params)
self.check_status(status)
def describe_index(self):
status, result = self._milvus.describe_index(self._table_name)
index_type = None
for k, v in INDEX_MAP.items():
if result._index_type == v:
index_type = k
break
nlist = result._nlist
res = {"index_type": index_type, "nlist": nlist}
return res
def drop_index(self):
logger.info("Drop index: %s" % self._table_name)
return self._milvus.drop_index(self._table_name)
@time_wrapper
def query(self, X, top_k, nprobe):
status, result = self._milvus.search_vectors(self._table_name, top_k,
nprobe, X)
self.check_status(status)
return result
def count(self):
return self._milvus.get_table_row_count(self._table_name)[1]
def delete(self, timeout=60):
logger.info("Start delete table: %s" % self._table_name)
self._milvus.delete_table(self._table_name)
i = 0
while i < timeout:
if self.count():
time.sleep(1)
i = i + 1
continue
else:
break
if i >= timeout:
logger.error("Delete table timeout")
def describe(self):
return self._milvus.describe_table(self._table_name)
def show_tables(self):
return self._milvus.show_tables()
def exists_table(self, table_name=None):
if table_name is None:
table_name = self._table_name
status, res = self._milvus.has_table(table_name)
self.check_status(status)
return res
@time_wrapper
def preload_table(self):
return self._milvus.preload_table(self._table_name, timeout=3000)
def get_server_version(self):
status, res = self._milvus.server_version()
return res
def get_server_mode(self):
return self.cmd("mode")
def get_server_commit(self):
return self.cmd("build_commit_id")
def get_server_config(self):
return json.loads(self.cmd("get_config *"))
def get_mem_info(self):
result = json.loads(self.cmd("get_system_info"))
result_human = {
# unit: Gb
"memory_used":
round(int(result["memory_used"]) / (1024 * 1024 * 1024), 2)
}
return result_human
def cmd(self, command):
status, res = self._milvus._cmd(command)
logger.info("Server command: %s, result: %s" % (command, res))
self.check_status(status)
return res
import random
import time
milvus = Milvus()
# Connect Milvus server.
# You may need to change HOST and PORT accordingly.
milvus.connect(host='localhost', port='19530')
# Table name is defined
table_name = 'demo_table_02'
# Create table: table name, vector dimension and index type
if not milvus.has_table(table_name):
milvus.create_table({
'table_name': table_name,
'dimension': 256,
'index_file_size': 1024,
'metric_type': MetricType.L2
})
# Insert 20 256-dim-vectors into demo_table
vectors = [[random.random() for _ in range(256)] for _ in range(20)]
milvus.add_vectors(table_name=table_name, records=vectors)
time.sleep(1)
# Get table row count
_, result = milvus.get_table_row_count(table_name=table_name)
print('Table {}, row counts: {}'.format(table_name, result))
from milvus import Milvus, Prepare, IndexType, Status
import random
milvus = Milvus()
# Connect Milvus server, please change HOST and PORT to correct one
milvus.connect(host='localhost', port='33001')
# Table name is defined
table_name = 'table_' + str(random.randint(0, 100))
# Create table: table name, vector dimension and index type
milvus.create_table(
Prepare.table_schema(table_name, dimension=256,
index_type=IndexType.IDMAP))
# Add 20 256-dim-vectors into table
vectors = Prepare.records([[random.random() for _ in range(256)]
for _ in range(20)])
milvus.add_vectors(table_name=table_name, records=vectors)
# Get table row count
_, result = milvus.get_table_row_count(table_name=table_name)
print('Table {}, row counts: {}'.format(table_name, result))
class MilvusClient(object):
def __init__(self, table_name=None, ip=None, port=None):
self._milvus = Milvus()
self._table_name = table_name
try:
if not ip:
self._milvus.connect(
host = SERVER_HOST_DEFAULT,
port = SERVER_PORT_DEFAULT)
else:
self._milvus.connect(
host = ip,
port = port)
except Exception as e:
raise e
def __str__(self):
return 'Milvus table %s' % self._table_name
def check_status(self, status):
if not status.OK():
logger.error(status.message)
raise Exception("Status not ok")
def create_table(self, table_name, dimension, index_file_size, metric_type):
if not self._table_name:
self._table_name = table_name
if metric_type == "l2":
metric_type = MetricType.L2
elif metric_type == "ip":
metric_type = MetricType.IP
else:
logger.error("Not supported metric_type: %s" % metric_type)
create_param = {'table_name': table_name,
'dimension': dimension,
'index_file_size': index_file_size,
"metric_type": metric_type}
status = self._milvus.create_table(create_param)
self.check_status(status)
@time_wrapper
def insert(self, X, ids=None):
status, result = self._milvus.add_vectors(self._table_name, X, ids)
self.check_status(status)
return status, result
@time_wrapper
def create_index(self, index_type, nlist):
if index_type == "flat":
index_type = IndexType.FLAT
elif index_type == "ivf_flat":
index_type = IndexType.IVFLAT
elif index_type == "ivf_sq8":
index_type = IndexType.IVF_SQ8
elif index_type == "nsg":
index_type = IndexType.NSG
elif index_type == "ivf_sq8h":
index_type = IndexType.IVF_SQ8H
elif index_type == "ivf_pq":
index_type = IndexType.IVF_PQ
index_params = {
"index_type": index_type,
"nlist": nlist,
}
logger.info("Building index start, table_name: %s, index_params: %s" % (self._table_name, json.dumps(index_params)))
status = self._milvus.create_index(self._table_name, index=index_params)
self.check_status(status)
def describe_index(self):
return self._milvus.describe_index(self._table_name)
def drop_index(self):
logger.info("Drop index: %s" % self._table_name)
return self._milvus.drop_index(self._table_name)
@time_wrapper
def query(self, X, top_k, nprobe):
status, result = self._milvus.search_vectors(self._table_name, top_k, nprobe, X)
self.check_status(status)
return status, result
def count(self):
return self._milvus.get_table_row_count(self._table_name)[1]
def delete(self, timeout=60):
logger.info("Start delete table: %s" % self._table_name)
self._milvus.delete_table(self._table_name)
i = 0
while i < timeout:
if self.count():
time.sleep(1)
i = i + 1
continue
else:
break
if i < timeout:
logger.error("Delete table timeout")
def describe(self):
return self._milvus.describe_table(self._table_name)
def show_tables(self):
return self._milvus.show_tables()
def exists_table(self):
status, res = self._milvus.has_table(self._table_name)
self.check_status(status)
return res
@time_wrapper
def preload_table(self):
return self._milvus.preload_table(self._table_name, timeout=3000)
def main():
milvus = Milvus(handler="HTTP")
# Connect to Milvus server
# You may need to change _HOST and _PORT accordingly
param = {'host': _HOST, 'port': _PORT}
status = milvus.connect(**param)
if status.OK():
print("Server connected.")
else:
print("Server connect fail.")
sys.exit(1)
# Create table demo_table if it dosen't exist.
table_name = 'demo_tables'
status, ok = milvus.has_table(table_name)
if not ok:
param = {
'table_name': table_name,
'dimension': _DIM,
'index_file_size': _INDEX_FILE_SIZE, # optional
'metric_type': MetricType.L2 # optional
}
milvus.create_table(param)
# Show tables in Milvus server
_, tables = milvus.show_tables()
# Describe demo_table
_, table = milvus.describe_table(table_name)
print(table)
# 10000 vectors with 16 dimension
# element per dimension is float32 type
# vectors should be a 2-D array
vectors = [[random.random() for _ in range(_DIM)] for _ in range(100000)]
# You can also use numpy to generate random vectors:
# `vectors = np.random.rand(10000, 16).astype(np.float32).tolist()`
# Insert vectors into demo_table, return status and vectors id list
status, ids = milvus.insert(table_name=table_name, records=vectors)
# Wait for 6 seconds, until Milvus server persist vector data.
time.sleep(6)
# Get demo_table row count
status, result = milvus.count_table(table_name)
# create index of vectors, search more rapidly
index_param = {
'index_type': IndexType.IVFLAT, # choice ivflat index
'nlist': 2048
}
# Create ivflat index in demo_table
# You can search vectors without creating index. however, Creating index help to
# search faster
status = milvus.create_index(table_name, index_param)
# describe index, get information of index
status, index = milvus.describe_index(table_name)
print(index)
# Use the top 10 vectors for similarity search
query_vectors = vectors[0:10]
# execute vector similarity search
param = {
'table_name': table_name,
'query_records': query_vectors,
'top_k': 1,
'nprobe': 16
}
status, results = milvus.search(**param)
if status.OK():
# indicate search result
# also use by:
# `results.distance_array[0][0] == 0.0 or results.id_array[0][0] == ids[0]`
if results[0][0].distance == 0.0 or results[0][0].id == ids[0]:
print('Query result is correct')
else:
print('Query result isn\'t correct')
# print results
print(results)
# Delete demo_table
status = milvus.drop_table(table_name)
# Disconnect from Milvus
status = milvus.disconnect()
def main():
milvus = Milvus()
# Connect to Milvus server
# You may need to change _HOST and _PORT accordingly
param = {'host': _HOST, 'port': _PORT}
status = milvus.connect(**param)
# Create table demo_table if it dosen't exist.
table_name = 'demo_table_01'
if not milvus.has_table(table_name):
param = {
'table_name': table_name,
'dimension': 16,
'index_file_size': 1024,
'metric_type': MetricType.L2
}
milvus.create_table(param)
# Show tables in Milvus server
_, tables = milvus.show_tables()
# Describe demo_table
_, table = milvus.describe_table(table_name)
# 10 vectors with 16 dimension
vectors = [
[
0.66, 0.01, 0.29, 0.64, 0.75, 0.94, 0.26, 0.79, 0.61, 0.11, 0.25,
0.50, 0.74, 0.37, 0.28, 0.63
],
[
0.77, 0.65, 0.57, 0.68, 0.29, 0.93, 0.17, 0.15, 0.95, 0.09, 0.78,
0.37, 0.76, 0.21, 0.42, 0.15
],
[
0.61, 0.38, 0.32, 0.39, 0.54, 0.93, 0.09, 0.81, 0.52, 0.30, 0.20,
0.59, 0.15, 0.27, 0.04, 0.37
],
[
0.33, 0.03, 0.87, 0.47, 0.79, 0.61, 0.46, 0.77, 0.62, 0.70, 0.85,
0.01, 0.30, 0.41, 0.74, 0.98
],
[
0.19, 0.80, 0.03, 0.75, 0.22, 0.49, 0.52, 0.91, 0.40, 0.91, 0.79,
0.08, 0.27, 0.16, 0.07, 0.24
],
[
0.44, 0.36, 0.16, 0.88, 0.30, 0.79, 0.45, 0.31, 0.45, 0.99, 0.15,
0.93, 0.37, 0.25, 0.78, 0.84
],
[
0.33, 0.37, 0.59, 0.66, 0.76, 0.11, 0.19, 0.38, 0.14, 0.37, 0.97,
0.50, 0.08, 0.69, 0.16, 0.67
],
[
0.68, 0.97, 0.20, 0.13, 0.30, 0.16, 0.85, 0.21, 0.26, 0.17, 0.81,
0.96, 0.18, 0.40, 0.13, 0.74
],
[
0.11, 0.26, 0.44, 0.91, 0.89, 0.79, 0.98, 0.91, 0.09, 0.45, 0.07,
0.88, 0.71, 0.35, 0.97, 0.41
],
[
0.17, 0.54, 0.61, 0.58, 0.25, 0.63, 0.65, 0.71, 0.26, 0.80, 0.28,
0.77, 0.69, 0.02, 0.63, 0.60
],
]
# Insert vectors into demo_table
status, ids = milvus.add_vectors(table_name=table_name, records=vectors)
# Wait for 6 seconds, since Milvus server persist vector data every 5 seconds by default.
# You can set data persist interval in Milvus config file.
time.sleep(6)
# Get demo_table row count
status, result = milvus.get_table_row_count(table_name)
# Use the 3rd vector for similarity search
query_vectors = [vectors[3]]
# execute vector similarity search
param = {
'table_name': table_name,
'query_records': query_vectors,
'top_k': 1,
'nprobe': 16
}
status, results = milvus.search_vectors(**param)
if results[0][0].distance == 0.0 or results[0][0].id == ids[3]:
print('Query result is correct')
else:
print('Query result isn\'t correct')
# Delete demo_table
status = milvus.delete_table(table_name)
# Disconnect from Milvus
status = milvus.disconnect()
# connect.connect(host="192.168.1.27")
# print(connect.show_tables())
# print(connect.get_table_row_count(table))
# sys.exit()
connect.connect(host="127.0.0.1")
connect.delete_table(table)
# sys.exit()
# time.sleep(2)
print(connect.get_table_row_count(table))
param = {
'table_name': table,
'dimension': dim,
'metric_type': MetricType.L2,
'index_file_size': 10
}
status = connect.create_table(param)
print(status)
print(connect.get_table_row_count(table))
# add vectors
for i in range(10):
status, ids = connect.add_vectors(table, vectors)
print(status)
print(ids[0])
# print(ids[0])
index_params = {"index_type": IndexType.IVFLAT, "nlist": 16384}
status = connect.create_index(table, index_params)
print(status)
# sys.exit()
query_vec = [vectors[0]]
# print(numpy.inner(numpy.array(query_vec[0]), numpy.array(query_vec[0])))
top_k = 12
# This program demos how to create a Milvus table, insert 20 vectors and get the table row count.
from milvus import Milvus, Prepare, IndexType, Status
import random, time
milvus = Milvus()
# Connect Milvus server.
# You may need to change HOST and PORT accordingly.
milvus.connect(host='localhost', port='19530')
# Table name is defined
table_name = 'demo_table'
# Create table: table name, vector dimension and index type
if not milvus.has_table(table_name):
milvus.create_table({
'table_name': table_name,
'dimension': 256,
'index_type': IndexType.FLAT
})
# Insert 20 256-dim-vectors into demo_table
vectors = [[random.random() for _ in range(256)] for _ in range(20)]
milvus.add_vectors(table_name=table_name, records=vectors)
time.sleep(1)
# Get table row count
_, result = milvus.get_table_row_count(table_name=table_name)
print('Table {}, row counts: {}'.format(table_name, result))