def test_compact_merge_inside_time_travel(self):
"""
target: test compact and merge segments inside time_travel range
method: search with time travel after merge compact
expected: Verify segments inside time_travel merged
"""
from pymilvus import utility
# create collection shard_num=1, insert 2 segments, each with tmp_nb entities
collection_w = self.init_collection_wrap(name=cf.gen_unique_str(prefix), shards_num=1)
# insert twice
df1 = cf.gen_default_dataframe_data(tmp_nb)
collection_w.insert(df1)[0]
assert collection_w.num_entities == tmp_nb
df2 = cf.gen_default_dataframe_data(tmp_nb, start=tmp_nb)
insert_two = collection_w.insert(df2)[0]
assert collection_w.num_entities == tmp_nb * 2
tt = utility.mkts_from_hybridts(insert_two.timestamp, milliseconds=0.1)
collection_w.compact()
collection_w.wait_for_compaction_completed()
collection_w.get_compaction_plans(check_task=CheckTasks.check_merge_compact)
collection_w.load()
search_res, _ = collection_w.search(df2[ct.default_float_vec_field_name][:1].to_list(),
ct.default_float_vec_field_name,
ct.default_search_params, ct.default_limit,
travel_timestamp=tt)
assert tmp_nb in search_res[0].ids
assert len(search_res[0]) == ct.default_limit
def test_compact_delete_inside_time_travel(self):
"""
target: test compact inside time_travel range
method: 1.insert data and get ts
2.delete all ids
4.compact
5.search with ts
expected: Verify search result
"""
from pymilvus import utility
collection_w = self.init_collection_wrap(cf.gen_unique_str(prefix), shards_num=1)
# insert and get tt
df = cf.gen_default_dataframe_data(tmp_nb)
insert_res, _ = collection_w.insert(df)
tt = utility.mkts_from_hybridts(insert_res.timestamp, milliseconds=0.)
# delete all
expr = f'{ct.default_int64_field_name} in {insert_res.primary_keys}'
delete_res, _ = collection_w.delete(expr)
log.debug(collection_w.num_entities)
collection_w.compact()
collection_w.wait_for_compaction_completed()
collection_w.get_compaction_plans()
collection_w.load()
search_one, _ = collection_w.search(df[ct.default_float_vec_field_name][:1].to_list(),
ct.default_float_vec_field_name,
ct.default_search_params, ct.default_limit,
travel_timestamp=tt)
assert 0 in search_one[0].ids
def test_compact_delete_outside_time_travel(self):
"""
target: test compact outside time_travel range
method: 1.create and insert
2.get time stamp
3.delete
4.compact after compact_retention_duration
5.load and search with travel time tt
expected: Empty search result
But no way to verify, because travel time does not support travel back to retentionDuration ago so far
"""
from pymilvus import utility
collection_w = self.init_collection_wrap(cf.gen_unique_str(prefix),
shards_num=1)
# insert
df = cf.gen_default_dataframe_data(tmp_nb)
insert_res, _ = collection_w.insert(df)
tt = utility.mkts_from_hybridts(insert_res.timestamp, milliseconds=0.)
expr = f'{ct.default_int64_field_name} in {insert_res.primary_keys}'
delete_res, _ = collection_w.delete(expr)
log.debug(collection_w.num_entities)
# ensure compact remove delta data that delete outside retention range
sleep(ct.compact_retention_duration + 1)
collection_w.compact()
collection_w.wait_for_compaction_completed()
collection_w.get_compaction_plans(
check_task=CheckTasks.check_delete_compact)
collection_w.load()
# search with travel_time tt
collection_w.search(df[ct.default_float_vec_field_name][:1].to_list(),
ct.default_float_vec_field_name,
ct.default_search_params,
ct.default_limit,
travel_timestamp=tt,
check_task=CheckTasks.err_res,
check_items={
ct.err_code: 1,
ct.err_msg: "only support to travel back to"
})
def test_compact_delete_outside_time_travel(self):
"""
target: test compact outside time_travel range
method: 1.create and insert
2.get time stamp
3.delete
4.compact after compact_retention_duration
5.load and search with travel time tt
expected: Empty search result
"""
from pymilvus import utility
collection_w = self.init_collection_wrap(cf.gen_unique_str(prefix),
shards_num=1)
# insert
df = cf.gen_default_dataframe_data(tmp_nb)
insert_res, _ = collection_w.insert(df)
tt = utility.mkts_from_hybridts(insert_res.timestamp, milliseconds=0.)
expr = f'{ct.default_int64_field_name} in {insert_res.primary_keys}'
delete_res, _ = collection_w.delete(expr)
log.debug(collection_w.num_entities)
# ensure compact remove delta data that delete outside retention range
# sleep(ct.compact_retention_duration)
sleep(60)
collection_w.compact()
collection_w.wait_for_compaction_completed()
collection_w.load()
# search with travel_time tt
search_res, _ = collection_w.search(
df[ct.default_float_vec_field_name][:1].to_list(),
ct.default_float_vec_field_name,
ct.default_search_params,
ct.default_limit,
travel_timestamp=tt)
log.debug(search_res[0].ids)
assert len(search_res[0]) == 0