def test_and_argegarion(all_values):
for values1 in [{'guid_0', 'guid_1'}, {'guid_3', 'guid_6', 'guid_20'},
{'guid_2'}]:
for values2 in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_9'},
{'guid_2'}]:
test_predicate = in_reduce([
in_set(values1, 'volume_guid'),
in_set(values2, 'volume_guid')
], all)
included_values = set()
for val in all_values:
if test_predicate.do_include({'volume_guid': val}):
included_values.add(val)
assert included_values == values1.intersection(values2)
def test_or_argegarion(self):
for values1 in [{'guid_0', 'guid_1'}, {'guid_3', 'guid_6', 'guid_20'},
{'guid_2'}]:
for values2 in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_9'},
{'guid_2'}]:
test_predicate = in_reduce([
in_set(values1, 'volume_guid'),
in_set(values2, 'volume_guid')
], any)
included_values = set()
for val in PredicatesTest.all_values:
if test_predicate.do_include({'volume_guid': val}):
included_values.add(val)
self.assertEqual(included_values, values1.union(values2))
def test_negation(all_values):
for values in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_9'}, {'guid_2'}]:
test_predicate = in_negate(in_set(values, 'volume_guid'))
included_values = set()
for val in all_values:
if test_predicate.do_include({'volume_guid': val}):
included_values.add(val)
assert included_values == all_values.difference(values)
def test_inclusion(all_values):
for values in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_XXX'}, {'guid_2'}]:
test_predicate = in_set(values, 'volume_guid')
included_values = set()
for val in all_values:
if test_predicate.do_include({'volume_guid': val}):
included_values.add(val)
assert included_values == all_values.intersection(values)
def test_negation(self):
for values in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_9'}, {'guid_2'}]:
test_predicate = in_negate(in_set(values, 'volume_guid'))
included_values = set()
for val in PredicatesTest.all_values:
if test_predicate.do_include({'volume_guid': val}):
included_values.add(val)
self.assertEqual(included_values,
PredicatesTest.all_values.difference(values))
def test_inclusion(self):
for values in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_XXX'},
{'guid_2'}]:
test_predicate = in_set(values, 'volume_guid')
included_values = set()
for val in PredicatesTest.all_values:
if test_predicate.do_include({'volume_guid': val}):
included_values.add(val)
self.assertEqual(included_values,
PredicatesTest.all_values.intersection(values))
def just_read_batch(dataset_url='file:///tmp/benchmark_dataset'):
values = [5]
predicate = in_set(values, "id")
with make_batch_carbon_reader(dataset_url, num_epochs=1, workers_count=16,
predicate=predicate) as train_reader:
i = 0
for schema_view in train_reader:
assert schema_view.id == 5
i += len(schema_view.id)
print(i)
def just_unified_read(dataset_url='file:///tmp/benchmark_dataset'):
values = [5]
predicate = in_set(values, "id")
properties = {
"predicate": predicate,
}
with make_reader(dataset_url,
is_batch=False,
num_epochs=1,
workers_count=16,
**properties) as train_reader:
i = 0
for schema_view in train_reader:
assert schema_view.id == 5
i += 1
assert i == 1
return i
