def test_and_argegarion(all_values): for values1 in [{'guid_0', 'guid_1'}, {'guid_3', 'guid_6', 'guid_20'}, {'guid_2'}]: for values2 in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_9'}, {'guid_2'}]: test_predicate = in_reduce([ in_set(values1, 'volume_guid'), in_set(values2, 'volume_guid') ], all) included_values = set() for val in all_values: if test_predicate.do_include({'volume_guid': val}): included_values.add(val) assert included_values == values1.intersection(values2)
def test_or_argegarion(self): for values1 in [{'guid_0', 'guid_1'}, {'guid_3', 'guid_6', 'guid_20'}, {'guid_2'}]: for values2 in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_9'}, {'guid_2'}]: test_predicate = in_reduce([ in_set(values1, 'volume_guid'), in_set(values2, 'volume_guid') ], any) included_values = set() for val in PredicatesTest.all_values: if test_predicate.do_include({'volume_guid': val}): included_values.add(val) self.assertEqual(included_values, values1.union(values2))
def test_negation(all_values): for values in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_9'}, {'guid_2'}]: test_predicate = in_negate(in_set(values, 'volume_guid')) included_values = set() for val in all_values: if test_predicate.do_include({'volume_guid': val}): included_values.add(val) assert included_values == all_values.difference(values)
def test_inclusion(all_values): for values in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_XXX'}, {'guid_2'}]: test_predicate = in_set(values, 'volume_guid') included_values = set() for val in all_values: if test_predicate.do_include({'volume_guid': val}): included_values.add(val) assert included_values == all_values.intersection(values)
def test_negation(self): for values in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_9'}, {'guid_2'}]: test_predicate = in_negate(in_set(values, 'volume_guid')) included_values = set() for val in PredicatesTest.all_values: if test_predicate.do_include({'volume_guid': val}): included_values.add(val) self.assertEqual(included_values, PredicatesTest.all_values.difference(values))
def test_inclusion(self): for values in [{'guid_2', 'guid_1'}, {'guid_5', 'guid_XXX'}, {'guid_2'}]: test_predicate = in_set(values, 'volume_guid') included_values = set() for val in PredicatesTest.all_values: if test_predicate.do_include({'volume_guid': val}): included_values.add(val) self.assertEqual(included_values, PredicatesTest.all_values.intersection(values))
def just_read_batch(dataset_url='file:///tmp/benchmark_dataset'): values = [5] predicate = in_set(values, "id") with make_batch_carbon_reader(dataset_url, num_epochs=1, workers_count=16, predicate=predicate) as train_reader: i = 0 for schema_view in train_reader: assert schema_view.id == 5 i += len(schema_view.id) print(i)
def just_unified_read(dataset_url='file:///tmp/benchmark_dataset'): values = [5] predicate = in_set(values, "id") properties = { "predicate": predicate, } with make_reader(dataset_url, is_batch=False, num_epochs=1, workers_count=16, **properties) as train_reader: i = 0 for schema_view in train_reader: assert schema_view.id == 5 i += 1 assert i == 1 return i