def test_filter_with_in_key_value(self): keys = [{ 'name': 'country_code', 'value': 'es', 'operator': 'contains' }] items = [{ 'name': 'item1', 'country_code': ['es', 'us'] }, { 'name': 'item2', 'country_code': ['es', 'us'] }, { 'name': 'item3', 'country_code': ['uk'] }] batch = [] for item in items: record = BaseRecord(item) batch.append(record) filter = KeyValueFilter({'options': {'keys': keys}}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertEqual(2, len(batch))
def test_filter_with_no_nested_key(self): keys = [{'name': 'not_a_key', 'value': 'val'}] batch = [{ 'country': { 'state': { 'city': random.choice(['val', 'es', 'uk']) } }, 'value': random.randint(0, 1000) } for i in range(100)] filter = KeyValueFilter({'options': {'keys': keys}}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertEqual(len(batch), 0, 'Resulting filtered batch should be empty')
def test_filter_with_nested_key_value(self): keys = [{'name': 'country.state.city', 'value': 'val'}] batch = [{ 'country': { 'state': { 'city': random.choice(['val', 'es', 'uk']) } }, 'value': random.randint(0, 1000) } for i in range(100)] filter = KeyValueFilter({'options': {'keys': keys}}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertGreater(len(batch), 0) for item in batch: self.assertEqual(item['country']['state']['city'], 'val')
def test_filter_with_no_nested_key(self): keys = [ {'name': 'not_a_key', 'value': 'val'} ] batch = [ {'country': { 'state': { 'city': random.choice(['val', 'es', 'uk']) } }, 'value': random.randint(0, 1000)} for i in range(100) ] filter = KeyValueFilter( {'options': {'keys': keys}}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertEqual(len(batch), 0, 'Resulting filtered batch should be empty')
def test_filter_with_nested_key_value(self): keys = [ {'name': 'country.state.city', 'value': 'val'} ] batch = [ {'country': { 'state': { 'city': random.choice(['val', 'es', 'uk']) } }, 'value': random.randint(0, 1000)} for i in range(100) ] filter = KeyValueFilter({'options': {'keys': keys}}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertGreater(len(batch), 0) for item in batch: self.assertEqual(item['country']['state']['city'], 'val')
def test_filter_with_in_key_value(self): keys = [ {'name': 'country_code', 'value': 'es', 'operator': 'contains'} ] items = [ {'name': 'item1', 'country_code': ['es', 'us']}, {'name': 'item2', 'country_code': ['es', 'us']}, {'name': 'item3', 'country_code': ['uk']} ] batch = [] for item in items: record = BaseRecord(item) batch.append(record) filter = KeyValueFilter({'options': {'keys': keys}}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertEqual(2, len(batch))
def setUp(self): self.options = { 'exporter_options': { 'log_level': 'DEBUG', 'logger_name': 'export-pipeline' } } self.keys = [{'name': 'country_code', 'value': 'es'}] items = [{ 'name': 'item1', 'country_code': 'es' }, { 'name': 'item2', 'country_code': 'uk' }] self.batch = [] for item in items: record = BaseRecord(item) self.batch.append(record) self.filter = KeyValueFilter({'options': {'keys': self.keys}}, meta())
def setUp(self): self.options = { 'exporter_options': { 'log_level': 'DEBUG', 'logger_name': 'export-pipeline' } } self.keys = [ {'name': 'country_code', 'value': 'es'} ] items = [{'name': 'item1', 'country_code': 'es'}, {'name': 'item2', 'country_code': 'uk'}] self.batch = [] for item in items: record = BaseRecord(item) self.batch.append(record) self.filter = KeyValueFilter({'options': {'keys': self.keys}}, meta())
def test_filter_with_non_existing_op(self): keys = [{ 'name': 'country_code', 'value': ['es', 'us'], 'operator': 'not_an_operator' }] items = [{ 'name': 'item1', 'country_code': 'es' }, { 'name': 'item2', 'country_code': 'us' }, { 'name': 'item3', 'country_code': 'uk' }] batch = [] for item in items: record = BaseRecord(item) batch.append(record) with self.assertRaisesRegexp(InvalidOperator, 'operator not valid'): KeyValueFilter({'options': {'keys': keys}}, meta())
class KeyValueFilterTest(unittest.TestCase): def setUp(self): self.options = { 'exporter_options': { 'log_level': 'DEBUG', 'logger_name': 'export-pipeline' } } self.keys = [{'name': 'country_code', 'value': 'es'}] items = [{ 'name': 'item1', 'country_code': 'es' }, { 'name': 'item2', 'country_code': 'uk' }] self.batch = [] for item in items: record = BaseRecord(item) self.batch.append(record) self.filter = KeyValueFilter({'options': {'keys': self.keys}}, meta()) def test_filter_with_key_value(self): batch = self.filter.filter_batch(self.batch) batch = list(batch) self.assertEqual(1, len(batch)) self.assertEqual('es', dict(batch[0])['country_code']) def test_filter_logs(self): batch = [{ 'country': random.choice(['es', 'uk']), 'value': random.randint(0, 1000) } for i in range(5000)] # No exception should be raised self.filter.filter_batch(batch) def test_filter_with_nested_key_value(self): keys = [{'name': 'country.state.city', 'value': 'val'}] batch = [{ 'country': { 'state': { 'city': random.choice(['val', 'es', 'uk']) } }, 'value': random.randint(0, 1000) } for i in range(100)] filter = KeyValueFilter({'options': {'keys': keys}}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertGreater(len(batch), 0) for item in batch: self.assertEqual(item['country']['state']['city'], 'val') def test_filter_with_nested_key_value_with_comma(self): keys = [{'name': 'country,state,city', 'value': 'val'}] batch = [{ 'country': { 'state': { 'city': random.choice(['val', 'es', 'uk']) } }, 'value': random.randint(0, 1000) } for i in range(100)] filter = KeyValueFilter( {'options': { 'keys': keys, 'nested_field_separator': ',' }}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertGreater(len(batch), 0) for item in batch: self.assertEqual(item['country']['state']['city'], 'val') def test_filter_with_no_nested_key(self): keys = [{'name': 'not_a_key', 'value': 'val'}] batch = [{ 'country': { 'state': { 'city': random.choice(['val', 'es', 'uk']) } }, 'value': random.randint(0, 1000) } for i in range(100)] filter = KeyValueFilter({'options': {'keys': keys}}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertEqual(len(batch), 0, 'Resulting filtered batch should be empty')
class KeyValueFilterTest(unittest.TestCase): def setUp(self): self.options = { 'exporter_options': { 'log_level': 'DEBUG', 'logger_name': 'export-pipeline' } } self.keys = [ {'name': 'country_code', 'value': 'es'} ] items = [{'name': 'item1', 'country_code': 'es'}, {'name': 'item2', 'country_code': 'uk'}] self.batch = [] for item in items: record = BaseRecord(item) self.batch.append(record) self.filter = KeyValueFilter({'options': {'keys': self.keys}}, meta()) def test_filter_with_key_value(self): batch = self.filter.filter_batch(self.batch) batch = list(batch) self.assertEqual(1, len(batch)) self.assertEqual('es', dict(batch[0])['country_code']) def test_filter_logs(self): batch = [ { 'country': random.choice(['es', 'uk']), 'value': random.randint(0, 1000)} for i in range(5000) ] # No exception should be raised self.filter.filter_batch(batch) def test_filter_with_nested_key_value(self): keys = [ {'name': 'country.state.city', 'value': 'val'} ] batch = [ {'country': { 'state': { 'city': random.choice(['val', 'es', 'uk']) } }, 'value': random.randint(0, 1000)} for i in range(100) ] filter = KeyValueFilter({'options': {'keys': keys}}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertGreater(len(batch), 0) for item in batch: self.assertEqual(item['country']['state']['city'], 'val') def test_filter_with_nested_key_value_with_comma(self): keys = [ {'name': 'country,state,city', 'value': 'val'} ] batch = [ {'country': { 'state': { 'city': random.choice(['val', 'es', 'uk']) } }, 'value': random.randint(0, 1000)} for i in range(100) ] filter = KeyValueFilter( {'options': {'keys': keys, 'nested_field_separator': ','}}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertGreater(len(batch), 0) for item in batch: self.assertEqual(item['country']['state']['city'], 'val') def test_filter_with_no_nested_key(self): keys = [ {'name': 'not_a_key', 'value': 'val'} ] batch = [ {'country': { 'state': { 'city': random.choice(['val', 'es', 'uk']) } }, 'value': random.randint(0, 1000)} for i in range(100) ] filter = KeyValueFilter( {'options': {'keys': keys}}, meta()) batch = filter.filter_batch(batch) batch = list(batch) self.assertEqual(len(batch), 0, 'Resulting filtered batch should be empty')