def test_filter_with_in_key_value(self):
keys = [{
'name': 'country_code',
'value': 'es',
'operator': 'contains'
}]
items = [{
'name': 'item1',
'country_code': ['es', 'us']
}, {
'name': 'item2',
'country_code': ['es', 'us']
}, {
'name': 'item3',
'country_code': ['uk']
}]
batch = []
for item in items:
record = BaseRecord(item)
batch.append(record)
filter = KeyValueFilter({'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertEqual(2, len(batch))
def test_filter_with_no_nested_key(self):
keys = [{'name': 'not_a_key', 'value': 'val'}]
batch = [{
'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
},
'value': random.randint(0, 1000)
} for i in range(100)]
filter = KeyValueFilter({'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertEqual(len(batch), 0,
'Resulting filtered batch should be empty')
def test_filter_with_nested_key_value(self):
keys = [{'name': 'country.state.city', 'value': 'val'}]
batch = [{
'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
},
'value': random.randint(0, 1000)
} for i in range(100)]
filter = KeyValueFilter({'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertGreater(len(batch), 0)
for item in batch:
self.assertEqual(item['country']['state']['city'], 'val')
def test_filter_with_no_nested_key(self):
keys = [
{'name': 'not_a_key', 'value': 'val'}
]
batch = [
{'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
}, 'value': random.randint(0, 1000)} for i in range(100)
]
filter = KeyValueFilter(
{'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertEqual(len(batch), 0, 'Resulting filtered batch should be empty')
def test_filter_with_nested_key_value(self):
keys = [
{'name': 'country.state.city', 'value': 'val'}
]
batch = [
{'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
}, 'value': random.randint(0, 1000)} for i in range(100)
]
filter = KeyValueFilter({'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertGreater(len(batch), 0)
for item in batch:
self.assertEqual(item['country']['state']['city'], 'val')
def test_filter_with_in_key_value(self):
keys = [
{'name': 'country_code', 'value': 'es', 'operator': 'contains'}
]
items = [
{'name': 'item1', 'country_code': ['es', 'us']},
{'name': 'item2', 'country_code': ['es', 'us']},
{'name': 'item3', 'country_code': ['uk']}
]
batch = []
for item in items:
record = BaseRecord(item)
batch.append(record)
filter = KeyValueFilter({'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertEqual(2, len(batch))
def setUp(self):
self.options = {
'exporter_options': {
'log_level': 'DEBUG',
'logger_name': 'export-pipeline'
}
}
self.keys = [{'name': 'country_code', 'value': 'es'}]
items = [{
'name': 'item1',
'country_code': 'es'
}, {
'name': 'item2',
'country_code': 'uk'
}]
self.batch = []
for item in items:
record = BaseRecord(item)
self.batch.append(record)
self.filter = KeyValueFilter({'options': {'keys': self.keys}}, meta())
def setUp(self):
self.options = {
'exporter_options': {
'log_level': 'DEBUG',
'logger_name': 'export-pipeline'
}
}
self.keys = [
{'name': 'country_code', 'value': 'es'}
]
items = [{'name': 'item1', 'country_code': 'es'}, {'name': 'item2', 'country_code': 'uk'}]
self.batch = []
for item in items:
record = BaseRecord(item)
self.batch.append(record)
self.filter = KeyValueFilter({'options': {'keys': self.keys}}, meta())
def test_filter_with_non_existing_op(self):
keys = [{
'name': 'country_code',
'value': ['es', 'us'],
'operator': 'not_an_operator'
}]
items = [{
'name': 'item1',
'country_code': 'es'
}, {
'name': 'item2',
'country_code': 'us'
}, {
'name': 'item3',
'country_code': 'uk'
}]
batch = []
for item in items:
record = BaseRecord(item)
batch.append(record)
with self.assertRaisesRegexp(InvalidOperator, 'operator not valid'):
KeyValueFilter({'options': {'keys': keys}}, meta())
class KeyValueFilterTest(unittest.TestCase):
def setUp(self):
self.options = {
'exporter_options': {
'log_level': 'DEBUG',
'logger_name': 'export-pipeline'
}
}
self.keys = [{'name': 'country_code', 'value': 'es'}]
items = [{
'name': 'item1',
'country_code': 'es'
}, {
'name': 'item2',
'country_code': 'uk'
}]
self.batch = []
for item in items:
record = BaseRecord(item)
self.batch.append(record)
self.filter = KeyValueFilter({'options': {'keys': self.keys}}, meta())
def test_filter_with_key_value(self):
batch = self.filter.filter_batch(self.batch)
batch = list(batch)
self.assertEqual(1, len(batch))
self.assertEqual('es', dict(batch[0])['country_code'])
def test_filter_logs(self):
batch = [{
'country': random.choice(['es', 'uk']),
'value': random.randint(0, 1000)
} for i in range(5000)]
# No exception should be raised
self.filter.filter_batch(batch)
def test_filter_with_nested_key_value(self):
keys = [{'name': 'country.state.city', 'value': 'val'}]
batch = [{
'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
},
'value': random.randint(0, 1000)
} for i in range(100)]
filter = KeyValueFilter({'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertGreater(len(batch), 0)
for item in batch:
self.assertEqual(item['country']['state']['city'], 'val')
def test_filter_with_nested_key_value_with_comma(self):
keys = [{'name': 'country,state,city', 'value': 'val'}]
batch = [{
'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
},
'value': random.randint(0, 1000)
} for i in range(100)]
filter = KeyValueFilter(
{'options': {
'keys': keys,
'nested_field_separator': ','
}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertGreater(len(batch), 0)
for item in batch:
self.assertEqual(item['country']['state']['city'], 'val')
def test_filter_with_no_nested_key(self):
keys = [{'name': 'not_a_key', 'value': 'val'}]
batch = [{
'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
},
'value': random.randint(0, 1000)
} for i in range(100)]
filter = KeyValueFilter({'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertEqual(len(batch), 0,
'Resulting filtered batch should be empty')
class KeyValueFilterTest(unittest.TestCase):
def setUp(self):
self.options = {
'exporter_options': {
'log_level': 'DEBUG',
'logger_name': 'export-pipeline'
}
}
self.keys = [
{'name': 'country_code', 'value': 'es'}
]
items = [{'name': 'item1', 'country_code': 'es'}, {'name': 'item2', 'country_code': 'uk'}]
self.batch = []
for item in items:
record = BaseRecord(item)
self.batch.append(record)
self.filter = KeyValueFilter({'options': {'keys': self.keys}}, meta())
def test_filter_with_key_value(self):
batch = self.filter.filter_batch(self.batch)
batch = list(batch)
self.assertEqual(1, len(batch))
self.assertEqual('es', dict(batch[0])['country_code'])
def test_filter_logs(self):
batch = [
{
'country': random.choice(['es', 'uk']),
'value': random.randint(0, 1000)} for i in range(5000)
]
# No exception should be raised
self.filter.filter_batch(batch)
def test_filter_with_nested_key_value(self):
keys = [
{'name': 'country.state.city', 'value': 'val'}
]
batch = [
{'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
}, 'value': random.randint(0, 1000)} for i in range(100)
]
filter = KeyValueFilter({'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertGreater(len(batch), 0)
for item in batch:
self.assertEqual(item['country']['state']['city'], 'val')
def test_filter_with_nested_key_value_with_comma(self):
keys = [
{'name': 'country,state,city', 'value': 'val'}
]
batch = [
{'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
}, 'value': random.randint(0, 1000)} for i in range(100)
]
filter = KeyValueFilter(
{'options': {'keys': keys, 'nested_field_separator': ','}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertGreater(len(batch), 0)
for item in batch:
self.assertEqual(item['country']['state']['city'], 'val')
def test_filter_with_no_nested_key(self):
keys = [
{'name': 'not_a_key', 'value': 'val'}
]
batch = [
{'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
}, 'value': random.randint(0, 1000)} for i in range(100)
]
filter = KeyValueFilter(
{'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertEqual(len(batch), 0, 'Resulting filtered batch should be empty')
