def test_filter_batch_with_key_value_regex(self):
# given:
items = [
{
'name': 'item1',
'country': u'es'
},
{
'name': 'item2',
'country': u'egypt'
},
{
'name': 'item3',
'country': u'uk'
},
{
'name': 'item4',
'country': u'españa'
},
]
batch = [BaseRecord(it) for it in items]
keys = [{'name': 'country', 'value': 'e[sg]'}]
regex_filter = KeyValueRegexFilter({'options': {'keys': keys}}, meta())
# when:
result = list(regex_filter.filter_batch(batch))
# then:
self.assertEqual(['es', 'egypt', u'españa'],
[d['country'] for d in result])
def test_filter_with_nonstring_values(self):
# given:
batch = [
{
'address': {
'country': None
}
},
{
'address': {
'country': 'US'
}
},
{
'address': {
'country': 3
}
},
{
'address': {
'country': 'BR'
}
},
]
options = {
'options': {
'keys': [{
'name': 'address.country',
'value': 'US|3'
}]
}
}
filter = KeyValueRegexFilter(options, meta())
# when:
result = list(filter.filter_batch(batch))
# then:
expected = [
{
'address': {
'country': 'US'
}
},
{
'address': {
'country': 3
}
},
]
self.assertEqual(expected, result)
def test_filter_with_no_nested_key(self):
keys = [{'name': 'not_a_key', 'value': 'val'}]
batch = [{
'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
},
'value': random.randint(0, 1000)
} for i in range(100)]
filter = KeyValueRegexFilter({'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertEqual(len(batch), 0,
'Resulting filtered batch should be empty')
def test_filter_with_nested_key_value(self):
keys = [{'name': 'country.state.city', 'value': 'val'}]
batch = [{
'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
},
'value': random.randint(0, 1000)
} for i in range(100)]
filter = KeyValueRegexFilter({'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertGreater(len(batch), 0)
for item in batch:
self.assertEqual(item['country']['state']['city'], 'val')
def test_filter_with_no_nested_key(self):
keys = [
{'name': 'not_a_key', 'value': 'val'}
]
batch = [
{'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
}, 'value': random.randint(0, 1000)} for i in range(100)
]
filter = KeyValueRegexFilter(
{'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertEqual(len(batch), 0, 'Resulting filtered batch should be empty')
def test_filter_with_nested_key_value_with_comma(self):
keys = [
{'name': 'country,state,city', 'value': 'val'}
]
batch = [
{'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
}, 'value': random.randint(0, 1000)} for i in range(100)
]
filter = KeyValueRegexFilter(
{'options': {'keys': keys, 'nested_field_separator': ','}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertGreater(len(batch), 0)
self.assertEqual(['val'] * len(batch), [e['country']['state']['city'] for e in batch])
def test_filter_with_nested_key_value(self):
keys = [
{'name': 'country.state.city', 'value': 'val'}
]
batch = [
{'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
}, 'value': random.randint(0, 1000)} for i in range(100)
]
filter = KeyValueRegexFilter({'options': {'keys': keys}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertGreater(len(batch), 0)
for item in batch:
self.assertEqual(item['country']['state']['city'], 'val')
def test_filter_batch_with_key_value_regex(self):
# given:
items = [
{'name': 'item1', 'country': u'es'},
{'name': 'item2', 'country': u'egypt'},
{'name': 'item3', 'country': u'uk'},
{'name': 'item4', 'country': u'españa'},
]
batch = [BaseRecord(it) for it in items]
keys = [{'name': 'country', 'value': 'e[sg]'}]
regex_filter = KeyValueRegexFilter({'options': {'keys': keys}}, meta())
# when:
result = list(regex_filter.filter_batch(batch))
# then:
self.assertEqual(['es', 'egypt', u'españa'], [d['country'] for d in result])
def test_filter_with_nested_key_value_with_comma(self):
keys = [{'name': 'country,state,city', 'value': 'val'}]
batch = [{
'country': {
'state': {
'city': random.choice(['val', 'es', 'uk'])
}
},
'value': random.randint(0, 1000)
} for i in range(100)]
filter = KeyValueRegexFilter(
{'options': {
'keys': keys,
'nested_field_separator': ','
}}, meta())
batch = filter.filter_batch(batch)
batch = list(batch)
self.assertGreater(len(batch), 0)
self.assertEqual(['val'] * len(batch),
[e['country']['state']['city'] for e in batch])
def test_filter_with_nonstring_values(self):
# given:
batch = [
{'address': {'country': None}},
{'address': {'country': 'US'}},
{'address': {'country': 3}},
{'address': {'country': 'BR'}},
]
options = {
'options': {
'keys': [{'name': 'address.country', 'value': 'US|3'}]
}
}
filter = KeyValueRegexFilter(options, meta())
# when:
result = list(filter.filter_batch(batch))
# then:
expected = [
{'address': {'country': 'US'}},
{'address': {'country': 3}},
]
self.assertEqual(expected, result)
