def setUp(self):
self._columns = {
'f__o_o': ColumnDefinition('f__o_o', type_name='text'),
'bar': ColumnDefinition('bar', type_name='int'),
'baz': ColumnDefinition('baz', type_name='text[]'),
'quux': ColumnDefinition('quux', type_name='int')
}
self._fdw = ESForeignDataWrapper(
{
'doc_type': 'foo_doc',
'index': 'our_index'
}, self._columns)
self._quals = [Qual('f__o_o', '=', 'value'), Qual('bar', '>', 5)]
def test_execute_column_name_translation(self, scan_mock,
_elasticsearch_mock):
columns = {
'object__nested_field':
ColumnDefinition('object__nested_field', type_name='text')
}
fdw = ESForeignDataWrapper(
{
'doc_type': 'foo_doc',
'index': 'our_index',
'column_name_translation': 'true'
}, columns)
quals = [Qual('object__nested_field', '=', 'value')]
scan_mock.return_value = [{
'fields': {
'object.nested-field': ['value']
}
}]
rows = list(fdw.execute(quals, ['object__nested_field']))
expected_query = {
'fields': ['object.nested-field'],
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'object.nested-field': 'value'
}
}]
}
}
}
}
}
scan_mock.assert_called_once_with(fdw.esclient,
query=expected_query,
index='our_index',
doc_type='foo_doc',
size=fdw._SCROLL_SIZE,
scroll=fdw._SCROLL_LENGTH)
expected_rows = [{'object__nested_field': 'value'}]
self.assertEqual(rows, expected_rows)
def test_id(
self, scan_mock, _elasticsearch_mock):
columns = {
'_id': ColumnDefinition(
'_id',
type_name='text')}
fdw = ESForeignDataWrapper({'doc_type': 'foo_doc',
'index': 'our_index',
'column_name_translation': 'true'},
columns)
quals = [Qual('_id', '=', 'value')]
scan_mock.return_value = [
{'_id': 'value'}]
rows = list(fdw.execute(quals, ['_id']))
expected_query = {
'fields': ['_id'],
'query': {
'filtered': {
'filter': {
'bool': {
'must': [
{
'term': {
'_id': 'value'
}
}
]
}
}
}
}
}
scan_mock.assert_called_once_with(
fdw.esclient,
query=expected_query,
index='our_index',
doc_type='foo_doc',
size=fdw._SCROLL_SIZE,
scroll=fdw._SCROLL_LENGTH)
expected_rows = [{'_id': 'value'}]
self.assertEqual(rows, expected_rows)
def setUp(self):
self._columns = {
'f__o_o': ColumnDefinition('f__o_o', type_name='text', options={'es_property': 'es.foo'}),
'bar': ColumnDefinition('bar', type_name='int'),
'baz': ColumnDefinition('baz', type_name='text[]'),
'quux': ColumnDefinition('quux', type_name='int', options={'es_property': 'qq.quux'})
}
self._fdw = ESForeignDataWrapper({'doc_type': 'foo_doc',
'index': 'our_index'},
self._columns)
self._quals = [
Qual('f__o_o', '=', 'value'),
Qual('bar', '>', 5)
]
def setUp(self):
self._columns = {
'foo': ColumnDefinition('foo', type_name='text', options={'es_property': 'es.foo'}),
'bar': ColumnDefinition('bar', type_name='int', options={'es_property': 'es.bar'}),
'baz': ColumnDefinition('baz', type_name='text[]', options={'es_property': 'es.baz'}),
'num': ColumnDefinition('num', type_name='int', options={'es_property': 'es.num'}),
'a': ColumnDefinition('num', type_name='text'),
'b': ColumnDefinition('num', type_name='text'),
'c': ColumnDefinition('num', type_name='text', options={'es_property': 'es.c'}),
'd': ColumnDefinition('num', type_name='text', options={'es_property': 'es.d'}),
'e': ColumnDefinition('num', type_name='int', options={'es_property': 'es.e'}),
'quux': ColumnDefinition('quux', type_name='int')
}
self._fdw = ESForeignDataWrapper({'doc_type': 'foo_doc'}, self._columns)
def setUp(self):
self._columns = {
'f__o_o': ColumnDefinition('f__o_o', type_name='text'),
'bar': ColumnDefinition('bar', type_name='int', options={'es_property': 'es.bar'}),
'baz': ColumnDefinition('baz', type_name='text[]'),
'quux': ColumnDefinition('quux', type_name='int')
}
self._fdw = ESForeignDataWrapper({'doc_type': 'foo_doc',
'index': 'our_index',
'query': '{ "bool" : { "must": [ { "range": { "es.bar": { "lte": 30 } } }, { "term": { "f__o_o": { "value": "bar" } } } ] } }'},
self._columns)
self._quals = [
Qual('quux', '=', '100'),
Qual('bar', '>', 5)
]
class TestESIntegrationPointsQueryBased(unittest.TestCase):
def setUp(self):
self._columns = {
'f__o_o': ColumnDefinition('f__o_o', type_name='text'),
'bar': ColumnDefinition('bar', type_name='int', options={'es_property': 'es.bar'}),
'baz': ColumnDefinition('baz', type_name='text[]'),
'quux': ColumnDefinition('quux', type_name='int')
}
self._fdw = ESForeignDataWrapper({'doc_type': 'foo_doc',
'index': 'our_index',
'query': '{ "bool" : { "must": [ { "range": { "es.bar": { "lte": 30 } } }, { "term": { "f__o_o": { "value": "bar" } } } ] } }'},
self._columns)
self._quals = [
Qual('quux', '=', '100'),
Qual('bar', '>', 5)
]
@patch('esfdw.esfdw.scan')
def test_execute(self, scan_mock, _elasticsearch_mock):
scan_mock.return_value = [
{'fields': {'f__o_o': ['bar'], 'es.bar': [6], 'baz': ['a', 'b', 'c'], 'quux': [100]}},
{'fields': {'f__o_o': ['bar'], 'es.bar': [7], 'baz': ['d', 'e', 'f'], 'quux': [100]}},
{'fields': {'f__o_o': ['bar'], 'es.bar': [8], 'baz': ['g', 'h'], 'quux': [100]}}
]
rows = list(self._fdw.execute([], ['f__o_o', 'bar', 'baz', 'quux']))
expected_query = {
'query': {
'bool': {
'must': [
{
'range': {
'es.bar': {
'lte': 30
}
}
},
{
'term': {
'f__o_o': {
'value': 'bar'
}
}
}
]
}
},
'fields': ['f__o_o', 'es.bar', 'baz', 'quux']
}
scan_mock.assert_called_once_with(
self._fdw.esclient,
query=expected_query,
index='our_index',
doc_type='foo_doc',
size=self._fdw._SCROLL_SIZE,
scroll=self._fdw._SCROLL_LENGTH)
expected_rows = [
{'f__o_o': 'bar', 'bar': 6, 'baz': ['a', 'b', 'c'], 'quux': 100},
{'f__o_o': 'bar', 'bar': 7, 'baz': ['d', 'e', 'f'], 'quux': 100},
{'f__o_o': 'bar', 'bar': 8, 'baz': ['g', 'h'], 'quux': 100}
]
self.assertEqual(rows, expected_rows)
def test_get_rel_size(self, _elasticsearch_mock):
self._fdw.esclient.search.return_value = {
'hits': {
'total': 100
}
}
rel_size = self._fdw.get_rel_size([], self._columns.keys())
expected_query = {
'size': 0,
'query': {
'bool': {
'must': [
{
'range': {
'es.bar': {
'lte': 30
}
}
},
{
'term': {
'f__o_o': {
'value': 'bar'
}
}
}
]
}
}
}
self._fdw.esclient.search.assert_called_once_with(
index='our_index', body=expected_query, doc_type='foo_doc')
self.assertEqual(rel_size, (100, 400))
@patch('esfdw.esfdw.scan')
def test_execute_quals(self, scan_mock, _elasticsearch_mock):
scan_mock.return_value = [
{'fields': {'f__o_o': ['bar'], 'es.bar': [6], 'baz': ['a', 'b', 'c'], 'quux': [100]}},
{'fields': {'f__o_o': ['bar'], 'es.bar': [7], 'baz': ['d', 'e', 'f'], 'quux': [100]}},
{'fields': {'f__o_o': ['bar'], 'es.bar': [8], 'baz': ['g', 'h'], 'quux': [100]}}
]
rows = list(
self._fdw.execute(
self._quals, [
'f__o_o', 'bar', 'baz', 'quux']))
expected_query = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [
{
'term': {
'quux': '100'
},
},
{
'range': {
'es.bar': {
'gt': 5
}
}
}
]
}
},
'query': {
'bool': {
'must': [
{
'range': {
'es.bar': {
'lte': 30
}
}
},
{
'term': {
'f__o_o': {
'value': 'bar'
}
}
}
]
},
}
}
},
'fields': ['f__o_o', 'es.bar', 'baz', 'quux']
}
scan_mock.assert_called_once_with(
self._fdw.esclient,
query=expected_query,
index='our_index',
doc_type='foo_doc',
size=self._fdw._SCROLL_SIZE,
scroll=self._fdw._SCROLL_LENGTH)
expected_rows = [
{'f__o_o': 'bar', 'bar': 6, 'baz': ['a', 'b', 'c'], 'quux': 100},
{'f__o_o': 'bar', 'bar': 7, 'baz': ['d', 'e', 'f'], 'quux': 100},
{'f__o_o': 'bar', 'bar': 8, 'baz': ['g', 'h'], 'quux': 100}
]
self.assertEqual(rows, expected_rows)
def test_get_rel_size_quals(self, _elasticsearch_mock):
self._fdw.esclient.search.return_value = {
'hits': {
'total': 100
}
}
rel_size = self._fdw.get_rel_size(self._quals, self._columns.keys())
expected_query = {
'size': 0,
'query': {
'filtered': {
'filter': {
'bool': {
'must': [
{
'term': {
'quux': '100'
},
},
{
'range': {
'es.bar': {
'gt': 5
}
}
}
]
}
},
'query': {
'bool': {
'must': [
{
'range': {
'es.bar': {
'lte': 30
}
}
},
{
'term': {
'f__o_o': {
'value': 'bar'
}
}
}
]
},
}
}
}
}
self._fdw.esclient.search.assert_called_once_with(
index='our_index', body=expected_query, doc_type='foo_doc')
self.assertEqual(rel_size, (100, 400))
class TestESIntegrationPoints(unittest.TestCase):
def setUp(self):
self._columns = {
'f__o_o': ColumnDefinition('f__o_o', type_name='text', options={'es_property': 'es.foo'}),
'bar': ColumnDefinition('bar', type_name='int'),
'baz': ColumnDefinition('baz', type_name='text[]'),
'quux': ColumnDefinition('quux', type_name='int', options={'es_property': 'qq.quux'})
}
self._fdw = ESForeignDataWrapper({'doc_type': 'foo_doc',
'index': 'our_index'},
self._columns)
self._quals = [
Qual('f__o_o', '=', 'value'),
Qual('bar', '>', 5)
]
@patch('esfdw.esfdw.scan')
def test_execute(self, scan_mock, _elasticsearch_mock):
scan_mock.return_value = [
{'fields': {'es.foo': ['value'], 'bar': [6], 'baz': ['a', 'b', 'c']}},
{'fields': {'es.foo': ['value'], 'bar': [7], 'baz': ['d', 'e', 'f']}},
{'fields': {'es.foo': ['value'], 'bar': [8], 'baz': ['g', 'h'], 'qq.quux': ['hi']}}
]
rows = list(
self._fdw.execute(
self._quals, [
'f__o_o', 'bar', 'baz', 'quux']))
expected_query = {
'fields': ['es.foo', 'bar', 'baz', 'qq.quux'],
'query': {
'filtered': {
'filter': {
'bool': {
'must': [
{
'term': {
'es.foo': 'value'
},
},
{
'range': {
'bar': {
'gt': 5
}
}
}
]
}
}
}
}
}
scan_mock.assert_called_once_with(
self._fdw.esclient,
query=expected_query,
index='our_index',
doc_type='foo_doc',
size=self._fdw._SCROLL_SIZE,
scroll=self._fdw._SCROLL_LENGTH)
expected_rows = [
{'f__o_o': 'value', 'bar': 6, 'baz': ['a', 'b', 'c'], 'quux': None},
{'f__o_o': 'value', 'bar': 7, 'baz': ['d', 'e', 'f'], 'quux': None},
{'f__o_o': 'value', 'bar': 8, 'baz': ['g', 'h'], 'quux': 'hi'}
]
self.assertEqual(rows, expected_rows)
def test_get_rel_size(self, _elasticsearch_mock):
self._fdw.esclient.search.return_value = {
'hits': {
'total': 200
}
}
rel_size = self._fdw.get_rel_size(self._quals, self._columns.keys())
expected_query = {
'size': 0,
'query': {
'filtered': {
'filter': {
'bool': {
'must': [
{
'term': {
'es.foo': 'value'
},
},
{
'range': {
'bar': {
'gt': 5
}
}
}
]
}
}
}
}
}
self._fdw.esclient.search.assert_called_once_with(
index='our_index', body=expected_query, doc_type='foo_doc')
self.assertEqual(rel_size, (200, 400))
@patch('esfdw.esfdw.scan')
def test_execute_column_name_translation(
self, scan_mock, _elasticsearch_mock):
columns = {
'object__nested_field': ColumnDefinition(
'object__nested_field',
type_name='text')}
fdw = ESForeignDataWrapper({'doc_type': 'foo_doc',
'index': 'our_index',
'column_name_translation': 'true'},
columns)
quals = [Qual('object__nested_field', '=', 'value')]
scan_mock.return_value = [
{'fields': {'object.nested-field': ['value']}}]
rows = list(fdw.execute(quals, ['object__nested_field']))
expected_query = {
'fields': ['object.nested-field'],
'query': {
'filtered': {
'filter': {
'bool': {
'must': [
{
'term': {
'object.nested-field': 'value'
}
}
]
}
}
}
}
}
scan_mock.assert_called_once_with(
fdw.esclient,
query=expected_query,
index='our_index',
doc_type='foo_doc',
size=fdw._SCROLL_SIZE,
scroll=fdw._SCROLL_LENGTH)
expected_rows = [{'object__nested_field': 'value'}]
self.assertEqual(rows, expected_rows)
@patch('esfdw.esfdw.scan')
def test_id(
self, scan_mock, _elasticsearch_mock):
columns = {
'_id': ColumnDefinition(
'_id',
type_name='text')}
fdw = ESForeignDataWrapper({'doc_type': 'foo_doc',
'index': 'our_index',
'column_name_translation': 'true'},
columns)
quals = [Qual('_id', '=', 'value')]
scan_mock.return_value = [
{'_id': 'value'}]
rows = list(fdw.execute(quals, ['_id']))
expected_query = {
'fields': ['_id'],
'query': {
'filtered': {
'filter': {
'bool': {
'must': [
{
'term': {
'_id': 'value'
}
}
]
}
}
}
}
}
scan_mock.assert_called_once_with(
fdw.esclient,
query=expected_query,
index='our_index',
doc_type='foo_doc',
size=fdw._SCROLL_SIZE,
scroll=fdw._SCROLL_LENGTH)
expected_rows = [{'_id': 'value'}]
self.assertEqual(rows, expected_rows)
class TestAppendFilter(unittest.TestCase):
def setUp(self):
self._fdw = ESForeignDataWrapper({'doc_type': 'foo'}, [])
self._ml = MatchList()
def test_equals(self):
self._fdw._append_filter(self._ml, 'foo', '=', 'bar')
self.assertEqual(self._ml, [{'term': {'foo': 'bar'}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '=', 100)
self.assertEqual(self._ml, [{'term': {'foo': 100}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '=',
datetime(year=2015, month=7, day=1))
self.assertEqual(self._ml, [{
'term': {
'foo': datetime(year=2015, month=7, day=1)
}
}])
def test_not_equals(self):
self._fdw._append_filter(self._ml, 'foo', '<>', 'baz')
self.assertEqual(self._ml, [{'not': {'term': {'foo': 'baz'}}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '<>', 4)
self.assertEqual(self._ml, [{'not': {'term': {'foo': 4}}}])
self._ml = MatchList()
self._fdw._append_filter(
self._ml, 'foo', '<>',
datetime(year=2015, month=8, day=31, hour=10, minute=0, second=30))
self.assertEqual(self._ml, [{
'not': {
'term': {
'foo':
datetime(year=2015,
month=8,
day=31,
hour=10,
minute=0,
second=30)
}
}
}])
def test_is_none(self):
self._fdw._append_filter(self._ml, 'foo', '=', None)
self.assertEqual(self._ml, [{'missing': {'field': 'foo'}}])
def test_is_not_none(self):
self._fdw._append_filter(self._ml, 'foo', '<>', None)
self.assertEqual(self._ml, [{'exists': {'field': 'foo'}}])
def test_like(self):
self._fdw._append_filter(self._ml, 'foo', '~~', 'bar%')
self.assertEqual(self._ml, [{'prefix': {'foo': r'bar'}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '~~', '%bar%b_z')
self.assertEqual(self._ml, [{'regexp': {'foo': r'.*bar.*b.z'}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '~~', 'foo%bar%')
self.assertEqual(self._ml, [{'regexp': {'foo': r'foo.*bar.*'}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '~~', '_bar%')
self.assertEqual(self._ml, [{'regexp': {'foo': r'.bar.*'}}])
def test_range(self):
self._fdw._append_filter(
self._ml, 'foo', '<@',
'["2015-12-01 00:00:00","2015-12-02 01:00:00.123456")')
self.assertEqual(self._ml, [{
'range': {
'foo': {
'gte': '2015-12-01T00:00:00.000000',
'lt': '2015-12-02T01:00:00.123456'
}
}
}])
self._ml = MatchList()
self._fdw._append_filter(
self._ml, 'foo', '<@',
'["2015-12-01 00:00:00.000001","2015-12-02 01:00:00"]')
self.assertEqual(self._ml, [{
'range': {
'foo': {
'gte': '2015-12-01T00:00:00.000001',
'lte': '2015-12-02T01:00:00.000000'
}
}
}])
self._ml = MatchList()
self._fdw._append_filter(
self._ml, 'foo', '<@',
'("2015-12-01 00:00:00","2015-12-02 01:00:00"]')
self.assertEqual(self._ml, [{
'range': {
'foo': {
'gt': '2015-12-01T00:00:00.000000',
'lte': '2015-12-02T01:00:00.000000'
}
}
}])
self._ml = MatchList()
self._fdw._append_filter(
self._ml, 'foo', '<@',
'("2015-12-01 00:00:00.123456","2015-12-02 01:00:00.987654")')
self.assertEqual(self._ml, [{
'range': {
'foo': {
'gt': '2015-12-01T00:00:00.123456',
'lt': '2015-12-02T01:00:00.987654'
}
}
}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '<@', '[-1,5]')
self.assertEqual(self._ml, [{
'range': {
'foo': {
'gte': '-1',
'lte': '5'
}
}
}])
def test_lt(self):
self._fdw._append_filter(
self._ml, 'foo', '<',
datetime(year=2015, month=12, day=1, hour=13, minute=24,
second=59))
self.assertEqual(self._ml, [{
'range': {
'foo': {
'lt':
datetime(year=2015,
month=12,
day=1,
hour=13,
minute=24,
second=59)
}
}
}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '<', 4)
self.assertEqual(self._ml, [{'range': {'foo': {'lt': 4}}}])
def test_lte(self):
self._fdw._append_filter(
self._ml, 'foo', '<=',
datetime(year=2015, month=12, day=1, hour=13, minute=24,
second=59))
self.assertEqual(self._ml, [{
'range': {
'foo': {
'lte':
datetime(year=2015,
month=12,
day=1,
hour=13,
minute=24,
second=59)
}
}
}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '<=', 4)
self.assertEqual(self._ml, [{'range': {'foo': {'lte': 4}}}])
def test_gt(self):
self._fdw._append_filter(
self._ml, 'foo', '>',
datetime(year=2015, month=12, day=1, hour=13, minute=24,
second=59))
self.assertEqual(self._ml, [{
'range': {
'foo': {
'gt':
datetime(year=2015,
month=12,
day=1,
hour=13,
minute=24,
second=59)
}
}
}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '>', 4)
self.assertEqual(self._ml, [{'range': {'foo': {'gt': 4}}}])
def test_gte(self):
self._fdw._append_filter(
self._ml, 'foo', '>=',
datetime(year=2015, month=12, day=1, hour=13, minute=24,
second=59))
self.assertEqual(self._ml, [{
'range': {
'foo': {
'gte':
datetime(year=2015,
month=12,
day=1,
hour=13,
minute=24,
second=59)
}
}
}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '>=', 4)
self.assertEqual(self._ml, [{'range': {'foo': {'gte': 4}}}])
def setUp(self):
self._fdw = ESForeignDataWrapper({'doc_type': 'foo_doc'}, [])
class TestESIntegrationPoints(unittest.TestCase):
def setUp(self):
self._columns = {
'f__o_o': ColumnDefinition('f__o_o', type_name='text'),
'bar': ColumnDefinition('bar', type_name='int'),
'baz': ColumnDefinition('baz', type_name='text[]'),
'quux': ColumnDefinition('quux', type_name='int')
}
self._fdw = ESForeignDataWrapper(
{
'doc_type': 'foo_doc',
'index': 'our_index'
}, self._columns)
self._quals = [Qual('f__o_o', '=', 'value'), Qual('bar', '>', 5)]
@patch('esfdw.esfdw.scan')
def test_execute(self, scan_mock, _elasticsearch_mock):
scan_mock.return_value = [{
'fields': {
'f__o_o': ['value'],
'bar': [6],
'baz': ['a', 'b', 'c']
}
}, {
'fields': {
'f__o_o': ['value'],
'bar': [7],
'baz': ['d', 'e', 'f']
}
}, {
'fields': {
'f__o_o': ['value'],
'bar': [8],
'baz': ['g', 'h'],
'quux': ['hi']
}
}]
rows = list(
self._fdw.execute(self._quals, ['f__o_o', 'bar', 'baz', 'quux']))
expected_query = {
'fields': ['f__o_o', 'bar', 'baz', 'quux'],
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'f__o_o': 'value'
},
}, {
'range': {
'bar': {
'gt': 5
}
}
}]
}
}
}
}
}
scan_mock.assert_called_once_with(self._fdw.esclient,
query=expected_query,
index='our_index',
doc_type='foo_doc',
size=self._fdw._SCROLL_SIZE,
scroll=self._fdw._SCROLL_LENGTH)
expected_rows = [{
'f__o_o': 'value',
'bar': 6,
'baz': ['a', 'b', 'c'],
'quux': None
}, {
'f__o_o': 'value',
'bar': 7,
'baz': ['d', 'e', 'f'],
'quux': None
}, {
'f__o_o': 'value',
'bar': 8,
'baz': ['g', 'h'],
'quux': 'hi'
}]
self.assertEqual(rows, expected_rows)
def test_get_rel_size(self, _elasticsearch_mock):
self._fdw.esclient.search.return_value = {'hits': {'total': 200}}
rel_size = self._fdw.get_rel_size(self._quals, self._columns.keys())
expected_query = {
'size': 0,
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'f__o_o': 'value'
},
}, {
'range': {
'bar': {
'gt': 5
}
}
}]
}
}
}
}
}
self._fdw.esclient.search.assert_called_once_with(index='our_index',
body=expected_query,
doc_type='foo_doc')
self.assertEqual(rel_size, (200, 400))
@patch('esfdw.esfdw.scan')
def test_execute_column_name_translation(self, scan_mock,
_elasticsearch_mock):
columns = {
'object__nested_field':
ColumnDefinition('object__nested_field', type_name='text')
}
fdw = ESForeignDataWrapper(
{
'doc_type': 'foo_doc',
'index': 'our_index',
'column_name_translation': 'true'
}, columns)
quals = [Qual('object__nested_field', '=', 'value')]
scan_mock.return_value = [{
'fields': {
'object.nested-field': ['value']
}
}]
rows = list(fdw.execute(quals, ['object__nested_field']))
expected_query = {
'fields': ['object.nested-field'],
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'object.nested-field': 'value'
}
}]
}
}
}
}
}
scan_mock.assert_called_once_with(fdw.esclient,
query=expected_query,
index='our_index',
doc_type='foo_doc',
size=fdw._SCROLL_SIZE,
scroll=fdw._SCROLL_LENGTH)
expected_rows = [{'object__nested_field': 'value'}]
self.assertEqual(rows, expected_rows)
def setUp(self):
self._fdw = ESForeignDataWrapper({'doc_type': 'foo_doc'}, [])
def setUp(self):
self._fdw = ESForeignDataWrapper({'doc_type': 'foo'}, [])
self._ml = MatchList()
class TestAppendFilter(unittest.TestCase):
def setUp(self):
self._fdw = ESForeignDataWrapper({'doc_type': 'foo'}, [])
self._ml = MatchList()
def test_equals(self):
self._fdw._append_filter(self._ml, 'foo', '=', 'bar')
self.assertEqual(self._ml, [{'term': {'foo': 'bar'}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '=', 100)
self.assertEqual(self._ml, [{'term': {'foo': 100}}])
self._ml = MatchList()
self._fdw._append_filter(
self._ml, 'foo', '=', datetime(
year=2015, month=7, day=1))
self.assertEqual(
self._ml, [{'term': {'foo': datetime(year=2015, month=7, day=1)}}])
def test_not_equals(self):
self._fdw._append_filter(self._ml, 'foo', '<>', 'baz')
self.assertEqual(self._ml, [{'not': {'term': {'foo': 'baz'}}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '<>', 4)
self.assertEqual(self._ml, [{'not': {'term': {'foo': 4}}}])
self._ml = MatchList()
self._fdw._append_filter(
self._ml,
'foo',
'<>',
datetime(
year=2015,
month=8,
day=31,
hour=10,
minute=0,
second=30))
self.assertEqual(self._ml, [{'not': {'term': {'foo': datetime(
year=2015, month=8, day=31, hour=10, minute=0, second=30)}}}])
def test_is_none(self):
self._fdw._append_filter(self._ml, 'foo', '=', None)
self.assertEqual(self._ml, [{'missing': {'field': 'foo'}}])
def test_is_not_none(self):
self._fdw._append_filter(self._ml, 'foo', '<>', None)
self.assertEqual(self._ml, [{'exists': {'field': 'foo'}}])
def test_like(self):
self._fdw._append_filter(self._ml, 'foo', '~~', 'bar%')
self.assertEqual(self._ml, [{'prefix': {'foo': r'bar'}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '~~', '%bar%b_z')
self.assertEqual(self._ml, [{'regexp': {'foo': r'.*bar.*b.z'}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '~~', 'foo%bar%')
self.assertEqual(self._ml, [{'regexp': {'foo': r'foo.*bar.*'}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '~~', '_bar%')
self.assertEqual(self._ml, [{'regexp': {'foo': r'.bar.*'}}])
def test_range(self):
self._fdw._append_filter(
self._ml,
'foo',
'<@',
'["2015-12-01 00:00:00","2015-12-02 01:00:00.123456")')
self.assertEqual(self._ml, [{'range': {
'foo': {
'gte': '2015-12-01T00:00:00.000000',
'lt': '2015-12-02T01:00:00.123456'
}}}])
self._ml = MatchList()
self._fdw._append_filter(
self._ml,
'foo',
'<@',
'["2015-12-01 00:00:00.000001","2015-12-02 01:00:00"]')
self.assertEqual(self._ml, [{'range': {
'foo': {
'gte': '2015-12-01T00:00:00.000001',
'lte': '2015-12-02T01:00:00.000000'
}}}])
self._ml = MatchList()
self._fdw._append_filter(
self._ml,
'foo',
'<@',
'("2015-12-01 00:00:00","2015-12-02 01:00:00"]')
self.assertEqual(self._ml, [{'range': {
'foo': {
'gt': '2015-12-01T00:00:00.000000',
'lte': '2015-12-02T01:00:00.000000'
}}}])
self._ml = MatchList()
self._fdw._append_filter(
self._ml,
'foo',
'<@',
'("2015-12-01 00:00:00.123456","2015-12-02 01:00:00.987654")')
self.assertEqual(self._ml, [{'range': {
'foo': {
'gt': '2015-12-01T00:00:00.123456',
'lt': '2015-12-02T01:00:00.987654'
}}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '<@', '[-1,5]')
self.assertEqual(self._ml, [{'range': {
'foo': {
'gte': '-1',
'lte': '5'
}}}])
def test_lt(self):
self._fdw._append_filter(
self._ml,
'foo',
'<',
datetime(
year=2015,
month=12,
day=1,
hour=13,
minute=24,
second=59))
self.assertEqual(self._ml, [{'range': {'foo': {'lt': datetime(
year=2015, month=12, day=1, hour=13, minute=24, second=59)}}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '<', 4)
self.assertEqual(self._ml, [{'range': {
'foo': {
'lt': 4
}}}])
def test_lte(self):
self._fdw._append_filter(
self._ml,
'foo',
'<=',
datetime(
year=2015,
month=12,
day=1,
hour=13,
minute=24,
second=59))
self.assertEqual(self._ml, [{'range': {'foo': {'lte': datetime(
year=2015, month=12, day=1, hour=13, minute=24, second=59)}}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '<=', 4)
self.assertEqual(self._ml, [{'range': {
'foo': {
'lte': 4
}}}])
def test_gt(self):
self._fdw._append_filter(
self._ml,
'foo',
'>',
datetime(
year=2015,
month=12,
day=1,
hour=13,
minute=24,
second=59))
self.assertEqual(self._ml, [{'range': {'foo': {'gt': datetime(
year=2015, month=12, day=1, hour=13, minute=24, second=59)}}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '>', 4)
self.assertEqual(self._ml, [{'range': {
'foo': {
'gt': 4
}}}])
def test_gte(self):
self._fdw._append_filter(
self._ml,
'foo',
'>=',
datetime(
year=2015,
month=12,
day=1,
hour=13,
minute=24,
second=59))
self.assertEqual(self._ml, [{'range': {'foo': {'gte': datetime(
year=2015, month=12, day=1, hour=13, minute=24, second=59)}}}])
self._ml = MatchList()
self._fdw._append_filter(self._ml, 'foo', '>=', 4)
self.assertEqual(self._ml, [{'range': {
'foo': {
'gte': 4
}}}])
class TestQualProcessing(unittest.TestCase):
def setUp(self):
self._columns = {
'foo': ColumnDefinition('foo', type_name='text', options={'es_property': 'es.foo'}),
'bar': ColumnDefinition('bar', type_name='int', options={'es_property': 'es.bar'}),
'baz': ColumnDefinition('baz', type_name='text[]', options={'es_property': 'es.baz'}),
'num': ColumnDefinition('num', type_name='int', options={'es_property': 'es.num'}),
'a': ColumnDefinition('num', type_name='text'),
'b': ColumnDefinition('num', type_name='text'),
'c': ColumnDefinition('num', type_name='text', options={'es_property': 'es.c'}),
'd': ColumnDefinition('num', type_name='text', options={'es_property': 'es.d'}),
'e': ColumnDefinition('num', type_name='int', options={'es_property': 'es.e'}),
'quux': ColumnDefinition('quux', type_name='int')
}
self._fdw = ESForeignDataWrapper({'doc_type': 'foo_doc'}, self._columns)
def test_normalize_operator(self):
for op in ('=', '~~', '<@', '<', '>', '<=', '>='):
self.assertEqual(self._fdw._normalize_operator(op, 'foo'),
(op, False))
self.assertEqual(self._fdw._normalize_operator(op, None),
(op, False))
self.assertEqual(self._fdw._normalize_operator('!' + op, 'foo'),
(op, True))
self.assertEqual(self._fdw._normalize_operator('!' + op, None),
(op, True))
# <> is handled specially
self.assertEqual(self._fdw._normalize_operator('<>', 'foo'),
('=', True))
self.assertEqual(self._fdw._normalize_operator('<>', None),
('<>', False))
def test_process_qual(self):
must_list = MatchList()
must_not_list = MatchList()
self._fdw._process_qual(must_list, must_not_list, 'foo', '=', 'bar')
self.assertEqual(len(must_list), 1)
self.assertEqual(len(must_not_list), 0)
self._fdw._process_qual(must_list, must_not_list, 'foo', '<>', 'bar')
self.assertEqual(len(must_list), 1)
self.assertEqual(len(must_not_list), 1)
self._fdw._process_qual(must_list, must_not_list, 'foo', '~~', 'bar%')
self.assertEqual(len(must_list), 2)
self.assertEqual(len(must_not_list), 1)
# Not pushing down ILIKE since we can't write a case-insensitive regexp
# filter
self._fdw._process_qual(must_list, must_not_list, 'foo', '~~*', 'bar%')
self.assertEqual(len(must_list), 2)
self.assertEqual(len(must_not_list), 1)
def test_make_match_lists(self):
quals = [
Qual('foo', '=', 'bar'),
Qual('quux', '<>', 'baz'),
Qual('num', '<@', '[1, 10]'),
Qual('a', ('=', True), ['x', 'y', 'z']),
Qual('b', ('~~', True), ['d', 'e', 'f']),
Qual('c', ('!~~', True), ['a%b', 'c_d', '_e%f%']),
Qual('d', ('~~', False), ['g%', 'h_%', 'i%']),
Qual('e', ('<>', False), [1, 2])
]
ml, mnl = self._fdw._make_match_lists(quals)
self.assertEqual(len(ml), 7)
self.assertIn({'terms': {'a': ['x', 'y', 'z']}}, ml)
self.assertIn({'or': [
{'regexp': {'b': 'd'}},
{'regexp': {'b': 'e'}},
{'regexp': {'b': 'f'}}
]
}, ml)
self.assertIn({'prefix': {'es.d': 'g'}}, ml)
self.assertIn({'regexp': {'es.d': 'h..*'}}, ml)
self.assertIn({'prefix': {'es.d': 'i'}}, ml)
self.assertEqual(len(mnl), 3)
self.assertIn({'and': [
{'regexp': {'es.c': 'a.*b'}},
{'regexp': {'es.c': 'c.d'}},
{'regexp': {'es.c': '.e.*f.*'}}
]
}, mnl)
self.assertIn({'terms': {'es.e': [1, 2]}}, mnl)
def setUp(self):
self._fdw = ESForeignDataWrapper({'doc_type': 'foo'}, [])
self._ml = MatchList()
class TestQualProcessing(unittest.TestCase):
def setUp(self):
self._fdw = ESForeignDataWrapper({'doc_type': 'foo_doc'}, [])
def test_normalize_operator(self):
for op in ('=', '~~', '<@', '<', '>', '<=', '>='):
self.assertEqual(self._fdw._normalize_operator(op, 'foo'),
(op, False))
self.assertEqual(self._fdw._normalize_operator(op, None),
(op, False))
self.assertEqual(self._fdw._normalize_operator('!' + op, 'foo'),
(op, True))
self.assertEqual(self._fdw._normalize_operator('!' + op, None),
(op, True))
# <> is handled specially
self.assertEqual(self._fdw._normalize_operator('<>', 'foo'),
('=', True))
self.assertEqual(self._fdw._normalize_operator('<>', None),
('<>', False))
def test_process_qual(self):
must_list = MatchList()
must_not_list = MatchList()
self._fdw._process_qual(must_list, must_not_list, 'foo', '=', 'bar')
self.assertEqual(len(must_list), 1)
self.assertEqual(len(must_not_list), 0)
self._fdw._process_qual(must_list, must_not_list, 'foo', '<>', 'bar')
self.assertEqual(len(must_list), 1)
self.assertEqual(len(must_not_list), 1)
self._fdw._process_qual(must_list, must_not_list, 'foo', '~~', 'bar%')
self.assertEqual(len(must_list), 2)
self.assertEqual(len(must_not_list), 1)
# Not pushing down ILIKE since we can't write a case-insensitive regexp
# filter
self._fdw._process_qual(must_list, must_not_list, 'foo', '~~*', 'bar%')
self.assertEqual(len(must_list), 2)
self.assertEqual(len(must_not_list), 1)
def test_make_match_lists(self):
quals = [
Qual('foo', '=', 'bar'),
Qual('quux', '<>', 'baz'),
Qual('num', '<@', '[1, 10]'),
Qual('a', ('=', True), ['x', 'y', 'z']),
Qual('b', ('~~', True), ['d', 'e', 'f']),
Qual('c', ('!~~', True), ['a%b', 'c_d', '_e%f%']),
Qual('d', ('~~', False), ['g%', 'h_%', 'i%']),
Qual('e', ('<>', False), [1, 2])
]
ml, mnl = self._fdw._make_match_lists(quals)
self.assertEqual(len(ml), 7)
self.assertIn({'terms': {'a': ['x', 'y', 'z']}}, ml)
self.assertIn(
{
'or': [{
'regexp': {
'b': 'd'
}
}, {
'regexp': {
'b': 'e'
}
}, {
'regexp': {
'b': 'f'
}
}]
}, ml)
self.assertIn({'prefix': {'d': 'g'}}, ml)
self.assertIn({'regexp': {'d': 'h..*'}}, ml)
self.assertIn({'prefix': {'d': 'i'}}, ml)
self.assertEqual(len(mnl), 3)
self.assertIn(
{
'and': [{
'regexp': {
'c': 'a.*b'
}
}, {
'regexp': {
'c': 'c.d'
}
}, {
'regexp': {
'c': '.e.*f.*'
}
}]
}, mnl)
self.assertIn({'terms': {'e': [1, 2]}}, mnl)
