def test_execute_column_name_translation(self, scan_mock, _elasticsearch_mock): columns = { 'object__nested_field': ColumnDefinition('object__nested_field', type_name='text') } fdw = ESForeignDataWrapper( { 'doc_type': 'foo_doc', 'index': 'our_index', 'column_name_translation': 'true' }, columns) quals = [Qual('object__nested_field', '=', 'value')] scan_mock.return_value = [{ 'fields': { 'object.nested-field': ['value'] } }] rows = list(fdw.execute(quals, ['object__nested_field'])) expected_query = { 'fields': ['object.nested-field'], 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'object.nested-field': 'value' } }] } } } } } scan_mock.assert_called_once_with(fdw.esclient, query=expected_query, index='our_index', doc_type='foo_doc', size=fdw._SCROLL_SIZE, scroll=fdw._SCROLL_LENGTH) expected_rows = [{'object__nested_field': 'value'}] self.assertEqual(rows, expected_rows)
def test_id( self, scan_mock, _elasticsearch_mock): columns = { '_id': ColumnDefinition( '_id', type_name='text')} fdw = ESForeignDataWrapper({'doc_type': 'foo_doc', 'index': 'our_index', 'column_name_translation': 'true'}, columns) quals = [Qual('_id', '=', 'value')] scan_mock.return_value = [ {'_id': 'value'}] rows = list(fdw.execute(quals, ['_id'])) expected_query = { 'fields': ['_id'], 'query': { 'filtered': { 'filter': { 'bool': { 'must': [ { 'term': { '_id': 'value' } } ] } } } } } scan_mock.assert_called_once_with( fdw.esclient, query=expected_query, index='our_index', doc_type='foo_doc', size=fdw._SCROLL_SIZE, scroll=fdw._SCROLL_LENGTH) expected_rows = [{'_id': 'value'}] self.assertEqual(rows, expected_rows)
class TestESIntegrationPointsQueryBased(unittest.TestCase): def setUp(self): self._columns = { 'f__o_o': ColumnDefinition('f__o_o', type_name='text'), 'bar': ColumnDefinition('bar', type_name='int', options={'es_property': 'es.bar'}), 'baz': ColumnDefinition('baz', type_name='text[]'), 'quux': ColumnDefinition('quux', type_name='int') } self._fdw = ESForeignDataWrapper({'doc_type': 'foo_doc', 'index': 'our_index', 'query': '{ "bool" : { "must": [ { "range": { "es.bar": { "lte": 30 } } }, { "term": { "f__o_o": { "value": "bar" } } } ] } }'}, self._columns) self._quals = [ Qual('quux', '=', '100'), Qual('bar', '>', 5) ] @patch('esfdw.esfdw.scan') def test_execute(self, scan_mock, _elasticsearch_mock): scan_mock.return_value = [ {'fields': {'f__o_o': ['bar'], 'es.bar': [6], 'baz': ['a', 'b', 'c'], 'quux': [100]}}, {'fields': {'f__o_o': ['bar'], 'es.bar': [7], 'baz': ['d', 'e', 'f'], 'quux': [100]}}, {'fields': {'f__o_o': ['bar'], 'es.bar': [8], 'baz': ['g', 'h'], 'quux': [100]}} ] rows = list(self._fdw.execute([], ['f__o_o', 'bar', 'baz', 'quux'])) expected_query = { 'query': { 'bool': { 'must': [ { 'range': { 'es.bar': { 'lte': 30 } } }, { 'term': { 'f__o_o': { 'value': 'bar' } } } ] } }, 'fields': ['f__o_o', 'es.bar', 'baz', 'quux'] } scan_mock.assert_called_once_with( self._fdw.esclient, query=expected_query, index='our_index', doc_type='foo_doc', size=self._fdw._SCROLL_SIZE, scroll=self._fdw._SCROLL_LENGTH) expected_rows = [ {'f__o_o': 'bar', 'bar': 6, 'baz': ['a', 'b', 'c'], 'quux': 100}, {'f__o_o': 'bar', 'bar': 7, 'baz': ['d', 'e', 'f'], 'quux': 100}, {'f__o_o': 'bar', 'bar': 8, 'baz': ['g', 'h'], 'quux': 100} ] self.assertEqual(rows, expected_rows) def test_get_rel_size(self, _elasticsearch_mock): self._fdw.esclient.search.return_value = { 'hits': { 'total': 100 } } rel_size = self._fdw.get_rel_size([], self._columns.keys()) expected_query = { 'size': 0, 'query': { 'bool': { 'must': [ { 'range': { 'es.bar': { 'lte': 30 } } }, { 'term': { 'f__o_o': { 'value': 'bar' } } } ] } } } self._fdw.esclient.search.assert_called_once_with( index='our_index', body=expected_query, doc_type='foo_doc') self.assertEqual(rel_size, (100, 400)) @patch('esfdw.esfdw.scan') def test_execute_quals(self, scan_mock, _elasticsearch_mock): scan_mock.return_value = [ {'fields': {'f__o_o': ['bar'], 'es.bar': [6], 'baz': ['a', 'b', 'c'], 'quux': [100]}}, {'fields': {'f__o_o': ['bar'], 'es.bar': [7], 'baz': ['d', 'e', 'f'], 'quux': [100]}}, {'fields': {'f__o_o': ['bar'], 'es.bar': [8], 'baz': ['g', 'h'], 'quux': [100]}} ] rows = list( self._fdw.execute( self._quals, [ 'f__o_o', 'bar', 'baz', 'quux'])) expected_query = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [ { 'term': { 'quux': '100' }, }, { 'range': { 'es.bar': { 'gt': 5 } } } ] } }, 'query': { 'bool': { 'must': [ { 'range': { 'es.bar': { 'lte': 30 } } }, { 'term': { 'f__o_o': { 'value': 'bar' } } } ] }, } } }, 'fields': ['f__o_o', 'es.bar', 'baz', 'quux'] } scan_mock.assert_called_once_with( self._fdw.esclient, query=expected_query, index='our_index', doc_type='foo_doc', size=self._fdw._SCROLL_SIZE, scroll=self._fdw._SCROLL_LENGTH) expected_rows = [ {'f__o_o': 'bar', 'bar': 6, 'baz': ['a', 'b', 'c'], 'quux': 100}, {'f__o_o': 'bar', 'bar': 7, 'baz': ['d', 'e', 'f'], 'quux': 100}, {'f__o_o': 'bar', 'bar': 8, 'baz': ['g', 'h'], 'quux': 100} ] self.assertEqual(rows, expected_rows) def test_get_rel_size_quals(self, _elasticsearch_mock): self._fdw.esclient.search.return_value = { 'hits': { 'total': 100 } } rel_size = self._fdw.get_rel_size(self._quals, self._columns.keys()) expected_query = { 'size': 0, 'query': { 'filtered': { 'filter': { 'bool': { 'must': [ { 'term': { 'quux': '100' }, }, { 'range': { 'es.bar': { 'gt': 5 } } } ] } }, 'query': { 'bool': { 'must': [ { 'range': { 'es.bar': { 'lte': 30 } } }, { 'term': { 'f__o_o': { 'value': 'bar' } } } ] }, } } } } self._fdw.esclient.search.assert_called_once_with( index='our_index', body=expected_query, doc_type='foo_doc') self.assertEqual(rel_size, (100, 400))
class TestESIntegrationPoints(unittest.TestCase): def setUp(self): self._columns = { 'f__o_o': ColumnDefinition('f__o_o', type_name='text', options={'es_property': 'es.foo'}), 'bar': ColumnDefinition('bar', type_name='int'), 'baz': ColumnDefinition('baz', type_name='text[]'), 'quux': ColumnDefinition('quux', type_name='int', options={'es_property': 'qq.quux'}) } self._fdw = ESForeignDataWrapper({'doc_type': 'foo_doc', 'index': 'our_index'}, self._columns) self._quals = [ Qual('f__o_o', '=', 'value'), Qual('bar', '>', 5) ] @patch('esfdw.esfdw.scan') def test_execute(self, scan_mock, _elasticsearch_mock): scan_mock.return_value = [ {'fields': {'es.foo': ['value'], 'bar': [6], 'baz': ['a', 'b', 'c']}}, {'fields': {'es.foo': ['value'], 'bar': [7], 'baz': ['d', 'e', 'f']}}, {'fields': {'es.foo': ['value'], 'bar': [8], 'baz': ['g', 'h'], 'qq.quux': ['hi']}} ] rows = list( self._fdw.execute( self._quals, [ 'f__o_o', 'bar', 'baz', 'quux'])) expected_query = { 'fields': ['es.foo', 'bar', 'baz', 'qq.quux'], 'query': { 'filtered': { 'filter': { 'bool': { 'must': [ { 'term': { 'es.foo': 'value' }, }, { 'range': { 'bar': { 'gt': 5 } } } ] } } } } } scan_mock.assert_called_once_with( self._fdw.esclient, query=expected_query, index='our_index', doc_type='foo_doc', size=self._fdw._SCROLL_SIZE, scroll=self._fdw._SCROLL_LENGTH) expected_rows = [ {'f__o_o': 'value', 'bar': 6, 'baz': ['a', 'b', 'c'], 'quux': None}, {'f__o_o': 'value', 'bar': 7, 'baz': ['d', 'e', 'f'], 'quux': None}, {'f__o_o': 'value', 'bar': 8, 'baz': ['g', 'h'], 'quux': 'hi'} ] self.assertEqual(rows, expected_rows) def test_get_rel_size(self, _elasticsearch_mock): self._fdw.esclient.search.return_value = { 'hits': { 'total': 200 } } rel_size = self._fdw.get_rel_size(self._quals, self._columns.keys()) expected_query = { 'size': 0, 'query': { 'filtered': { 'filter': { 'bool': { 'must': [ { 'term': { 'es.foo': 'value' }, }, { 'range': { 'bar': { 'gt': 5 } } } ] } } } } } self._fdw.esclient.search.assert_called_once_with( index='our_index', body=expected_query, doc_type='foo_doc') self.assertEqual(rel_size, (200, 400)) @patch('esfdw.esfdw.scan') def test_execute_column_name_translation( self, scan_mock, _elasticsearch_mock): columns = { 'object__nested_field': ColumnDefinition( 'object__nested_field', type_name='text')} fdw = ESForeignDataWrapper({'doc_type': 'foo_doc', 'index': 'our_index', 'column_name_translation': 'true'}, columns) quals = [Qual('object__nested_field', '=', 'value')] scan_mock.return_value = [ {'fields': {'object.nested-field': ['value']}}] rows = list(fdw.execute(quals, ['object__nested_field'])) expected_query = { 'fields': ['object.nested-field'], 'query': { 'filtered': { 'filter': { 'bool': { 'must': [ { 'term': { 'object.nested-field': 'value' } } ] } } } } } scan_mock.assert_called_once_with( fdw.esclient, query=expected_query, index='our_index', doc_type='foo_doc', size=fdw._SCROLL_SIZE, scroll=fdw._SCROLL_LENGTH) expected_rows = [{'object__nested_field': 'value'}] self.assertEqual(rows, expected_rows) @patch('esfdw.esfdw.scan') def test_id( self, scan_mock, _elasticsearch_mock): columns = { '_id': ColumnDefinition( '_id', type_name='text')} fdw = ESForeignDataWrapper({'doc_type': 'foo_doc', 'index': 'our_index', 'column_name_translation': 'true'}, columns) quals = [Qual('_id', '=', 'value')] scan_mock.return_value = [ {'_id': 'value'}] rows = list(fdw.execute(quals, ['_id'])) expected_query = { 'fields': ['_id'], 'query': { 'filtered': { 'filter': { 'bool': { 'must': [ { 'term': { '_id': 'value' } } ] } } } } } scan_mock.assert_called_once_with( fdw.esclient, query=expected_query, index='our_index', doc_type='foo_doc', size=fdw._SCROLL_SIZE, scroll=fdw._SCROLL_LENGTH) expected_rows = [{'_id': 'value'}] self.assertEqual(rows, expected_rows)
class TestESIntegrationPoints(unittest.TestCase): def setUp(self): self._columns = { 'f__o_o': ColumnDefinition('f__o_o', type_name='text'), 'bar': ColumnDefinition('bar', type_name='int'), 'baz': ColumnDefinition('baz', type_name='text[]'), 'quux': ColumnDefinition('quux', type_name='int') } self._fdw = ESForeignDataWrapper( { 'doc_type': 'foo_doc', 'index': 'our_index' }, self._columns) self._quals = [Qual('f__o_o', '=', 'value'), Qual('bar', '>', 5)] @patch('esfdw.esfdw.scan') def test_execute(self, scan_mock, _elasticsearch_mock): scan_mock.return_value = [{ 'fields': { 'f__o_o': ['value'], 'bar': [6], 'baz': ['a', 'b', 'c'] } }, { 'fields': { 'f__o_o': ['value'], 'bar': [7], 'baz': ['d', 'e', 'f'] } }, { 'fields': { 'f__o_o': ['value'], 'bar': [8], 'baz': ['g', 'h'], 'quux': ['hi'] } }] rows = list( self._fdw.execute(self._quals, ['f__o_o', 'bar', 'baz', 'quux'])) expected_query = { 'fields': ['f__o_o', 'bar', 'baz', 'quux'], 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'f__o_o': 'value' }, }, { 'range': { 'bar': { 'gt': 5 } } }] } } } } } scan_mock.assert_called_once_with(self._fdw.esclient, query=expected_query, index='our_index', doc_type='foo_doc', size=self._fdw._SCROLL_SIZE, scroll=self._fdw._SCROLL_LENGTH) expected_rows = [{ 'f__o_o': 'value', 'bar': 6, 'baz': ['a', 'b', 'c'], 'quux': None }, { 'f__o_o': 'value', 'bar': 7, 'baz': ['d', 'e', 'f'], 'quux': None }, { 'f__o_o': 'value', 'bar': 8, 'baz': ['g', 'h'], 'quux': 'hi' }] self.assertEqual(rows, expected_rows) def test_get_rel_size(self, _elasticsearch_mock): self._fdw.esclient.search.return_value = {'hits': {'total': 200}} rel_size = self._fdw.get_rel_size(self._quals, self._columns.keys()) expected_query = { 'size': 0, 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'f__o_o': 'value' }, }, { 'range': { 'bar': { 'gt': 5 } } }] } } } } } self._fdw.esclient.search.assert_called_once_with(index='our_index', body=expected_query, doc_type='foo_doc') self.assertEqual(rel_size, (200, 400)) @patch('esfdw.esfdw.scan') def test_execute_column_name_translation(self, scan_mock, _elasticsearch_mock): columns = { 'object__nested_field': ColumnDefinition('object__nested_field', type_name='text') } fdw = ESForeignDataWrapper( { 'doc_type': 'foo_doc', 'index': 'our_index', 'column_name_translation': 'true' }, columns) quals = [Qual('object__nested_field', '=', 'value')] scan_mock.return_value = [{ 'fields': { 'object.nested-field': ['value'] } }] rows = list(fdw.execute(quals, ['object__nested_field'])) expected_query = { 'fields': ['object.nested-field'], 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'object.nested-field': 'value' } }] } } } } } scan_mock.assert_called_once_with(fdw.esclient, query=expected_query, index='our_index', doc_type='foo_doc', size=fdw._SCROLL_SIZE, scroll=fdw._SCROLL_LENGTH) expected_rows = [{'object__nested_field': 'value'}] self.assertEqual(rows, expected_rows)