def test_parse_json(self, mock_es, mock_helpers, index_mock): test_cases = [ ('{"bool": true, "number": 1234, "string": "foo"}', True), ('{regular meta}', False) ] obj = 'object' obj_meta = { 'x-timestamp': 0, 'last-modified': 'Wed, 06 Jun 2018 22:24:19 GMT'} doc_id = self.compute_id(self.test_account, self.test_container, obj) es_mock = mock_es.return_value es_mock.info.return_value = {'version': {'number': '5.4.0'}} es_mock.mget.return_value = {'docs': [{'_id': doc_id, 'found': False}]} index_mock.return_value.get_mapping.return_value = { self.test_index: { 'mappings': { 'properties': metadata_sync.MetadataSync.DOC_MAPPING}}} sync_conf = dict(self.sync_conf) sync_conf['parse_json'] = True sync = metadata_sync.MetadataSync(self.status_dir, sync_conf) for meta, is_json in test_cases: mock_helpers.reset_mock() mock_helpers.bulk.return_value = (None, []) test_meta = dict(obj_meta) test_meta['x-object-meta-test'] = meta internal_client = mock.Mock() internal_client.get_object_metadata.return_value = test_meta sync.handle([{'name': obj, 'deleted': False, 'created_at': 0}], internal_client) if is_json: expected = json.loads(meta) else: expected = meta mock_helpers.bulk.assert_called_once_with( es_mock, [ {'_op_type': 'index', '_id': doc_id, '_index': self.test_index, '_type': metadata_sync.MetadataSync.DEFAULT_DOC_TYPE, '_source': { 'test': expected, 'x-timestamp': 0, 'last-modified': 1528323859000, 'x-swift-account': self.test_account, 'x-swift-container': self.test_container, 'x-swift-object': obj}}], raise_on_error=False, raise_on_exception=False)
def test_verify_mapping(self, es_mock, index_mock): full_mapping = { key: metadata_sync.MetadataSync._update_string_mapping(val) for key, val in metadata_sync.MetadataSync.DOC_MAPPING.items()} # List of tuples of mappings to test: the mapping returned by ES and # the mapping we expect to submit to the put_mapping call. test_mappings = [ ({self.test_index: {"mappings": {}}}, full_mapping), ({self.test_index: {"mappings": { "properties": { "content-length": {"type": "string"}, "x-timestamp": {"type": "string"}, "x-trans-id": {"type": "string"} } }}}, { "content-type": {"type": "string"}, "last-modified": {"type": "date"}, "etag": {"type": "string", "index": "not_analyzed"}, "x-object-manifest": {"type": "string"}, "x-static-large-object": {"type": "boolean"}, "x-swift-container": {"type": "string"}, "x-swift-account": {"type": "string"}, "x-swift-object": {"type": "string"}, }), ({self.test_index: {"mappings": { "properties": full_mapping}}}, {}), ({self.test_index: {'mappings': { metadata_sync.MetadataSync.OLD_DOC_TYPE: {}}}}, full_mapping), ] for return_mapping, expected_put_mapping in test_mappings: es_conn = mock.Mock() es_conn.info.return_value = {'version': {'number': '7.4.0'}} index_conn = mock.Mock() index_conn.get_mapping.return_value = return_mapping es_mock.return_value = es_conn index_mock.return_value = index_conn metadata_sync.MetadataSync(self.status_dir, self.sync_conf) expected_put_mapping = { key: metadata_sync.MetadataSync._update_string_mapping(val) for key, val in expected_put_mapping.items()} return_mappings = return_mapping[self.test_index]['mappings'] if expected_put_mapping: if not return_mappings or 'properties' in return_mappings: index_conn.put_mapping.assert_called_once_with( index=self.test_index, body={"properties": expected_put_mapping}, include_type_name=False) else: index_conn.put_mapping.assert_called_once_with( index=self.test_index, body={"properties": expected_put_mapping}, doc_type=return_mappings.keys()[0]) else: index_conn.put_mapping.assert_not_called()
def test_verify_mapping_5x(self, es_mock, index_mock): full_mapping = metadata_sync.MetadataSync.DOC_MAPPING swift_type = metadata_sync.MetadataSync.DOC_TYPE # Test that "string" mappings are converted to keyword or text or both. current_mapping = dict([(k, v) for k, v in full_mapping.items() if v['type'] != 'string']) text_and_keyword = { 'type': 'text', 'fields': { 'keyword': { 'type': 'keyword' } } } expected_mapping = { 'content-type': text_and_keyword, 'etag': { 'type': 'keyword' }, 'x-object-manifest': text_and_keyword, 'x-swift-container': text_and_keyword, 'x-swift-account': text_and_keyword, 'x-swift-object': text_and_keyword, 'x-trans-id': { 'type': 'keyword' } } es_conn = mock.Mock() es_conn.info.return_value = {'version': {'number': '5.0'}} index_conn = mock.Mock() index_conn.get_mapping.return_value = { self.test_index: { 'mappings': { swift_type: { 'properties': current_mapping } } } } es_mock.return_value = es_conn index_mock.return_value = index_conn metadata_sync.MetadataSync(self.status_dir, self.sync_conf) index_conn.put_mapping.assert_called_once_with( index=self.test_index, doc_type=swift_type, body={'properties': expected_mapping})
def test_tls_parameters(self, mock_verify_mapping, mock_es): sync_conf = {'es_hosts': self.es_hosts, 'index': self.test_index, 'account': self.test_account, 'container': self.test_container, 'verify_certs': False, 'ca_certs': "/no/such/pemfile"} es_mock = mock_es.return_value es_mock.info.return_value = {'version': {'number': '5.4.0'}} metadata_sync.MetadataSync(self.status_dir, sync_conf) mock_es.assert_called_once_with( self.es_hosts, ca_certs='/no/such/pemfile', verify_certs=False)
def test_set_pipeline(self, es_mock, helpers_mock): pipeline = 'test-pipeline' obj = 'object' doc_id = self.compute_id(self.test_account, self.test_container, obj) config = dict(self.sync_conf) config['pipeline'] = pipeline es_mock.return_value.info.return_value = { 'version': { 'number': '5.4.0' } } es_mock.return_value.mget.return_value = { 'docs': [{ '_id': doc_id, 'found': False }] } internal_client = mock.Mock() internal_client.get_object_metadata.return_value = { 'x-timestamp': 0, 'last-modified': 'Wed, 06 Jun 2018 22:24:19 GMT' } helpers_mock.bulk.return_value = (None, []) sync = metadata_sync.MetadataSync(self.status_dir, config) sync.handle([{ 'name': obj, 'deleted': False, 'created_at': 0 }], internal_client) helpers_mock.bulk.assert_called_once_with( es_mock.return_value, [{ '_op_type': 'index', '_id': doc_id, '_index': self.test_index, '_type': metadata_sync.MetadataSync.DOC_TYPE, '_source': { 'x-timestamp': 0, 'last-modified': 1528323859000, 'x-swift-account': self.test_account, 'x-swift-container': self.test_container, 'x-swift-object': obj, }, 'pipeline': 'test-pipeline' }], raise_on_error=False, raise_on_exception=False)
def test_verify_mapping_multiple_mappings(self, es_mock, index_mock): mapping_resp = {self.test_index: { "mappings": { "bogus_type": metadata_sync.MetadataSync.DOC_MAPPING}}} es_conn = mock.Mock() es_conn.info.return_value = {'version': {'number': '6.8.0'}} index_conn = mock.Mock() index_conn.get_mapping.return_value = mapping_resp es_mock.return_value = es_conn index_mock.return_value = index_conn with self.assertRaises(RuntimeError) as err_ctx: metadata_sync.MetadataSync(self.status_dir, self.sync_conf) self.assertEqual('Cannot set more than one mapping type for index {}. ' "Known types: ['bogus_type']".format(self.test_index), err_ctx.exception.message)
def setUp(self, mock_verify_mapping, mock_es): self.status_dir = '/status/dir' self.es_hosts = 'es.example.com' self.test_index = 'test_index' self.test_account = u'test_account' self.test_container = u'test_container' self.sync_conf = { 'es_hosts': self.es_hosts, 'index': self.test_index, 'account': self.test_account, 'container': self.test_container } self.es_mock = mock.Mock() self.es_mock.info.return_value = {'version': {'number': '2.2.0'}} mock_es.return_value = self.es_mock self.sync = metadata_sync.MetadataSync(self.status_dir, self.sync_conf)
def test_index_booleans(self, es_mock, helpers_mock, index_mock): obj = 'object' doc_id = self.compute_id(self.test_account, self.test_container, obj) config = dict(self.sync_conf) es_mock.return_value.info.return_value = { 'version': {'number': '7.4.0'}} es_mock.return_value.mget.return_value = { 'docs': [{'_id': doc_id, 'found': False}]} internal_client = mock.Mock() internal_client.get_object_metadata.return_value = { 'x-static-large-object': 'True', 'x-timestamp': 0, 'last-modified': 'Wed, 06 Jun 2018 22:24:19 GMT' } helpers_mock.bulk.return_value = (None, []) index_mock.return_value.get_mapping.return_value = { self.test_index: { 'mappings': { 'properties': metadata_sync.MetadataSync.DOC_MAPPING}}} sync = metadata_sync.MetadataSync(self.status_dir, config) sync.handle([{'name': obj, 'deleted': False, 'created_at': 0}], internal_client) helpers_mock.bulk.assert_called_once_with( es_mock.return_value, [{'_op_type': 'index', '_id': doc_id, '_index': self.test_index, '_source': { 'x-timestamp': 0, 'last-modified': 1528323859000, 'x-swift-account': self.test_account, 'x-swift-container': self.test_container, 'x-swift-object': obj, 'x-static-large-object': 'true', }}], raise_on_error=False, raise_on_exception=False)
def test_verify_mapping(self, es_mock, index_mock): full_mapping = metadata_sync.MetadataSync.DOC_MAPPING swift_type = metadata_sync.MetadataSync.DOC_TYPE # List of tuples of mappings to test: the mapping returned by ES and # the mapping we expect to submit to the put_mapping call. test_mappings = [({ self.test_index: { "mappings": {} } }, full_mapping), ({ self.test_index: { "mappings": { "bogus_type": full_mapping } } }, full_mapping), ({ self.test_index: { "mappings": { swift_type: { "properties": { "content-length": { "type": "string" }, "x-timestamp": { "type": "string" }, "x-trans-id": { "type": "string" } } } } } }, { "content-type": { "type": "string" }, "last-modified": { "type": "date" }, "etag": { "type": "string", "index": "not_analyzed" }, "x-object-manifest": { "type": "string" }, "x-static-large-object": { "type": "boolean" }, "x-swift-container": { "type": "string" }, "x-swift-account": { "type": "string" }, "x-swift-object": { "type": "string" }, }), ({ self.test_index: { "mappings": { swift_type: { "properties": full_mapping } } } }, {})] for return_mapping, expected_put_mapping in test_mappings: es_conn = mock.Mock() es_conn.info.return_value = {'version': {'number': '2.2.0'}} index_conn = mock.Mock() index_conn.get_mapping.return_value = return_mapping es_mock.return_value = es_conn index_mock.return_value = index_conn metadata_sync.MetadataSync(self.status_dir, self.sync_conf) if expected_put_mapping: index_conn.put_mapping.assert_called_once_with( index=self.test_index, doc_type=swift_type, body={"properties": expected_put_mapping}) else: index_conn.put_mapping.assert_not_called()