def test_parse_json(self, mock_es, mock_helpers, index_mock):
test_cases = [
('{"bool": true, "number": 1234, "string": "foo"}', True),
('{regular meta}', False)
]
obj = 'object'
obj_meta = {
'x-timestamp': 0,
'last-modified': 'Wed, 06 Jun 2018 22:24:19 GMT'}
doc_id = self.compute_id(self.test_account, self.test_container, obj)
es_mock = mock_es.return_value
es_mock.info.return_value = {'version': {'number': '5.4.0'}}
es_mock.mget.return_value = {'docs': [{'_id': doc_id, 'found': False}]}
index_mock.return_value.get_mapping.return_value = {
self.test_index: {
'mappings': {
'properties': metadata_sync.MetadataSync.DOC_MAPPING}}}
sync_conf = dict(self.sync_conf)
sync_conf['parse_json'] = True
sync = metadata_sync.MetadataSync(self.status_dir, sync_conf)
for meta, is_json in test_cases:
mock_helpers.reset_mock()
mock_helpers.bulk.return_value = (None, [])
test_meta = dict(obj_meta)
test_meta['x-object-meta-test'] = meta
internal_client = mock.Mock()
internal_client.get_object_metadata.return_value = test_meta
sync.handle([{'name': obj,
'deleted': False,
'created_at': 0}],
internal_client)
if is_json:
expected = json.loads(meta)
else:
expected = meta
mock_helpers.bulk.assert_called_once_with(
es_mock, [
{'_op_type': 'index',
'_id': doc_id,
'_index': self.test_index,
'_type': metadata_sync.MetadataSync.DEFAULT_DOC_TYPE,
'_source': {
'test': expected,
'x-timestamp': 0,
'last-modified': 1528323859000,
'x-swift-account': self.test_account,
'x-swift-container': self.test_container,
'x-swift-object': obj}}],
raise_on_error=False,
raise_on_exception=False)
def test_verify_mapping(self, es_mock, index_mock):
full_mapping = {
key: metadata_sync.MetadataSync._update_string_mapping(val)
for key, val in metadata_sync.MetadataSync.DOC_MAPPING.items()}
# List of tuples of mappings to test: the mapping returned by ES and
# the mapping we expect to submit to the put_mapping call.
test_mappings = [
({self.test_index: {"mappings": {}}}, full_mapping),
({self.test_index: {"mappings": {
"properties": {
"content-length": {"type": "string"},
"x-timestamp": {"type": "string"},
"x-trans-id": {"type": "string"}
}
}}}, {
"content-type": {"type": "string"},
"last-modified": {"type": "date"},
"etag": {"type": "string", "index": "not_analyzed"},
"x-object-manifest": {"type": "string"},
"x-static-large-object": {"type": "boolean"},
"x-swift-container": {"type": "string"},
"x-swift-account": {"type": "string"},
"x-swift-object": {"type": "string"},
}),
({self.test_index: {"mappings": {
"properties": full_mapping}}},
{}),
({self.test_index: {'mappings': {
metadata_sync.MetadataSync.OLD_DOC_TYPE: {}}}},
full_mapping),
]
for return_mapping, expected_put_mapping in test_mappings:
es_conn = mock.Mock()
es_conn.info.return_value = {'version': {'number': '7.4.0'}}
index_conn = mock.Mock()
index_conn.get_mapping.return_value = return_mapping
es_mock.return_value = es_conn
index_mock.return_value = index_conn
metadata_sync.MetadataSync(self.status_dir, self.sync_conf)
expected_put_mapping = {
key: metadata_sync.MetadataSync._update_string_mapping(val)
for key, val in expected_put_mapping.items()}
return_mappings = return_mapping[self.test_index]['mappings']
if expected_put_mapping:
if not return_mappings or 'properties' in return_mappings:
index_conn.put_mapping.assert_called_once_with(
index=self.test_index,
body={"properties": expected_put_mapping},
include_type_name=False)
else:
index_conn.put_mapping.assert_called_once_with(
index=self.test_index,
body={"properties": expected_put_mapping},
doc_type=return_mappings.keys()[0])
else:
index_conn.put_mapping.assert_not_called()
def test_verify_mapping_5x(self, es_mock, index_mock):
full_mapping = metadata_sync.MetadataSync.DOC_MAPPING
swift_type = metadata_sync.MetadataSync.DOC_TYPE
# Test that "string" mappings are converted to keyword or text or both.
current_mapping = dict([(k, v) for k, v in full_mapping.items()
if v['type'] != 'string'])
text_and_keyword = {
'type': 'text',
'fields': {
'keyword': {
'type': 'keyword'
}
}
}
expected_mapping = {
'content-type': text_and_keyword,
'etag': {
'type': 'keyword'
},
'x-object-manifest': text_and_keyword,
'x-swift-container': text_and_keyword,
'x-swift-account': text_and_keyword,
'x-swift-object': text_and_keyword,
'x-trans-id': {
'type': 'keyword'
}
}
es_conn = mock.Mock()
es_conn.info.return_value = {'version': {'number': '5.0'}}
index_conn = mock.Mock()
index_conn.get_mapping.return_value = {
self.test_index: {
'mappings': {
swift_type: {
'properties': current_mapping
}
}
}
}
es_mock.return_value = es_conn
index_mock.return_value = index_conn
metadata_sync.MetadataSync(self.status_dir, self.sync_conf)
index_conn.put_mapping.assert_called_once_with(
index=self.test_index,
doc_type=swift_type,
body={'properties': expected_mapping})
def test_tls_parameters(self, mock_verify_mapping, mock_es):
sync_conf = {'es_hosts': self.es_hosts,
'index': self.test_index,
'account': self.test_account,
'container': self.test_container,
'verify_certs': False,
'ca_certs': "/no/such/pemfile"}
es_mock = mock_es.return_value
es_mock.info.return_value = {'version': {'number': '5.4.0'}}
metadata_sync.MetadataSync(self.status_dir, sync_conf)
mock_es.assert_called_once_with(
self.es_hosts,
ca_certs='/no/such/pemfile',
verify_certs=False)
def test_set_pipeline(self, es_mock, helpers_mock):
pipeline = 'test-pipeline'
obj = 'object'
doc_id = self.compute_id(self.test_account, self.test_container, obj)
config = dict(self.sync_conf)
config['pipeline'] = pipeline
es_mock.return_value.info.return_value = {
'version': {
'number': '5.4.0'
}
}
es_mock.return_value.mget.return_value = {
'docs': [{
'_id': doc_id,
'found': False
}]
}
internal_client = mock.Mock()
internal_client.get_object_metadata.return_value = {
'x-timestamp': 0,
'last-modified': 'Wed, 06 Jun 2018 22:24:19 GMT'
}
helpers_mock.bulk.return_value = (None, [])
sync = metadata_sync.MetadataSync(self.status_dir, config)
sync.handle([{
'name': obj,
'deleted': False,
'created_at': 0
}], internal_client)
helpers_mock.bulk.assert_called_once_with(
es_mock.return_value, [{
'_op_type': 'index',
'_id': doc_id,
'_index': self.test_index,
'_type': metadata_sync.MetadataSync.DOC_TYPE,
'_source': {
'x-timestamp': 0,
'last-modified': 1528323859000,
'x-swift-account': self.test_account,
'x-swift-container': self.test_container,
'x-swift-object': obj,
},
'pipeline': 'test-pipeline'
}],
raise_on_error=False,
raise_on_exception=False)
def test_verify_mapping_multiple_mappings(self, es_mock, index_mock):
mapping_resp = {self.test_index: {
"mappings": {
"bogus_type": metadata_sync.MetadataSync.DOC_MAPPING}}}
es_conn = mock.Mock()
es_conn.info.return_value = {'version': {'number': '6.8.0'}}
index_conn = mock.Mock()
index_conn.get_mapping.return_value = mapping_resp
es_mock.return_value = es_conn
index_mock.return_value = index_conn
with self.assertRaises(RuntimeError) as err_ctx:
metadata_sync.MetadataSync(self.status_dir, self.sync_conf)
self.assertEqual('Cannot set more than one mapping type for index {}. '
"Known types: ['bogus_type']".format(self.test_index),
err_ctx.exception.message)
def setUp(self, mock_verify_mapping, mock_es):
self.status_dir = '/status/dir'
self.es_hosts = 'es.example.com'
self.test_index = 'test_index'
self.test_account = u'test_account'
self.test_container = u'test_container'
self.sync_conf = {
'es_hosts': self.es_hosts,
'index': self.test_index,
'account': self.test_account,
'container': self.test_container
}
self.es_mock = mock.Mock()
self.es_mock.info.return_value = {'version': {'number': '2.2.0'}}
mock_es.return_value = self.es_mock
self.sync = metadata_sync.MetadataSync(self.status_dir, self.sync_conf)
def test_index_booleans(self, es_mock, helpers_mock, index_mock):
obj = 'object'
doc_id = self.compute_id(self.test_account, self.test_container, obj)
config = dict(self.sync_conf)
es_mock.return_value.info.return_value = {
'version': {'number': '7.4.0'}}
es_mock.return_value.mget.return_value = {
'docs': [{'_id': doc_id, 'found': False}]}
internal_client = mock.Mock()
internal_client.get_object_metadata.return_value = {
'x-static-large-object': 'True',
'x-timestamp': 0,
'last-modified': 'Wed, 06 Jun 2018 22:24:19 GMT'
}
helpers_mock.bulk.return_value = (None, [])
index_mock.return_value.get_mapping.return_value = {
self.test_index: {
'mappings': {
'properties': metadata_sync.MetadataSync.DOC_MAPPING}}}
sync = metadata_sync.MetadataSync(self.status_dir, config)
sync.handle([{'name': obj, 'deleted': False, 'created_at': 0}],
internal_client)
helpers_mock.bulk.assert_called_once_with(
es_mock.return_value,
[{'_op_type': 'index',
'_id': doc_id,
'_index': self.test_index,
'_source': {
'x-timestamp': 0,
'last-modified': 1528323859000,
'x-swift-account': self.test_account,
'x-swift-container': self.test_container,
'x-swift-object': obj,
'x-static-large-object': 'true',
}}],
raise_on_error=False,
raise_on_exception=False)
def test_verify_mapping(self, es_mock, index_mock):
full_mapping = metadata_sync.MetadataSync.DOC_MAPPING
swift_type = metadata_sync.MetadataSync.DOC_TYPE
# List of tuples of mappings to test: the mapping returned by ES and
# the mapping we expect to submit to the put_mapping call.
test_mappings = [({
self.test_index: {
"mappings": {}
}
}, full_mapping),
({
self.test_index: {
"mappings": {
"bogus_type": full_mapping
}
}
}, full_mapping),
({
self.test_index: {
"mappings": {
swift_type: {
"properties": {
"content-length": {
"type": "string"
},
"x-timestamp": {
"type": "string"
},
"x-trans-id": {
"type": "string"
}
}
}
}
}
}, {
"content-type": {
"type": "string"
},
"last-modified": {
"type": "date"
},
"etag": {
"type": "string",
"index": "not_analyzed"
},
"x-object-manifest": {
"type": "string"
},
"x-static-large-object": {
"type": "boolean"
},
"x-swift-container": {
"type": "string"
},
"x-swift-account": {
"type": "string"
},
"x-swift-object": {
"type": "string"
},
}),
({
self.test_index: {
"mappings": {
swift_type: {
"properties": full_mapping
}
}
}
}, {})]
for return_mapping, expected_put_mapping in test_mappings:
es_conn = mock.Mock()
es_conn.info.return_value = {'version': {'number': '2.2.0'}}
index_conn = mock.Mock()
index_conn.get_mapping.return_value = return_mapping
es_mock.return_value = es_conn
index_mock.return_value = index_conn
metadata_sync.MetadataSync(self.status_dir, self.sync_conf)
if expected_put_mapping:
index_conn.put_mapping.assert_called_once_with(
index=self.test_index,
doc_type=swift_type,
body={"properties": expected_put_mapping})
else:
index_conn.put_mapping.assert_not_called()
