def test_create_es_action_without_id_field_in_translation_map(self):
self.assertEqual(
'{"index": {"_index": "baz", "_type": "doc", "_id": "123"}}\n{"id": "123", "foo": "bar"}',
create_es_actions('baz', [{
'id': '123',
'foo': 'bar'
}], TranslationMap()))
def test_create_es_action_with_multivalued_and_named_field(self):
self.assertEqual(
[({
'index': {
'_index': 'baz',
'_type': 'doc',
'_id': '808'
}
}, {
'id': '808',
'path': 'value_01'
}),
({
'index': {
'_index': 'baz',
'_type': 'doc',
'_id': hashlib.sha256(b'value_02').hexdigest()
}
}, {
'path': 'value_02',
'documentId': '808',
'type': 'Duplicate'
})],
create_es_actions(
'baz', [{
'id': '808',
'my_field': ['value_01', 'value_02']
}],
TranslationMap(
{'my_field': {
'multivalued': False,
'name': 'path'
}})))
def test_with_empty_array_as_default_value(self):
self.assertEqual({'array_field': []},
translate_doc({},
TranslationMap(
{'array_field': {
'default': []
}})))
def test_with_nested_fields_and_multivalued_field(self):
self.assertEqual({'a': {
'b': 'value1'
}},
translate_doc({'a_b': ['value1']},
TranslationMap({'a_b': {
'name': 'a.b'
}})))
def test_create_es_action_with_nonexistent_routing_field(self):
self.assertEqual(
'{"index": {"_index": "baz", "_type": "doc", "_id": "808"}}\n{"id": "808"}',
create_es_actions(
'baz', [{
'id': '808'
}], TranslationMap({'root_id': {
'routing_field': True
}})))
def test_create_es_actions(self):
self.assertEqual(
'{"index": {"_index": "baz", "_type": "doc", "_id": "123"}}\n{"my_id": "123", "foo": "bar"}',
create_es_actions('baz', [{
'my_id': '123',
'foo': 'bar'
}], TranslationMap({'my_id': {
'name': '_id'
}})))
def test_create_es_action_with_routing_field_false(self):
self.assertEqual(
'{"index": {"_index": "baz", "_type": "doc", "_id": "808"}}\n{"id": "808", "root_id": "654"}',
create_es_actions(
'baz', [{
'id': '808',
'root_id': '654'
}], TranslationMap({'root_id': {
'routing_field': False
}})))
def test_create_es_action_with_routing_field_in_translation_map(self):
self.assertEqual(
'{"index": {"_index": "baz", "_type": "doc", "_id": "321", "_routing": "456"}}\n{"id": "321", "root_id": "456"}',
create_es_actions(
'baz', [{
'id': '321',
'root_id': '456'
}], TranslationMap({'root_id': {
'routing_field': True
}})))
def test_with_nested_field(self):
self.assertEqual({'a': {
'b': {
'c': 'value'
}
}},
translate_doc({'a_b_c': 'value'},
TranslationMap(
{'a_b_c': {
'name': 'a.b.c'
}})))
def test_migrate_with_ignored_fields(self):
TestMigration.solr.add([{"id": "12"}])
self.solr2es.migrate('foo', {},
TranslationMap({'_version_': {
'ignore': True
}}))
doc = self.es.get_source(index='foo',
doc_type=DEFAULT_ES_DOC_TYPE,
id="12")
self.assertEqual({"id": "12"}, doc)
def test_with_multivalued_field_ignored(self):
self.assertEqual({
'id': '1234',
'ignored_mv_field': 'value1'
},
translate_doc(
{
'id': '1234',
'ignored_mv_field': ['value1', 'value2']
},
TranslationMap(
{'ignored_mv_field': {
'multivalued': False
}})))
def test_create_es_action_with_more_than_one_routing_field_in_translation_map(
self):
create_es_actions(
'baz', [{
'id': '321'
}],
TranslationMap({
'route1': {
'routing_field': True
},
'route2': {
'routing_field': True
}
}))
def test_create_es_action_without_id_field_in_translation_map(self):
self.assertEqual([({
'index': {
'_index': 'baz',
'_type': 'doc',
'_id': '123'
}
}, {
'id': '123',
'foo': 'bar'
})],
create_es_actions('baz', [{
'id': '123',
'foo': 'bar'
}], TranslationMap()))
def test_with_sibling_nested_fields(self):
self.assertEqual({'a': {
'b': 'value1',
'c': 'value2'
}},
translate_doc({
'a_b': 'value1',
'a_c': 'value2'
},
TranslationMap({
'a_b': {
'name': 'a.b'
},
'a_c': {
'name': 'a.c'
}
})))
def test_create_es_action_with_nonexistent_routing_field(self):
self.assertEqual([({
'index': {
'_index': 'baz',
'_type': 'doc',
'_id': '808'
}
}, {
'id': '808'
})],
create_es_actions(
'baz', [{
'id': '808'
}],
TranslationMap(
{'root_id': {
'routing_field': True
}})))
def test_create_es_actions(self):
self.assertEqual([({
'index': {
'_index': 'baz',
'_type': 'doc',
'_id': '123'
}
}, {
'my_id': '123',
'foo': 'bar'
})],
create_es_actions(
'baz', [{
'my_id': '123',
'foo': 'bar'
}], TranslationMap({'my_id': {
'name': '_id'
}})))
def test_migrate_with_multiple_matching_fields_one_exact(self):
TestMigration.solr.add([{"id": "678", "title": "this is a title"}])
self.solr2es.migrate(
'foo', None,
TranslationMap({
"title": {
"name": "new_title"
},
"ti(.*)": {
"name": "regexp_title"
}
}))
doc = self.es.get_source(index='foo',
doc_type=DEFAULT_ES_DOC_TYPE,
id="678")
self.assertEqual('this is a title', doc['new_title'])
def test_migrate_with_default_field_value_on_unexisting_field(self):
TestMigration.solr.add([{"id": "142"}])
self.solr2es.migrate(
'foo',
'{"mappings": {"doc": {"properties": {"nested": {"type": "object"}}}}}',
TranslationMap({
"new_field": {
'default': 'john doe'
},
'field1': {
'name': 'field1'
}
}))
doc = self.es.get_source(index='foo',
doc_type=DEFAULT_ES_DOC_TYPE,
id="142")
self.assertEqual('john doe', doc['new_field'])
def test_migrate_nested_fields_with_translation_map(self):
requests.post(
'http://solr:8983/solr/test_core/schema',
headers={'Content-Type': 'application/json'},
data=
'{ "add-field":{ "name":"nested_field", "type":"string", "stored":true }}'
)
TestMigration.solr.add([{"id": "142", "nested_field": "content"}])
self.solr2es.migrate(
'foo',
'{"mappings": {"doc": {"properties": {"nested": {"type": "object"}}}}}',
TranslationMap({"nested_field": {
"name": "nested.a.b.c"
}}))
doc = self.es.get_source(index='foo',
doc_type=DEFAULT_ES_DOC_TYPE,
id="142")
self.assertEqual({'a': {'b': {'c': 'content'}}}, doc['nested'])
def test_create_es_action_with_routing_field_false(self):
self.assertEqual([({
'index': {
'_index': 'baz',
'_type': 'doc',
'_id': '808'
}
}, {
'id': '808',
'root_id': '654'
})],
create_es_actions(
'baz', [{
'id': '808',
'root_id': '654'
}],
TranslationMap(
{'root_id': {
'routing_field': False
}})))
def test_migrate_different_fields_with_translation_map(self):
requests.post(
'http://solr:8983/solr/test_core/schema',
headers={'Content-Type': 'application/json'},
data=
'{ "add-field":{ "name":"my_bar", "type":"string", "stored":true }}'
)
TestMigration.solr.add([{"id": "142", "my_bar": "content"}])
self.solr2es.migrate(
'foo',
'{"mappings": {"doc": {"properties": {"my_baz": {"type": "text"}}}}}',
TranslationMap({"my_bar": {
'name': "my_baz"
}}))
doc = self.es.get_source(index='foo',
doc_type=DEFAULT_ES_DOC_TYPE,
id="142")
self.assertEqual(doc['my_baz'], "content")
def test_migrate_with_default_field_value_on_existing_field(self):
requests.post(
'http://solr:8983/solr/test_core/schema',
headers={'Content-Type': 'application/json'},
data=
'{ "add-field":{ "name":"field1", "type":"string", "stored":true }}'
)
TestMigration.solr.add([{"id": "142", "field1": "content1"}])
self.solr2es.migrate(
'foo',
'{"mappings": {"doc": {"properties": {"nested": {"type": "object"}}}}}',
TranslationMap({"field1": {
'default': 'content2'
}}))
doc = self.es.get_source(index='foo',
doc_type=DEFAULT_ES_DOC_TYPE,
id="142")
self.assertEqual('content1', doc['field1'])
def test_create_es_action_with_routing_field_in_translation_map(self):
self.assertEqual([({
'index': {
'_index': 'baz',
'_type': 'doc',
'_id': '321',
'_routing': '456'
}
}, {
'id': '321',
'root_id': '456'
})],
create_es_actions(
'baz', [{
'id': '321',
'root_id': '456'
}],
TranslationMap(
{'root_id': {
'routing_field': True
}})))
def test_with_sibling_nested_fields_in_depth(self):
self.assertEqual({'a': {
'b': {
'c': {
'd': 'value1'
},
'e': 'value2'
}
}},
translate_doc({
'a_b_c_d': 'value1',
'a_b_e': 'value2'
},
TranslationMap({
'a_b_c_d': {
'name': 'a.b.c.d'
},
'a_b_e': {
'name': 'a.b.e'
}
})))
def test_migrate_sibling_nested_fields_with_wildcard(self):
TestMigration.solr.add([{
"id": "142",
"nested_field1": "content1",
"nested_field2": "content2"
}])
self.solr2es.migrate(
'foo',
'{"mappings": {"doc": {"properties": {"nested": {"type": "object"}}}}}',
TranslationMap(
{re.compile(r"nested_(.*)"): {
"name": "nested.\\1"
}}))
doc = self.es.get_source(index='foo',
doc_type=DEFAULT_ES_DOC_TYPE,
id="142")
self.assertEqual({
'field1': 'content1',
'field2': 'content2'
}, doc['nested'])
def test_migrate_with_multiple_matching_fields_against_mapping(self):
requests.post(
'http://solr:8983/solr/test_core/schema',
headers={'Content-Type': 'application/json'},
data=
'{ "add-field":{ "name":"flag_field_test", "type":"string", "stored":true }}'
)
TestMigration.solr.add([{"id": "142", "flag_field_test": "content1"}])
with assert_raises(IllegalStateError) as e:
self.solr2es.migrate(
'foo',
'{"mappings": {"doc": {"properties": {"nested": {"type": "object"}}}}}',
TranslationMap({
re.compile(r"flag_field_(.*)"): {
"name": "flag1_\\1"
},
re.compile(r"flag_(.*)"): {
"name": "flag2_\\1"
}
}))
self.assertTrue(
'Too many doc fields matching key flag_field_test in translation map'
in str(e.exception))
def test_with_multivalued_field_multiple_values(self):
self.assertEqual({'mv_field': ['value1', 'value2']},
translate_doc({'mv_field': ['value1', 'value2']},
TranslationMap({})))
def test_with_multivalued_field_one_value(self):
self.assertEqual({'mv_field': 'value1'},
translate_doc({'mv_field': ['value1']},
TranslationMap({})))