def test_redefine_attribute():
"""Check that redefining an attribute with a different type fails."""
json_schema = {
'id':
'https://example.org/root_schema#',
'type':
'object',
'anyOf': [
{
'type': 'object',
'properties': {
# first definition
'attr': {
'type': 'string'
},
},
},
{
'type': 'object',
'properties': {
# redefinition
'attr': {
'type': 'boolean'
},
},
}
]
}
with pytest.raises(JsonSchemaSupportError):
schema_to_mapping(json_schema, json_schema['id'], {},
ElasticMappingGeneratorConfig())
def test_depencency_extension():
"""Test ampping generation from schema containing a dependency"""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'properties': {
"name": {"type": "string"},
},
"dependencies": {
"name": {
"properties": {
"address": {"type": "boolean"}
},
"required": ["address"]
}
}
}
es_mapping = {
'_all': {'enabled': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'name': {'type': 'string'},
'address': {'type': 'boolean'},
},
}
result_mapping = schema_to_mapping(json_schema, json_schema['id'],
{},
ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
def test_type_mapping():
"""Test mapping a json type to another elasticsearch type."""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'properties': {
'attr1': {'type': 'string', 'format': 'custom-date'},
'attr2': {'type': 'number'},
},
}
es_mapping = {
'_all': {'enable': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'attr1': {'type': 'date', 'format': 'YYYY'},
'attr2': {'type': 'integer', 'coerce': False},
}
}
config = ElasticMappingGeneratorConfig() \
.map_type(es_type='integer',
json_type='number',
es_props={'coerce': False}) \
.map_type(es_type='date',
json_type='string',
json_format='custom-date')
config.date_format = 'YYYY'
assert config.date_format == 'YYYY'
result_mapping = schema_to_mapping(json_schema, json_schema['id'],
{}, config)
assert result_mapping == es_mapping
def test_additionnalproperties_value_to_false():
"""Check that putting additionalProperties to False doesn't stop."""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'properties': {
'attr1': {'type': 'string'},
'attr2': {'type': 'boolean'},
'attr3': {'type': 'number'},
'attr4': {'enum': ['Hello', 'world']},
'attr5': {'enum': [0, 1, 2]},
},
"additionalProperties": False
}
es_mapping = {
'_all': {'enable': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'attr1': {'type': 'string'},
'attr2': {'type': 'boolean'},
'attr3': {'type': 'double'},
'attr4': {'type': 'string'},
'attr5': {'type': 'double'},
},
}
result_mapping = schema_to_mapping(json_schema, json_schema['id'],
{},
ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
def test_references():
"""Test mapping generation from a schema containing references."""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'properties': {
'orig': {'type': 'string'},
'local_ref': {'$ref': '#/properties/orig'},
'ext_ref': {
'$ref':
'https://example.org/external_schema#/definitions/ext_def'
},
},
}
external_json_schema = {
'id': 'https://example.org/external_schema#',
'definitions': {
'ext_def': {'type': 'boolean'},
},
}
es_mapping = {
'_all': {'enable': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'orig': {'type': 'string'},
'local_ref': {'type': 'string'},
'ext_ref': {'type': 'boolean'},
},
}
result_mapping = schema_to_mapping(
json_schema, json_schema['id'], {
external_json_schema['id']: external_json_schema
}, ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
def test_additionnalproperties_value_to_false():
"""Check that putting additionalProperties to False doesn't stop."""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'properties': {
'attr1': {'type': 'string'},
'attr2': {'type': 'boolean'},
'attr3': {'type': 'number'},
'attr4': {'enum': ['Hello', 'world']},
'attr5': {'enum': [0, 1, 2]},
},
"additionalProperties": False
}
es_mapping = {
'_all': {'enabled': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'attr1': {'type': 'string'},
'attr2': {'type': 'boolean'},
'attr3': {'type': 'double'},
'attr4': {'type': 'string'},
'attr5': {'type': 'double'},
},
}
result_mapping = schema_to_mapping(json_schema, json_schema['id'],
{},
ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
def test_simple_properties():
"""Test generation of a very simple mapping"""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'properties': {
'attr1': {'type': 'string'},
'attr2': {'type': 'boolean'},
'attr3': {'type': 'number'},
'attr4': {'enum': ['Hello', 'world']},
'attr5': {'enum': [0, 1, 2]},
},
}
es_mapping = {
'_all': {'enable': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'attr1': {'type': 'string'},
'attr2': {'type': 'boolean'},
'attr3': {'type': 'double'},
'attr4': {'type': 'string'},
'attr5': {'type': 'double'},
},
}
result_mapping = schema_to_mapping(json_schema, json_schema['id'],
{},
ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
def test_simple_properties():
"""Test generation of a very simple mapping"""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'properties': {
'attr1': {'type': 'string'},
'attr2': {'type': 'boolean'},
'attr3': {'type': 'number'},
'attr4': {'enum': ['Hello', 'world']},
'attr5': {'enum': [0, 1, 2]},
},
}
es_mapping = {
'_all': {'enabled': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'attr1': {'type': 'string'},
'attr2': {'type': 'boolean'},
'attr3': {'type': 'double'},
'attr4': {'type': 'string'},
'attr5': {'type': 'double'},
},
}
result_mapping = schema_to_mapping(json_schema, json_schema['id'],
{},
ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
def test_type_mapping():
"""Test mapping a json type to another elasticsearch type."""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'properties': {
'attr1': {'type': 'string', 'format': 'custom-date'},
'attr2': {'type': 'number'},
},
}
es_mapping = {
'_all': {'enabled': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'attr1': {'type': 'date', 'format': 'YYYY'},
'attr2': {'type': 'integer', 'coerce': False},
}
}
config = ElasticMappingGeneratorConfig() \
.map_type(es_type='integer',
json_type='number',
es_props={'coerce': False}) \
.map_type(es_type='date',
json_type='string',
json_format='custom-date')
config.date_format = 'YYYY'
assert config.date_format == 'YYYY'
result_mapping = schema_to_mapping(json_schema, json_schema['id'],
{}, config)
assert result_mapping == es_mapping
def test_depencency_extension():
"""Test ampping generation from schema containing a dependency"""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'properties': {
"name": {"type": "string"},
},
"dependencies": {
"name": {
"properties": {
"address": {"type": "boolean"}
},
"required": ["address"]
}
}
}
es_mapping = {
'_all': {'enable': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'name': {'type': 'string'},
'address': {'type': 'boolean'},
},
}
result_mapping = schema_to_mapping(json_schema, json_schema['id'],
{},
ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
def test_references():
"""Test mapping generation from a schema containing references."""
json_schema = {
'id': 'https://example.org/root_schema.json',
'type': 'object',
'properties': {
'orig': {'type': 'string'},
'local_ref': {'$ref': '#/properties/orig'},
'cached_ext_ref': {
'$ref':
'https://example.org/cached_external_schema.json#'
'/definitions/cached_ext_def'
},
'ext_ref': {
'$ref':
'external_schema.json#/definitions/ext_def'
},
},
}
cached_external_json_schema = {
'id': 'https://example.org/cached_external_schema.json',
'definitions': {
'cached_ext_def': {'type': 'boolean'},
},
}
non_cached_external_json_schema = {
'id': 'https://example.org/external_schema.json',
'definitions': {
'ext_def': {'type': 'boolean'},
},
}
es_mapping = {
'_all': {'enabled': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'orig': {'type': 'string'},
'local_ref': {'type': 'string'},
'cached_ext_ref': {'type': 'boolean'},
'ext_ref': {'type': 'boolean'},
},
}
with responses.RequestsMock() as rsps:
# serve the external_schema.json file
rsps.add(responses.GET, 'https://example.org/external_schema.json',
body=json.dumps(non_cached_external_json_schema),
status=200,
content_type='application/json')
result_mapping = schema_to_mapping(
json_schema, json_schema['id'], {
cached_external_json_schema['id']: cached_external_json_schema
}, ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
def test_references():
"""Test mapping generation from a schema containing references."""
json_schema = {
'id': 'https://example.org/root_schema.json',
'type': 'object',
'properties': {
'orig': {'type': 'string'},
'local_ref': {'$ref': '#/properties/orig'},
'cached_ext_ref': {
'$ref':
'https://example.org/cached_external_schema.json#'
'/definitions/cached_ext_def'
},
'ext_ref': {
'$ref':
'external_schema.json#/definitions/ext_def'
},
},
}
cached_external_json_schema = {
'id': 'https://example.org/cached_external_schema.json',
'definitions': {
'cached_ext_def': {'type': 'boolean'},
},
}
non_cached_external_json_schema = {
'id': 'https://example.org/external_schema.json',
'definitions': {
'ext_def': {'type': 'boolean'},
},
}
es_mapping = {
'_all': {'enable': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'orig': {'type': 'string'},
'local_ref': {'type': 'string'},
'cached_ext_ref': {'type': 'boolean'},
'ext_ref': {'type': 'boolean'},
},
}
with responses.RequestsMock() as rsps:
# serve the external_schema.json file
rsps.add(responses.GET, 'https://example.org/external_schema.json',
body=json.dumps(non_cached_external_json_schema),
status=200,
content_type='application/json')
result_mapping = schema_to_mapping(
json_schema, json_schema['id'], {
cached_external_json_schema['id']: cached_external_json_schema
}, ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
def test_redefine_attribute():
"""Check that redefining an attribute with a different type fails."""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'anyOf': [{
'type': 'object',
'properties': {
# first definition
'attr': {'type': 'string'},
},
}, {
'type': 'object',
'properties': {
# redefinition
'attr': {'type': 'boolean'},
},
}]
}
with pytest.raises(JsonSchemaSupportError):
schema_to_mapping(json_schema, json_schema['id'],
{}, ElasticMappingGeneratorConfig())
def test_allOf_anyOf_oneOf():
"""Test mapping generation from a schema containing (all|any|one)Of"""
json_schema = {
'id':
'https://example.org/root_schema#',
'anyOf': [{
'type': 'object',
'properties': {
'defined_twice1': {
'type': 'string'
},
'allof_attr': {
'allOf': [{
'type': 'object',
'properties': {
'attr1': {
'type': 'string'
},
'defined_twice2': {
'type': 'string'
},
},
}, {
'type': 'object',
'properties': {
'attr2': {
'type': 'boolean'
},
'defined_twice2': {
'type': 'string'
},
},
}]
},
},
}, {
'type': 'object',
'properties': {
'defined_twice1': {
'type': 'string'
},
'oneof_attr': {
'oneOf': [{
'type': 'object',
'properties': {
'attr3': {
'type': 'string'
},
'defined_twice3': {
'type': 'string'
},
},
}, {
'type': 'object',
'properties': {
'attr4': {
'type': 'boolean'
},
'defined_twice3': {
'type': 'string'
},
},
}]
},
},
}]
}
es_mapping = {
'_all': {
'enable': True
},
'numeric_detection': True,
'date_detection': True,
'properties': {
'defined_twice1': {
'type': 'string'
},
'allof_attr': {
'type': 'object',
'properties': {
'attr1': {
'type': 'string'
},
'attr2': {
'type': 'boolean'
},
'defined_twice2': {
'type': 'string'
},
},
},
'oneof_attr': {
'type': 'object',
'properties': {
'attr3': {
'type': 'string'
},
'attr4': {
'type': 'boolean'
},
'defined_twice3': {
'type': 'string'
},
},
},
},
}
result_mapping = schema_to_mapping(json_schema, json_schema['id'], {},
ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
from domapping.mapping import ElasticMappingGeneratorConfig, schema_to_mapping
from domapping.templating import jinja_to_mapping, mapping_to_jinja
config = ElasticMappingGeneratorConfig()
with open('schema.json') as schema_file:
# read the schema
schema = json.load(schema_file)
with open('config.json') as config_file:
# load the configuration file
config.load(json.load(config_file))
# generate the intermediate mapping
mapping = schema_to_mapping(schema, schema['id'], {}, config)
print('>> Intermediate mapping')
print(json.dumps(mapping, indent=4))
print('=' * 50)
# generate the jinja template
generated_template = mapping_to_jinja(mapping, 'person')
generated_template_dir = 'generated'
if not os.path.exists(generated_template_dir):
os.mkdir(generated_template_dir)
generated_template_path = os.path.join(generated_template_dir,
'generated_schema.json')
def test_complex_array():
"""Test mapping generation from schema containing a complex array."""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'properties': {
'myarray': {
"type":
"array",
"items": [{
"type": "object",
"properties": {
"first_attr": {
"type": "string",
},
},
}, {
"type": "object",
"properties": {
"second_attr": {
"type": "boolean",
},
},
}]
},
'myarray2': {
"type": "array",
"items": {
"type": "object",
"properties": {
"third_attr": {
"type": "string",
},
},
},
},
},
}
es_mapping = {
'_all': {
'enable': True
},
'numeric_detection': True,
'date_detection': True,
'properties': {
'myarray': {
'type': 'object',
'properties': {
'first_attr': {
'type': 'string'
},
'second_attr': {
'type': 'boolean'
},
},
},
'myarray2': {
'type': 'object',
'properties': {
'third_attr': {
'type': 'string'
},
},
},
},
}
result_mapping = schema_to_mapping(json_schema, json_schema['id'], {},
ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
def test_complex_array():
"""Test mapping generation from schema containing a complex array."""
json_schema = {
'id': 'https://example.org/root_schema#',
'type': 'object',
'properties': {
'myarray': {
"type": "array",
"items": [{
"type": "object",
"properties": {
"first_attr": {
"type": "string",
},
},
}, {
"type": "object",
"properties": {
"second_attr": {
"type": "boolean",
},
},
}]
},
'myarray2': {
"type": "array",
"items": {
"type": "object",
"properties": {
"third_attr": {
"type": "string",
},
},
},
},
},
}
es_mapping = {
'_all': {'enable': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'myarray': {
'type': 'object',
'properties': {
'first_attr': {'type': 'string'},
'second_attr': {'type': 'boolean'},
},
},
'myarray2': {
'type': 'object',
'properties': {
'third_attr': {'type': 'string'},
},
},
},
}
result_mapping = schema_to_mapping(json_schema, json_schema['id'],
{},
ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
def test_allOf_anyOf_oneOf():
"""Test mapping generation from a schema containing (all|any|one)Of"""
json_schema = {
'id': 'https://example.org/root_schema#',
'anyOf': [{
'type': 'object',
'properties': {
'defined_twice1': {'type': 'string'},
'allof_attr': {
'allOf': [{
'type': 'object',
'properties': {
'attr1': {'type': 'string'},
'defined_twice2': {'type': 'string'},
},
}, {
'type': 'object',
'properties': {
'attr2': {'type': 'boolean'},
'defined_twice2': {'type': 'string'},
},
}]
},
},
}, {
'type': 'object',
'properties': {
'defined_twice1': {'type': 'string'},
'oneof_attr': {
'oneOf': [{
'type': 'object',
'properties': {
'attr3': {'type': 'string'},
'defined_twice3': {'type': 'string'},
},
}, {
'type': 'object',
'properties': {
'attr4': {'type': 'boolean'},
'defined_twice3': {'type': 'string'},
},
}]
},
},
}]
}
es_mapping = {
'_all': {'enable': True},
'numeric_detection': True,
'date_detection': True,
'properties': {
'defined_twice1': {'type': 'string'},
'allof_attr': {
'type': 'object',
'properties': {
'attr1': {'type': 'string'},
'attr2': {'type': 'boolean'},
'defined_twice2': {'type': 'string'},
},
},
'oneof_attr': {
'type': 'object',
'properties': {
'attr3': {'type': 'string'},
'attr4': {'type': 'boolean'},
'defined_twice3': {'type': 'string'},
},
},
},
}
result_mapping = schema_to_mapping(json_schema, json_schema['id'],
{},
ElasticMappingGeneratorConfig())
assert result_mapping == es_mapping
from domapping.mapping import ElasticMappingGeneratorConfig, schema_to_mapping
from domapping.templating import jinja_to_mapping, mapping_to_jinja
config = ElasticMappingGeneratorConfig()
with open("schema.json") as schema_file:
# read the schema
schema = json.load(schema_file)
with open("config.json") as config_file:
# load the configuration file
config.load(json.load(config_file))
# generate the intermediate mapping
mapping = schema_to_mapping(schema, schema["id"], {}, config)
print(">> Intermediate mapping")
print(json.dumps(mapping, indent=4))
print("=" * 50)
# generate the jinja template
generated_template = mapping_to_jinja(mapping, "person")
generated_template_dir = "generated"
if not os.path.exists(generated_template_dir):
os.mkdir(generated_template_dir)
generated_template_path = os.path.join(generated_template_dir, "generated_schema.json")
# write the template to a file so that it can be used by jinja
