def add_schema_from_fixture(data=None):
"""Add or update schema."""
allow_all = data.pop("allow_all", False)
name = data['name']
try:
with db.session.begin_nested():
with db.session.begin_nested():
try:
schema = Schema.get(name=data['name'],
version=data['version'])
click.secho('{} already exist in the db.'.format(
str(name)))
return
except JSONSchemaNotFound:
schema = Schema(**data)
db.session.add(schema)
if allow_all:
schema.add_read_access_for_all_users()
except IntegrityError:
click.secho('Error occured during adding {} to the db. \n'.format(
str(name)),
fg='red')
return
db.session.commit()
click.secho('{} has been added.'.format(str(name)), fg='green')
def _add_schema(schema, is_deposit=True, experiment=None, json=None, roles=None):
"""
Add new schema into db
"""
default_json = {
'title': {
'type': 'string'
}
}
try:
schema = Schema.get_by_fullpath(schema)
except SchemaDoesNotExist:
schema = Schema(
fullpath=schema,
experiment=experiment,
is_deposit=is_deposit,
json=json or default_json
)
db.session.add(schema)
db.session.commit()
if roles:
roles = Role.query.filter(Role.name.in_(roles)).all()
for role in roles:
schema.add_read_access(role)
return schema
def test_add_or_update_schema_when_schema_already_exist_updates_json_for_schema(
db): # noqa
db.session.add(
Schema(
**{
'name': 'records/ana1',
'major': 1,
'minor': 0,
'patch': 2,
'experiment': 'CMS',
'fullname': 'Old Schema',
'json': {
'field': 'initial'
}
}))
db.session.commit()
updated_data = {
'experiment': 'Updated',
'fullname': 'Updated Schema',
'jsonschema': {
'updated': 'string'
}
}
add_or_update_schema(fullpath='records/ana1-v1.0.2.json',
data=updated_data)
schema = Schema.get_by_fullpath('records/ana1-v1.0.2.json')
assert schema.name == 'records/ana1'
assert schema.fullname == updated_data['fullname']
assert schema.experiment == updated_data['experiment']
assert schema.json == updated_data['jsonschema']
assert schema.major == 1
assert schema.minor == 0
assert schema.patch == 2
def _add_schema(name,
deposit_schema=None,
is_indexed=True,
use_deposit_as_record=True,
version="1.0.0",
**kwargs):
"""
Add new schema into db
"""
default_json = {'title': {'type': 'string'}}
try:
schema = Schema.get(name, version)
except JSONSchemaNotFound:
schema = Schema(name=name,
version=version,
is_indexed=is_indexed,
use_deposit_as_record=use_deposit_as_record,
deposit_schema=deposit_schema or default_json,
**kwargs)
db.session.add(schema)
db.session.commit()
if not schema.experiment:
schema.add_read_access_for_all_users()
db.session.commit()
return schema
def test_get_only_latest_version_of_schemas(client, db, users,
auth_headers_for_user):
cms_user = users['cms_user']
latest_schema = Schema(name='schema1', version='1.2.3', experiment='CMS')
latest_schema2 = Schema(name='schema2', version='3.0.0', experiment='CMS')
db.session.add(latest_schema)
db.session.add(latest_schema2)
db.session.add(Schema(name='schema1', experiment='CMS', version='1.1.0'))
db.session.add(Schema(name='schema2', experiment='CMS'))
db.session.add(Schema(name='schema3', experiment='LHCb'))
db.session.commit()
resp = client.get('/jsonschemas?latest=True',
headers=auth_headers_for_user(cms_user))
assert resp.status_code == 200
assert resp.json == [
{
'name': 'schema1',
'version': '1.2.3',
'deposit_options': {},
'record_schema': {},
'fullname': None,
'use_deposit_as_record': False,
'is_indexed': False,
'record_options': {},
'deposit_mapping': {},
'deposit_schema': {},
'record_mapping': {},
'links': {
'record':
'http://analysispreservation.cern.ch/api/schemas/records/schema1-v1.2.3.json',
'self':
'http://analysispreservation.cern.ch/api/jsonschemas/schema1/1.2.3',
'deposit':
'http://analysispreservation.cern.ch/api/schemas/deposits/records/schema1-v1.2.3.json',
},
},
{
'links': {
'record':
'http://analysispreservation.cern.ch/api/schemas/records/schema2-v3.0.0.json',
'self':
'http://analysispreservation.cern.ch/api/jsonschemas/schema2/3.0.0',
'deposit':
'http://analysispreservation.cern.ch/api/schemas/deposits/records/schema2-v3.0.0.json',
},
'deposit_options': {},
'record_schema': {},
'fullname': None,
'use_deposit_as_record': False,
'version': '3.0.0',
'is_indexed': False,
'record_options': {},
'deposit_mapping': {},
'deposit_schema': {},
'record_mapping': {},
'name': 'schema2',
},
]
def test_get(db):
schema = Schema(name='test-schema')
db.session.add(schema)
db.session.add(Schema(name='test-schema', version='2.0.0'))
db.session.add(Schema(name='another-schema', version='1.0.1'))
db.session.commit()
assert Schema.get('test-schema', '1.0.0') == schema
def test_schema_name_to_url(db):
db.session.add(Schema(name='my-schema', version='1.5.5'))
db.session.add(Schema(name='my-schema', version='2.4.0'))
db.session.add(Schema(name='my-schema', version='2.4.3'))
db.session.add(Schema(name='different-schema', version='3.5.4'))
db.session.commit()
assert schema_name_to_url(
'my-schema'
) == 'https://analysispreservation.cern.ch/schemas/deposits/records/my-schema-v2.4.3.json'
def test_when_schema_with_same_name_and_version_raises_IntegrityError(db):
with raises(IntegrityError):
schema = {
'name': 'records/ana1',
'major': 1,
'minor': 0,
'patch': 1
}
db.session.add(Schema(**schema))
db.session.add(Schema(**schema))
db.session.commit()
def test_when_schema_with_same_name_and_version_raises_IntegrityError(db):
with raises(IntegrityError):
schema = {
'name': 'cms-analysis',
'deposit_schema': {},
'major': 1,
'minor': 0,
'patch': 1
}
db.session.add(Schema(**schema))
db.session.add(Schema(**schema))
db.session.commit()
def test_parse_fullpath():
wrong_string = "deposits/records/alice-analysis-111v0.0.1.json"
correct_string = "deposits/records/alice-analysis-v0.0.1.json"
with raises(JSONSchemaNotFound):
Schema._parse_fullpath(wrong_string)
resp = Schema._parse_fullpath(correct_string)
assert resp[0] == "deposits/records/alice-analysis"
assert resp[1] == "0"
assert resp[2] == "0"
assert resp[3] == "1"
def test_create_newer_version_of_existing_schema(db):
schema = {
'name': 'records/ana1',
'major': 1,
'minor': 0,
'patch': 1,
}
db.session.add(Schema(**schema))
schema.update({'patch': 2})
db.session.add(Schema(**schema))
db.session.commit()
assert Schema.query.count() == 2
def _validate_data(cls, data):
if not isinstance(data, dict) or data == {}:
raise DepositValidationError('Empty deposit data.')
try:
schema_fullpath = data['$schema']
except KeyError:
raise DepositValidationError('Schema not specified.')
try:
Schema.get_by_fullpath(schema_fullpath)
except (AttributeError, JSONSchemaNotFound):
raise DepositValidationError('Schema {} is not a valid option.'
.format(schema_fullpath))
def test_create_newer_version_of_existing_schema(db):
schema = {
'name': 'cms-analysis',
'deposit_schema': {},
'major': 1,
'minor': 0,
'patch': 1
}
db.session.add(Schema(**schema))
schema.update({'patch': 2})
db.session.add(Schema(**schema))
db.session.commit()
assert Schema.query.count() == 2
def _validate_data(cls, data):
if not isinstance(data, dict) or data == {}:
raise DepositValidationError('Empty deposit data.')
try:
schema_fullpath = data['$schema']
except KeyError:
raise DepositValidationError('Schema not specified.')
try:
Schema.get_by_fullpath(schema_fullpath)
except (AttributeError, JSONSchemaNotFound):
raise DepositValidationError(
'Schema {} is not a valid option.'.format(schema_fullpath))
def test_get_latest_version_of_schema(db):
schemas = [
{'name': 'name', 'major': 0, 'minor': 6, 'patch': 6},
{'name': 'name', 'major': 1, 'minor': 5, 'patch': 5},
{'name': 'name', 'major': 2, 'minor': 3, 'patch': 4},
{'name': 'name', 'major': 2, 'minor': 4, 'patch': 0},
{'name': 'name', 'major': 2, 'minor': 4, 'patch': 3},
]
for x in schemas:
db.session.add(Schema(**x))
db.session.commit()
latest = Schema.get_latest(name='name')
assert latest.version == "2.4.3"
def test_add_or_update_schema_when_schema_already_exist_updates_json_for_schema(db): # noqa
db.session.add(Schema(**{
'name': 'records/ana1',
'major': 1,
'minor': 0,
'patch': 2,
'experiment': 'CMS',
'fullname': 'Old Schema',
'json': {'field': 'initial'}
}))
db.session.commit()
updated_data = {
'experiment': 'Updated',
'fullname': 'Updated Schema',
'jsonschema': {
'updated': 'string'
}
}
add_or_update_schema(fullpath='records/ana1-v1.0.2.json',
data=updated_data)
schema = Schema.get_by_fullpath('records/ana1-v1.0.2.json')
assert schema.name == 'records/ana1'
assert schema.fullname == updated_data['fullname']
assert schema.experiment == updated_data['experiment']
assert schema.json == updated_data['jsonschema']
assert schema.major == 1
assert schema.minor == 0
assert schema.patch == 2
def test_add_schema_from_fixture_when_schema_already_exist_updates_json_for_schema(
db):
updated_data = dict(name='new-schema',
version='1.1.1',
experiment='LHCb',
fullname='New fullname',
deposit_schema={'title': 'deposit_schema'},
deposit_options={'title': 'deposit_options'},
record_schema={'title': 'record_schema'},
record_options={'title': 'record_options'},
record_mapping={
'mappings': {
'doc': {
'properties': {
"title": {
"type": "text"
}
}
}
}
},
deposit_mapping={
'mappings': {
'doc': {
'properties': {
"keyword": {
"type": "keyword"
}
}
}
}
},
is_indexed=True,
use_deposit_as_record=False)
db.session.add(
Schema(**{
'name': 'new-schema',
'experiment': 'CMS',
'fullname': 'Old Schema',
}))
db.session.commit()
add_schema_from_json(data=updated_data)
schema = Schema.get('new-schema', version='1.1.1')
for key, value in updated_data.items():
assert getattr(schema, key) == value
def test_get_when_user_outside_of_experiment_returns_403(
client, db, users, auth_headers_for_user):
other_user = users['lhcb_user']
schema = Schema(
name='cms-schema',
version='1.2.3',
fullname='CMS Schema 1.2.3',
experiment='CMS',
deposit_schema={'title': 'deposit_schema'},
deposit_options={'title': 'deposit_options'},
record_schema={'title': 'record_schema'},
record_options={'title': 'record_options'},
record_mapping={
'mappings': {
'doc': {
'properties': {
'title': {
'type': 'text'
}
}
}
}
},
deposit_mapping={
'mappings': {
'doc': {
'properties': {
'keyword': {
'type': 'keyword'
}
}
}
}
},
is_indexed=True,
)
db.session.add(schema)
db.session.add(Schema(name='cms-schema', version='1.1.0'))
db.session.add(Schema(name='lhcb-schema'))
db.session.commit()
resp = client.get('/jsonschemas/cms-schema/1.2.3',
headers=auth_headers_for_user(other_user))
assert resp.status_code == 403
def resolve_cap_schemas(path):
"""Resolve CAP JSON schemas."""
schema = Schema.get_by_fullpath(path)
if not has_request_context() or ReadSchemaPermission(schema).can():
return schema.json
else:
abort(403)
def resolve_cap_schemas(path):
"""Resolve CAP JSON schemas."""
schema = Schema.get_by_fullpath(path)
if not has_request_context() or ReadSchemaPermission(schema).can():
return schema.json
else:
abort(403)
def test_when_schema_name_contains_forbidden_characters_raises_AssertionError(
db):
with raises(ValueError):
schema = {
'name': 'cmsanalysis,\\',
'deposit_schema': {},
}
db.session.add(Schema(**schema))
db.session.commit()
def test_create_schema_by_fullpath(db):
schema = Schema('records/ana1-v1.3.2')
db.session.add(schema)
db.session.commit()
assert schema.name == 'records/ana1'
assert schema.major == 1
assert schema.minor == 3
assert schema.patch == 2
def test_add_schemas_with_url_with_replace(app, db, cli_runner):
schema = Schema(
name='test',
experiment='CMS',
version='1.0.0',
deposit_schema={
'type': 'object',
'properties': {
'title': {'type': 'string'}
}
})
db.session.add(schema)
db.session.commit()
responses.add(
responses.GET,
'http://schemas.org/my-schema.json',
json={
'version': '1.0.0',
'name': 'test',
'use_deposit_as_record': True,
'deposit_schema': {
'properties': {
'_buckets': {},
'_deposit': {},
'_files': {},
'_experiment': {},
'_fetched_from': {},
'_user_edited': {},
'_access': {},
'title': {'type': 'string'},
'abstract': {'type': 'string'}
}
}
},
status=200
)
res = cli_runner('fixtures schemas --url http://schemas.org/my-schema.json -r')
assert res.exit_code == 0
assert 'test has been added.' in res.output
schema = Schema.get_latest('test')
assert 'abstract' in schema.deposit_schema['properties'].keys()
def get_user_deposit_groups():
"""Get Deposit Groups."""
# Set deposit groups for user
schemas = Schema.get_user_deposit_schemas()
dep_groups = [{
'name': schema.fullname,
'deposit_group': schema.name.replace('deposits/records/', '')
} for schema in schemas]
return dep_groups
def get_user_deposit_groups():
"""Get Deposit Groups."""
# Set deposit groups for user
schemas = Schema.get_user_deposit_schemas()
dep_groups = [{
'name': schema.fullname,
'deposit_group': schema.name.replace('deposits/records/', '')
} for schema in schemas]
return dep_groups
def build_cadi_deposit(cadi_id, cadi_info):
"""Build CMS analysis deposit, based on CADI entry."""
schema = Schema.get_latest('deposits/records/cms-analysis').fullpath
return {
'$schema': schema,
'cadi_info': cadi_info,
'general_title': cadi_id,
'basic_info': {
'cadi_id': cadi_id
}
}
def test_on_save_mapping_is_created_and_index_name_added_to_mappings_map(schema_params, index_name, db, es):
schema = Schema(**schema_params)
db.session.add(schema)
db.session.commit()
assert index_name in current_search.mappings.keys()
assert es.indices.exists(index_name)
db.session.delete(schema)
db.session.commit()
assert not es.indices.exists(index_name)
assert not schema.name in current_search.mappings.keys()
def test_schema_stringify(db):
schema = Schema(
**{
'name': 'cms-analysis',
'deposit_schema': {},
'major': 1,
'minor': 0,
'patch': 2
})
db.session.add(schema)
db.session.commit()
assert str(schema) == 'cms-analysis-v1.0.2'
def test_get_latest_version_of_schema(db):
latest_schema = Schema(name='my-schema', version='2.4.3')
db.session.add(Schema(name='my-schema', version='0.6.6'))
db.session.add(Schema(name='my-schema', version='1.5.5'))
db.session.add(Schema(name='my-schema', version='2.3.4'))
db.session.add(Schema(name='my-schema', version='2.4.0'))
db.session.add(Schema(name='different-schema', version='3.5.4'))
db.session.add(latest_schema)
db.session.commit()
assert Schema.get_latest('my-schema') == latest_schema
def _add_schema(schema, is_deposit=True, experiment=None, json=None):
"""
Add new schema into db
"""
default_json = {
'title': {
'type': 'string'
}
}
try:
schema = Schema.get_by_fullpath(schema)
except JSONSchemaNotFound:
schema = Schema(
fullpath=schema,
experiment=experiment,
is_deposit=is_deposit,
json=json or default_json
)
db.session.add(schema)
db.session.commit()
return schema
def test_deposit_path_and_index(db):
schema = Schema(
**{
'name': 'cms-analysis',
'deposit_schema': {},
'major': 1,
'minor': 0,
'patch': 2
})
db.session.add(schema)
db.session.commit()
assert schema.deposit_path == 'deposits/records/cms-analysis-v1.0.2.json'
assert schema.deposit_index == 'deposits-records-cms-analysis-v1.0.2'
def _get_schema_needs(self, deposit):
"""Create deposit permissions are based on schema's permissions."""
if '$schema' in deposit:
try:
schema = Schema.get_by_fullpath(deposit['$schema'])
except SchemaDoesNotExist:
raise WrongJSONSchemaError('Schema {} doesnt exist.'.
format(deposit['$schema']))
elif '$ana_type' in deposit:
try:
schema = Schema.get_latest(
'deposits/records/{}'.format(deposit['$ana_type'])
)
except SchemaDoesNotExist:
raise WrongJSONSchemaError('Schema with name {} doesnt exist.'.
format(deposit['$ana_type']))
else:
raise WrongJSONSchemaError(
'You have to specify either $schema or $ana_type')
return ReadSchemaPermission(schema).needs
def _get_schema_needs(self, deposit):
"""Create deposit permissions are based on schema's permissions."""
if '$schema' in deposit:
try:
schema = Schema.get_by_fullpath(deposit['$schema'])
except JSONSchemaNotFound:
raise WrongJSONSchemaError('Schema {} doesnt exist.'.
format(deposit['$schema']))
elif '$ana_type' in deposit:
try:
schema = Schema.get_latest(
'deposits/records/{}'.format(deposit['$ana_type'])
)
except JSONSchemaNotFound:
raise WrongJSONSchemaError('Schema with name {} doesnt exist.'.
format(deposit['$ana_type']))
else:
raise WrongJSONSchemaError(
'You have to specify either $schema or $ana_type')
return ReadSchemaPermission(schema).needs
def test_add_schema_from_fixture_when_schema_does_not_exist_create_new_one(
app, users): # noqa
data = dict(name='new-schema',
version='1.2.3',
experiment='CMS',
fullname='New fullname',
deposit_schema={'title': 'deposit_schema'},
deposit_options={'title': 'deposit_options'},
record_schema={'title': 'record_schema'},
record_options={'title': 'record_options'},
record_mapping={
'mappings': {
'doc': {
'properties': {
"title": {
"type": "text"
}
}
}
}
},
deposit_mapping={
'mappings': {
'doc': {
'properties': {
"keyword": {
"type": "keyword"
}
}
}
}
},
is_indexed=True,
use_deposit_as_record=False)
add_schema_from_json(data=data)
schema = Schema.get('new-schema', version='1.2.3')
for key, value in data.items():
assert getattr(schema, key) == value
with app.test_request_context():
login_user(users['cms_user'])
assert ReadSchemaPermission(schema).can()
assert not AdminSchemaPermission(schema).can()
login_user(users['lhcb_user'])
assert not ReadSchemaPermission(schema).can()
assert not AdminSchemaPermission(schema).can()
def add_schema_from_json(data, replace=None, force_version=None):
"""Add or update schema, using a json file."""
allow_all = data.pop("allow_all", False)
version = data['version']
name = data['name']
try:
with db.session.begin_nested():
try:
_schema = Schema.get(name=name, version=version)
if replace:
_schema.update(**data)
db.session.add(_schema)
else:
click.secho(f'{name}-{version} already exists in the db.')
return
except JSONSchemaNotFound:
try:
_schema = Schema.get_latest(name=name)
if is_later_version(_schema.version,
version) and not force_version:
click.secho(
f'A later version ({_schema.version}) of '
f'{name} is already in the db. Error while '
f'adding {name}-{version}',
fg='red')
return
else:
_schema = Schema(**data)
db.session.add(_schema)
except JSONSchemaNotFound:
_schema = Schema(**data)
db.session.add(_schema)
# throws error if you try to re-insert the same action roles
if allow_all and not replace:
_schema.add_read_access_for_all_users()
db.session.commit()
click.secho(f'{name} has been added.', fg='green')
except IntegrityError:
click.secho(f'An db error occurred while adding {name}.', fg='red')
def test_put_when_validation_errors_returns_400(client, db, users,
auth_headers_for_user,
json_headers):
owner = users['superuser']
db.session.add(
Schema(**{
'name': 'cms-schema',
'experiment': 'CMS',
'fullname': 'Old Schema',
}))
db.session.commit()
resp = client.put('/jsonschemas/cms-schema/1.0.0',
data='{}',
headers=json_headers + auth_headers_for_user(owner))
assert resp.status_code == 400
assert resp.json['message'] == {'_schema': ['Empty data']}
def test_add_or_update_schema_when_schema_does_not_exist_create_new_one(db): # noqa
data = {
'experiment': 'CMS',
'fullname': 'Test Schema',
'jsonschema': {'field': 'string'}
}
add_or_update_schema(fullpath='records/ana1-v1.0.2.json',
data=data)
schema = Schema.get_by_fullpath('records/ana1-v1.0.2.json')
assert schema.name == 'records/ana1'
assert schema.fullname == data['fullname']
assert schema.experiment == data['experiment']
assert schema.json == data['jsonschema']
assert schema.major == 1
assert schema.minor == 0
assert schema.patch == 2
def _preprocess_data(cls, data):
# data can be sent without specifying particular version of schema,
# but just with a type, e.g. cms-analysis
# this be resolved to the last version of deposit schema of this type
if '$ana_type' in data:
try:
schema = Schema.get_latest('deposits/records/{}'.format(
data['$ana_type']))
except JSONSchemaNotFound:
raise DepositValidationError(
'Schema {} is not a valid deposit schema.'.format(
data['$ana_type']))
data['$schema'] = schema.fullpath
data.pop('$ana_type')
return data
def _preprocess_data(cls, data):
# data can be sent without specifying particular version of schema,
# but just with a type, e.g. cms-analysis
# this be resolved to the last version of deposit schema of this type
if '$ana_type' in data:
try:
schema = Schema.get_latest(
'deposits/records/{}'.format(data['$ana_type'])
)
except JSONSchemaNotFound:
raise DepositValidationError(
'Schema {} is not a valid deposit schema.'
.format(data['$ana_type']))
data['$schema'] = schema.fullpath
data.pop('$ana_type')
return data
def _add_schema(schema, is_deposit=True, experiment=None, json=None):
"""
Add new schema into db
"""
default_json = {
'title': {
'type': 'string'
}
}
try:
schema = Schema.get_by_fullpath(schema)
except JSONSchemaNotFound:
schema = Schema(
fullpath=schema,
experiment=experiment,
is_deposit=is_deposit,
json=json or default_json
)
db.session.add(schema)
db.session.commit()
return schema
def test_get_by_fullpath(db):
schema = Schema(**{'name': 'records/ana1',
'major': 1,
'minor': 0,
'patch': 1})
schema2 = Schema(**{'name': 'deposits/records/ana2',
'major': 2,
'minor': 1,
'patch': 1})
db.session.add(schema)
db.session.add(schema2)
db.session.commit()
assert Schema.get_by_fullpath('https://some-host.com/schemas/records/ana1-v1.0.1.json') == schema
assert Schema.get_by_fullpath('records/ana1-v1.0.1.json') == schema
assert Schema.get_by_fullpath('https://some-host.com/schemas/deposits/records/ana2-v2.1.1') == schema2
assert Schema.get_by_fullpath('deposits/records/ana2-v2.1.1.json') == schema2
assert Schema.get_by_fullpath('/deposits/records/ana2-v2.1.1.json') == schema2
assert Schema.get_by_fullpath('/deposits/records/ana2-v2.1.1') == schema2
def test_get_latest_version_of_schema_when_schema_with_given_name_doesnt_exist_raises_SchemaDoesNotExist(db):
with raises(SchemaDoesNotExist):
Schema.get_latest(name='non-existing')
def test_get_by_fullpath_when_non_existing_raise_JSONSchemaNotFound(db):
with raises(JSONSchemaNotFound):
Schema.get_by_fullpath('/non-existing/schema/ana2-v2.1.1')
def test_get_by_fullpath_when_non_existing_raise_SchemaDoesNotExist(db):
with raises(SchemaDoesNotExist):
Schema.get_by_fullpath('/non-existing/schema/ana2-v2.1.1')
def resolve_cap_schemas(path):
"""Resolve CAP JSON schemas."""
return Schema.get_by_fullpath(path).json
def test_get_latest_version_of_schema_when_schema_with_given_name_doesnt_exist_raises_JSONSchemaNotFound(db):
with raises(JSONSchemaNotFound):
Schema.get_latest(name='non-existing')
def schema(self):
"""Schema property."""
return Schema.get_by_fullpath(self['$schema'])
def _set_experiment(self):
schema = Schema.get_by_fullpath(self['$schema'])
self['_experiment'] = schema.experiment
