def test_get_only_latest_version_of_schemas(client, db, users, auth_headers_for_user): cms_user = users['cms_user'] latest_schema = Schema(name='schema1', version='1.2.3', experiment='CMS') latest_schema2 = Schema(name='schema2', version='3.0.0', experiment='CMS') db.session.add(latest_schema) db.session.add(latest_schema2) db.session.add(Schema(name='schema1', experiment='CMS', version='1.1.0')) db.session.add(Schema(name='schema2', experiment='CMS')) db.session.add(Schema(name='schema3', experiment='LHCb')) db.session.commit() resp = client.get('/jsonschemas?latest=True', headers=auth_headers_for_user(cms_user)) assert resp.status_code == 200 assert resp.json == [ { 'name': 'schema1', 'version': '1.2.3', 'deposit_options': {}, 'record_schema': {}, 'fullname': None, 'use_deposit_as_record': False, 'is_indexed': False, 'record_options': {}, 'deposit_mapping': {}, 'deposit_schema': {}, 'record_mapping': {}, 'links': { 'record': 'http://analysispreservation.cern.ch/api/schemas/records/schema1-v1.2.3.json', 'self': 'http://analysispreservation.cern.ch/api/jsonschemas/schema1/1.2.3', 'deposit': 'http://analysispreservation.cern.ch/api/schemas/deposits/records/schema1-v1.2.3.json', }, }, { 'links': { 'record': 'http://analysispreservation.cern.ch/api/schemas/records/schema2-v3.0.0.json', 'self': 'http://analysispreservation.cern.ch/api/jsonschemas/schema2/3.0.0', 'deposit': 'http://analysispreservation.cern.ch/api/schemas/deposits/records/schema2-v3.0.0.json', }, 'deposit_options': {}, 'record_schema': {}, 'fullname': None, 'use_deposit_as_record': False, 'version': '3.0.0', 'is_indexed': False, 'record_options': {}, 'deposit_mapping': {}, 'deposit_schema': {}, 'record_mapping': {}, 'name': 'schema2', }, ]
def test_get(db): schema = Schema(name='test-schema') db.session.add(schema) db.session.add(Schema(name='test-schema', version='2.0.0')) db.session.add(Schema(name='another-schema', version='1.0.1')) db.session.commit() assert Schema.get('test-schema', '1.0.0') == schema
def test_schema_name_to_url(db): db.session.add(Schema(name='my-schema', version='1.5.5')) db.session.add(Schema(name='my-schema', version='2.4.0')) db.session.add(Schema(name='my-schema', version='2.4.3')) db.session.add(Schema(name='different-schema', version='3.5.4')) db.session.commit() assert schema_name_to_url( 'my-schema' ) == 'https://analysispreservation.cern.ch/schemas/deposits/records/my-schema-v2.4.3.json'
def test_when_schema_with_same_name_and_version_raises_IntegrityError(db): with raises(IntegrityError): schema = { 'name': 'records/ana1', 'major': 1, 'minor': 0, 'patch': 1 } db.session.add(Schema(**schema)) db.session.add(Schema(**schema)) db.session.commit()
def test_when_schema_with_same_name_and_version_raises_IntegrityError(db): with raises(IntegrityError): schema = { 'name': 'cms-analysis', 'deposit_schema': {}, 'major': 1, 'minor': 0, 'patch': 1 } db.session.add(Schema(**schema)) db.session.add(Schema(**schema)) db.session.commit()
def test_create_newer_version_of_existing_schema(db): schema = { 'name': 'records/ana1', 'major': 1, 'minor': 0, 'patch': 1, } db.session.add(Schema(**schema)) schema.update({'patch': 2}) db.session.add(Schema(**schema)) db.session.commit() assert Schema.query.count() == 2
def test_create_newer_version_of_existing_schema(db): schema = { 'name': 'cms-analysis', 'deposit_schema': {}, 'major': 1, 'minor': 0, 'patch': 1 } db.session.add(Schema(**schema)) schema.update({'patch': 2}) db.session.add(Schema(**schema)) db.session.commit() assert Schema.query.count() == 2
def test_add_or_update_schema_when_schema_already_exist_updates_json_for_schema( db): # noqa db.session.add( Schema( **{ 'name': 'records/ana1', 'major': 1, 'minor': 0, 'patch': 2, 'experiment': 'CMS', 'fullname': 'Old Schema', 'json': { 'field': 'initial' } })) db.session.commit() updated_data = { 'experiment': 'Updated', 'fullname': 'Updated Schema', 'jsonschema': { 'updated': 'string' } } add_or_update_schema(fullpath='records/ana1-v1.0.2.json', data=updated_data) schema = Schema.get_by_fullpath('records/ana1-v1.0.2.json') assert schema.name == 'records/ana1' assert schema.fullname == updated_data['fullname'] assert schema.experiment == updated_data['experiment'] assert schema.json == updated_data['jsonschema'] assert schema.major == 1 assert schema.minor == 0 assert schema.patch == 2
def add_schema_from_fixture(data=None): """Add or update schema.""" allow_all = data.pop("allow_all", False) name = data['name'] try: with db.session.begin_nested(): with db.session.begin_nested(): try: schema = Schema.get(name=data['name'], version=data['version']) click.secho('{} already exist in the db.'.format( str(name))) return except JSONSchemaNotFound: schema = Schema(**data) db.session.add(schema) if allow_all: schema.add_read_access_for_all_users() except IntegrityError: click.secho('Error occured during adding {} to the db. \n'.format( str(name)), fg='red') return db.session.commit() click.secho('{} has been added.'.format(str(name)), fg='green')
def _add_schema(name, deposit_schema=None, is_indexed=True, use_deposit_as_record=True, version="1.0.0", **kwargs): """ Add new schema into db """ default_json = {'title': {'type': 'string'}} try: schema = Schema.get(name, version) except JSONSchemaNotFound: schema = Schema(name=name, version=version, is_indexed=is_indexed, use_deposit_as_record=use_deposit_as_record, deposit_schema=deposit_schema or default_json, **kwargs) db.session.add(schema) db.session.commit() if not schema.experiment: schema.add_read_access_for_all_users() db.session.commit() return schema
def test_get_when_user_outside_of_experiment_returns_403( client, db, users, auth_headers_for_user): other_user = users['lhcb_user'] schema = Schema( name='cms-schema', version='1.2.3', fullname='CMS Schema 1.2.3', experiment='CMS', deposit_schema={'title': 'deposit_schema'}, deposit_options={'title': 'deposit_options'}, record_schema={'title': 'record_schema'}, record_options={'title': 'record_options'}, record_mapping={ 'mappings': { 'doc': { 'properties': { 'title': { 'type': 'text' } } } } }, deposit_mapping={ 'mappings': { 'doc': { 'properties': { 'keyword': { 'type': 'keyword' } } } } }, is_indexed=True, ) db.session.add(schema) db.session.add(Schema(name='cms-schema', version='1.1.0')) db.session.add(Schema(name='lhcb-schema')) db.session.commit() resp = client.get('/jsonschemas/cms-schema/1.2.3', headers=auth_headers_for_user(other_user)) assert resp.status_code == 403
def test_create_schema_by_fullpath(db): schema = Schema('records/ana1-v1.3.2') db.session.add(schema) db.session.commit() assert schema.name == 'records/ana1' assert schema.major == 1 assert schema.minor == 3 assert schema.patch == 2
def test_when_schema_name_contains_forbidden_characters_raises_AssertionError( db): with raises(ValueError): schema = { 'name': 'cmsanalysis,\\', 'deposit_schema': {}, } db.session.add(Schema(**schema)) db.session.commit()
def add_schema_from_json(data, replace=None, force_version=None): """Add or update schema, using a json file.""" allow_all = data.pop("allow_all", False) version = data['version'] name = data['name'] try: with db.session.begin_nested(): try: _schema = Schema.get(name=name, version=version) if replace: _schema.update(**data) db.session.add(_schema) else: click.secho(f'{name}-{version} already exists in the db.') return except JSONSchemaNotFound: try: _schema = Schema.get_latest(name=name) if is_later_version(_schema.version, version) and not force_version: click.secho( f'A later version ({_schema.version}) of ' f'{name} is already in the db. Error while ' f'adding {name}-{version}', fg='red') return else: _schema = Schema(**data) db.session.add(_schema) except JSONSchemaNotFound: _schema = Schema(**data) db.session.add(_schema) # throws error if you try to re-insert the same action roles if allow_all and not replace: _schema.add_read_access_for_all_users() db.session.commit() click.secho(f'{name} has been added.', fg='green') except IntegrityError: click.secho(f'An db error occurred while adding {name}.', fg='red')
def test_on_save_mapping_is_created_and_index_name_added_to_mappings_map(schema_params, index_name, db, es): schema = Schema(**schema_params) db.session.add(schema) db.session.commit() assert index_name in current_search.mappings.keys() assert es.indices.exists(index_name) db.session.delete(schema) db.session.commit() assert not es.indices.exists(index_name) assert not schema.name in current_search.mappings.keys()
def test_schema_stringify(db): schema = Schema( **{ 'name': 'cms-analysis', 'deposit_schema': {}, 'major': 1, 'minor': 0, 'patch': 2 }) db.session.add(schema) db.session.commit() assert str(schema) == 'cms-analysis-v1.0.2'
def test_get_by_fullpath(db): schema = Schema(**{'name': 'records/ana1', 'major': 1, 'minor': 0, 'patch': 1}) schema2 = Schema(**{'name': 'deposits/records/ana2', 'major': 2, 'minor': 1, 'patch': 1}) db.session.add(schema) db.session.add(schema2) db.session.commit() assert Schema.get_by_fullpath( 'https://some-host.com/schemas/records/ana1-v1.0.1.json') == schema assert Schema.get_by_fullpath('records/ana1-v1.0.1.json') == schema assert Schema.get_by_fullpath( 'https://some-host.com/schemas/deposits/records/ana2-v2.1.1') == schema2 assert Schema.get_by_fullpath( 'deposits/records/ana2-v2.1.1.json') == schema2 assert Schema.get_by_fullpath( '/deposits/records/ana2-v2.1.1.json') == schema2 assert Schema.get_by_fullpath('/deposits/records/ana2-v2.1.1') == schema2
def test_deposit_path_and_index(db): schema = Schema( **{ 'name': 'cms-analysis', 'deposit_schema': {}, 'major': 1, 'minor': 0, 'patch': 2 }) db.session.add(schema) db.session.commit() assert schema.deposit_path == 'deposits/records/cms-analysis-v1.0.2.json' assert schema.deposit_index == 'deposits-records-cms-analysis-v1.0.2'
def test_get_latest_version_of_schema(db): schemas = [ {'name': 'name', 'major': 0, 'minor': 6, 'patch': 6}, {'name': 'name', 'major': 1, 'minor': 5, 'patch': 5}, {'name': 'name', 'major': 2, 'minor': 3, 'patch': 4}, {'name': 'name', 'major': 2, 'minor': 4, 'patch': 0}, {'name': 'name', 'major': 2, 'minor': 4, 'patch': 3}, ] for x in schemas: db.session.add(Schema(**x)) db.session.commit() latest = Schema.get_latest(name='name') assert latest.version == "2.4.3"
def test_add_schema_from_fixture_when_schema_already_exist_updates_json_for_schema( db): updated_data = dict(name='new-schema', version='1.1.1', experiment='LHCb', fullname='New fullname', deposit_schema={'title': 'deposit_schema'}, deposit_options={'title': 'deposit_options'}, record_schema={'title': 'record_schema'}, record_options={'title': 'record_options'}, record_mapping={ 'mappings': { 'doc': { 'properties': { "title": { "type": "text" } } } } }, deposit_mapping={ 'mappings': { 'doc': { 'properties': { "keyword": { "type": "keyword" } } } } }, is_indexed=True, use_deposit_as_record=False) db.session.add( Schema(**{ 'name': 'new-schema', 'experiment': 'CMS', 'fullname': 'Old Schema', })) db.session.commit() add_schema_from_json(data=updated_data) schema = Schema.get('new-schema', version='1.1.1') for key, value in updated_data.items(): assert getattr(schema, key) == value
def test_get_latest_version_of_schema(db): latest_schema = Schema(name='my-schema', version='2.4.3') db.session.add(Schema(name='my-schema', version='0.6.6')) db.session.add(Schema(name='my-schema', version='1.5.5')) db.session.add(Schema(name='my-schema', version='2.3.4')) db.session.add(Schema(name='my-schema', version='2.4.0')) db.session.add(Schema(name='different-schema', version='3.5.4')) db.session.add(latest_schema) db.session.commit() assert Schema.get_latest('my-schema') == latest_schema
def test_put_when_validation_errors_returns_400(client, db, users, auth_headers_for_user, json_headers): owner = users['superuser'] db.session.add( Schema(**{ 'name': 'cms-schema', 'experiment': 'CMS', 'fullname': 'Old Schema', })) db.session.commit() resp = client.put('/jsonschemas/cms-schema/1.0.0', data='{}', headers=json_headers + auth_headers_for_user(owner)) assert resp.status_code == 400 assert resp.json['message'] == {'_schema': ['Empty data']}
def test_add_schemas_with_url_with_replace(app, db, cli_runner): schema = Schema( name='test', experiment='CMS', version='1.0.0', deposit_schema={ 'type': 'object', 'properties': { 'title': {'type': 'string'} } }) db.session.add(schema) db.session.commit() responses.add( responses.GET, 'http://schemas.org/my-schema.json', json={ 'version': '1.0.0', 'name': 'test', 'use_deposit_as_record': True, 'deposit_schema': { 'properties': { '_buckets': {}, '_deposit': {}, '_files': {}, '_experiment': {}, '_fetched_from': {}, '_user_edited': {}, '_access': {}, 'title': {'type': 'string'}, 'abstract': {'type': 'string'} } } }, status=200 ) res = cli_runner('fixtures schemas --url http://schemas.org/my-schema.json -r') assert res.exit_code == 0 assert 'test has been added.' in res.output schema = Schema.get_latest('test') assert 'abstract' in schema.deposit_schema['properties'].keys()
def _add_schema(schema, is_deposit=True, experiment=None, json=None): """ Add new schema into db """ default_json = { 'title': { 'type': 'string' } } try: schema = Schema.get_by_fullpath(schema) except JSONSchemaNotFound: schema = Schema( fullpath=schema, experiment=experiment, is_deposit=is_deposit, json=json or default_json ) db.session.add(schema) db.session.commit() return schema
def test_get_indexed_schemas_for_user_when_latest(app, db, users): db.session.add( Schema(name='schema1', version='1.3.1', is_indexed=True, experiment='CMS')) db.session.add( Schema(name='schema1', version='2.5.7', is_indexed=True, experiment='CMS')) db.session.add( Schema(name='schema1', version='3.0.0', is_indexed=False, experiment='CMS')) db.session.add( Schema(name='schema3', version='1.3.1', is_indexed=True, experiment='LHCb')) latest_schema1 = Schema(name='schema1', version='3.4.5', is_indexed=True, experiment='CMS') latest_schema2 = Schema(name='schema2', version='3.0.0', is_indexed=True, experiment='CMS') db.session.add(latest_schema1) db.session.add(latest_schema2) db.session.commit() login_user(users['cms_user']) schemas = get_indexed_schemas_for_user(latest=True) assert schemas == [latest_schema1, latest_schema2]
def test_on_save_mapping_is_created_and_index_name_added_to_mappings_map( db, es): schema = Schema( name='cms-schema', is_indexed=True, record_mapping={'doc': { 'properties': { "title": { "type": "text" } } }}, deposit_mapping={ 'doc': { 'properties': { "keyword": { "type": "keyword" } } } }) db.session.add(schema) db.session.commit() assert 'deposits-records-cms-schema-v1.0.0' in current_search.mappings.keys( ) assert 'records-cms-schema-v1.0.0' in current_search.mappings.keys() assert es.indices.exists('deposits-records-cms-schema-v1.0.0') assert es.indices.exists('records-cms-schema-v1.0.0') assert es.indices.get_mapping('records-cms-schema-v1.0.0') == { 'records-cms-schema-v1.0.0': { 'mappings': { 'doc': { 'properties': { 'title': { 'type': 'text' } } } } } } assert es.indices.get_mapping('deposits-records-cms-schema-v1.0.0') == { 'deposits-records-cms-schema-v1.0.0': { 'mappings': { 'doc': { 'properties': { 'keyword': { 'type': 'keyword' } } } } } } db.session.delete(schema) db.session.commit() assert not es.indices.exists('deposits-records-cms-schema-v1.0.0') assert not es.indices.exists('records-cms-schema-v1.0.0') assert 'deposits-records-cms-schema-v1.0.0' not in current_search.mappings.keys( ) assert 'records-cms-schema-v1.0.0' not in current_search.mappings.keys()
def schema(db): """Example schema.""" schema = Schema(name='test-schema', fullname='Test Schema') db.session.add(schema) db.session.commit() return schema
def test_get(client, db, users, auth_headers_for_user): cms_user = users['cms_user'] schema = Schema( name='cms-schema', version='1.2.3', fullname='CMS Schema 1.2.3', experiment='CMS', deposit_schema={'title': 'deposit_schema'}, deposit_options={'title': 'deposit_options'}, record_schema={'title': 'record_schema'}, record_options={'title': 'record_options'}, record_mapping={'doc': { 'properties': { "title": { "type": "text" } } }}, deposit_mapping={ 'doc': { 'properties': { "keyword": { "type": "keyword" } } } }, is_indexed=True, ) db.session.add(schema) db.session.add(Schema(name='cms-schema', version='1.1.0')) db.session.add(Schema(name='lhcb-schema')) db.session.commit() resp = client.get('/jsonschemas/', headers=auth_headers_for_user(cms_user)) assert resp.status_code == 200 assert resp.json == [{ 'name': 'cms-schema', 'version': '1.2.3', 'fullname': 'CMS Schema 1.2.3', 'is_indexed': True, 'use_deposit_as_record': False, 'deposit_schema': { 'title': 'deposit_schema' }, 'deposit_options': { 'title': 'deposit_options' }, 'record_schema': { 'title': 'record_schema' }, 'record_options': { 'title': 'record_options' }, 'record_mapping': { 'doc': { 'properties': { "title": { "type": "text" } } } }, 'deposit_mapping': { 'doc': { 'properties': { "keyword": { "type": "keyword" } } } }, 'links': { 'self': u'http://analysispreservation.cern.ch/api/jsonschemas/cms-schema/1.2.3', 'deposit': 'http://analysispreservation.cern.ch/api/schemas/deposits/records/cms-schema-v1.2.3.json', 'record': u'http://analysispreservation.cern.ch/api/schemas/records/cms-schema-v1.2.3.json', # 'versions': u'http://analysispreservation.cern.ch/api/jsonschemas/cms-schema/versions' } }] resp = client.get('/jsonschemas/cms-schema/1.2.3', headers=auth_headers_for_user(cms_user)) assert resp.status_code == 200 assert resp.json == { 'name': 'cms-schema', 'version': '1.2.3', 'fullname': 'CMS Schema 1.2.3', 'is_indexed': True, 'use_deposit_as_record': False, 'deposit_schema': { 'title': 'deposit_schema' }, 'deposit_options': { 'title': 'deposit_options' }, 'record_schema': { 'title': 'record_schema' }, 'record_options': { 'title': 'record_options' }, 'record_mapping': { 'doc': { 'properties': { "title": { "type": "text" } } } }, 'deposit_mapping': { 'doc': { 'properties': { "keyword": { "type": "keyword" } } } }, 'links': { 'self': u'http://analysispreservation.cern.ch/api/jsonschemas/cms-schema/1.2.3', 'deposit': 'http://analysispreservation.cern.ch/api/schemas/deposits/records/cms-schema-v1.2.3.json', 'record': u'http://analysispreservation.cern.ch/api/schemas/records/cms-schema-v1.2.3.json', # 'versions': u'http://analysispreservation.cern.ch/api/jsonschemas/cms-schema/versions' } } resp = client.get('/jsonschemas/cms-schema', headers=auth_headers_for_user(cms_user)) assert resp.status_code == 200 assert resp.json == { 'name': 'cms-schema', 'version': '1.2.3', 'fullname': 'CMS Schema 1.2.3', 'is_indexed': True, 'use_deposit_as_record': False, 'deposit_schema': { 'title': 'deposit_schema' }, 'deposit_options': { 'title': 'deposit_options' }, 'record_schema': { 'title': 'record_schema' }, 'record_options': { 'title': 'record_options' }, 'record_mapping': { 'doc': { 'properties': { "title": { "type": "text" } } } }, 'deposit_mapping': { 'doc': { 'properties': { "keyword": { "type": "keyword" } } } }, 'links': { 'self': u'http://analysispreservation.cern.ch/api/jsonschemas/cms-schema/1.2.3', 'deposit': 'http://analysispreservation.cern.ch/api/schemas/deposits/records/cms-schema-v1.2.3.json', 'record': u'http://analysispreservation.cern.ch/api/schemas/records/cms-schema-v1.2.3.json', # 'versions': u'http://analysispreservation.cern.ch/api/jsonschemas/cms-schema/versions' } }