def add_schema_from_fixture(data=None): """Add or update schema.""" allow_all = data.pop("allow_all", False) name = data['name'] try: with db.session.begin_nested(): with db.session.begin_nested(): try: schema = Schema.get(name=data['name'], version=data['version']) click.secho('{} already exist in the db.'.format( str(name))) return except JSONSchemaNotFound: schema = Schema(**data) db.session.add(schema) if allow_all: schema.add_read_access_for_all_users() except IntegrityError: click.secho('Error occured during adding {} to the db. \n'.format( str(name)), fg='red') return db.session.commit() click.secho('{} has been added.'.format(str(name)), fg='green')
def _add_schema(schema, is_deposit=True, experiment=None, json=None, roles=None): """ Add new schema into db """ default_json = { 'title': { 'type': 'string' } } try: schema = Schema.get_by_fullpath(schema) except SchemaDoesNotExist: schema = Schema( fullpath=schema, experiment=experiment, is_deposit=is_deposit, json=json or default_json ) db.session.add(schema) db.session.commit() if roles: roles = Role.query.filter(Role.name.in_(roles)).all() for role in roles: schema.add_read_access(role) return schema
def test_add_or_update_schema_when_schema_already_exist_updates_json_for_schema( db): # noqa db.session.add( Schema( **{ 'name': 'records/ana1', 'major': 1, 'minor': 0, 'patch': 2, 'experiment': 'CMS', 'fullname': 'Old Schema', 'json': { 'field': 'initial' } })) db.session.commit() updated_data = { 'experiment': 'Updated', 'fullname': 'Updated Schema', 'jsonschema': { 'updated': 'string' } } add_or_update_schema(fullpath='records/ana1-v1.0.2.json', data=updated_data) schema = Schema.get_by_fullpath('records/ana1-v1.0.2.json') assert schema.name == 'records/ana1' assert schema.fullname == updated_data['fullname'] assert schema.experiment == updated_data['experiment'] assert schema.json == updated_data['jsonschema'] assert schema.major == 1 assert schema.minor == 0 assert schema.patch == 2
def _add_schema(name, deposit_schema=None, is_indexed=True, use_deposit_as_record=True, version="1.0.0", **kwargs): """ Add new schema into db """ default_json = {'title': {'type': 'string'}} try: schema = Schema.get(name, version) except JSONSchemaNotFound: schema = Schema(name=name, version=version, is_indexed=is_indexed, use_deposit_as_record=use_deposit_as_record, deposit_schema=deposit_schema or default_json, **kwargs) db.session.add(schema) db.session.commit() if not schema.experiment: schema.add_read_access_for_all_users() db.session.commit() return schema
def test_get_only_latest_version_of_schemas(client, db, users, auth_headers_for_user): cms_user = users['cms_user'] latest_schema = Schema(name='schema1', version='1.2.3', experiment='CMS') latest_schema2 = Schema(name='schema2', version='3.0.0', experiment='CMS') db.session.add(latest_schema) db.session.add(latest_schema2) db.session.add(Schema(name='schema1', experiment='CMS', version='1.1.0')) db.session.add(Schema(name='schema2', experiment='CMS')) db.session.add(Schema(name='schema3', experiment='LHCb')) db.session.commit() resp = client.get('/jsonschemas?latest=True', headers=auth_headers_for_user(cms_user)) assert resp.status_code == 200 assert resp.json == [ { 'name': 'schema1', 'version': '1.2.3', 'deposit_options': {}, 'record_schema': {}, 'fullname': None, 'use_deposit_as_record': False, 'is_indexed': False, 'record_options': {}, 'deposit_mapping': {}, 'deposit_schema': {}, 'record_mapping': {}, 'links': { 'record': 'http://analysispreservation.cern.ch/api/schemas/records/schema1-v1.2.3.json', 'self': 'http://analysispreservation.cern.ch/api/jsonschemas/schema1/1.2.3', 'deposit': 'http://analysispreservation.cern.ch/api/schemas/deposits/records/schema1-v1.2.3.json', }, }, { 'links': { 'record': 'http://analysispreservation.cern.ch/api/schemas/records/schema2-v3.0.0.json', 'self': 'http://analysispreservation.cern.ch/api/jsonschemas/schema2/3.0.0', 'deposit': 'http://analysispreservation.cern.ch/api/schemas/deposits/records/schema2-v3.0.0.json', }, 'deposit_options': {}, 'record_schema': {}, 'fullname': None, 'use_deposit_as_record': False, 'version': '3.0.0', 'is_indexed': False, 'record_options': {}, 'deposit_mapping': {}, 'deposit_schema': {}, 'record_mapping': {}, 'name': 'schema2', }, ]
def test_get(db): schema = Schema(name='test-schema') db.session.add(schema) db.session.add(Schema(name='test-schema', version='2.0.0')) db.session.add(Schema(name='another-schema', version='1.0.1')) db.session.commit() assert Schema.get('test-schema', '1.0.0') == schema
def test_schema_name_to_url(db): db.session.add(Schema(name='my-schema', version='1.5.5')) db.session.add(Schema(name='my-schema', version='2.4.0')) db.session.add(Schema(name='my-schema', version='2.4.3')) db.session.add(Schema(name='different-schema', version='3.5.4')) db.session.commit() assert schema_name_to_url( 'my-schema' ) == 'https://analysispreservation.cern.ch/schemas/deposits/records/my-schema-v2.4.3.json'
def test_when_schema_with_same_name_and_version_raises_IntegrityError(db): with raises(IntegrityError): schema = { 'name': 'records/ana1', 'major': 1, 'minor': 0, 'patch': 1 } db.session.add(Schema(**schema)) db.session.add(Schema(**schema)) db.session.commit()
def test_when_schema_with_same_name_and_version_raises_IntegrityError(db): with raises(IntegrityError): schema = { 'name': 'cms-analysis', 'deposit_schema': {}, 'major': 1, 'minor': 0, 'patch': 1 } db.session.add(Schema(**schema)) db.session.add(Schema(**schema)) db.session.commit()
def test_parse_fullpath(): wrong_string = "deposits/records/alice-analysis-111v0.0.1.json" correct_string = "deposits/records/alice-analysis-v0.0.1.json" with raises(JSONSchemaNotFound): Schema._parse_fullpath(wrong_string) resp = Schema._parse_fullpath(correct_string) assert resp[0] == "deposits/records/alice-analysis" assert resp[1] == "0" assert resp[2] == "0" assert resp[3] == "1"
def test_create_newer_version_of_existing_schema(db): schema = { 'name': 'records/ana1', 'major': 1, 'minor': 0, 'patch': 1, } db.session.add(Schema(**schema)) schema.update({'patch': 2}) db.session.add(Schema(**schema)) db.session.commit() assert Schema.query.count() == 2
def _validate_data(cls, data): if not isinstance(data, dict) or data == {}: raise DepositValidationError('Empty deposit data.') try: schema_fullpath = data['$schema'] except KeyError: raise DepositValidationError('Schema not specified.') try: Schema.get_by_fullpath(schema_fullpath) except (AttributeError, JSONSchemaNotFound): raise DepositValidationError('Schema {} is not a valid option.' .format(schema_fullpath))
def test_create_newer_version_of_existing_schema(db): schema = { 'name': 'cms-analysis', 'deposit_schema': {}, 'major': 1, 'minor': 0, 'patch': 1 } db.session.add(Schema(**schema)) schema.update({'patch': 2}) db.session.add(Schema(**schema)) db.session.commit() assert Schema.query.count() == 2
def _validate_data(cls, data): if not isinstance(data, dict) or data == {}: raise DepositValidationError('Empty deposit data.') try: schema_fullpath = data['$schema'] except KeyError: raise DepositValidationError('Schema not specified.') try: Schema.get_by_fullpath(schema_fullpath) except (AttributeError, JSONSchemaNotFound): raise DepositValidationError( 'Schema {} is not a valid option.'.format(schema_fullpath))
def test_get_latest_version_of_schema(db): schemas = [ {'name': 'name', 'major': 0, 'minor': 6, 'patch': 6}, {'name': 'name', 'major': 1, 'minor': 5, 'patch': 5}, {'name': 'name', 'major': 2, 'minor': 3, 'patch': 4}, {'name': 'name', 'major': 2, 'minor': 4, 'patch': 0}, {'name': 'name', 'major': 2, 'minor': 4, 'patch': 3}, ] for x in schemas: db.session.add(Schema(**x)) db.session.commit() latest = Schema.get_latest(name='name') assert latest.version == "2.4.3"
def test_add_or_update_schema_when_schema_already_exist_updates_json_for_schema(db): # noqa db.session.add(Schema(**{ 'name': 'records/ana1', 'major': 1, 'minor': 0, 'patch': 2, 'experiment': 'CMS', 'fullname': 'Old Schema', 'json': {'field': 'initial'} })) db.session.commit() updated_data = { 'experiment': 'Updated', 'fullname': 'Updated Schema', 'jsonschema': { 'updated': 'string' } } add_or_update_schema(fullpath='records/ana1-v1.0.2.json', data=updated_data) schema = Schema.get_by_fullpath('records/ana1-v1.0.2.json') assert schema.name == 'records/ana1' assert schema.fullname == updated_data['fullname'] assert schema.experiment == updated_data['experiment'] assert schema.json == updated_data['jsonschema'] assert schema.major == 1 assert schema.minor == 0 assert schema.patch == 2
def test_add_schema_from_fixture_when_schema_already_exist_updates_json_for_schema( db): updated_data = dict(name='new-schema', version='1.1.1', experiment='LHCb', fullname='New fullname', deposit_schema={'title': 'deposit_schema'}, deposit_options={'title': 'deposit_options'}, record_schema={'title': 'record_schema'}, record_options={'title': 'record_options'}, record_mapping={ 'mappings': { 'doc': { 'properties': { "title": { "type": "text" } } } } }, deposit_mapping={ 'mappings': { 'doc': { 'properties': { "keyword": { "type": "keyword" } } } } }, is_indexed=True, use_deposit_as_record=False) db.session.add( Schema(**{ 'name': 'new-schema', 'experiment': 'CMS', 'fullname': 'Old Schema', })) db.session.commit() add_schema_from_json(data=updated_data) schema = Schema.get('new-schema', version='1.1.1') for key, value in updated_data.items(): assert getattr(schema, key) == value
def test_get_when_user_outside_of_experiment_returns_403( client, db, users, auth_headers_for_user): other_user = users['lhcb_user'] schema = Schema( name='cms-schema', version='1.2.3', fullname='CMS Schema 1.2.3', experiment='CMS', deposit_schema={'title': 'deposit_schema'}, deposit_options={'title': 'deposit_options'}, record_schema={'title': 'record_schema'}, record_options={'title': 'record_options'}, record_mapping={ 'mappings': { 'doc': { 'properties': { 'title': { 'type': 'text' } } } } }, deposit_mapping={ 'mappings': { 'doc': { 'properties': { 'keyword': { 'type': 'keyword' } } } } }, is_indexed=True, ) db.session.add(schema) db.session.add(Schema(name='cms-schema', version='1.1.0')) db.session.add(Schema(name='lhcb-schema')) db.session.commit() resp = client.get('/jsonschemas/cms-schema/1.2.3', headers=auth_headers_for_user(other_user)) assert resp.status_code == 403
def resolve_cap_schemas(path): """Resolve CAP JSON schemas.""" schema = Schema.get_by_fullpath(path) if not has_request_context() or ReadSchemaPermission(schema).can(): return schema.json else: abort(403)
def test_when_schema_name_contains_forbidden_characters_raises_AssertionError( db): with raises(ValueError): schema = { 'name': 'cmsanalysis,\\', 'deposit_schema': {}, } db.session.add(Schema(**schema)) db.session.commit()
def test_create_schema_by_fullpath(db): schema = Schema('records/ana1-v1.3.2') db.session.add(schema) db.session.commit() assert schema.name == 'records/ana1' assert schema.major == 1 assert schema.minor == 3 assert schema.patch == 2
def test_add_schemas_with_url_with_replace(app, db, cli_runner): schema = Schema( name='test', experiment='CMS', version='1.0.0', deposit_schema={ 'type': 'object', 'properties': { 'title': {'type': 'string'} } }) db.session.add(schema) db.session.commit() responses.add( responses.GET, 'http://schemas.org/my-schema.json', json={ 'version': '1.0.0', 'name': 'test', 'use_deposit_as_record': True, 'deposit_schema': { 'properties': { '_buckets': {}, '_deposit': {}, '_files': {}, '_experiment': {}, '_fetched_from': {}, '_user_edited': {}, '_access': {}, 'title': {'type': 'string'}, 'abstract': {'type': 'string'} } } }, status=200 ) res = cli_runner('fixtures schemas --url http://schemas.org/my-schema.json -r') assert res.exit_code == 0 assert 'test has been added.' in res.output schema = Schema.get_latest('test') assert 'abstract' in schema.deposit_schema['properties'].keys()
def get_user_deposit_groups(): """Get Deposit Groups.""" # Set deposit groups for user schemas = Schema.get_user_deposit_schemas() dep_groups = [{ 'name': schema.fullname, 'deposit_group': schema.name.replace('deposits/records/', '') } for schema in schemas] return dep_groups
def build_cadi_deposit(cadi_id, cadi_info): """Build CMS analysis deposit, based on CADI entry.""" schema = Schema.get_latest('deposits/records/cms-analysis').fullpath return { '$schema': schema, 'cadi_info': cadi_info, 'general_title': cadi_id, 'basic_info': { 'cadi_id': cadi_id } }
def test_on_save_mapping_is_created_and_index_name_added_to_mappings_map(schema_params, index_name, db, es): schema = Schema(**schema_params) db.session.add(schema) db.session.commit() assert index_name in current_search.mappings.keys() assert es.indices.exists(index_name) db.session.delete(schema) db.session.commit() assert not es.indices.exists(index_name) assert not schema.name in current_search.mappings.keys()
def test_schema_stringify(db): schema = Schema( **{ 'name': 'cms-analysis', 'deposit_schema': {}, 'major': 1, 'minor': 0, 'patch': 2 }) db.session.add(schema) db.session.commit() assert str(schema) == 'cms-analysis-v1.0.2'
def test_get_latest_version_of_schema(db): latest_schema = Schema(name='my-schema', version='2.4.3') db.session.add(Schema(name='my-schema', version='0.6.6')) db.session.add(Schema(name='my-schema', version='1.5.5')) db.session.add(Schema(name='my-schema', version='2.3.4')) db.session.add(Schema(name='my-schema', version='2.4.0')) db.session.add(Schema(name='different-schema', version='3.5.4')) db.session.add(latest_schema) db.session.commit() assert Schema.get_latest('my-schema') == latest_schema
def _add_schema(schema, is_deposit=True, experiment=None, json=None): """ Add new schema into db """ default_json = { 'title': { 'type': 'string' } } try: schema = Schema.get_by_fullpath(schema) except JSONSchemaNotFound: schema = Schema( fullpath=schema, experiment=experiment, is_deposit=is_deposit, json=json or default_json ) db.session.add(schema) db.session.commit() return schema
def test_deposit_path_and_index(db): schema = Schema( **{ 'name': 'cms-analysis', 'deposit_schema': {}, 'major': 1, 'minor': 0, 'patch': 2 }) db.session.add(schema) db.session.commit() assert schema.deposit_path == 'deposits/records/cms-analysis-v1.0.2.json' assert schema.deposit_index == 'deposits-records-cms-analysis-v1.0.2'
def _get_schema_needs(self, deposit): """Create deposit permissions are based on schema's permissions.""" if '$schema' in deposit: try: schema = Schema.get_by_fullpath(deposit['$schema']) except SchemaDoesNotExist: raise WrongJSONSchemaError('Schema {} doesnt exist.'. format(deposit['$schema'])) elif '$ana_type' in deposit: try: schema = Schema.get_latest( 'deposits/records/{}'.format(deposit['$ana_type']) ) except SchemaDoesNotExist: raise WrongJSONSchemaError('Schema with name {} doesnt exist.'. format(deposit['$ana_type'])) else: raise WrongJSONSchemaError( 'You have to specify either $schema or $ana_type') return ReadSchemaPermission(schema).needs
def _get_schema_needs(self, deposit): """Create deposit permissions are based on schema's permissions.""" if '$schema' in deposit: try: schema = Schema.get_by_fullpath(deposit['$schema']) except JSONSchemaNotFound: raise WrongJSONSchemaError('Schema {} doesnt exist.'. format(deposit['$schema'])) elif '$ana_type' in deposit: try: schema = Schema.get_latest( 'deposits/records/{}'.format(deposit['$ana_type']) ) except JSONSchemaNotFound: raise WrongJSONSchemaError('Schema with name {} doesnt exist.'. format(deposit['$ana_type'])) else: raise WrongJSONSchemaError( 'You have to specify either $schema or $ana_type') return ReadSchemaPermission(schema).needs
def test_add_schema_from_fixture_when_schema_does_not_exist_create_new_one( app, users): # noqa data = dict(name='new-schema', version='1.2.3', experiment='CMS', fullname='New fullname', deposit_schema={'title': 'deposit_schema'}, deposit_options={'title': 'deposit_options'}, record_schema={'title': 'record_schema'}, record_options={'title': 'record_options'}, record_mapping={ 'mappings': { 'doc': { 'properties': { "title": { "type": "text" } } } } }, deposit_mapping={ 'mappings': { 'doc': { 'properties': { "keyword": { "type": "keyword" } } } } }, is_indexed=True, use_deposit_as_record=False) add_schema_from_json(data=data) schema = Schema.get('new-schema', version='1.2.3') for key, value in data.items(): assert getattr(schema, key) == value with app.test_request_context(): login_user(users['cms_user']) assert ReadSchemaPermission(schema).can() assert not AdminSchemaPermission(schema).can() login_user(users['lhcb_user']) assert not ReadSchemaPermission(schema).can() assert not AdminSchemaPermission(schema).can()
def add_schema_from_json(data, replace=None, force_version=None): """Add or update schema, using a json file.""" allow_all = data.pop("allow_all", False) version = data['version'] name = data['name'] try: with db.session.begin_nested(): try: _schema = Schema.get(name=name, version=version) if replace: _schema.update(**data) db.session.add(_schema) else: click.secho(f'{name}-{version} already exists in the db.') return except JSONSchemaNotFound: try: _schema = Schema.get_latest(name=name) if is_later_version(_schema.version, version) and not force_version: click.secho( f'A later version ({_schema.version}) of ' f'{name} is already in the db. Error while ' f'adding {name}-{version}', fg='red') return else: _schema = Schema(**data) db.session.add(_schema) except JSONSchemaNotFound: _schema = Schema(**data) db.session.add(_schema) # throws error if you try to re-insert the same action roles if allow_all and not replace: _schema.add_read_access_for_all_users() db.session.commit() click.secho(f'{name} has been added.', fg='green') except IntegrityError: click.secho(f'An db error occurred while adding {name}.', fg='red')
def test_put_when_validation_errors_returns_400(client, db, users, auth_headers_for_user, json_headers): owner = users['superuser'] db.session.add( Schema(**{ 'name': 'cms-schema', 'experiment': 'CMS', 'fullname': 'Old Schema', })) db.session.commit() resp = client.put('/jsonschemas/cms-schema/1.0.0', data='{}', headers=json_headers + auth_headers_for_user(owner)) assert resp.status_code == 400 assert resp.json['message'] == {'_schema': ['Empty data']}
def test_add_or_update_schema_when_schema_does_not_exist_create_new_one(db): # noqa data = { 'experiment': 'CMS', 'fullname': 'Test Schema', 'jsonschema': {'field': 'string'} } add_or_update_schema(fullpath='records/ana1-v1.0.2.json', data=data) schema = Schema.get_by_fullpath('records/ana1-v1.0.2.json') assert schema.name == 'records/ana1' assert schema.fullname == data['fullname'] assert schema.experiment == data['experiment'] assert schema.json == data['jsonschema'] assert schema.major == 1 assert schema.minor == 0 assert schema.patch == 2
def _preprocess_data(cls, data): # data can be sent without specifying particular version of schema, # but just with a type, e.g. cms-analysis # this be resolved to the last version of deposit schema of this type if '$ana_type' in data: try: schema = Schema.get_latest('deposits/records/{}'.format( data['$ana_type'])) except JSONSchemaNotFound: raise DepositValidationError( 'Schema {} is not a valid deposit schema.'.format( data['$ana_type'])) data['$schema'] = schema.fullpath data.pop('$ana_type') return data
def _preprocess_data(cls, data): # data can be sent without specifying particular version of schema, # but just with a type, e.g. cms-analysis # this be resolved to the last version of deposit schema of this type if '$ana_type' in data: try: schema = Schema.get_latest( 'deposits/records/{}'.format(data['$ana_type']) ) except JSONSchemaNotFound: raise DepositValidationError( 'Schema {} is not a valid deposit schema.' .format(data['$ana_type'])) data['$schema'] = schema.fullpath data.pop('$ana_type') return data
def test_get_by_fullpath(db): schema = Schema(**{'name': 'records/ana1', 'major': 1, 'minor': 0, 'patch': 1}) schema2 = Schema(**{'name': 'deposits/records/ana2', 'major': 2, 'minor': 1, 'patch': 1}) db.session.add(schema) db.session.add(schema2) db.session.commit() assert Schema.get_by_fullpath('https://some-host.com/schemas/records/ana1-v1.0.1.json') == schema assert Schema.get_by_fullpath('records/ana1-v1.0.1.json') == schema assert Schema.get_by_fullpath('https://some-host.com/schemas/deposits/records/ana2-v2.1.1') == schema2 assert Schema.get_by_fullpath('deposits/records/ana2-v2.1.1.json') == schema2 assert Schema.get_by_fullpath('/deposits/records/ana2-v2.1.1.json') == schema2 assert Schema.get_by_fullpath('/deposits/records/ana2-v2.1.1') == schema2
def test_get_latest_version_of_schema_when_schema_with_given_name_doesnt_exist_raises_SchemaDoesNotExist(db): with raises(SchemaDoesNotExist): Schema.get_latest(name='non-existing')
def test_get_by_fullpath_when_non_existing_raise_JSONSchemaNotFound(db): with raises(JSONSchemaNotFound): Schema.get_by_fullpath('/non-existing/schema/ana2-v2.1.1')
def test_get_by_fullpath_when_non_existing_raise_SchemaDoesNotExist(db): with raises(SchemaDoesNotExist): Schema.get_by_fullpath('/non-existing/schema/ana2-v2.1.1')
def resolve_cap_schemas(path): """Resolve CAP JSON schemas.""" return Schema.get_by_fullpath(path).json
def test_get_latest_version_of_schema_when_schema_with_given_name_doesnt_exist_raises_JSONSchemaNotFound(db): with raises(JSONSchemaNotFound): Schema.get_latest(name='non-existing')
def schema(self): """Schema property.""" return Schema.get_by_fullpath(self['$schema'])
def _set_experiment(self): schema = Schema.get_by_fullpath(self['$schema']) self['_experiment'] = schema.experiment