def test_jsonschemas_with_additional_properties_fails(app, db, location): with app.test_request_context(): metadata = minimal_deposits_metadata('test-schema-v0.0.1') id_ = uuid4() deposit_minter(id_, metadata) with pytest.raises(ValidationError): Deposit.create(metadata, id_=id_)
def test_create_deposit_with_non_object_data_returns_wrong_schema_error( app, users, location, jsonschemas_host): with app.test_request_context(): metadata = 5 login_user(users['superuser']) id_ = uuid4() with raises(EmptyDepositError): Deposit.create(metadata, id_=id_)
def test_create_deposit_with_empty_schema_returns_wrong_schema_error( app, users, location, jsonschemas_host): with app.test_request_context(): metadata = {'$schema': ''} login_user(users['superuser']) id_ = uuid4() with raises(WrongJSONSchemaError): Deposit.create(metadata, id_=id_)
def test_create_deposit_with_empty_schema_raises_DepositValidationError( app, users, location, jsonschemas_host): metadata = {'$schema': ''} with app.test_request_context(): login_user(users['superuser']) id_ = uuid4() with raises(DepositValidationError): Deposit.create(metadata, id_=id_)
def test_create_deposit_with_empty_schema_raises_DepositValidationError(app, users, location, jsonschemas_host): metadata = {'$schema': ''} with app.test_request_context(): login_user(users['superuser']) id_ = uuid4() with raises(DepositValidationError): Deposit.create(metadata, id_=id_)
def test_create_deposit_atlas_analysis_index(db, es, location, deposit_index): """Test if deposit atlas analysis index is created.""" deposit_index_name = deposit_index + '-' + 'atlas-analysis-v0.0.1' Deposit.create({ '$schema': 'https://analysispreservation.cern.ch/schemas/deposits/records/atlas-analysis-v0.0.1.json' }) db.session.commit() current_search.flush_and_refresh(deposit_index_name) res = current_search.client.search(index=deposit_index_name) assert 'id' in res['hits']['hits'][0]['_source']['_deposit']
def test_create_deposit_with_wrong_schema_returns_wrong_schema_error( app, users, location, jsonschemas_host): with app.test_request_context(): metadata = { '$schema': 'https://{}/schemas/deposits/records/lhcb-wrong.json'.format( jsonschemas_host) } login_user(users['superuser']) id_ = uuid4() with raises(WrongJSONSchemaError): Deposit.create(metadata, id_=id_)
def test_create_deposit_with_wrong_schema_raises_DepositValidationError( app, users, location): metadata = { '$schema': 'https://analysispreservation.cern.ch/schemas/deposits/records/non-existing-schema.json' # noqa } with app.test_request_context(): login_user(users['superuser']) id_ = uuid4() with raises(DepositValidationError): Deposit.create(metadata, id_=id_)
def test_create_deposit_with_wrong_data_returns_validation_error( app, users, location, jsonschemas_host): with app.test_request_context(): metadata = { '$schema': 'https://{}/schemas/deposits/records/lhcb-v0.0.1.json'.format( jsonschemas_host), 'general_title': ['I am an array, not a string'] } login_user(users['superuser']) id_ = uuid4() with raises(ValidationError): Deposit.create(metadata, id_=id_)
def test_create_deposit_with_wrong_schema_raises_DepositValidationError( app, users, location, jsonschemas_host): metadata = { '$schema': 'https://{}/schemas/deposits/records/non-existing-schema.json'.format( jsonschemas_host) } with app.test_request_context(): login_user(users['superuser']) id_ = uuid4() with raises(DepositValidationError): Deposit.create(metadata, id_=id_)
def test_create_deposit_with_wrong_schema_raises_DepositValidationError(app, users, location, jsonschemas_host): metadata = { '$schema': 'https://{}/schemas/deposits/records/non-existing-schema.json'.format( jsonschemas_host) } with app.test_request_context(): login_user(users['superuser']) id_ = uuid4() with raises(DepositValidationError): Deposit.create(metadata, id_=id_)
def questionnaires(file): """Load and save CMS questionnaire data.""" answers = extract_questionnaires_from_excel(file) click.secho(f"Total Answers: {len(answers)}", fg='green') for answer in answers: deposit = None with db.session.begin_nested(): try: title = f"Statistics Questionnaire for " \ f"{answer['analysis_context']['cadi_id']}" \ if answer['analysis_context']['cadi_id'] \ else "--" extracted = remove_none_keys( _questionnaire_data(answer, title=title)) if title: extracted["general_title"] = title deposit = CAPDeposit.create(data=extracted, owner=None) # give read access to members of CMS experiment deposit._add_experiment_permissions( 'CMS', ['deposit-read'], ) # give permission to stat committee admin_egroups = ["*****@*****.**"] for role in admin_egroups: _role = get_existing_or_register_role(role) deposit._add_egroup_permissions( _role, [ 'deposit-read', 'deposit-update', 'deposit-admin', ], db.session, ) deposit.commit() click.secho( f"Success: {answer['_general_info']['serial']} - " f"{answer['_general_info']['user']}", fg='green') click.secho(f"{title}", fg='yellow') except DepositValidationError as e: click.secho("---------------") click.secho(f"Validation Error", fg='red') for err in e.errors: click.secho(f"{err.to_dict()}", fg='red') click.secho("---------------") pass except ValidationError: click.secho("---------------") click.secho(f"Validation Error", fg='red') click.secho("---------------") pass db.session.commit()
def _create_deposit(user, schema_name, metadata=None, experiment=None, publish=False): """ Create a new deposit for given user and schema name e.g cms-analysis-v0.0.1, with minimal metadata defined for this schema type. """ with app.test_request_context(): # create schema for record create_schema('records/{}'.format(schema_name), is_deposit=False, experiment='CMS') # create schema for deposit schema = create_schema('deposits/records/{}'.format(schema_name), experiment=experiment) metadata = metadata or minimal_metadata(jsonschemas_host, 'deposits/records/{}'.format(schema_name)) login_user(user) id_ = uuid4() deposit_minter(id_, metadata) deposit = Deposit.create(metadata, id_=id_) if publish: deposit.publish() _, record = deposit.fetch_published() RecordIndexer().index(record) current_search.flush_and_refresh('records') current_search.flush_and_refresh(schema.index_name) return Deposit.get_record(deposit.id)
def synchronize_cadi_entries(limit=None): """Add/update all CADI entries connecting with CADI database.""" entries = get_entries_from_cadi_db() for entry in entries[0:limit]: # remove artefact from code names cadi_id = re.sub('^d', '', entry.get('code', None)) try: # update if already exists parser = HTMLParser() uuid = get_entry_uuid_by_unique_field('deposits-records-cms-analysis-v0.0.1', {'basic_info__cadi_id': cadi_id}) deposit = CAPDeposit.get_record(uuid) if 'cadi_info' not in deposit: deposit['cadi_info'] = {} for cadi_key, cap_key in CADI_FIELD_TO_CAP_MAP.items(): # sometimes they store data in HTML format.. need to escape chars deposit['cadi_info'][cap_key] = parser.unescape(entry.get(cadi_key, '')) or '' deposit.commit() print('Cadi entry {} updated.'.format(cadi_id)) except DepositDoesNotExist: # or create new cadi entry data = construct_cadi_entry(cadi_id, { 'cadi_info': {v: parser.unescape(entry.get(k, '')) or '' for k, v in CADI_FIELD_TO_CAP_MAP.items()} }) deposit = CAPDeposit.create(data=data) add_read_permission_for_egroup(deposit, '*****@*****.**') print('Cadi entry {} added.'.format(cadi_id))
def add_drafts_from_file(file_path, schema, egroup=None, user=None, limit=None): """Adds drafts from a specified file. Drafts with specified pid will be registered under those. For drafts without pid, new pids will be minted. """ with open(file_path, 'r') as fp: entries = json.load(fp) for entry in entries[0:limit]: data = construct_draft_obj(schema, entry) pid = cap_deposit_fetcher(None, data) pid_value = pid.pid_value if pid else None try: PersistentIdentifier.get('depid', pid_value) print('Draft with id {} already exist!'.format(pid_value)) except PIDDoesNotExistError: record_uuid = uuid.uuid4() pid = cap_deposit_minter(record_uuid, data) if user: user = User.query.filter_by(email=user).one() if egroup: role = Role.query.filter_by(name=egroup).one() deposit = CAPDeposit.create(data, record_uuid, user) deposit.commit() if egroup: add_read_permission_for_egroup(deposit, egroup) print('Draft {} added.'.format(pid.pid_value))
def add_drafts_from_file(file_path, schema, egroup, limit=None): with open(file_path, 'r') as fp: entries = json.load(fp) for entry in entries[0:limit]: deposit = CAPDeposit.create(construct_draft_obj(schema, entry)) add_read_permission_for_egroup(deposit, egroup) print('Draft {} added.'.format(deposit.id))
def synchronize_cadi_entries(limit=None): """Synchronize CMS analysis with CADI database. Updates cadi_info in existing analysis. If analysis with given CADI id doesn't exist yet, new deposit will be created. All members of CMS will get a r access and cms-admin egroup rw access. :params int limit: number of entries to update """ def _cadi_deposit(cadi_id, cadi_info): return { '$ana_type': 'cms-analysis', 'cadi_info': cadi_info, 'general_title': cadi_info.get('name') or cadi_id, '_fetched_from': 'cadi', '_user_edited': False, 'basic_info': { 'cadi_id': cadi_id } } entries = get_all_from_cadi() cms_admin_group = get_existing_or_register_role('*****@*****.**') for entry in entries[:limit]: cadi_info = cadi_serializer.dump(entry).data cadi_id = cadi_info['cadi_id'] try: # update if cadi deposit already exists deposit = get_deposit_by_cadi_id(cadi_id) if deposit.get('cadi_info') == cadi_info: print('No changes in cadi entry {}.'.format(cadi_id)) else: deposit['cadi_info'] = cadi_info deposit.commit() db.session.commit() print('Cadi entry {} updated.'.format(cadi_id)) except DepositDoesNotExist: deposit = CAPDeposit.create(data=_cadi_deposit(cadi_id, cadi_info), owner=None) deposit._add_experiment_permissions('CMS', ['deposit-read']) deposit._add_egroup_permissions( cms_admin_group, ['deposit-read', 'deposit-update', 'deposit-admin'], db.session) # noqa deposit.commit() db.session.commit() print('Cadi entry {} added.'.format(cadi_id))
def _create_deposit(user, schema): """ Create a new deposit for given user and schema name e.g cms-analysis-v0.0.1, with minimal metadata defined for this schema type. """ db_.session.begin_nested() with app.test_request_context(): metadata = minimal_deposits_metadata(schema) login_user(user) id_ = uuid4() deposit_minter(id_, metadata) deposit = Deposit.create(metadata, id_=id_) current_search.flush_and_refresh('deposits-records-{}'.format(schema)) return deposit
def _create_deposit(user, schema_name, metadata=None, experiment=None, files={}, publish=False, mapping=None): """Create a new deposit for given user and schema name. e.g cms-analysis-v0.0.1, with minimal metadata defined for this schema type. """ # create schema for record with app.test_request_context(): schema = create_schema(schema_name, experiment=experiment, deposit_mapping=mapping) deposit_schema_url = current_jsonschemas.path_to_url( schema.deposit_path) metadata = metadata or minimal_metadata(deposit_schema_url) login_user(user) id_ = uuid4() deposit_minter(id_, metadata) deposit = Deposit.create(metadata, id_=id_) for k, v in files.items(): deposit.files[k] = v if files: deposit.commit() db.session.commit() if publish: deposit.publish() _, record = deposit.fetch_published() RecordIndexer().index(record) current_search.flush_and_refresh(schema.record_index) current_search.flush_and_refresh(schema.deposit_index) return Deposit.get_record(deposit.id)
def synchronize_cadi_entries(limit=None): """Add/update CADI info in all cms-analysis syncing with CADI db.""" parser = HTMLParser() entries = get_all_entries_from_cadi() def parse_field(field): """Escape HTML characters.""" if isinstance(field, unicode): return parser.unescape(parser.unescape(field)) else: return '' for entry in entries[:limit]: # remove artefact from cadi names cadi_id = re.sub('^d', '', entry.get('code', None)) cadi_info = { cap_key: parse_field(entry.get(cadi_key, None)) for cadi_key, cap_key in CADI_FIELD_TO_CAP_MAP.items() } try: # update if already exists uuid = get_entry_uuid_by_unique_field( 'deposits-records-cms-analysis-v0.0.1', {'basic_info__cadi_id__keyword': cadi_id} ) deposit = CAPDeposit.get_record(uuid) deposit['cadi_info'] = cadi_info deposit.commit() print('Cadi entry {} updated.'.format(cadi_id)) except (DepositDoesNotExist, NoResultFound): data = construct_cadi_entry(cadi_id, { 'cadi_info': cadi_info }) deposit = CAPDeposit.create(data=data) add_read_permission_for_egroup(deposit, '*****@*****.**') print('Cadi entry {} added.'.format(cadi_id))
def add_drafts_from_file(file_path, schema, egroup=None, usermail=None, limit=None): """Adds drafts from a specified file. Drafts with specified pid will be registered under those. For drafts without pid, new pids will be minted. """ if usermail: user = get_existing_or_register_user(usermail) else: user = None with open(file_path, 'r') as fp: entries = json.load(fp) for entry in entries[0:limit]: data = construct_draft_obj(schema, entry) pid = cap_deposit_fetcher(None, data) pid_value = pid.pid_value if pid else None try: PersistentIdentifier.get('depid', pid_value) print('Draft with id {} already exist!'.format(pid_value)) except PIDDoesNotExistError: record_uuid = uuid.uuid4() pid = cap_deposit_minter(record_uuid, data) deposit = CAPDeposit.create(data, record_uuid, user) deposit.commit() if egroup: add_read_permission_for_egroup(deposit, egroup) print('Draft {} added.'.format(pid.pid_value)) db.session.commit()
def synchronize_cadi_entries(limit=None): """Add/update CADI info in all cms-analysis syncing with CADI db.""" parser = HTMLParser() entries = get_all_entries_from_cadi() def parse_field(field): """Escape HTML characters.""" if isinstance(field, unicode): return parser.unescape(parser.unescape(field)) else: return '' for entry in entries[:limit]: # remove artefact from cadi names cadi_id = re.sub('^d', '', entry.get('code', None)) cadi_info = { cap_key: parse_field(entry.get(cadi_key, None)) for cadi_key, cap_key in CADI_FIELD_TO_CAP_MAP.items() } try: # update if already exists uuid = get_entry_uuid_by_unique_field( 'deposits-records-cms-analysis-v0.0.1', {'basic_info__cadi_id__keyword': cadi_id}) deposit = CAPDeposit.get_record(uuid) deposit['cadi_info'] = cadi_info deposit.commit() print('Cadi entry {} updated.'.format(cadi_id)) except (DepositDoesNotExist, NoResultFound): data = construct_cadi_entry(cadi_id, {'cadi_info': cadi_info}) deposit = CAPDeposit.create(data=data) add_read_permission_for_egroup(deposit, '*****@*****.**') print('Cadi entry {} added.'.format(cadi_id))
def add(file_path, schema, version, egroup, usermail, limit): """Add drafts from a file. Drafts with specified pid will be registered under those. For drafts without pid, new pids will be minted. """ if usermail: user = get_existing_or_register_user(usermail) else: user = None with open(file_path, 'r') as fp: entries = json.load(fp) for entry in entries[0:limit]: pid = cap_deposit_fetcher(None, entry) pid_value = pid.pid_value if pid else None try: PersistentIdentifier.get('depid', pid_value) click.secho( 'Draft with id {} already exist!'.format(pid_value), fg='red') except PIDDoesNotExistError: record_uuid = uuid.uuid4() pid = cap_deposit_minter(record_uuid, entry) deposit = CAPDeposit.create(entry, record_uuid, user) deposit.commit() if egroup: add_read_permission_for_egroup(deposit, egroup) click.secho('Draft {} added.'.format(pid.pid_value), fg='green') db.session.commit()
def synchronize_cadi_entries(limit=None): """Synchronize CMS analysis with CADI database. Updates cadi_info in existing analysis. If analysis with given CADI id doesn't exist yet, new deposit will be created. All members of CMS will get a read access. :params int limit: number of entries to update """ entries = get_all_from_cadi() for entry in entries[:limit]: cadi_id, cadi_info = parse_cadi_entry(entry) try: # update if cadi deposit already exists deposit = get_deposit_by_cadi_id(cadi_id) if deposit.get('cadi_info') == cadi_info: print('No changes in cadi entry {}.'.format(cadi_id)) else: deposit['cadi_info'] = cadi_info deposit.commit() db.session.commit() print('Cadi entry {} updated.'.format(cadi_id)) except (DepositDoesNotExist): cadi_deposit = build_cadi_deposit(cadi_id, cadi_info) deposit = CAPDeposit.create(data=cadi_deposit, owner=None) deposit._add_experiment_permissions('CMS', ['deposit-read']) deposit.commit() db.session.commit() print('Cadi entry {} added.'.format(cadi_id))
def synchronize_cadi_entries(limit=None): """Add/update all CADI entries connecting with CADI database.""" entries = get_entries_from_cadi_db() for entry in entries[:limit]: # remove artefact from code names cadi_id = re.sub('^d', '', entry.get('code', None)) try: # update if already exists parser = HTMLParser() uuid = get_entry_uuid_by_unique_field( 'deposits-records-cms-analysis-v0.0.1', {'basic_info__cadi_id__keyword': cadi_id} ) deposit = CAPDeposit.get_record(uuid) if 'cadi_info' not in deposit: deposit['cadi_info'] = {} for cadi_key, cap_key in CADI_FIELD_TO_CAP_MAP.items(): # sometimes they store data in HTML format.. need to escape # chars deposit['cadi_info'][cap_key] = parser.unescape( entry.get(cadi_key, '') or '') or '' deposit.commit() print('Cadi entry {} updated.'.format(cadi_id)) except (DepositDoesNotExist, NoResultFound): data = construct_cadi_entry(cadi_id, { 'cadi_info': {v: parser.unescape(entry.get(k, '') or '') or '' for k, v in CADI_FIELD_TO_CAP_MAP.items()} }) deposit = CAPDeposit.create(data=data) add_read_permission_for_egroup(deposit, '*****@*****.**') print('Cadi entry {} added.'.format(cadi_id))
def create_deposit_with_permissions(data, roles, users, owner, ana, errors): """Create a deposit and add privileges and owner information.""" from cap.modules.deposit.api import CAPDeposit # make sure the schema is valid first if not check_and_update_data_with_schema(data, ana): return with db.session.begin_nested(): try: # saving original to return to user if wrong _data = copy.deepcopy(data) owner = get_existing_or_register_user(owner) if owner else None deposit = CAPDeposit.create(data=data, owner=owner) # add roles and users if roles: for role in roles: _role = get_existing_or_register_role(role.strip()) deposit._add_egroup_permissions(_role, ['deposit-read'], db.session) if users: for user in users: _user = get_existing_or_register_user(user.strip()) deposit._add_user_permissions(_user, ['deposit-read'], db.session) deposit.commit() except ValidationError as err: click.secho(f'Validation Error: {err.message}', fg='red') errors.append(_data) return db.session.commit() click.secho(f"Created deposit with id: {deposit['_deposit']['id']}", fg='green') # noqa
def _create_deposit(user, schema_name, metadata=None, experiment=None, publish=False): """ Create a new deposit for given user and schema name e.g cms-analysis-v0.0.1, with minimal metadata defined for this schema type. """ with app.test_request_context(): # create schema for record schema = create_schema('records/{}'.format(schema_name), is_deposit=False, experiment=experiment) if not experiment: schema.add_read_access_to_all() # create schema for deposit schema = create_schema('deposits/records/{}'.format(schema_name), experiment=experiment) if not experiment: schema.add_read_access_to_all() metadata = metadata or minimal_metadata(jsonschemas_host, 'deposits/records/{}'.format(schema_name)) login_user(user) id_ = uuid4() deposit_minter(id_, metadata) deposit = Deposit.create(metadata, id_=id_) if publish: deposit.publish() _, record = deposit.fetch_published() RecordIndexer().index(record) current_search.flush_and_refresh('records') current_search.flush_and_refresh(schema.index_name) return Deposit.get_record(deposit.id)
def synchronize_cadi_entries(limit=None): """Synchronize CMS analysis with CADI database. Updates cadi_info in existing analysis. If analysis with given CADI id doesn't exist yet, new deposit will be created. All members of CMS will get a r access and cms-admin egroup rw access. :params int limit: number of entries to update """ def _cadi_deposit(cadi_id, cadi_info): return { '$ana_type': 'cms-analysis', 'cadi_info': cadi_info, 'general_title': cadi_info.get('name') or cadi_id, '_fetched_from': 'cadi', '_user_edited': False, 'basic_info': { 'cadi_id': cadi_id } } entries = get_all_from_cadi() for entry in islice(entries, limit): deposit = None cadi_info = cadi_serializer.dump(entry).data cadi_id = cadi_info.pop('cadi_id') try: # update if cadi deposit already exists deposit = get_deposit_by_cadi_id(cadi_id) if deposit.get('cadi_info') == cadi_info: current_app.logger.info(f'No changes in entry {cadi_id}.') else: deposit['cadi_info'] = cadi_info try: deposit.commit() except DepositValidationError: current_app.logger.exception( f'Error during updating cadi info in {cadi_id}.') continue db.session.commit() current_app.logger.info(f'Cadi entry {cadi_id} updated.') except DepositDoesNotExist: try: with db.session.begin_nested(): deposit = CAPDeposit.create( data=_cadi_deposit(cadi_id, cadi_info)) # give read access to members of CMS experiment deposit._add_experiment_permissions( 'CMS', ['deposit-read'], ) # give admin access to the contact person (if in ldap) owner = _get_owner_account(cadi_info['contact']) if owner: deposit._add_user_permissions( owner, [ 'deposit-read', 'deposit-update', 'deposit-admin', ], db.session, ) # give admin access to cms admin egroups admin_egroups = _get_admin_egroups(wg=cadi_id[:3]) for role in admin_egroups: deposit._add_egroup_permissions( role, [ 'deposit-read', 'deposit-update', 'deposit-admin', ], db.session, ) deposit.commit() db.session.commit() current_app.logger.info(f'Cadi entry {cadi_id} added.') except Exception: if deposit and deposit.id: deposit.indexer.delete(deposit) current_app.logger.exception(f'Error during adding {cadi_id}.')