def test_jsonschemas_with_additional_properties_fails(app, db, location):
with app.test_request_context():
metadata = minimal_deposits_metadata('test-schema-v0.0.1')
id_ = uuid4()
deposit_minter(id_, metadata)
with pytest.raises(ValidationError):
Deposit.create(metadata, id_=id_)
def test_create_deposit_with_non_object_data_returns_wrong_schema_error(
app, users, location, jsonschemas_host):
with app.test_request_context():
metadata = 5
login_user(users['superuser'])
id_ = uuid4()
with raises(EmptyDepositError):
Deposit.create(metadata, id_=id_)
def test_create_deposit_with_empty_schema_returns_wrong_schema_error(
app, users, location, jsonschemas_host):
with app.test_request_context():
metadata = {'$schema': ''}
login_user(users['superuser'])
id_ = uuid4()
with raises(WrongJSONSchemaError):
Deposit.create(metadata, id_=id_)
def test_create_deposit_with_empty_schema_raises_DepositValidationError(
app, users, location, jsonschemas_host):
metadata = {'$schema': ''}
with app.test_request_context():
login_user(users['superuser'])
id_ = uuid4()
with raises(DepositValidationError):
Deposit.create(metadata, id_=id_)
def test_create_deposit_with_empty_schema_raises_DepositValidationError(app,
users,
location,
jsonschemas_host):
metadata = {'$schema': ''}
with app.test_request_context():
login_user(users['superuser'])
id_ = uuid4()
with raises(DepositValidationError):
Deposit.create(metadata, id_=id_)
def test_create_deposit_atlas_analysis_index(db, es, location, deposit_index):
"""Test if deposit atlas analysis index is created."""
deposit_index_name = deposit_index + '-' + 'atlas-analysis-v0.0.1'
Deposit.create({
'$schema':
'https://analysispreservation.cern.ch/schemas/deposits/records/atlas-analysis-v0.0.1.json'
})
db.session.commit()
current_search.flush_and_refresh(deposit_index_name)
res = current_search.client.search(index=deposit_index_name)
assert 'id' in res['hits']['hits'][0]['_source']['_deposit']
def test_create_deposit_with_wrong_schema_returns_wrong_schema_error(
app, users, location, jsonschemas_host):
with app.test_request_context():
metadata = {
'$schema':
'https://{}/schemas/deposits/records/lhcb-wrong.json'.format(
jsonschemas_host)
}
login_user(users['superuser'])
id_ = uuid4()
with raises(WrongJSONSchemaError):
Deposit.create(metadata, id_=id_)
def test_create_deposit_with_wrong_schema_raises_DepositValidationError(
app, users, location):
metadata = {
'$schema': 'https://analysispreservation.cern.ch/schemas/deposits/records/non-existing-schema.json' # noqa
}
with app.test_request_context():
login_user(users['superuser'])
id_ = uuid4()
with raises(DepositValidationError):
Deposit.create(metadata, id_=id_)
def test_create_deposit_with_wrong_data_returns_validation_error(
app, users, location, jsonschemas_host):
with app.test_request_context():
metadata = {
'$schema':
'https://{}/schemas/deposits/records/lhcb-v0.0.1.json'.format(
jsonschemas_host),
'general_title': ['I am an array, not a string']
}
login_user(users['superuser'])
id_ = uuid4()
with raises(ValidationError):
Deposit.create(metadata, id_=id_)
def test_create_deposit_with_wrong_schema_raises_DepositValidationError(
app, users, location, jsonschemas_host):
metadata = {
'$schema':
'https://{}/schemas/deposits/records/non-existing-schema.json'.format(
jsonschemas_host)
}
with app.test_request_context():
login_user(users['superuser'])
id_ = uuid4()
with raises(DepositValidationError):
Deposit.create(metadata, id_=id_)
def test_create_deposit_with_wrong_schema_raises_DepositValidationError(app,
users,
location,
jsonschemas_host):
metadata = {
'$schema': 'https://{}/schemas/deposits/records/non-existing-schema.json'.format(
jsonschemas_host)
}
with app.test_request_context():
login_user(users['superuser'])
id_ = uuid4()
with raises(DepositValidationError):
Deposit.create(metadata, id_=id_)
def questionnaires(file):
"""Load and save CMS questionnaire data."""
answers = extract_questionnaires_from_excel(file)
click.secho(f"Total Answers: {len(answers)}", fg='green')
for answer in answers:
deposit = None
with db.session.begin_nested():
try:
title = f"Statistics Questionnaire for " \
f"{answer['analysis_context']['cadi_id']}" \
if answer['analysis_context']['cadi_id'] \
else "--"
extracted = remove_none_keys(
_questionnaire_data(answer, title=title))
if title:
extracted["general_title"] = title
deposit = CAPDeposit.create(data=extracted, owner=None)
# give read access to members of CMS experiment
deposit._add_experiment_permissions(
'CMS',
['deposit-read'],
)
# give permission to stat committee
admin_egroups = ["*****@*****.**"]
for role in admin_egroups:
_role = get_existing_or_register_role(role)
deposit._add_egroup_permissions(
_role,
[
'deposit-read',
'deposit-update',
'deposit-admin',
],
db.session,
)
deposit.commit()
click.secho(
f"Success: {answer['_general_info']['serial']} - "
f"{answer['_general_info']['user']}",
fg='green')
click.secho(f"{title}", fg='yellow')
except DepositValidationError as e:
click.secho("---------------")
click.secho(f"Validation Error", fg='red')
for err in e.errors:
click.secho(f"{err.to_dict()}", fg='red')
click.secho("---------------")
pass
except ValidationError:
click.secho("---------------")
click.secho(f"Validation Error", fg='red')
click.secho("---------------")
pass
db.session.commit()
def _create_deposit(user, schema_name, metadata=None, experiment=None, publish=False):
"""
Create a new deposit for given user and schema name
e.g cms-analysis-v0.0.1,
with minimal metadata defined for this schema type.
"""
with app.test_request_context():
# create schema for record
create_schema('records/{}'.format(schema_name), is_deposit=False,
experiment='CMS')
# create schema for deposit
schema = create_schema('deposits/records/{}'.format(schema_name),
experiment=experiment)
metadata = metadata or minimal_metadata(jsonschemas_host,
'deposits/records/{}'.format(schema_name))
login_user(user)
id_ = uuid4()
deposit_minter(id_, metadata)
deposit = Deposit.create(metadata, id_=id_)
if publish:
deposit.publish()
_, record = deposit.fetch_published()
RecordIndexer().index(record)
current_search.flush_and_refresh('records')
current_search.flush_and_refresh(schema.index_name)
return Deposit.get_record(deposit.id)
def synchronize_cadi_entries(limit=None):
"""Add/update all CADI entries connecting with CADI database."""
entries = get_entries_from_cadi_db()
for entry in entries[0:limit]:
# remove artefact from code names
cadi_id = re.sub('^d', '', entry.get('code', None))
try: # update if already exists
parser = HTMLParser()
uuid = get_entry_uuid_by_unique_field('deposits-records-cms-analysis-v0.0.1',
{'basic_info__cadi_id': cadi_id})
deposit = CAPDeposit.get_record(uuid)
if 'cadi_info' not in deposit:
deposit['cadi_info'] = {}
for cadi_key, cap_key in CADI_FIELD_TO_CAP_MAP.items():
# sometimes they store data in HTML format.. need to escape chars
deposit['cadi_info'][cap_key] = parser.unescape(entry.get(cadi_key, '')) or ''
deposit.commit()
print('Cadi entry {} updated.'.format(cadi_id))
except DepositDoesNotExist: # or create new cadi entry
data = construct_cadi_entry(cadi_id, {
'cadi_info': {v: parser.unescape(entry.get(k, '')) or ''
for k, v in CADI_FIELD_TO_CAP_MAP.items()}
})
deposit = CAPDeposit.create(data=data)
add_read_permission_for_egroup(deposit, '*****@*****.**')
print('Cadi entry {} added.'.format(cadi_id))
def add_drafts_from_file(file_path, schema,
egroup=None, user=None, limit=None):
"""Adds drafts from a specified file.
Drafts with specified pid will be registered under those.
For drafts without pid, new pids will be minted.
"""
with open(file_path, 'r') as fp:
entries = json.load(fp)
for entry in entries[0:limit]:
data = construct_draft_obj(schema, entry)
pid = cap_deposit_fetcher(None, data)
pid_value = pid.pid_value if pid else None
try:
PersistentIdentifier.get('depid', pid_value)
print('Draft with id {} already exist!'.format(pid_value))
except PIDDoesNotExistError:
record_uuid = uuid.uuid4()
pid = cap_deposit_minter(record_uuid, data)
if user:
user = User.query.filter_by(email=user).one()
if egroup:
role = Role.query.filter_by(name=egroup).one()
deposit = CAPDeposit.create(data, record_uuid, user)
deposit.commit()
if egroup:
add_read_permission_for_egroup(deposit, egroup)
print('Draft {} added.'.format(pid.pid_value))
def add_drafts_from_file(file_path, schema, egroup, limit=None):
with open(file_path, 'r') as fp:
entries = json.load(fp)
for entry in entries[0:limit]:
deposit = CAPDeposit.create(construct_draft_obj(schema, entry))
add_read_permission_for_egroup(deposit, egroup)
print('Draft {} added.'.format(deposit.id))
def synchronize_cadi_entries(limit=None):
"""Synchronize CMS analysis with CADI database.
Updates cadi_info in existing analysis.
If analysis with given CADI id doesn't exist yet,
new deposit will be created.
All members of CMS will get a r access and cms-admin egroup rw access.
:params int limit: number of entries to update
"""
def _cadi_deposit(cadi_id, cadi_info):
return {
'$ana_type': 'cms-analysis',
'cadi_info': cadi_info,
'general_title': cadi_info.get('name') or cadi_id,
'_fetched_from': 'cadi',
'_user_edited': False,
'basic_info': {
'cadi_id': cadi_id
}
}
entries = get_all_from_cadi()
cms_admin_group = get_existing_or_register_role('*****@*****.**')
for entry in entries[:limit]:
cadi_info = cadi_serializer.dump(entry).data
cadi_id = cadi_info['cadi_id']
try: # update if cadi deposit already exists
deposit = get_deposit_by_cadi_id(cadi_id)
if deposit.get('cadi_info') == cadi_info:
print('No changes in cadi entry {}.'.format(cadi_id))
else:
deposit['cadi_info'] = cadi_info
deposit.commit()
db.session.commit()
print('Cadi entry {} updated.'.format(cadi_id))
except DepositDoesNotExist:
deposit = CAPDeposit.create(data=_cadi_deposit(cadi_id, cadi_info),
owner=None)
deposit._add_experiment_permissions('CMS', ['deposit-read'])
deposit._add_egroup_permissions(
cms_admin_group,
['deposit-read', 'deposit-update', 'deposit-admin'],
db.session) # noqa
deposit.commit()
db.session.commit()
print('Cadi entry {} added.'.format(cadi_id))
def _create_deposit(user, schema):
"""
Create a new deposit for given user and schema name
e.g cms-analysis-v0.0.1,
with minimal metadata defined for this schema type.
"""
db_.session.begin_nested()
with app.test_request_context():
metadata = minimal_deposits_metadata(schema)
login_user(user)
id_ = uuid4()
deposit_minter(id_, metadata)
deposit = Deposit.create(metadata, id_=id_)
current_search.flush_and_refresh('deposits-records-{}'.format(schema))
return deposit
def _create_deposit(user,
schema_name,
metadata=None,
experiment=None,
files={},
publish=False,
mapping=None):
"""Create a new deposit for given user and schema name.
e.g cms-analysis-v0.0.1,
with minimal metadata defined for this schema type.
"""
# create schema for record
with app.test_request_context():
schema = create_schema(schema_name,
experiment=experiment,
deposit_mapping=mapping)
deposit_schema_url = current_jsonschemas.path_to_url(
schema.deposit_path)
metadata = metadata or minimal_metadata(deposit_schema_url)
login_user(user)
id_ = uuid4()
deposit_minter(id_, metadata)
deposit = Deposit.create(metadata, id_=id_)
for k, v in files.items():
deposit.files[k] = v
if files:
deposit.commit()
db.session.commit()
if publish:
deposit.publish()
_, record = deposit.fetch_published()
RecordIndexer().index(record)
current_search.flush_and_refresh(schema.record_index)
current_search.flush_and_refresh(schema.deposit_index)
return Deposit.get_record(deposit.id)
def synchronize_cadi_entries(limit=None):
"""Add/update CADI info in all cms-analysis syncing with CADI db."""
parser = HTMLParser()
entries = get_all_entries_from_cadi()
def parse_field(field):
"""Escape HTML characters."""
if isinstance(field, unicode):
return parser.unescape(parser.unescape(field))
else:
return ''
for entry in entries[:limit]:
# remove artefact from cadi names
cadi_id = re.sub('^d', '', entry.get('code', None))
cadi_info = {
cap_key: parse_field(entry.get(cadi_key, None))
for cadi_key, cap_key in CADI_FIELD_TO_CAP_MAP.items()
}
try: # update if already exists
uuid = get_entry_uuid_by_unique_field(
'deposits-records-cms-analysis-v0.0.1',
{'basic_info__cadi_id__keyword': cadi_id}
)
deposit = CAPDeposit.get_record(uuid)
deposit['cadi_info'] = cadi_info
deposit.commit()
print('Cadi entry {} updated.'.format(cadi_id))
except (DepositDoesNotExist, NoResultFound):
data = construct_cadi_entry(cadi_id, {
'cadi_info': cadi_info
})
deposit = CAPDeposit.create(data=data)
add_read_permission_for_egroup(deposit, '*****@*****.**')
print('Cadi entry {} added.'.format(cadi_id))
def add_drafts_from_file(file_path,
schema,
egroup=None,
usermail=None,
limit=None):
"""Adds drafts from a specified file.
Drafts with specified pid will be registered under those.
For drafts without pid, new pids will be minted.
"""
if usermail:
user = get_existing_or_register_user(usermail)
else:
user = None
with open(file_path, 'r') as fp:
entries = json.load(fp)
for entry in entries[0:limit]:
data = construct_draft_obj(schema, entry)
pid = cap_deposit_fetcher(None, data)
pid_value = pid.pid_value if pid else None
try:
PersistentIdentifier.get('depid', pid_value)
print('Draft with id {} already exist!'.format(pid_value))
except PIDDoesNotExistError:
record_uuid = uuid.uuid4()
pid = cap_deposit_minter(record_uuid, data)
deposit = CAPDeposit.create(data, record_uuid, user)
deposit.commit()
if egroup:
add_read_permission_for_egroup(deposit, egroup)
print('Draft {} added.'.format(pid.pid_value))
db.session.commit()
def synchronize_cadi_entries(limit=None):
"""Add/update CADI info in all cms-analysis syncing with CADI db."""
parser = HTMLParser()
entries = get_all_entries_from_cadi()
def parse_field(field):
"""Escape HTML characters."""
if isinstance(field, unicode):
return parser.unescape(parser.unescape(field))
else:
return ''
for entry in entries[:limit]:
# remove artefact from cadi names
cadi_id = re.sub('^d', '', entry.get('code', None))
cadi_info = {
cap_key: parse_field(entry.get(cadi_key, None))
for cadi_key, cap_key in CADI_FIELD_TO_CAP_MAP.items()
}
try: # update if already exists
uuid = get_entry_uuid_by_unique_field(
'deposits-records-cms-analysis-v0.0.1',
{'basic_info__cadi_id__keyword': cadi_id})
deposit = CAPDeposit.get_record(uuid)
deposit['cadi_info'] = cadi_info
deposit.commit()
print('Cadi entry {} updated.'.format(cadi_id))
except (DepositDoesNotExist, NoResultFound):
data = construct_cadi_entry(cadi_id, {'cadi_info': cadi_info})
deposit = CAPDeposit.create(data=data)
add_read_permission_for_egroup(deposit, '*****@*****.**')
print('Cadi entry {} added.'.format(cadi_id))
def add(file_path, schema, version, egroup, usermail, limit):
"""Add drafts from a file.
Drafts with specified pid will be registered under those.
For drafts without pid, new pids will be minted.
"""
if usermail:
user = get_existing_or_register_user(usermail)
else:
user = None
with open(file_path, 'r') as fp:
entries = json.load(fp)
for entry in entries[0:limit]:
pid = cap_deposit_fetcher(None, entry)
pid_value = pid.pid_value if pid else None
try:
PersistentIdentifier.get('depid', pid_value)
click.secho(
'Draft with id {} already exist!'.format(pid_value),
fg='red')
except PIDDoesNotExistError:
record_uuid = uuid.uuid4()
pid = cap_deposit_minter(record_uuid, entry)
deposit = CAPDeposit.create(entry, record_uuid, user)
deposit.commit()
if egroup:
add_read_permission_for_egroup(deposit, egroup)
click.secho('Draft {} added.'.format(pid.pid_value),
fg='green')
db.session.commit()
def synchronize_cadi_entries(limit=None):
"""Synchronize CMS analysis with CADI database.
Updates cadi_info in existing analysis.
If analysis with given CADI id doesn't exist yet,
new deposit will be created.
All members of CMS will get a read access.
:params int limit: number of entries to update
"""
entries = get_all_from_cadi()
for entry in entries[:limit]:
cadi_id, cadi_info = parse_cadi_entry(entry)
try: # update if cadi deposit already exists
deposit = get_deposit_by_cadi_id(cadi_id)
if deposit.get('cadi_info') == cadi_info:
print('No changes in cadi entry {}.'.format(cadi_id))
else:
deposit['cadi_info'] = cadi_info
deposit.commit()
db.session.commit()
print('Cadi entry {} updated.'.format(cadi_id))
except (DepositDoesNotExist):
cadi_deposit = build_cadi_deposit(cadi_id, cadi_info)
deposit = CAPDeposit.create(data=cadi_deposit, owner=None)
deposit._add_experiment_permissions('CMS', ['deposit-read'])
deposit.commit()
db.session.commit()
print('Cadi entry {} added.'.format(cadi_id))
def synchronize_cadi_entries(limit=None):
"""Add/update all CADI entries connecting with CADI database."""
entries = get_entries_from_cadi_db()
for entry in entries[:limit]:
# remove artefact from code names
cadi_id = re.sub('^d', '', entry.get('code', None))
try: # update if already exists
parser = HTMLParser()
uuid = get_entry_uuid_by_unique_field(
'deposits-records-cms-analysis-v0.0.1',
{'basic_info__cadi_id__keyword': cadi_id}
)
deposit = CAPDeposit.get_record(uuid)
if 'cadi_info' not in deposit:
deposit['cadi_info'] = {}
for cadi_key, cap_key in CADI_FIELD_TO_CAP_MAP.items():
# sometimes they store data in HTML format.. need to escape
# chars
deposit['cadi_info'][cap_key] = parser.unescape(
entry.get(cadi_key, '') or '') or ''
deposit.commit()
print('Cadi entry {} updated.'.format(cadi_id))
except (DepositDoesNotExist, NoResultFound):
data = construct_cadi_entry(cadi_id, {
'cadi_info': {v: parser.unescape(entry.get(k, '') or '') or ''
for k, v in CADI_FIELD_TO_CAP_MAP.items()}
})
deposit = CAPDeposit.create(data=data)
add_read_permission_for_egroup(deposit, '*****@*****.**')
print('Cadi entry {} added.'.format(cadi_id))
def create_deposit_with_permissions(data, roles, users, owner, ana, errors):
"""Create a deposit and add privileges and owner information."""
from cap.modules.deposit.api import CAPDeposit
# make sure the schema is valid first
if not check_and_update_data_with_schema(data, ana):
return
with db.session.begin_nested():
try:
# saving original to return to user if wrong
_data = copy.deepcopy(data)
owner = get_existing_or_register_user(owner) if owner else None
deposit = CAPDeposit.create(data=data, owner=owner)
# add roles and users
if roles:
for role in roles:
_role = get_existing_or_register_role(role.strip())
deposit._add_egroup_permissions(_role, ['deposit-read'],
db.session)
if users:
for user in users:
_user = get_existing_or_register_user(user.strip())
deposit._add_user_permissions(_user, ['deposit-read'],
db.session)
deposit.commit()
except ValidationError as err:
click.secho(f'Validation Error: {err.message}', fg='red')
errors.append(_data)
return
db.session.commit()
click.secho(f"Created deposit with id: {deposit['_deposit']['id']}",
fg='green') # noqa
def _create_deposit(user, schema_name, metadata=None, experiment=None, publish=False):
"""
Create a new deposit for given user and schema name
e.g cms-analysis-v0.0.1,
with minimal metadata defined for this schema type.
"""
with app.test_request_context():
# create schema for record
schema = create_schema('records/{}'.format(schema_name), is_deposit=False,
experiment=experiment)
if not experiment:
schema.add_read_access_to_all()
# create schema for deposit
schema = create_schema('deposits/records/{}'.format(schema_name),
experiment=experiment)
if not experiment:
schema.add_read_access_to_all()
metadata = metadata or minimal_metadata(jsonschemas_host,
'deposits/records/{}'.format(schema_name))
login_user(user)
id_ = uuid4()
deposit_minter(id_, metadata)
deposit = Deposit.create(metadata, id_=id_)
if publish:
deposit.publish()
_, record = deposit.fetch_published()
RecordIndexer().index(record)
current_search.flush_and_refresh('records')
current_search.flush_and_refresh(schema.index_name)
return Deposit.get_record(deposit.id)
def synchronize_cadi_entries(limit=None):
"""Synchronize CMS analysis with CADI database.
Updates cadi_info in existing analysis.
If analysis with given CADI id doesn't exist yet,
new deposit will be created.
All members of CMS will get a r access and cms-admin egroup rw access.
:params int limit: number of entries to update
"""
def _cadi_deposit(cadi_id, cadi_info):
return {
'$ana_type': 'cms-analysis',
'cadi_info': cadi_info,
'general_title': cadi_info.get('name') or cadi_id,
'_fetched_from': 'cadi',
'_user_edited': False,
'basic_info': {
'cadi_id': cadi_id
}
}
entries = get_all_from_cadi()
for entry in islice(entries, limit):
deposit = None
cadi_info = cadi_serializer.dump(entry).data
cadi_id = cadi_info.pop('cadi_id')
try: # update if cadi deposit already exists
deposit = get_deposit_by_cadi_id(cadi_id)
if deposit.get('cadi_info') == cadi_info:
current_app.logger.info(f'No changes in entry {cadi_id}.')
else:
deposit['cadi_info'] = cadi_info
try:
deposit.commit()
except DepositValidationError:
current_app.logger.exception(
f'Error during updating cadi info in {cadi_id}.')
continue
db.session.commit()
current_app.logger.info(f'Cadi entry {cadi_id} updated.')
except DepositDoesNotExist:
try:
with db.session.begin_nested():
deposit = CAPDeposit.create(
data=_cadi_deposit(cadi_id, cadi_info))
# give read access to members of CMS experiment
deposit._add_experiment_permissions(
'CMS',
['deposit-read'],
)
# give admin access to the contact person (if in ldap)
owner = _get_owner_account(cadi_info['contact'])
if owner:
deposit._add_user_permissions(
owner,
[
'deposit-read',
'deposit-update',
'deposit-admin',
],
db.session,
)
# give admin access to cms admin egroups
admin_egroups = _get_admin_egroups(wg=cadi_id[:3])
for role in admin_egroups:
deposit._add_egroup_permissions(
role,
[
'deposit-read',
'deposit-update',
'deposit-admin',
],
db.session,
)
deposit.commit()
db.session.commit()
current_app.logger.info(f'Cadi entry {cadi_id} added.')
except Exception:
if deposit and deposit.id:
deposit.indexer.delete(deposit)
current_app.logger.exception(f'Error during adding {cadi_id}.')
