def load_submission(self,
record_information,
file_base_path,
submission_yaml_file_location,
update=False):
"""
:param record_information:
:param file_base_path:
:param files:
:return:
"""
# create publication record.
# load data tables
# create data table records (call finalise(recid))
admin_user_id = 1
# consume data payload and store in db.
get_or_create_hepsubmission(record_information["recid"], admin_user_id)
errors = process_submission_directory(file_base_path,
submission_yaml_file_location,
record_information["recid"],
update=update)
if len(errors) > 0:
print("ERRORS ARE: ")
print(errors)
if errors:
raise FailedSubmission(
"Submission failed for {0}.".format(
record_information["recid"]), errors,
record_information["recid"])
else:
return record_information["recid"]
def test_upload_max_size(app):
# Test uploading a file with size greater than UPLOAD_MAX_SIZE
app.config.update({'UPLOAD_MAX_SIZE': 1000000})
with app.app_context():
user = User.query.first()
login_user(user)
recid = '12345'
get_or_create_hepsubmission(recid, 1)
base_dir = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(base_dir, 'test_data/TestHEPSubmission.zip'),
"rb") as stream:
test_file = FileStorage(stream=stream,
filename="TestHEPSubmission.zip")
response, code = process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
assert (code == 413)
assert (response.json == {
'message':
'TestHEPSubmission.zip too large (1818265 bytes > 1000000 bytes)'
})
def load_submission(self, record_information, file_base_path,
submission_yaml_file_location, update=False):
"""
:param record_information:
:param file_base_path:
:param files:
:return:
"""
# create publication record.
# load data tables
# create data table records (call finalise(recid))
admin_user_id = 1
# consume data payload and store in db.
get_or_create_hepsubmission(record_information["recid"], admin_user_id)
errors = process_submission_directory(file_base_path,
submission_yaml_file_location,
record_information["recid"], update=update)
if len(errors) > 0:
print('ERRORS ARE: ')
print(errors)
if errors:
raise FailedSubmission("Submission failed for {0}.".format(
record_information['recid']), errors,
record_information['recid'])
else:
return record_information["recid"]
def consume_sandbox_payload():
# generate a unique id
import time
id = (int(current_user.get_id())) + int(round(time.time()))
get_or_create_hepsubmission(id, current_user.get_id(), status="sandbox")
file = request.files["hep_archive"]
return process_payload(id, file, "/record/sandbox/{}")
def consume_sandbox_payload():
"""
Creates a new sandbox submission with a new file upload
:param recid:
"""
import time
id = (int(current_user.get_id())) + int(round(time.time()))
get_or_create_hepsubmission(id, current_user.get_id(), status="sandbox")
file = request.files['hep_archive']
return process_payload(id, file, '/record/sandbox/{}')
def test_create_record_for_dashboard(app):
with app.app_context():
record_information = create_record({
'journal_info': 'Phys. Letts',
'title': 'My Journal Paper',
'inspire_id': '1487726'
})
get_or_create_hepsubmission(record_information['recid'])
record = get_record_by_id(record_information['recid'])
user = User(email='*****@*****.**',
password='******',
active=True,
id=101)
test_submissions = {}
create_record_for_dashboard(record['recid'], test_submissions, user)
assert (test_submissions == {
record_information['recid']: {
'metadata': {
'coordinator': {
'name': 'No coordinator'
},
'recid': record_information['recid'],
'role': ['coordinator'],
'start_date': record.created,
'title': u'My Journal Paper',
'versions': 1
},
'stats': {
'attention': 0,
'passed': 0,
'todo': 0
},
'status': 'todo'
}
})
test_submissions = {
record_information['recid']: {
"metadata": {
"role": []
}
}
}
create_record_for_dashboard(record['recid'], test_submissions, user)
assert (test_submissions == {
record_information['recid']: {
"metadata": {
"role": [['coordinator']]
}
}
})
def consume_sandbox_payload():
"""
Creates a new sandbox submission with a new file upload.
:param recid:
"""
if request.method == 'GET':
return redirect('/record/sandbox')
id = (int(current_user.get_id())) + int(round(time.time()))
get_or_create_hepsubmission(id, current_user.get_id(), status="sandbox")
file = request.files['hep_archive']
redirect_url = request.url_root + "record/sandbox/{}"
return process_payload(id, file, redirect_url)
def process_submission_payload(*args, **kwargs):
"""
Processes the submission payload.
:param inspire_id:
:param title:
:param reviewer:
:param uploader:
:param send_upload_email:
:return:
"""
if kwargs.get('inspire_id'):
content, status = get_inspire_record_information(kwargs.get('inspire_id'))
content["inspire_id"] = kwargs.get('inspire_id')
elif kwargs.get('title'):
content = {'title': kwargs.get('title')}
else:
raise ValueError(message="A title or inspire_id must be provided.")
record_information = create_record(content)
submitter_id = kwargs.get('submitter_id')
if submitter_id is None:
submitter_id = kwargs.get('user_id') if 'user_id' in kwargs else int(current_user.get_id())
hepsubmission = get_or_create_hepsubmission(record_information["recid"], submitter_id)
if kwargs.get('inspire_id'):
hepsubmission.inspire_id = kwargs.get('inspire_id')
db.session.add(hepsubmission)
reviewer_details = kwargs.get('reviewer')
reviewer = create_participant_record(
reviewer_details.get('name'),
reviewer_details.get('email'), 'reviewer', 'primary',
record_information['recid'])
hepsubmission.participants.append(reviewer)
uploader_details = kwargs.get('uploader')
uploader = create_participant_record(uploader_details.get('name'), uploader_details.get('email'),
'uploader', 'primary',
record_information['recid'])
hepsubmission.participants.append(uploader)
db.session.commit()
if kwargs.get('send_upload_email', True):
# Now Send Email only to the uploader first. The reviewer will be asked to
# review only when an upload has been performed.
message = kwargs.get('message', None)
send_cookie_email(uploader, record_information, message)
admin_idx = AdminIndexer()
admin_idx.index_submission(hepsubmission)
return hepsubmission
def test_prepare_submissions_admin(app, load_submission):
with app.app_context():
record_information = create_record({
'journal_info': 'Phys. Letts',
'title': 'My Journal Paper',
'inspire_id': '1487726'
})
get_or_create_hepsubmission(record_information['recid'])
record = get_record_by_id(record_information['recid'])
role = Role(name='admin')
user = User(email='*****@*****.**',
password='******',
active=True,
id=101,
roles=[role])
submissions = prepare_submissions(user)
assert (len(submissions) == 1)
assert (submissions[str(record_information['recid'])] == {
'metadata': {
'coordinator': {
'email': u'*****@*****.**',
'name': u'*****@*****.**',
'id': 1
},
'recid': str(record_information['recid']),
'role': ['coordinator'],
'show_coord_view': False,
'start_date': record.created,
'title': u'My Journal Paper',
'versions': 1
},
'stats': {
'attention': 0L,
'passed': 0L,
'todo': 0L
},
'status': u'todo'
})
def test_upload_invalid_file(app):
# Test uploading an invalid file
with app.app_context():
user = User.query.first()
login_user(user)
recid = '12345'
get_or_create_hepsubmission(recid, 1)
with StringIO("test") as stream:
test_file = FileStorage(stream=stream, filename="test.txt")
response, code = process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
assert (code == 400)
assert (response.json == {
'message':
'You must upload a .zip, .tar, .tar.gz or .tgz file'
' (or a .oldhepdata or single .yaml or .yaml.gz file).'
})
def test_has_upload_permissions(app):
# Test uploader permissions
with app.app_context():
# Create a record
recid = '12345'
get_or_create_hepsubmission(recid, 1)
# Check admin user has upload permissions to new record
admin_user = user = User.query.first()
assert has_upload_permissions(recid, admin_user)
# Create a user who is not admin and not associated with a record
user = User(email='*****@*****.**',
password='******',
active=True)
db.session.add(user)
db.session.commit()
login_user(user)
assert not has_upload_permissions(recid, user)
# Add the user as an uploader but not primary - should not be allowed
submission_participant = SubmissionParticipant(user_account=user.id,
publication_recid=recid,
email=user.email,
role='uploader')
db.session.add(submission_participant)
db.session.commit()
assert not has_upload_permissions(recid, user)
# Make the participant primary uploader - should now work
submission_participant.status = 'primary'
db.session.add(submission_participant)
db.session.commit()
assert has_upload_permissions(recid, user)
def process_submission_payload(*args, **kwargs):
"""
Processes the submission payload
:param inspire_id:
:param title:
:param reviewer:
:param uploader:
:param send_upload_email:
:return:
"""
if kwargs.get('inspire_id'):
content, status = get_inspire_record_information(kwargs.get('inspire_id'))
content["inspire_id"] = kwargs.get('inspire_id')
elif kwargs.get('title'):
content = {'title': kwargs.get('title')}
else:
raise ValueError(message="A title or inspire_id must be provided.")
record_information = create_record(content)
submitter_id = kwargs.get('submitter_id')
if submitter_id is None:
submitter_id = kwargs.get('user_id') if 'user_id' in kwargs else int(current_user.get_id())
hepsubmission = get_or_create_hepsubmission(record_information["recid"], submitter_id)
reviewer_details = kwargs.get('reviewer')
reviewer = create_participant_record(
reviewer_details.get('name'),
reviewer_details.get('email'), 'reviewer', 'primary',
record_information['recid'])
hepsubmission.participants.append(reviewer)
uploader_details = kwargs.get('uploader')
uploader = create_participant_record(uploader_details.get('name'), uploader_details.get('email'),
'uploader', 'primary',
record_information['recid'])
hepsubmission.participants.append(uploader)
db.session.commit()
if kwargs.get('send_upload_email', True):
# Now Send Email only to the uploader first. The reviewer will be asked to
# review only when an upload has been performed.
message = kwargs.get('message', None)
send_cookie_email(uploader, record_information, message)
return hepsubmission
def test_has_coordinator_permissions(app):
# Test coordinator permissions
with app.app_context():
recid = '12345'
hepsubmission = get_or_create_hepsubmission(recid, 1)
# Check admin user has coordinator permissions to new record
admin_user = user = User.query.first()
assert has_coordinator_permissions(recid, admin_user)
# Create a user who is not admin and not associated with a record
user = User(email='*****@*****.**',
password='******',
active=True)
db.session.add(user)
db.session.commit()
login_user(user)
assert not has_coordinator_permissions(recid, user)
# Add the user as an uploader - should not have permission
submission_participant = SubmissionParticipant(user_account=user.id,
publication_recid=recid,
email=user.email,
role='uploader')
db.session.add(submission_participant)
db.session.commit()
assert not has_coordinator_permissions(recid, user)
# Modify record to add this user as coordinator - should now work
hepsubmission.coordinator = user.get_id()
db.session.add(hepsubmission)
db.session.commit()
assert has_coordinator_permissions(recid, user)
def test_upload_valid_file_yaml_gz(app):
# Test uploading and processing a file for a record
with app.app_context():
base_dir = os.path.dirname(os.path.realpath(__file__))
user = User.query.first()
login_user(user)
recid = '1512299'
get_or_create_hepsubmission(recid, 1)
hepdata_submission = HEPSubmission.query.filter_by(
publication_recid=recid).first()
assert (hepdata_submission is not None)
assert (hepdata_submission.data_abstract is None)
assert (hepdata_submission.created < hepdata_submission.last_updated)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == 'todo')
with open(os.path.join(base_dir, 'test_data/1512299.yaml.gz'),
"rb") as stream:
test_file = FileStorage(stream=stream, filename="1512299.yaml.gz")
response = process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
assert (response.json == {'url': '/test_redirect_url'})
# Check the submission has been updated
hepdata_submission = HEPSubmission.query.filter_by(
publication_recid=recid).first()
assert (hepdata_submission.data_abstract.startswith(
'Unfolded differential decay rates of four kinematic variables'))
assert (hepdata_submission.created < hepdata_submission.last_updated)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == 'todo')
# Set the status to finished and try again, to check versioning
hepdata_submission.overall_status = 'finished'
db.session.add(hepdata_submission)
# Refresh user
user = User.query.first()
login_user(user)
with open(os.path.join(base_dir, 'test_data/1512299.yaml.gz'),
"rb") as stream:
test_file = FileStorage(stream=stream, filename="1512299.yaml.gz")
process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
# Check the submission has been updated
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=recid).order_by(
HEPSubmission.last_updated).all()
assert (len(hepdata_submissions) == 2)
assert (hepdata_submissions[0].version == 1)
assert (hepdata_submissions[0].overall_status == 'finished')
assert (hepdata_submissions[1].data_abstract.startswith(
'Unfolded differential decay rates of four kinematic variables'))
assert (hepdata_submissions[1].version == 2)
assert (hepdata_submissions[1].overall_status == 'todo')
def test_upload_valid_file(app):
# Test uploading and processing a file for a record
with app.app_context():
base_dir = os.path.dirname(os.path.realpath(__file__))
for i, status in enumerate(["todo", "sandbox"]):
user = User.query.first()
login_user(user)
recid = f'12345{i}'
get_or_create_hepsubmission(recid, 1, status=status)
hepdata_submission = HEPSubmission.query.filter_by(
publication_recid=recid).first()
assert (hepdata_submission is not None)
assert (hepdata_submission.data_abstract is None)
assert (hepdata_submission.created <
hepdata_submission.last_updated)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == status)
with open(
os.path.join(base_dir, 'test_data/TestHEPSubmission.zip'),
"rb") as stream:
test_file = FileStorage(stream=stream,
filename="TestHEPSubmission.zip")
response = process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
assert (response.json == {'url': '/test_redirect_url'})
# Check the submission has been updated
hepdata_submission = HEPSubmission.query.filter_by(
publication_recid=recid).first()
assert (hepdata_submission.data_abstract.startswith(
'CERN-LHC. Measurements of the cross section for ZZ production'
))
assert (hepdata_submission.created <
hepdata_submission.last_updated)
assert (hepdata_submission.version == 1)
assert (hepdata_submission.overall_status == status)
# Set the status to finished and try again, to check versioning
if status == "todo":
hepdata_submission.overall_status = 'finished'
db.session.add(hepdata_submission)
# Sleep before uploading new version to avoid dir name conflict
sleep(1)
# Refresh user
user = User.query.first()
login_user(user)
# Upload a new version
with open(
os.path.join(base_dir, 'test_data/TestHEPSubmission.zip'),
"rb") as stream:
test_file = FileStorage(stream=stream,
filename="TestHEPSubmission.zip")
process_payload(recid,
test_file,
'/test_redirect_url',
synchronous=True)
# Check the submission has been updated (overridden for a sandbox;
# new version for normal submission)
expected_versions = 2 if status == "todo" else 1
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=recid).order_by(
HEPSubmission.last_updated).all()
assert (len(hepdata_submissions) == expected_versions)
assert (hepdata_submissions[0].version == 1)
if status == "todo":
assert (hepdata_submissions[0].overall_status == 'finished')
assert (hepdata_submissions[-1].data_abstract.startswith(
'CERN-LHC. Measurements of the cross section for ZZ production'
))
assert (hepdata_submissions[-1].version == expected_versions)
assert (hepdata_submissions[-1].overall_status == status)
# Check that there are the expected number of subdirectories and
# zip files under the record's main path
# For status = 'todo' (standard submission) there will be 1 file
# and 1 dir for each of 2 versions; for the sandbox submission
# there will just be 1 file and 1 dir.
directory = get_data_path_for_record(
hepdata_submission.publication_recid)
assert (os.path.exists(directory))
filepaths = os.listdir(directory)
assert (len(filepaths) == 2 * expected_versions)
dir_count = 0
file_count = 0
for path in filepaths:
if os.path.isdir(os.path.join(directory, path)):
dir_count += 1
assert (re.match(r"\d{10}", path) is not None)
else:
file_count += 1
assert (re.match(r"HEPData-%s-v[12]-yaml.zip" % recid,
path) is not None)
assert (dir_count == expected_versions)
assert (file_count == expected_versions)
if status == "todo":
# Delete the v2 submission and check db and v2 files have been removed
unload_submission(hepdata_submission.publication_recid,
version=2)
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=recid).order_by(
HEPSubmission.last_updated).all()
assert (len(hepdata_submissions) == 1)
assert (hepdata_submissions[0].version == 1)
assert (hepdata_submissions[0].overall_status == 'finished')
filepaths = os.listdir(directory)
assert (len(filepaths) == 2)
assert (f"HEPData-12345{i}-v1-yaml.zip" in filepaths)
# Delete the submission and check everything has been removed
unload_submission(hepdata_submission.publication_recid, version=1)
hepdata_submissions = HEPSubmission.query.filter_by(
publication_recid=recid).order_by(
HEPSubmission.last_updated).all()
assert (len(hepdata_submissions) == 0)
assert (not os.path.exists(directory))
def test_submissions_participant(app, load_submission):
with app.app_context():
record_information = create_record({
'journal_info': 'Phys. Letts',
'title': 'My Journal Paper',
'inspire_id': '1487726'
})
hepsubmission = get_or_create_hepsubmission(
record_information['recid'])
db.session.add(hepsubmission)
user = User(email='*****@*****.**', password='******', active=True)
db.session.add(user)
db.session.commit()
# Check the user doesn't see the record before they are a participant
assert (get_submission_count(user) == 0)
assert (list_submission_titles(user) == [])
# Add the user as a participant
participant = SubmissionParticipant(
publication_recid=record_information['recid'],
role="uploader",
email='*****@*****.**',
status='primary',
user_account=user.id)
db.session.add(participant)
hepsubmission.participants.append(participant)
db.session.add(hepsubmission)
db.session.commit()
assert (get_submission_count(user) == 1)
assert (list_submission_titles(user) == [{
'id':
record_information['recid'],
'title':
'My Journal Paper'
}])
participant_submissions = prepare_submissions(user)
assert (len(participant_submissions) == 1)
assert (participant_submissions[str(record_information['recid'])] == {
'metadata': {
'coordinator': {
'email': u'*****@*****.**',
'name': u'*****@*****.**',
'id': 1
},
'recid': str(record_information['recid']),
'role': ['uploader'],
'show_coord_view': False,
'start_date': hepsubmission.created,
'last_updated': hepsubmission.last_updated,
'title': u'My Journal Paper',
'versions': 1
},
'stats': {
'attention': 0,
'passed': 0,
'todo': 0
},
'status': u'todo'
})
# Add a new submission as coordinator
record_information2 = create_record({
'journal_info': 'Another Journal',
'title': 'My New Journal Paper',
'inspire_id': '123456'
})
hepsubmission = get_or_create_hepsubmission(
record_information2['recid'], coordinator=user.id)
assert (get_submission_count(user) == 2)
assert (list_submission_titles(user) == [{
'id':
record_information2['recid'],
'title':
'My New Journal Paper'
}, {
'id':
record_information['recid'],
'title':
'My Journal Paper'
}])
all_submissions = prepare_submissions(user)
assert (len(all_submissions) == 2)
assert (all_submissions[str(record_information2['recid'])] == {
'metadata': {
'coordinator': {
'email': u'*****@*****.**',
'name': u'*****@*****.**',
'id': user.id
},
'recid': str(record_information2['recid']),
'role': ['coordinator'],
'show_coord_view': True,
'start_date': hepsubmission.created,
'last_updated': hepsubmission.last_updated,
'title': u'My New Journal Paper',
'versions': 1
},
'stats': {
'attention': 0,
'passed': 0,
'todo': 0
},
'status': u'todo'
})
# Check pagination
page1_submissions = prepare_submissions(user, 1)
assert (len(page1_submissions) == 1)
assert (page1_submissions[str(
record_information2['recid'])] == all_submissions[str(
record_information2['recid'])])
page2_submissions = prepare_submissions(user, 1, 2)
assert (len(page2_submissions) == 1)
assert (page2_submissions[str(
record_information['recid'])] == all_submissions[str(
record_information['recid'])])
# Check filtering by record id
record_submissions = prepare_submissions(
user, record_id=record_information2['recid'])
assert (len(record_submissions) == 1)
assert (record_submissions[str(
record_information2['recid'])] == all_submissions[str(
record_information2['recid'])])
# change status to 'finished' and check new submission no longer appears
hepsubmission.overall_status = 'finished'
db.session.add(hepsubmission)
db.session.commit()
assert (get_submission_count(user) == 1)
all_submissions = prepare_submissions(user)
assert (len(all_submissions) == 1)
assert (list(
all_submissions.keys()) == [str(record_information['recid'])])
def _import_record(inspire_id,
update_existing=False,
base_url='https://hepdata.net',
send_email=False):
publication_information, status = get_inspire_record_information(
inspire_id)
if status != "success":
log.error("Failed to retrieve publication information for " +
inspire_id)
return False
current_submission = get_latest_hepsubmission(inspire_id=inspire_id)
if not current_submission:
log.info(
"The record with id {0} does not exist in the database, so we're loading it."
.format(inspire_id))
publication_information["inspire_id"] = inspire_id
record_information = create_record(publication_information)
recid = record_information['recid']
else:
log.info("The record with inspire id {0} already exists.".format(
inspire_id))
if update_existing:
log.info("Updating instead")
recid = current_submission.publication_recid
else:
log.info("Not updating as update_existing is False")
return False
try:
download_path = _download_file(base_url, inspire_id)
filename = os.path.basename(download_path)
time_stamp = str(int(round(time.time())))
file_save_directory = get_data_path_for_record(str(recid), time_stamp)
if not os.path.exists(file_save_directory):
os.makedirs(file_save_directory)
file_path = os.path.join(file_save_directory, filename)
log.info("Moving file to %s" % file_path)
shutil.copy(download_path, file_path)
# Create submission
admin_user_id = 1
hepsubmission = get_or_create_hepsubmission(recid, admin_user_id)
db.session.add(hepsubmission)
db.session.commit()
# Then process the payload as for any other record
errors = process_zip_archive(file_path, recid)
if errors:
log.info("Errors processing archive. Re-trying with old schema.")
# Try again with old schema
# Need to clean up first to avoid errors
# First delete tables
cleanup_submission(recid, 1, [])
# Next remove remaining files
file_save_directory = os.path.dirname(file_path)
submission_path = os.path.join(file_save_directory,
remove_file_extension(filename))
shutil.rmtree(submission_path)
errors = process_zip_archive(file_path,
recid,
old_submission_schema=True,
old_data_schema=True)
if errors:
log.error("Could not process zip archive: ")
for file, file_errors in errors.items():
log.error(" %s:" % file)
for error in file_errors:
log.error(" %s" % error['message'])
raise ValueError("Could not validate record.")
# Delete any previous upload folders
cleanup_old_files(hepsubmission)
log.info("Finalising record %s" % recid)
result_json = do_finalise(recid,
force_finalise=True,
update=(current_submission is not None),
convert=False,
send_email=send_email)
result = json.loads(result_json)
if result and result['success']:
log.info("Imported record %s with %s submissions" %
(recid, result['data_count']))
return True
else:
raise ValueError("Failed to finalise record.")
except Exception as e:
# Unload record
unload_submission(recid)
log.error(e)
return False
def mock_import_old_record(inspire_id=mock_inspire_ids[1], send_email=False):
"""Creates a submission but mimics the old migrated paths. (See hepdata
master branch at ccd691b for old migrator module.)
"""
if inspire_id not in mock_inspire_ids:
raise ValueError('Invalid inspire id %s. Accepted values are: %s' %
(inspire_id, ', '.join(mock_inspire_ids)))
# Use zipped test data for specific record(s)
publication_information, status = get_inspire_record_information(
inspire_id)
publication_information["inspire_id"] = inspire_id
# Create record
if status == "success":
record_information = create_record(publication_information)
else:
log.error("Failed to retrieve publication information for " +
inspire_id)
return False
# Unzip into correct data dir
data_path = get_data_path_for_record(record_information['recid'])
base_dir = os.path.dirname(os.path.realpath(__file__))
zip_path = os.path.join(base_dir, 'old_hepdata_zips',
'ins%s.zip' % inspire_id)
if os.path.isfile(zip_path):
log.info('Unzipping %s to %s' % (zip_path, data_path))
shutil.unpack_archive(zip_path, data_path)
time_stamp = str(int(round(time.time())))
yaml_path = os.path.join(data_path, time_stamp)
sub_zip_path = os.path.join(data_path, 'ins%s.zip' % inspire_id)
shutil.unpack_archive(sub_zip_path, yaml_path)
else:
log.error('Invalid path %s' % zip_path)
return False
# Create submission
admin_user_id = 1
# Consume data payload and store in db.
get_or_create_hepsubmission(record_information["recid"], admin_user_id)
errors = process_submission_directory(yaml_path,
os.path.join(yaml_path,
"submission.yaml"),
record_information["recid"],
old_submission_schema=True,
old_data_schema=True)
if errors:
log.error(
"Submission failed for {0}.".format(record_information["recid"]),
errors, record_information["recid"])
return False
do_finalise(record_information['recid'],
publication_record=record_information,
force_finalise=True,
convert=False,
send_email=send_email)
def test_dashboard(live_server, logged_in_browser):
"""
Test dashboard functions
"""
browser = logged_in_browser
# Create some submissions so that there'll be something on the dashboard
# and on 2 pages. Current user will be coordinator and uploader.
for i in range(26):
content = {'title': f'Dashboard Test {i}'}
record_information = create_record(content)
hepsubmission = get_or_create_hepsubmission(record_information["recid"], 1)
participant_record = SubmissionParticipant(email='*****@*****.**',
status='primary',
role='uploader',
user_account=1,
publication_recid=record_information["recid"])
db.session.add(hepsubmission)
db.session.add(participant_record)
db.session.commit()
# Confirm there are 26 'todo' submissions
submissions = HEPSubmission.query \
.filter_by(overall_status='todo').all()
assert len(submissions) == 26
# Click on dashboard link
browser.find_element_by_link_text('Dashboard').click()
e2e_assert_url(browser, 'hep_dashboard.dashboard')
# Check links in top section work
# Submissions Overview link
browser.find_element_by_link_text('Submissions Overview').click()
e2e_assert_url(browser, 'hep_dashboard.submissions')
# Wait for graph to load
WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.CSS_SELECTOR, "#submission_vis svg"))
)
# Go back
browser.back()
e2e_assert_url(browser, 'hep_dashboard.dashboard')
# Edit Profile link
browser.find_element_by_link_text('Edit Profile').click()
e2e_assert_url(browser, 'invenio_userprofiles.profile')
# Go back
browser.back()
e2e_assert_url(browser, 'hep_dashboard.dashboard')
# Wait for submissions to load
submissions_list = WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.ID, "hep-submissions"))
)
submission_items = submissions_list.find_elements_by_class_name('submission-item')
assert len(submission_items) == 25
# Check pagination works
browser.find_element_by_css_selector(".pagination-bar a[href='/dashboard/?page=2']").click()
# Wait for loader, then new items appear
WebDriverWait(browser, 10).until(
EC.text_to_be_present_in_element(
(By.CSS_SELECTOR, '.submission-item h4 a'),
'Dashboard Test 0'
)
)
# Should just be 1 submission on page 2
submission_items = browser.find_elements_by_class_name('submission-item')
assert len(submission_items) == 1
# Check settings modal appears
submission_items[0].find_element_by_class_name('manage-submission-trigger').click()
manage_widget = WebDriverWait(browser, 10).until(
EC.visibility_of_element_located((By.ID, 'manageWidget'))
)
assert manage_widget.find_element_by_class_name('modal-title').text == 'Manage Submission'
# Close modal
manage_widget.find_element_by_css_selector('.modal-footer .btn-default').click()
WebDriverWait(browser, 10).until(
EC.invisibility_of_element(manage_widget)
)
# Click delete button
# Check settings modal appears
submission_items[0].find_element_by_class_name('delete-submission-trigger').click()
delete_widget = WebDriverWait(browser, 10).until(
EC.visibility_of_element_located((By.ID, 'deleteWidget'))
)
assert delete_widget.find_element_by_class_name('modal-title').text == 'Delete Submission'
# Confirm deletion
delete_widget.find_element_by_class_name('confirm-delete').click()
# Wait for confirmation of deletion
WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.ID, 'delete-success'))
)
assert 'Submission deleted' in \
delete_widget.find_element_by_css_selector('#delete-success p').text
# Should now be 25 submissions not 26
submissions = HEPSubmission.query \
.filter_by(overall_status='todo').all()
assert len(submissions) == 25
# Reload the dashboard (rather than waiting)
browser.refresh()
# Check permissions widget
# Coordinator tab should have 5 items (restricted as we are user id 1)
coordinator_pane = WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.ID, 'coordinator'))
)
coordinator_rows = coordinator_pane.find_elements_by_class_name('row')
assert len(coordinator_rows) == 5
# Click on uploader pane - should be all 25 items
browser.find_element_by_link_text('uploader').click()
uploader_pane = browser.find_element_by_id('uploader')
uploader_rows = uploader_pane.find_elements_by_class_name('row')
assert len(uploader_rows) == 25
# Only first 5 should be visible
assert all(row.is_displayed() for row in uploader_rows[:5])
assert all(not row.is_displayed() for row in uploader_rows[5:])
# Scroll down to find paginator
ActionChains(browser).move_to_element(uploader_rows[4]).perform()
# Click on last page
uploader_pane.find_element_by_css_selector(".pagination-bar li a[title=last]").click()
# Now last 5 items should be visible
assert all(not row.is_displayed() for row in uploader_rows[:20])
assert all(row.is_displayed() for row in uploader_rows[20:])
