def test_074_run_modifies_records_appropriately(self):
from invenio_checker.clients.master import StatusMaster
from invenio_checker.api import create_task, run_task
from invenio_records.api import get_record
# Given a task that forces re-running on unmodified records
task_data = TestApi.task_data
Query = get_Query(task_data)
create_task(task_data)
self.create_records(small_rng)
with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
with patch('invenio_checker.models.Query', Query):
task_id = run_task(task_data['name'])
# Ensure that it modifies the records as coded
execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
assert execution.status == StatusMaster.completed
for i in small_rng:
assert get_record(i)['field1'] == 3.4
with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
with patch('invenio_checker.models.Query', Query):
task_id = run_task(task_data['name'])
# And that it picks up the changes it did last time on the next run
execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
assert execution.status == StatusMaster.completed
for i in small_rng:
assert get_record(i)['field1'] == 3.4 * 2
def test_0101_run_reruns_once_record_has_been_modified(self):
from invenio_checker.clients.master import StatusMaster
from invenio_checker.api import create_task, run_task
from invenio_records.api import get_record
# Given a task that does not force re-running on unmodified records
task_data = dict(TestApi.task_data)
task_data['force_run_on_unmodified_records'] = False
Query = get_Query(task_data)
create_task(task_data)
self.create_records(small_rng)
with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
with patch('invenio_checker.models.Query', Query):
# Run once
task_id = run_task(task_data['name'])
execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
assert execution.status == StatusMaster.completed
# Mark first record as modified
rec = get_record(small_rng[0])
rec['unrelated_thing'] = 'unrelated_key'
rec.commit()
# Run again
task_id = run_task(task_data['name'])
execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
assert execution.status == StatusMaster.completed
assert get_record(small_rng[0])['field1'] == 3.4 * 2
assert get_record(small_rng[1])['field1'] == 3.4
def accessrequest(request_id):
"""Accept/reject access request."""
r = AccessRequest.get_by_receiver(request_id, current_user)
if not r or r.status != RequestStatus.PENDING:
abort(404)
form = ApprovalForm(request.form)
if form.validate_on_submit():
if form.accept.data:
r.accept(message=form.data['message'],
expires_at=form.expires_at.data)
flash(_("Request accepted."))
return redirect(url_for(".index"))
elif form.reject.data:
r.reject(message=form.data['message'])
flash(_("Request rejected."))
return redirect(url_for(".index"))
return render_template(
"accessrequests/settings/request.html",
accessrequest=r,
record=get_record(r.recid),
form=form,
)
def record_id_process(form, field, submit=False):
value = field.data or ''
if value == "" or value.isspace():
return
def is_number(s):
try:
float(s)
return True
except ValueError:
return False
if is_number(field.data):
json_reader = get_record(value)
else:
field.add_message("Record id must be a number!", state='error')
return
if json_reader is not None:
webdeposit_json = form.uncook_json(json_reader, {}, value)
#FIXME: update current json, past self, what do you mean?? :S
field.add_message('<a href="/record/"' + value +
'>Record</a> was loaded successfully',
state='info')
form.process(MultiDict(webdeposit_json))
else:
field.add_message("Record doesn't exist", state='info')
def export_as(ids, export_format):
"""
Export file according to format
:param ids: the ids of the records.
:param export_format: the type of the file the user needs to download.
:return: the downloaded file.
"""
ids = request.values.getlist('ids[]')
if len(ids) > cfg['EXPORT_LIMIT']:
ids = ids[:cfg['EXPORT_LIMIT']]
out = []
for id in ids:
record = get_record(id)
if export_format == 'bibtex':
results = Bibtex(record).format() + '\n' * 2
elif export_format in ('latex_eu', 'latex_us'):
results = Latex(record, export_format).format() + '\n'
elif export_format == 'cv_latex':
results = Cv_latex(record).format() + '\n'
elif export_format == 'cv_latex_html':
results = Cv_latex_html_text(record, export_format, '<br/>').format() + '\n'
elif export_format == 'cv_latex_text':
results = Cv_latex_html_text(record, export_format, '\n').format() + '\n'
generator = (cell for row in results
for cell in row)
out.append(generator)
output = itertools.chain()
for gen in out:
output = itertools.chain(output, gen)
return Response(output)
def tearDown(self):
from invenio_records.api import get_record
from invenio_search.api import Query
import time
# clear cache
inspire_cache.clear()
# undo changes in the record
record = get_record(self.changed_record_id)
record['references'] = self.old_references
record['titles'][0]['title'] = self.old_title
record.commit()
# make sure changes have been applied to ES and all citation_counts have been recalculated
timeout = 15 # [seconds]
timeout_start = time.time()
while time.time() < timeout_start + timeout:
es_record = Query('control_number:' + self.changed_record_id).search().records().pop()
references = [unicode(ref['recid']) for ref in es_record['references'] if ref.get('recid')]
title = es_record['titles'][0]['title']
if self.removed_reference in references \
and self.added_reference not in references \
and title == self.old_title:
break
else:
time.sleep(1)
update_citation_count_for_records.delay([int(self.removed_reference), int(self.added_reference)])
# make sure citation_counts have been updated
time.sleep(4)
def setUp(self):
from invenio_records.api import get_record
from invenio_search.api import Query
import copy
self.test_recid = u'1196797'
record = get_record(recid=self.test_recid)
self.removed_reference_recid = u'454197'
self.citation_count_before_update = Query('control_number:' + str(self.removed_reference_recid))\
.search().records()[0]['citation_count']
self.expected_citation_count_after_update = self.citation_count_before_update - 1
self.references_before_update = copy.deepcopy(record['references'])
# remove a reference
reference_to_remove = [ref for ref in record['references']
if ref.get('recid') and ref['recid'] == self.removed_reference_recid].pop()
record['references'].remove(reference_to_remove)
record.commit()
# we need to make sure the record has been already updated also in es
timeout = 15 # [seconds]
timeout_start = time.time()
while time.time() < timeout_start + timeout:
es_record = Query('control_number:' + self.test_recid).search().records()[0]
references = [ref['recid'] for ref in es_record['references'] if ref.get('recid')]
if self.removed_reference_recid not in references:
break
else:
time.sleep(1)
# wait a few seconds to make sure that the field citation_count of removed reference has been updated
time.sleep(6)
def record_id_process(form, field, submit=False):
value = field.data or ''
if value == "" or value.isspace():
return
def is_number(s):
try:
float(s)
return True
except ValueError:
return False
if is_number(field.data):
json_reader = get_record(value)
else:
field.add_message("Record id must be a number!", state='error')
return
if json_reader is not None:
webdeposit_json = form.uncook_json(json_reader, {}, value)
#FIXME: update current json, past self, what do you mean?? :S
field.add_message('<a href="/record/"' + value +
'>Record</a> was loaded successfully',
state='info')
form.process(MultiDict(webdeposit_json))
else:
field.add_message("Record doesn't exist", state='info')
def accessrequest(request_id):
"""Accept/reject access request."""
r = AccessRequest.get_by_receiver(request_id, current_user)
if not r or r.status != RequestStatus.PENDING:
abort(404)
form = ApprovalForm(request.form)
if form.validate_on_submit():
if form.accept.data:
r.accept(message=form.data['message'],
expires_at=form.expires_at.data)
flash(_("Request accepted."))
return redirect(url_for(".index"))
elif form.reject.data:
r.reject(message=form.data['message'])
flash(_("Request rejected."))
return redirect(url_for(".index"))
return render_template(
"accessrequests/settings/request.html",
accessrequest=r,
record=get_record(r.recid),
form=form,
)
def get_bibtex_file(recid):
record = get_record(recid)
results = Bibtex(record).format()
generator = (cell for row in results
for cell in row)
return Response(generator,
mimetype="text/plain",
headers={"Content-Disposition":
"attachment;filename=Bibtex%d.bib" % recid})
def test_0102_run_without_record_fixture_runs_once(self):
from invenio_checker.clients.master import StatusMaster
from invenio_checker.api import create_task, run_task
from invenio_records.api import get_record
# Given a task
task_data = TestApi.task_data
# Query = get_Query(task_data)
create_task(task_data)
self.create_records(small_rng)
with patch('invenio_checker.models.CheckerRule.filepath', filepath_without_record_fixture):
task_id = run_task(task_data['name'])
execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
assert execution.status == StatusMaster.completed
assert get_record(small_rng[0])['this_record_has_id_1'] == True
assert 'this_record_has_id_1' not in get_record(small_rng[1])
def get_restricted_collections_for_recid(recid, recreate_cache_if_needed=True):
"""Return the list of restricted collections to which recid belongs."""
from invenio_records.api import get_record
if recreate_cache_if_needed:
restricted_collection_cache.recreate_cache_if_needed()
record = get_record(recid)
return set(record.get("_collections", [])) & set([collection for collection in restricted_collection_cache.cache])
def get_record(recid):
"""Directly the record object corresponding to the recid."""
import warnings
warnings.warn('Deprecated get_record({}).'.format(str(recid)),
stacklevel=2)
from invenio_records import api
try:
return api.get_record(recid).legacy_create_recstruct()
except AttributeError:
return api.Record.create({'recid': recid}, 'json').legacy_create_recstruct()
def receive_after_insert(mapper, connection, target):
"""Check 'after_insert' signal for Tags."""
from invenio_records.api import get_record
record = get_record(target.id_bibrec)
if "files" in record:
for rec in record["files"]:
if "full_path" in rec and target.tag.name in cfg["CLOUDCONNECTOR_SERVICE_NAME_MAPPING"]:
# FIXME make the upload asynchronous
upload(target.tag.name, rec["full_path"], cfg["CLOUDCONNECTOR_UPLOAD_FOLDER"] + "/" + rec["full_path"])
def get_record(recid):
"""Directly the record object corresponding to the recid."""
import warnings
warnings.warn('Deprecated get_record({}).'.format(str(recid)),
stacklevel=2)
from invenio_records import api
try:
return api.get_record(recid).legacy_create_recstruct()
except AttributeError:
return api.Record.create({'recid': recid}, 'json').legacy_create_recstruct()
def is_record_in_any_collection(recID, recreate_cache_if_needed=True):
"""Return True if the record belongs to at least one collection.
This is a good, although not perfect, indicator to guess if webcoll has
already run after this record has been entered into the system.
"""
from invenio_records.api import get_record
warnings.warn("Use record['_collections'] directly.", DeprecationWarning,
stacklevel=2)
return bool(get_record(recID).get('_collections', []))
def access_request(recid=None):
"""Create an access request."""
record = get_record(recid)
# Record must be in restricted access mode.
if record.get('access_right') != 'restricted' or \
not record.get('access_conditions'):
abort(404)
# Record must have an owner and owner must still exists.
try:
record_owner = User.query.get_or_404(record['owner']['id'])
except KeyError:
abort(404)
sender = None
initialdata = dict()
# Prepare initial form data
if current_user.is_authenticated():
sender = current_user
initialdata = dict(
# FIXME: add full_name attribute to user accounts.
full_name=" ".join([sender["family_name"], sender["given_names"]]),
email=sender["email"],
)
# Normal form validation
form = AccessRequestForm(formdata=request.form, **initialdata)
if form.validate_on_submit():
accreq = AccessRequest.create(
recid=recid,
receiver=record_owner,
sender_full_name=form.data['full_name'],
sender_email=form.data['email'],
justification=form.data['justification'],
sender=sender
)
if accreq.status == RequestStatus.EMAIL_VALIDATION:
flash(_(
"Email confirmation needed: We have sent you an email to "
"verify your address. Please check the email and follow the "
"instructions to complete the access request."),
category='info')
else:
flash(_("Access request submitted."), category='info')
return redirect(url_for("record.metadata", recid=recid))
return render_template(
'accessrequests/access_request.html',
record=record,
form=form,
)
def test_records_dereference(self):
"""Record - Getting record with remote references"""
with patch('invenio_records.api.RecordMetadata') as mock_metadata:
from invenio_records.api import get_record
record_tuple = namedtuple('RecordTuple', ['json'])
url_prefix = current_app.config['CFG_SITE_URL']
fake_records = {
1: record_tuple({"param1": {"$ref": urlparse.urljoin(url_prefix, "record/2")},
"param2": {"$ref": urlparse.urljoin(url_prefix, "record/3")}}),
2: record_tuple({"param3": "test_param_3"}),
3: record_tuple({"param4": "test_param_4"}),
4: record_tuple({
"param_top": {
"param_bottom": {"$ref": urlparse.urljoin(url_prefix, "record/3")}
}
}),
5: record_tuple({
"param5": [
{"$ref": urlparse.urljoin(url_prefix, "record/3")}
]
})
}
mock_metadata.query = fake_records
record = get_record(1)
assert record['param1'] == fake_records[2].json
assert record['param2'] == fake_records[3].json
record = get_record(2)
assert record['param3'] == "test_param_3"
record = get_record(3)
assert record['param4'] == "test_param_4"
record = get_record(4)
assert record['param_top']['param_bottom']['param4'] == "test_param_4"
record = get_record(5)
assert record['param5'][0]['param4'] == "test_param_4"
def is_record_in_any_collection(recID, recreate_cache_if_needed=True):
"""Return True if the record belongs to at least one collection.
This is a good, although not perfect, indicator to guess if webcoll has
already run after this record has been entered into the system.
"""
from invenio_records.api import get_record
warnings.warn("Use record['_collections'] directly.",
DeprecationWarning,
stacklevel=2)
return bool(get_record(recID).get('_collections', []))
def test_077_run_on_non_record_centric_calls_check_function(self):
from invenio_checker.clients.master import StatusMaster
from invenio_checker.api import create_task, run_task
from invenio_records.api import get_record
# Given a task that does not force re-running on unmodified records
task_data = dict(TestApi.task_data)
task_data['force_run_on_unmodified_records'] = False
Query = get_Query(task_data)
create_task(task_data)
self.create_records([6000000, 6000001])
with patch('invenio_checker.models.CheckerRule.filepath', filepath_non_record_centric):
with patch('invenio_checker.models.Query', Query):
run_task(task_data['name'])
task_id = run_task(task_data['name'])
execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
assert execution.status == StatusMaster.completed
assert get_record(6000000)['a_field'] == 'a_value_2'
assert get_record(6000001)['a_field'] == 'another_value'
def access_request(recid=None):
"""Create an access request."""
record = get_record(recid)
# Record must be in restricted access mode.
if record.get('access_right') != 'restricted' or \
not record.get('access_conditions'):
abort(404)
# Record must have an owner and owner must still exists.
try:
record_owner = User.query.get_or_404(record['owner']['id'])
except KeyError:
abort(404)
sender = None
initialdata = dict()
# Prepare initial form data
if current_user.is_authenticated():
sender = current_user
initialdata = dict(
# FIXME: add full_name attribute to user accounts.
full_name=" ".join([sender["family_name"], sender["given_names"]]),
email=sender["email"],
)
# Normal form validation
form = AccessRequestForm(formdata=request.form, **initialdata)
if form.validate_on_submit():
accreq = AccessRequest.create(recid=recid,
receiver=record_owner,
sender_full_name=form.data['full_name'],
sender_email=form.data['email'],
justification=form.data['justification'],
sender=sender)
if accreq.status == RequestStatus.EMAIL_VALIDATION:
flash(_(
"Email confirmation needed: We have sent you an email to "
"verify your address. Please check the email and follow the "
"instructions to complete the access request."),
category='info')
else:
flash(_("Access request submitted."), category='info')
return redirect(url_for("record.metadata", recid=recid))
return render_template(
'accessrequests/access_request.html',
record=record,
form=form,
)
def export(collection, of, ot):
"""
Export requested records to defined output format.
It uses following request values:
* of (string): output format
* recid ([int]): list of record IDs
"""
# Get list of integers with record IDs.
recids = request.values.getlist('recid', type=int)
return response_formated_records([get_record(recid) for recid in recids],
collection, of, ot=ot)
def _store_record(record):
"""Update existing record via `control_number` or create new."""
if "control_number" in record:
existing_record = get_record(record['control_number'])
if existing_record is not None:
patch = jsonpatch.JsonPatch.from_diff(existing_record, record)
updated_record = existing_record.patch(patch=patch)
updated_record.commit()
else:
# New record with some hardcoded recid/control_number
create_record(data=record)
else:
create_record(data=record)
def receive_after_insert(mapper, connection, target):
"""Check 'after_insert' signal for Tags."""
from invenio_records.api import get_record
record = get_record(target.id_bibrec)
if 'files' in record:
for rec in record['files']:
if 'full_path' in rec and target.tag.name in \
cfg['CLOUDCONNECTOR_SERVICE_NAME_MAPPING']:
# FIXME make the upload asynchronous
upload(target.tag.name, rec['full_path'],
cfg['CLOUDCONNECTOR_UPLOAD_FOLDER'] + '/' +
rec['full_path'])
def _merge_record(obj, eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
# Get the current record, which contains all fields.
current_record = get_record(
sip.metadata.get('recid'), reset_cache=True
)
form_class = d.get_draft(draft_id).form_class
# Create a simplified record from the current record, that only
# contains fields concerning this deposition.
current_simple_record = deposition_record(
current_record,
[form_class],
pre_process_load=pre_process_load,
post_process_load=post_process_load,
process_export=partial(process_export, d),
)
# Create a simplified record from the changes the user have made.
changed_simple_record = make_record(sip.metadata, is_dump=True)
# Make an initial patch of current record (e.g. some default values set
# by the form, might not exists in the current record)
for k in current_simple_record:
if k not in current_record:
current_record[k] = current_simple_record[k]
# Export clean dumps
current_simple_json = current_simple_record.dumps(clean=True)
changed_simple_json = changed_simple_record.dumps(clean=True)
current_full_json = current_record.dumps(clean=True)
# Merge changes from changed record into the current record.
sip.metadata = merge_func(
d,
current_full_json,
current_simple_json,
changed_simple_json,
)
# Ensure we are based on latest version_id to prevent being rejected in
# the bibupload queue.
hst_record = HstRECORD.query.filter_by(
id_bibrec=sip.metadata.get('recid')
).order_by(HstRECORD.job_date.desc()).first()
sip.metadata['modification_date'] = hst_record.job_date.isoformat()
d.update()
def _merge_record(obj, eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
# Get the current record, which contains all fields.
current_record = get_record(
sip.metadata.get('recid'), reset_cache=True
)
form_class = d.get_draft(draft_id).form_class
# Create a simplified record from the current record, that only
# contains fields concerning this deposition.
current_simple_record = deposition_record(
current_record,
[form_class],
pre_process_load=pre_process_load,
post_process_load=post_process_load,
process_export=partial(process_export, d),
)
# Create a simplified record from the changes the user have made.
changed_simple_record = make_record(sip.metadata, is_dump=True)
# Make an initial patch of current record (e.g. some default values set
# by the form, might not exists in the current record)
for k in current_simple_record:
if k not in current_record:
current_record[k] = current_simple_record[k]
# Export clean dumps
current_simple_json = current_simple_record.dumps(clean=True)
changed_simple_json = changed_simple_record.dumps(clean=True)
current_full_json = current_record.dumps(clean=True)
# Merge changes from changed record into the current record.
sip.metadata = merge_func(
d,
current_full_json,
current_simple_json,
changed_simple_json,
)
# Ensure we are based on latest version_id to prevent being rejected in
# the bibupload queue.
hst_record = HstRECORD.query.filter_by(
id_bibrec=sip.metadata.get('recid')
).order_by(HstRECORD.job_date.desc()).first()
sip.metadata['modification_date'] = hst_record.job_date.isoformat()
d.update()
def get_latex_file(recid, latex_format):
record = get_record(recid)
results = Latex(record, latex_format).format()
generator = (cell for row in results
for cell in row)
return Response(
generator,
mimetype="text/plain",
headers={
"Content-Disposition": "attachment;filename=Latex_{0}{1}.tex".format(
latex_format, recid
)
}
)
def receive_after_insert(mapper, connection, target):
"""Check 'after_insert' signal for Tags."""
from invenio_records.api import get_record
record = get_record(target.id_bibrec)
if 'files' in record:
for rec in record['files']:
if 'full_path' in rec and target.tag.name in \
cfg['CLOUDCONNECTOR_SERVICE_NAME_MAPPING']:
# FIXME make the upload asynchronous
upload(
target.tag.name, rec['full_path'],
cfg['CLOUDCONNECTOR_UPLOAD_FOLDER'] + '/' +
rec['full_path'])
def export(collection, of, ot):
"""
Export requested records to defined output format.
It uses following request values:
* of (string): output format
* recid ([int]): list of record IDs
"""
# Get list of integers with record IDs.
recids = request.values.getlist('recid', type=int)
return response_formated_records([get_record(recid) for recid in recids],
collection,
of,
ot=ot)
def _modify_record(self, recid, test_func, replace_func, include_func,
append_colls=[], replace_colls=[]):
"""Generate record a MARCXML file.
@param test_func: Function to test if a collection id should be changed
@param replace_func: Function to replace the collection id.
@param include_func: Function to test if collection should be included
"""
rec = get_record(recid).legacy_create_recstruct()
newcolls = []
dirty = False
try:
colls = rec['980']
if replace_colls:
for c in replace_colls:
newcolls.append([('a', c)])
dirty = True
else:
for c in colls:
try:
# We are only interested in subfield 'a'
code, val = c[0][0]
if test_func(code, val):
c[0][0] = replace_func(code, val)
dirty = True
if include_func(code, val):
newcolls.append(c[0])
else:
dirty = True
except IndexError:
pass
for c in append_colls:
newcolls.append([('a', c)])
dirty = True
except KeyError:
return False
if not dirty:
return False
rec = {}
record_add_field(rec, '001', controlfield_value=str(recid))
for subfields in newcolls:
record_add_field(rec, '980', subfields=subfields)
return rec
def update_records_citations(new_citations):
"""Gets a set of records that need to be updated
and querys database to get all citations for each record.
Saves this ciattions on a new set and updates the record.
"""
citees = set()
for id in new_citations:
cit = Citation.query.filter_by(citer=id).all()
for rec in cit:
citees.add(rec.citee)
rec = get_record(id)
if rec is not None:
rec.update({"references_id": list(citees)})
else:
current_app.logger.exception(
"citations: record with id:%d not found", id)
citees.clear()
def _merge_record(obj, eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
# Get the current record, which contains all fields.
current_record = get_record(
sip.metadata.get('recid'), reset_cache=True
)
form_class = d.get_draft(draft_id).form_class
# Create a simplified record from the current record, that only
# contains fields concerning this deposition.
current_simple_record = deposition_record(
current_record,
[form_class],
pre_process_load=pre_process_load,
post_process_load=post_process_load,
process_export=partial(process_export, d),
)
# Create a simplified record from the changes the user have made.
changed_simple_record = make_record(sip.metadata, is_dump=True)
# Make an initial patch of current record (e.g. some default values set
# by the form, might not exists in the current record)
for k in current_simple_record:
if k not in current_record:
current_record[k] = current_simple_record[k]
# Export clean dumps
current_simple_json = current_simple_record.dumps(clean=True)
changed_simple_json = changed_simple_record.dumps(clean=True)
current_full_json = current_record.dumps(clean=True)
# Merge changes from changed record into the current record.
sip.metadata = merge_func(
d,
current_full_json,
current_simple_json,
changed_simple_json,
)
# TODO check sip.metadata['modification_date']
d.update()
def _merge_record(obj, eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
# Get the current record, which contains all fields.
current_record = get_record(
sip.metadata.get('recid'), reset_cache=True
)
form_class = d.get_draft(draft_id).form_class
# Create a simplified record from the current record, that only
# contains fields concerning this deposition.
current_simple_record = deposition_record(
current_record,
[form_class],
pre_process_load=pre_process_load,
post_process_load=post_process_load,
process_export=partial(process_export, d),
)
# Create a simplified record from the changes the user have made.
changed_simple_record = make_record(sip.metadata, is_dump=True)
# Make an initial patch of current record (e.g. some default values set
# by the form, might not exists in the current record)
for k in current_simple_record:
if k not in current_record:
current_record[k] = current_simple_record[k]
# Export clean dumps
current_simple_json = current_simple_record.dumps(clean=True)
changed_simple_json = changed_simple_record.dumps(clean=True)
current_full_json = current_record.dumps(clean=True)
# Merge changes from changed record into the current record.
sip.metadata = merge_func(
d,
current_full_json,
current_simple_json,
changed_simple_json,
)
# TODO check sip.metadata['modification_date']
d.update()
def datacite_register(recid):
"""
Register a DOI for new publication
If it fails, it will retry every 10 minutes for 1 hour.
"""
record = get_record(recid)
if record is None:
logger.debug("Record %s not found" % recid)
return
doi_val = record.get(cfg['PIDSTORE_DATACITE_RECORD_DOI_FIELD'], None)
logger.debug("Found DOI %s in record %s" % (doi_val, recid))
pid = PersistentIdentifier.get("doi", doi_val)
if not pid:
logger.debug("DOI not locally managed.")
return
else:
logger.debug("DOI locally managed.")
if not pid.has_object("rec", recid):
raise Exception(
"DOI %s is not assigned to record %s." % (doi_val, recid))
if pid.is_new() or pid.is_reserved():
logger.info("Registering DOI %s for record %s" % (doi_val, recid))
url = "%s/record/%s" % (
cfg.get('PIDSTORE_DATACITE_SITE_URL', cfg['CFG_SITE_URL']),
recid
)
doc = format_record(record, cfg['PIDSTORE_DATACITE_OUTPUTFORMAT'])
if not pid.register(url=url, doc=doc):
m = "Failed to register DOI %s" % doi_val
logger.error(m + "\n%s\n%s" % (url, doc))
if not datacite_register.request.is_eager:
raise datacite_register.retry(exc=Exception(m))
else:
logger.info("Successfully registered DOI %s." % doi_val)
def get_record_documents(recid, filename):
"""Yield LegacyBibDoc files from Documents."""
from invenio_records.api import get_record
from invenio_documents.api import Document
from invenio.legacy.bibdocfile.api import decompose_file
record = get_record(recid)
duuids = [uuid for (k, uuid) in record.get('_documents', [])
if k == filename]
for duuid in duuids:
document = Document.get_document(duuid)
if not document.is_authorized(current_user):
current_app.logger.info(
"Unauthorized access to /{recid}/files/{filename} "
"({document}) by {current_user}".format(
recid=recid, filename=filename, document=document,
current_user=current_user))
continue
if document.get('linked', False) and (
document.get('uri').startswith('http://') or
document.get('uri').startswith('https://')):
url = document.get('uri')
else:
url = url_for('record.file', recid=recid, filename=filename)
(dummy, name, superformat) = decompose_file(filename)
class LegacyBibDoc(object):
def __init__(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
def get_full_path(self):
return document.get('uri')
def get_recid(self):
return recid
yield LegacyBibDoc(name=name, superformat=superformat, url=url)
def test_075_run_with_dry_run_does_not_modify_records(self):
from invenio_checker.clients.master import StatusMaster
from invenio_checker.api import create_task, run_task
from invenio_records.api import get_record
# Given a task that forces re-running on unmodified records
task_data = TestApi.task_data
Query = get_Query(task_data)
create_task(task_data)
self.create_records(small_rng)
with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
with patch('invenio_checker.models.Query', Query):
task_id = run_task(task_data['name'], dry_run=True)
# Ensure that it modifies the records as coded
execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
assert execution.status == StatusMaster.completed
for i in small_rng:
assert 'field1' not in get_record(i)
def is_sip_uploaded(sip, record=None):
"""Check if a submission information package for a record has been uploaded."""
if not sip.is_sealed():
return False
if record is None:
record = get_record(sip.metadata.get('recid'), reset_cache=True)
sip_version_id = sip.metadata.get('modification_date')
if sip_version_id:
sip_version_id = dateutil.parser.parse(sip_version_id)
record_version_id = record.get('modification_date') if record else None
# Check of record in latest SIP has been uploaded (record version must
# be newer than SIP record version.
if record_version_id is None or (sip_version_id and
sip_version_id >= record_version_id):
return False
else:
return True
def is_sip_uploaded(sip, record=None):
"""Check if a SIP for a record has been uploaded."""
if not sip.is_sealed():
return False
if record is None:
record = get_record(sip.metadata.get('recid'), reset_cache=True)
sip_version_id = sip.metadata.get('modification_date')
if sip_version_id:
sip_version_id = dateutil.parser.parse(sip_version_id)
record_version_id = record.get('modification_date') if record else None
# Check of record in latest SIP has been uploaded (record version must
# be newer than SIP record version.
if record_version_id is None or (sip_version_id and
sip_version_id >= record_version_id):
return False
else:
return True
def test_0100_run_does_not_run_twice_on_records_that_did_not_change_since_last_run(self):
from invenio_checker.clients.master import StatusMaster
from invenio_checker.api import create_task, run_task
from invenio_records.api import get_record
# Given a task that does not force re-running on unmodified records
task_data = dict(TestApi.task_data)
task_data['force_run_on_unmodified_records'] = False
Query = get_Query(task_data)
create_task(task_data)
self.create_records(small_rng)
with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
with patch('invenio_checker.models.Query', Query):
run_task(task_data['name'])
task_id = run_task(task_data['name'])
execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
assert execution.status == StatusMaster.completed
for i in small_rng:
assert get_record(i)['field1'] == 3.4
def from_recid(cls, recid, provisional=False):
"""Get user communities specified in recid."""
rec = get_record(recid).legacy_create_recstruct()
prefix = "%s-" % (
cfg['COMMUNITIES_ID_PREFIX_PROVISIONAL']
if provisional else cfg['COMMUNITIES_ID_PREFIX'])
colls = rec.get('980', [])
usercomm = []
for c in colls:
try:
# We are only interested in subfield 'a'
code, val = c[0][0]
if code == 'a' and val.startswith(prefix):
val = val[len(prefix):]
u = cls.query.filter_by(id=val).first()
if u:
usercomm.append(u)
except IndexError:
pass
return usercomm
def tearDown(self):
from invenio_search.api import Query
from invenio_records.api import get_record
record = get_record(recid=self.test_recid)
record['references'] = self.references_before_update
record.commit()
# we need to make sure the record has been already restored to initial state also in es
timeout = 15 # [seconds]
timeout_start = time.time()
while time.time() < timeout_start + timeout:
es_record = Query('control_number:' + self.test_recid).search().records().pop()
references = [ref['recid'] for ref in es_record['references'] if ref.get('recid')]
if self.removed_reference_recid in references:
break
else:
time.sleep(1)
update_citation_count_for_records.delay([self.removed_reference_recid])
def datacite_sync(recid):
"""
Check DOI in DataCite.
"""
record = get_record(recid)
if record is None:
logger.debug("Record %s not found" % recid)
return
doi_val = record.get(cfg['PIDSTORE_DATACITE_RECORD_DOI_FIELD'], None)
logger.debug("Found DOI %s in record %s" % (doi_val, recid))
pid = PersistentIdentifier.get("doi", doi_val)
if not pid:
logger.debug("DOI not locally managed.")
return
else:
logger.debug("DOI locally managed.")
if pid.sync_status():
logger.info("Successfully synchronized DOI %s." % doi_val)
def datacite_delete(recid):
"""
Delete DOI in DataCite
If it fails, it will retry every 10 minutes for 1 hour.
"""
record = get_record(recid)
if record is None:
logger.debug("Record %s not found" % recid)
return
doi_val = record.get(cfg['PIDSTORE_DATACITE_RECORD_DOI_FIELD'], None)
logger.debug("Found DOI %s in record %s" % (doi_val, recid))
pid = PersistentIdentifier.get("doi", doi_val)
if not pid:
logger.debug("DOI not locally managed.")
return
else:
logger.debug("DOI locally managed.")
if not pid.has_object("rec", recid):
raise Exception(
"DOI %s is not assigned to record %s." % (doi_val, recid))
if pid.is_registered():
logger.info("Inactivating DOI %s for record %s" % (doi_val, recid))
if not pid.delete():
m = "Failed to inactive DOI %s" % doi_val
logger.error(m)
if not datacite_delete.request.is_eager:
raise datacite_delete.retry(exc=Exception(m))
else:
logger.info("Successfully inactivated DOI %s." % doi_val)
def record(self):
"""Get the Invenio record in JSON format.
:return: an instance of a record in JSON format
"""
return get_record(self.id_bibrec)
def getfile(req, form):
args = wash_urlargd(form,
bibdocfile_templates.files_default_urlargd)
ln = args['ln']
_ = gettext_set_language(ln)
uid = getUid(req)
user_info = collect_user_info(req)
verbose = args['verbose']
if verbose >= 1 and not isUserSuperAdmin(user_info):
# Only SuperUser can see all the details!
verbose = 0
if uid == -1 or CFG_ACCESS_CONTROL_LEVEL_SITE > 1:
return page_not_authorized(req,
"/%s/%s" %
(CFG_SITE_RECORD, self.recid),
navmenuid='submit')
if record_exists(self.recid) < 1:
msg = "<p>%s</p>" % _(
"Requested record does not seem to exist.")
return warning_page(msg, req, ln)
if record_empty(get_record(self.recid).legacy_create_recstruct()):
msg = "<p>%s</p>" % _(
"Requested record does not seem to have been integrated.")
return warning_page(msg, req, ln)
(auth_code,
auth_message) = check_user_can_view_record(user_info, self.recid)
if auth_code and user_info['email'] == 'guest':
cookie = mail_cookie_create_authorize_action(
VIEWRESTRCOLL, {
'collection':
guess_primary_collection_of_a_record(self.recid)
})
target = CFG_SITE_SECURE_URL + '/youraccount/login' + \
make_canonical_urlargd({'action': cookie, 'ln' : ln, 'referer' : \
CFG_SITE_SECURE_URL + user_info['uri']}, {})
return redirect_to_url(req, target, norobot=True)
elif auth_code:
return page_not_authorized(req, "../", \
text = auth_message)
readonly = CFG_ACCESS_CONTROL_LEVEL_SITE == 1
# From now on: either the user provided a specific file
# name (and a possible version), or we return a list of
# all the available files. In no case are the docids
# visible.
try:
bibarchive = BibRecDocs(self.recid)
except InvenioBibDocFileError:
register_exception(req=req, alert_admin=True)
msg = "<p>%s</p><p>%s</p>" % (
_("The system has encountered an error in retrieving the list of files for this document."
),
_("The error has been logged and will be taken in consideration as soon as possible."
))
return warning_page(msg, req, ln)
if bibarchive.deleted_p():
req.status = apache.HTTP_GONE
return warning_page(
_("Requested record does not seem to exist."), req, ln)
docname = ''
docformat = ''
version = ''
warn = ''
if filename:
# We know the complete file name, guess which docid it
# refers to
## TODO: Change the extension system according to ext.py from setlink
## and have a uniform extension mechanism...
docname = file_strip_ext(filename)
docformat = filename[len(docname):]
if docformat and docformat[0] != '.':
docformat = '.' + docformat
if args['subformat']:
docformat += ';%s' % args['subformat']
else:
docname = args['docname']
if not docformat:
docformat = args['format']
if args['subformat']:
docformat += ';%s' % args['subformat']
if not version:
version = args['version']
## Download as attachment
is_download = False
if args['download']:
is_download = True
# version could be either empty, or all or an integer
try:
int(version)
except ValueError:
if version != 'all':
version = ''
display_hidden = isUserSuperAdmin(user_info)
if version != 'all':
# search this filename in the complete list of files
for doc in bibarchive.list_bibdocs():
if docname == bibarchive.get_docname(doc.id):
try:
try:
docfile = doc.get_file(docformat, version)
except InvenioBibDocFileError as msg:
req.status = apache.HTTP_NOT_FOUND
if not CFG_INSPIRE_SITE and req.headers_in.get(
'referer'):
## There must be a broken link somewhere.
## Maybe it's good to alert the admin
register_exception(req=req,
alert_admin=True)
warn += write_warning(
_("The format %(x_form)s does not exist for the given version: %(x_vers)s",
x_form=cgi.escape(docformat),
x_vers=cgi.escape(str(msg))))
break
(auth_code,
auth_message) = docfile.is_restricted(user_info)
if auth_code != 0 and not is_user_owner_of_record(
user_info, self.recid):
if CFG_BIBDOCFILE_ICON_SUBFORMAT_RE.match(
get_subformat_from_format(docformat)):
return stream_restricted_icon(req)
if user_info['email'] == 'guest':
cookie = mail_cookie_create_authorize_action(
'viewrestrdoc',
{'status': docfile.get_status()})
target = CFG_SITE_SECURE_URL + '/youraccount/login' + \
make_canonical_urlargd({'action': cookie, 'ln' : ln, 'referer' : \
CFG_SITE_SECURE_URL + user_info['uri']}, {})
redirect_to_url(req, target)
else:
req.status = apache.HTTP_UNAUTHORIZED
warn += write_warning(
_("This file is restricted: ") +
str(auth_message))
break
if not docfile.hidden_p():
if not readonly:
ip = str(req.remote_ip)
doc.register_download(
ip, docfile.get_version(), docformat,
uid, self.recid)
try:
return docfile.stream(req,
download=is_download)
except InvenioBibDocFileError as msg:
register_exception(req=req,
alert_admin=True)
req.status = apache.HTTP_INTERNAL_SERVER_ERROR
warn += write_warning(
_("An error has happened in trying to stream the request file."
))
else:
req.status = apache.HTTP_UNAUTHORIZED
warn += write_warning(
_("The requested file is hidden and can not be accessed."
))
except InvenioBibDocFileError as msg:
register_exception(req=req, alert_admin=True)
if docname and docformat and not warn:
req.status = apache.HTTP_NOT_FOUND
warn += write_warning(
_("Requested file does not seem to exist."))
# filelist = bibarchive.display("", version, ln=ln, verbose=verbose, display_hidden=display_hidden)
filelist = bibdocfile_templates.tmpl_display_bibrecdocs(
bibarchive,
"",
version,
ln=ln,
verbose=verbose,
display_hidden=display_hidden)
t = warn + bibdocfile_templates.tmpl_filelist(ln=ln,
filelist=filelist)
cc = guess_primary_collection_of_a_record(self.recid)
cc_id = Collection.query.filter_by(name=cc).value('id')
unordered_tabs = None # get_detailed_page_tabs(cc_id, self.recid, ln)
ordered_tabs_id = [(tab_id, values['order'])
for (tab_id,
values) in iteritems(unordered_tabs)]
ordered_tabs_id.sort(lambda x, y: cmp(x[1], y[1]))
link_ln = ''
if ln != CFG_SITE_LANG:
link_ln = '?ln=%s' % ln
tabs = [
(unordered_tabs[tab_id]['label'], '%s/%s/%s/%s%s' %
(CFG_SITE_URL, CFG_SITE_RECORD, self.recid, tab_id, link_ln),
tab_id == 'files', unordered_tabs[tab_id]['enabled'])
for (tab_id, dummy_order) in ordered_tabs_id
if unordered_tabs[tab_id]['visible'] is True
]
tabs_counts = {} # get_detailed_page_tabs_counts(self.recid)
top = webstyle_templates.detailed_record_container_top(
self.recid,
tabs,
args['ln'],
citationnum=tabs_counts['Citations'],
referencenum=tabs_counts['References'],
discussionnum=tabs_counts['Discussions'])
bottom = webstyle_templates.detailed_record_container_bottom(
self.recid, tabs, args['ln'])
title, description, keywords = websearch_templates.tmpl_record_page_header_content(
req, self.recid, args['ln'])
return pageheaderonly(title=title,
navtrail=create_navtrail_links(cc=cc, aas=0, ln=ln) + \
''' > <a class="navtrail" href="%s/%s/%s">%s</a>
> %s''' % \
(CFG_SITE_URL, CFG_SITE_RECORD, self.recid, title, _("Access to Fulltext")),
description=description,
keywords=keywords,
uid=uid,
language=ln,
req=req,
navmenuid='search',
navtrail_append_title_p=0) + \
websearch_templates.tmpl_search_pagestart(ln) + \
top + t + bottom + \
websearch_templates.tmpl_search_pageend(ln) + \
pagefooteronly(language=ln, req=req)
def metadata(self):
"""Return record metadata."""
from invenio_records.api import get_record
return get_record(self.recid).dumps()
def _load_record(obj, eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=True)
record = get_record(sip.metadata.get('recid'), reset_cache=True)
if not is_sip_uploaded(sip, record=record):
if getattr(request, 'is_api_request', False):
d.set_render_context(dict(
response=dict(
message="Conflict",
status=409,
errors="Upload not yet fully integrated. Please wait"
" a few moments.",
),
status=409,
))
else:
from flask import flash
flash(
"Editing is only possible after your upload have been"
" fully integrated. Please wait a few moments, then try"
" to reload the page.",
category='warning'
)
d.set_render_context(dict(
template_name_or_list="deposit/completed.html",
deposition=d,
deposition_type=(
None if d.type.is_default() else
d.type.get_identifier()
),
uuid=d.id,
sip=sip,
my_depositions=Deposition.get_depositions(
current_user, type=d.type
),
format_record=format_record,
))
d.update()
eng.halt("Wait for record to be uploaded")
# Check if record is already loaded, if so, skip.
if d.drafts:
eng.jumpCallForward(1)
# Load draft
draft = d.get_or_create_draft(draft_id)
# Fill draft with values from recjson
record_to_draft(
record, draft=draft, post_process=post_process, producer=producer
)
d.update()
# Stop API request
if getattr(request, 'is_api_request', False):
d.set_render_context(dict(
response=d.marshal(),
status=201,
))
eng.halt("API request")