def dropbox_upload(pub_id=None, fileurl=''):
"""
Dropbox upload backend
"""
if pub_id:
pub_id = pub_id.encode('utf8')
if fileurl:
fileurl = fileurl.encode('utf8')
uid = current_user.get_id()
if not fileurl:
abort(400)
if not (fileurl.startswith("https://dl.dropbox.com/") or fileurl.startswith("https://dl.dropboxusercontent.com/")):
abort(400)
publication = OpenAIREPublication(uid)
if not is_editable(publication):
flash("You cannot upload new files when your upload has already been submitted!")
abort(400)
# Pre-fill user collection
c = request.values.get('c', None)
if c:
publication.add_usercollection(c)
uploaded_file = download_external_url(fileurl)
publication.add_a_fulltext(uploaded_file, secure_filename(os.path.basename(fileurl)))
return redirect(url_for('deposit.edit', pub_id=publication.publicationid))
def getfile(pub_id='', file_id='', action='view'):
"""
View for stream file or deleting it.
"""
pub_id = pub_id.encode('utf8')
file_id = file_id.encode('utf8')
action = action.encode('utf8')
uid = current_user.get_id()
if action not in ['view', 'delete']:
abort(404)
try:
pub = OpenAIREPublication(uid, pub_id)
fulltext = pub.fulltexts[file_id]
except (ValueError, KeyError):
abort(404)
if action == 'view':
return send_file(fulltext.get_full_path(),
attachment_filename=fulltext.get_full_name(),
as_attachment=True)
elif action == 'delete':
if not is_editable(pub):
flash("You cannot delete files when your upload has already been submitted!")
return redirect(url_for('.edit', pub_id=pub.publicationid))
if len(pub.fulltexts.keys()) > 1:
if pub.remove_a_fulltext(file_id):
flash("File was deleted", category='success')
else:
flash("File could not be deleted. Please contact support.", category='danger')
else:
flash("File cannot be deleted. You must provide minimum one file.")
return redirect(url_for('.edit', pub_id=pub.publicationid))
def upload(pub_id=None):
"""
PLUpload backend
"""
if pub_id:
pub_id = pub_id.encode('utf8')
uid = current_user.get_id()
if 'file' not in request.files:
abort(400)
afile = request.files['file']
filename = secure_filename(afile.filename)
publication = OpenAIREPublication(uid, publicationid=pub_id)
# Pre-fill user collection:
c = request.values.get('c', None)
if c:
publication.add_usercollection(c)
if not is_editable(publication):
flash("You cannot upload new files when your upload has already been submitted!")
abort(400)
publication.add_a_fulltext(None, filename, req_file=afile)
return publication.publicationid
def setUp(self):
if self.user_id == None:
res = run_sql("SELECT id FROM user WHERE nickname='admin'")
assert(len(res) == 1, "Couldn't find admin user")
self.user_id = int(res[0][0])
self.pub = OpenAIREPublication(self.user_id)
self.pub_id = self.pub.publicationid
def assertPublicationMetadata(self, pub_id, expected_metadata):
"""
Assert that field values of a publication is equal to those
given in the expected_metadata dictionary.
"""
pub = OpenAIREPublication(self.user_id, publicationid=pub_id)
metadata = pub.metadata
for field, expected_val in expected_metadata.items():
if field == 'projects':
continue
expected_val = wash_for_xml(expected_val)
real_val = metadata.get(field, None)
if field in ['related_publications', 'related_datasets']:
# Remove "doi:" and filter out blank strings.
real_val = filter(lambda x: x, real_val.split("\n"))
def _map_func(x):
if x.startswith("doi:"):
return x[4:]
else:
return x
expected_val = filter(lambda x: x,
map(_map_func, expected_val.split("\n")))
self.assertEqual(
real_val, expected_val, "Field %s: expected %s but got %s" %
(field, expected_val, real_val))
def setUp(self):
if self.user_id == None:
res = run_sql("SELECT id FROM user WHERE nickname='admin'")
assert (len(res) == 1, "Couldn't find admin user")
self.user_id = int(res[0][0])
self.pub = OpenAIREPublication(self.user_id)
self.pub_id = self.pub.publicationid
def setUp(self):
"""
Create test client, login with the given user and create a new publication.
"""
if self.user_id == None:
res = run_sql("SELECT id FROM user WHERE nickname='admin'")
assert (len(res) == 1, "Couldn't find admin user")
self.user_id = int(res[0][0])
uname = get_nickname(self.user_id)
self.assertEqual(uname, "admin")
self.client = TestClient(response_wrapper=JSONResponse)
self.client.login(uname, '')
self.client_noauth = TestClient(response_wrapper=JSONResponse)
self.pub = OpenAIREPublication(self.user_id)
self.pub_id = self.pub.publicationid
def upload(pub_id=None):
"""
PLUpload backend
"""
if pub_id:
pub_id = pub_id.encode('utf8')
uid = current_user.get_id()
if 'file' not in request.files:
abort(400)
afile = request.files['file']
filename = secure_filename(afile.filename)
publication = OpenAIREPublication(uid, publicationid=pub_id)
# Pre-fill user collection:
c = request.values.get('c', None)
if c:
publication.add_usercollection(c)
if not is_editable(publication):
flash(
"You cannot upload new files when your upload has already been submitted!"
)
abort(400)
publication.add_a_fulltext(None, filename, req_file=afile)
return publication.publicationid
def dropbox_upload(pub_id=None, fileurl=''):
"""
Dropbox upload backend
"""
if pub_id:
pub_id = pub_id.encode('utf8')
if fileurl:
fileurl = fileurl.encode('utf8')
uid = current_user.get_id()
if not fileurl:
abort(400)
if not (fileurl.startswith("https://dl.dropbox.com/")
or fileurl.startswith("https://dl.dropboxusercontent.com/")):
abort(400)
publication = OpenAIREPublication(uid)
if not is_editable(publication):
flash(
"You cannot upload new files when your upload has already been submitted!"
)
abort(400)
# Pre-fill user collection
c = request.values.get('c', None)
if c:
publication.add_usercollection(c)
uploaded_file = download_external_url(fileurl)
publication.add_a_fulltext(uploaded_file,
secure_filename(os.path.basename(fileurl)))
return redirect(url_for('deposit.edit', pub_id=publication.publicationid))
def getfile(pub_id='', file_id='', action='view'):
"""
View for stream file or deleting it.
"""
pub_id = pub_id.encode('utf8')
file_id = file_id.encode('utf8')
action = action.encode('utf8')
uid = current_user.get_id()
if action not in ['view', 'delete']:
abort(404)
try:
pub = OpenAIREPublication(uid, pub_id)
fulltext = pub.fulltexts[file_id]
except (ValueError, KeyError):
abort(404)
if action == 'view':
return send_file(fulltext.get_full_path(),
attachment_filename=fulltext.get_full_name(),
as_attachment=True)
elif action == 'delete':
if not is_editable(pub):
flash(
"You cannot delete files when your upload has already been submitted!"
)
return redirect(url_for('.edit', pub_id=pub.publicationid))
if len(pub.fulltexts.keys()) > 1:
if pub.remove_a_fulltext(file_id):
flash("File was deleted", category='success')
else:
flash("File could not be deleted. Please contact support.",
category='danger')
else:
flash("File cannot be deleted. You must provide minimum one file.")
return redirect(url_for('.edit', pub_id=pub.publicationid))
def setUp(self):
"""
Create test client, login with the given user and create a new publication.
"""
if self.user_id == None:
res = run_sql("SELECT id FROM user WHERE nickname='admin'")
assert(len(res) == 1, "Couldn't find admin user")
self.user_id = int(res[0][0])
uname = get_nickname(self.user_id)
self.assertEqual(uname, "admin")
self.client = TestClient(response_wrapper=JSONResponse)
self.client.login(uname, '')
self.client_noauth = TestClient(response_wrapper=JSONResponse)
self.pub = OpenAIREPublication(self.user_id)
self.pub_id = self.pub.publicationid
class EngineTest(unittest.TestCase):
user_id = None
project_id = "283595"
def record_marc_output(self, rec):
"""
Print MARC for a given record.
Used to compare a record generated by engine with the content of the record.
"""
rec = rec.items()
rec.sort()
marc = ""
for field_code, fields in rec:
for subfields, ind1, ind2, a, b in fields:
line = "\n%s%s%s " % (field_code, ind1 or "_", ind2 or "_")
for subcode, subvalue in subfields:
line += "$$%s%s" % (subcode, subvalue)
marc += line
return marc
def get_fixture(self, type):
""" Retrieve fixutre and expected MARC """
fixture = {}
# Append pub id to field name.
for field, val in FIXTURES[type].items():
fixture["%s_%s" % (field, self.pub_id)] = val
return fixture, MARC_FIXTURES[type]
def assertMarcIsEqual(self, expected_marc, marc):
"""
Assert that two MARC representations are equal, but exclude
eg. 001 control fields.
"""
# Filter out the control field.
rm_pat = re.compile("^001.*$")
rm_func = lambda x: not rm_pat.match(x)
expected_marc_list = filter(rm_func, [x.strip() for x in expected_marc.split("\n")])
marc_list = filter(rm_func, [x.strip() for x in marc.split("\n")])
i = 0
for i in range(0, max(len(expected_marc_list), len(marc_list))):
try:
l1 = expected_marc_list[i]
except IndexError:
l1 = ""
try:
l2 = marc_list[i]
except IndexError:
l2 = ""
i += 1
if l1.startswith("RE:"):
if not re.match(l1[3:], l2):
raise AssertionError("%s doesn't match %s" % (l1, l2))
else:
if l1 != l2:
raise AssertionError("%s != %s" % (l1, l2))
#
# Tests
#
def setUp(self):
if self.user_id == None:
res = run_sql("SELECT id FROM user WHERE nickname='admin'")
assert (len(res) == 1, "Couldn't find admin user")
self.user_id = int(res[0][0])
self.pub = OpenAIREPublication(self.user_id)
self.pub_id = self.pub.publicationid
def tearDown(self):
self.pub.delete()
self.pub = None
self.pub_id = None
def _test_pubtype(self, type):
"""
Test MARC generated for a given publication type.
"""
fixture, expected_marc = self.get_fixture(type)
self.pub.merge_form(fixture)
self.pub.check_metadata()
self.pub.check_projects()
self.pub.link_project(self.project_id)
rec = self.pub.get_record()
marc = self.record_marc_output(rec)
self.assertMarcIsEqual(expected_marc, marc)
def test_publishedArticle(self):
self._test_pubtype("publishedArticle")
def test_report(self):
self._test_pubtype("report")
def test_data(self):
self._test_pubtype("data")
def test_thesis(self):
self._test_pubtype("thesis")
def test_book(self):
self._test_pubtype("book")
def test_bookpart(self):
self._test_pubtype("bookpart")
def test_conference(self):
self._test_pubtype("conferenceContribution")
def test_preprint(self):
self._test_pubtype("preprint")
def test_workingpaper(self):
self._test_pubtype("workingPaper")
def bst_openaire_keywords():
"""
Tasklet to update the list of keywords and flag potential
keywords which should be split.
"""
current_uid = None
current_pubid = None
current_keywords = {}
# Iterate over all keywords (order by publication id)
res = run_sql(
"SELECT k.keyword, p.publicationid, p.uid, p.id_bibrec FROM eupublication AS p JOIN OpenAIREkeywords AS k ON p.publicationid=k.publicationid ORDER BY p.publicationid"
)
total = len(res)
i = 0
# Any last minute regrets to stop?
task_sleep_now_if_required(can_stop_too=True)
write_message("Updating OpenAIREkeywords table (%s entries)..." % total)
for keyword, pubid, uid, recid in res:
i += 1
task_update_progress("%s - %s%%" % (total, i * 100 / total))
# Check if we reached a new publication
if current_pubid != pubid:
# Add keywords which was not currently not there.
add_missing_keywords(current_uid, current_pubid, current_keywords)
# Stop task if required (we just reached a new publication, so we can
# stop now in a consistent state.
task_sleep_now_if_required(can_stop_too=True)
# Reset state (i.e new record encountered)
current_uid = uid
current_pubid = pubid
current_keywords = {}
# Get current keywords for publication (either from record if exists,
# otherwise from submitted form.
if recid:
current_keywords = dict([(x[0], 1) for x in run_sql(
"SELECT value FROM bibrec_bib65x JOIN bib65x ON id=id_bibxxx WHERE id_bibrec=%s AND tag='6531_a' ORDER BY id_bibrec, field_number",
(recid, ))])
else:
pub = OpenAIREPublication(uid, publicationid=pubid)
current_keywords = dict([(x, 1)
for x in normalize_multivalue_field(
pub._metadata.get('keywords', ''),
sort=True).splitlines()])
# Check if keyword is in the current list of keywords.
if keyword not in current_keywords:
# If not, remove it.
write_message("Removing keyword '%s'" % keyword)
res = run_sql(
"DELETE FROM OpenAIREkeywords WHERE keyword=%s AND publicationid=%s",
(keyword, pubid))
else:
del current_keywords[
keyword] # Remove from dictionary, so we know if all keywords
add_missing_keywords(current_uid, current_pubid, current_keywords)
class EngineTest(unittest.TestCase):
user_id = None
project_id = '283595'
def record_marc_output(self, rec):
"""
Print MARC for a given record.
Used to compare a record generated by engine with the content of the record.
"""
rec = rec.items()
rec.sort()
marc = ""
for field_code, fields in rec:
for subfields, ind1, ind2, a, b in fields:
line = "\n%s%s%s " % (field_code, ind1 or '_', ind2 or '_')
for subcode, subvalue in subfields:
line += "$$%s%s" % (subcode, subvalue)
marc += line
return marc
def get_fixture(self, type):
""" Retrieve fixutre and expected MARC """
fixture = {}
# Append pub id to field name.
for field, val in FIXTURES[type].items():
fixture['%s_%s' % (field, self.pub_id)] = val
return fixture, MARC_FIXTURES[type]
def assertMarcIsEqual(self, expected_marc, marc):
"""
Assert that two MARC representations are equal, but exclude
eg. 001 control fields.
"""
# Filter out the control field.
rm_pat = re.compile("^001.*$")
rm_func = lambda x: not rm_pat.match(x)
expected_marc_list = filter(
rm_func, [x.strip() for x in expected_marc.split('\n')])
marc_list = filter(rm_func, [x.strip() for x in marc.split('\n')])
i = 0
for i in range(0, max(len(expected_marc_list), len(marc_list))):
try:
l1 = expected_marc_list[i]
except IndexError:
l1 = ''
try:
l2 = marc_list[i]
except IndexError:
l2 = ''
i += 1
if l1.startswith("RE:"):
if not re.match(l1[3:], l2):
raise AssertionError("%s doesn't match %s" % (l1, l2))
else:
if l1 != l2:
raise AssertionError("%s != %s" % (l1, l2))
#
# Tests
#
def setUp(self):
if self.user_id == None:
res = run_sql("SELECT id FROM user WHERE nickname='admin'")
assert(len(res) == 1, "Couldn't find admin user")
self.user_id = int(res[0][0])
self.pub = OpenAIREPublication(self.user_id)
self.pub_id = self.pub.publicationid
def tearDown(self):
self.pub.delete()
self.pub = None
self.pub_id = None
def _test_pubtype(self, type):
"""
Test MARC generated for a given publication type.
"""
fixture, expected_marc = self.get_fixture(type)
self.pub.merge_form(fixture)
self.pub.check_metadata()
self.pub.check_projects()
self.pub.link_project(self.project_id)
rec = self.pub.get_record()
marc = self.record_marc_output(rec)
self.assertMarcIsEqual(expected_marc, marc)
def test_publishedArticle(self):
self._test_pubtype('publishedArticle')
def test_report(self):
self._test_pubtype('report')
def test_data(self):
self._test_pubtype('data')
def test_thesis(self):
self._test_pubtype('thesis')
def test_book(self):
self._test_pubtype('book')
def test_bookpart(self):
self._test_pubtype('bookpart')
def test_conference(self):
self._test_pubtype('conferenceContribution')
def test_preprint(self):
self._test_pubtype('preprint')
def test_workingpaper(self):
self._test_pubtype('workingPaper')
def edit(pub_id=u'', action=u'edit'):
"""
Edit an upload
"""
uid = current_user.get_id()
if action not in ['edit', 'save', 'delete', 'reserve-doi']:
abort(404)
try:
pub = OpenAIREPublication(uid, publicationid=pub_id)
title = pub.metadata.get('title', 'Untitled') or 'Untitled'
editable = is_editable(pub)
except ValueError:
abort(404)
# All POST requests change the publication, and are not allowed if the
# publication is not editable anymore.
if request.method == 'POST':
if not editable:
flash(
"You cannot edit an already submitted upload. Please contact %s if you would like to make changes!"
% CFG_SITE_SUPPORT_EMAIL)
return redirect(url_for('.edit', pub_id=pub.publicationid))
#
# Action handling
#
ctx = {}
if action == 'reserve-doi':
#
# Reserve DOI action (AjAX)
#
if request.method == 'POST':
doi = pub.create_doi()
return json.dumps({'doi': doi})
else:
abort(405)
elif action == 'delete':
#
# Delete action
#
if not editable:
flash(
"You cannot delete an already submitted upload. Please contact %s if you would like to have it removed!"
% CFG_SITE_SUPPORT_EMAIL)
return redirect(url_for('.edit', pub_id=pub.publicationid))
pub.delete()
flash("Upload '%s' was deleted." % title, 'success')
return redirect(url_for('.index'))
elif action == 'edit':
#
# Edit action
#
upload_url = url_for('deposit.upload', pub_id=pub.publicationid)
dropbox_upload_url = url_for('deposit.dropbox_upload',
pub_id=pub.publicationid)
ctx = {
'pub': pub,
'recid': pub.metadata.get('__recid__', None),
'title': title,
'is_editable': editable,
'upload_url': upload_url,
'dropbox_upload_url': dropbox_upload_url,
}
if request.method == 'POST':
form = DepositionForm(request.values, crsf_enabled=False)
mapper = DepositionFormMapper(pub)
pub = mapper.map(form)
form._pub = pub
if form.validate():
pub.save()
pub.upload_record()
flash(
"Upload was successfully submitted - it may take up 5 minutes before it has been fully integrated into %s."
% CFG_SITE_NAME,
category='success')
return redirect(url_for('.index'))
else:
pub.save()
ctx['form'] = form
ctx['form_message'] = "The form was saved, but there were errors. Please see below."
elif editable:
mapper = PublicationMapper()
form = DepositionForm(mapper.map(pub), crsf_enabled=False)
ctx['form'] = form
else:
ctx['record_hd'] = format_record(recID=pub.recid,
xml_record=pub.marcxml,
ln=g.ln,
of='hd')
ctx['record_hdinfo'] = format_record(recID=pub.recid,
xml_record=pub.marcxml,
ln=g.ln,
of='HDINFO')
elif action == 'save':
#
# Save action (AjAX)
#
if request.method == 'POST':
form = DepositionForm(request.values, crsf_enabled=False)
mapper = DepositionFormMapper(pub)
pub = mapper.map(form)
form._pub = pub
if form.validate():
pub.save()
return json.dumps({
'status': 'success',
'form': 'Successfully saved.'
})
else:
pub.save()
errors = dict([(x, '') for x in form._fields.keys()])
errors.update(form.errors)
return json.dumps({
'status': 'warning',
'form':
'The form was saved, but there were errors. Please see below.',
'fields': errors,
})
else:
abort(405)
return render_template("openaire_edit.html",
myresearch=get_exisiting_publications_for_uid(
current_user.get_id()),
**ctx)
def ajaxgateway(self, req, form):
"""
"""
argd = wash_urlargd(form, {'projectid': (str, ''), 'publicationid': (
str, ''), 'action': (str, ''), 'current_field': (str, '')})
# Get parameters
action = argd['action']
publicationid = argd['publicationid']
projectid = argd['projectid']
# Check if action is supported
assert(action in (
'save', 'verify_field', 'submit', 'unlinkproject', 'linkproject'))
# JSON return dictionary
out = {
'errors': {},
'warnings': {},
'addclasses': {},
'delclasses': {},
'substitutions': {},
'appends': {},
'hiddens': [],
'showns': [],
'action': action,
}
if action == 'verify_field':
current_field = argd['current_field']
assert(current_field)
metadata = wash_form(form, publicationid)
out["errors"], out["warnings"] = OpenAIREPublication.static_check_metadata(metadata, publicationid, check_only_field=current_field, ln=argd['ln'])
else:
user_info = collect_user_info(req)
auth_code, auth_message = acc_authorize_action(
user_info, 'submit', doctype='OpenAIRE')
assert(auth_code == 0)
uid = user_info['uid']
publication = OpenAIREPublication(
uid, publicationid, ln=argd['ln'])
if action == 'unlinkproject':
publication.unlink_project(projectid)
out["substitutions"]["#projectsbox_%s" %
publicationid] = publication.get_projects_information()
publication.check_projects()
out["errors"], out["warnings"] = simple_metadata2namespaced_metadata(publication.errors, publicationid), simple_metadata2namespaced_metadata(publication.warnings, publicationid)
elif action == 'linkproject':
publication.link_project(projectid)
out["substitutions"]["#projectsbox_%s" %
publicationid] = publication.get_projects_information()
publication.check_projects()
out["errors"], out["warnings"] = simple_metadata2namespaced_metadata(publication.errors, publicationid), simple_metadata2namespaced_metadata(publication.warnings, publicationid)
else:
publication.merge_form(form)
publication.check_metadata()
publication.check_projects()
out["errors"], out["warnings"] = simple_metadata2namespaced_metadata(publication.errors, publicationid), simple_metadata2namespaced_metadata(publication.warnings, publicationid)
if "".join(out["errors"].values()).strip(): # FIXME bad hack, we need a cleaner way to discover if there are errors
out['addclasses']['#status_%s' % publicationid] = 'error'
out['delclasses']['#status_%s' %
publicationid] = 'warning ok empty'
elif "".join(out["warnings"].values()).strip():
out['addclasses']['#status_%s' % publicationid] = 'warning'
out['delclasses']['#status_%s' %
publicationid] = 'error ok empty'
else:
out['addclasses']['#status_%s' % publicationid] = 'ok'
out['delclasses']['#status_%s' %
publicationid] = 'warning error empty'
if action == 'save':
out["substitutions"]['#publication_information_%s' % publicationid] = publication.get_publication_information()
elif action == 'submit':
if not "".join(out["errors"].values()).strip():
publication.upload_record()
out["appends"]['#submitted_publications'] = publication.get_publication_preview()
out["showns"].append('#submitted_publications')
out["hiddens"].append('#header_row_%s' % publicationid)
out["hiddens"].append('#body_row_%s' % publicationid)
req.content_type = 'application/json'
return json.dumps(out)
class AjaxGatewayTest(unittest.TestCase):
"""
Testing of AJAX Gateway
"""
user_id = None
""" User for which we want to add publications. """
project_id = '283595'
test_full_submit = True
""" Set to True if you want to test full submission. By default this
is not enabled, since it will pollute your installation.
"""
# ================================
# Helpers
# ================================
def _base_form(self, action='', projectid='0', current_field='', data={}, override_pubid=None):
"""
Helper for creating required POST form parameters
@param action: Name of action (e.g. save, submit, verify_field, ...)
@param projectid: Id of project, can be left empty (defaults to NOPROJECT).
@param current_field: Name of the current field without publication id suffix (e.g. language).
@param data: Dictionary of fields/values for the form (without publication id suffix)/
"""
pub_id = override_pubid or self.pub_id
form = {}
for field in CFG_METADATA_FIELDS:
form['%s_%s' % (field, pub_id)] = ''
if current_field:
current_field_pub_id = ('%s_%s' % (current_field, pub_id))
else:
current_field_pub_id = ''
form.update({
'projectid': projectid,
'publicationid': pub_id,
'action': action,
'current_field': current_field_pub_id,
'save_%s' % pub_id: "Save+publication",
'submit_%s' % pub_id: "Submit+publication",
})
# Default form values
form.update({
'publication_type_%s' % pub_id: 'publishedArticle',
'access_rights_%s' % pub_id: 'openAccess',
'language_%s' % pub_id: 'eng',
})
# Add form data
for k, v in data.items():
form['%s_%s' % (k, pub_id)] = v
return form
#
# Actions
#
def action_verify_field(self, field, value, extra_data={}, **kwargs):
""" Helper for creating required POST parameters for verify_field action """
data = {field: value}
data.update(extra_data)
return self._base_form(
action='verify_field',
current_field=field,
data=data,
**kwargs
)
def action_save(self, data={}, **kwargs):
""" Helper for creating required POST parameters for save action """
return self._base_form(
action='save',
current_field='save',
data=data,
**kwargs
)
def action_submit(self, data={}, **kwargs):
""" Helper for creating required POST parameters for save action """
return self._base_form(
action='submit',
current_field='submit',
data=data,
**kwargs
)
#
# Utility methods
#
def get_file_fixture(self, type, style):
"""
Retrieve file fixture for publication type if it exists, or generate a
PDF on the file.
"""
if type in FILE_FIXTURES:
name, base64_file = FILE_FIXTURES[type]
return (StringIO(binascii.a2b_base64(base64_file)), name)
else:
return (StringIO(self.make_stringio_pdf("Type: %s Style: %s" % (type, style))), '%s_%s_file.pdf' % (type.lower(), style))
def make_stringio_pdf(self, text):
"""
Generate a PDF which includes the text given as parameter to this function.
"""
from reportlab.pdfgen import canvas
output = StringIO()
c = canvas.Canvas(output)
c.drawString(100, 100, text)
c.showPage()
c.save()
return output.getvalue()
def get_publications_for_project(self, project_id, style='invenio'):
""" Get all publications for a project """
RE_DELETE_LINK = re.compile("/deposit\?projectid=%s&delete=([a-z0-9]+)&ln=" % project_id, re.IGNORECASE)
resp = self.client.get("/deposit", query_string={'style':
style, 'ln': 'en', 'projectid': project_id})
return filter(lambda x: x != self.pub_id, RE_DELETE_LINK.findall(resp.data)) # Filter out pub from setUp
def clear_publications(self, project_id, style='invenio'):
""" Clear all publications for a project """
pub_ids = self.get_publications_for_project(project_id, style=style)
for pub_id in pub_ids:
resp = self.client.get("/deposit", query_string={'projectid':
project_id, 'delete': pub_id, 'ln': ''})
self.assertEqual(resp.status_code, 200)
def save_metadata(self, project_id, pub_id, fixture):
""" Load a fixture and save it for the given publication """
res = self.client.post("/deposit/ajaxgateway", data=self.action_save(
data=fixture, override_pubid=pub_id, projectid=project_id))
for field, error in res.json['errors'].items():
self.assertEqual(error, '', msg="Save unsuccessful - %s error: %s" % (field, error))
self.assertPublicationMetadata(pub_id, fixture)
def approve_record(self, recid):
""" Approve a record to make it publicly available """
# Make MARCXML to approve record
rec = {}
record_add_field(rec, '001', controlfield_value=str(recid))
record_add_field(rec, '980', subfields=[('a', 'OPENAIRE')])
output = "<collection>%s</collection>" % record_xml_output(rec)
# Upload MARCXML
run_sql("TRUNCATE schTASK") # Ensures we run bibupload
(hdl, marcxml_path) = mkstemp(suffix=".xml", text=True)
open(marcxml_path, 'w').write(output)
task_low_level_submission(
'bibupload', 'openaire', '-c', marcxml_path, '-P5')
task_low_level_submission('bibindex', 'openaire')
task_low_level_submission('webcoll', 'openaire')
os.system("%s/bin/bibupload 1 > /dev/null" % CFG_PREFIX)
os.system("%s/bin/bibindex 2 > /dev/null" % CFG_PREFIX)
os.system("%s/bin/webcoll 3 > /dev/null" % CFG_PREFIX)
#
# Assert methods
#
def assertPublicationMetadata(self, pub_id, expected_metadata):
"""
Assert that field values of a publication is equal to those
given in the expected_metadata dictionary.
"""
pub = OpenAIREPublication(self.user_id, publicationid=pub_id)
metadata = pub.metadata
for field, expected_val in expected_metadata.items():
if field == 'projects':
continue
expected_val = wash_for_xml(expected_val)
real_val = metadata.get(field, None)
if field in ['related_publications','related_datasets']:
# Remove "doi:" and filter out blank strings.
real_val = filter(lambda x: x, real_val.split("\n"))
def _map_func(x):
if x.startswith("doi:"):
return x[4:]
else:
return x
expected_val = filter(
lambda x: x, map(_map_func, expected_val.split("\n")))
self.assertEqual(real_val, expected_val, "Field %s: expected %s but got %s" % (field, expected_val, real_val))
# ================================
# Tests
# ================================
def setUp(self):
"""
Create test client, login with the given user and create a new publication.
"""
if self.user_id == None:
res = run_sql("SELECT id FROM user WHERE nickname='admin'")
assert(len(res) == 1, "Couldn't find admin user")
self.user_id = int(res[0][0])
uname = get_nickname(self.user_id)
self.assertEqual(uname, "admin")
self.client = TestClient(response_wrapper=JSONResponse)
self.client.login(uname, '')
self.client_noauth = TestClient(response_wrapper=JSONResponse)
self.pub = OpenAIREPublication(self.user_id)
self.pub_id = self.pub.publicationid
def tearDown(self):
"""
Remove the publication again.
"""
self.pub.delete()
self.pub = None
self.pub_id = None
def test_verify_field(self):
"""
Testing of verify field action
python -m unittest invenio.openaire_deposit_webinterface_tests.AjaxGatewayTest.test_verify_field
"""
# Language
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('language', 'eng'))
self.assertEqual(res.json['errors']['language_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field('language', ''))
self.assertNotEqual(
res.json['errors']['language_%s' % self.pub_id], '')
# Publication type
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('publication_type', 'publishedArticle'))
self.assertEqual(
res.json['errors']['publication_type_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('publication_type', 'dfgsdfgsd'))
self.assertNotEqual(
res.json['errors']['publication_type_%s' % self.pub_id], '')
# Report pages no
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('report_pages_no', '6545'))
self.assertEqual(
res.json['errors']['report_pages_no_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('report_pages_no', 'dfgsdfgsd'))
self.assertNotEqual(
res.json['errors']['report_pages_no_%s' % self.pub_id], '')
# Keywords
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('keywords', 'multiline\nkeywords'))
self.assertEqual(res.json['errors']['keywords_%s' % self.pub_id], '')
self.assertEqual(res.json['warnings']['keywords_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('keywords', 'this;should;probably;be;separated'))
self.assertEqual(res.json['errors']['keywords_%s' % self.pub_id], '')
self.assertNotEqual(
res.json['warnings']['keywords_%s' % self.pub_id], '')
# Accept CC0 license
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('accept_cc0_license', 'yes', extra_data={'publication_type': 'data'}))
self.assertEqual(
res.json['errors']['accept_cc0_license_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['accept_cc0_license_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('accept_cc0_license', '', extra_data={'publication_type': 'data'}))
self.assertNotEqual(
res.json['errors']['accept_cc0_license_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['accept_cc0_license_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('accept_cc0_license', '', extra_data={'publication_type': 'report'}))
self.assertEqual(
res.json['errors']['accept_cc0_license_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['accept_cc0_license_%s' % self.pub_id], '')
# Related publications
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('related_publications', '10.1234/1234', extra_data={'publication_type': 'data'}))
self.assertEqual(
res.json['errors']['related_publications_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['related_publications_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('related_publications', '10.1234/1234\n\n12.2323', extra_data={'publication_type': 'data'}))
self.assertNotEqual(
res.json['errors']['related_publications_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['related_publications_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field('related_publications', '10.1234/1234\n\n10.1234/12342\n10.1234/12342', extra_data={'publication_type': 'data'}))
self.assertEqual(
res.json['errors']['related_publications_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['related_publications_%s' % self.pub_id], '')
# Ensure all checks are executed to test for possible runtime errors. Validate is not needed
for field in CFG_METADATA_FIELDS_CHECKS:
res = self.client.post("/deposit/ajaxgateway", data=self.action_verify_field(field, 'test'))
def test_action_save_published_article(self):
"""
Testing of save action
"""
data = {
'access_rights': 'openAccess',
'embargo_date': '',
'authors': '',
'title': 'some title',
'abstract': '',
'language': '',
'original_title': '',
'original_abstract': '',
'publication_type': 'publishedArticle',
'publication_date': '',
'journal_title': '',
'doi': '',
'volume': '',
'issue': '',
'pages': '',
'keywords': '',
'notes': '',
}
res = self.client.post(
"/deposit/ajaxgateway", data=self.action_save(data=data))
self.assertPublicationMetadata(
self.pub_id, {'publication_type': 'publishedArticle'})
def _submit(self, type, style, submit=True):
"""
Make a submission
"""
resp = self.client.get(
"/deposit", query_string={'style': style, 'ln': 'en'})
self.assertEqual(resp.status_code, 200)
# Check if there's any existing submissions
self.clear_publications('0', style=style)
self.clear_publications(self.project_id, style=style)
# Simple file upload
(file_stringio, filename) = self.get_file_fixture(type, style)
resp = self.client.post(
"/deposit",
query_string={'style': style, 'ln': 'en', 'projectid': '0'},
data={'Filedata': (file_stringio, filename), 'upload': 'Upload'},
)
self.assertEqual(resp.status_code, 200)
pub_ids = self.get_publications_for_project('0', style=style)
self.assertEqual(len(pub_ids), 1)
pub_id = pub_ids[0]
# Link with project
resp = self.client.post(
"/deposit/ajaxgateway",
data={'projectid': self.project_id,
'publicationid': pub_id, 'action': 'linkproject'},
)
self.assertEqual(resp.json['errors']['projects_%s' % pub_id], '')
# Save metadata
fixture = FIXTURES[type]
self.save_metadata(self.project_id, pub_id, fixture)
# Submit publication
if submit:
run_sql("TRUNCATE schTASK") # Ensures we run bibupload
res = self.client.post("/deposit/ajaxgateway", data=self.action_submit(data=fixture, override_pubid=pub_id, projectid=self.project_id))
self.assertIn("This is a preview of the submitted publication. If approved, it will be available at ", res.json['appends']['#submitted_publications'])
# Get record id
m = re.search("/record/(\d+)\"", res.json[
'appends']['#submitted_publications'])
self.assertIsNotNone(m, "Couldn't find record id.")
rec_id = m.group(1)
# Run bibupload to get record
from invenio.config import CFG_PREFIX
os.system("%s/bin/bibupload 1" % CFG_PREFIX)
# Approve record so it's publicly available.
self.approve_record(rec_id)
# Check if record is reachable
res = self.client_noauth.get("/record/%s" % rec_id)
self.assertEqual(res.status_code, 200)
res = self.client_noauth.get("/record/%s/files/%s" % (
rec_id, filename))
if fixture["access_rights"] in ["embargoedAccess", \
"restrictedAccess", "closedAccess"] and \
type not in CFG_OPENAIRE_CC0_PUBLICATION_TYPES:
self.assertEqual(res.status_code, 302) # Restricted access.
else:
self.assertEqual(res.status_code, 200)
# Remove submission again
self.clear_publications('0', style=style)
self.clear_publications(self.project_id, style=style)
def test_submission_published_article(self):
self._submit('publishedArticle', 'portal', submit=self.test_full_submit)
def test_submission_report(self):
self._submit('report', 'invenio', submit=self.test_full_submit)
def test_submission_data(self):
self._submit('data', 'portal', submit=self.test_full_submit)
def test_submission_thesis(self):
self._submit('thesis', 'invenio', submit=self.test_full_submit)
def test_submission_book(self):
self._submit('book', 'portal', submit=self.test_full_submit)
def test_submission_bookpart(self):
self._submit('bookpart', 'portal', submit=self.test_full_submit)
def test_submission_conference(self):
self._submit('conferenceContribution', 'portal', submit=self.test_full_submit)
def test_submission_preprint(self):
self._submit('preprint', 'portal', submit=self.test_full_submit)
def test_submission_workingpaper(self):
self._submit('workingPaper', 'portal', submit=self.test_full_submit)
def edit(self, req, form):
argd = wash_urlargd(
form,
{
'projectid': (int, -1),
#'delete': (str, ''),
'id': (str, ''),
'a': (str, 'edit'),
#'linkproject': (int, -1),
#'unlinkproject': (int, -1),
#style': (str, None),
#'upload': (str, ''),
#'dropbox': (str, '')
})
_ = gettext_set_language(argd['ln'])
# Check if user is authorized to deposit publications
user_info = collect_user_info(req)
auth_code, auth_message = acc_authorize_action(user_info,
'submit',
doctype='OpenAIRE')
if auth_code:
if user_info['guest'] == '1':
return redirect_to_url(
req, "%s/youraccount/login%s" %
(CFG_SITE_SECURE_URL,
make_canonical_urlargd(
{
'referer': "%s%s" %
(CFG_SITE_URL, req.unparsed_uri),
"ln": argd['ln']
}, {})))
else:
return page(
req=req,
body=_(
"You are not authorized to use OpenAIRE deposition."),
title=_("Authorization failure"),
navmenuid="submit")
# Validate action
action = argd['a']
if action not in ['edit', 'save', 'submit', 'delete']:
abort(404)
uid = user_info['uid']
if not argd['id']:
return redirect("/deposit/")
try:
pub = OpenAIREPublication(uid, publicationid=argd['id'])
title = pub.metadata.get('title', 'Untitled')
except ValueError:
abort(404)
#
# Action handling
#
if action == 'delete':
pub.delete()
flash("Upload '%s' was deleted." % title, 'info')
return redirect('/deposit/')
elif action == 'edit':
ctx = {
'pub': pub,
'title': title,
'recid': pub.metadata.get('__recid__', None),
}
ctx['rendered_form'] = openaire_deposit_templates.tmpl_form(
pub.metadata['__publicationid__'],
-1,
"", # projects_information
"", # publication_information
"", # fulltext_information
form=None,
metadata_status='empty',
warnings=None,
errors=None,
ln='en',
)
#if action == "edit":
#elif action == "delete":
# pass
# if projectid >= 0:
# ## There is a project on which we are working good!
# publications = get_all_publications_for_project(
# uid, projectid, ln=argd['ln'], style=style)
# if argd['publicationid'] in publications:
# if argd['addproject'] in all_project_ids:
# publications[argd['publicationid']
# ].link_project(argd['linkproject'])
# if argd['delproject'] in all_project_ids:
# publications[argd['publicationid']
# ].unlink_project(argd['unlinkproject'])
# if argd['delete'] and argd['delete'] in publications:
# ## there was a request to delete a publication
# publications[argd['delete']].delete()
# del publications[argd['delete']]
# forms = ""
# submitted_publications = ""
# for index, (publicationid, publication) in enumerate(publications.iteritems()):
# if req.method.upper() == 'POST':
# publication.merge_form(form, ln=argd['ln'])
# if publication.status == 'edited':
# publication.check_metadata()
# publication.check_projects()
# if 'submit_%s' % publicationid in form and not "".join(publication.errors.values()).strip():
# ## i.e. if the button submit for the corresponding publication has been pressed...
# publication.upload_record()
# if publication.status in ('initialized', 'edited'):
# forms += publication.get_publication_form(projectid)
# else:
# submitted_publications += publication.get_publication_preview()
# body += openaire_deposit_templates.tmpl_add_publication_data_and_submit(projectid, forms, submitted_publications, project_information=upload_to_project_information, ln=argd['ln'])
# body += openaire_deposit_templates.tmpl_upload_publications(projectid=upload_to_projectid, project_information=upload_to_project_information, session=get_session(req).sid.encode('utf8'), style=style, ln=argd['ln'])
# else:
ctx.update({
'myresearch': get_exisiting_publications_for_uid(uid),
})
return render_template("openaire_edit.html",
req=req,
breadcrumbs=[
(_('Home'), ''),
(_('Upload'), 'deposit'),
(title, ''),
],
**ctx).encode('utf8')
def ajaxgateway(self, req, form):
"""
"""
argd = wash_urlargd(
form, {"projectid": (str, ""), "publicationid": (str, ""), "action": (str, ""), "current_field": (str, "")}
)
# Get parameters
action = argd["action"]
publicationid = argd["publicationid"]
projectid = argd["projectid"]
# Check if action is supported
assert action in ("save", "verify_field", "submit", "unlinkproject", "linkproject")
# JSON return dictionary
out = {
"errors": {},
"warnings": {},
"addclasses": {},
"delclasses": {},
"substitutions": {},
"appends": {},
"hiddens": [],
"showns": [],
"action": action,
}
if action == "verify_field":
current_field = argd["current_field"]
assert current_field
metadata = wash_form(form, publicationid)
out["errors"], out["warnings"] = OpenAIREPublication.static_check_metadata(
metadata, publicationid, check_only_field=current_field, ln=argd["ln"]
)
else:
user_info = collect_user_info(req)
auth_code, auth_message = acc_authorize_action(user_info, "submit", doctype="OpenAIRE")
assert auth_code == 0
uid = user_info["uid"]
publication = OpenAIREPublication(uid, publicationid, ln=argd["ln"])
if action == "unlinkproject":
publication.unlink_project(projectid)
out["substitutions"]["#projectsbox_%s" % publicationid] = publication.get_projects_information()
publication.check_projects()
out["errors"], out["warnings"] = (
simple_metadata2namespaced_metadata(publication.errors, publicationid),
simple_metadata2namespaced_metadata(publication.warnings, publicationid),
)
elif action == "linkproject":
publication.link_project(projectid)
out["substitutions"]["#projectsbox_%s" % publicationid] = publication.get_projects_information()
publication.check_projects()
out["errors"], out["warnings"] = (
simple_metadata2namespaced_metadata(publication.errors, publicationid),
simple_metadata2namespaced_metadata(publication.warnings, publicationid),
)
else:
publication.merge_form(form)
publication.check_metadata()
publication.check_projects()
out["errors"], out["warnings"] = (
simple_metadata2namespaced_metadata(publication.errors, publicationid),
simple_metadata2namespaced_metadata(publication.warnings, publicationid),
)
if "".join(
out["errors"].values()
).strip(): # FIXME bad hack, we need a cleaner way to discover if there are errors
out["addclasses"]["#status_%s" % publicationid] = "error"
out["delclasses"]["#status_%s" % publicationid] = "warning ok empty"
elif "".join(out["warnings"].values()).strip():
out["addclasses"]["#status_%s" % publicationid] = "warning"
out["delclasses"]["#status_%s" % publicationid] = "error ok empty"
else:
out["addclasses"]["#status_%s" % publicationid] = "ok"
out["delclasses"]["#status_%s" % publicationid] = "warning error empty"
if action == "save":
out["substitutions"][
"#publication_information_%s" % publicationid
] = publication.get_publication_information()
elif action == "submit":
if not "".join(out["errors"].values()).strip():
publication.upload_record()
out["appends"]["#submitted_publications"] = publication.get_publication_preview()
out["showns"].append("#submitted_publications")
out["hiddens"].append("#header_row_%s" % publicationid)
out["hiddens"].append("#body_row_%s" % publicationid)
req.content_type = "application/json"
return json.dumps(out)
def edit(pub_id=u'', action=u'edit'):
"""
Edit an upload
"""
uid = current_user.get_id()
if action not in ['edit', 'save', 'delete', 'reserve-doi']:
abort(404)
try:
pub = OpenAIREPublication(uid, publicationid=pub_id)
title = pub.metadata.get('title', 'Untitled') or 'Untitled'
editable = is_editable(pub)
except ValueError:
abort(404)
# All POST requests change the publication, and are not allowed if the
# publication is not editable anymore.
if request.method == 'POST':
if not editable:
flash("You cannot edit an already submitted upload. Please contact %s if you would like to make changes!" % CFG_SITE_SUPPORT_EMAIL)
return redirect(url_for('.edit', pub_id=pub.publicationid))
#
# Action handling
#
ctx = {}
if action == 'reserve-doi':
#
# Reserve DOI action (AjAX)
#
if request.method == 'POST':
doi = pub.create_doi()
return json.dumps({'doi': doi})
else:
abort(405)
elif action == 'delete':
#
# Delete action
#
if not editable:
flash("You cannot delete an already submitted upload. Please contact %s if you would like to have it removed!" % CFG_SITE_SUPPORT_EMAIL)
return redirect(url_for('.edit', pub_id=pub.publicationid))
pub.delete()
flash("Upload '%s' was deleted." % title, 'success')
return redirect(url_for('.index'))
elif action == 'edit':
#
# Edit action
#
upload_url = url_for('deposit.upload', pub_id=pub.publicationid)
dropbox_upload_url = url_for('deposit.dropbox_upload', pub_id=pub.publicationid)
ctx = {
'pub': pub,
'recid': pub.metadata.get('__recid__', None),
'title': title,
'is_editable': editable,
'upload_url': upload_url,
'dropbox_upload_url': dropbox_upload_url,
}
if request.method == 'POST':
form = DepositionForm(request.values, crsf_enabled=False)
mapper = DepositionFormMapper(pub)
pub = mapper.map(form)
form._pub = pub
if form.validate():
pub.save()
pub.upload_record()
flash("Upload was successfully submitted - it may take up 5 minutes before it has been fully integrated into %s." % CFG_SITE_NAME, category='success')
return redirect(url_for('.index'))
else:
pub.save()
ctx['form'] = form
ctx['form_message'] = "The form was saved, but there were errors. Please see below."
elif editable:
mapper = PublicationMapper()
form = DepositionForm(mapper.map(pub), crsf_enabled=False)
ctx['form'] = form
else:
ctx['record_hd'] = format_record(recID=pub.recid, xml_record=pub.marcxml, ln=g.ln, of='hd')
ctx['record_hdinfo'] = format_record(recID=pub.recid, xml_record=pub.marcxml, ln=g.ln, of='HDINFO')
elif action == 'save':
#
# Save action (AjAX)
#
if request.method == 'POST':
form = DepositionForm(request.values, crsf_enabled=False)
mapper = DepositionFormMapper(pub)
pub = mapper.map(form)
form._pub = pub
if form.validate():
pub.save()
return json.dumps({'status': 'success', 'form': 'Successfully saved.'})
else:
pub.save()
errors = dict([(x, '') for x in form._fields.keys()])
errors.update(form.errors)
return json.dumps({
'status': 'warning',
'form': 'The form was saved, but there were errors. Please see below.',
'fields': errors,
})
else:
abort(405)
return render_template(
"openaire_edit.html",
myresearch=get_exisiting_publications_for_uid(current_user.get_id()),
**ctx
)
class AjaxGatewayTest(unittest.TestCase):
"""
Testing of AJAX Gateway
"""
user_id = None
""" User for which we want to add publications. """
project_id = '283595'
test_full_submit = True
""" Set to True if you want to test full submission. By default this
is not enabled, since it will pollute your installation.
"""
# ================================
# Helpers
# ================================
def _base_form(self,
action='',
projectid='0',
current_field='',
data={},
override_pubid=None):
"""
Helper for creating required POST form parameters
@param action: Name of action (e.g. save, submit, verify_field, ...)
@param projectid: Id of project, can be left empty (defaults to NOPROJECT).
@param current_field: Name of the current field without publication id suffix (e.g. language).
@param data: Dictionary of fields/values for the form (without publication id suffix)/
"""
pub_id = override_pubid or self.pub_id
form = {}
for field in CFG_METADATA_FIELDS:
form['%s_%s' % (field, pub_id)] = ''
if current_field:
current_field_pub_id = ('%s_%s' % (current_field, pub_id))
else:
current_field_pub_id = ''
form.update({
'projectid': projectid,
'publicationid': pub_id,
'action': action,
'current_field': current_field_pub_id,
'save_%s' % pub_id: "Save+publication",
'submit_%s' % pub_id: "Submit+publication",
})
# Default form values
form.update({
'publication_type_%s' % pub_id: 'publishedArticle',
'access_rights_%s' % pub_id: 'openAccess',
'language_%s' % pub_id: 'eng',
})
# Add form data
for k, v in data.items():
form['%s_%s' % (k, pub_id)] = v
return form
#
# Actions
#
def action_verify_field(self, field, value, extra_data={}, **kwargs):
""" Helper for creating required POST parameters for verify_field action """
data = {field: value}
data.update(extra_data)
return self._base_form(action='verify_field',
current_field=field,
data=data,
**kwargs)
def action_save(self, data={}, **kwargs):
""" Helper for creating required POST parameters for save action """
return self._base_form(action='save',
current_field='save',
data=data,
**kwargs)
def action_submit(self, data={}, **kwargs):
""" Helper for creating required POST parameters for save action """
return self._base_form(action='submit',
current_field='submit',
data=data,
**kwargs)
#
# Utility methods
#
def get_file_fixture(self, type, style):
"""
Retrieve file fixture for publication type if it exists, or generate a
PDF on the file.
"""
if type in FILE_FIXTURES:
name, base64_file = FILE_FIXTURES[type]
return (StringIO(binascii.a2b_base64(base64_file)), name)
else:
return (StringIO(
self.make_stringio_pdf("Type: %s Style: %s" % (type, style))),
'%s_%s_file.pdf' % (type.lower(), style))
def make_stringio_pdf(self, text):
"""
Generate a PDF which includes the text given as parameter to this function.
"""
from reportlab.pdfgen import canvas
output = StringIO()
c = canvas.Canvas(output)
c.drawString(100, 100, text)
c.showPage()
c.save()
return output.getvalue()
def get_publications_for_project(self, project_id, style='invenio'):
""" Get all publications for a project """
RE_DELETE_LINK = re.compile(
"/deposit\?projectid=%s&delete=([a-z0-9]+)&ln=" %
project_id, re.IGNORECASE)
resp = self.client.get("/deposit",
query_string={
'style': style,
'ln': 'en',
'projectid': project_id
})
return filter(lambda x: x != self.pub_id,
RE_DELETE_LINK.findall(
resp.data)) # Filter out pub from setUp
def clear_publications(self, project_id, style='invenio'):
""" Clear all publications for a project """
pub_ids = self.get_publications_for_project(project_id, style=style)
for pub_id in pub_ids:
resp = self.client.get("/deposit",
query_string={
'projectid': project_id,
'delete': pub_id,
'ln': ''
})
self.assertEqual(resp.status_code, 200)
def save_metadata(self, project_id, pub_id, fixture):
""" Load a fixture and save it for the given publication """
res = self.client.post("/deposit/ajaxgateway",
data=self.action_save(data=fixture,
override_pubid=pub_id,
projectid=project_id))
for field, error in res.json['errors'].items():
self.assertEqual(error,
'',
msg="Save unsuccessful - %s error: %s" %
(field, error))
self.assertPublicationMetadata(pub_id, fixture)
def approve_record(self, recid):
""" Approve a record to make it publicly available """
# Make MARCXML to approve record
rec = {}
record_add_field(rec, '001', controlfield_value=str(recid))
record_add_field(rec, '980', subfields=[('a', 'OPENAIRE')])
output = "<collection>%s</collection>" % record_xml_output(rec)
# Upload MARCXML
run_sql("TRUNCATE schTASK") # Ensures we run bibupload
(hdl, marcxml_path) = mkstemp(suffix=".xml", text=True)
open(marcxml_path, 'w').write(output)
task_low_level_submission('bibupload', 'openaire', '-c', marcxml_path,
'-P5')
task_low_level_submission('bibindex', 'openaire')
task_low_level_submission('webcoll', 'openaire')
os.system("%s/bin/bibupload 1 > /dev/null" % CFG_PREFIX)
os.system("%s/bin/bibindex 2 > /dev/null" % CFG_PREFIX)
os.system("%s/bin/webcoll 3 > /dev/null" % CFG_PREFIX)
#
# Assert methods
#
def assertPublicationMetadata(self, pub_id, expected_metadata):
"""
Assert that field values of a publication is equal to those
given in the expected_metadata dictionary.
"""
pub = OpenAIREPublication(self.user_id, publicationid=pub_id)
metadata = pub.metadata
for field, expected_val in expected_metadata.items():
if field == 'projects':
continue
expected_val = wash_for_xml(expected_val)
real_val = metadata.get(field, None)
if field in ['related_publications', 'related_datasets']:
# Remove "doi:" and filter out blank strings.
real_val = filter(lambda x: x, real_val.split("\n"))
def _map_func(x):
if x.startswith("doi:"):
return x[4:]
else:
return x
expected_val = filter(lambda x: x,
map(_map_func, expected_val.split("\n")))
self.assertEqual(
real_val, expected_val, "Field %s: expected %s but got %s" %
(field, expected_val, real_val))
# ================================
# Tests
# ================================
def setUp(self):
"""
Create test client, login with the given user and create a new publication.
"""
if self.user_id == None:
res = run_sql("SELECT id FROM user WHERE nickname='admin'")
assert (len(res) == 1, "Couldn't find admin user")
self.user_id = int(res[0][0])
uname = get_nickname(self.user_id)
self.assertEqual(uname, "admin")
self.client = TestClient(response_wrapper=JSONResponse)
self.client.login(uname, '')
self.client_noauth = TestClient(response_wrapper=JSONResponse)
self.pub = OpenAIREPublication(self.user_id)
self.pub_id = self.pub.publicationid
def tearDown(self):
"""
Remove the publication again.
"""
self.pub.delete()
self.pub = None
self.pub_id = None
def test_verify_field(self):
"""
Testing of verify field action
python -m unittest invenio.openaire_deposit_webinterface_tests.AjaxGatewayTest.test_verify_field
"""
# Language
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'language', 'eng'))
self.assertEqual(res.json['errors']['language_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field('language', ''))
self.assertNotEqual(res.json['errors']['language_%s' % self.pub_id],
'')
# Publication type
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'publication_type', 'publishedArticle'))
self.assertEqual(
res.json['errors']['publication_type_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'publication_type', 'dfgsdfgsd'))
self.assertNotEqual(
res.json['errors']['publication_type_%s' % self.pub_id], '')
# Report pages no
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'report_pages_no', '6545'))
self.assertEqual(
res.json['errors']['report_pages_no_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'report_pages_no', 'dfgsdfgsd'))
self.assertNotEqual(
res.json['errors']['report_pages_no_%s' % self.pub_id], '')
# Keywords
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'keywords', 'multiline\nkeywords'))
self.assertEqual(res.json['errors']['keywords_%s' % self.pub_id], '')
self.assertEqual(res.json['warnings']['keywords_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'keywords',
'this;should;probably;be;separated'))
self.assertEqual(res.json['errors']['keywords_%s' % self.pub_id], '')
self.assertNotEqual(res.json['warnings']['keywords_%s' % self.pub_id],
'')
# Accept CC0 license
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'accept_cc0_license',
'yes',
extra_data={'publication_type': 'data'}))
self.assertEqual(
res.json['errors']['accept_cc0_license_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['accept_cc0_license_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'accept_cc0_license',
'',
extra_data={'publication_type': 'data'}))
self.assertNotEqual(
res.json['errors']['accept_cc0_license_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['accept_cc0_license_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'accept_cc0_license',
'',
extra_data={'publication_type': 'report'}))
self.assertEqual(
res.json['errors']['accept_cc0_license_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['accept_cc0_license_%s' % self.pub_id], '')
# Related publications
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'related_publications',
'10.1234/1234',
extra_data={'publication_type': 'data'}))
self.assertEqual(
res.json['errors']['related_publications_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['related_publications_%s' % self.pub_id], '')
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
'related_publications',
'10.1234/1234\n\n12.2323',
extra_data={'publication_type': 'data'}))
self.assertNotEqual(
res.json['errors']['related_publications_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['related_publications_%s' % self.pub_id], '')
res = self.client.post(
"/deposit/ajaxgateway",
data=self.action_verify_field(
'related_publications',
'10.1234/1234\n\n10.1234/12342\n10.1234/12342',
extra_data={'publication_type': 'data'}))
self.assertEqual(
res.json['errors']['related_publications_%s' % self.pub_id], '')
self.assertEqual(
res.json['warnings']['related_publications_%s' % self.pub_id], '')
# Ensure all checks are executed to test for possible runtime errors. Validate is not needed
for field in CFG_METADATA_FIELDS_CHECKS:
res = self.client.post("/deposit/ajaxgateway",
data=self.action_verify_field(
field, 'test'))
def test_action_save_published_article(self):
"""
Testing of save action
"""
data = {
'access_rights': 'openAccess',
'embargo_date': '',
'authors': '',
'title': 'some title',
'abstract': '',
'language': '',
'original_title': '',
'original_abstract': '',
'publication_type': 'publishedArticle',
'publication_date': '',
'journal_title': '',
'doi': '',
'volume': '',
'issue': '',
'pages': '',
'keywords': '',
'notes': '',
}
res = self.client.post("/deposit/ajaxgateway",
data=self.action_save(data=data))
self.assertPublicationMetadata(
self.pub_id, {'publication_type': 'publishedArticle'})
def _submit(self, type, style, submit=True):
"""
Make a submission
"""
resp = self.client.get("/deposit",
query_string={
'style': style,
'ln': 'en'
})
self.assertEqual(resp.status_code, 200)
# Check if there's any existing submissions
self.clear_publications('0', style=style)
self.clear_publications(self.project_id, style=style)
# Simple file upload
(file_stringio, filename) = self.get_file_fixture(type, style)
resp = self.client.post(
"/deposit",
query_string={
'style': style,
'ln': 'en',
'projectid': '0'
},
data={
'Filedata': (file_stringio, filename),
'upload': 'Upload'
},
)
self.assertEqual(resp.status_code, 200)
pub_ids = self.get_publications_for_project('0', style=style)
self.assertEqual(len(pub_ids), 1)
pub_id = pub_ids[0]
# Link with project
resp = self.client.post(
"/deposit/ajaxgateway",
data={
'projectid': self.project_id,
'publicationid': pub_id,
'action': 'linkproject'
},
)
self.assertEqual(resp.json['errors']['projects_%s' % pub_id], '')
# Save metadata
fixture = FIXTURES[type]
self.save_metadata(self.project_id, pub_id, fixture)
# Submit publication
if submit:
run_sql("TRUNCATE schTASK") # Ensures we run bibupload
res = self.client.post("/deposit/ajaxgateway",
data=self.action_submit(
data=fixture,
override_pubid=pub_id,
projectid=self.project_id))
self.assertIn(
"This is a preview of the submitted publication. If approved, it will be available at ",
res.json['appends']['#submitted_publications'])
# Get record id
m = re.search("/record/(\d+)\"",
res.json['appends']['#submitted_publications'])
self.assertIsNotNone(m, "Couldn't find record id.")
rec_id = m.group(1)
# Run bibupload to get record
from invenio.config import CFG_PREFIX
os.system("%s/bin/bibupload 1" % CFG_PREFIX)
# Approve record so it's publicly available.
self.approve_record(rec_id)
# Check if record is reachable
res = self.client_noauth.get("/record/%s" % rec_id)
self.assertEqual(res.status_code, 200)
res = self.client_noauth.get("/record/%s/files/%s" %
(rec_id, filename))
if fixture["access_rights"] in ["embargoedAccess", \
"restrictedAccess", "closedAccess"] and \
type not in CFG_OPENAIRE_CC0_PUBLICATION_TYPES:
self.assertEqual(res.status_code, 302) # Restricted access.
else:
self.assertEqual(res.status_code, 200)
# Remove submission again
self.clear_publications('0', style=style)
self.clear_publications(self.project_id, style=style)
def test_submission_published_article(self):
self._submit('publishedArticle',
'portal',
submit=self.test_full_submit)
def test_submission_report(self):
self._submit('report', 'invenio', submit=self.test_full_submit)
def test_submission_data(self):
self._submit('data', 'portal', submit=self.test_full_submit)
def test_submission_thesis(self):
self._submit('thesis', 'invenio', submit=self.test_full_submit)
def test_submission_book(self):
self._submit('book', 'portal', submit=self.test_full_submit)
def test_submission_bookpart(self):
self._submit('bookpart', 'portal', submit=self.test_full_submit)
def test_submission_conference(self):
self._submit('conferenceContribution',
'portal',
submit=self.test_full_submit)
def test_submission_preprint(self):
self._submit('preprint', 'portal', submit=self.test_full_submit)
def test_submission_workingpaper(self):
self._submit('workingPaper', 'portal', submit=self.test_full_submit)
def edit(self, req, form):
argd = wash_urlargd(
form,
{
"projectid": (int, -1),
#'delete': (str, ''),
"id": (str, ""),
"a": (str, "edit"),
#'linkproject': (int, -1),
#'unlinkproject': (int, -1),
# style': (str, None),
#'upload': (str, ''),
#'dropbox': (str, '')
},
)
_ = gettext_set_language(argd["ln"])
# Check if user is authorized to deposit publications
user_info = collect_user_info(req)
auth_code, auth_message = acc_authorize_action(user_info, "submit", doctype="OpenAIRE")
if auth_code:
if user_info["guest"] == "1":
return redirect_to_url(
req,
"%s/youraccount/login%s"
% (
CFG_SITE_SECURE_URL,
make_canonical_urlargd(
{"referer": "%s%s" % (CFG_SITE_URL, req.unparsed_uri), "ln": argd["ln"]}, {}
),
),
)
else:
return page(
req=req,
body=_("You are not authorized to use OpenAIRE deposition."),
title=_("Authorization failure"),
navmenuid="submit",
)
# Validate action
action = argd["a"]
if action not in ["edit", "save", "submit", "delete"]:
abort(404)
uid = user_info["uid"]
if not argd["id"]:
return redirect("/deposit/")
try:
pub = OpenAIREPublication(uid, publicationid=argd["id"])
title = pub.metadata.get("title", "Untitled")
except ValueError:
abort(404)
#
# Action handling
#
if action == "delete":
pub.delete()
flash("Upload '%s' was deleted." % title, "info")
return redirect("/deposit/")
elif action == "edit":
ctx = {"pub": pub, "title": title, "recid": pub.metadata.get("__recid__", None)}
ctx["rendered_form"] = openaire_deposit_templates.tmpl_form(
pub.metadata["__publicationid__"],
-1,
"", # projects_information
"", # publication_information
"", # fulltext_information
form=None,
metadata_status="empty",
warnings=None,
errors=None,
ln="en",
)
# if action == "edit":
# elif action == "delete":
# pass
# if projectid >= 0:
# ## There is a project on which we are working good!
# publications = get_all_publications_for_project(
# uid, projectid, ln=argd['ln'], style=style)
# if argd['publicationid'] in publications:
# if argd['addproject'] in all_project_ids:
# publications[argd['publicationid']
# ].link_project(argd['linkproject'])
# if argd['delproject'] in all_project_ids:
# publications[argd['publicationid']
# ].unlink_project(argd['unlinkproject'])
# if argd['delete'] and argd['delete'] in publications:
# ## there was a request to delete a publication
# publications[argd['delete']].delete()
# del publications[argd['delete']]
# forms = ""
# submitted_publications = ""
# for index, (publicationid, publication) in enumerate(publications.iteritems()):
# if req.method.upper() == 'POST':
# publication.merge_form(form, ln=argd['ln'])
# if publication.status == 'edited':
# publication.check_metadata()
# publication.check_projects()
# if 'submit_%s' % publicationid in form and not "".join(publication.errors.values()).strip():
# ## i.e. if the button submit for the corresponding publication has been pressed...
# publication.upload_record()
# if publication.status in ('initialized', 'edited'):
# forms += publication.get_publication_form(projectid)
# else:
# submitted_publications += publication.get_publication_preview()
# body += openaire_deposit_templates.tmpl_add_publication_data_and_submit(projectid, forms, submitted_publications, project_information=upload_to_project_information, ln=argd['ln'])
# body += openaire_deposit_templates.tmpl_upload_publications(projectid=upload_to_projectid, project_information=upload_to_project_information, session=get_session(req).sid.encode('utf8'), style=style, ln=argd['ln'])
# else:
ctx.update({"myresearch": get_exisiting_publications_for_uid(uid)})
return render_template(
"openaire_edit.html", req=req, breadcrumbs=[(_("Home"), ""), (_("Upload"), "deposit"), (title, "")], **ctx
).encode("utf8")
def ajaxgateway(self, req, form):
"""
"""
argd = wash_urlargd(
form, {
'projectid': (str, ''),
'publicationid': (str, ''),
'action': (str, ''),
'current_field': (str, '')
})
# Get parameters
action = argd['action']
publicationid = argd['publicationid']
projectid = argd['projectid']
# Check if action is supported
assert (action in ('save', 'verify_field', 'submit', 'unlinkproject',
'linkproject'))
# JSON return dictionary
out = {
'errors': {},
'warnings': {},
'addclasses': {},
'delclasses': {},
'substitutions': {},
'appends': {},
'hiddens': [],
'showns': [],
'action': action,
}
if action == 'verify_field':
current_field = argd['current_field']
assert (current_field)
metadata = wash_form(form, publicationid)
out["errors"], out[
"warnings"] = OpenAIREPublication.static_check_metadata(
metadata,
publicationid,
check_only_field=current_field,
ln=argd['ln'])
else:
user_info = collect_user_info(req)
auth_code, auth_message = acc_authorize_action(user_info,
'submit',
doctype='OpenAIRE')
assert (auth_code == 0)
uid = user_info['uid']
publication = OpenAIREPublication(uid,
publicationid,
ln=argd['ln'])
if action == 'unlinkproject':
publication.unlink_project(projectid)
out["substitutions"][
"#projectsbox_%s" %
publicationid] = publication.get_projects_information()
publication.check_projects()
out["errors"], out[
"warnings"] = simple_metadata2namespaced_metadata(
publication.errors,
publicationid), simple_metadata2namespaced_metadata(
publication.warnings, publicationid)
elif action == 'linkproject':
publication.link_project(projectid)
out["substitutions"][
"#projectsbox_%s" %
publicationid] = publication.get_projects_information()
publication.check_projects()
out["errors"], out[
"warnings"] = simple_metadata2namespaced_metadata(
publication.errors,
publicationid), simple_metadata2namespaced_metadata(
publication.warnings, publicationid)
else:
publication.merge_form(form)
publication.check_metadata()
publication.check_projects()
out["errors"], out[
"warnings"] = simple_metadata2namespaced_metadata(
publication.errors,
publicationid), simple_metadata2namespaced_metadata(
publication.warnings, publicationid)
if "".join(out["errors"].values()).strip(
): # FIXME bad hack, we need a cleaner way to discover if there are errors
out['addclasses']['#status_%s' % publicationid] = 'error'
out['delclasses']['#status_%s' %
publicationid] = 'warning ok empty'
elif "".join(out["warnings"].values()).strip():
out['addclasses']['#status_%s' % publicationid] = 'warning'
out['delclasses']['#status_%s' %
publicationid] = 'error ok empty'
else:
out['addclasses']['#status_%s' % publicationid] = 'ok'
out['delclasses']['#status_%s' %
publicationid] = 'warning error empty'
if action == 'save':
out["substitutions"][
'#publication_information_%s' %
publicationid] = publication.get_publication_information(
)
elif action == 'submit':
if not "".join(out["errors"].values()).strip():
publication.upload_record()
out["appends"][
'#submitted_publications'] = publication.get_publication_preview(
)
out["showns"].append('#submitted_publications')
out["hiddens"].append('#header_row_%s' % publicationid)
out["hiddens"].append('#body_row_%s' % publicationid)
req.content_type = 'application/json'
return json.dumps(out)
