def test_unmerge_orcid_nones(self):
# First, fetch a few DOIs
dois = [
"10.1075/aicr.90.09ngo",
"10.1075/aicr.90.04wad",
]
for doi in dois:
Paper.create_by_doi(doi)
# Then, fetch an ORCID profile with a buggy version of the ORCID interface, which incorrectly merges papers together
with patch.object(OrcidPaperSource, '_oai_id_for_doi') as mock_identifier:
mock_identifier.return_value = "https://pub.orcid.org/v2.1/0000-0002-1909-134X/work/None"
profile = OrcidProfileStub('0000-0002-1909-134X', instance='orcid.org')
trung = Researcher.get_or_create_by_orcid('0000-0002-1909-134X', profile=profile)
OrcidPaperSource().fetch_and_save(trung, profile=profile)
# The two papers are incorrectly merged!
papers = [Paper.get_by_doi(doi) for doi in dois]
self.assertEqual(papers[0], papers[1])
# We unmerge them
unmerge_orcid_nones()
# The two papers are now distinct
papers = [Paper.get_by_doi(doi) for doi in dois]
self.assertTrue(papers[0] != papers[1])
def test_query(self):
invalid_payloads = [
'test', '{}',
'{"doi":"anurisecbld"}',
'{"title":""}',
'{"title":"this is a test"}',
'{"title":"this is a test","date":"aunriset"}',
'{"title":"this is a test","date":"2008"}',
'{"title":"this is a test","date":"2008","authors":"test"}',
'{"title":"this is a test","date":"2008-03","authors":[]}',
'{"title":"this is a test","date":"2008-03","authors":["lsc"]}',
'{"title":"test","date":"2008-03","authors":[{"error":"test"}]}',
]
for payload in invalid_payloads:
self.checkJson(self.postPage('api-paper-query', postargs=payload,
postkwargs={'content_type': 'application/json'}), 400)
# Paper not found
payload = '{"title":"Refining the Conceptualization of a Future-Oriented Self-Regulatory Behavior: Proactive Coping", "date":"2009-07-01","authors":[{"first":"Stephanie Jean","last":"Sohl"},{"first":"Anne","last":"Moyer"}]}'
self.checkJson(self.postPage('api-paper-query', postargs=payload,
postkwargs={'content_type': 'application/json'}), 404)
Paper.create_by_doi('10.1016/j.paid.2009.02.013')
# Paper now found
self.checkJson(self.postPage('api-paper-query', postargs=payload,
postkwargs={'content_type': 'application/json'}), 200)
self.checkJson(self.postPage('api-paper-query', postargs='{"doi":"10.1016/j.paid.2009.02.013"}',
postkwargs={'content_type': 'application/json'}), 200)
def unmerge_paper_by_dois(paper):
"""
Given a paper that was merged by mistake, delete it and re-create
it with each DOI found in it.
"""
dois = [record.doi for record in paper.oairecords if record.doi]
paper.delete()
for doi in dois:
try:
Paper.create_by_doi(doi)
except ValueError:
continue
def test_ingest_dump(self):
doi = '10.1080/21645515.2017.1330236'
p = Paper.create_by_doi(doi)
self.assertEqual(p.pdf_url, None)
Paper.create_by_doi(doi)
# then load an OAdoi dump
oadoi = OadoiAPI()
oadoi.load_dump(os.path.join(self.testdir, 'data/sample_unpaywall_snapshot.jsonl.gz'))
# the paper is now OA, yay!
p = Paper.get_by_doi(doi)
self.assertEqual(p.pdf_url, 'http://europepmc.org/articles/pmc5718814?pdf=render')
def get_object(self, queryset=None):
if queryset is None:
queryset = self.get_queryset()
pk = self.kwargs.get('pk', None)
doi = self.kwargs.get('doi', None)
if doi:
doi = unquote(doi)
doi = to_doi(doi)
paper = None
try:
if pk is not None:
paper = queryset.get(pk=pk)
elif doi is not None:
paper = Paper.get_by_doi(doi)
else:
raise Http404(_("Paper view expects a DOI or a pk"))
except ObjectDoesNotExist:
pass
if not paper:
paper = Paper.create_by_doi(doi)
if paper is None or paper.is_orphan():
raise Http404(_("No %(verbose_name)s found matching the query") %
{'verbose_name': Paper._meta.verbose_name})
if not paper.visible:
raise Http404(_("This paper has been deleted."))
return paper
def test_refresh_deposit_status(self):
# This is the identifier of a paper which should
# currently be published on HAL preprod
hal_id = 'hal-01211282'
# First, fake the deposition of a paper
p = Paper.create_by_doi('10.1109/lics.2015.37')
r = OaiRecord.new(source=self.repo.oaisource,
identifier='deposition:1:'+hal_id,
splash_url='https://hal-preprod.archives-ouvertes.fr/'+hal_id,
pdf_url=None,
about=p)
f = UploadedPDF.objects.create(
user=self.user,
orig_name='File.pdf',
file=os.path.join(self.testdir, 'testdata/blank.pdf'),
thumbnail='my_thumbnail.png')
d = DepositRecord.objects.create(
paper=p,
oairecord=r,
repository=self.repo,
user=self.user,
status='pending',
identifier=hal_id,
upload_type='postprint',
file=f)
self.proto.refresh_deposit_status(d)
self.assertEqual(d.status, 'published')
self.assertTrue(r.pdf_url)
def test_paper_already_in_hal_but_not_in_dissemin(self):
"""
In this case, Dissemin missed the paper on HAL
(for some reason) and so the deposit interface was
enabled. But HAL refuses the deposit! We have to
give a good error message to the user.
"""
# this paper is currently in HAL-preprod
p = Paper.create_by_doi('10.1051/jphys:01975003607-8060700')
# this is just to make sure that we are depositing with
# a single author (otherwise, the deposit would fail because
# we are not providing enough affiliations).
p.authors_list = [p.authors_list[0]]
r = self.dry_deposit(p,
abstract='this is an abstract',
topic='INFO',
depositing_author=0,
affiliation=59704) # ENS
# Deposit fails: a duplicate is found
self.assertEqualOrLog(r.status, 'failed')
# The error message should be specific
self.assertTrue('already in HAL' in r.message)
def test_500_error(self):
with requests_mock.Mocker(real_http=True) as mocker:
mocker.get(re.compile(r'.*\.zenodo\.org/.*'), status_code=500)
p = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
r = self.dry_deposit(p,
abstract = lorem_ipsum,
license = ZENODO_DEFAULT_LICENSE_CHOICE)
self.assertEqual(r.status, 'failed')
def test_unmerge_paper(self):
# First we merge two unrelated papers
p1 = Paper.create_by_doi("10.1016/j.bmc.2005.06.035")
title1 = p1.title
p2 = Paper.create_by_doi("10.1016/j.ijar.2017.06.011")
title2 = p2.title
p1.merge(p2)
# Then we unmerge them
unmerge_paper_by_dois(p1)
# We have two new papers!
p3 = Paper.get_by_doi("10.1016/j.bmc.2005.06.035")
self.assertTrue(p3.id != p1.id)
self.assertEqual(p3.title, title1)
p4 = Paper.get_by_doi("10.1016/j.ijar.2017.06.011")
self.assertTrue(p4.id != p1.id)
self.assertTrue(p4.id != p3.id)
self.assertEqual(p4.title, title2)
def test_invisible_paper(self):
"""
If a paper is marked as invisible, then accessing it returns 404
"""
p = Paper.create_by_doi('10.1007/978-3-642-14363-2_7')
p.visible = False
p.save()
self.check404('paper', kwargs={'pk': p.id, 'slug': p.slug})
def test_bibtex_formatting(self):
dois_bibtex = {
'10.1007/978-3-662-49214-7_4': '''@incollection{Tang2016,
author = {Tang, Ruiming and Amarilli, Antoine and Senellart, Pierre and Bressan, Stéphane},
doi = {10.1007/978-3-662-49214-7_4},
journal = {Transactions on Large-Scale Data- and Knowledge-Centered Systems XXIV},
month = {jan},
pages = {116-138},
title = {A Framework for Sampling-Based XML Data Pricing},
url = {https://oadoi.org/10.1007/978-3-662-49214-7_4},
year = {2016}
}''',
'10.1145/3034786.3056121': '''@misc{Amarilli2017,
author = {Amarilli, Antoine and Monet, Mikaël and Senellart, Pierre},
doi = {10.1145/3034786.3056121},
journal = {Proceedings of the 36th ACM SIGMOD-SIGACT-SIGAI Symposium on Principles of Database Systems - PODS '17},
month = {jan},
title = {Conjunctive Queries on Probabilistic Graphs: Combined Complexity},
url = {https://oadoi.org/10.1145/3034786.3056121},
year = {2017}
}''',
'10.1007/978-3-319-45856-4_22': '''@incollection{Amarilli2016,
author = {Amarilli, Antoine and Maniu, Silviu and Monet, Mikaël},
doi = {10.1007/978-3-319-45856-4_22},
journal = {Lecture Notes in Computer Science},
month = {jan},
pages = {323-330},
title = {Challenges for Efficient Query Evaluation on Structured Probabilistic Data},
url = {https://oadoi.org/10.1007/978-3-319-45856-4_22},
year = {2016}
}''',
'10.1103/physrevapplied.11.024003': '''@misc{Verney2019,
author = {Verney, Lucas and Lescanne, Raphaël and Devoret, Michel H. and Leghtas, Zaki and Mirrahimi, Mazyar},
doi = {10.1103/physrevapplied.11.024003},
journal = {Physical Review Applied},
month = {feb},
title = {Structural Instability of Driven Josephson Circuits Prevented by an Inductive Shunt},
url = {https://oadoi.org/10.1103/physrevapplied.11.024003},
volume = {11},
year = {2019}
}''',
}
for doi, bibtex in dois_bibtex.items():
p = Paper.create_by_doi(doi)
resp = self.getPage('api-paper-doi',
args=[doi], getargs={'format': 'bibtex'})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content.decode('utf-8').strip(),
bibtex.strip())
resp = self.getPage('api-paper-pk',
args=[p.id], getargs={'format': 'bibtex'})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content.decode('utf-8').strip(),
bibtex.strip())
def test_paper_by_doi_escaped(self):
"""
Automatically unescape DOIs, for issue
https://github.com/dissemin/dissemin/issues/517
"""
paper = Paper.create_by_doi('10.1175/1520-0426(2003)020<0383%3ARCAACO>2.0.CO%3B2')
paper.save()
self.checkPermanentRedirect('paper-doi',
kwargs={'doi':'10.1175%2F1520-0426%282003%29020%3C0383%3ARCAACO%3E2.0.CO%3B2'},
url=paper.url)
def test_consolidate_paper(self):
p = Paper.create_by_doi('10.1175/jas-d-15-0240.1')
self.client.login(username='******', password='******')
result = self.checkJson(self.getPage(
'ajax-waitForConsolidatedField', getargs={
'field': 'abstract',
'id': p.id}))
self.client.logout()
self.assertTrue(result['success'])
self.assertTrue(len(result['value']) > 10)
def test_submit_deposit(self):
paper = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
request = self.dry_deposit(
paper,
license='58fd62fcda3e2400012ca5d3',
abstract='Salagadoola menchicka boola bibbidi-bobbidi-boo.',
subjects=['59552884da3e240081ba32de'],
tags='Pumpkin, Mouse, Godmother')
self.assertEqualOrLog(request.status, 'published')
def test_submit_deposit_nolicense(self):
paper = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
request = self.dry_deposit(
paper,
license='58fd62fcda3e2400012ca5cc',
abstract='Higitus Figitus Migitus Mum.',
subjects=['59552884da3e240081ba32de'],
tags='Sword, King, Wizard')
self.assertEqualOrLog(request.status, 'published')
def test_get_form_initial_data(self):
paper = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
record = paper.oairecords[0]
record_value = "Supercalifragilisticexpialidocious."
record.description = record_value
record.save()
self.proto.init_deposit(paper, self.user)
data = self.proto.get_form_initial_data()
self.assertIsInstance(data, dict)
self.assertEqual(data.get('abstract'), record_value)
def test_lncs(self):
"""
Same as test_lncs but with only one author
"""
p = Paper.create_by_doi('10.1007/978-3-319-63342-8_1')
p.authors_list = [p.authors_list[0]]
r = self.dry_deposit(p,
abstract='this is an abstract',
topic='INFO',
depositing_author=0,
affiliation=59704) # ENS
self.assertEqualOrLog(r.status, 'faked')
def test_keywords(self):
"""
Keywords are mandatory
"""
p = Paper.create_by_doi('10.1007/s00268-016-3429-x')
p.authors_list = [p.authors_list[0]]
r = self.dry_deposit(p,
abstract='bla ble bli blo blu',
topic='SDV',
depositing_author=0,
affiliation=128940)
self.assertEqualOrLog(r.status, 'faked')
def test_lics(self):
"""
Submit a paper from LICS (type: conference-proceedings)
"""
p = Paper.create_by_doi('10.1109/lics.2015.37')
p.authors_list = [p.authors_list[0]]
r = self.dry_deposit(p,
abstract='here is my great result',
topic='NLIN',
depositing_author=0,
affiliation=128940)
self.assertEqualOrLog(r.status, 'faked')
def test_journal_article(self):
"""
Submit a journal article
"""
p = Paper.create_by_doi('10.1016/j.agee.2004.10.001')
p.authors_list = [p.authors_list[0]]
r = self.dry_deposit(p,
abstract='here is my great result',
topic='SDV',
depositing_author=0,
affiliation=128940)
self.assertEqualOrLog(r.status, 'faked')
def test_bibtex_formatting_search(self):
bibtex_output = """@inproceedings{Amarilli2015,
author = {Amarilli, Antoine},
doi = {10.1145/2744680.2744690},
journal = {Proceedings of the 2015 ACM SIGMOD on PhD Symposium - SIGMOD '15 PhD Symposium},
month = {jan},
title = {Structurally Tractable Uncertain Data},
url = {https://oadoi.org/10.1145/2744680.2744690},
year = {2015}
}
@inproceedings{Amarilli2015_2,
author = {Amarilli, Antoine and Benedikt, Michael},
doi = {10.1109/lics.2015.37},
journal = {2015 30th Annual ACM/IEEE Symposium on Logic in Computer Science},
month = {jul},
title = {Finite Open-World Query Answering with Number Restrictions},
url = {https://oadoi.org/10.1109/lics.2015.37},
year = {2015}
}"""
r1 = Researcher.create_by_name('John', 'Doe')
p1 = Paper.create_by_doi('10.1109/lics.2015.37')
p1.set_researcher(0, r1.id)
p1.update_index() # Ensure index is updated
p2 = Paper.create_by_doi('10.1145/2744680.2744690')
p2.set_researcher(0, r1.id)
p2.update_index() # Ensure index is updated
resp = self.getPage(
'api-paper-search',
getargs={'authors': 'amarilli', 'format': 'bibtex'}
)
self.assertEqual(resp.status_code, 200)
self.assertEqual(
resp.content.decode('utf-8').strip(),
bibtex_output.strip()
)
def api_paper_doi(request, doi):
p = None
try:
p = Paper.get_by_doi(doi)
if not p:
p = Paper.create_by_doi(doi)
except MetadataSourceException:
pass
if p is None:
return JsonResponse({
'error': 404,
'message': 'The paper you requested could not be found.',
}, status=404)
return api_paper_common(request, p)
def test_lncs_many_authors(self):
"""
Submit a paper from LNCS (type: book-chapter).
This fails with the default test account because
it does not have the right to deposit with only one
affiliation.
"""
# the DOI below should *not* already exist in HAL
# so it may need to be changed if the test fails
p = Paper.create_by_doi('10.1007/978-3-319-63342-8_1')
r = self.dry_deposit(p,
abstract='this is an abstract',
topic='INFO',
depositing_author=0,
affiliation=59704) # ENS
self.assertEqualOrLog(r.status, 'faked')
def test_deposit_on_behalf_of(self):
paper = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
prefs = self.proto.get_preferences(self.user)
prefs.on_behalf_of = 'mweg3' # sample user id on the sandbox
# with name "Jean Saisrien"
paper.add_author(BareAuthor(name=BareName.create_bare('Jean', 'Saisrien')))
paper.save()
request = self.dry_deposit(
paper,
license='58fd62fcda3e2400012ca5d3',
abstract='Salagadoola menchicka boola bibbidi-bobbidi-boo.',
subjects=['59552884da3e240081ba32de'],
tags='Pumpkin, Mouse, Godmother')
self.assertEqualOrLog(request.status, 'published')
def test_deposit_on_behalf_of(self):
paper = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
prefs = self.proto.get_preferences(self.user)
prefs.on_behalf_of = 'mweg3' # sample user id on the sandbox
# with name "Jean Saisrien"
paper.add_author(
BareAuthor(name=BareName.create_bare('Jean', 'Saisrien')))
paper.save()
request = self.dry_deposit(
paper,
license='58fd62fcda3e2400012ca5d3',
abstract='Salagadoola menchicka boola bibbidi-bobbidi-boo.',
subjects=['59552884da3e240081ba32de'],
tags='Pumpkin, Mouse, Godmother')
self.assertEqualOrLog(request.status, 'published')
def test_on_behalf_of(self):
# Set on-behalf-of to some user
# Currently we are using "test_ws" as deposit account
# and we own "dissemin" so let's deposit on-behalf-of dissemin
# although normally this would be replaced by the user's account.
preferences = self.proto.get_preferences(self.user)
preferences.on_behalf_of = 'pintochtest'
preferences.save()
p = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
p.authors_list = [p.authors_list[0]]
r = self.dry_deposit(p,
abstract='this is an abstract',
topic='INFO',
depositing_author=0,
affiliation=59704) # ENS
self.assertEqualOrLog(r.status, 'faked')
def test_on_behalf_of(self):
# Set on-behalf-of to some user
# Currently we are using "test_ws" as deposit account
preferences = self.proto.get_preferences(self.user)
preferences.on_behalf_of = 'dissemin_test'
preferences.save()
# the DOI here should *not* already exist in HAL
# so it may need to be changed if the test fails
p = Paper.create_by_doi('10.1007/978-3-319-63342-8_1')
p.authors_list = [p.authors_list[0]]
r = self.dry_deposit(p,
abstract='this is an abstract',
topic='INFO',
depositing_author=0,
affiliation=59704) # ENS
self.assertEqualOrLog(r.status, 'faked')
def api_paper_doi(request, doi):
p = None
try:
p = Paper.get_by_doi(doi)
if not p:
p = Paper.create_by_doi(doi)
except MetadataSourceException:
pass
if p is None:
return JsonResponse(
{
'error': 404,
'message': 'The paper you requested could not be found.',
},
status=404)
return api_paper_common(request, p)
def test_on_behalf_of(self):
# Set on-behalf-of to some user
# Currently we are using "test_ws" as deposit account
preferences = self.proto.get_preferences(self.user)
preferences.on_behalf_of = 'dissemin_test'
preferences.save()
# the DOI here should *not* already exist in HAL
# so it may need to be changed if the test fails
p = Paper.create_by_doi('10.1007/978-3-319-63342-8_1')
p.authors_list = [p.authors_list[0]]
r = self.dry_deposit(p,
abstract='this is an abstract',
topic='INFO',
depositing_author=0,
affiliation=59704) # ENS
self.assertEqualOrLog(r.status, 'faked')
def test_owned_by(self):
p1 = Paper.create_by_doi('10.4049/jimmunol.167.12.6786')
r1 = Researcher.create_by_name('Stephan', 'Hauschildt')
r1.user, _ = User.objects.get_or_create(username='******')
r1.save()
p1.set_researcher(4, r1.id)
# The user is associated to the author in the model,
# so it is considered an owner.
self.assertTrue(p1.is_owned_by(r1.user))
other_user, _ = User.objects.get_or_create(username='******',
first_name='Andrea',
last_name='Thiele')
# This other user is not associated to any researcher,
# so it isn't associated to the paper.
self.assertFalse(p1.is_owned_by(other_user))
# But if we ask for a flexible check, as her name matches
# one of the author names of the paper, she is recognized.
self.assertTrue(p1.is_owned_by(other_user, flexible=True))
def test_paper_already_on_zenodo(self):
"""
In this case, Dissemin missed the paper on Zenodo
(for some reason) and so the deposit interface was
enabled. But Zenodo refuses the deposit! We have to
give a good error message to the user.
"""
p = Paper.create_by_doi('10.5281/zenodo.50134')
r = self.deposit(
p,
abstract=lorem_ipsum,
license=self.lc,
)
# Deposit fails: a duplicate is found
self.assertEqualOrLog(r.status, 'failed')
# The error message should be specific
self.assertTrue('already in Zenodo' in r.message)
def test_paper_already_on_zenodo(self):
"""
In this case, Dissemin missed the paper on Zenodo
(for some reason) and so the deposit interface was
enabled. But Zenodo refuses the deposit! We have to
give a good error message to the user.
"""
p = Paper.create_by_doi('10.5281/zenodo.50134')
r = self.deposit(
p,
abstract = lorem_ipsum,
license = ZENODO_DEFAULT_LICENSE_CHOICE
)
# Deposit fails: a duplicate is found
self.assertEqualOrLog(r.status, 'failed')
# The error message should be specific
self.assertTrue('already in Zenodo' in r.message)
def test_topic_set_to_other(self):
"""
Submit a journal article with "OTHER" as topic,
which is forbidden by HAL
"""
p = Paper.create_by_doi('10.1016/j.agee.2004.10.001')
self.proto.init_deposit(p, self.user)
# the user is presented with initial data
args = self.proto.get_form_initial_data()
# they fill the form with an invalid topic
form_fields = {'abstract':'here is my great result',
'topic':'OTHER',
'depositing_author':0,
'affiliation':128940}
args.update(form_fields)
# the form should reject the "OTHER" topic
form = self.proto.get_bound_form(args)
self.assertFalse(form.is_valid())
def test_topic_set_to_other(self):
"""
Submit a journal article with "OTHER" as topic,
which is forbidden by HAL
"""
p = Paper.create_by_doi('10.1016/j.agee.2004.10.001')
self.proto.init_deposit(p, self.user)
# the user is presented with initial data
args = self.proto.get_form_initial_data()
# they fill the form with an invalid topic
form_fields = {'abstract':'here is my great result',
'topic':'OTHER',
'depositing_author':0,
'affiliation':128940}
args.update(form_fields)
# the form should reject the "OTHER" topic
form = self.proto.get_bound_form(args)
self.assertFalse(form.is_valid())
def test_update_pdf_url(self):
"""
Two OAI records share the same splash URL, but
the second one has a pdf_url. We should add the PDF
url to the existing OAI record (merge the two records).
"""
# first, make sure the paper isn't there already
self.delete('oai:crossref.org:10.1007/s10858-015-9994-8')
# Create a paper from Crossref
first = Paper.create_by_doi('10.1007/s10858-015-9994-8')
# initially the PDF url should be empty
assert not first.oairecords[0].pdf_url
# then we import a new identifier
new_paper = self.create('ftspringeroc:10.1007/s10858-015-9994-8',
'base_dc')
self.assertEqual(first, new_paper)
# no new record should be created
self.assertEqual(len(new_paper.oairecords), 1)
self.assertNotEqual(new_paper.oairecords[0].pdf_url, None)
def test_form_with_no_subject(self):
"""
Submit a preprint with no subject selected,
which is a problem if we want to make it public.
"""
paper = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
self.proto.init_deposit(paper, self.user)
# These are the initial data. No subject given.
data = self.proto.get_form_initial_data()
form_fields = {'licence': '563c1cf88c5e4a3877f9e965',
'abstract': 'This is a fake abstract.',
'tags': 'One, Two, Three, Four',
'subjects': []}
data.update(form_fields)
form = self.proto.get_bound_form(data)
self.assertEqual(form.has_error('subjects', code=None), True)
self.assertEqual(form.errors['subjects'],
['At least one subject is required.'])
self.assertFalse(form.is_valid())
def test_change_publisher(self):
"""
Changing the publisher of a journal should update the
associated papers.
"""
journal = RomeoAPIStub().fetch_journal({'issn':'0892-7537'})
paper = Paper.create_by_doi('10.1007/bf02221836')
self.assertEqual(paper.oairecords[0].journal, journal)
self.assertEqual(paper.oairecords[0].publisher.oa_status, 'OK')
self.assertEqual(paper.oa_status, 'OK')
closed_publisher = Publisher(romeo_id='249384', preprint='cannot', postprint='cannot', pdfversion='cannot')
closed_publisher.save()
journal.change_publisher(closed_publisher)
paper = Paper.objects.get(id=paper.id)
journal = Journal.objects.get(id=journal.id)
self.assertEqual(paper.oa_status, 'UNK')
self.assertEqual(paper.publisher(), closed_publisher)
self.assertEqual(journal.publisher, closed_publisher)
def test_create_match_doi(self):
"""
Addition of an OAI record when it is matched
to an existing paper by DOI
"""
first_id = 'ftunivmacedonia:oai:dspace.lib.uom.gr:2159/6240'
doi = '10.1111/j.1574-0862.2005.00325.x'
# first, make sure the paper isn't there already
self.delete(first_id)
# Create a paper from BASE
first = Paper.create_by_doi(doi)
self.assertEqual(first.oairecords[0].doi, doi)
records = set(first.oairecords)
new_paper = self.create(first_id, 'base_dc')
# Make sure that, if a merge happens, the oldest
# paper remains (otherwise we create broken links!)
self.assertEqual(first, new_paper)
records.add(OaiRecord.objects.get(identifier=first_id))
self.assertEqual(set(new_paper.oairecords), records)
def test_change_publisher(self):
"""
Changing the publisher of a journal should update the
associated papers.
"""
journal = RomeoAPIStub().fetch_journal({'issn': '0892-7537'})
paper = Paper.create_by_doi('10.1007/bf02221836')
self.assertEqual(paper.oairecords[0].journal, journal)
self.assertEqual(paper.oairecords[0].publisher.oa_status, 'OK')
self.assertEqual(paper.oa_status, 'OK')
closed_publisher = Publisher(romeo_id='249384',
preprint='cannot',
postprint='cannot',
pdfversion='cannot')
closed_publisher.save()
journal.change_publisher(closed_publisher)
paper = Paper.objects.get(id=paper.id)
journal = Journal.objects.get(id=journal.id)
self.assertEqual(paper.oa_status, 'UNK')
self.assertEqual(paper.publisher(), closed_publisher)
self.assertEqual(journal.publisher, closed_publisher)
def test_form_with_no_subject(self):
"""
Submit a preprint with no subject selected,
which is a problem if we want to make it public.
"""
paper = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
self.proto.init_deposit(paper, self.user)
# These are the initial data. No subject given.
data = self.proto.get_form_initial_data()
form_fields = {
'licence': '563c1cf88c5e4a3877f9e965',
'abstract': 'This is a fake abstract.',
'tags': 'One, Two, Three, Four',
'subjects': []
}
data.update(form_fields)
form = self.proto.get_bound_form(data)
self.assertEqual(form.has_error('subjects', code=None), True)
self.assertEqual(form.errors['subjects'],
['At least one subject is required.'])
self.assertFalse(form.is_valid())
def test_merge(self):
# Get a paper with metadata
p = Paper.create_by_doi('10.1111/j.1744-6570.1953.tb01038.x')
p = Paper.from_bare(p)
# Create a copy with slight variations
names = [BareName.create_bare(f, l) for (f, l) in
[('M. H.', 'Jones'), ('R. H.', 'Haase'), ('S. F.', 'Hulbert')]]
p2 = Paper.get_or_create(
'A Survey of the Literature on Technical Positions', names,
date(year=2011, month=0o1, day=0o1))
# The two are not merged because of the difference in the title
assert p != p2
# Fix the title of the second one
p2.title = 'A Survey of the Literature on Job Analysis of Technical Positions'
p2.save()
# Check that the new fingerprint is equal to that of the first paper
assert p2.new_fingerprint() == p.fingerprint
# and that the new fingerprint and the current differ
assert p2.new_fingerprint() != p2.fingerprint
# and that the first paper matches its own shit
assert Paper.objects.filter(fingerprint=p.fingerprint).first() == p
# The two papers should hence be merged together
new_paper = p2.recompute_fingerprint_and_merge_if_needed()
assert new_paper.pk == p.pk
def test_create_no_authors(self):
p = Paper.create_by_doi('10.1021/cen-v043n050.p033')
self.assertEqual(p, None)
def test_create_invalid_doi(self):
p = Paper.create_by_doi('10.1021/eiaeuiebop134223cen-v043n050.p033')
self.assertEqual(p, None)
def test_publisher_url(self):
p = Paper.create_by_doi('10.1007/978-3-642-14363-2_7')
for publi in p.publications:
self.checkPage('publisher', kwargs={
'pk': publi.publisher_id, 'slug': publi.publisher.slug})
def test_500_error(self):
with requests_mock.Mocker(real_http=True) as mocker:
mocker.get(re.compile(r'.*\.zenodo\.org/.*'), status_code=500)
p = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
r = self.dry_deposit(p, abstract=lorem_ipsum, license=self.lc)
self.assertEqual(r.status, 'failed')
def test_generate_metadata_doi(self):
# f =
AOFRFormatter()
dois = ['10.1175/jas-d-15-0240.1']
for doi in dois:
Paper.create_by_doi(doi)
def test_lncs(self):
p = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
r = self.dry_deposit(p,
abstract = lorem_ipsum,
license = 'other-open')
self.assertEqual(r.status, 'faked')
def test_lncs(self):
p = Paper.create_by_doi('10.1007/978-3-662-47666-6_5')
r = self.dry_deposit(p,
abstract=lorem_ipsum,
license=ZENODO_DEFAULT_LICENSE_CHOICE)
self.assertEqualOrLog(r.status, 'faked')
def test_create_by_doi_no_authors(self):
p = Paper.create_by_doi('10.1021/cen-v043n050.p033')
assert p is None
def test_visible_paper(self, check_page):
"""
By default, a paper accessed with its pk and slug is visible
"""
p = Paper.create_by_doi('10.1007/978-3-642-14363-2_7')
check_page(200, 'paper', kwargs={'pk': p.id, 'slug': p.slug})
def test_missing_info_in_pub(self, db, check_page):
p = Paper.create_by_doi('10.1007/978-3-642-14363-2_7')
check_page(200, 'paper', kwargs={'pk': p.id, 'slug': p.slug})
def api_paper_query(request):
try:
fields = json.loads(request.body.decode('utf-8'))
except (ValueError, UnicodeDecodeError):
raise BadRequest('Invalid JSON payload')
doi = fields.get('doi')
if doi:
p = None
try:
p = Paper.create_by_doi(doi, bare=True)
except MetadataSourceException:
pass
if p is None:
raise BadRequest('Could not find a paper with this DOI')
return {'status': 'ok', 'paper': p.json()}
title = fields.get('title')
if not isinstance(title, unicode) or not title or len(title) > 512:
raise BadRequest(
'Invalid title, has to be a non-empty string shorter than 512 characters'
)
date = fields.get('date')
if not isinstance(date, unicode):
raise BadRequest('A date is required')
try:
date = tolerant_datestamp_to_datetime(date)
except ValueError as e:
raise BadRequest(unicode(e))
authors = fields.get('authors')
if not isinstance(authors, list):
raise BadRequest('A list of authors is expected')
parsed_authors = []
for a in authors:
author = None
if not isinstance(a, dict):
raise BadRequest('Invalid author')
if 'first' in a and 'last' in a:
if not isinstance(a['first'], unicode) or not isinstance(
a['last'], unicode) or not a['last']:
raise BadRequest('Invalid (first,last) name provided')
else:
author = (a['first'], a['last'])
elif 'plain' in a:
if not isinstance(a['plain'], unicode) or not a['plain']:
raise BadRequest('Invalid plain name provided')
else:
author = parse_comma_name(a['plain'])
if author is None:
raise BadRequest('Invalid author')
parsed_authors.append(BareName.create(author[0], author[1]))
if not authors:
raise BadRequest('No authors provided')
try:
p = BarePaper.create(title, parsed_authors, date)
except ValueError:
raise BadRequest('Invalid paper')
return {'status': 'ok', 'paper': p.json()}
def test_create_by_doi_invalid_doi(self):
p = Paper.create_by_doi('10.1021/eiaeuiebop134223cen-v043n050.p033')
assert p is None
def test_name_initial(self):
n = self.r2.name
p = Paper.create_by_doi("10.1002/ange.19941062339")
n1 = p.authors[0].name
self.assertEqual((n1.first, n1.last), (n.first, n.last))
def api_paper_query(request):
try:
fields = json.loads(request.body.decode('utf-8'))
except (ValueError, UnicodeDecodeError):
raise BadRequest('Invalid JSON payload')
doi = fields.get('doi')
if doi:
p = None
try:
p = Paper.get_by_doi(doi)
if not p:
p = Paper.create_by_doi(doi)
except MetadataSourceException:
pass
if p is None:
raise BadRequest('Could not find a paper with this DOI')
return {'status': 'ok', 'paper': p.json()}
title = fields.get('title')
if not isinstance(title, str) or not title or len(title) > 512:
raise BadRequest(
'Invalid title, has to be a non-empty string shorter than 512 characters'
)
date = fields.get('date')
if not isinstance(date, str):
raise BadRequest('A date is required')
try:
date = tolerant_datestamp_to_datetime(date)
except ValueError as e:
raise BadRequest(str(e))
authors = fields.get('authors')
if not isinstance(authors, list):
raise BadRequest('A list of authors is expected')
parsed_authors = []
for a in authors:
author = None
if not isinstance(a, dict):
raise BadRequest('Invalid author')
if 'first' in a and 'last' in a:
if not isinstance(a['first'], str) or not isinstance(
a['last'], str) or not a['last']:
raise BadRequest('Invalid (first,last) name provided')
else:
author = (a['first'], a['last'])
elif 'plain' in a:
if not isinstance(a['plain'], str) or not a['plain']:
raise BadRequest('Invalid plain name provided')
else:
author = parse_comma_name(a['plain'])
if author is None:
raise BadRequest('Invalid author')
parsed_authors.append(BareName.create(author[0], author[1]))
if not authors:
raise BadRequest('No authors provided')
try:
# Validate the metadata against our data model,
# and compute the fingerprint to look up the paper in the DB.
# This does NOT create a paper in the database - we do not want
# to create papers for every search query we get!
p = BarePaper.create(title, parsed_authors, date)
except ValueError as e:
raise BadRequest('Invalid paper: {}'.format(e))
try:
model_paper = Paper.objects.get(fingerprint=p.fingerprint)
return {'status': 'ok', 'paper': model_paper.json()}
except Paper.DoesNotExist:
return {'status': 'not found'}, 404
def test_missing_info_in_pub(self):
p = Paper.create_by_doi('10.1007/978-3-642-14363-2_7')
self.checkPage('paper', kwargs={'pk': p.id, 'slug': p.slug})
def api_paper_query(request):
try:
fields = json.loads(request.body.decode('utf-8'))
except (ValueError, UnicodeDecodeError):
raise BadRequest('Invalid JSON payload')
doi = fields.get('doi')
if doi:
p = None
try:
p = Paper.get_by_doi(doi)
if not p:
p = Paper.create_by_doi(doi)
except MetadataSourceException:
pass
if p is None:
raise BadRequest('Could not find a paper with this DOI')
return {'status': 'ok', 'paper': p.json()}
title = fields.get('title')
if not isinstance(title, str) or not title or len(title) > 512:
raise BadRequest(
'Invalid title, has to be a non-empty string shorter than 512 characters')
date = fields.get('date')
if not isinstance(date, str):
raise BadRequest('A date is required')
try:
date = tolerant_datestamp_to_datetime(date)
except ValueError as e:
raise BadRequest(str(e))
authors = fields.get('authors')
if not isinstance(authors, list):
raise BadRequest('A list of authors is expected')
parsed_authors = []
for a in authors:
author = None
if not isinstance(a, dict):
raise BadRequest('Invalid author')
if 'first' in a and 'last' in a:
if not isinstance(a['first'], str) or not isinstance(a['last'], str) or not a['last']:
raise BadRequest('Invalid (first,last) name provided')
else:
author = (a['first'], a['last'])
elif 'plain' in a:
if not isinstance(a['plain'], str) or not a['plain']:
raise BadRequest('Invalid plain name provided')
else:
author = parse_comma_name(a['plain'])
if author is None:
raise BadRequest('Invalid author')
parsed_authors.append(BareName.create(author[0], author[1]))
if not authors:
raise BadRequest('No authors provided')
try:
# Validate the metadata against our data model,
# and compute the fingerprint to look up the paper in the DB.
# This does NOT create a paper in the database - we do not want
# to create papers for every search query we get!
p = BarePaper.create(title, parsed_authors, date)
except ValueError as e:
raise BadRequest('Invalid paper: {}'.format(e))
try:
model_paper = Paper.objects.get(fingerprint=p.fingerprint)
return {'status': 'ok', 'paper': model_paper.json()}
except Paper.DoesNotExist:
return {'status': 'not found'}, 404
def test_valid_doi(self):
Paper.create_by_doi('10.1016/0379-6779(91)91572-r')
self.checkJson(self.getPage('api-paper-doi',
args=['10.1016/0379-6779(91)91572-r']))
def test_valid_pk(self):
paper = Paper.create_by_doi('10.1016/0379-6779(91)91572-r')
self.checkJson(self.getPage('api-paper-pk',
args=[paper.id]))
def create_oairecord(self, record, update_index=True, create_missing_dois=True):
"""
Given one line of the dump (represented as a dict),
add it to the corresponding paper (if it exists)
"""
doi = to_doi(record['doi'])
if not doi:
return
prefix = doi.split('/')[0]
if prefix in free_doi_prefixes:
return
if not record.get('oa_locations'):
return
paper = Paper.get_by_doi(doi)
if not paper:
if not create_missing_dois:
return
try:
paper = Paper.create_by_doi(doi)
except (MetadataSourceException, ValueError):
return
if not paper:
logger.info('no such paper for doi {doi}'.format(doi=doi))
return
logger.info(doi)
paper.cache_oairecords()
for oa_location in record.get('oa_locations') or []:
url = oa_location['url']
# just to speed things up a bit...
if paper.pdf_url == url:
return
identifier='oadoi:'+url
source = self.oadoi_source
if oa_location['host_type'] == 'publisher':
url = doi_to_url(doi)
identifier = doi_to_crossref_identifier(doi)
source = self.crossref_source
record = BareOaiRecord(
paper=paper,
doi=doi,
pubtype=paper.doctype,
source=source,
identifier=identifier,
splash_url=url,
pdf_url=oa_location['url'])
try:
# We disable checks by DOI since we know the paper has been looked up by DOI already.
old_pdf_url = paper.pdf_url
paper.add_oairecord(record, check_by_doi=False)
super(Paper, paper).update_availability()
if old_pdf_url != paper.pdf_url:
paper.save()
if update_index:
paper.update_index()
except (DataError, ValueError):
logger.warning('Record does not fit in the DB')
