def test_get_publication_date(self):
self.assertEqual(
get_publication_date(
fetch_metadata_by_DOI('10.5281/zenodo.18898')),
datetime.date(year=2015, month=1, day=1))
self.assertEqual(
get_publication_date(
fetch_metadata_by_DOI('10.5380/dp.v1i1.1919')),
datetime.date(year=2005, month=3, day=18))
def test_get_publication_date(self):
self.assertEqual(
get_publication_date(
fetch_metadata_by_DOI('10.5281/zenodo.18898')),
datetime.date(year=2015, month=01, day=01))
self.assertEqual(
get_publication_date(
fetch_metadata_by_DOI('10.5380/dp.v1i1.1919')),
datetime.date(year=2005, month=03, day=18))
def fetch_oa(doi):
metadata = fetch_metadata_by_DOI(doi)
if metadata is None:
return False
licenses = set([(license or {}).get('URL')
for license in metadata.get('license', [])])
return any(map(is_oa_license, licenses))
def test_fetch_single_doi(self):
doi = '10.5380/dp.v1i1.1922'
metadata = fetch_metadata_by_DOI(doi)
self.assertEqual(metadata,
{'publisher': 'Universidade Federal do Parana',
'DOI': '10.5380/dp.v1i1.1922',
'subtitle': [],
'author': [{'given': 'Frederic', 'family': 'Worms'}],
'URL': 'http://dx.doi.org/10.5380/dp.v1i1.1922',
'issued': {'date-parts': [[2005, 3, 18]]},
'reference-count': 0,
'title': 'A concep\xe7\xe3o bergsoniana do tempo',
'volume': '1',
'source': 'CrossRef',
'prefix': 'http://id.crossref.org/prefix/10.5380',
'score': 1.0,
'deposited': {'timestamp': 1421107200000, 'date-parts': [[2015, 1, 13]]},
'type': 'journal-article',
'container-title': 'DoisPontos',
'indexed': {'timestamp': 1421405831942, 'date-parts': [[2015, 1, 16]]},
'issue': '1',
'ISSN': ['2179-7412', '1807-3883'],
'member': 'http://id.crossref.org/member/3785'})
def test_fetch_single_doi(self):
doi = '10.5380/dp.v1i1.1922'
metadata = fetch_metadata_by_DOI(doi)
self.assertEqual(
metadata, {
'publisher': 'Universidade Federal do Parana',
'DOI': '10.5380/dp.v1i1.1922',
'subtitle': [],
'author': [{
'given': 'Frederic',
'family': 'Worms'
}],
'URL': 'http://dx.doi.org/10.5380/dp.v1i1.1922',
'issued': {
'date-parts': [[2005, 3, 18]]
},
'reference-count': 0,
'title': 'A concep\xe7\xe3o bergsoniana do tempo',
'volume': '1',
'source': 'CrossRef',
'prefix': 'http://id.crossref.org/prefix/10.5380',
'score': 1.0,
'deposited': {
'timestamp': 1421107200000,
'date-parts': [[2015, 1, 13]]
},
'type': 'journal-article',
'container-title': 'DoisPontos',
'indexed': {
'timestamp': 1421405831942,
'date-parts': [[2015, 1, 16]]
},
'issue': '1',
'ISSN': ['2179-7412', '1807-3883'],
'member': 'http://id.crossref.org/member/3785'
})
def fetch_oa(doi):
metadata = fetch_metadata_by_DOI(doi)
if metadata is None:
return False
licenses = set([(license or {}).get('URL') for license in metadata.get('license', [])])
return any(map(is_oa_license, licenses))
def process_records(self, listRecords):
for record in listRecords:
metadata = record[1]._map
authors = get_oai_authors(metadata)
# Filter the record
if all(not elem.is_known for elem in authors):
print "No relevant author, continue"
continue
if not 'title' in metadata or metadata['title'] == []:
continue
# Find the source
sets = record[0].setSpec()
source_identifier = None
for s in sets:
if s.startswith(PROXY_SOURCE_PREFIX):
source_identifier = s[len(PROXY_SOURCE_PREFIX):]
break
source = None
if source_identifier:
try:
source = OaiSource.objects.get(
identifier=source_identifier)
except OaiSource.DoesNotExist:
pass
if not source:
print "Invalid source '" + str(
source_identifier) + "' from the proxy, skipping"
continue
# Find the DOI, if any
doi = None
for identifier in metadata['identifier'] + metadata['relation']:
if not doi:
doi = to_doi(identifier)
# A publication date is necessary
pubdate = find_earliest_oai_date(record)
if not pubdate:
print "No publication date, skipping"
continue
print 'Saving record %s' % record[0].identifier()
paper = BarePaper.create(metadata['title'][0], authors, pubdate)
if doi:
try:
metadata = crossref.fetch_metadata_by_DOI(doi)
crossref.create_publication(paper, metadata)
except MetadataSourceException as e:
print(
"Warning, metadata source exception while fetching DOI "
+ doi + ":\n" + unicode(e))
pass
if paper is None:
print "Paper creation failed, skipping"
continue
# Save the record
# TODO: we should check record validity *BEFORE* creating the paper
try:
add_oai_record(record, source, paper)
yield paper
except ValueError as e:
print "Warning, OAI record " + record[0].identifier(
) + " skipped:\n" + unicode(e)
paper.update_availability()
def process_records(self, listRecords):
for record in listRecords:
metadata = record[1]._map
authors = get_oai_authors(metadata)
# Filter the record
if all(not elem.is_known for elem in authors):
print "No relevant author, continue"
continue
if not 'title' in metadata or metadata['title'] == []:
continue
# Find the source
sets = record[0].setSpec()
source_identifier = None
for s in sets:
if s.startswith(PROXY_SOURCE_PREFIX):
source_identifier = s[len(PROXY_SOURCE_PREFIX):]
break
source = None
if source_identifier:
try:
source = OaiSource.objects.get(identifier=source_identifier)
except OaiSource.DoesNotExist:
pass
if not source:
print "Invalid source '"+str(source_identifier)+"' from the proxy, skipping"
continue
# Find the DOI, if any
doi = None
for identifier in metadata['identifier']+metadata['relation']:
if not doi:
doi = to_doi(identifier)
# A publication date is necessary
pubdate = find_earliest_oai_date(record)
if not pubdate:
print "No publication date, skipping"
continue
print 'Saving record %s' % record[0].identifier()
paper = BarePaper.create(metadata['title'][0], authors, pubdate)
if doi:
try:
metadata = crossref.fetch_metadata_by_DOI(doi)
crossref.create_publication(paper, metadata)
except MetadataSourceException as e:
print("Warning, metadata source exception while fetching DOI "+doi+":\n"+unicode(e))
pass
if paper is None:
print "Paper creation failed, skipping"
continue
# Save the record
# TODO: we should check record validity *BEFORE* creating the paper
try:
add_oai_record(record, source, paper)
yield paper
except ValueError as e:
print "Warning, OAI record "+record[0].identifier()+" skipped:\n"+unicode(e)
paper.update_availability()
