def test_get_publication_date(self): self.assertEqual( get_publication_date( fetch_metadata_by_DOI('10.5281/zenodo.18898')), datetime.date(year=2015, month=1, day=1)) self.assertEqual( get_publication_date( fetch_metadata_by_DOI('10.5380/dp.v1i1.1919')), datetime.date(year=2005, month=3, day=18))
def test_get_publication_date(self): self.assertEqual( get_publication_date( fetch_metadata_by_DOI('10.5281/zenodo.18898')), datetime.date(year=2015, month=01, day=01)) self.assertEqual( get_publication_date( fetch_metadata_by_DOI('10.5380/dp.v1i1.1919')), datetime.date(year=2005, month=03, day=18))
def fetch_oa(doi): metadata = fetch_metadata_by_DOI(doi) if metadata is None: return False licenses = set([(license or {}).get('URL') for license in metadata.get('license', [])]) return any(map(is_oa_license, licenses))
def test_fetch_single_doi(self): doi = '10.5380/dp.v1i1.1922' metadata = fetch_metadata_by_DOI(doi) self.assertEqual(metadata, {'publisher': 'Universidade Federal do Parana', 'DOI': '10.5380/dp.v1i1.1922', 'subtitle': [], 'author': [{'given': 'Frederic', 'family': 'Worms'}], 'URL': 'http://dx.doi.org/10.5380/dp.v1i1.1922', 'issued': {'date-parts': [[2005, 3, 18]]}, 'reference-count': 0, 'title': 'A concep\xe7\xe3o bergsoniana do tempo', 'volume': '1', 'source': 'CrossRef', 'prefix': 'http://id.crossref.org/prefix/10.5380', 'score': 1.0, 'deposited': {'timestamp': 1421107200000, 'date-parts': [[2015, 1, 13]]}, 'type': 'journal-article', 'container-title': 'DoisPontos', 'indexed': {'timestamp': 1421405831942, 'date-parts': [[2015, 1, 16]]}, 'issue': '1', 'ISSN': ['2179-7412', '1807-3883'], 'member': 'http://id.crossref.org/member/3785'})
def test_fetch_single_doi(self): doi = '10.5380/dp.v1i1.1922' metadata = fetch_metadata_by_DOI(doi) self.assertEqual( metadata, { 'publisher': 'Universidade Federal do Parana', 'DOI': '10.5380/dp.v1i1.1922', 'subtitle': [], 'author': [{ 'given': 'Frederic', 'family': 'Worms' }], 'URL': 'http://dx.doi.org/10.5380/dp.v1i1.1922', 'issued': { 'date-parts': [[2005, 3, 18]] }, 'reference-count': 0, 'title': 'A concep\xe7\xe3o bergsoniana do tempo', 'volume': '1', 'source': 'CrossRef', 'prefix': 'http://id.crossref.org/prefix/10.5380', 'score': 1.0, 'deposited': { 'timestamp': 1421107200000, 'date-parts': [[2015, 1, 13]] }, 'type': 'journal-article', 'container-title': 'DoisPontos', 'indexed': { 'timestamp': 1421405831942, 'date-parts': [[2015, 1, 16]] }, 'issue': '1', 'ISSN': ['2179-7412', '1807-3883'], 'member': 'http://id.crossref.org/member/3785' })
def process_records(self, listRecords): for record in listRecords: metadata = record[1]._map authors = get_oai_authors(metadata) # Filter the record if all(not elem.is_known for elem in authors): print "No relevant author, continue" continue if not 'title' in metadata or metadata['title'] == []: continue # Find the source sets = record[0].setSpec() source_identifier = None for s in sets: if s.startswith(PROXY_SOURCE_PREFIX): source_identifier = s[len(PROXY_SOURCE_PREFIX):] break source = None if source_identifier: try: source = OaiSource.objects.get( identifier=source_identifier) except OaiSource.DoesNotExist: pass if not source: print "Invalid source '" + str( source_identifier) + "' from the proxy, skipping" continue # Find the DOI, if any doi = None for identifier in metadata['identifier'] + metadata['relation']: if not doi: doi = to_doi(identifier) # A publication date is necessary pubdate = find_earliest_oai_date(record) if not pubdate: print "No publication date, skipping" continue print 'Saving record %s' % record[0].identifier() paper = BarePaper.create(metadata['title'][0], authors, pubdate) if doi: try: metadata = crossref.fetch_metadata_by_DOI(doi) crossref.create_publication(paper, metadata) except MetadataSourceException as e: print( "Warning, metadata source exception while fetching DOI " + doi + ":\n" + unicode(e)) pass if paper is None: print "Paper creation failed, skipping" continue # Save the record # TODO: we should check record validity *BEFORE* creating the paper try: add_oai_record(record, source, paper) yield paper except ValueError as e: print "Warning, OAI record " + record[0].identifier( ) + " skipped:\n" + unicode(e) paper.update_availability()
def process_records(self, listRecords): for record in listRecords: metadata = record[1]._map authors = get_oai_authors(metadata) # Filter the record if all(not elem.is_known for elem in authors): print "No relevant author, continue" continue if not 'title' in metadata or metadata['title'] == []: continue # Find the source sets = record[0].setSpec() source_identifier = None for s in sets: if s.startswith(PROXY_SOURCE_PREFIX): source_identifier = s[len(PROXY_SOURCE_PREFIX):] break source = None if source_identifier: try: source = OaiSource.objects.get(identifier=source_identifier) except OaiSource.DoesNotExist: pass if not source: print "Invalid source '"+str(source_identifier)+"' from the proxy, skipping" continue # Find the DOI, if any doi = None for identifier in metadata['identifier']+metadata['relation']: if not doi: doi = to_doi(identifier) # A publication date is necessary pubdate = find_earliest_oai_date(record) if not pubdate: print "No publication date, skipping" continue print 'Saving record %s' % record[0].identifier() paper = BarePaper.create(metadata['title'][0], authors, pubdate) if doi: try: metadata = crossref.fetch_metadata_by_DOI(doi) crossref.create_publication(paper, metadata) except MetadataSourceException as e: print("Warning, metadata source exception while fetching DOI "+doi+":\n"+unicode(e)) pass if paper is None: print "Paper creation failed, skipping" continue # Save the record # TODO: we should check record validity *BEFORE* creating the paper try: add_oai_record(record, source, paper) yield paper except ValueError as e: print "Warning, OAI record "+record[0].identifier()+" skipped:\n"+unicode(e) paper.update_availability()