def extract_bibliographic(self, element): identifiers = [] contributors = [] identifiers.append(IdentifierData(Identifier.ISBN, element["isbn"])) sort_name = element["author"] if not sort_name: sort_name = Edition.UNKNOWN_AUTHOR contributors.append(ContributorData(sort_name=sort_name)) primary_identifier = IdentifierData(EnkiAPI.ENKI_ID, element["id"]) image_url = element["large_image"] thumbnail_url = element["large_image"] images = [ LinkData(rel=Hyperlink.THUMBNAIL_IMAGE, href=thumbnail_url, media_type=Representation.PNG_MEDIA_TYPE), LinkData(rel=Hyperlink.IMAGE, href=image_url, media_type=Representation.PNG_MEDIA_TYPE) ] metadata = Metadata( data_source=DataSource.ENKI, title=element["title"], language="eng", medium=Edition.BOOK_MEDIUM, publisher=element["publisher"], primary_identifier=primary_identifier, identifiers=identifiers, contributors=contributors, links=images, ) licenses_owned = element["availability"]["totalCopies"] licenses_available = element["availability"]["availableCopies"] hold = element["availability"]["onHold"] drm_type = EnkiAPI.adobe_drm if (element["availability"]["accessType"] == 'acs') else EnkiAPI.no_drm formats = [] formats.append( FormatData(content_type=Representation.EPUB_MEDIA_TYPE, drm_scheme=drm_type)) circulationdata = CirculationData( data_source=DataSource.ENKI, primary_identifier=primary_identifier, formats=formats, licenses_owned=int(licenses_owned), licenses_available=int(licenses_available), patrons_in_hold_queue=int(hold)) metadata.circulation = circulationdata return metadata
def extract_bibliographic(self, element): """Extract Metadata and CirculationData from a dictionary of information from Enki. :return: A Metadata with attached CirculationData. """ # TODO: it's not clear what these are or whether we'd find them # useful: # dateSaved # length # publishDate primary_identifier = IdentifierData(EnkiAPI.ENKI_ID, element["id"]) identifiers = [] identifiers.append(IdentifierData(Identifier.ISBN, element["isbn"])) contributors = [] sort_name = element.get("author", None) or Edition.UNKNOWN_AUTHOR contributors.append(ContributorData(sort_name=sort_name)) links = [] description = element.get("description") if description: links.append( LinkData( rel=Hyperlink.DESCRIPTION, content=description, media_type="text/html", ) ) # NOTE: When this method is called by, e.g. updated_titles(), # the large and small images are available separately. When # this method is called by get_item(), we only get a single # image, in 'cover'. In get_item() we ask that that image be 'large', # which means we'll be filing it as a normal-sized image. # full_image = None thumbnail_image = None for key, rel in ( ("cover", Hyperlink.IMAGE), ("small_image", Hyperlink.THUMBNAIL_IMAGE), ("large_image", Hyperlink.IMAGE), ): url = element.get(key) if not url: continue link = LinkData(rel=rel, href=url, media_type=Representation.PNG_MEDIA_TYPE) if rel == Hyperlink.THUMBNAIL_IMAGE: # Don't add a thumbnail to the list of links -- wait # until the end and then make it a thumbnail of the # primary image. thumbnail_image = link else: if rel == Hyperlink.IMAGE: full_image = link links.append(link) if thumbnail_image: if full_image: # Set the thumbnail as the thumbnail _of_ the full image. full_image.thumbnail = thumbnail_image else: # Treat the thumbnail as the full image. thumbnail_image.rel = Hyperlink.IMAGE links.append(thumbnail_image) # We treat 'subject', 'topic', and 'genre' as interchangeable # sets of tags. This data is based on BISAC but it's not reliably # presented in a form that can be parsed as BISAC. subjects = [] seen_topics = set() for key in ("subject", "topic", "genre"): for topic in element.get(key, []): if not topic or topic in seen_topics: continue subjects.append( SubjectData( Subject.TAG, topic, weight=Classification.TRUSTED_DISTRIBUTOR_WEIGHT, ) ) seen_topics.add(topic) language_code = element.get("language", "English") language = self.LANGUAGE_CODES.get(language_code, "eng") metadata = Metadata( data_source=DataSource.ENKI, title=element.get("title"), language=language, medium=Edition.BOOK_MEDIUM, publisher=element.get("publisher"), primary_identifier=primary_identifier, identifiers=identifiers, contributors=contributors, links=links, subjects=subjects, ) circulationdata = self.extract_circulation( primary_identifier, element.get("availability", {}), element.get("formattype", None), ) metadata.circulation = circulationdata return metadata
def record_info_to_metadata(cls, book, availability): """Turn Odilo's JSON representation of a book into a Metadata object. Note: The json data passed into this method is from a different file/stream from the json data that goes into the book_info_to_circulation() method. """ if 'id' not in book: return None odilo_id = book['id'] primary_identifier = IdentifierData(Identifier.ODILO_ID, odilo_id) active = book.get('active') title = book.get('title') subtitle = book.get('subtitle') series = book.get('series') series_position = book.get('seriesPosition') contributors = [] sort_author = book.get('author') if sort_author: roles = [Contributor.AUTHOR_ROLE] display_author = sort_name_to_display_name(sort_author) contributor = ContributorData(sort_name=sort_author, display_name=display_author, roles=roles, biography=None) contributors.append(contributor) publisher = book.get('publisher') # Metadata --> Marc21 260$c published = book.get('publicationDate') if not published: # yyyyMMdd --> record creation date published = book.get('releaseDate') if published: try: published = datetime.datetime.strptime(published, "%Y%m%d") except ValueError as e: cls.log.warn('Cannot parse publication date from: ' + published + ', message: ' + e.message) # yyyyMMdd --> record last modification date last_update = book.get('modificationDate') if last_update: try: last_update = datetime.datetime.strptime(last_update, "%Y%m%d") except ValueError as e: cls.log.warn('Cannot parse last update date from: ' + last_update + ', message: ' + e.message) language = book.get('language', 'spa') subjects = [] for subject in book.get('subjects', []): subjects.append( SubjectData(type=Subject.TAG, identifier=subject, weight=100)) for subjectBisacCode in book.get('subjectsBisacCodes', []): subjects.append( SubjectData(type=Subject.BISAC, identifier=subjectBisacCode, weight=100)) grade_level = book.get('gradeLevel') if grade_level: subject = SubjectData(type=Subject.GRADE_LEVEL, identifier=grade_level, weight=10) subjects.append(subject) medium = None file_format = book.get('fileFormat') formats = [] for format_received in book.get('formats', []): if format_received in cls.format_data_for_odilo_format: medium = cls.set_format(format_received, formats) elif format_received == cls.ACSM and file_format: medium = cls.set_format( format_received + '_' + file_format.upper(), formats) else: cls.log.warn('Unrecognized format received: ' + format_received) if not medium: medium = Edition.BOOK_MEDIUM identifiers = [] isbn = book.get('isbn') if isbn: if isbnlib.is_isbn10(isbn): isbn = isbnlib.to_isbn13(isbn) identifiers.append(IdentifierData(Identifier.ISBN, isbn, 1)) # A cover links = [] cover_image_url = book.get('coverImageUrl') if cover_image_url: image_data = cls.image_link_to_linkdata(cover_image_url, Hyperlink.THUMBNAIL_IMAGE) if image_data: links.append(image_data) original_image_url = book.get('originalImageUrl') if original_image_url: image_data = cls.image_link_to_linkdata(original_image_url, Hyperlink.IMAGE) if image_data: links.append(image_data) # Descriptions become links. description = book.get('description') if description: links.append( LinkData(rel=Hyperlink.DESCRIPTION, content=description, media_type="text/html")) metadata = Metadata(data_source=DataSource.ODILO, title=title, subtitle=subtitle, language=language, medium=medium, series=series, series_position=series_position, publisher=publisher, published=published, primary_identifier=primary_identifier, identifiers=identifiers, subjects=subjects, contributors=contributors, links=links, data_source_last_updated=last_update) metadata.circulation = OdiloRepresentationExtractor.record_info_to_circulation( availability) # 'active' --> means that the book exists but it's no longer in the collection # (it could be available again in the future) if not active: metadata.circulation.licenses_owned = 0 metadata.circulation.formats = formats return metadata, active
def test_annotate_metadata(self): """Verify that annotate_metadata calls load_circulation_data and load_cover_link appropriately. """ # First, test an unsuccessful annotation. class MockNoCirculationData(DirectoryImportScript): """Do nothing when load_circulation_data is called. Explode if load_cover_link is called. """ def load_circulation_data(self, *args): self.load_circulation_data_args = args return None def load_cover_link(self, *args): raise Exception("Explode!") gutenberg = DataSource.lookup(self._db, DataSource.GUTENBERG) identifier = IdentifierData(Identifier.GUTENBERG_ID, "11111") identifier_obj, ignore = identifier.load(self._db) metadata = Metadata( title=self._str, data_source=gutenberg, primary_identifier=identifier ) mirror = object() policy = ReplacementPolicy(mirror=mirror) cover_directory = object() ebook_directory = object() rights_uri = object() script = MockNoCirculationData(self._db) args = (metadata, policy, cover_directory, ebook_directory, rights_uri) script.annotate_metadata(*args) # load_circulation_data was called. eq_( (identifier_obj, gutenberg, ebook_directory, mirror, metadata.title, rights_uri), script.load_circulation_data_args ) # But because load_circulation_data returned None, # metadata.circulation_data was not modified and # load_cover_link was not called (which would have raised an # exception). eq_(None, metadata.circulation) # Test a successful annotation with no cover image. class MockNoCoverLink(DirectoryImportScript): """Return an object when load_circulation_data is called. Do nothing when load_cover_link is called. """ def load_circulation_data(self, *args): return "Some circulation data" def load_cover_link(self, *args): self.load_cover_link_args = args return None script = MockNoCoverLink(self._db) script.annotate_metadata(*args) # The Metadata object was annotated with the return value of # load_circulation_data. eq_("Some circulation data", metadata.circulation) # load_cover_link was called. eq_( (identifier_obj, gutenberg, cover_directory, mirror), script.load_cover_link_args ) # But since it provided no cover link, metadata.links was empty. eq_([], metadata.links) # Finally, test a completely successful annotation. class MockWithCoverLink(DirectoryImportScript): """Mock success for both load_circulation_data and load_cover_link. """ def load_circulation_data(self, *args): return "Some circulation data" def load_cover_link(self, *args): return "A cover link" metadata.circulation = None script = MockWithCoverLink(self._db) script.annotate_metadata(*args) eq_("Some circulation data", metadata.circulation) eq_(['A cover link'], metadata.links)
def test_annotate_metadata(self): """Verify that annotate_metadata calls load_circulation_data and load_cover_link appropriately. """ # First, test an unsuccessful annotation. class MockNoCirculationData(DirectoryImportScript): """Do nothing when load_circulation_data is called. Explode if load_cover_link is called. """ def load_circulation_data(self, *args): self.load_circulation_data_args = args return None def load_cover_link(self, *args): raise Exception("Explode!") gutenberg = DataSource.lookup(self._db, DataSource.GUTENBERG) identifier = IdentifierData(Identifier.GUTENBERG_ID, "11111") identifier_obj, ignore = identifier.load(self._db) metadata = Metadata( title=self._str, data_source=gutenberg, primary_identifier=identifier ) mirror = object() policy = ReplacementPolicy(mirror=mirror) cover_directory = object() ebook_directory = object() rights_uri = object() script = MockNoCirculationData(self._db) args = (metadata, policy, cover_directory, ebook_directory, rights_uri) script.annotate_metadata(*args) # load_circulation_data was called. eq_( (identifier_obj, gutenberg, ebook_directory, mirror, metadata.title, rights_uri), script.load_circulation_data_args ) # But because load_circulation_data returned None, # metadata.circulation_data was not modified and # load_cover_link was not called (which would have raised an # exception). eq_(None, metadata.circulation) # Test a successful annotation with no cover image. class MockNoCoverLink(DirectoryImportScript): """Return an object when load_circulation_data is called. Do nothing when load_cover_link is called. """ def load_circulation_data(self, *args): return "Some circulation data" def load_cover_link(self, *args): self.load_cover_link_args = args return None script = MockNoCoverLink(self._db) script.annotate_metadata(*args) # The Metadata object was annotated with the return value of # load_circulation_data. eq_("Some circulation data", metadata.circulation) # load_cover_link was called. eq_( (identifier_obj, gutenberg, cover_directory, mirror), script.load_cover_link_args ) # But since it provided no cover link, metadata.links was empty. eq_([], metadata.links) # Finally, test a completely successful annotation. class MockWithCoverLink(DirectoryImportScript): """Mock success for both load_circulation_data and load_cover_link. """ def load_circulation_data(self, *args): return "Some circulation data" def load_cover_link(self, *args): return "A cover link" metadata.circulation = None script = MockWithCoverLink(self._db) script.annotate_metadata(*args) eq_("Some circulation data", metadata.circulation) eq_(['A cover link'], metadata.links)
def extract_bibliographic(self, element): """Extract Metadata and CirculationData from a dictionary of information from Enki. :return: A Metadata with attached CirculationData. """ # TODO: it's not clear what these are or whether we'd find them # useful: # dateSaved # length # publishDate primary_identifier = IdentifierData(EnkiAPI.ENKI_ID, element["id"]) identifiers = [] identifiers.append(IdentifierData(Identifier.ISBN, element["isbn"])) contributors = [] sort_name = element.get("author", None) or Edition.UNKNOWN_AUTHOR contributors.append(ContributorData(sort_name=sort_name)) links = [] description = element.get('description') if description: links.append( LinkData(rel=Hyperlink.DESCRIPTION, content=description, media_type="text/html") ) # NOTE: When this method is called by, e.g. updated_titles(), # the large and small images are available separately. When # this method is called by get_item(), we only get a single # image, in 'cover'. In get_item() we ask that that image be 'large', # which means we'll be filing it as a normal-sized image. # full_image = None thumbnail_image = None for key, rel in ( ('cover', Hyperlink.IMAGE), ('small_image', Hyperlink.THUMBNAIL_IMAGE), ('large_image', Hyperlink.IMAGE) ): url = element.get(key) if not url: continue link = LinkData( rel=rel, href=url, media_type=Representation.PNG_MEDIA_TYPE ) if rel == Hyperlink.THUMBNAIL_IMAGE: # Don't add a thumbnail to the list of links -- wait # until the end and then make it a thumbnail of the # primary image. thumbnail_image = link else: if rel == Hyperlink.IMAGE: full_image = link links.append(link) if thumbnail_image: if full_image: # Set the thumbnail as the thumbnail _of_ the full image. full_image.thumbnail = thumbnail_image else: # Treat the thumbnail as the full image. thumbnail_image.rel = Hyperlink.IMAGE links.append(thumbnail_image) # We treat 'subject', 'topic', and 'genre' as interchangeable # sets of tags. This data is based on BISAC but it's not reliably # presented in a form that can be parsed as BISAC. subjects = [] seen_topics = set() for key in ('subject', 'topic', 'genre'): for topic in element.get(key, []): if not topic or topic in seen_topics: continue subjects.append(SubjectData(Subject.TAG, topic)) seen_topics.add(topic) language_code = element.get("language", "English") language = self.LANGUAGE_CODES.get(language_code, "eng") metadata = Metadata( data_source=DataSource.ENKI, title=element.get("title"), language=language, medium=Edition.BOOK_MEDIUM, publisher=element.get("publisher"), primary_identifier=primary_identifier, identifiers=identifiers, contributors=contributors, links=links, subjects=subjects, ) circulationdata = self.extract_circulation( primary_identifier, element.get('availability', {}), element.get('formattype', None) ) metadata.circulation = circulationdata return metadata