def test_publisher_s(self): ''' Checks to see if the BookData's publisher_s property works. ''' book = BookData() self.assertEquals(book.publisher_s, BookData.blank("publisher_s")) book.publisher_s = " DC " self.assertEquals(book.publisher_s, "DC") del book.publisher_s self.assertEquals(book.publisher_s, BookData.blank("publisher_s"))
def test_editors_sl(self): ''' Checks to see if the BookData's editors_sl property works. ''' book = BookData() self.assertEquals(book.editors_sl, BookData.blank("editors_sl")) book.editors_sl = [None, "Warren Ellis", "", "" ] self.assertEquals(book.editors_sl, ["Warren Ellis"]) del book.editors_sl self.assertEquals(book.editors_sl, BookData.blank("editors_sl"))
def test_crossovers_sl(self): ''' Checks to see if the BookData's crossovers_sl property works. ''' book = BookData() self.assertEquals(book.crossovers_sl, BookData.blank("crossovers_sl")) book.crossovers_sl = ["Killing Joke", None, "", "The Last Laugh"] self.assertEquals(book.crossovers_sl, ["Killing Joke", "The Last Laugh"]) del book.crossovers_sl self.assertEquals(book.crossovers_sl, BookData.blank("crossovers_sl"))
def test_path_s(self): ''' Checks to see if the BookData's path_s property works. ''' book = BookData() self.assertEquals(book.path_s, BookData.blank("path_s")) book.path_s = "d:\bob\bob2\file.cbz" self.assertEquals(book.path_s, "d:\bob\bob2\file.cbz") del book.path_s self.assertEquals(book.path_s, BookData.blank("path_s"))
def test_tags_sl(self): ''' Checks to see if the BookData's tags_sl property works. ''' book = BookData() self.assertEquals(book.tags_sl, BookData.blank("tags_sl")) book.tags_sl = [None, "Hentai", "Marvel", "" ] self.assertEquals(book.tags_sl, ["Hentai", "Marvel"]) del book.tags_sl self.assertEquals(book.tags_sl, BookData.blank("tags_sl"))
def test_cover_url_s(self): ''' Checks to see if the BookData's cover_url_s property works. ''' book = BookData() self.assertEquals(book.cover_url_s, BookData.blank("cover_url_s")) book.cover_url_s = "http:\\batman.com\pic.jpg" self.assertEquals(book.cover_url_s, "http:\\batman.com\pic.jpg") del book.cover_url_s self.assertEquals(book.cover_url_s, BookData.blank("cover_url_s"))
def test_webpage_s(self): ''' Checks to see if the BookData's webpage_s property works. ''' book = BookData() self.assertEquals(book.webpage_s, BookData.blank("webpage_s")) book.webpage_s = "http:\\batman.com" self.assertEquals(book.webpage_s, "http:\\batman.com") del book.webpage_s self.assertEquals(book.webpage_s, BookData.blank("webpage_s"))
def test_imprint_s(self): ''' Checks to see if the BookData's imprint_s property works. ''' book = BookData() self.assertEquals(book.imprint_s, BookData.blank("imprint_s")) book.imprint_s = "Wildstorm" self.assertEquals(book.imprint_s, "Wildstorm") del book.imprint_s self.assertEquals(book.imprint_s, BookData.blank("imprint_s"))
def test_title_s(self): ''' Checks to see if the BookData's title_s property works. ''' book = BookData() self.assertEquals(book.title_s, BookData.blank("title_s")) book.title_s = " Batman!! " self.assertEquals(book.title_s, "Batman!!") del book.title_s self.assertEquals(book.title_s, BookData.blank("title_s"))
def test_summary_s(self): ''' Checks to see if the BookData's summary_s property works. ''' book = BookData() self.assertEquals(book.summary_s, BookData.blank("summary_s")) book.summary_s = " Batman beats up people!" self.assertEquals(book.summary_s, "Batman beats up people!") del book.summary_s self.assertEquals(book.summary_s, BookData.blank("summary_s"))
def test_format_s(self): ''' Checks to see if the BookData's format_s property works. ''' book = BookData() self.assertEquals(book.format_s, BookData.blank("format_s")) book.format_s = "BIG" self.assertEquals(book.format_s, "BIG") del book.format_s self.assertEquals(book.format_s, BookData.blank("format_s"))
def test_series_key_s(self): ''' Checks to see if the BookData's series_key_s property works. ''' book = BookData() self.assertEquals(book.series_key_s, BookData.blank("series_key_s")) book.series_key_s = "9393" self.assertEquals(book.series_key_s, "9393") del book.series_key_s self.assertEquals(book.series_key_s, BookData.blank("series_key_s"))
def test_characters_sl(self): ''' Checks to see if the BookData's characters_sl property works. ''' book = BookData() self.assertEquals(book.characters_sl, BookData.blank("characters_sl")) book.characters_sl = ["Hawkgirl", "", "Batman"] self.assertEquals(book.characters_sl, ["Hawkgirl", "Batman"]) del book.characters_sl self.assertEquals(book.characters_sl, BookData.blank("characters_sl"))
def test_locations_sl(self): ''' Checks to see if the BookData's locations_sl property works. ''' book = BookData() self.assertEquals(book.locations_sl, BookData.blank("locations_sl")) book.locations_sl = [None, "Edmonton"] self.assertEquals(book.locations_sl, ["Edmonton"]) del book.locations_sl self.assertEquals(book.locations_sl, BookData.blank("locations_sl"))
def test_teams_sl(self): ''' Checks to see if the BookData's teams_sl property works. ''' book = BookData() self.assertEquals(book.teams_sl, BookData.blank("teams_sl")) book.teams_sl = [None, "The Avengers", "Justice League"] self.assertEquals(book.teams_sl, ["The Avengers", "Justice League"]) del book.teams_sl self.assertEquals(book.teams_sl, BookData.blank("teams_sl"))
def test_editors_sl(self): ''' Checks to see if the BookData's editors_sl property works. ''' book = BookData() self.assertEquals(book.editors_sl, BookData.blank("editors_sl")) book.editors_sl = [None, "Warren Ellis", "", ""] self.assertEquals(book.editors_sl, ["Warren Ellis"]) del book.editors_sl self.assertEquals(book.editors_sl, BookData.blank("editors_sl"))
def test_notes_s(self): ''' Checks to see if the BookData's notes_s property works. ''' book = BookData() self.assertEquals(book.notes_s, BookData.blank("notes_s")) book.notes_s = " A nice note" self.assertEquals(book.notes_s, "A nice note") del book.notes_s self.assertEquals(book.notes_s, BookData.blank("notes_s"))
def test_tags_sl(self): ''' Checks to see if the BookData's tags_sl property works. ''' book = BookData() self.assertEquals(book.tags_sl, BookData.blank("tags_sl")) book.tags_sl = [None, "Hentai", "Marvel", ""] self.assertEquals(book.tags_sl, ["Hentai", "Marvel"]) del book.tags_sl self.assertEquals(book.tags_sl, BookData.blank("tags_sl"))
def test_issue_num_s(self): ''' Checks to see if the BookData's issue_num_s property works. ''' book = BookData() self.assertEquals(book.issue_num_s, BookData.blank("issue_num_s")) book.issue_num_s = "1a" self.assertEquals(book.issue_num_s, "1a") del book.issue_num_s self.assertEquals(book.issue_num_s, BookData.blank("issue_num_s"))
def books(input_stream): reader = csv.DictReader(input_stream) for row in reader: if row["Title"] == "": continue # ignore blank line row_data = map_obp_csv_columns(row) yield BookData(**row_data)
def test_page_count_n(self): ''' Checks to see if the BookData's page_count_n property works. ''' book = BookData() self.assertEquals(book.page_count_n, BookData.blank("page_count_n")) book.page_count_n = 3.4 self.assertEquals(book.page_count_n, 3) book.page_count_n = "44" self.assertEquals(book.page_count_n, 44) book.page_count_n = None self.assertEquals(book.page_count_n, BookData.blank("page_count_n")) del book.page_count_n self.assertEquals(book.page_count_n, BookData.blank("page_count_n"))
def test_rel_year_n(self): ''' Checks to see if the BookData's rel_year_n property works. ''' book = BookData() self.assertEquals(book.rel_year_n, BookData.blank("rel_year_n")) book.rel_year_n = 2012 self.assertEquals(book.rel_year_n, 2012) book.rel_year_n = "2013" self.assertEquals(book.rel_year_n, 2013) book.rel_year_n = None self.assertEquals(book.rel_year_n, BookData.blank("rel_year_n")) del book.rel_year_n self.assertEquals(book.rel_year_n, BookData.blank("rel_year_n"))
def test_rating_n(self): ''' Checks to see if the BookData's rating_n property works. ''' book = BookData() self.assertEquals(book.rating_n, BookData.blank("rating_n")) book.rating_n = 3.4 self.assertEquals(book.rating_n, 3.4) book.rating_n = "2.42" self.assertEquals(book.rating_n, 2.42) book.rating_n = None self.assertEquals(book.rating_n, BookData.blank("rating_n")) del book.rating_n self.assertEquals(book.rating_n, BookData.blank("rating_n"))
def test_volume_year_n(self): ''' Checks to see if the BookData's volume_year_n property works. ''' book = BookData() self.assertEquals(book.volume_year_n, BookData.blank("volume_year_n")) book.volume_year_n = 1934 self.assertEquals(book.volume_year_n, 1934) book.volume_year_n = "2015" self.assertEquals(book.volume_year_n, 2015) book.volume_year_n = None self.assertEquals(book.volume_year_n, BookData.blank("volume_year_n")) del book.volume_year_n self.assertEquals(book.volume_year_n, BookData.blank("volume_year_n"))
def test_pub_month_n(self): ''' Checks to see if the BookData's pub_month_n property works. ''' book = BookData() self.assertEquals(book.pub_month_n, BookData.blank("pub_month_n")) book.pub_month_n = 9 self.assertEquals(book.pub_month_n, 9) book.pub_month_n = " 11 " self.assertEquals(book.pub_month_n, 11) book.pub_month_n = None self.assertEquals(book.pub_month_n, BookData.blank("pub_month_n")) del book.pub_month_n self.assertEquals(book.pub_month_n, BookData.blank("pub_month_n"))
def test_pub_day_n(self): ''' Checks to see if the BookData's pub_day_n property works. ''' book = BookData() self.assertEquals(book.pub_day_n, BookData.blank("pub_day_n")) book.pub_day_n = 15 self.assertEquals(book.pub_day_n, 15) book.pub_day_n = "16" self.assertEquals(book.pub_day_n, 16) book.pub_day_n = None self.assertEquals(book.pub_day_n, BookData.blank("pub_day_n")) del book.pub_year_n self.assertEquals(book.pub_day_n, BookData.blank("pub_day_n"))
def get_book_by_doi(doi): with psycopg2.connect(DSN) as conn: with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as curs: sql = """select book_uuid, title, publisher, doi, isbn1, subtitle, page_count, bisac_subject_code_1 from book where doi = %(doi)s;""" curs.execute(sql, {"doi": doi}) row = curs.fetchone() row_data = db_row_to_book_columns(row) return BookData(**row_data)
def read_from_db(): with psycopg2.connect(DSN) as conn: with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as curs: sql = """select book_uuid, title, publisher, doi, isbn1, subtitle, page_count, bisac_subject_code_1 from book;""" curs.execute(sql) for row in curs.fetchall(): row_data = db_row_to_book_columns(row) yield BookData(**row_data)
def __parse_extra_details_from_path(self): ''' Series name, issue number, and volume year are all critical bits of data for scraping purposes--yet fresh, unscraped files often do not have them. So when some or all of these values are missing, this method tries to fill them in by parsing them out of the comic's path. ''' bd = self.__bookdata no_series = BookData.blank("series_s") == bd.series_s no_issuenum = BookData.blank("issue_num_s") == bd.issue_num_s no_year = BookData.blank("pub_year_n") == bd.pub_year_n if no_series or no_issuenum or no_year: if bd.path_s: # 1. at least one detail is missing, and we have a path name to # work with, so lets try to extract some details that way. filename = Path.GetFileName(bd.path_s) config = self.__scraper.config regex = config.alt_search_regex_s extracted = None # 2. first, extract using the user specified regex, if there is one if regex: extracted = fnameparser.regex(filename, regex) if not extracted: extracted = fnameparser.extract(filename) # never fails # 3. now that we have some extracted data, use it to fill in # any gaps in our details. if no_series: bd.series_s = extracted[0] if no_issuenum: bd.issue_num_s = extracted[1] if no_year: bd.pub_year_n = int(extracted[2]) \ if is_number(extracted[2])\ else BookData.blank("pub_year_n")
def update(self): ''' Overridden to implement abstract method defined in superclass. Writes all eligible properties in this object out to their counterparts in ComicRack (i.e. back into the ComicBook object that was passed into __init__.) ''' ok_to_update = self.updated_properties() # removes commas from the the given string def cleanup(s): s = re.sub(r",(\s+)", r"\1", s) if s else "" return re.sub(r",", r" ", s) if "series_s" in ok_to_update: self.__crbook.Series = self.series_s ok_to_update.remove("series_s") if "issue_num_s" in ok_to_update: self.__crbook.Number = self.issue_num_s ok_to_update.remove("issue_num_s") if "volume_year_n" in ok_to_update: self.__crbook.Volume = self.volume_year_n ok_to_update.remove("volume_year_n") if "format_s" in ok_to_update: self.__crbook.Format = self.format_s ok_to_update.remove("format_s") if "title_s" in ok_to_update: self.__crbook.Title = self.title_s ok_to_update.remove("title_s") if "crossovers_sl" in ok_to_update: self.__crbook.AlternateSeries = \ ', '.join([cleanup(x) for x in self.crossovers_sl]) ok_to_update.remove("crossovers_sl") if "summary_s" in ok_to_update: self.__crbook.Summary = self.summary_s ok_to_update.remove("summary_s") if "publisher_s" in ok_to_update: self.__crbook.Publisher = self.publisher_s ok_to_update.remove("publisher_s") if "imprint_s" in ok_to_update: self.__crbook.Imprint = self.imprint_s ok_to_update.remove("imprint_s") if "characters_sl" in ok_to_update: self.__crbook.Characters = \ ', '.join([cleanup(x) for x in self.characters_sl]) ok_to_update.remove("characters_sl") if "teams_sl" in ok_to_update: self.__crbook.Teams = \ ', '.join([cleanup(x) for x in self.teams_sl]) ok_to_update.remove("teams_sl") if "locations_sl" in ok_to_update: self.__crbook.Locations = \ ', '.join([cleanup(x) for x in self.locations_sl]) ok_to_update.remove("locations_sl") if "writers_sl" in ok_to_update: self.__crbook.Writer = \ ', '.join([cleanup(x) for x in self.writers_sl]) ok_to_update.remove("writers_sl") if "pencillers_sl" in ok_to_update: self.__crbook.Penciller = \ ', '.join([cleanup(x) for x in self.pencillers_sl]) ok_to_update.remove("pencillers_sl") if "inkers_sl" in ok_to_update: self.__crbook.Inker = \ ', '.join([cleanup(x) for x in self.inkers_sl]) ok_to_update.remove("inkers_sl") if "colorists_sl" in ok_to_update: self.__crbook.Colorist = \ ', '.join([cleanup(x) for x in self.colorists_sl]) ok_to_update.remove("colorists_sl") if "letterers_sl" in ok_to_update: self.__crbook.Letterer = \ ', '.join([cleanup(x) for x in self.letterers_sl]) ok_to_update.remove("letterers_sl") if "cover_artists_sl" in ok_to_update: self.__crbook.CoverArtist = \ ', '.join([cleanup(x) for x in self.cover_artists_sl]) ok_to_update.remove("cover_artists_sl") if "editors_sl" in ok_to_update: self.__crbook.Editor = \ ', '.join([cleanup(x) for x in self.editors_sl]) ok_to_update.remove("editors_sl") if "tags_sl" in ok_to_update: self.__crbook.Tags = \ ', '.join([cleanup(x) for x in self.tags_sl]) ok_to_update.remove("tags_sl") if "notes_s" in ok_to_update: self.__crbook.Notes = self.notes_s ok_to_update.remove("notes_s") if "webpage_s" in ok_to_update: self.__crbook.Web = self.webpage_s ok_to_update.remove("webpage_s") if "rating_n" in ok_to_update: self.__crbook.CommunityRating = self.rating_n ok_to_update.remove("rating_n") if "issue_key_s" in ok_to_update: self.__crbook.SetCustomValue( PluginBookData.__ISSUE_KEY, sstr(self.issue_key_s)) ok_to_update.remove("issue_key_s") if "series_key_s" in ok_to_update: self.__crbook.SetCustomValue( PluginBookData.__SERIES_KEY, sstr(self.series_key_s)) ok_to_update.remove("series_key_s") # dates are a little special. any element in the data could be blank # (missing), and we only update the released date if NONE of the # elements are missing. the published date, however, can have a missing # day, or a missing day and month, and we'll still update the rest if "rel_year_n" in ok_to_update and \ "rel_month_n" in ok_to_update and \ "rel_day_n" in ok_to_update: if self.rel_year_n != BookData.blank("rel_year_n") and \ self.rel_month_n != BookData.blank("rel_month_n") and \ self.rel_day_n != BookData.blank("rel_day_n"): date = DateTime(self.rel_year_n, self.rel_month_n, self.rel_day_n) self.__crbook.ReleasedTime = date ok_to_update.remove("rel_year_n") ok_to_update.remove("rel_month_n") ok_to_update.remove("rel_day_n") if "pub_year_n" in ok_to_update: if self.pub_year_n != BookData.blank("pub_year_n"): self.__crbook.Year = self.pub_year_n ok_to_update.remove("pub_year_n") if "pub_month_n" in ok_to_update: if self.pub_year_n != BookData.blank("pub_year_n") and \ self.pub_month_n != BookData.blank("pub_month_n"): self.__crbook.Month = self.pub_month_n ok_to_update.remove("pub_month_n") if "pub_day_n" in ok_to_update: if self.pub_year_n != BookData.blank("pub_year_n") and \ self.pub_month_n != BookData.blank("pub_month_n") and \ self.pub_day_n != BookData.blank("pub_day_n"): self.__crbook.Day = self.pub_day_n ok_to_update.remove("pub_day_n") # we only download and install a thumbnail for fileless CR books, and # even then, only if the user's prefs indicate that they want us to if "cover_url_s" in ok_to_update: already_has_thumb = self.__crbook.CustomThumbnailKey book_is_fileless = not self.path_s config = self.__scraper.config if not self.cover_url_s or not book_is_fileless or \ not config.download_thumbs_b or \ (already_has_thumb and config.preserve_thumbs_b): pass else: image = db.query_image(self.cover_url_s) if not image: log.debug("ERROR: can't download thumbnail: ", self.cover_url_s) else: cr = self.__scraper.comicrack.App success = cr.SetCustomBookThumbnail(self.__crbook, image) if not success: log.debug("ERROR: can't set thumbnail: ", self.cover_url_s) ok_to_update.remove("cover_url_s") # a nice safety check to make sure we didn't miss anything if len(ok_to_update) > 0: for s in ok_to_update: log.debug(self.__class__.__name__ + " can't update property: " + s) raise Exception()