def __extract_issue_ref(self):
'''
This method attempts to rebuild the IssueRef that the user chose the
last time that they scraped this comic. If it can do so, it will
return that IssueRef. If not, it will return None, or the
string "skip" (see below).
If the user has manually added the magic CVDBSKIP flag to the tags or
notes for this book, then this method will return the string "skip",
which should be interpreted as "never scrape this book".
'''
# in this method, its easier to work with tags as a single string
bd = self.__bookdata
tagstring = ', '.join(bd.tags_sl)
# check for the magic CVDBSKIP skip flag
skip_found = re.search(r'(?i)' + ComicBook.CVDBSKIP, tagstring)
if not skip_found:
skip_found = re.search(r'(?i)' + ComicBook.CVDBSKIP, bd.notes_s)
retval = "skip" if skip_found else None
if retval is None:
# if no skip tag, see if there's a key tag in the tags or notes
issue_key = db.parse_key_tag(tagstring)
if issue_key == None:
issue_key = db.parse_key_tag(bd.notes_s)
if issue_key == None:
issue_key = int(bd.issue_key_s) if \
utils.is_number(bd.issue_key_s) else None
if issue_key != None:
# found a key tag! convert to an IssueRef
retval = IssueRef(self.issue_num_s, issue_key,
self.__bookdata.title_s,
self.__bookdata.cover_url_s)
return retval
def __extract_issue_ref(self):
'''
This method attempts to rebuild the IssueRef that the user chose the
last time that they scraped this comic. If it can do so, it will
return that IssueRef. If not, it will return None, or the
string "skip" (see below).
If the user has manually added the magic CVDBSKIP flag to the tags or
notes for this book, then this method will return the string "skip",
which should be interpreted as "never scrape this book".
'''
# in this method, its easier to work with tags as a single string
bd = self.__bookdata
tagstring = ', '.join(bd.tags_sl)
# check for the magic CVDBSKIP skip flag
skip_found = re.search(r'(?i)'+ComicBook.CVDBSKIP, tagstring)
if not skip_found:
skip_found = re.search(r'(?i)'+ComicBook.CVDBSKIP, bd.notes_s)
retval = "skip" if skip_found else None
if retval is None:
# if no skip tag, see if there's a key tag in the tags or notes
issue_key = db.parse_key_tag(tagstring)
if issue_key == None:
issue_key = db.parse_key_tag(bd.notes_s)
if issue_key == None:
issue_key = int(bd.issue_key_s) if \
utils.is_number(bd.issue_key_s) else None
if issue_key != None:
# found a key tag! convert to an IssueRef
retval = IssueRef(self.issue_num_s, issue_key,
self.__bookdata.title_s, self.__bookdata.cover_url_s);
return retval
def __add_key_to_tags(self, tags_sl, issue_key):
'''
Returns the given tag list, but with a "key tag" for the given issue_key
added in (iff key tags are supported by the current database
implementation.) If the given string already contains a valid
key tag, the tag will be REPLACED with the new one, otherwise the new key
tag will be appended to the end of the list.
If the given issue_key is None, the tags will be updated with the
magic CVDBSKIP tag instead of a regular key tag.
This method never returns None.
'''
updated_tagstring_s = None # our return value
# 1. clean up whitespace and None in our tagstring parameter
tagstring_s = ', '.join(tags_sl).strip() if tags_sl else ''
key_tag_s = db.create_key_tag_s(issue_key) \
if issue_key != None else ComicBook.CVDBSKIP
if key_tag_s and tagstring_s:
# 2. we have both a new key tag AND a non-empty tagstring; find and
# replace the existing tag (if it exists in the tagtring)
# with the new one.
matches = False
prev_issue_key = db.parse_key_tag(tagstring_s)
if prev_issue_key:
prev_key_tag_s = db.create_key_tag_s(prev_issue_key)
if prev_key_tag_s:
regexp = re.compile(r"(?i)" + prev_key_tag_s)
matches = regexp.search(tagstring_s)
if matches:
# 2a. yup, found an existing key tag--replace it with the new one
updated_tagstring_s = re.sub(regexp, key_tag_s, tagstring_s)
else:
# 2b. nope, no previous key tag found--just append the new key tag
if tagstring_s[-1] == ",":
tagstring_s = tagstring_s[:-1]
updated_tagstring_s = tagstring_s + ", " + key_tag_s
elif key_tag_s:
# 3. no previous tagstring, so the key tag *becomes* the new tagstring
updated_tagstring_s = key_tag_s
else:
# 4. there's no key tag available, so don't change the tagstring.
updated_tagstring_s = tagstring_s
return updated_tagstring_s.split(", ")
def __add_key_to_tags(self, tags_sl, issue_key):
'''
Returns the given tag list, but with a "key tag" for the given issue_key
added in (iff key tags are supported by the current database
implementation.) If the given string already contains a valid
key tag, the tag will be REPLACED with the new one, otherwise the new key
tag will be appended to the end of the list.
If the given issue_key is None, the tags will be updated with the
magic CVDBSKIP tag instead of a regular key tag.
This method never returns None.
'''
updated_tagstring_s = None # our return value
# 1. clean up whitespace and None in our tagstring parameter
tagstring_s = ', '.join(tags_sl).strip() if tags_sl else ''
key_tag_s = db.create_key_tag_s(issue_key) \
if issue_key != None else ComicBook.CVDBSKIP
if key_tag_s and tagstring_s:
# 2. we have both a new key tag AND a non-empty tagstring; find and
# replace the existing tag (if it exists in the tagtring)
# with the new one.
matches = False
prev_issue_key = db.parse_key_tag(tagstring_s)
if prev_issue_key:
prev_key_tag_s = db.create_key_tag_s(prev_issue_key)
if prev_key_tag_s:
regexp = re.compile(r"(?i)" + prev_key_tag_s)
matches = regexp.search(tagstring_s)
if matches:
# 2a. yup, found an existing key tag--replace it with the new one
updated_tagstring_s = re.sub(regexp, key_tag_s, tagstring_s)
else:
# 2b. nope, no previous key tag found--just append the new key tag
if tagstring_s[-1] == ",":
tagstring_s = tagstring_s[:-1]
updated_tagstring_s = tagstring_s +", " + key_tag_s
elif key_tag_s:
# 3. no previous tagstring, so the key tag *becomes* the new tagstring
updated_tagstring_s = key_tag_s
else:
# 4. there's no key tag available, so don't change the tagstring.
updated_tagstring_s = tagstring_s
return updated_tagstring_s.split(", ")
def __add_key_to_notes(self, notestring_s, issue_key):
'''
Returns a copy of the given comic book note string, but with a "key tag"
for the given issue_key appended onto the end (iff key tags are
supported by the current database implementation.) If the given
notes string already contains a valid key tag, the existing tag will be
REPLACED with the new one.
If the given issue_key is None, the note string will be updated with the
magic CVDBSKIP tag instead of a regular key tag.
This method never returns None.
'''
updated_notestring_s = None # our return value
# 1. clean up whitespace and None in our notestring parameter
notestring_s = notestring_s.strip() if notestring_s else ''
# Create a key-note that looks like either:
# 1) Scraped metadata from ComicVine [CVDB9999].
# 2) Scraped metadata from ComicVine [CVDB9999] on 2013.05.14 21:43:06.
key_tag_s = db.create_key_tag_s(issue_key) \
if issue_key != None else ComicBook.CVDBSKIP
date_s = " on "+strftime(r'%Y.%m.%d %X') \
if self.__scraper.config.note_scrape_date_b else ""
key_note_s = 'Scraped metadata from {0} [{1}]{2}.'.format(
"ComicVine", key_tag_s, date_s) if key_tag_s else ''
if key_note_s and notestring_s:
# 2. we have both a new key-note (based on the key tag), AND a
# non-empty notestring; find and replace the existing key-note (if
# it exists in the notestring) with the new one.
matches = False
prev_issue_key = db.parse_key_tag(notestring_s)
if prev_issue_key:
prev_key_tag = db.create_key_tag_s(prev_issue_key)
if prev_key_tag:
regexp = re.compile( r"(?i)Scraped.*?" + prev_key_tag \
+ "(]\.|.*?[\d\.]{8,} [\d:]{6,}\.)")
matches = regexp.search(notestring_s)
if matches:
# 2a. yup, found an existing key-note--replace it with the new one
updated_notestring_s = re.sub(regexp, key_note_s, notestring_s)
else:
# 2b. nope, no previous key-note found. try looking for the key
# tag on it's own (i.e. if the user added it by hand)
if prev_issue_key:
prev_key_tag = db.create_key_tag_s(prev_issue_key)
if prev_key_tag:
regexp = re.compile(r"(?i)" + prev_key_tag)
matches = regexp.search(notestring_s)
if matches:
# 2c. yup, found a key tag--replace it with the new one
updated_notestring_s = re.sub(regexp, key_tag_s,
notestring_s)
else:
# 2b. nope, no previous key found--just append the new key-note
updated_notestring_s = notestring_s + "\n\n" + key_note_s
elif key_note_s:
# 3. no previous notestring, so the key-note *becomes* the new string
updated_notestring_s = key_note_s
else:
# 4. there's no key tag available, so don't change the tagstring.
updated_notestring_s = notestring_s
return updated_notestring_s
def __add_key_to_notes(self, notestring_s, issue_key):
'''
Returns a copy of the given comic book note string, but with a "key tag"
for the given issue_key appended onto the end (iff key tags are
supported by the current database implementation.) If the given
notes string already contains a valid key tag, the existing tag will be
REPLACED with the new one.
If the given issue_key is None, the note string will be updated with the
magic CVDBSKIP tag instead of a regular key tag.
This method never returns None.
'''
updated_notestring_s = None # our return value
# 1. clean up whitespace and None in our notestring parameter
notestring_s = notestring_s.strip() if notestring_s else ''
# Create a key-note that looks like either:
# 1) Scraped metadata from ComicVine [CVDB9999].
# 2) Scraped metadata from ComicVine [CVDB9999] on 2013.05.14 21:43:06.
key_tag_s = db.create_key_tag_s(issue_key) \
if issue_key != None else ComicBook.CVDBSKIP
date_s = " on "+strftime(r'%Y.%m.%d %X') \
if self.__scraper.config.note_scrape_date_b else ""
key_note_s = 'Scraped metadata from {0} [{1}]{2}.'.format(
"ComicVine", key_tag_s, date_s) if key_tag_s else ''
if key_note_s and notestring_s:
# 2. we have both a new key-note (based on the key tag), AND a
# non-empty notestring; find and replace the existing key-note (if
# it exists in the notestring) with the new one.
matches = False
prev_issue_key = db.parse_key_tag(notestring_s)
if prev_issue_key:
prev_key_tag = db.create_key_tag_s(prev_issue_key)
if prev_key_tag:
regexp = re.compile( r"(?i)Scraped.*?" + prev_key_tag \
+ "(]\.|.*?[\d\.]{8,} [\d:]{6,}\.)")
matches = regexp.search(notestring_s)
if matches:
# 2a. yup, found an existing key-note--replace it with the new one
updated_notestring_s = re.sub(regexp, key_note_s, notestring_s)
else:
# 2b. nope, no previous key-note found. try looking for the key
# tag on it's own (i.e. if the user added it by hand)
if prev_issue_key:
prev_key_tag = db.create_key_tag_s(prev_issue_key)
if prev_key_tag:
regexp = re.compile(r"(?i)" + prev_key_tag)
matches = regexp.search(notestring_s)
if matches:
# 2c. yup, found a key tag--replace it with the new one
updated_notestring_s = re.sub(regexp, key_tag_s, notestring_s)
else:
# 2b. nope, no previous key found--just append the new key-note
updated_notestring_s = notestring_s + "\n\n" + key_note_s
elif key_note_s:
# 3. no previous notestring, so the key-note *becomes* the new string
updated_notestring_s = key_note_s
else:
# 4. there's no key tag available, so don't change the tagstring.
updated_notestring_s = notestring_s
return updated_notestring_s
