def log(self, message, level):
logger = logging.getLogger('mylar')
threadname = threading.currentThread().getName()
if level != 'DEBUG':
if mylar.OS_DETECT == "Windows" and mylar.OS_ENCODING is not "utf-8":
tmpthedate = unicodedata.normalize(
'NFKD',
helpers.now().decode(mylar.OS_ENCODING, "replace"))
else:
tmpthedate = helpers.now()
mylar.LOG_LIST.insert(0, (tmpthedate, message, level, threadname))
message = threadname + ' : ' + message
if level == 'DEBUG':
logger.debug(message)
elif level == 'INFO':
logger.info(message)
elif level == 'WARNING':
logger.warn(message)
elif level == 'FDEBUG':
logger.debug(message)
else:
logger.error(message)
def log(self, message, level):
logger = logging.getLogger('mylar')
threadname = threading.currentThread().getName()
if level != 'DEBUG':
if mylar.OS_DETECT == "Windows" and mylar.OS_ENCODING is not "utf-8":
tmpthedate = unicodedata.normalize('NFKD', helpers.now().decode(mylar.OS_ENCODING, "replace"))
else:
tmpthedate = helpers.now()
mylar.LOG_LIST.insert(0, (tmpthedate, message, level, threadname))
message = threadname + ' : ' + message
if level == 'DEBUG':
logger.debug(message)
elif level == 'INFO':
logger.info(message)
elif level == 'WARNING':
logger.warn(message)
elif level == 'FDEBUG':
logger.debug(message)
else:
logger.error(message)
def foundsearch(ComicID, IssueID, down=None):
# When doing a Force Search (Wanted tab), the resulting search calls this to update.
# this is all redudant code that forceRescan already does.
# should be redone at some point so that instead of rescanning entire
# series directory, it just scans for the issue it just downloaded and
# and change the status to Snatched accordingly. It is not to increment the have count
# at this stage as it's not downloaded - just the .nzb has been snatched and sent to SAB.
myDB = db.DBConnection()
comic = myDB.action('SELECT * FROM comics WHERE ComicID=?',
[ComicID]).fetchone()
issue = myDB.action('SELECT * FROM issues WHERE IssueID=?',
[IssueID]).fetchone()
CYear = issue['IssueDate'][:4]
if down is None:
# update the status to Snatched (so it won't keep on re-downloading!)
logger.fdebug("updating status to snatched")
controlValue = {"IssueID": IssueID}
newValue = {"Status": "Snatched"}
myDB.upsert("issues", newValue, controlValue)
# update the snatched DB
snatchedupdate = {"IssueID": IssueID, "Status": "Snatched"}
newsnatchValues = {
"ComicName": comic['ComicName'],
"ComicID": ComicID,
"Issue_Number": issue['Issue_Number'],
"DateAdded": helpers.now(),
"Status": "Snatched"
}
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
else:
snatchedupdate = {"IssueID": IssueID, "Status": "Downloaded"}
newsnatchValues = {
"ComicName": comic['ComicName'],
"ComicID": ComicID,
"Issue_Number": issue['Issue_Number'],
"DateAdded": helpers.now(),
"Status": "Downloaded"
}
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
#print ("finished updating snatched db.")
logger.info(u"Updating now complete for " + comic['ComicName'] +
" issue: " + str(issue['Issue_Number']))
return
def foundsearch(ComicID, IssueID, down=None):
# When doing a Force Search (Wanted tab), the resulting search calls this to update.
# this is all redudant code that forceRescan already does.
# should be redone at some point so that instead of rescanning entire
# series directory, it just scans for the issue it just downloaded and
# and change the status to Snatched accordingly. It is not to increment the have count
# at this stage as it's not downloaded - just the .nzb has been snatched and sent to SAB.
myDB = db.DBConnection()
comic = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone()
issue = myDB.action('SELECT * FROM issues WHERE IssueID=?', [IssueID]).fetchone()
CYear = issue['IssueDate'][:4]
if down is None:
# update the status to Snatched (so it won't keep on re-downloading!)
logger.fdebug("updating status to snatched")
controlValue = {"IssueID": IssueID}
newValue = {"Status": "Snatched"}
myDB.upsert("issues", newValue, controlValue)
# update the snatched DB
snatchedupdate = {"IssueID": IssueID,
"Status": "Snatched"
}
newsnatchValues = {"ComicName": comic['ComicName'],
"ComicID": ComicID,
"Issue_Number": issue['Issue_Number'],
"DateAdded": helpers.now(),
"Status": "Snatched"
}
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
else:
snatchedupdate = {"IssueID": IssueID,
"Status": "Downloaded"
}
newsnatchValues = {"ComicName": comic['ComicName'],
"ComicID": ComicID,
"Issue_Number": issue['Issue_Number'],
"DateAdded": helpers.now(),
"Status": "Downloaded"
}
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
#print ("finished updating snatched db.")
logger.info(u"Updating now complete for " + comic['ComicName'] + " issue: " + str(issue['Issue_Number']))
return
def log(message, level):
logger = logging.getLogger('mylar')
threadname = threading.currentThread().getName()
# Get the frame data of the method that made the original logger call
if len(inspect.stack()) > 2:
frame = inspect.getframeinfo(inspect.stack()[2][0])
program = os.path.basename(frame.filename)
method = frame.function
lineno = frame.lineno
else:
program = ""
method = ""
lineno = ""
if PY2:
message = safe_unicode(message)
message = message.encode(mylar.SYS_ENCODING)
if level != 'DEBUG' or mylar.LOG_LEVEL >= 2:
mylar.LOGLIST.insert(
0, (helpers.now(), message, level, threadname))
if len(mylar.LOGLIST) > 2500:
del mylar.LOGLIST[-1]
message = "%s : %s:%s:%s : %s" % (threadname, program, method,
lineno, message)
if level == 'DEBUG':
logger.debug(message)
elif level == 'INFO':
logger.info(message)
elif level == 'WARNING':
logger.warning(message)
else:
logger.error(message)
def markFailed(self):
#use this to forcibly mark a single issue as being Failed (ie. if a search result is sent to a client, but the result
#ends up passing in a 404 or something that makes it so that the download can't be initiated).
module = '[FAILED-DOWNLOAD]'
myDB = db.DBConnection()
logger.info(module + ' Marking as a Failed Download.')
logger.fdebug(module + 'nzb_name: ' + self.nzb_name)
logger.fdebug(module + 'issueid: ' + str(self.issueid))
logger.fdebug(module + 'nzb_id: ' + str(self.id))
logger.fdebug(module + 'prov: ' + self.prov)
logger.fdebug('oneoffinfo: ' + str(self.oneoffinfo))
if self.oneoffinfo:
ComicName = self.oneoffinfo['ComicName']
IssueNumber = self.oneoffinfo['IssueNumber']
else:
if 'annual' in self.nzb_name.lower():
logger.info(module + ' Annual detected.')
annchk = "yes"
issuenzb = myDB.selectone(
"SELECT * from annuals WHERE IssueID=? AND ComicName NOT NULL",
[self.issueid]).fetchone()
else:
issuenzb = myDB.selectone(
"SELECT * from issues WHERE IssueID=? AND ComicName NOT NULL",
[self.issueid]).fetchone()
ctrlVal = {"IssueID": self.issueid}
Vals = {"Status": 'Failed'}
myDB.upsert("issues", Vals, ctrlVal)
ComicName = issuenzb['ComicName']
IssueNumber = issuenzb['Issue_Number']
ctrlVal = {
"ID": self.id,
"Provider": self.prov,
"NZBName": self.nzb_name
}
Vals = {
"Status": 'Failed',
"ComicName": ComicName,
"Issue_Number": IssueNumber,
"IssueID": self.issueid,
"ComicID": self.comicid,
"DateFailed": helpers.now()
}
myDB.upsert("failed", Vals, ctrlVal)
logger.info(module + ' Successfully marked as Failed.')
def tehMain(forcerss=None):
logger.info('RSS Feed Check was last run at : ' + str(mylar.RSS_LASTRUN))
firstrun = "no"
#check the last run of rss to make sure it's not hammering.
if mylar.RSS_LASTRUN is None or mylar.RSS_LASTRUN == '' or mylar.RSS_LASTRUN == '0' or forcerss == True:
logger.info('RSS Feed Check First Ever Run.')
firstrun = "yes"
mins = 0
else:
c_obj_date = datetime.datetime.strptime(mylar.RSS_LASTRUN,
"%Y-%m-%d %H:%M:%S")
n_date = datetime.datetime.now()
absdiff = abs(n_date - c_obj_date)
mins = (absdiff.days * 24 * 60 * 60 +
absdiff.seconds) / 60.0 #3600 is for hours.
if firstrun == "no" and mins < int(mylar.RSS_CHECKINTERVAL):
logger.fdebug(
'RSS Check has taken place less than the threshold - not initiating at this time.'
)
return
mylar.RSS_LASTRUN = helpers.now()
logger.fdebug('Updating RSS Run time to : ' + str(mylar.RSS_LASTRUN))
mylar.config_write()
#function for looping through nzbs/torrent feeds
if mylar.ENABLE_TORRENTS:
logger.fdebug('[RSS] Initiating Torrent RSS Check.')
if mylar.ENABLE_KAT:
logger.fdebug('[RSS] Initiating Torrent RSS Feed Check on KAT.')
torrents(pickfeed='3')
if mylar.ENABLE_CBT:
logger.fdebug('[RSS] Initiating Torrent RSS Feed Check on CBT.')
torrents(pickfeed='1')
torrents(pickfeed='4')
logger.fdebug('[RSS] Initiating RSS Feed Check for NZB Providers.')
nzbs()
logger.fdebug('[RSS] RSS Feed Check/Update Complete')
logger.fdebug('[RSS] Watchlist Check for new Releases')
#if mylar.ENABLE_TORRENTS:
# if mylar.ENABLE_KAT:
# search.searchforissue(rsscheck='yes')
# if mylar.ENABLE_CBT:
mylar.search.searchforissue(rsscheck='yes')
#nzbcheck here
#nzbs(rsscheck='yes')
logger.fdebug('[RSS] Watchlist Check complete.')
return
def foundsearch(ComicID, IssueID):
myDB = db.DBConnection()
#print ("Updater-ComicID: " + str(ComicID))
#print ("Updater-IssueID: " + str(IssueID))
comic = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone()
issue = myDB.action('SELECT * FROM issues WHERE IssueID=?', [IssueID]).fetchone()
#print ("comic location: " + comic['ComicLocation'])
#this is too soon - file hasn't downloaded even yet.
#fixed and addressed in search.py and follow-thru here!
#check sab history for completion here :)
CYear = issue['IssueDate'][:4]
print ("year:" + str(CYear))
#slog = myDB.action('SELECT * FROM sablog WHERE ComicName=? AND ComicYEAR=?', [issue['ComicName'], str(CYear)]).fetchone()
#this checks the active queue for downloading/non-existant jobs
#--end queue check
#this checks history for completed jobs...
#---
#-- end history check
fc = filechecker.listFiles(comic['ComicLocation'], comic['ComicName'])
HaveDict = {"ComicID": ComicID}
newHave = { "Have": fc['comiccount'] }
myDB.upsert("comics", newHave, HaveDict)
#---
issue = myDB.action('SELECT * FROM issues WHERE IssueID=? AND ComicID=?', [IssueID, ComicID]).fetchone()
#print ("updating status to snatched")
controlValueDict = {"IssueID": IssueID}
newValueDict = {"Status": "Snatched"}
#print ("updating snatched db.")
myDB.upsert("issues", newValueDict, controlValueDict)
snatchedupdate = {"IssueID": IssueID}
newsnatchValues = {"ComicName": comic['ComicName'],
"ComicID": ComicID,
"Issue_Number": issue['Issue_Number'],
"DateAdded": helpers.now(),
"Status": "Snatched"
}
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
#we need to update sablog now to mark the nzo_id row as being completed and not used again.
#this becomes an issue with files downloaded x2 or same name...
#print ("finished updating snatched db.")
logger.info(u"Updating now complete for " + str(comic['ComicName']) + " issue: " + str(issue['Issue_Number']))
return
def tehMain():
logger.info('RSS Feed Check was last run at : ' + str(mylar.RSS_LASTRUN))
firstrun = "no"
#check the last run of rss to make sure it's not hammering.
if mylar.RSS_LASTRUN is None or mylar.RSS_LASTRUN == '' or mylar.RSS_LASTRUN == '0':
logger.info('RSS Feed Check First Ever Run.')
firstrun = "yes"
mins = 0
else:
c_obj_date = datetime.datetime.strptime(mylar.RSS_LASTRUN, "%Y-%m-%d %H:%M:%S")
n_date = datetime.datetime.now()
absdiff = abs(n_date - c_obj_date)
mins = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 60.0 #3600 is for hours.
if firstrun == "no" and mins < int(mylar.RSS_CHECKINTERVAL):
logger.fdebug('RSS Check has taken place less than the threshold - not initiating at this time.')
return
mylar.RSS_LASTRUN = helpers.now()
logger.fdebug('Updating RSS Run time to : ' + str(mylar.RSS_LASTRUN))
mylar.config_write()
#function for looping through nzbs/torrent feeds
if mylar.ENABLE_TORRENTS:
logger.fdebug("[RSS] Initiating Torrent RSS Check.")
if mylar.ENABLE_KAT:
logger.fdebug('[RSS] Initiating Torrent RSS Feed Check on KAT.')
torrents(pickfeed='3')
if mylar.ENABLE_CBT:
logger.fdebug('[RSS] Initiating Torrent RSS Feed Check on CBT.')
torrents(pickfeed='1')
torrents(pickfeed='4')
logger.fdebug('RSS] Initiating RSS Feed Check for NZB Providers.')
nzbs()
logger.fdebug('[RSS] RSS Feed Check/Update Complete')
logger.fdebug('[RSS] Watchlist Check for new Releases')
#if mylar.ENABLE_TORRENTS:
# if mylar.ENABLE_KAT:
# search.searchforissue(rsscheck='yes')
# if mylar.ENABLE_CBT:
mylar.search.searchforissue(rsscheck='yes')
#nzbcheck here
#nzbs(rsscheck='yes')
logger.fdebug('[RSS] Watchlist Check complete.')
return
def markFailed(self):
#use this to forcibly mark a single issue as being Failed (ie. if a search result is sent to a client, but the result
#ends up passing in a 404 or something that makes it so that the download can't be initiated).
module = '[FAILED-DOWNLOAD]'
myDB = db.DBConnection()
logger.info(module + ' Marking as a Failed Download.')
logger.fdebug(module + 'nzb_name: ' + self.nzb_name)
logger.fdebug(module + 'issueid: ' + str(self.issueid))
logger.fdebug(module + 'nzb_id: ' + str(self.id))
logger.fdebug(module + 'prov: ' + self.prov)
logger.fdebug('oneoffinfo: ' + str(self.oneoffinfo))
if self.oneoffinfo:
ComicName = self.oneoffinfo['ComicName']
IssueNumber = self.oneoffinfo['IssueNumber']
else:
if 'annual' in self.nzb_name.lower():
logger.info(module + ' Annual detected.')
annchk = "yes"
issuenzb = myDB.selectone("SELECT * from annuals WHERE IssueID=? AND ComicName NOT NULL", [self.issueid]).fetchone()
else:
issuenzb = myDB.selectone("SELECT * from issues WHERE IssueID=? AND ComicName NOT NULL", [self.issueid]).fetchone()
ctrlVal = {"IssueID": self.issueid}
Vals = {"Status": 'Failed'}
myDB.upsert("issues", Vals, ctrlVal)
ComicName = issuenzb['ComicName']
IssueNumber = issuenzb['Issue_Number']
ctrlVal = {"ID": self.id,
"Provider": self.prov,
"NZBName": self.nzb_name}
Vals = {"Status": 'Failed',
"ComicName": ComicName,
"Issue_Number": IssueNumber,
"IssueID": self.issueid,
"ComicID": self.comicid,
"DateFailed": helpers.now()}
myDB.upsert("failed", Vals, ctrlVal)
logger.info(module + ' Successfully marked as Failed.')
def log(self, message, level):
logger = logging.getLogger('mylar')
threadname = threading.currentThread().getName()
if level != 'DEBUG':
mylar.LOG_LIST.insert(0, (helpers.now(), message, level, threadname))
message = threadname + ' : ' + message
if level == 'DEBUG':
logger.debug(message)
elif level == 'INFO':
logger.info(message)
elif level == 'WARNING':
logger.warn(message)
else:
logger.error(message)
def foundsearch(ComicID, IssueID):
# When doing a Force Search (Wanted tab), the resulting search calls this to update.
# this is all redudant code that forceRescan already does.
# should be redone at some point so that instead of rescanning entire
# series directory, it just scans for the issue it just downloaded and
# and change the status to Snatched accordingly. It is not to increment the have count
# at this stage as it's not downloaded - just the .nzb has been snatched and sent to SAB.
myDB = db.DBConnection()
comic = myDB.action("SELECT * FROM comics WHERE ComicID=?", [ComicID]).fetchone()
issue = myDB.action("SELECT * FROM issues WHERE IssueID=?", [IssueID]).fetchone()
CYear = issue["IssueDate"][:4]
# fc = filechecker.listFiles(comic['ComicLocation'], comic['ComicName'])
# HaveDict = {"ComicID": ComicID}
# newHave = { "Have": fc['comiccount'] }
# myDB.upsert("comics", newHave, HaveDict)
# #---
issue = myDB.action("SELECT * FROM issues WHERE IssueID=? AND ComicID=?", [IssueID, ComicID]).fetchone()
# update the status to Snatched (so it won't keep on re-downloading!)
logger.fdebug("updating status to snatched")
controlValue = {"IssueID": IssueID}
newValue = {"Status": "Snatched"}
myDB.upsert("issues", newValue, controlValue)
# update the snatched DB
controlValueDict = {"IssueID": IssueID}
newValueDict = {"Status": "Snatched"}
logger.fdebug("updating snatched db.")
myDB.upsert("issues", newValueDict, controlValueDict)
snatchedupdate = {"IssueID": IssueID}
newsnatchValues = {
"ComicName": comic["ComicName"],
"ComicID": ComicID,
"Issue_Number": issue["Issue_Number"],
"DateAdded": helpers.now(),
"Status": "Snatched",
}
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
# print ("finished updating snatched db.")
logger.info(u"Updating now complete for " + str(comic["ComicName"]) + " issue: " + str(issue["Issue_Number"]))
return
def log(self, message, level):
logger = logging.getLogger('mylar')
threadname = threading.currentThread().getName()
if level != 'DEBUG':
mylar.LOG_LIST.insert(0,
(helpers.now(), message, level, threadname))
message = threadname + ' : ' + message
if level == 'DEBUG':
logger.debug(message)
elif level == 'INFO':
logger.info(message)
elif level == 'WARNING':
logger.warn(message)
elif level == 'FDEBUG':
logger.debug(message)
else:
logger.error(message)
#print ("recentchk: " + str(recentchk))
if recentchk <= 55:
lastpubdate = 'Present'
else:
lastpubdate = str(ltmonth) + ' ' + str(ltyear)
publishfigure = str(stmonth) + ' ' + str(styear) + ' - ' + str(lastpubdate)
controlValueStat = {"ComicID": comicid}
newValueStat = {
"Status": "Active",
"LatestIssue": latestiss,
"LatestDate": latestdate,
"ComicPublished": publishfigure,
"LastUpdated": helpers.now()
}
myDB.upsert("comics", newValueStat, controlValueStat)
if mylar.CVINFO or (mylar.CV_ONLY and mylar.CVINFO):
if not os.path.exists(os.path.join(comlocation,
"cvinfo")) or mylar.CV_ONETIMER:
with open(os.path.join(comlocation, "cvinfo"), "w") as text_file:
text_file.write(str(comic['ComicURL']))
logger.info(u"Updating complete for: " + comic['ComicName'])
#move the files...if imported is not empty (meaning it's not from the mass importer.)
if imported is None or imported == 'None':
pass
def emit(self, record):
message = self.format(record)
message = message.replace("\n", "<br />")
mylar.LOG_LIST.insert(0, (helpers.now(), message, record.levelname, record.threadName))
def pullitcheck(comic1off_name=None,comic1off_id=None,forcecheck=None, futurepull=None, issue=None):
if futurepull is None:
logger.info(u"Checking the Weekly Releases list for comics I'm watching...")
else:
logger.info('Checking the Future Releases list for upcoming comics I am watching for...')
myDB = db.DBConnection()
not_t = ['TP',
'NA',
'HC',
'PI']
not_c = ['PTG',
'COMBO PACK',
'(PP #']
lines = []
unlines = []
llen = []
ccname = []
pubdate = []
latestissue = []
w = 0
wc = 0
tot = 0
chkout = []
watchfnd = []
watchfndiss = []
watchfndextra = []
alternate = []
#print ("----------WATCHLIST--------")
a_list = []
b_list = []
comicid = []
mylardb = os.path.join(mylar.DATA_DIR, "mylar.db")
con = sqlite3.connect(str(mylardb))
with con:
cur = con.cursor()
# if it's a one-off check (during an add series), load the comicname here and ignore below.
if comic1off_name:
logger.fdebug("this is a one-off" + comic1off_name)
lines.append(comic1off_name.strip())
unlines.append(comic1off_name.strip())
comicid.append(comic1off_id)
latestissue.append(issue)
w = 1
else:
#let's read in the comic.watchlist from the db here
cur.execute("SELECT ComicID, ComicName, ComicYear, ComicPublisher, ComicPublished, LatestDate, ForceContinuing, AlternateSearch, LatestIssue from comics")
while True:
watchd = cur.fetchone()
#print ("watchd: " + str(watchd))
if watchd is None:
break
if 'Present' in watchd[4] or (helpers.now()[:4] in watchd[4]) or watchd[6] == 1:
# this gets buggered up when series are named the same, and one ends in the current
# year, and the new series starts in the same year - ie. Avengers
# lets' grab the latest issue date and see how far it is from current
# anything > 45 days we'll assume it's a false match ;)
logger.fdebug("ComicName: " + watchd[1])
latestdate = watchd[5]
logger.fdebug("latestdate: " + str(latestdate))
if latestdate[8:] == '':
logger.fdebug("invalid date " + str(latestdate) + " appending 01 for day for continuation.")
latest_day = '01'
else:
latest_day = latestdate[8:]
c_date = datetime.date(int(latestdate[:4]),int(latestdate[5:7]),int(latest_day))
n_date = datetime.date.today()
logger.fdebug("c_date : " + str(c_date) + " ... n_date : " + str(n_date))
recentchk = (n_date - c_date).days
logger.fdebug("recentchk: " + str(recentchk) + " days")
chklimit = helpers.checkthepub(watchd[0])
logger.fdebug("Check date limit set to : " + str(chklimit))
logger.fdebug(" ----- ")
if recentchk < int(chklimit) or watchd[6] == 1:
if watchd[6] == 1:
logger.fdebug('Forcing Continuing Series enabled for series...')
# let's not even bother with comics that are not in the Present.
a_list.append(watchd[1])
b_list.append(watchd[2])
comicid.append(watchd[0])
pubdate.append(watchd[4])
latestissue.append(watchd[8])
lines.append(a_list[w].strip())
unlines.append(a_list[w].strip())
w+=1 # we need to increment the count here, so we don't count the same comics twice (albeit with alternate names)
#here we load in the alternate search names for a series and assign them the comicid and
#alternate names
Altload = helpers.LoadAlternateSearchNames(watchd[7], watchd[0])
if Altload == 'no results':
pass
else:
wc = 0
alt_cid = Altload['ComicID']
n = 0
iscnt = Altload['Count']
while (n <= iscnt):
try:
altval = Altload['AlternateName'][n]
except IndexError:
break
cleanedname = altval['AlternateName']
a_list.append(altval['AlternateName'])
b_list.append(watchd[2])
comicid.append(alt_cid)
pubdate.append(watchd[4])
latestissue.append(watchd[8])
lines.append(a_list[w+wc].strip())
unlines.append(a_list[w+wc].strip())
logger.fdebug('loading in Alternate name for ' + str(cleanedname))
n+=1
wc+=1
w+=wc
#-- to be removed -
#print ( "Comic:" + str(a_list[w]) + " Year: " + str(b_list[w]) )
#if "WOLVERINE AND THE X-MEN" in str(a_list[w]): a_list[w] = "WOLVERINE AND X-MEN"
#lines.append(a_list[w].strip())
#unlines.append(a_list[w].strip())
#llen.append(a_list[w].splitlines())
#ccname.append(a_list[w].strip())
#tmpwords = a_list[w].split(None)
#ltmpwords = len(tmpwords)
#ltmp = 1
#-- end to be removed
else:
logger.fdebug("Determined to not be a Continuing series at this time.")
cnt = int(w-1)
cntback = int(w-1)
kp = []
ki = []
kc = []
otot = 0
logger.fdebug("You are watching for: " + str(w) + " comics")
#print ("----------THIS WEEK'S PUBLISHED COMICS------------")
if w > 0:
while (cnt > -1):
latestiss = latestissue[cnt]
lines[cnt] = lines[cnt].upper()
#llen[cnt] = str(llen[cnt])
logger.fdebug("looking for : " + lines[cnt])
sqlsearch = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\'\?\@]', ' ', lines[cnt])
sqlsearch = re.sub("\&", '%', sqlsearch)
sqlsearch = re.sub("\\bAND\\b", '%', sqlsearch)
sqlsearch = re.sub("\\bTHE\\b", '', sqlsearch)
if '+' in sqlsearch: sqlsearch = re.sub('\+', '%PLUS%', sqlsearch)
sqlsearch = re.sub(r'\s', '%', sqlsearch)
sqlsearch = sqlsearch + '%'
#logger.fdebug("searchsql: " + sqlsearch)
if futurepull is None:
weekly = myDB.select('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)', [sqlsearch])
else:
weekly = myDB.select('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM future WHERE COMIC LIKE (?)', [sqlsearch])
#cur.execute('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)', [lines[cnt]])
for week in weekly:
if week == None:
break
for nono in not_t:
if nono in week['PUBLISHER']:
#logger.fdebug("nono present")
break
if nono in week['ISSUE']:
#logger.fdebug("graphic novel/tradeback detected..ignoring.")
break
for nothere in not_c:
if nothere in week['EXTRA']:
#logger.fdebug("nothere present")
break
else:
comicnm = week['COMIC']
#here's the tricky part, ie. BATMAN will match on
#every batman comic, not exact
logger.fdebug("comparing" + comicnm + "..to.." + unlines[cnt].upper())
#-NEW-
# strip out all special characters and compare
watchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\'\?\@]', '', unlines[cnt])
comicnm = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\'\?\@]', '', comicnm)
if "THE" in watchcomic.upper() or "THE" in comicnm.upper():
modwatchcomic = re.sub("\\bTHE\\b", "", watchcomic.upper())
modcomicnm = re.sub("\\bTHE\\b", "", comicnm)
else:
modwatchcomic = watchcomic
modcomicnm = comicnm
if '&' in watchcomic.upper():
modwatchcomic = re.sub('\&', 'AND', modwatchcomic.upper())
modcomicnm = re.sub('\&', 'AND', modcomicnm)
if '&' in comicnm:
modwatchcom = re.sub('\&', 'AND', modwatchcomic.upper())
modcomicnm = re.sub('\&', 'AND', modcomicnm)
#thnx to A+X for this...
if '+' in watchcomic:
logger.fdebug("+ detected...adjusting.")
#logger.fdebug("comicnm:" + comicnm)
#logger.fdebug("watchcomic:" + watchcomic)
modwatchcomic = re.sub('\+', 'PLUS', modwatchcomic)
#logger.fdebug("modcomicnm:" + modcomicnm)
#logger.fdebug("modwatchcomic:" + modwatchcomic)
#annuals!
if 'ANNUAL' in comicnm.upper():
modcomicnm = re.sub("\\bANNUAL\\b", "", modcomicnm.upper())
watchcomic = re.sub(r'\s', '', watchcomic)
comicnm = re.sub(r'\s', '', comicnm)
modwatchcomic = re.sub(r'\s', '', modwatchcomic)
modcomicnm = re.sub(r'\s', '', modcomicnm)
logger.fdebug("watchcomic : " + str(watchcomic) + " / mod :" + str(modwatchcomic))
logger.fdebug("comicnm : " + str(comicnm) + " / mod :" + str(modcomicnm))
if comicnm == watchcomic.upper() or modcomicnm == modwatchcomic.upper():
logger.fdebug("matched on:" + comicnm + "..." + watchcomic.upper())
pass
# elif ("ANNUAL" in week['EXTRA']):
# pass
# print ( row[3] + " matched on ANNUAL")
else:
break
if ("NA" not in week['ISSUE']) and ("HC" not in week['ISSUE']):
if ("COMBO PACK" not in week['EXTRA']) and ("2ND PTG" not in week['EXTRA']) and ("3RD PTG" not in week['EXTRA']):
#this all needs to get redone, so the ability to compare issue dates can be done systematically.
#Everything below should be in it's own function - at least the callable sections - in doing so, we can
#then do comparisons when two titles of the same name exist and are by definition 'current'. Issue date comparisons
#would identify the difference between two #1 titles within the same series year, but have different publishing dates.
#Wolverine (2013) & Wolverine (2014) are good examples of this situation.
#of course initially, the issue data for the newer series wouldn't have any issue data associated with it so it would be
#a null value, but given that the 2013 series (as an example) would be from 2013-05-01, it obviously wouldn't be a match to
#the current date & year (2014). Throwing out that, we could just assume that the 2014 would match the #1.
#get the issue number of the 'weeklypull' series.
#load in the actual series issue number's store-date (not publishing date)
#---use a function to check db, then return the results in a tuple/list to avoid db locks.
#if the store-date is >= weeklypull-list date then continue processing below.
#if the store-date is <= weeklypull-list date then break.
### week['ISSUE'] #issue # from pullist
### week['SHIPDATE'] #weeklypull-list date
### comicid[cnt] #comicid of matched series
## if it's a futurepull, the dates get mixed up when two titles exist of the same name
## ie. Wolverine-2011 & Wolverine-2014
## we need to set the compare date to today's date ( Now() ) in this case.
if futurepull:
usedate = datetime.datetime.now().strftime('%Y%m%d') #convert to yyyymmdd
else:
usedate = re.sub("[^0-9]", "", week['SHIPDATE'])
if 'ANNUAL' in comicnm.upper():
chktype = 'annual'
else:
chktype = 'series'
datevalues = loaditup(watchcomic, comicid[cnt], week['ISSUE'], chktype)
date_downloaded = None
altissuenum = None
if datevalues == 'no results':
#if a series is a .NOW on the pullist, it won't match up against anything (probably) on CV
#let's grab the digit from the .NOW, poll it against CV to see if there's any data
#if there is, check the store date to make sure it's a 'new' release.
#if it is a new release that has the same store date as the .NOW, then we assume
#it's the same, and assign it the AltIssueNumber to do extra searches.
if week['ISSUE'].isdigit() == False and '.' not in week['ISSUE']:
altissuenum = re.sub("[^0-9]", "", week['ISSUE']) # carry this through to get added to db later if matches
logger.fdebug('altissuenum is: ' + str(altissuenum))
altvalues = loaditup(watchcomic, comicid[cnt], altissuenum, chktype)
if altvalues == 'no results':
logger.fdebug('No alternate Issue numbering - something is probably wrong somewhere.')
pass
validcheck = checkthis(altvalues[0]['issuedate'], altvalues[0]['status'], usedate)
if validcheck == False:
if date_downloaded is None:
break
if chktype == 'series':
latest_int = helpers.issuedigits(latestiss)
weekiss_int = helpers.issuedigits(week['ISSUE'])
logger.fdebug('comparing ' + str(latest_int) + ' to ' + str(weekiss_int))
if (latest_int > weekiss_int) or (latest_int == 0 or weekiss_int == 0):
logger.fdebug(str(week['ISSUE']) + ' should not be the next issue in THIS volume of the series.')
logger.fdebug('it should be either greater than ' + str(latestiss) + ' or an issue #0')
break
else:
#logger.fdebug('issuedate:' + str(datevalues[0]['issuedate']))
#logger.fdebug('status:' + str(datevalues[0]['status']))
datestatus = datevalues[0]['status']
validcheck = checkthis(datevalues[0]['issuedate'], datestatus, usedate)
if validcheck == True:
if datestatus != 'Downloaded' and datestatus != 'Archived':
pass
else:
logger.fdebug('Issue #' + str(week['ISSUE']) + ' already downloaded.')
date_downloaded = datestatus
else:
if date_downloaded is None:
break
otot+=1
dontadd = "no"
if dontadd == "no":
#print (row[0], row[1], row[2])
tot+=1
#kp.append(row[0])
#ki.append(row[1])
#kc.append(comicnm)
if "ANNUAL" in comicnm.upper():
watchfndextra.append("annual")
ComicName = str(unlines[cnt]) + " Annual"
else:
ComicName = str(unlines[cnt])
watchfndextra.append("none")
watchfnd.append(comicnm)
watchfndiss.append(week['ISSUE'])
ComicID = comicid[cnt]
if not mylar.CV_ONLY:
ComicIssue = str(watchfndiss[tot -1] + ".00")
else:
ComicIssue = str(watchfndiss[tot -1])
ComicDate = str(week['SHIPDATE'])
#ComicName = str(unlines[cnt])
logger.fdebug("Watchlist hit for : " + ComicName + " ISSUE: " + str(watchfndiss[tot -1]))
if futurepull is None:
# here we add to comics.latest
updater.latest_update(ComicID=ComicID, LatestIssue=ComicIssue, LatestDate=ComicDate)
# here we add to upcoming table...
statusupdate = updater.upcoming_update(ComicID=ComicID, ComicName=ComicName, IssueNumber=ComicIssue, IssueDate=ComicDate, forcecheck=forcecheck)
else:
# here we add to upcoming table...
statusupdate = updater.upcoming_update(ComicID=ComicID, ComicName=ComicName, IssueNumber=ComicIssue, IssueDate=ComicDate, forcecheck=forcecheck, futurepull='yes', altissuenumber=altissuenum)
# here we update status of weekly table...
if statusupdate is not None:
cstatus = statusupdate['Status']
cstatusid = statusupdate['ComicID']
else:
cstatus = None
cstatusid = None
#set the variable fp to denote updating the futurepull list ONLY
if futurepull is None:
fp = None
else:
cstatusid = ComicID
fp = "yes"
if date_downloaded is None:
updater.weekly_update(ComicName=week['COMIC'], IssueNumber=ComicIssue, CStatus=cstatus, CID=cstatusid, futurepull=fp, altissuenumber=altissuenum)
else:
updater.weekly_update(ComicName=week['COMIC'], IssueNumber=ComicIssue, CStatus=date_downloaded, CID=cstatusid, futurepull=fp, altissuenumber=altissuenum)
break
break
break
cnt-=1
#print ("-------------------------")
logger.fdebug("There are " + str(otot) + " comics this week to get!")
#print ("However I've already grabbed " + str(btotal) )
#print ("I need to get " + str(tot) + " comic(s)!" )
logger.info(u"Finished checking for comics on my watchlist.")
#con.close()
return
recentchk = (n_date - c_date).days
#print ("recentchk: " + str(recentchk))
if recentchk <= 55:
lastpubdate = 'Present'
else:
lastpubdate = str(ltmonth) + ' ' + str(ltyear)
publishfigure = str(stmonth) + ' ' + str(styear) + ' - ' + str(lastpubdate)
controlValueStat = {"ComicID": comicid}
newValueStat = {"Status": "Active",
"LatestIssue": latestiss,
"LatestDate": latestdate,
"ComicPublished": publishfigure,
"LastUpdated": helpers.now()
}
myDB.upsert("comics", newValueStat, controlValueStat)
if mylar.CVINFO or (mylar.CV_ONLY and mylar.CVINFO):
if not os.path.exists(os.path.join(comlocation,"cvinfo")) or mylar.CV_ONETIMER:
with open(os.path.join(comlocation,"cvinfo"),"w") as text_file:
text_file.write(str(comic['ComicURL']))
logger.info(u"Updating complete for: " + comic['ComicName'])
#move the files...if imported is not empty (meaning it's not from the mass importer.)
if imported is None or imported == 'None':
pass
else:
def GCDimport(gcomicid, pullupd=None,imported=None,ogcname=None):
# this is for importing via GCD only and not using CV.
# used when volume spanning is discovered for a Comic (and can't be added using CV).
# Issue Counts are wrong (and can't be added).
# because Comicvine ComicID and GCD ComicID could be identical at some random point, let's distinguish.
# CV = comicid, GCD = gcomicid :) (ie. CV=2740, GCD=G3719)
gcdcomicid = gcomicid
myDB = db.DBConnection()
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ComicID": gcdcomicid}
comic = myDB.action('SELECT ComicName, ComicYear, Total, ComicPublished, ComicImage, ComicLocation, ComicPublisher FROM comics WHERE ComicID=?', [gcomicid]).fetchone()
ComicName = comic[0]
ComicYear = comic[1]
ComicIssues = comic[2]
ComicPublished = comic[3]
comlocation = comic[5]
ComicPublisher = comic[6]
#ComicImage = comic[4]
#print ("Comic:" + str(ComicName))
newValueDict = {"Status": "Loading"}
myDB.upsert("comics", newValueDict, controlValueDict)
# we need to lookup the info for the requested ComicID in full now
#comic = cv.getComic(comicid,'comic')
if not comic:
logger.warn("Error fetching comic. ID for : " + gcdcomicid)
if dbcomic is None:
newValueDict = {"ComicName": "Fetch failed, try refreshing. (%s)" % (gcdcomicid),
"Status": "Active"}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("comics", newValueDict, controlValueDict)
return
#run the re-sortorder here in order to properly display the page
if pullupd is None:
helpers.ComicSort(comicorder=mylar.COMICSORT, imported=gcomicid)
if ComicName.startswith('The '):
sortname = ComicName[4:]
else:
sortname = ComicName
logger.info(u"Now adding/updating: " + ComicName)
#--Now that we know ComicName, let's try some scraping
#--Start
# gcd will return issue details (most importantly publishing date)
comicid = gcomicid[1:]
resultURL = "/series/" + str(comicid) + "/"
gcdinfo=parseit.GCDdetails(comseries=None, resultURL=resultURL, vari_loop=0, ComicID=gcdcomicid, TotalIssues=ComicIssues, issvariation=None, resultPublished=None)
if gcdinfo == "No Match":
logger.warn("No matching result found for " + ComicName + " (" + ComicYear + ")" )
updater.no_searchresults(gcomicid)
nomatch = "true"
return nomatch
logger.info(u"Sucessfully retrieved details for " + ComicName )
# print ("Series Published" + parseit.resultPublished)
#--End
ComicImage = gcdinfo['ComicImage']
#comic book location on machine
# setup default location here
if comlocation is None:
# let's remove the non-standard characters here.
u_comicnm = ComicName
u_comicname = u_comicnm.encode('ascii', 'ignore').strip()
if ':' in u_comicname or '/' in u_comicname or ',' in u_comicname:
comicdir = u_comicname
if ':' in comicdir:
comicdir = comicdir.replace(':','')
if '/' in comicdir:
comicdir = comicdir.replace('/','-')
if ',' in comicdir:
comicdir = comicdir.replace(',','')
else: comicdir = u_comicname
series = comicdir
publisher = ComicPublisher
year = ComicYear
#do work to generate folder path
values = {'$Series': series,
'$Publisher': publisher,
'$Year': year,
'$series': series.lower(),
'$publisher': publisher.lower(),
'$Volume': year
}
if mylar.FOLDER_FORMAT == '':
comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + comic['ComicYear'] + ")"
else:
comlocation = mylar.DESTINATION_DIR + "/" + helpers.replace_all(mylar.FOLDER_FORMAT, values)
#comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + ComicYear + ")"
if mylar.DESTINATION_DIR == "":
logger.error(u"There is no general directory specified - please specify in Config/Post-Processing.")
return
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
comlocation = comlocation.replace(' ', mylar.REPLACE_CHAR)
#if it doesn't exist - create it (otherwise will bugger up later on)
if os.path.isdir(str(comlocation)):
logger.info(u"Directory (" + str(comlocation) + ") already exists! Continuing...")
else:
#print ("Directory doesn't exist!")
#try:
# os.makedirs(str(comlocation))
# logger.info(u"Directory successfully created at: " + str(comlocation))
#except OSError:
# logger.error(u"Could not create comicdir : " + str(comlocation))
filechecker.validateAndCreateDirectory(comlocation, True)
comicIssues = gcdinfo['totalissues']
#let's download the image...
if os.path.exists(mylar.CACHE_DIR):pass
else:
#let's make the dir.
try:
os.makedirs(str(mylar.CACHE_DIR))
logger.info(u"Cache Directory successfully created at: " + str(mylar.CACHE_DIR))
except OSError:
logger.error(u"Could not create cache dir : " + str(mylar.CACHE_DIR))
coverfile = os.path.join(mylar.CACHE_DIR, str(gcomicid) + ".jpg")
#try:
urllib.urlretrieve(str(ComicImage), str(coverfile))
try:
with open(str(coverfile)) as f:
ComicImage = os.path.join('cache',str(gcomicid) + ".jpg")
#this is for Firefox when outside the LAN...it works, but I don't know how to implement it
#without breaking the normal flow for inside the LAN (above)
#ComicImage = "http://" + str(mylar.HTTP_HOST) + ":" + str(mylar.HTTP_PORT) + "/cache/" + str(comi$
logger.info(u"Sucessfully retrieved cover for " + ComicName)
#if the comic cover local is checked, save a cover.jpg to the series folder.
if mylar.COMIC_COVER_LOCAL:
comiclocal = os.path.join(str(comlocation) + "/cover.jpg")
shutil.copy(ComicImage,comiclocal)
except IOError as e:
logger.error(u"Unable to save cover locally at this time.")
#if comic['ComicVersion'].isdigit():
# comicVol = "v" + comic['ComicVersion']
#else:
# comicVol = None
controlValueDict = {"ComicID": gcomicid}
newValueDict = {"ComicName": ComicName,
"ComicSortName": sortname,
"ComicYear": ComicYear,
"Total": comicIssues,
"ComicLocation": comlocation,
#"ComicVersion": comicVol,
"ComicImage": ComicImage,
#"ComicPublisher": comic['ComicPublisher'],
#"ComicPublished": comicPublished,
"DateAdded": helpers.today(),
"Status": "Loading"}
myDB.upsert("comics", newValueDict, controlValueDict)
#comicsort here...
#run the re-sortorder here in order to properly display the page
if pullupd is None:
helpers.ComicSort(sequence='update')
logger.info(u"Sucessfully retrieved issue details for " + ComicName )
n = 0
iscnt = int(comicIssues)
issnum = []
issname = []
issdate = []
int_issnum = []
#let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :)
latestiss = "0"
latestdate = "0000-00-00"
#print ("total issues:" + str(iscnt))
#---removed NEW code here---
logger.info(u"Now adding/updating issues for " + ComicName)
bb = 0
while (bb <= iscnt):
#---NEW.code
try:
gcdval = gcdinfo['gcdchoice'][bb]
#print ("gcdval: " + str(gcdval))
except IndexError:
#account for gcd variation here
if gcdinfo['gcdvariation'] == 'gcd':
#print ("gcd-variation accounted for.")
issdate = '0000-00-00'
int_issnum = int ( issis / 1000 )
break
if 'nn' in str(gcdval['GCDIssue']):
#no number detected - GN, TP or the like
logger.warn(u"Non Series detected (Graphic Novel, etc) - cannot proceed at this time.")
updater.no_searchresults(comicid)
return
elif '.' in str(gcdval['GCDIssue']):
issst = str(gcdval['GCDIssue']).find('.')
issb4dec = str(gcdval['GCDIssue'])[:issst]
#if the length of decimal is only 1 digit, assume it's a tenth
decis = str(gcdval['GCDIssue'])[issst+1:]
if len(decis) == 1:
decisval = int(decis) * 10
issaftdec = str(decisval)
if len(decis) == 2:
decisval = int(decis)
issaftdec = str(decisval)
if int(issaftdec) == 0: issaftdec = "00"
gcd_issue = issb4dec + "." + issaftdec
gcdis = (int(issb4dec) * 1000) + decisval
else:
gcdis = int(str(gcdval['GCDIssue'])) * 1000
gcd_issue = str(gcdval['GCDIssue'])
#get the latest issue / date using the date.
int_issnum = int( gcdis / 1000 )
issdate = str(gcdval['GCDDate'])
issid = "G" + str(gcdval['IssueID'])
if gcdval['GCDDate'] > latestdate:
latestiss = str(gcd_issue)
latestdate = str(gcdval['GCDDate'])
#print("(" + str(bb) + ") IssueID: " + str(issid) + " IssueNo: " + str(gcd_issue) + " Date" + str(issdate) )
#---END.NEW.
# check if the issue already exists
iss_exists = myDB.action('SELECT * from issues WHERE IssueID=?', [issid]).fetchone()
# Only change the status & add DateAdded if the issue is not already in the database
if iss_exists is None:
newValueDict['DateAdded'] = helpers.today()
#adjust for inconsistencies in GCD date format - some dates have ? which borks up things.
if "?" in str(issdate):
issdate = "0000-00-00"
controlValueDict = {"IssueID": issid}
newValueDict = {"ComicID": gcomicid,
"ComicName": ComicName,
"Issue_Number": gcd_issue,
"IssueDate": issdate,
"Int_IssueNumber": int_issnum
}
#print ("issueid:" + str(controlValueDict))
#print ("values:" + str(newValueDict))
if mylar.AUTOWANT_ALL:
newValueDict['Status'] = "Wanted"
elif issdate > helpers.today() and mylar.AUTOWANT_UPCOMING:
newValueDict['Status'] = "Wanted"
else:
newValueDict['Status'] = "Skipped"
if iss_exists:
#print ("Existing status : " + str(iss_exists['Status']))
newValueDict['Status'] = iss_exists['Status']
myDB.upsert("issues", newValueDict, controlValueDict)
bb+=1
# logger.debug(u"Updating comic cache for " + ComicName)
# cache.getThumb(ComicID=issue['issueid'])
# logger.debug(u"Updating cache for: " + ComicName)
# cache.getThumb(ComicIDcomicid)
controlValueStat = {"ComicID": gcomicid}
newValueStat = {"Status": "Active",
"LatestIssue": latestiss,
"LatestDate": latestdate,
"LastUpdated": helpers.now()
}
myDB.upsert("comics", newValueStat, controlValueStat)
if mylar.CVINFO:
if not os.path.exists(comlocation + "/cvinfo"):
with open(comlocation + "/cvinfo","w") as text_file:
text_file.write("http://www.comicvine.com/volume/49-" + str(comicid))
logger.info(u"Updating complete for: " + ComicName)
#move the files...if imported is not empty (meaning it's not from the mass importer.)
if imported is None or imported == 'None':
pass
else:
if mylar.IMP_MOVE:
logger.info("Mass import - Move files")
moveit.movefiles(gcomicid,comlocation,ogcname)
else:
logger.info("Mass import - Moving not Enabled. Setting Archived Status for import.")
moveit.archivefiles(gcomicid,ogcname)
#check for existing files...
updater.forceRescan(gcomicid)
if pullupd is None:
# lets' check the pullist for anyting at this time as well since we're here.
if mylar.AUTOWANT_UPCOMING and 'Present' in ComicPublished:
logger.info(u"Checking this week's pullist for new issues of " + ComicName)
updater.newpullcheck(comic['ComicName'], gcomicid)
#here we grab issues that have been marked as wanted above...
results = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [gcomicid])
if results:
logger.info(u"Attempting to grab wanted issues for : " + ComicName)
for result in results:
foundNZB = "none"
if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB or mylar.NZBX) and (mylar.SAB_HOST):
foundNZB = search.searchforissue(result['IssueID'])
if foundNZB == "yes":
updater.foundsearch(result['ComicID'], result['IssueID'])
else: logger.info(u"No issues marked as wanted for " + ComicName)
logger.info(u"Finished grabbing what I could.")
def run(self):
with rss_lock:
logger.info('RSS Feed Check was last run at : ' + str(mylar.RSS_LASTRUN))
firstrun = "no"
#check the last run of rss to make sure it's not hammering.
if mylar.RSS_LASTRUN is None or mylar.RSS_LASTRUN == '' or mylar.RSS_LASTRUN == '0' or self.forcerss == True:
logger.info('RSS Feed Check First Ever Run.')
firstrun = "yes"
mins = 0
else:
c_obj_date = datetime.datetime.strptime(mylar.RSS_LASTRUN, "%Y-%m-%d %H:%M:%S")
n_date = datetime.datetime.now()
absdiff = abs(n_date - c_obj_date)
mins = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 60.0 #3600 is for hours.
if firstrun == "no" and mins < int(mylar.RSS_CHECKINTERVAL):
logger.fdebug('RSS Check has taken place less than the threshold - not initiating at this time.')
return
mylar.RSS_LASTRUN = helpers.now()
logger.fdebug('Updating RSS Run time to : ' + str(mylar.RSS_LASTRUN))
mylar.config_write()
#function for looping through nzbs/torrent feeds
if mylar.ENABLE_TORRENT_SEARCH:
logger.info('[RSS] Initiating Torrent RSS Check.')
if mylar.ENABLE_KAT:
logger.info('[RSS] Initiating Torrent RSS Feed Check on KAT.')
rsscheck.torrents(pickfeed='3')
rsscheck.torrents(pickfeed='6')
if mylar.ENABLE_32P:
logger.info('[RSS] Initiating Torrent RSS Feed Check on 32P.')
if mylar.MODE_32P == 0:
logger.fdebug('[RSS] 32P mode set to Legacy mode. Monitoring New Releases feed only.')
if any([mylar.PASSKEY_32P is None, mylar.PASSKEY_32P == '', mylar.RSSFEED_32P is None, mylar.RSSFEED_32P == '']):
logger.error('[RSS] Unable to validate information from provided RSS Feed. Verify that the feed provided is a current one.')
else:
rsscheck.torrents(pickfeed='1', feedinfo=mylar.KEYS_32P)
else:
logger.fdebug('[RSS] 32P mode set to Auth mode. Monitoring all personal notification feeds & New Releases feed')
if any([mylar.USERNAME_32P is None, mylar.USERNAME_32P == '', mylar.PASSWORD_32P is None]):
logger.error('[RSS] Unable to sign-on to 32P to validate settings. Please enter/check your username password in the configuration.')
else:
if mylar.KEYS_32P is None:
feed32p = auth32p.info32p()
feedinfo = feed32p.authenticate()
if feedinfo == "disable":
mylar.ENABLE_32P = 0
mylar.config_write()
else:
feedinfo = mylar.FEEDINFO_32P
if feedinfo is None or len(feedinfo) == 0 or feedinfo == "disable":
logger.error('[RSS] Unable to retrieve any information from 32P for RSS Feeds. Skipping for now.')
else:
rsscheck.torrents(pickfeed='1', feedinfo=feedinfo[0])
x = 0
#assign personal feeds for 32p > +8
for fi in feedinfo:
x+=1
pfeed_32p = str(7 + x)
rsscheck.torrents(pickfeed=pfeed_32p, feedinfo=fi)
logger.info('[RSS] Initiating RSS Feed Check for NZB Providers.')
rsscheck.nzbs(forcerss=self.forcerss)
logger.info('[RSS] RSS Feed Check/Update Complete')
logger.info('[RSS] Watchlist Check for new Releases')
mylar.search.searchforissue(rsscheck='yes')
logger.info('[RSS] Watchlist Check complete.')
if self.forcerss:
logger.info('[RSS] Successfully ran a forced RSS Check.')
return
def pullitcheck(comic1off_name=None,comic1off_id=None,forcecheck=None):
logger.info(u"Checking the Weekly Releases list for comics I'm watching...")
myDB = db.DBConnection()
not_t = ['TP',
'NA',
'HC',
'PI']
not_c = ['PTG',
'COMBO PACK',
'(PP #']
lines = []
unlines = []
llen = []
ccname = []
pubdate = []
w = 0
tot = 0
chkout = []
watchfnd = []
watchfndiss = []
watchfndextra = []
#print ("----------WATCHLIST--------")
a_list = []
b_list = []
comicid = []
mylardb = os.path.join(mylar.DATA_DIR, "mylar.db")
con = sqlite3.connect(str(mylardb))
with con:
cur = con.cursor()
# if it's a one-off check (during an add series), load the comicname here and ignore below.
if comic1off_name:
logger.fdebug("this is a one-off" + str(comic1off_name))
lines.append(comic1off_name.strip())
unlines.append(comic1off_name.strip())
comicid.append(comic1off_id)
w = 1
else:
#let's read in the comic.watchlist from the db here
cur.execute("SELECT ComicID, ComicName, ComicYear, ComicPublisher, ComicPublished, LatestDate from comics")
while True:
watchd = cur.fetchone()
#print ("watchd: " + str(watchd))
if watchd is None:
break
if 'Present' in watchd[4] or (helpers.now()[:4] in watchd[4]):
# this gets buggered up when series are named the same, and one ends in the current
# year, and the new series starts in the same year - ie. Avengers
# lets' grab the latest issue date and see how far it is from current
# anything > 45 days we'll assume it's a false match ;)
#logger.fdebug("ComicName: " + watchd[1])
latestdate = watchd[5]
#logger.fdebug("latestdate: " + str(latestdate))
c_date = datetime.date(int(latestdate[:4]),int(latestdate[5:7]),1)
n_date = datetime.date.today()
#logger.fdebug("c_date : " + str(c_date) + " ... n_date : " + str(n_date))
recentchk = (n_date - c_date).days
#logger.fdebug("recentchk: " + str(recentchk) + " days")
#logger.fdebug(" ----- ")
if recentchk < 55:
# let's not even bother with comics that are in the Present.
a_list.append(watchd[1])
b_list.append(watchd[2])
comicid.append(watchd[0])
pubdate.append(watchd[4])
#print ( "Comic:" + str(a_list[w]) + " Year: " + str(b_list[w]) )
#if "WOLVERINE AND THE X-MEN" in str(a_list[w]): a_list[w] = "WOLVERINE AND X-MEN"
lines.append(a_list[w].strip())
unlines.append(a_list[w].strip())
llen.append(a_list[w].splitlines())
ccname.append(a_list[w].strip())
tmpwords = a_list[w].split(None)
ltmpwords = len(tmpwords)
ltmp = 1
w+=1
cnt = int(w-1)
cntback = int(w-1)
kp = []
ki = []
kc = []
otot = 0
logger.fdebug("You are watching for: " + str(w) + " comics")
#print ("----------THIS WEEK'S PUBLISHED COMICS------------")
if w > 0:
while (cnt > -1):
lines[cnt] = lines[cnt].upper()
#llen[cnt] = str(llen[cnt])
#logger.fdebug("looking for : " + str(lines[cnt]))
sqlsearch = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\'\?\@]', ' ', lines[cnt])
sqlsearch = re.sub(r'\s', '%', sqlsearch)
if 'THE' in sqlsearch: sqlsearch = re.sub('THE', '', sqlsearch)
if '+' in sqlsearch: sqlsearch = re.sub('\+', '%PLUS%', sqlsearch)
#logger.fdebug("searchsql: " + str(sqlsearch))
weekly = myDB.select('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)', [sqlsearch])
#cur.execute('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)', [lines[cnt]])
for week in weekly:
if week == None:
break
for nono in not_t:
if nono in week['PUBLISHER']:
#logger.fdebug("nono present")
break
if nono in week['ISSUE']:
#logger.fdebug("graphic novel/tradeback detected..ignoring.")
break
for nothere in not_c:
if nothere in week['EXTRA']:
#logger.fdebug("nothere present")
break
else:
comicnm = week['COMIC']
#here's the tricky part, ie. BATMAN will match on
#every batman comic, not exact
#logger.fdebug("comparing" + str(comicnm) + "..to.." + str(unlines[cnt]).upper())
#-NEW-
# strip out all special characters and compare
watchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\'\?\@]', '', unlines[cnt])
comicnm = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\'\?\@]', '', comicnm)
watchcomic = re.sub(r'\s', '', watchcomic)
comicnm = re.sub(r'\s', '', comicnm)
#logger.fdebug("Revised_Watch: " + watchcomic)
#logger.fdebug("ComicNM: " + comicnm)
if 'THE' in watchcomic.upper():
modwatchcomic = re.sub('THE', '', watchcomic.upper())
modcomicnm = re.sub('THE', '', comicnm)
else:
modwatchcomic = watchcomic
modcomicnm = comicnm
#thnx to A+X for this...
if '+' in watchcomic:
logger.fdebug("+ detected...adjusting.")
#logger.fdebug("comicnm:" + comicnm)
#logger.fdebug("watchcomic:" + watchcomic)
modwatchcomic = re.sub('\+', 'PLUS', modwatchcomic)
#logger.fdebug("modcomicnm:" + modcomicnm)
#logger.fdebug("modwatchcomic:" + modwatchcomic)
if comicnm == watchcomic.upper() or modcomicnm == modwatchcomic.upper():
logger.fdebug("matched on:" + str(comicnm) + "..." + str(watchcomic).upper())
pass
elif ("ANNUAL" in week['EXTRA']):
pass
#print ( row[3] + " matched on ANNUAL")
else:
break
if ("NA" not in week['ISSUE']) and ("HC" not in week['ISSUE']):
if ("COMBO PACK" not in week['EXTRA']) and ("2ND PTG" not in week['EXTRA']) and ("3RD PTG" not in week['EXTRA']):
otot+=1
dontadd = "no"
if dontadd == "no":
#print (row[0], row[1], row[2])
tot+=1
#kp.append(row[0])
#ki.append(row[1])
#kc.append(comicnm)
if ("ANNUAL" in week['EXTRA']):
watchfndextra.append("annual")
else:
watchfndextra.append("none")
watchfnd.append(comicnm)
watchfndiss.append(week['ISSUE'])
ComicID = comicid[cnt]
if not mylar.CV_ONLY:
ComicIssue = str(watchfndiss[tot -1] + ".00")
else:
ComicIssue = str(watchfndiss[tot -1])
ComicDate = str(week['SHIPDATE'])
ComicName = str(unlines[cnt])
logger.fdebug("Watchlist hit for : " + ComicName + " ISSUE: " + str(watchfndiss[tot -1]))
# here we add to comics.latest
updater.latest_update(ComicID=ComicID, LatestIssue=ComicIssue, LatestDate=ComicDate)
# here we add to upcoming table...
statusupdate = updater.upcoming_update(ComicID=ComicID, ComicName=ComicName, IssueNumber=ComicIssue, IssueDate=ComicDate, forcecheck=forcecheck)
# here we update status of weekly table...
if statusupdate is not None:
cstatus = statusupdate['Status']
cstatusid = statusupdate['ComicID']
else:
cstatus = None
cstatusid = None
updater.weekly_update(ComicName=week['COMIC'], IssueNumber=ComicIssue, CStatus=cstatus, CID=cstatusid)
break
break
break
cnt-=1
#print ("-------------------------")
logger.fdebug("There are " + str(otot) + " comics this week to get!")
#print ("However I've already grabbed " + str(btotal) )
#print ("I need to get " + str(tot) + " comic(s)!" )
logger.info(u"Finished checking for comics on my watchlist.")
#con.close()
return
def Process(self):
module = '[FAILED-DOWNLOAD]'
myDB = db.DBConnection()
if self.nzb_name and self.nzb_folder:
self._log('Failed download has been detected: ' + self.nzb_name + ' in ' + self.nzb_folder)
#since this has already been passed through the search module, which holds the IssueID in the nzblog,
#let's find the matching nzbname and pass it the IssueID in order to mark it as Failed and then return
#to the search module and continue trucking along.
nzbname = self.nzb_name
#remove extensions from nzb_name if they somehow got through (Experimental most likely)
extensions = ('.cbr', '.cbz')
if nzbname.lower().endswith(extensions):
fd, ext = os.path.splitext(nzbname)
self._log("Removed extension from nzb: " + ext)
nzbname = re.sub(str(ext), '', str(nzbname))
#replace spaces
nzbname = re.sub(' ', '.', str(nzbname))
nzbname = re.sub('[\,\:\?\'\(\)]', '', str(nzbname))
nzbname = re.sub('[\&]', 'and', str(nzbname))
nzbname = re.sub('_', '.', str(nzbname))
logger.fdebug(module + ' After conversions, nzbname is : ' + str(nzbname))
self._log("nzbname: " + str(nzbname))
nzbiss = myDB.selectone("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
self._log("Failure - could not initially locate nzbfile in my database to rename.")
logger.fdebug(module + ' Failure - could not locate nzbfile initially')
# if failed on spaces, change it all to decimals and try again.
nzbname = re.sub('_', '.', str(nzbname))
self._log("trying again with this nzbname: " + str(nzbname))
logger.fdebug(module + ' Trying to locate nzbfile again with nzbname of : ' + str(nzbname))
nzbiss = myDB.selectone("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
logger.error(module + ' Unable to locate downloaded file to rename. PostProcessing aborted.')
self._log('Unable to locate downloaded file to rename. PostProcessing aborted.')
self.valreturn.append({"self.log": self.log,
"mode": 'stop'})
return self.queue.put(self.valreturn)
else:
self._log("I corrected and found the nzb as : " + str(nzbname))
logger.fdebug(module + ' Auto-corrected and found the nzb as : ' + str(nzbname))
issueid = nzbiss['IssueID']
else:
issueid = nzbiss['IssueID']
logger.fdebug(module + ' Issueid: ' + str(issueid))
sarc = nzbiss['SARC']
#use issueid to get publisher, series, year, issue number
else:
issueid = self.issueid
nzbiss = myDB.selectone("SELECT * from nzblog WHERE IssueID=?", [issueid]).fetchone()
if nzbiss is None:
logger.info(module + ' Cannot locate corresponding record in download history. This will be implemented soon.')
self.valreturn.append({"self.log": self.log,
"mode": 'stop'})
return self.queue.put(self.valreturn)
nzbname = nzbiss['NZBName']
# find the provider.
self.prov = nzbiss['PROVIDER']
logger.info(module + ' Provider: ' + self.prov)
# grab the id.
self.id = nzbiss['ID']
logger.info(module + ' ID: ' + self.id)
annchk = "no"
if 'annual' in nzbname.lower():
logger.info(module + ' Annual detected.')
annchk = "yes"
issuenzb = myDB.selectone("SELECT * from annuals WHERE IssueID=? AND ComicName NOT NULL", [issueid]).fetchone()
else:
issuenzb = myDB.selectone("SELECT * from issues WHERE IssueID=? AND ComicName NOT NULL", [issueid]).fetchone()
if issuenzb is not None:
logger.info(module + ' issuenzb found.')
if helpers.is_number(issueid):
sandwich = int(issuenzb['IssueID'])
else:
logger.info(module + ' issuenzb not found.')
#if it's non-numeric, it contains a 'G' at the beginning indicating it's a multi-volume
#using GCD data. Set sandwich to 1 so it will bypass and continue post-processing.
if 'S' in issueid:
sandwich = issueid
elif 'G' in issueid or '-' in issueid:
sandwich = 1
if helpers.is_number(sandwich):
if sandwich < 900000:
# if sandwich is less than 900000 it's a normal watchlist download. Bypass.
pass
else:
logger.info('Failed download handling for story-arcs and one-off\'s are not supported yet. Be patient!')
self._log(' Unable to locate downloaded file to rename. PostProcessing aborted.')
self.valreturn.append({"self.log": self.log,
"mode": 'stop'})
return self.queue.put(self.valreturn)
comicid = issuenzb['ComicID']
issuenumOG = issuenzb['Issue_Number']
logger.info(module + ' Successfully detected as : ' + issuenzb['ComicName'] + ' issue: ' + str(issuenzb['Issue_Number']) + ' that was downloaded using ' + self.prov)
self._log('Successfully detected as : ' + issuenzb['ComicName'] + ' issue: ' + str(issuenzb['Issue_Number']) + ' downloaded using ' + self.prov)
logger.info(module + ' Marking as a Failed Download.')
self._log('Marking as a Failed Download.')
ctrlVal = {"IssueID": issueid}
Vals = {"Status": 'Failed'}
myDB.upsert("issues", Vals, ctrlVal)
ctrlVal = {"ID": self.id,
"Provider": self.prov,
"NZBName": nzbname}
Vals = {"Status": 'Failed',
"ComicName": issuenzb['ComicName'],
"Issue_Number": issuenzb['Issue_Number'],
"IssueID": issueid,
"ComicID": comicid,
"DateFailed": helpers.now()}
myDB.upsert("failed", Vals, ctrlVal)
logger.info(module + ' Successfully marked as Failed.')
self._log('Successfully marked as Failed.')
if mylar.CONFIG.FAILED_AUTO:
logger.info(module + ' Sending back to search to see if we can find something that will not fail.')
self._log('Sending back to search to see if we can find something better that will not fail.')
self.valreturn.append({"self.log": self.log,
"mode": 'retry',
"issueid": issueid,
"comicid": comicid,
"comicname": issuenzb['ComicName'],
"issuenumber": issuenzb['Issue_Number'],
"annchk": annchk})
return self.queue.put(self.valreturn)
else:
logger.info(module + ' Stopping search here as automatic handling of failed downloads is not enabled *hint*')
self._log('Stopping search here as automatic handling of failed downloads is not enabled *hint*')
self.valreturn.append({"self.log": self.log,
"mode": 'stop'})
return self.queue.put(self.valreturn)
def _OneOffs(self, **kwargs):
index = 0
if 'index' in kwargs:
index = int(kwargs['index'])
links = []
entries = []
flist = []
book = ''
gbd = str(mylar.CONFIG.GRABBAG_DIR + '/*').encode('utf-8')
flist = glob.glob(gbd)
readlist = []
for book in flist:
issue = {}
fileexists = True
book = book.encode('utf-8')
issue['Title'] = book
issue['IssueID'] = book
issue['fileloc'] = book
issue['filename'] = book
issue['image'] = None
issue['thumbnail'] = None
issue['updated'] = helpers.now()
if not os.path.isfile(issue['fileloc']):
fileexists = False
if fileexists:
readlist.append(issue)
if len(readlist) > 0:
if index <= len(readlist):
subset = readlist[index:(index + self.PAGE_SIZE)]
for issue in subset:
metainfo = None
metainfo = [{'writer': None,'summary': ''}]
entries.append(
{
'title': escape(issue['Title']),
'id': escape('comic:%s' % issue['IssueID']),
'updated': issue['updated'],
'content': escape('%s' % (metainfo[0]['summary'])),
'href': '%s?cmd=deliverFile&file=%s&filename=%s' % (self.opdsroot, quote_plus(issue['fileloc']), quote_plus(issue['filename'])),
'kind': 'acquisition',
'rel': 'file',
'author': metainfo[0]['writer'],
'image': issue['image'],
'thumbnail': issue['thumbnail'],
}
)
feed = {}
feed['title'] = 'Mylar OPDS - One-Offs'
feed['id'] = escape('OneOffs')
feed['updated'] = mylar.helpers.now()
links.append(getLink(href=self.opdsroot,type='application/atom+xml; profile=opds-catalog; kind=navigation', rel='start', title='Home'))
links.append(getLink(href='%s?cmd=OneOffs' % self.opdsroot,type='application/atom+xml; profile=opds-catalog; kind=navigation',rel='self'))
if len(readlist) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=OneOffs&index=%s' % (self.opdsroot, index+self.PAGE_SIZE), type='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Read&index=%s' % (self.opdsroot, index-self.PAGE_SIZE), type='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
self.data = feed
return
def emit(self, record):
message = self.format(record)
message = message.replace("\n", "<br />")
mylar.LOG_LIST.insert(
0, (helpers.now(), message, record.levelname, record.threadName))
def GCDimport(gcomicid):
# this is for importing via GCD only and not using CV.
# used when volume spanning is discovered for a Comic (and can't be added using CV).
# Issue Counts are wrong (and can't be added).
# because Comicvine ComicID and GCD ComicID could be identical at some random point, let's distinguish.
# CV = comicid, GCD = gcomicid :) (ie. CV=2740, GCD=G3719)
gcdcomicid = gcomicid
myDB = db.DBConnection()
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ComicID": gcdcomicid}
comic = myDB.action(
"SELECT ComicName, ComicYear, Total, ComicPublished, ComicImage, ComicLocation FROM comics WHERE ComicID=?",
[gcomicid],
).fetchone()
ComicName = comic[0]
ComicYear = comic[1]
ComicIssues = comic[2]
comlocation = comic[5]
# ComicImage = comic[4]
# print ("Comic:" + str(ComicName))
newValueDict = {"Status": "Loading"}
myDB.upsert("comics", newValueDict, controlValueDict)
# we need to lookup the info for the requested ComicID in full now
# comic = cv.getComic(comicid,'comic')
if not comic:
logger.warn("Error fetching comic. ID for : " + gcdcomicid)
if dbcomic is None:
newValueDict = {"ComicName": "Fetch failed, try refreshing. (%s)" % (gcdcomicid), "Status": "Active"}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("comics", newValueDict, controlValueDict)
return
if ComicName.startswith("The "):
sortname = ComicName[4:]
else:
sortname = ComicName
logger.info(u"Now adding/updating: " + ComicName)
# --Now that we know ComicName, let's try some scraping
# --Start
# gcd will return issue details (most importantly publishing date)
comicid = gcomicid[1:]
resultURL = "/series/" + str(comicid) + "/"
gcdinfo = parseit.GCDdetails(
comseries=None,
resultURL=resultURL,
vari_loop=0,
ComicID=gcdcomicid,
TotalIssues=ComicIssues,
issvariation=None,
resultPublished=None,
)
if gcdinfo == "No Match":
logger.warn("No matching result found for " + ComicName + " (" + ComicYear + ")")
updater.no_searchresults(gcomicid)
nomatch = "true"
return nomatch
logger.info(u"Sucessfully retrieved details for " + ComicName)
# print ("Series Published" + parseit.resultPublished)
# --End
ComicImage = gcdinfo["ComicImage"]
# comic book location on machine
# setup default location here
if comlocation is None:
if ":" in ComicName or "/" in ComicName or "," in ComicName:
comicdir = ComicName
if ":" in comicdir:
comicdir = comicdir.replace(":", "")
if "/" in comicdir:
comicdir = comicdir.replace("/", "-")
if "," in comicdir:
comicdir = comicdir.replace(",", "")
else:
comicdir = ComicName
comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + ComicYear + ")"
if mylar.DESTINATION_DIR == "":
logger.error(u"There is no general directory specified - please specify in Config/Post-Processing.")
return
if mylar.REPLACE_SPACES:
# mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
comlocation = comlocation.replace(" ", mylar.REPLACE_CHAR)
# if it doesn't exist - create it (otherwise will bugger up later on)
if os.path.isdir(str(comlocation)):
logger.info(u"Directory (" + str(comlocation) + ") already exists! Continuing...")
else:
# print ("Directory doesn't exist!")
try:
os.makedirs(str(comlocation))
logger.info(u"Directory successfully created at: " + str(comlocation))
except OSError:
logger.error(u"Could not create comicdir : " + str(comlocation))
comicIssues = gcdinfo["totalissues"]
# let's download the image...
if os.path.exists(mylar.CACHE_DIR):
pass
else:
# let's make the dir.
try:
os.makedirs(str(mylar.CACHE_DIR))
logger.info(u"Cache Directory successfully created at: " + str(mylar.CACHE_DIR))
except OSError:
logger.error(u"Could not create cache dir : " + str(mylar.CACHE_DIR))
coverfile = mylar.CACHE_DIR + "/" + str(gcomicid) + ".jpg"
urllib.urlretrieve(str(ComicImage), str(coverfile))
try:
with open(str(coverfile)) as f:
ComicImage = "cache/" + str(gcomicid) + ".jpg"
logger.info(u"Sucessfully retrieved cover for " + str(ComicName))
except IOError as e:
logger.error(u"Unable to save cover locally at this time.")
controlValueDict = {"ComicID": gcomicid}
newValueDict = {
"ComicName": ComicName,
"ComicSortName": sortname,
"ComicYear": ComicYear,
"Total": comicIssues,
"ComicLocation": comlocation,
"ComicImage": ComicImage,
# "ComicPublisher": comic['ComicPublisher'],
# "ComicPublished": comicPublished,
"DateAdded": helpers.today(),
"Status": "Loading",
}
myDB.upsert("comics", newValueDict, controlValueDict)
logger.info(u"Sucessfully retrieved issue details for " + ComicName)
n = 0
iscnt = int(comicIssues)
issnum = []
issname = []
issdate = []
int_issnum = []
# let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :)
latestiss = "0"
latestdate = "0000-00-00"
# print ("total issues:" + str(iscnt))
# ---removed NEW code here---
logger.info(u"Now adding/updating issues for " + ComicName)
bb = 0
while bb <= iscnt:
# ---NEW.code
try:
gcdval = gcdinfo["gcdchoice"][bb]
# print ("gcdval: " + str(gcdval))
except IndexError:
# account for gcd variation here
if gcdinfo["gcdvariation"] == "gcd":
# print ("gcd-variation accounted for.")
issdate = "0000-00-00"
int_issnum = int(issis / 1000)
break
if "nn" in str(gcdval["GCDIssue"]):
# no number detected - GN, TP or the like
logger.warn(u"Non Series detected (Graphic Novel, etc) - cannot proceed at this time.")
updater.no_searchresults(comicid)
return
elif "." in str(gcdval["GCDIssue"]):
issst = str(gcdval["GCDIssue"]).find(".")
issb4dec = str(gcdval["GCDIssue"])[:issst]
# if the length of decimal is only 1 digit, assume it's a tenth
decis = str(gcdval["GCDIssue"])[issst + 1 :]
if len(decis) == 1:
decisval = int(decis) * 10
issaftdec = str(decisval)
if len(decis) == 2:
decisval = int(decis)
issaftdec = str(decisval)
if int(issaftdec) == 0:
issaftdec = "00"
gcd_issue = issb4dec + "." + issaftdec
gcdis = (int(issb4dec) * 1000) + decisval
else:
gcdis = int(str(gcdval["GCDIssue"])) * 1000
gcd_issue = str(gcdval["GCDIssue"])
# get the latest issue / date using the date.
int_issnum = int(gcdis / 1000)
issdate = str(gcdval["GCDDate"])
issid = "G" + str(gcdval["IssueID"])
if gcdval["GCDDate"] > latestdate:
latestiss = str(gcd_issue)
latestdate = str(gcdval["GCDDate"])
# print("(" + str(bb) + ") IssueID: " + str(issid) + " IssueNo: " + str(gcd_issue) + " Date" + str(issdate) )
# ---END.NEW.
# check if the issue already exists
iss_exists = myDB.action("SELECT * from issues WHERE IssueID=?", [issid]).fetchone()
# Only change the status & add DateAdded if the issue is not already in the database
if iss_exists is None:
newValueDict["DateAdded"] = helpers.today()
# adjust for inconsistencies in GCD date format - some dates have ? which borks up things.
if "?" in str(issdate):
issdate = "0000-00-00"
controlValueDict = {"IssueID": issid}
newValueDict = {
"ComicID": gcomicid,
"ComicName": ComicName,
"Issue_Number": gcd_issue,
"IssueDate": issdate,
"Int_IssueNumber": int_issnum,
}
# print ("issueid:" + str(controlValueDict))
# print ("values:" + str(newValueDict))
if mylar.AUTOWANT_ALL:
newValueDict["Status"] = "Wanted"
# elif release_dict['releasedate'] > helpers.today() and mylar.AUTOWANT_UPCOMING:
# newValueDict['Status'] = "Wanted"
else:
newValueDict["Status"] = "Skipped"
if iss_exists:
# print ("Existing status : " + str(iss_exists['Status']))
newValueDict["Status"] = iss_exists["Status"]
myDB.upsert("issues", newValueDict, controlValueDict)
bb += 1
# logger.debug(u"Updating comic cache for " + ComicName)
# cache.getThumb(ComicID=issue['issueid'])
# logger.debug(u"Updating cache for: " + ComicName)
# cache.getThumb(ComicIDcomicid)
# check for existing files...
updater.forceRescan(gcomicid)
controlValueStat = {"ComicID": gcomicid}
newValueStat = {
"Status": "Active",
"LatestIssue": latestiss,
"LatestDate": latestdate,
"LastUpdated": helpers.now(),
}
myDB.upsert("comics", newValueStat, controlValueStat)
logger.info(u"Updating complete for: " + ComicName)
# lets' check the pullist for anyting at this time as well since we're here.
if mylar.AUTOWANT_UPCOMING:
logger.info(u"Checking this week's pullist for new issues of " + str(ComicName))
updater.newpullcheck()
# here we grab issues that have been marked as wanted above...
results = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [gcomicid])
if results:
logger.info(u"Attempting to grab wanted issues for : " + ComicName)
for result in results:
foundNZB = "none"
if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL) and (mylar.SAB_HOST):
foundNZB = search.searchforissue(result["IssueID"])
if foundNZB == "yes":
updater.foundsearch(result["ComicID"], result["IssueID"])
else:
logger.info(u"No issues marked as wanted for " + ComicName)
logger.info(u"Finished grabbing what I could.")
def foundsearch(ComicID, IssueID, mode=None, down=None, provider=None, SARC=None, IssueArcID=None):
# When doing a Force Search (Wanted tab), the resulting search calls this to update.
# this is all redudant code that forceRescan already does.
# should be redone at some point so that instead of rescanning entire
# series directory, it just scans for the issue it just downloaded and
# and change the status to Snatched accordingly. It is not to increment the have count
# at this stage as it's not downloaded - just the .nzb has been snatched and sent to SAB.
myDB = db.DBConnection()
logger.info('comicid: ' + str(ComicID))
logger.info('issueid: ' + str(IssueID))
if mode != 'story_arc':
comic = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone()
ComicName = comic['ComicName']
if mode == 'want_ann':
issue = myDB.action('SELECT * FROM annuals WHERE IssueID=?', [IssueID]).fetchone()
else:
issue = myDB.action('SELECT * FROM issues WHERE IssueID=?', [IssueID]).fetchone()
CYear = issue['IssueDate'][:4]
else:
issue = myDB.action('SELECT * FROM readinglist WHERE IssueArcID=?', [IssueArcID]).fetchone()
ComicName = issue['ComicName']
CYear = issue['IssueYEAR']
if down is None:
# update the status to Snatched (so it won't keep on re-downloading!)
logger.fdebug('updating status to snatched')
logger.fdebug('provider is ' + provider)
newValue = {"Status": "Snatched"}
if mode == 'story_arc':
cValue = {"IssueArcID": IssueArcID}
snatchedupdate = {"IssueArcID": IssueArcID}
myDB.upsert("readinglist", newValue, cValue)
# update the snatched DB
snatchedupdate = {"IssueID": IssueArcID,
"Status": "Snatched",
"Provider": provider
}
else:
if mode == 'want_ann':
controlValue = {"IssueID": IssueID}
myDB.upsert("annuals", newValue, controlValue)
else:
controlValue = {"IssueID": IssueID}
myDB.upsert("issues", newValue, controlValue)
# update the snatched DB
snatchedupdate = {"IssueID": IssueID,
"Status": "Snatched",
"Provider": provider
}
if mode == 'story_arc':
IssueNum = issue['IssueNumber']
newsnatchValues = {"ComicName": ComicName,
"ComicID": 'None',
"Issue_Number": IssueNum,
"DateAdded": helpers.now(),
"Status": "Snatched"
}
else:
if mode == 'want_ann':
IssueNum = "Annual " + issue['Issue_Number']
else:
IssueNum = issue['Issue_Number']
newsnatchValues = {"ComicName": ComicName,
"ComicID": ComicID,
"Issue_Number": IssueNum,
"DateAdded": helpers.now(),
"Status": "Snatched"
}
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
logger.info("updated the snatched.")
else:
logger.info("updating the downloaded.")
if mode == 'want_ann':
IssueNum = "Annual " + issue['Issue_Number']
elif mode == 'story_arc':
IssueNum = issue['IssueNumber']
IssueID = IssueArcID
else:
IssueNum = issue['Issue_Number']
snatchedupdate = {"IssueID": IssueID,
"Status": "Downloaded",
"Provider": provider
}
newsnatchValues = {"ComicName": ComicName,
"ComicID": ComicID,
"Issue_Number": IssueNum,
"DateAdded": helpers.now(),
"Status": "Downloaded"
}
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
if mode == 'story_arc':
cValue = {"IssueArcID": IssueArcID}
nValue = {"Status": "Downloaded"}
myDB.upsert("readinglist", nValue, cValue)
else:
controlValue = {"IssueID": IssueID}
newValue = {"Status": "Downloaded"}
myDB.upsert("issues", newValue, controlValue)
#print ("finished updating snatched db.")
logger.info('Updating now complete for ' + ComicName + ' issue: ' + str(IssueNum))
return
def _OneOffs(self, **kwargs):
index = 0
if 'index' in kwargs:
index = int(kwargs['index'])
links = []
entries = []
flist = []
book = ''
gbd = str(mylar.CONFIG.GRABBAG_DIR + '/*')
flist = glob.glob(gbd)
readlist = []
for book in flist:
issue = {}
fileexists = True
book = book
issue['Title'] = book
issue['IssueID'] = book
issue['fileloc'] = book
issue['filename'] = book
issue['image'] = None
issue['thumbnail'] = None
issue['updated'] = helpers.now()
if not os.path.isfile(issue['fileloc']):
fileexists = False
if fileexists:
readlist.append(issue)
if len(readlist) > 0:
if index <= len(readlist):
subset = readlist[index:(index + self.PAGE_SIZE)]
for issue in subset:
metainfo = None
metainfo = [{'writer': None,'summary': ''}]
entries.append(
{
'title': escape(issue['Title']),
'id': escape('comic:%s' % issue['IssueID']),
'updated': issue['updated'],
'content': escape('%s' % (metainfo[0]['summary'])),
'href': '%s?cmd=deliverFile&file=%s&filename=%s' % (self.opdsroot, quote_plus(issue['fileloc']), quote_plus(issue['filename'])),
'kind': 'acquisition',
'rel': 'file',
'author': metainfo[0]['writer'],
'image': issue['image'],
'thumbnail': issue['thumbnail'],
}
)
feed = {}
feed['title'] = 'Mylar OPDS - One-Offs'
feed['id'] = escape('OneOffs')
feed['updated'] = mylar.helpers.now()
links.append(getLink(href=self.opdsroot,type='application/atom+xml; profile=opds-catalog; kind=navigation', rel='start', title='Home'))
links.append(getLink(href='%s?cmd=OneOffs' % self.opdsroot,type='application/atom+xml; profile=opds-catalog; kind=navigation',rel='self'))
if len(readlist) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=OneOffs&index=%s' % (self.opdsroot, index+self.PAGE_SIZE), type='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Read&index=%s' % (self.opdsroot, index-self.PAGE_SIZE), type='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
self.data = feed
return
def GCDimport(gcomicid, pullupd=None, imported=None, ogcname=None):
# this is for importing via GCD only and not using CV.
# used when volume spanning is discovered for a Comic (and can't be added using CV).
# Issue Counts are wrong (and can't be added).
# because Comicvine ComicID and GCD ComicID could be identical at some random point, let's distinguish.
# CV = comicid, GCD = gcomicid :) (ie. CV=2740, GCD=G3719)
gcdcomicid = gcomicid
myDB = db.DBConnection()
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ComicID": gcdcomicid}
comic = myDB.action(
'SELECT ComicName, ComicYear, Total, ComicPublished, ComicImage, ComicLocation, ComicPublisher FROM comics WHERE ComicID=?',
[gcomicid]).fetchone()
ComicName = comic[0]
ComicYear = comic[1]
ComicIssues = comic[2]
ComicPublished = comic[3]
comlocation = comic[5]
ComicPublisher = comic[6]
#ComicImage = comic[4]
#print ("Comic:" + str(ComicName))
newValueDict = {"Status": "Loading"}
myDB.upsert("comics", newValueDict, controlValueDict)
# we need to lookup the info for the requested ComicID in full now
#comic = cv.getComic(comicid,'comic')
if not comic:
logger.warn("Error fetching comic. ID for : " + gcdcomicid)
if dbcomic is None:
newValueDict = {
"ComicName":
"Fetch failed, try refreshing. (%s)" % (gcdcomicid),
"Status": "Active"
}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("comics", newValueDict, controlValueDict)
return
#run the re-sortorder here in order to properly display the page
if pullupd is None:
helpers.ComicSort(comicorder=mylar.COMICSORT, imported=gcomicid)
if ComicName.startswith('The '):
sortname = ComicName[4:]
else:
sortname = ComicName
logger.info(u"Now adding/updating: " + ComicName)
#--Now that we know ComicName, let's try some scraping
#--Start
# gcd will return issue details (most importantly publishing date)
comicid = gcomicid[1:]
resultURL = "/series/" + str(comicid) + "/"
gcdinfo = parseit.GCDdetails(comseries=None,
resultURL=resultURL,
vari_loop=0,
ComicID=gcdcomicid,
TotalIssues=ComicIssues,
issvariation=None,
resultPublished=None)
if gcdinfo == "No Match":
logger.warn("No matching result found for " + ComicName + " (" +
ComicYear + ")")
updater.no_searchresults(gcomicid)
nomatch = "true"
return nomatch
logger.info(u"Sucessfully retrieved details for " + ComicName)
# print ("Series Published" + parseit.resultPublished)
#--End
ComicImage = gcdinfo['ComicImage']
#comic book location on machine
# setup default location here
if comlocation is None:
# let's remove the non-standard characters here.
u_comicnm = ComicName
u_comicname = u_comicnm.encode('ascii', 'ignore').strip()
if ':' in u_comicname or '/' in u_comicname or ',' in u_comicname:
comicdir = u_comicname
if ':' in comicdir:
comicdir = comicdir.replace(':', '')
if '/' in comicdir:
comicdir = comicdir.replace('/', '-')
if ',' in comicdir:
comicdir = comicdir.replace(',', '')
else:
comicdir = u_comicname
series = comicdir
publisher = ComicPublisher
year = ComicYear
#do work to generate folder path
values = {
'$Series': series,
'$Publisher': publisher,
'$Year': year,
'$series': series.lower(),
'$publisher': publisher.lower(),
'$Volume': year
}
if mylar.FOLDER_FORMAT == '':
comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + comic[
'ComicYear'] + ")"
else:
comlocation = mylar.DESTINATION_DIR + "/" + helpers.replace_all(
mylar.FOLDER_FORMAT, values)
#comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + ComicYear + ")"
if mylar.DESTINATION_DIR == "":
logger.error(
u"There is no general directory specified - please specify in Config/Post-Processing."
)
return
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
comlocation = comlocation.replace(' ', mylar.REPLACE_CHAR)
#if it doesn't exist - create it (otherwise will bugger up later on)
if os.path.isdir(str(comlocation)):
logger.info(u"Directory (" + str(comlocation) +
") already exists! Continuing...")
else:
#print ("Directory doesn't exist!")
#try:
# os.makedirs(str(comlocation))
# logger.info(u"Directory successfully created at: " + str(comlocation))
#except OSError:
# logger.error(u"Could not create comicdir : " + str(comlocation))
filechecker.validateAndCreateDirectory(comlocation, True)
comicIssues = gcdinfo['totalissues']
#let's download the image...
if os.path.exists(mylar.CACHE_DIR): pass
else:
#let's make the dir.
try:
os.makedirs(str(mylar.CACHE_DIR))
logger.info(u"Cache Directory successfully created at: " +
str(mylar.CACHE_DIR))
except OSError:
logger.error(u"Could not create cache dir : " +
str(mylar.CACHE_DIR))
coverfile = os.path.join(mylar.CACHE_DIR, str(gcomicid) + ".jpg")
#try:
urllib.urlretrieve(str(ComicImage), str(coverfile))
try:
with open(str(coverfile)) as f:
ComicImage = os.path.join('cache', str(gcomicid) + ".jpg")
#this is for Firefox when outside the LAN...it works, but I don't know how to implement it
#without breaking the normal flow for inside the LAN (above)
#ComicImage = "http://" + str(mylar.HTTP_HOST) + ":" + str(mylar.HTTP_PORT) + "/cache/" + str(comi$
logger.info(u"Sucessfully retrieved cover for " + ComicName)
#if the comic cover local is checked, save a cover.jpg to the series folder.
if mylar.COMIC_COVER_LOCAL:
comiclocal = os.path.join(str(comlocation) + "/cover.jpg")
shutil.copy(ComicImage, comiclocal)
except IOError as e:
logger.error(u"Unable to save cover locally at this time.")
#if comic['ComicVersion'].isdigit():
# comicVol = "v" + comic['ComicVersion']
#else:
# comicVol = None
controlValueDict = {"ComicID": gcomicid}
newValueDict = {
"ComicName": ComicName,
"ComicSortName": sortname,
"ComicYear": ComicYear,
"Total": comicIssues,
"ComicLocation": comlocation,
#"ComicVersion": comicVol,
"ComicImage": ComicImage,
#"ComicPublisher": comic['ComicPublisher'],
#"ComicPublished": comicPublished,
"DateAdded": helpers.today(),
"Status": "Loading"
}
myDB.upsert("comics", newValueDict, controlValueDict)
#comicsort here...
#run the re-sortorder here in order to properly display the page
if pullupd is None:
helpers.ComicSort(sequence='update')
logger.info(u"Sucessfully retrieved issue details for " + ComicName)
n = 0
iscnt = int(comicIssues)
issnum = []
issname = []
issdate = []
int_issnum = []
#let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :)
latestiss = "0"
latestdate = "0000-00-00"
#print ("total issues:" + str(iscnt))
#---removed NEW code here---
logger.info(u"Now adding/updating issues for " + ComicName)
bb = 0
while (bb <= iscnt):
#---NEW.code
try:
gcdval = gcdinfo['gcdchoice'][bb]
#print ("gcdval: " + str(gcdval))
except IndexError:
#account for gcd variation here
if gcdinfo['gcdvariation'] == 'gcd':
#print ("gcd-variation accounted for.")
issdate = '0000-00-00'
int_issnum = int(issis / 1000)
break
if 'nn' in str(gcdval['GCDIssue']):
#no number detected - GN, TP or the like
logger.warn(
u"Non Series detected (Graphic Novel, etc) - cannot proceed at this time."
)
updater.no_searchresults(comicid)
return
elif '.' in str(gcdval['GCDIssue']):
issst = str(gcdval['GCDIssue']).find('.')
issb4dec = str(gcdval['GCDIssue'])[:issst]
#if the length of decimal is only 1 digit, assume it's a tenth
decis = str(gcdval['GCDIssue'])[issst + 1:]
if len(decis) == 1:
decisval = int(decis) * 10
issaftdec = str(decisval)
if len(decis) == 2:
decisval = int(decis)
issaftdec = str(decisval)
if int(issaftdec) == 0: issaftdec = "00"
gcd_issue = issb4dec + "." + issaftdec
gcdis = (int(issb4dec) * 1000) + decisval
else:
gcdis = int(str(gcdval['GCDIssue'])) * 1000
gcd_issue = str(gcdval['GCDIssue'])
#get the latest issue / date using the date.
int_issnum = int(gcdis / 1000)
issdate = str(gcdval['GCDDate'])
issid = "G" + str(gcdval['IssueID'])
if gcdval['GCDDate'] > latestdate:
latestiss = str(gcd_issue)
latestdate = str(gcdval['GCDDate'])
#print("(" + str(bb) + ") IssueID: " + str(issid) + " IssueNo: " + str(gcd_issue) + " Date" + str(issdate) )
#---END.NEW.
# check if the issue already exists
iss_exists = myDB.action('SELECT * from issues WHERE IssueID=?',
[issid]).fetchone()
# Only change the status & add DateAdded if the issue is not already in the database
if iss_exists is None:
newValueDict['DateAdded'] = helpers.today()
#adjust for inconsistencies in GCD date format - some dates have ? which borks up things.
if "?" in str(issdate):
issdate = "0000-00-00"
controlValueDict = {"IssueID": issid}
newValueDict = {
"ComicID": gcomicid,
"ComicName": ComicName,
"Issue_Number": gcd_issue,
"IssueDate": issdate,
"Int_IssueNumber": int_issnum
}
#print ("issueid:" + str(controlValueDict))
#print ("values:" + str(newValueDict))
if mylar.AUTOWANT_ALL:
newValueDict['Status'] = "Wanted"
elif issdate > helpers.today() and mylar.AUTOWANT_UPCOMING:
newValueDict['Status'] = "Wanted"
else:
newValueDict['Status'] = "Skipped"
if iss_exists:
#print ("Existing status : " + str(iss_exists['Status']))
newValueDict['Status'] = iss_exists['Status']
myDB.upsert("issues", newValueDict, controlValueDict)
bb += 1
# logger.debug(u"Updating comic cache for " + ComicName)
# cache.getThumb(ComicID=issue['issueid'])
# logger.debug(u"Updating cache for: " + ComicName)
# cache.getThumb(ComicIDcomicid)
controlValueStat = {"ComicID": gcomicid}
newValueStat = {
"Status": "Active",
"LatestIssue": latestiss,
"LatestDate": latestdate,
"LastUpdated": helpers.now()
}
myDB.upsert("comics", newValueStat, controlValueStat)
if mylar.CVINFO:
if not os.path.exists(comlocation + "/cvinfo"):
with open(comlocation + "/cvinfo", "w") as text_file:
text_file.write("http://www.comicvine.com/volume/49-" +
str(comicid))
logger.info(u"Updating complete for: " + ComicName)
#move the files...if imported is not empty (meaning it's not from the mass importer.)
if imported is None or imported == 'None':
pass
else:
if mylar.IMP_MOVE:
logger.info("Mass import - Move files")
moveit.movefiles(gcomicid, comlocation, ogcname)
else:
logger.info(
"Mass import - Moving not Enabled. Setting Archived Status for import."
)
moveit.archivefiles(gcomicid, ogcname)
#check for existing files...
updater.forceRescan(gcomicid)
if pullupd is None:
# lets' check the pullist for anyting at this time as well since we're here.
if mylar.AUTOWANT_UPCOMING and 'Present' in ComicPublished:
logger.info(u"Checking this week's pullist for new issues of " +
ComicName)
updater.newpullcheck(comic['ComicName'], gcomicid)
#here we grab issues that have been marked as wanted above...
results = myDB.select(
"SELECT * FROM issues where ComicID=? AND Status='Wanted'",
[gcomicid])
if results:
logger.info(u"Attempting to grab wanted issues for : " + ComicName)
for result in results:
foundNZB = "none"
if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL
or mylar.NEWZNAB or mylar.NZBX) and (mylar.SAB_HOST):
foundNZB = search.searchforissue(result['IssueID'])
if foundNZB == "yes":
updater.foundsearch(result['ComicID'],
result['IssueID'])
else:
logger.info(u"No issues marked as wanted for " + ComicName)
logger.info(u"Finished grabbing what I could.")
def Process(self):
module = '[FAILED-DOWNLOAD]'
myDB = db.DBConnection()
if self.nzb_name and self.nzb_folder:
self._log('Failed download has been detected: ' + self.nzb_name +
' in ' + self.nzb_folder)
#since this has already been passed through the search module, which holds the IssueID in the nzblog,
#let's find the matching nzbname and pass it the IssueID in order to mark it as Failed and then return
#to the search module and continue trucking along.
nzbname = self.nzb_name
#remove extensions from nzb_name if they somehow got through (Experimental most likely)
extensions = ('.cbr', '.cbz')
if nzbname.lower().endswith(extensions):
fd, ext = os.path.splitext(nzbname)
self._log("Removed extension from nzb: " + ext)
nzbname = re.sub(str(ext), '', str(nzbname))
#replace spaces
nzbname = re.sub(' ', '.', str(nzbname))
nzbname = re.sub('[\,\:\?\'\(\)]', '', str(nzbname))
nzbname = re.sub('[\&]', 'and', str(nzbname))
nzbname = re.sub('_', '.', str(nzbname))
logger.fdebug(module + ' After conversions, nzbname is : ' +
str(nzbname))
self._log("nzbname: " + str(nzbname))
nzbiss = myDB.selectone("SELECT * from nzblog WHERE nzbname=?",
[nzbname]).fetchone()
if nzbiss is None:
self._log(
"Failure - could not initially locate nzbfile in my database to rename."
)
logger.fdebug(module +
' Failure - could not locate nzbfile initially')
# if failed on spaces, change it all to decimals and try again.
nzbname = re.sub('_', '.', str(nzbname))
self._log("trying again with this nzbname: " + str(nzbname))
logger.fdebug(
module +
' Trying to locate nzbfile again with nzbname of : ' +
str(nzbname))
nzbiss = myDB.selectone("SELECT * from nzblog WHERE nzbname=?",
[nzbname]).fetchone()
if nzbiss is None:
logger.error(
module +
' Unable to locate downloaded file to rename. PostProcessing aborted.'
)
self._log(
'Unable to locate downloaded file to rename. PostProcessing aborted.'
)
self.valreturn.append({
"self.log": self.log,
"mode": 'stop'
})
return self.queue.put(self.valreturn)
else:
self._log("I corrected and found the nzb as : " +
str(nzbname))
logger.fdebug(module +
' Auto-corrected and found the nzb as : ' +
str(nzbname))
issueid = nzbiss['IssueID']
else:
issueid = nzbiss['IssueID']
logger.fdebug(module + ' Issueid: ' + str(issueid))
sarc = nzbiss['SARC']
#use issueid to get publisher, series, year, issue number
else:
issueid = self.issueid
nzbiss = myDB.selectone("SELECT * from nzblog WHERE IssueID=?",
[issueid]).fetchone()
if nzbiss is None:
logger.info(
module +
' Cannot locate corresponding record in download history. This will be implemented soon.'
)
self.valreturn.append({"self.log": self.log, "mode": 'stop'})
return self.queue.put(self.valreturn)
nzbname = nzbiss['NZBName']
# find the provider.
self.prov = nzbiss['PROVIDER']
logger.info(module + ' Provider: ' + self.prov)
# grab the id.
self.id = nzbiss['ID']
logger.info(module + ' ID: ' + self.id)
annchk = "no"
if 'annual' in nzbname.lower():
logger.info(module + ' Annual detected.')
annchk = "yes"
issuenzb = myDB.selectone(
"SELECT * from annuals WHERE IssueID=? AND ComicName NOT NULL",
[issueid]).fetchone()
else:
issuenzb = myDB.selectone(
"SELECT * from issues WHERE IssueID=? AND ComicName NOT NULL",
[issueid]).fetchone()
if issuenzb is not None:
logger.info(module + ' issuenzb found.')
if helpers.is_number(issueid):
sandwich = int(issuenzb['IssueID'])
else:
logger.info(module + ' issuenzb not found.')
#if it's non-numeric, it contains a 'G' at the beginning indicating it's a multi-volume
#using GCD data. Set sandwich to 1 so it will bypass and continue post-processing.
if 'S' in issueid:
sandwich = issueid
elif 'G' in issueid or '-' in issueid:
sandwich = 1
if helpers.is_number(sandwich):
if sandwich < 900000:
# if sandwich is less than 900000 it's a normal watchlist download. Bypass.
pass
else:
logger.info(
'Failed download handling for story-arcs and one-off\'s are not supported yet. Be patient!'
)
self._log(
' Unable to locate downloaded file to rename. PostProcessing aborted.'
)
self.valreturn.append({"self.log": self.log, "mode": 'stop'})
return self.queue.put(self.valreturn)
comicid = issuenzb['ComicID']
issuenumOG = issuenzb['Issue_Number']
logger.info(module + ' Successfully detected as : ' +
issuenzb['ComicName'] + ' issue: ' +
str(issuenzb['Issue_Number']) +
' that was downloaded using ' + self.prov)
self._log('Successfully detected as : ' + issuenzb['ComicName'] +
' issue: ' + str(issuenzb['Issue_Number']) +
' downloaded using ' + self.prov)
logger.info(module + ' Marking as a Failed Download.')
self._log('Marking as a Failed Download.')
ctrlVal = {"IssueID": issueid}
Vals = {"Status": 'Failed'}
myDB.upsert("issues", Vals, ctrlVal)
ctrlVal = {"ID": self.id, "Provider": self.prov, "NZBName": nzbname}
Vals = {
"Status": 'Failed',
"ComicName": issuenzb['ComicName'],
"Issue_Number": issuenzb['Issue_Number'],
"IssueID": issueid,
"ComicID": comicid,
"DateFailed": helpers.now()
}
myDB.upsert("failed", Vals, ctrlVal)
logger.info(module + ' Successfully marked as Failed.')
self._log('Successfully marked as Failed.')
if mylar.CONFIG.FAILED_AUTO:
logger.info(
module +
' Sending back to search to see if we can find something that will not fail.'
)
self._log(
'Sending back to search to see if we can find something better that will not fail.'
)
self.valreturn.append({
"self.log": self.log,
"mode": 'retry',
"issueid": issueid,
"comicid": comicid,
"comicname": issuenzb['ComicName'],
"issuenumber": issuenzb['Issue_Number'],
"annchk": annchk
})
return self.queue.put(self.valreturn)
else:
logger.info(
module +
' Stopping search here as automatic handling of failed downloads is not enabled *hint*'
)
self._log(
'Stopping search here as automatic handling of failed downloads is not enabled *hint*'
)
self.valreturn.append({"self.log": self.log, "mode": 'stop'})
return self.queue.put(self.valreturn)
def run(self):
with rss_lock:
logger.info('RSS Feed Check was last run at : ' +
str(mylar.RSS_LASTRUN))
firstrun = "no"
#check the last run of rss to make sure it's not hammering.
if mylar.RSS_LASTRUN is None or mylar.RSS_LASTRUN == '' or mylar.RSS_LASTRUN == '0' or self.forcerss == True:
logger.info('RSS Feed Check First Ever Run.')
firstrun = "yes"
mins = 0
else:
c_obj_date = datetime.datetime.strptime(
mylar.RSS_LASTRUN, "%Y-%m-%d %H:%M:%S")
n_date = datetime.datetime.now()
absdiff = abs(n_date - c_obj_date)
mins = (absdiff.days * 24 * 60 * 60 +
absdiff.seconds) / 60.0 #3600 is for hours.
if firstrun == "no" and mins < int(mylar.RSS_CHECKINTERVAL):
logger.fdebug(
'RSS Check has taken place less than the threshold - not initiating at this time.'
)
return
mylar.RSS_LASTRUN = helpers.now()
logger.fdebug('Updating RSS Run time to : ' +
str(mylar.RSS_LASTRUN))
mylar.config_write()
#function for looping through nzbs/torrent feeds
if mylar.ENABLE_TORRENT_SEARCH:
logger.info('[RSS] Initiating Torrent RSS Check.')
if mylar.ENABLE_KAT:
logger.info(
'[RSS] Initiating Torrent RSS Feed Check on KAT.')
rsscheck.torrents(pickfeed='3')
rsscheck.torrents(pickfeed='6')
if mylar.ENABLE_32P:
logger.info(
'[RSS] Initiating Torrent RSS Feed Check on 32P.')
if mylar.MODE_32P == 0:
logger.fdebug(
'[RSS] 32P mode set to Legacy mode. Monitoring New Releases feed only.'
)
if any([
mylar.PASSKEY_32P is None,
mylar.PASSKEY_32P == '',
mylar.RSSFEED_32P is None,
mylar.RSSFEED_32P == ''
]):
logger.error(
'[RSS] Unable to validate information from provided RSS Feed. Verify that the feed provided is a current one.'
)
else:
rsscheck.torrents(pickfeed='1',
feedinfo=mylar.KEYS_32P)
else:
logger.fdebug(
'[RSS] 32P mode set to Auth mode. Monitoring all personal notification feeds & New Releases feed'
)
if any([
mylar.USERNAME_32P is None,
mylar.USERNAME_32P == '',
mylar.PASSWORD_32P is None
]):
logger.error(
'[RSS] Unable to sign-on to 32P to validate settings. Please enter/check your username password in the configuration.'
)
else:
if mylar.KEYS_32P is None:
feed32p = auth32p.info32p()
feedinfo = feed32p.authenticate()
else:
feedinfo = mylar.FEEDINFO_32P
if feedinfo is None or len(feedinfo) == 0:
logger.error(
'[RSS] Unable to retrieve any information from 32P for RSS Feeds. Skipping for now.'
)
else:
rsscheck.torrents(pickfeed='1',
feedinfo=feedinfo[0])
x = 0
#assign personal feeds for 32p > +8
for fi in feedinfo:
x += 1
pfeed_32p = str(7 + x)
rsscheck.torrents(pickfeed=pfeed_32p,
feedinfo=fi)
logger.info('[RSS] Initiating RSS Feed Check for NZB Providers.')
rsscheck.nzbs(forcerss=self.forcerss)
logger.info('[RSS] RSS Feed Check/Update Complete')
logger.info('[RSS] Watchlist Check for new Releases')
mylar.search.searchforissue(rsscheck='yes')
logger.info('[RSS] Watchlist Check complete.')
if self.forcerss:
logger.info('[RSS] Successfully ran a forced RSS Check.')
return
def addComictoDB(comicid, mismatch=None):
# Putting this here to get around the circular import. Will try to use this to update images at later date.
from mylar import cache
myDB = db.DBConnection()
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ComicID": comicid}
dbcomic = myDB.action("SELECT * FROM comics WHERE ComicID=?", [comicid]).fetchone()
if dbcomic is None:
newValueDict = {"ComicName": "Comic ID: %s" % (comicid), "Status": "Loading"}
comlocation = None
else:
newValueDict = {"Status": "Loading"}
comlocation = dbcomic["ComicLocation"]
myDB.upsert("comics", newValueDict, controlValueDict)
# we need to lookup the info for the requested ComicID in full now
comic = cv.getComic(comicid, "comic")
# comic = myDB.action('SELECT * FROM comics WHERE ComicID=?', [comicid]).fetchone()
if not comic:
logger.warn("Error fetching comic. ID for : " + comicid)
if dbcomic is None:
newValueDict = {"ComicName": "Fetch failed, try refreshing. (%s)" % (comicid), "Status": "Active"}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("comics", newValueDict, controlValueDict)
return
if comic["ComicName"].startswith("The "):
sortname = comic["ComicName"][4:]
else:
sortname = comic["ComicName"]
logger.info(u"Now adding/updating: " + comic["ComicName"])
# --Now that we know ComicName, let's try some scraping
# --Start
# gcd will return issue details (most importantly publishing date)
if mismatch == "no" or mismatch is None:
gcdinfo = parseit.GCDScraper(comic["ComicName"], comic["ComicYear"], comic["ComicIssues"], comicid)
mismatch_com = "no"
if gcdinfo == "No Match":
updater.no_searchresults(comicid)
nomatch = "true"
logger.info(
u"There was an error when trying to add " + comic["ComicName"] + " (" + comic["ComicYear"] + ")"
)
return nomatch
else:
mismatch_com = "yes"
# print ("gcdinfo:" + str(gcdinfo))
elif mismatch == "yes":
CV_EXcomicid = myDB.action("SELECT * from exceptions WHERE ComicID=?", [comicid]).fetchone()
if CV_EXcomicid["variloop"] is None:
pass
else:
vari_loop = CV_EXcomicid["variloop"]
NewComicID = CV_EXcomicid["NewComicID"]
gcomicid = CV_EXcomicid["GComicID"]
resultURL = "/series/" + str(NewComicID) + "/"
# print ("variloop" + str(CV_EXcomicid['variloop']))
# if vari_loop == '99':
gcdinfo = parseit.GCDdetails(
comseries=None,
resultURL=resultURL,
vari_loop=0,
ComicID=comicid,
TotalIssues=0,
issvariation="no",
resultPublished=None,
)
logger.info(u"Sucessfully retrieved details for " + comic["ComicName"])
# print ("Series Published" + parseit.resultPublished)
# comic book location on machine
# setup default location here
if comlocation is None:
if ":" in comic["ComicName"] or "/" in comic["ComicName"] or "," in comic["ComicName"]:
comicdir = comic["ComicName"]
if ":" in comicdir:
comicdir = comicdir.replace(":", "")
if "/" in comicdir:
comicdir = comicdir.replace("/", "-")
if "," in comicdir:
comicdir = comicdir.replace(",", "")
else:
comicdir = comic["ComicName"]
series = comicdir
publisher = comic["ComicPublisher"]
year = comic["ComicYear"]
# do work to generate folder path
values = {"$Series": series, "$Publisher": publisher, "$Year": year}
# print mylar.FOLDER_FORMAT
# print 'working dir:'
# print helpers.replace_all(mylar.FOLDER_FORMAT, values)
if mylar.FOLDER_FORMAT == "":
comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + comic["ComicYear"] + ")"
else:
comlocation = mylar.DESTINATION_DIR + "/" + helpers.replace_all(mylar.FOLDER_FORMAT, values)
# comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + comic['ComicYear'] + ")"
if mylar.DESTINATION_DIR == "":
logger.error(u"There is no general directory specified - please specify in Config/Post-Processing.")
return
if mylar.REPLACE_SPACES:
# mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
comlocation = comlocation.replace(" ", mylar.REPLACE_CHAR)
# if it doesn't exist - create it (otherwise will bugger up later on)
if os.path.isdir(str(comlocation)):
logger.info(u"Directory (" + str(comlocation) + ") already exists! Continuing...")
else:
# print ("Directory doesn't exist!")
try:
os.makedirs(str(comlocation))
logger.info(u"Directory successfully created at: " + str(comlocation))
except OSError:
logger.error(u"Could not create comicdir : " + str(comlocation))
# try to account for CV not updating new issues as fast as GCD
# seems CV doesn't update total counts
# comicIssues = gcdinfo['totalissues']
if gcdinfo["gcdvariation"] == "cv":
comicIssues = str(int(comic["ComicIssues"]) + 1)
else:
comicIssues = comic["ComicIssues"]
# let's download the image...
if os.path.exists(mylar.CACHE_DIR):
pass
else:
# let's make the dir.
try:
os.makedirs(str(mylar.CACHE_DIR))
logger.info(u"Cache Directory successfully created at: " + str(mylar.CACHE_DIR))
except OSError:
logger.error("Could not create cache dir. Check permissions of cache dir: " + str(mylar.CACHE_DIR))
coverfile = mylar.CACHE_DIR + "/" + str(comicid) + ".jpg"
# try:
urllib.urlretrieve(str(comic["ComicImage"]), str(coverfile))
try:
with open(str(coverfile)) as f:
ComicImage = "cache/" + str(comicid) + ".jpg"
logger.info(u"Sucessfully retrieved cover for " + str(comic["ComicName"]))
except IOError as e:
logger.error(u"Unable to save cover locally at this time.")
controlValueDict = {"ComicID": comicid}
newValueDict = {
"ComicName": comic["ComicName"],
"ComicSortName": sortname,
"ComicYear": comic["ComicYear"],
"ComicImage": ComicImage,
"Total": comicIssues,
"ComicLocation": comlocation,
"ComicPublisher": comic["ComicPublisher"],
"ComicPublished": gcdinfo["resultPublished"],
"DateAdded": helpers.today(),
"Status": "Loading",
}
myDB.upsert("comics", newValueDict, controlValueDict)
issued = cv.getComic(comicid, "issue")
logger.info(u"Sucessfully retrieved issue details for " + comic["ComicName"])
n = 0
iscnt = int(comicIssues)
issid = []
issnum = []
issname = []
issdate = []
int_issnum = []
# let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :)
latestiss = "0"
latestdate = "0000-00-00"
# print ("total issues:" + str(iscnt))
# ---removed NEW code here---
logger.info(u"Now adding/updating issues for " + comic["ComicName"])
# file check to see if issue exists
logger.info(u"Checking directory for existing issues.")
# fc = filechecker.listFiles(dir=comlocation, watchcomic=comic['ComicName'])
# havefiles = 0
# fccnt = int(fc['comiccount'])
# logger.info(u"Found " + str(fccnt) + "/" + str(iscnt) + " issues of " + comic['ComicName'] + "...verifying")
# fcnew = []
while n <= iscnt:
# ---NEW.code
try:
firstval = issued["issuechoice"][n]
except IndexError:
break
cleanname = helpers.cleanName(firstval["Issue_Name"])
issid = str(firstval["Issue_ID"])
issnum = str(firstval["Issue_Number"])
issname = cleanname
if "." in str(issnum):
issn_st = str(issnum).find(".")
issn_b4dec = str(issnum)[:issn_st]
# if the length of decimal is only 1 digit, assume it's a tenth
dec_is = str(issnum)[issn_st + 1 :]
if len(dec_is) == 1:
dec_nisval = int(dec_is) * 10
iss_naftdec = str(dec_nisval)
if len(dec_is) == 2:
dec_nisval = int(dec_is)
iss_naftdec = str(dec_nisval)
iss_issue = issn_b4dec + "." + iss_naftdec
issis = (int(issn_b4dec) * 1000) + dec_nisval
else:
issis = int(issnum) * 1000
bb = 0
while bb <= iscnt:
try:
gcdval = gcdinfo["gcdchoice"][bb]
except IndexError:
# account for gcd variation here
if gcdinfo["gcdvariation"] == "gcd":
# print ("gcd-variation accounted for.")
issdate = "0000-00-00"
int_issnum = int(issis / 1000)
break
if "nn" in str(gcdval["GCDIssue"]):
# no number detected - GN, TP or the like
logger.warn(u"Non Series detected (Graphic Novel, etc) - cannot proceed at this time.")
updater.no_searchresults(comicid)
return
elif "." in str(gcdval["GCDIssue"]):
# print ("g-issue:" + str(gcdval['GCDIssue']))
issst = str(gcdval["GCDIssue"]).find(".")
# print ("issst:" + str(issst))
issb4dec = str(gcdval["GCDIssue"])[:issst]
# print ("issb4dec:" + str(issb4dec))
# if the length of decimal is only 1 digit, assume it's a tenth
decis = str(gcdval["GCDIssue"])[issst + 1 :]
# print ("decis:" + str(decis))
if len(decis) == 1:
decisval = int(decis) * 10
issaftdec = str(decisval)
if len(decis) == 2:
decisval = int(decis)
issaftdec = str(decisval)
gcd_issue = issb4dec + "." + issaftdec
# print ("gcd_issue:" + str(gcd_issue))
gcdis = (int(issb4dec) * 1000) + decisval
else:
gcdis = int(str(gcdval["GCDIssue"])) * 1000
if gcdis == issis:
issdate = str(gcdval["GCDDate"])
int_issnum = int(gcdis / 1000)
# get the latest issue / date using the date.
if gcdval["GCDDate"] > latestdate:
latestiss = str(issnum)
latestdate = str(gcdval["GCDDate"])
break
# bb = iscnt
bb += 1
# print("(" + str(n) + ") IssueID: " + str(issid) + " IssueNo: " + str(issnum) + " Date" + str(issdate))
# ---END.NEW.
# check if the issue already exists
iss_exists = myDB.action("SELECT * from issues WHERE IssueID=?", [issid]).fetchone()
# Only change the status & add DateAdded if the issue is already in the database
if iss_exists is None:
newValueDict["DateAdded"] = helpers.today()
controlValueDict = {"IssueID": issid}
newValueDict = {
"ComicID": comicid,
"ComicName": comic["ComicName"],
"IssueName": issname,
"Issue_Number": issnum,
"IssueDate": issdate,
"Int_IssueNumber": int_issnum,
}
if mylar.AUTOWANT_ALL:
newValueDict["Status"] = "Wanted"
# elif release_dict['releasedate'] > helpers.today() and mylar.AUTOWANT_UPCOMING:
# newValueDict['Status'] = "Wanted"
else:
newValueDict["Status"] = "Skipped"
if iss_exists:
# print ("Existing status : " + str(iss_exists['Status']))
newValueDict["Status"] = iss_exists["Status"]
myDB.upsert("issues", newValueDict, controlValueDict)
n += 1
# logger.debug(u"Updating comic cache for " + comic['ComicName'])
# cache.getThumb(ComicID=issue['issueid'])
# logger.debug(u"Updating cache for: " + comic['ComicName'])
# cache.getThumb(ComicIDcomicid)
# check for existing files...
updater.forceRescan(comicid)
controlValueStat = {"ComicID": comicid}
newValueStat = {
"Status": "Active",
"LatestIssue": latestiss,
"LatestDate": latestdate,
"LastUpdated": helpers.now(),
}
myDB.upsert("comics", newValueStat, controlValueStat)
logger.info(u"Updating complete for: " + comic["ComicName"])
# lets' check the pullist for anyting at this time as well since we're here.
if mylar.AUTOWANT_UPCOMING:
logger.info(u"Checking this week's pullist for new issues of " + str(comic["ComicName"]))
updater.newpullcheck()
# here we grab issues that have been marked as wanted above...
results = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [comicid])
if results:
logger.info(u"Attempting to grab wanted issues for : " + comic["ComicName"])
for result in results:
foundNZB = "none"
if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL) and (mylar.SAB_HOST):
foundNZB = search.searchforissue(result["IssueID"])
if foundNZB == "yes":
updater.foundsearch(result["ComicID"], result["IssueID"])
else:
logger.info(u"No issues marked as wanted for " + comic["ComicName"])
logger.info(u"Finished grabbing what I could.")
def pullitcheck(comic1off_name=None, comic1off_id=None, forcecheck=None):
logger.info(u"Checking the Weekly Releases list for comics I'm watching...")
myDB = db.DBConnection()
not_t = ["TP", "NA", "HC", "PI"]
not_c = ["PTG", "COMBO PACK", "(PP #"]
lines = []
unlines = []
llen = []
ccname = []
pubdate = []
w = 0
tot = 0
chkout = []
watchfnd = []
watchfndiss = []
watchfndextra = []
# print ("----------WATCHLIST--------")
a_list = []
b_list = []
comicid = []
mylardb = os.path.join(mylar.DATA_DIR, "mylar.db")
con = sqlite3.connect(str(mylardb))
with con:
cur = con.cursor()
# if it's a one-off check (during an add series), load the comicname here and ignore below.
if comic1off_name:
lines.append(comic1off_name.strip())
unlines.append(comic1off_name.strip())
comicid.append(comic1off_id)
w = 1
else:
# let's read in the comic.watchlist from the db here
cur.execute("SELECT ComicID, ComicName, ComicYear, ComicPublisher, ComicPublished from comics")
while True:
watchd = cur.fetchone()
# print ("watchd: " + str(watchd))
if watchd is None:
break
if "Present" in watchd[4] or (helpers.now()[:4] in watchd[4]):
# let's not even bother with comics that are in the Present.
a_list.append(watchd[1])
b_list.append(watchd[2])
comicid.append(watchd[0])
pubdate.append(watchd[4])
# print ( "Comic:" + str(a_list[w]) + " Year: " + str(b_list[w]) )
# if "WOLVERINE AND THE X-MEN" in str(a_list[w]): a_list[w] = "WOLVERINE AND X-MEN"
lines.append(a_list[w].strip())
unlines.append(a_list[w].strip())
llen.append(a_list[w].splitlines())
ccname.append(a_list[w].strip())
tmpwords = a_list[w].split(None)
ltmpwords = len(tmpwords)
ltmp = 1
w += 1
cnt = int(w - 1)
cntback = int(w - 1)
kp = []
ki = []
kc = []
otot = 0
logger.fdebug("You are watching for: " + str(w) + " comics")
# print ("----------THIS WEEK'S PUBLISHED COMICS------------")
if w > 0:
while cnt > -1:
lines[cnt] = lines[cnt].upper()
# llen[cnt] = str(llen[cnt])
logger.fdebug("looking for : " + str(lines[cnt]))
sqlsearch = re.sub("[\_\#\,\/\:\;\.\-\!\$\%\&'\?\@]", " ", lines[cnt])
sqlsearch = re.sub(r"\s", "%", sqlsearch)
if "THE" in sqlsearch:
sqlsearch = re.sub("THE", "", sqlsearch)
if "+" in sqlsearch:
sqlsearch = re.sub("\+", "%PLUS%", sqlsearch)
logger.fdebug("searchsql: " + str(sqlsearch))
weekly = myDB.select(
"SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)", [sqlsearch]
)
# cur.execute('SELECT PUBLISHER, ISSUE, COMIC, EXTRA, SHIPDATE FROM weekly WHERE COMIC LIKE (?)', [lines[cnt]])
for week in weekly:
if week == None:
break
for nono in not_t:
if nono in week["PUBLISHER"]:
logger.fdebug("nono present")
break
if nono in week["ISSUE"]:
# logger.fdebug("graphic novel/tradeback detected..ignoring.")
break
for nothere in not_c:
if nothere in week["EXTRA"]:
# logger.fdebug("nothere present")
break
else:
comicnm = week["COMIC"]
# here's the tricky part, ie. BATMAN will match on
# every batman comic, not exact
# logger.fdebug("comparing" + str(comicnm) + "..to.." + str(unlines[cnt]).upper())
# -NEW-
# strip out all special characters and compare
watchcomic = re.sub("[\_\#\,\/\:\;\.\-\!\$\%\&'\?\@]", "", unlines[cnt])
comicnm = re.sub("[\_\#\,\/\:\;\.\-\!\$\%\&'\?\@]", "", comicnm)
watchcomic = re.sub(r"\s", "", watchcomic)
comicnm = re.sub(r"\s", "", comicnm)
# logger.fdebug("Revised_Watch: " + watchcomic)
# logger.fdebug("ComicNM: " + comicnm)
if "THE" in watchcomic.upper():
modwatchcomic = re.sub("THE", "", watchcomic.upper())
modcomicnm = re.sub("THE", "", comicnm)
else:
modwatchcomic = watchcomic
modcomicnm = comicnm
# thnx to A+X for this...
if "+" in watchcomic:
logger.fdebug("+ detected...adjusting.")
logger.fdebug("comicnm:" + comicnm)
logger.fdebug("watchcomic:" + watchcomic)
modwatchcomic = re.sub("\+", "PLUS", modwatchcomic)
logger.fdebug("modcomicnm:" + modcomicnm)
logger.fdebug("modwatchcomic:" + modwatchcomic)
if comicnm == watchcomic.upper() or modcomicnm == modwatchcomic.upper():
logger.fdebug("matched on:" + str(comicnm) + "..." + str(watchcomic).upper())
pass
elif "ANNUAL" in week["EXTRA"]:
pass
# print ( row[3] + " matched on ANNUAL")
else:
break
if ("NA" not in week["ISSUE"]) and ("HC" not in week["ISSUE"]):
if (
("COMBO PACK" not in week["EXTRA"])
and ("2ND PTG" not in week["EXTRA"])
and ("3RD PTG" not in week["EXTRA"])
):
otot += 1
dontadd = "no"
if dontadd == "no":
# print (row[0], row[1], row[2])
tot += 1
# kp.append(row[0])
# ki.append(row[1])
# kc.append(comicnm)
if "ANNUAL" in week["EXTRA"]:
watchfndextra.append("annual")
else:
watchfndextra.append("none")
watchfnd.append(comicnm)
watchfndiss.append(week["ISSUE"])
ComicID = comicid[cnt]
if not mylar.CV_ONLY:
ComicIssue = str(watchfndiss[tot - 1] + ".00")
else:
ComicIssue = str(watchfndiss[tot - 1])
ComicDate = str(week["SHIPDATE"])
ComicName = str(unlines[cnt])
logger.fdebug(
"Watchlist hit for : "
+ ComicName
+ " ISSUE: "
+ str(watchfndiss[tot - 1])
)
# here we add to comics.latest
updater.latest_update(
ComicID=ComicID, LatestIssue=ComicIssue, LatestDate=ComicDate
)
# here we add to upcoming table...
statusupdate = updater.upcoming_update(
ComicID=ComicID,
ComicName=ComicName,
IssueNumber=ComicIssue,
IssueDate=ComicDate,
forcecheck=forcecheck,
)
# here we update status of weekly table...
updater.weekly_update(
ComicName=week["COMIC"], IssueNumber=ComicIssue, CStatus=statusupdate
)
break
break
break
cnt -= 1
# print ("-------------------------")
logger.fdebug("There are " + str(otot) + " comics this week to get!")
# print ("However I've already grabbed " + str(btotal) )
# print ("I need to get " + str(tot) + " comic(s)!" )
logger.info(u"Finished checking for comics on my watchlist.")
# con.close()
return
