def forceRescan(ComicID): myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName']) iscnt = rescan['Total'] havefiles = 0 mylar.AUTOWANT_ALL = 0 fccnt = int(fc['comiccount']) issnum = 1 fcnew = [] n = 0 while (n < iscnt): fn = 0 haveissue = "no" print ("on issue " + str(int(n+1)) + " of " + str(iscnt) + " issues") print ("checking issue: " + str(issnum)) # stupid way to do this, but check each issue against file-list in fc. while (fn < fccnt): tmpfc = fc['comiclist'][fn] print (str(issnum) + "against ... " + str(tmpfc['ComicFilename'])) temploc = tmpfc['ComicFilename'].replace('_', ' ') fcnew = shlex.split(str(temploc)) fcn = len(fcnew) som = 0 # this loop searches each word in the filename for a match. while (som < fcn): print (fcnew[som]) #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' if ".cbr" in fcnew[som]: fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som]: fcnew[som] = fcnew[som].replace(".cbz", "") if fcnew[som].isdigit(): print ("digit detected") fcdigit = fcnew[som].lstrip('0') print ( "filename:" + str(int(fcnew[som])) + " - issue: " + str(issnum) ) #fcdigit = fcnew[som].lstrip('0') + ".00" if int(fcdigit) == int(issnum): print ("matched") print ("We have this issue - " + str(issnum) + " at " + tmpfc['ComicFilename'] ) havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) break print ("failed word match on:" + str(fcnew[som]) + "..continuing next word") som+=1 print (str(temploc) + " doesn't match anything...moving to next file.") fn+=1 issnum+=1 print ("you have " + str(havefiles) + " comics!") return
def foundsearch(ComicID, IssueID): myDB = db.DBConnection() #print ("Updater-ComicID: " + str(ComicID)) #print ("Updater-IssueID: " + str(IssueID)) comic = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() issue = myDB.action('SELECT * FROM issues WHERE IssueID=?', [IssueID]).fetchone() #print ("comic location: " + comic['ComicLocation']) #this is too soon - file hasn't downloaded even yet. #fixed and addressed in search.py and follow-thru here! #check sab history for completion here :) CYear = issue['IssueDate'][:4] print ("year:" + str(CYear)) #slog = myDB.action('SELECT * FROM sablog WHERE ComicName=? AND ComicYEAR=?', [issue['ComicName'], str(CYear)]).fetchone() #this checks the active queue for downloading/non-existant jobs #--end queue check #this checks history for completed jobs... #--- #-- end history check fc = filechecker.listFiles(comic['ComicLocation'], comic['ComicName']) HaveDict = {"ComicID": ComicID} newHave = { "Have": fc['comiccount'] } myDB.upsert("comics", newHave, HaveDict) #--- issue = myDB.action('SELECT * FROM issues WHERE IssueID=? AND ComicID=?', [IssueID, ComicID]).fetchone() #print ("updating status to snatched") controlValueDict = {"IssueID": IssueID} newValueDict = {"Status": "Snatched"} #print ("updating snatched db.") myDB.upsert("issues", newValueDict, controlValueDict) snatchedupdate = {"IssueID": IssueID} newsnatchValues = {"ComicName": comic['ComicName'], "ComicID": ComicID, "Issue_Number": issue['Issue_Number'], "DateAdded": helpers.now(), "Status": "Snatched" } myDB.upsert("snatched", newsnatchValues, snatchedupdate) #we need to update sablog now to mark the nzo_id row as being completed and not used again. #this becomes an issue with files downloaded x2 or same name... #print ("finished updating snatched db.") logger.info(u"Updating now complete for " + str(comic['ComicName']) + " issue: " + str(issue['Issue_Number'])) return
def forceRescan(ComicID): myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.action("SELECT * FROM comics WHERE ComicID=?", [ComicID]).fetchone() logger.info( u"Now checking files for " + str(rescan["ComicName"]) + " (" + str(rescan["ComicYear"]) + ") in " + str(rescan["ComicLocation"]) ) fc = filechecker.listFiles( dir=rescan["ComicLocation"], watchcomic=rescan["ComicName"], AlternateSearch=rescan["AlternateSearch"] ) iscnt = rescan["Total"] havefiles = 0 fccnt = int(fc["comiccount"]) issnum = 1 fcnew = [] fn = 0 issuedupechk = [] reissues = myDB.action("SELECT * FROM issues WHERE ComicID=?", [ComicID]).fetchall() # if filechecker returns 0 files (it doesn't find any), but some issues have a status of 'Archived' # the loop below won't work...let's adjust :) arcissues = myDB.select("SELECT * FROM issues WHERE ComicID=? and Status='Archived'", [ComicID]) if len(arcissues) > 0: havefiles = len(arcissues) logger.fdebug("Adjusting have total because of this many archive files:" + str(len(arcissues))) while fn < fccnt: haveissue = "no" issuedupe = "no" try: tmpfc = fc["comiclist"][fn] except IndexError: break temploc = tmpfc["ComicFilename"].replace("_", " ") temploc = re.sub("[\#']", "", temploc) # logger.fdebug("temploc: " + str(temploc)) if "annual" not in temploc: fcnew = shlex.split(str(temploc)) fcn = len(fcnew) n = 0 while n <= iscnt: som = 0 try: reiss = reissues[n] except IndexError: break int_iss = helpers.decimal_issue(reiss["Issue_Number"]) issyear = reiss["IssueDate"][:4] old_status = reiss["Status"] # logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status)) # if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die. # can't distinguish the 5(c2c) to tell it's the issue #... while som < fcn: # counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' # logger.fdebug("checking word - " + str(fcnew[som])) if ".cbr" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbz", "") if "(c2c)" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace("(c2c)", " ") get_issue = shlex.split(str(fcnew[som])) if fcnew[som] != " ": fcnew[som] = get_issue[0] if "." in fcnew[som]: # logger.fdebug("decimal detected...adjusting.") try: i = float(fcnew[som]) except ValueError, TypeError: # not numeric fcnew[som] = fcnew[som].replace(".", "") # logger.fdebug("NOT NUMERIC - new word: " + str(fcnew[som])) else: # numeric pass if fcnew[som].isdigit(): # this won't match on decimal issues - need to fix. # logger.fdebug("digit detected") if int(fcnew[som]) > 0: # fcdigit = fcnew[som].lstrip('0') # fcdigit = str(int(fcnew[som])) fcdigit = int(fcnew[som]) * 1000 else: # fcdigit = "0" fcdigit = 0 elif "." in fcnew[som]: # this will match on decimal issues IssueChk = fcnew[som] # logger.fdebug("decimal detected...analyzing if issue") isschk_find = IssueChk.find(".") isschk_b4dec = IssueChk[:isschk_find] isschk_decval = IssueChk[isschk_find + 1 :] if isschk_b4dec.isdigit(): # logger.fdebug("digit detected prior to decimal.") if isschk_decval.isdigit(): pass # logger.fdebug("digit detected after decimal.") else: # logger.fdebug("not an issue - no digit detected after decimal") break else: # logger.fdebug("not an issue - no digit detected prior to decimal") break # logger.fdebug("IssueNumber: " + str(IssueChk)) # logger.fdebug("..before decimal: " + str(isschk_b4dec)) # logger.fdebug("...after decimal: " + str(isschk_decval)) # --let's make sure we don't wipe out decimal issues ;) if int(isschk_decval) == 0: iss = isschk_b4dec intdec = int(isschk_decval) else: if len(isschk_decval) == 1: iss = isschk_b4dec + "." + isschk_decval intdec = int(isschk_decval) * 10 else: iss = isschk_b4dec + "." + isschk_decval.rstrip("0") intdec = int(isschk_decval.rstrip("0")) * 10 fcdigit = (int(isschk_b4dec) * 1000) + intdec # logger.fdebug("b4dec: " + str(isschk_b4dec)) # logger.fdebug("decval: " + str(isschk_decval)) # logger.fdebug("intdec: " + str(intdec)) # logger.fdebug("let's compare with this issue value: " + str(fcdigit)) else: # it's a word, skip it. fcdigit = 1000000 # logger.fdebug("fcdigit: " + str(fcdigit)) # logger.fdebug("int_iss: " + str(int_iss)) if "." in str(int_iss): int_iss = helpers.decimal_issue(int_iss) # logger.fdebug("this is the int issue:" + str(int_iss)) if int(fcdigit) == int_iss: # if issyear in fcnew[som+1]: # print "matched on year:" + str(issyear) # issuedupechk here. if int(fcdigit) in issuedupechk: logger.fdebug( "duplicate issue detected - not counting this: " + str(tmpfc["ComicFilename"]) ) issuedupe = "yes" break logger.fdebug("matched...issue: " + str(rescan["ComicName"]) + " --- " + str(int_iss)) havefiles += 1 haveissue = "yes" isslocation = str(tmpfc["ComicFilename"]) issSize = str(tmpfc["ComicSize"]) logger.fdebug(".......filename: " + str(isslocation)) logger.fdebug(".......filesize: " + str(tmpfc["ComicSize"])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... issuedupechk.append(int(fcdigit)) break # else: # if the issue # matches, but there is no year present - still match. # determine a way to match on year if present, or no year (currently). som += 1 if haveissue == "yes": break n += 1 # we have the # of comics, now let's update the db. # even if we couldn't find the physical issue, check the status. # if Archived, increase the 'Have' count. if haveissue == "no" and issuedupe == "no": isslocation = "None" if old_status == "Skipped": if mylar.AUTOWANT_ALL: issStatus = "Wanted" else: issStatus = "Skipped" elif old_status == "Archived": havefiles += 1 issStatus = "Archived" elif old_status == "Downloaded": issStatus = "Archived" havefiles += 1 elif old_status == "Wanted": issStatus = "Wanted" else: issStatus = "Skipped" controlValueDict = {"IssueID": reiss["IssueID"]} newValueDict = {"Status": issStatus} elif haveissue == "yes": issStatus = "Downloaded" controlValueDict = {"IssueID": reiss["IssueID"]} newValueDict = {"Location": isslocation, "ComicSize": issSize, "Status": issStatus} myDB.upsert("issues", newValueDict, controlValueDict) fn += 1
def Process(self): self._log("nzb name: " + str(self.nzb_name)) self._log("nzb folder: " + str(self.nzb_folder)) logger.fdebug("nzb name: " + str(self.nzb_name)) logger.fdebug("nzb folder: " + str(self.nzb_folder)) if mylar.USE_SABNZBD==0: logger.fdebug("Not using SABnzbd") elif mylar.USE_SABNZBD != 0 and self.nzb_name == 'Manual Run': logger.fdebug('Not using SABnzbd : Manual Run') else: # if the SAB Directory option is enabled, let's use that folder name and append the jobname. if mylar.SAB_DIRECTORY is not None and mylar.SAB_DIRECTORY is not 'None' and len(mylar.SAB_DIRECTORY) > 4: self.nzb_folder = os.path.join(mylar.SAB_DIRECTORY, self.nzb_name).encode(mylar.SYS_ENCODING) logger.fdebug('SABnzbd Download folder option enabled. Directory set to : ' + self.nzb_folder) # -- start. not used. #query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals #http://localhost:8080/sabnzbd/api?mode=set_config§ion=misc&keyword=dirscan_speed&value=5 #querysab = str(mylar.SAB_HOST) + "/api?mode=get_config§ion=misc&output=xml&apikey=" + str(mylar.SAB_APIKEY) #logger.info("querysab_string:" + str(querysab)) #file = urllib2.urlopen(querysab) #data = file.read() #file.close() #dom = parseString(data) #try: # sabreps = dom.getElementsByTagName('replace_spaces')[0].firstChild.wholeText #except: # errorm = dom.getElementsByTagName('error')[0].firstChild.wholeText # logger.error(u"Error detected attempting to retrieve SAB data : " + errorm) # return #sabrepd = dom.getElementsByTagName('replace_dots')[0].firstChild.wholeText #logger.fdebug("SAB Replace Spaces: " + str(sabreps)) #logger.fdebug("SAB Replace Dots: " + str(sabrepd)) # -- end. not used. if mylar.USE_NZBGET==1: if self.nzb_name != 'Manual Run': logger.fdebug("Using NZBGET") logger.fdebug("NZB name as passed from NZBGet: " + self.nzb_name) # if the NZBGet Directory option is enabled, let's use that folder name and append the jobname. if self.nzb_name == 'Manual Run': logger.fdebug('Manual Run Post-Processing enabled.') elif mylar.NZBGET_DIRECTORY is not None and mylar.NZBGET_DIRECTORY is not 'None' and len(mylar.NZBGET_DIRECTORY) > 4: self.nzb_folder = os.path.join(mylar.NZBGET_DIRECTORY, self.nzb_name).encode(mylar.SYS_ENCODING) logger.fdebug('NZBGET Download folder option enabled. Directory set to : ' + self.nzb_folder) myDB = db.DBConnection() if self.nzb_name == 'Manual Run': logger.fdebug ("manual run initiated") #Manual postprocessing on a folder. #use the nzb_folder to determine every file #walk the dir, #once a series name and issue are matched, #write the series/issue/filename to a tuple #when all done, iterate over the tuple until completion... comicseries = myDB.select("SELECT * FROM comics") manual_list = [] if comicseries is None: logger.error(u"No Series in Watchlist - aborting Manual Post Processing. Maybe you should be running Import?") return else: ccnt=0 nm=0 watchvals = {} for cs in comicseries: watchvals = {"SeriesYear": cs['ComicYear'], "LatestDate": cs['LatestDate'], "ComicVersion": cs['ComicVersion'], "Publisher": cs['ComicPublisher'], "Total": cs['Total']} watchmatch = filechecker.listFiles(self.nzb_folder,cs['ComicName'],cs['ComicPublisher'],cs['AlternateSearch'], manual=watchvals) if watchmatch['comiccount'] == 0: # is None: nm+=1 continue else: fn = 0 fccnt = int(watchmatch['comiccount']) if len(watchmatch) == 1: continue while (fn < fccnt): try: tmpfc = watchmatch['comiclist'][fn] except IndexError,KeyError: break temploc= tmpfc['JusttheDigits'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) if 'annual' in temploc.lower(): logger.info("annual detected.") annchk = "yes" fcdigit = helpers.issuedigits(re.sub('annual', '', str(temploc.lower())).strip()) issuechk = myDB.selectone("SELECT * from annuals WHERE ComicID=? AND Int_IssueNumber=?", [cs['ComicID'],fcdigit]).fetchone() else: fcdigit = helpers.issuedigits(temploc) issuechk = myDB.selectone("SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?", [cs['ComicID'],fcdigit]).fetchone() if issuechk is None: logger.fdebug("No corresponding issue # found for " + str(cs['ComicID'])) else: datematch = "True" if len(watchmatch) >= 1 and tmpfc['ComicYear'] is not None: #if the # of matches is more than 1, we need to make sure we get the right series #compare the ReleaseDate for the issue, to the found issue date in the filename. #if ReleaseDate doesn't exist, use IssueDate #if no issue date was found, then ignore. issyr = None #logger.fdebug('issuedate:' + str(issuechk['IssueDate'])) #logger.fdebug('issuechk: ' + str(issuechk['IssueDate'][5:7])) #logger.info('ReleaseDate: ' + str(issuechk['ReleaseDate'])) #logger.info('IssueDate: ' + str(issuechk['IssueDate'])) if issuechk['ReleaseDate'] is not None and issuechk['ReleaseDate'] != '0000-00-00': monthval = issuechk['ReleaseDate'] if int(issuechk['ReleaseDate'][:4]) < int(tmpfc['ComicYear']): logger.fdebug(str(issuechk['ReleaseDate']) + ' is before the issue year of ' + str(tmpfc['ComicYear']) + ' that was discovered in the filename') datematch = "False" else: monthval = issuechk['IssueDate'] if int(issuechk['IssueDate'][:4]) < int(tmpfc['ComicYear']): logger.fdebug(str(issuechk['IssueDate']) + ' is before the issue year ' + str(tmpfc['ComicYear']) + ' that was discovered in the filename') datematch = "False" if int(monthval[5:7]) == 11 or int(monthval[5:7]) == 12: issyr = int(monthval[:4]) + 1 logger.fdebug('issyr is ' + str(issyr)) elif int(monthval[5:7]) == 1 or int(monthval[5:7]) == 2: issyr = int(monthval[:4]) - 1 if datematch == "False" and issyr is not None: logger.fdebug(str(issyr) + ' comparing to ' + str(tmpfc['ComicYear']) + ' : rechecking by month-check versus year.') datematch = "True" if int(issyr) != int(tmpfc['ComicYear']): logger.fdebug('[fail] Issue is before the modified issue year of ' + str(issyr)) datematch = "False" else: logger.info("Found matching issue # " + str(fcdigit) + " for ComicID: " + str(cs['ComicID']) + " / IssueID: " + str(issuechk['IssueID'])) if datematch == "True": manual_list.append({"ComicLocation": tmpfc['ComicLocation'], "ComicID": cs['ComicID'], "IssueID": issuechk['IssueID'], "IssueNumber": issuechk['Issue_Number'], "ComicName": cs['ComicName']}) else: logger.fdebug('Incorrect series - not populating..continuing post-processing') #ccnt+=1 fn+=1 logger.fdebug("There are " + str(len(manual_list)) + " files found that match on your watchlist, " + str(nm) + " do not match anything and will be ignored.")
def forceRescan(ComicID,archive=None): myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() logger.info('Now checking files for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') in ' + rescan['ComicLocation'] ) if archive is None: fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) else: fc = filechecker.listFiles(dir=archive, watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) iscnt = rescan['Total'] havefiles = 0 if mylar.ANNUALS_ON: an_cnt = myDB.action("SELECT COUNT(*) FROM annuals WHERE ComicID=?", [ComicID]).fetchall() anncnt = an_cnt[0][0] else: anncnt = 0 fccnt = int(fc['comiccount']) issnum = 1 fcnew = [] fn = 0 issuedupechk = [] annualdupechk = [] issueexceptdupechk = [] reissues = myDB.action('SELECT * FROM issues WHERE ComicID=?', [ComicID]).fetchall() issID_to_ignore = [] issID_to_ignore.append(str(ComicID)) while (fn < fccnt): haveissue = "no" issuedupe = "no" try: tmpfc = fc['comiclist'][fn] except IndexError: logger.fdebug('Unable to properly retrieve a file listing for the given series.') logger.fdebug('Probably because the filenames being scanned are not in a parseable format') if fn == 0: return else: break temploc= tmpfc['JusttheDigits'].replace('_', ' ') # temploc = tmpfc['ComicFilename'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) logger.fdebug('temploc: ' + str(temploc)) if 'annual' not in temploc.lower(): #remove the extension here extensions = ('.cbr','.cbz') if temploc.lower().endswith(extensions): logger.fdebug('removed extension for issue: ' + str(temploc)) temploc = temploc[:-4] # deccnt = str(temploc).count('.') # if deccnt > 1: #logger.fdebug('decimal counts are :' + str(deccnt)) #if the file is formatted with '.' in place of spaces we need to adjust. #before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE # occur=1 # prevstart = 0 # digitfound = "no" # decimalfound = "no" # tempreconstruct = '' # while (occur <= deccnt): # n = occur # start = temploc.find('.') # while start >=0 and n > 1: # start = temploc.find('.', start+len('.')) # n-=1 # #logger.fdebug('occurance ' + str(occur) + ' of . at position: ' + str(start)) # if temploc[prevstart:start].isdigit(): # if digitfound == "yes": # #logger.fdebug('this is a decimal, assuming decimal issue.') # decimalfound = "yes" # reconst = "." + temploc[prevstart:start] + " " # else: # #logger.fdebug('digit detected.') # digitfound = "yes" # reconst = temploc[prevstart:start] # else: # reconst = temploc[prevstart:start] + " " # #logger.fdebug('word: ' + reconst) # tempreconstruct = tempreconstruct + reconst # #logger.fdebug('tempreconstruct is : ' + tempreconstruct) # prevstart = (start+1) # occur+=1 # #logger.fdebug('word: ' + temploc[prevstart:]) # tempreconstruct = tempreconstruct + " " + temploc[prevstart:] # #logger.fdebug('final filename to use is : ' + str(tempreconstruct)) # temploc = tempreconstruct #logger.fdebug("checking " + str(temploc)) #fcnew_b4 = shlex.split(str(temploc)) fcnew_af = re.findall('[^\()]+', temploc) fcnew = shlex.split(fcnew_af[0]) fcn = len(fcnew) n = 0 while (n <= iscnt): som = 0 try: reiss = reissues[n] except IndexError: break # int_iss, iss_except = helpers.decimal_issue(reiss['Issue_Number']) int_iss = helpers.issuedigits(reiss['Issue_Number']) issyear = reiss['IssueDate'][:4] old_status = reiss['Status'] issname = reiss['IssueName'] #logger.fdebug('integer_issue:' + str(int_iss) + ' ... status: ' + str(old_status)) #if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die. #can't distinguish the 5(c2c) to tell it's the issue #... fnd_iss_except = 'None' #print ("Issue, int_iss, iss_except: " + str(reiss['Issue_Number']) + "," + str(int_iss) + "," + str(iss_except)) while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' #logger.fdebug('checking word - ' + str(fcnew[som])) if ".cbr" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbz", "") if "(c2c)" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace("(c2c)", " ") get_issue = shlex.split(str(fcnew[som])) if fcnew[som] != " ": fcnew[som] = get_issue[0] if som+1 < len(fcnew) and len(fcnew[som+1]) == 2: #print "fcnew[som+1]: " + str(fcnew[som+1]) #print "fcnew[som]: " + str(fcnew[som]) if 'au' in fcnew[som+1].lower(): #if the 'AU' is in 005AU vs 005 AU it will yield different results. fcnew[som] = fcnew[som] + 'AU' fcnew[som+1] = '93939999919190933' logger.info('AU Detected seperate from issue - combining and continuing') elif 'ai' in fcnew[som+1].lower(): #if the 'AI' is in 005AI vs 005 AI it will yield different results. fcnew[som] = fcnew[som] + 'AI' fcnew[som+1] = '93939999919190933' logger.info('AI Detected seperate from issue - combining and continuing') #sometimes scanners refuse to use spaces between () and lump the issue right at the start #mylar assumes it's all one word in this case..let's dump the brackets. fcdigit = helpers.issuedigits(fcnew[som]) #logger.fdebug("fcdigit: " + str(fcdigit)) #logger.fdebug("int_iss: " + str(int_iss)) if int(fcdigit) == int_iss: logger.fdebug('issue match - fcdigit: ' + str(fcdigit) + ' ... int_iss: ' + str(int_iss)) if '-' in temploc and temploc.find(reiss['Issue_Number']) > temploc.find('-'): logger.fdebug('I have detected a possible Title in the filename') logger.fdebug('the issue # has occured after the -, so I assume that it is part of the Title') break for d in issuedupechk: if int(d['fcdigit']) == int(fcdigit): logger.fdebug('duplicate issue detected - not counting this: ' + str(tmpfc['ComicFilename'])) logger.fdebug('is a duplicate of ' + d['filename']) logger.fdebug('fcdigit:' + str(fcdigit) + ' === dupedigit: ' + str(d['fcdigit'])) issuedupe = "yes" break if issuedupe == "no": logger.fdebug('matched...issue: ' + rescan['ComicName'] + '#' + str(reiss['Issue_Number']) + ' --- ' + str(int_iss)) havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) issSize = str(tmpfc['ComicSize']) logger.fdebug('.......filename: ' + str(isslocation)) logger.fdebug('.......filesize: ' + str(tmpfc['ComicSize'])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... issuedupechk.append({'fcdigit': int(fcdigit), 'filename': tmpfc['ComicFilename']}) break #else: # if the issue # matches, but there is no year present - still match. # determine a way to match on year if present, or no year (currently). if issuedupe == "yes": logger.fdebug('I should break out here because of a dupe.') break som+=1 if haveissue == "yes" or issuedupe == "yes": break n+=1 else: # annual inclusion here. #logger.fdebug("checking " + str(temploc)) reannuals = myDB.action('SELECT * FROM annuals WHERE ComicID=?', [ComicID]).fetchall() fcnew = shlex.split(str(temploc)) fcn = len(fcnew) n = 0 while (n < anncnt): som = 0 try: reann = reannuals[n] except IndexError: break int_iss, iss_except = helpers.decimal_issue(reann['Issue_Number']) issyear = reann['IssueDate'][:4] old_status = reann['Status'] while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50$ #logger.fdebug('checking word - ' + str(fcnew[som])) if ".cbr" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbz", "") if "(c2c)" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace("(c2c)", " ") get_issue = shlex.split(str(fcnew[som])) if fcnew[som] != " ": fcnew[som] = get_issue[0] if fcnew[som].lower() == 'annual': logger.fdebug('Annual detected.') if fcnew[som+1].isdigit(): ann_iss = fcnew[som+1] logger.fdebug('Annual # ' + str(ann_iss) + ' detected.') fcdigit = helpers.issuedigits(ann_iss) logger.fdebug('fcdigit:' + str(fcdigit)) logger.fdebug('int_iss:' + str(int_iss)) if int(fcdigit) == int_iss: logger.fdebug('annual match - issue : ' + str(int_iss)) for d in annualdupechk: if int(d['fcdigit']) == int(fcdigit): logger.fdebug('duplicate annual issue detected - not counting this: ' + str(tmpfc['ComicFilename'])) issuedupe = "yes" break if issuedupe == "no": logger.fdebug('matched...annual issue: ' + rescan['ComicName'] + '#' + str(reann['Issue_Number']) + ' --- ' + str(int_iss)) havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) issSize = str(tmpfc['ComicSize']) logger.fdebug('.......filename: ' + str(isslocation)) logger.fdebug('.......filesize: ' + str(tmpfc['ComicSize'])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... annualdupechk.append({'fcdigit': int(fcdigit)}) break som+=1 if haveissue == "yes": break n+=1 if issuedupe == "yes": pass else: #we have the # of comics, now let's update the db. #even if we couldn't find the physical issue, check the status. #-- if annuals aren't enabled, this will bugger out. writeit = True if mylar.ANNUALS_ON: if 'annual' in temploc.lower(): iss_id = reann['IssueID'] else: iss_id = reiss['IssueID'] else: if 'annual' in temploc.lower(): logger.fdebug('Annual support not enabled, but annual issue present within directory. Ignoring annual.') writeit = False else: iss_id = reiss['IssueID'] if writeit == True: logger.fdebug('issueID to write to db:' + str(iss_id)) controlValueDict = {"IssueID": iss_id} #if Archived, increase the 'Have' count. #if archive: # issStatus = "Archived" if haveissue == "yes": issStatus = "Downloaded" newValueDict = {"Location": isslocation, "ComicSize": issSize, "Status": issStatus } issID_to_ignore.append(str(iss_id)) if 'annual' in temploc.lower(): myDB.upsert("annuals", newValueDict, controlValueDict) else: myDB.upsert("issues", newValueDict, controlValueDict) fn+=1 logger.fdebug('IssueID to ignore: ' + str(issID_to_ignore)) #here we need to change the status of the ones we DIDN'T FIND above since the loop only hits on FOUND issues. update_iss = [] tmpsql = "SELECT * FROM issues WHERE ComicID=? AND IssueID not in ({seq})".format(seq=','.join(['?']*(len(issID_to_ignore)-1))) chkthis = myDB.action(tmpsql, issID_to_ignore).fetchall() # chkthis = None if chkthis is None: pass else: for chk in chkthis: old_status = chk['Status'] logger.fdebug('old_status:' + str(old_status)) if old_status == "Skipped": if mylar.AUTOWANT_ALL: issStatus = "Wanted" else: issStatus = "Skipped" elif old_status == "Archived": issStatus = "Archived" elif old_status == "Downloaded": issStatus = "Archived" elif old_status == "Wanted": issStatus = "Wanted" elif old_status == "Ignored": issStatus = "Ignored" elif old_status == "Snatched": #this is needed for torrents, or else it'll keep on queuing.. issStatus = "Snatched" else: issStatus = "Skipped" logger.fdebug("new status: " + str(issStatus)) update_iss.append({"IssueID": chk['IssueID'], "Status": issStatus}) if len(update_iss) > 0: i = 0 #do it like this to avoid DB locks... for ui in update_iss: controlValueDict = {"IssueID": ui['IssueID']} newStatusValue = {"Status": ui['Status']} myDB.upsert("issues", newStatusValue, controlValueDict) i+=1 logger.info('Updated the status of ' + str(i) + ' issues for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') that were not found.') logger.info('Total files located: ' + str(havefiles)) foundcount = havefiles arcfiles = 0 # if filechecker returns 0 files (it doesn't find any), but some issues have a status of 'Archived' # the loop below won't work...let's adjust :) arcissues = myDB.action("SELECT count(*) FROM issues WHERE ComicID=? and Status='Archived'", [ComicID]).fetchall() if int(arcissues[0][0]) > 0: arcfiles = arcissues[0][0] havefiles = havefiles + arcfiles logger.fdebug('Adjusting have total to ' + str(havefiles) + ' because of this many archive files:' + str(arcfiles)) ignorecount = 0 if mylar.IGNORE_HAVETOTAL: # if this is enabled, will increase Have total as if in Archived Status ignores = myDB.action("SELECT count(*) FROM issues WHERE ComicID=? AND Status='Ignored'", [ComicID]).fetchall() if int(ignores[0][0]) > 0: ignorecount = ignores[0][0] havefiles = havefiles + ignorecount logger.fdebug('Adjusting have total to ' + str(havefiles) + ' because of this many Ignored files:' + str(ignorecount)) #now that we are finished... #adjust for issues that have been marked as Downloaded, but aren't found/don't exist. #do it here, because above loop only cycles though found comics using filechecker. downissues = myDB.select("SELECT * FROM issues WHERE ComicID=? and Status='Downloaded'", [ComicID]) if downissues is None: pass else: archivedissues = 0 #set this to 0 so it tallies correctly. for down in downissues: #print "downlocation:" + str(down['Location']) #remove special characters from #temploc = rescan['ComicLocation'].replace('_', ' ') #temploc = re.sub('[\#\'\/\.]', '', temploc) #print ("comiclocation: " + str(rescan['ComicLocation'])) #print ("downlocation: " + str(down['Location'])) if down['Location'] is None: logger.fdebug('location does not exist which means file was not downloaded successfully, or was moved.') controlValue = {"IssueID": down['IssueID']} newValue = {"Status": "Archived"} myDB.upsert("issues", newValue, controlValue) archivedissues+=1 pass else: comicpath = os.path.join(rescan['ComicLocation'], down['Location']) if os.path.exists(comicpath): pass #print "Issue exists - no need to change status." else: #print "Changing status from Downloaded to Archived - cannot locate file" controlValue = {"IssueID": down['IssueID']} newValue = {"Status": "Archived"} myDB.upsert("issues", newValue, controlValue) archivedissues+=1 totalarc = arcfiles + archivedissues havefiles = havefiles + archivedissues #arcfiles already tallied in havefiles in above segment logger.fdebug('I have changed the status of ' + str(archivedissues) + ' issues to a status of Archived, as I now cannot locate them in the series directory.') #let's update the total count of comics that was found. controlValueStat = {"ComicID": rescan['ComicID']} newValueStat = {"Have": havefiles } combined_total = rescan['Total'] + anncnt myDB.upsert("comics", newValueStat, controlValueStat) logger.info('I have physically found ' + str(foundcount) + ' issues, ignored ' + str(ignorecount) + ' issues, and accounted for ' + str(totalarc) + ' in an Archived state. Total Issue Count: ' + str(havefiles) + ' / ' + str(combined_total)) return
def addComictoDB(comicid): # Putting this here to get around the circular import. Will try to use this to update images at later date. from mylar import cache myDB = db.DBConnection() # myDB.action('DELETE from blacklist WHERE ComicID=?', [comicid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ComicID": comicid} dbcomic = myDB.action('SELECT * FROM comics WHERE ComicID=?', [comicid]).fetchone() if dbcomic is None: newValueDict = {"ComicName": "Comic ID: %s" % (comicid), "Status": "Loading"} else: newValueDict = {"Status": "Loading"} myDB.upsert("comics", newValueDict, controlValueDict) # we need to lookup the info for the requested ComicID in full now comic = cv.getComic(comicid,'comic') if not comic: logger.warn("Error fetching comic. ID for : " + comicid) if dbcomic is None: newValueDict = {"ComicName": "Fetch failed, try refreshing. (%s)" % (comicid), "Status": "Active"} else: newValueDict = {"Status": "Active"} myDB.upsert("comics", newValueDict, controlValueDict) return if comic['ComicName'].startswith('The '): sortname = comic['ComicName'][4:] else: sortname = comic['ComicName'] logger.info(u"Now adding/updating: " + comic['ComicName']) #--Now that we know ComicName, let's try some scraping #--Start # gcd will return issue details (most importantly publishing date) gcdinfo=parseit.GCDScraper(comic['ComicName'], comic['ComicYear'], comic['ComicIssues'], comicid) if gcdinfo == "No Match": logger.warn("No matching result found for " + comic['ComicName'] + " (" + comic['ComicYear'] + ")" ) return logger.info(u"Sucessfully retrieved details for " + comic['ComicName'] ) # print ("Series Published" + parseit.resultPublished) #--End #comic book location on machine # setup default location here comlocation = mylar.DESTINATION_DIR + "/" + comic['ComicName'] + " (" + comic['ComicYear'] + ")" #if mylar.REPLACE_SPACES == "yes": #mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot mylarREPLACE_CHAR = '_' comlocation = comlocation.replace(' ', mylarREPLACE_CHAR) #if it doesn't exist - create it (otherwise will bugger up later on) if os.path.isdir(str(comlocation)): logger.info(u"Directory (" + str(comlocation) + ") already exists! Continuing...") else: #print ("Directory doesn't exist!") try: os.makedirs(str(comlocation)) logger.info(u"Directory successfully created at: " + str(comlocation)) except OSError.e: if e.errno != errno.EEXIST: raise #print ("root dir for series: " + comlocation) #try to account for CV not updating new issues as fast as GCD if gcdinfo['gcdvariation'] == "yes": comicIssues = str(int(comic['ComicIssues']) + 1) else: comicIssues = comic['ComicIssues'] controlValueDict = {"ComicID": comicid} newValueDict = {"ComicName": comic['ComicName'], "ComicSortName": sortname, "ComicYear": comic['ComicYear'], "ComicImage": comic['ComicImage'], "Total": comicIssues, "Description": comic['ComicDesc'], "ComicLocation": comlocation, "ComicPublisher": comic['ComicPublisher'], "ComicPublished": parseit.resultPublished, "DateAdded": helpers.today(), "Status": "Loading"} myDB.upsert("comics", newValueDict, controlValueDict) issued = cv.getComic(comicid,'issue') logger.info(u"Sucessfully retrieved issue details for " + comic['ComicName'] ) n = 0 iscnt = int(comicIssues) issid = [] issnum = [] issname = [] issdate = [] int_issnum = [] #let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :) latestiss = "0" latestdate = "0000-00-00" while (n < iscnt): firstval = issued['issuechoice'][n] cleanname = helpers.cleanName(firstval['Issue_Name']) issid.append( str(firstval['Issue_ID']) ) issnum.append( str(firstval['Issue_Number']) ) issname.append(cleanname) bb = 0 while (bb < iscnt): gcdval = gcdinfo['gcdchoice'][bb] #print ("issuecompare: " + str(issnum[n])) #print ("issuecheck: " + str(gcdval['GCDIssue']) ) if str(gcdval['GCDIssue']) == str(issnum[n]): issdate.append( str(gcdval['GCDDate']) ) issnumchg = issnum[n].replace(".00", "") #print ("issnumchg" + str(issnumchg) + "...latestiss:" + str(latestiss)) int_issnum.append(int(issnumchg)) #get the latest issue / date using the date. if gcdval['GCDDate'] > latestdate: latestiss = str(issnumchg) latestdate = str(gcdval['GCDDate']) bb = iscnt bb+=1 #logger.info(u"IssueID: " + str(issid[n]) + " IssueNo: " + str(issnum[n]) + " Date" + str(issdate[n]) ) n+=1 latestiss = latestiss + ".00" #once again - thanks to the new 52 reboot...start n at 0. n = 0 logger.info(u"Now adding/updating issues for" + comic['ComicName']) # file check to see if issue exists logger.info(u"Checking directory for existing issues.") fc = filechecker.listFiles(dir=comlocation, watchcomic=comic['ComicName']) havefiles = 0 fccnt = int(fc['comiccount']) logger.info(u"Found " + str(fccnt) + " issues of " + comic['ComicName']) fcnew = [] while (n < iscnt): fn = 0 haveissue = "no" #print ("on issue " + str(int(n+1)) + " of " + str(iscnt) + " issues") # check if the issue already exists iss_exists = myDB.select('SELECT * from issues WHERE IssueID=?', [issid[n]]) #print ("checking issue: " + str(int_issnum[n])) # stupid way to do this, but check each issue against file-list in fc. while (fn < fccnt): tmpfc = fc['comiclist'][fn] #print (str(int_issnum[n]) + " against ... " + str(tmpfc['ComicFilename'])) temploc = tmpfc['ComicFilename'].replace('_', ' ') fcnew = shlex.split(str(temploc)) fcn = len(fcnew) som = 0 # this loop searches each word in the filename for a match. while (som < fcn): #print (fcnew[som]) #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' if ".cbr" in fcnew[som]: fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som]: fcnew[som] = fcnew[som].replace(".cbz", "") if fcnew[som].isdigit(): #print ("digit detected") #good ol' 52 again.... if int(fcnew[som]) > 0: fcdigit = fcnew[som].lstrip('0') else: fcdigit = "0" #print ( "filename:" + str(int(fcnew[som])) + " - issue: " + str(int_issnum[n]) ) if int(fcdigit) == int_issnum[n]: #print ("matched") #print ("We have this issue - " + str(issnum[n]) + " at " + tmpfc['ComicFilename'] ) havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) break #print ("failed word match on:" + str(fcnew[som]) + "..continuing next word") som+=1 #print (str(temploc) + " doesn't match anything...moving to next file.") fn+=1 if haveissue == "no": isslocation = "None" controlValueDict = {"IssueID": issid[n]} newValueDict = {"ComicID": comicid, "ComicName": comic['ComicName'], "IssueName": issname[n], "Issue_Number": issnum[n], "IssueDate": issdate[n], "Location": isslocation, "Int_IssueNumber": int_issnum[n] } # Only change the status & add DateAdded if the issue is not already in the database if not len(iss_exists): controlValueDict = {"IssueID": issid[n]} newValueDict['DateAdded'] = helpers.today() if haveissue == "no": if mylar.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" #elif release_dict['releasedate'] > helpers.today() and mylar.AUTOWANT_UPCOMING: # newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" elif haveissue == "yes": newValueDict['Status'] = "Downloaded" myDB.upsert("issues", newValueDict, controlValueDict) n+=1 # logger.debug(u"Updating comic cache for " + comic['ComicName']) # cache.getThumb(ComicID=issue['issueid']) # newValueDict['LastUpdated'] = helpers.now() # myDB.upsert("comics", newValueDict, controlValueDict) # logger.debug(u"Updating cache for: " + comic['ComicName']) # cache.getThumb(ComicIDcomicid) controlValueStat = {"ComicID": comicid} newValueStat = {"Status": "Active", "Have": havefiles, "LatestIssue": latestiss, "LatestDate": latestdate } myDB.upsert("comics", newValueStat, controlValueStat) logger.info(u"Updating complete for: " + comic['ComicName']) #here we grab issues that have been marked as wanted above... results = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [comicid]) if results: logger.info(u"Attempting to grab wanted issues for : " + comic['ComicName']) for result in results: foundNZB = "none" if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL) and (mylar.SAB_HOST): foundNZB = search.searchforissue(result['IssueID']) if foundNZB == "yes": updater.foundsearch(result['ComicID'], result['IssueID']) else: logger.info(u"No issues marked as wanted for " + comic['ComicName']) logger.info(u"Finished grabbing what I could.")
def forceRescan(ComicID,archive=None): myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() logger.info(u"Now checking files for " + rescan['ComicName'] + " (" + str(rescan['ComicYear']) + ") in " + str(rescan['ComicLocation']) ) if archive is None: fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) else: fc = filechecker.listFiles(dir=archive, watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) iscnt = rescan['Total'] havefiles = 0 fccnt = int(fc['comiccount']) issnum = 1 fcnew = [] fn = 0 issuedupechk = [] issueexceptdupechk = [] reissues = myDB.action('SELECT * FROM issues WHERE ComicID=?', [ComicID]).fetchall() while (fn < fccnt): haveissue = "no" issuedupe = "no" try: tmpfc = fc['comiclist'][fn] except IndexError: break temploc= tmpfc['JusttheDigits'].replace('_', ' ') # temploc = tmpfc['ComicFilename'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) logger.fdebug("temploc: " + str(temploc)) if 'annual' not in temploc: #remove the extension here extensions = ('.cbr','.cbz') if temploc.lower().endswith(extensions): #logger.fdebug("removed extension for issue:" + str(temploc)) temploc = temploc[:-4] deccnt = str(temploc).count('.') if deccnt > 1: #logger.fdebug("decimal counts are :" + str(deccnt)) #if the file is formatted with '.' in place of spaces we need to adjust. #before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE occur=1 prevstart = 0 digitfound = "no" decimalfound = "no" tempreconstruct = '' while (occur <= deccnt): n = occur start = temploc.find('.') while start >=0 and n > 1: start = temploc.find('.', start+len('.')) n-=1 #logger.fdebug("occurance " + str(occur) + " of . at position: " + str(start)) if temploc[prevstart:start].isdigit(): if digitfound == "yes": #logger.fdebug("this is a decimal, assuming decimal issue.") decimalfound = "yes" reconst = "." + temploc[prevstart:start] + " " else: #logger.fdebug("digit detected.") digitfound = "yes" reconst = temploc[prevstart:start] else: reconst = temploc[prevstart:start] + " " #logger.fdebug("word: " + reconst) tempreconstruct = tempreconstruct + reconst #logger.fdebug("tempreconstruct is : " + tempreconstruct) prevstart = (start+1) occur+=1 #logger.fdebug("word: " + temploc[prevstart:]) tempreconstruct = tempreconstruct + " " + temploc[prevstart:] #logger.fdebug("final filename to use is : " + str(tempreconstruct)) temploc = tempreconstruct #logger.fdebug("checking " + str(temploc)) fcnew = shlex.split(str(temploc)) fcn = len(fcnew) n = 0 while (n <= iscnt): som = 0 try: reiss = reissues[n] except IndexError: break int_iss, iss_except = helpers.decimal_issue(reiss['Issue_Number']) issyear = reiss['IssueDate'][:4] old_status = reiss['Status'] #logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status)) #if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die. #can't distinguish the 5(c2c) to tell it's the issue #... fnd_iss_except = 'None' #print ("Issue, int_iss, iss_except: " + str(reiss['Issue_Number']) + "," + str(int_iss) + "," + str(iss_except)) while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' #logger.fdebug("checking word - " + str(fcnew[som])) if ".cbr" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbz", "") if "(c2c)" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace("(c2c)", " ") get_issue = shlex.split(str(fcnew[som])) if fcnew[som] != " ": fcnew[som] = get_issue[0] if '.' in fcnew[som]: #logger.fdebug("decimal detected...adjusting.") try: i = float(fcnew[som]) except ValueError, TypeError: #not numeric #logger.fdebug("NOT NUMERIC - new word: " + str(fcnew[som])) fcnew[som] = fcnew[som].replace(".", "") else: #numeric pass if fcnew[som].isdigit(): #this won't match on decimal issues - need to fix. #logger.fdebug("digit detected") if int(fcnew[som]) > 0: # fcdigit = fcnew[som].lstrip('0') #fcdigit = str(int(fcnew[som])) fcdigit = int(fcnew[som]) * 1000 if som+1 < len(fcnew) and 'au' in fcnew[som+1].lower(): if len(fcnew[som+1]) == 2: #if the 'AU' is in 005AU vs 005 AU it will yield different results. fnd_iss_except = 'AU' #logger.info("AU Detected - fnd_iss_except set.") else: #fcdigit = "0" fcdigit = 0 elif "." in fcnew[som]: #this will match on decimal issues IssueChk = fcnew[som] #logger.fdebug("decimal detected...analyzing if issue") isschk_find = IssueChk.find('.') isschk_b4dec = IssueChk[:isschk_find] isschk_decval = IssueChk[isschk_find+1:] if isschk_b4dec.isdigit(): #logger.fdebug("digit detected prior to decimal.") if isschk_decval.isdigit(): pass #logger.fdebug("digit detected after decimal.") else: #logger.fdebug("not an issue - no digit detected after decimal") break else: #logger.fdebug("not an issue - no digit detected prior to decimal") break #logger.fdebug("IssueNumber: " + str(IssueChk)) #logger.fdebug("..before decimal: " + str(isschk_b4dec)) #logger.fdebug("...after decimal: " + str(isschk_decval)) #--let's make sure we don't wipe out decimal issues ;) if int(isschk_decval) == 0: iss = isschk_b4dec intdec = int(isschk_decval) else: if len(isschk_decval) == 1: iss = isschk_b4dec + "." + isschk_decval intdec = int(isschk_decval) * 10 else: iss = isschk_b4dec + "." + isschk_decval.rstrip('0') intdec = int(isschk_decval.rstrip('0')) * 10 fcdigit = (int(isschk_b4dec) * 1000) + intdec #logger.fdebug("b4dec: " + str(isschk_b4dec)) #logger.fdebug("decval: " + str(isschk_decval)) #logger.fdebug("intdec: " + str(intdec)) #logger.fdebug("let's compare with this issue value: " + str(fcdigit)) elif 'au' in fcnew[som].lower(): #if AU is part of issue (5AU instead of 5 AU) austart = fcnew[som].lower().find('au') if fcnew[som][:austart].isdigit(): fcdigit = int(fcnew[som][:austart]) * 1000 fnd_iss_except = 'AU' #logger.info("iss_except set to AU") else: # it's a word, skip it. fcdigit = 19283838380101193 #logger.fdebug("fcdigit: " + str(fcdigit)) #logger.fdebug("int_iss: " + str(int_iss)) if "." in str(int_iss): int_iss = helpers.decimal_issue(int_iss) #logger.fdebug("this is the int issue:" + str(int_iss)) #logger.fdebug("this is the fcdigit:" + str(fcdigit)) if int(fcdigit) == int_iss: #logger.fdebug("issue match") #logger.fdebug("fnd_iss_except: " + str(fnd_iss_except)) #logger.fdebug("iss_except: " + str(iss_except)) if str(fnd_iss_except) != 'None' and str(iss_except) == 'AU': if fnd_iss_except.lower() == iss_except.lower(): logger.fdebug("matched for AU") else: logger.fdebug("this is not an AU match..ignoring result.") break elif str(fnd_iss_except) == 'None' and str(iss_except) == 'AU':break elif str(fnd_iss_except) == 'AU' and str(iss_except) == 'None':break #if issyear in fcnew[som+1]: # print "matched on year:" + str(issyear) #issuedupechk here. #print ("fcdigit:" + str(fcdigit)) #print ("findiss_except:" + str(fnd_iss_except) + " = iss_except:" + str(iss_except)) #if int(fcdigit) in issuedupechk and str(fnd_iss_except) not in issueexceptdupechk: #str(fnd_iss_except) == str(iss_except): for d in issuedupechk: if int(d['fcdigit']) == int(fcdigit) and d['fnd_iss_except'] == str(fnd_iss_except): logger.fdebug("duplicate issue detected - not counting this: " + str(tmpfc['ComicFilename'])) issuedupe = "yes" break if issuedupe == "no": logger.fdebug("matched...issue: " + rescan['ComicName'] + "#" + str(reiss['Issue_Number']) + " --- " + str(int_iss)) havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) issSize = str(tmpfc['ComicSize']) logger.fdebug(".......filename: " + str(isslocation)) logger.fdebug(".......filesize: " + str(tmpfc['ComicSize'])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... issuedupechk.append({'fcdigit': int(fcdigit), 'fnd_iss_except': fnd_iss_except}) break #else: # if the issue # matches, but there is no year present - still match. # determine a way to match on year if present, or no year (currently). som+=1 if haveissue == "yes": break n+=1 #we have the # of comics, now let's update the db. #even if we couldn't find the physical issue, check the status. #if Archived, increase the 'Have' count. if archive: issStatus = "Archived" if haveissue == "no" and issuedupe == "no": isslocation = "None" if old_status == "Skipped": if mylar.AUTOWANT_ALL: issStatus = "Wanted" else: issStatus = "Skipped" elif old_status == "Archived": havefiles+=1 issStatus = "Archived" elif old_status == "Downloaded": issStatus = "Archived" havefiles+=1 elif old_status == "Wanted": issStatus = "Wanted" else: issStatus = "Skipped" controlValueDict = {"IssueID": reiss['IssueID']} newValueDict = {"Status": issStatus } elif haveissue == "yes": issStatus = "Downloaded" controlValueDict = {"IssueID": reiss['IssueID']} newValueDict = {"Location": isslocation, "ComicSize": issSize, "Status": issStatus } myDB.upsert("issues", newValueDict, controlValueDict) fn+=1
def Process(self): self._log("nzb name: " + str(self.nzb_name), logger.DEBUG) self._log("nzb folder: " + str(self.nzb_folder), logger.DEBUG) logger.fdebug("nzb name: " + str(self.nzb_name)) logger.fdebug("nzb folder: " + str(self.nzb_folder)) if mylar.USE_SABNZBD==0: logger.fdebug("Not using SABnzbd") elif mylar.USE_SABNZBD != 0 and self.nzb_name == 'Manual Run': logger.fdebug('Not using SABnzbd : Manual Run') else: # if the SAB Directory option is enabled, let's use that folder name and append the jobname. if mylar.SAB_DIRECTORY is not None and mylar.SAB_DIRECTORY is not 'None' and len(mylar.SAB_DIRECTORY) > 4: self.nzb_folder = os.path.join(mylar.SAB_DIRECTORY, self.nzb_name).encode(mylar.SYS_ENCODING) #lookup nzb_name in nzblog table to get issueid #query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals #http://localhost:8080/sabnzbd/api?mode=set_config§ion=misc&keyword=dirscan_speed&value=5 querysab = str(mylar.SAB_HOST) + "/api?mode=get_config§ion=misc&output=xml&apikey=" + str(mylar.SAB_APIKEY) #logger.info("querysab_string:" + str(querysab)) file = urllib2.urlopen(querysab) data = file.read() file.close() dom = parseString(data) try: sabreps = dom.getElementsByTagName('replace_spaces')[0].firstChild.wholeText except: errorm = dom.getElementsByTagName('error')[0].firstChild.wholeText logger.error(u"Error detected attempting to retrieve SAB data : " + errorm) return sabrepd = dom.getElementsByTagName('replace_dots')[0].firstChild.wholeText logger.fdebug("SAB Replace Spaces: " + str(sabreps)) logger.fdebug("SAB Replace Dots: " + str(sabrepd)) if mylar.USE_NZBGET==1: logger.fdebug("Using NZBGET") logger.fdebug("NZB name as passed from NZBGet: " + self.nzb_name) myDB = db.DBConnection() if self.nzb_name == 'Manual Run': print ("manual run initiated") #Manual postprocessing on a folder. #use the nzb_folder to determine every file #walk the dir, #once a series name and issue are matched, #write the series/issue/filename to a tuple #when all done, iterate over the tuple until completion... comicseries = myDB.action("SELECT * FROM comics") manual_list = [] if comicseries is None: logger.error(u"No Series in Watchlist - aborting Manual Post Processing. Maybe you should be running Import?") return else: ccnt=0 nm=0 watchvals = {} for cs in comicseries: watchvals = {"SeriesYear": cs['ComicYear'], "LatestDate": cs['LatestDate'], "ComicVersion": cs['ComicVersion'], "Total": cs['Total']} watchmatch = filechecker.listFiles(self.nzb_folder,cs['ComicName'],cs['AlternateSearch'], manual=watchvals) if watchmatch is None: nm+=1 continue else: fn = 0 fccnt = int(watchmatch['comiccount']) if len(watchmatch) == 1: continue while (fn < fccnt): try: tmpfc = watchmatch['comiclist'][fn] except IndexError,KeyError: break temploc= tmpfc['JusttheDigits'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) logger.fdebug("temploc: " + str(temploc)) ww = shlex.split(temploc) lnw = len(ww) wdc = 0 while (wdc < lnw): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' if ".cbr" in ww[wdc].lower(): ww[wdc] = ww[wdc].replace(".cbr", "") elif ".cbz" in ww[wdc].lower(): ww[wdc] = ww[wdc].replace(".cbz", "") if "(c2c)" in ww[wdc].lower(): ww[wdc] = ww[wdc].replace("(c2c)", " ") get_issue = shlex.split(str(ww[wdc])) if ww[wdc] != " ": ww[wdc] = get_issue[0] if '.' in ww[wdc]: #logger.fdebug("decimal detected...adjusting.") try: i = float(ww[wdc]) except ValueError, TypeError: #not numeric #logger.fdebug("NOT NUMERIC - new word: " + str(ww[wdc])) ww[wdc] = ww[wdc].replace(".", "") else: #numeric pass if ww[wdc].isdigit(): if int(ww[wdc]) > 0: if wdc+1 < len(ww) and 'au' in ww[wdc+1].lower(): if len(ww[wdc+1]) == 2: #if the 'AU' is in 005AU vs 005 AU it will yield different results. ww[wdc] = ww[wdc] + 'AU' ww[wdc+1] = '93939999919190933' logger.info("AU Detected seperate from issue - combining and continuing") fcdigit = helpers.issuedigits(ww[wdc]) if 'annual' in self.nzb_name.lower(): logger.info("annual detected.") annchk = "yes" issuechk = myDB.action("SELECT * from annuals WHERE ComicID=? AND Int_IssueNumber=?", [cs['ComicID'],fcdigit]).fetchone() else: issuechk = myDB.action("SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?", [cs['ComicID'],fcdigit]).fetchone() if issuechk is None: logger.info("No corresponding issue # found for " + str(cs['ComicID'])) else: logger.info("Found matching issue # " + str(fcdigit) + " for ComicID: " + str(cs['ComicID']) + " / IssueID: " + str(issuechk['IssueID'])) manual_list.append({"ComicLocation": tmpfc['ComicLocation'], "ComicID": cs['ComicID'], "IssueID": issuechk['IssueID'], "IssueNumber": issuechk['Issue_Number'], "ComicName": cs['ComicName']}) ccnt+=1 print manual_list wdc+=1 fn+=1 print("There are " + str(len(manual_list)) + " files found that match on your watchlist, " + str(nm) + " do not match anything and will be ignored.")
def forceRescan(ComicID): myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() logger.info(u"Now checking files for " + str(rescan['ComicName']) + " (" + str(rescan['ComicYear']) + ") in " + str(rescan['ComicLocation']) ) fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName']) iscnt = rescan['Total'] havefiles = 0 fccnt = int(fc['comiccount']) issnum = 1 fcnew = [] n = 0 reissues = myDB.action('SELECT * FROM issues WHERE ComicID=?', [ComicID]).fetchall() while (n < iscnt): reiss = reissues[n] int_iss = reiss['Int_IssueNumber'] fn = 0 haveissue = "no" while (fn < fccnt): tmpfc = fc['comiclist'][fn] temploc = tmpfc['ComicFilename'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) if 'annual' not in temploc: fcnew = shlex.split(str(temploc)) fcn = len(fcnew) som = 0 # this loop searches each word in the filename for a match. while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' if ".cbr" in fcnew[som]: fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som]: fcnew[som] = fcnew[som].replace(".cbz", "") if fcnew[som].isdigit(): if int(fcnew[som]) > 0: fcdigit = fcnew[som].lstrip('0') else: fcdigit = "0" if int(fcdigit) == int_iss: havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) break som+=1 else: pass fn+=1 if haveissue == "yes": break #we have the # of comics, now let's update the db. if haveissue == "no": isslocation = "None" if mylar.AUTOWANT_ALL: issStatus = "Wanted" else: issStatus = "Skipped" elif haveissue == "yes": issStatus = "Downloaded" controlValueDict = {"IssueID": reiss['IssueID']} newValueDict = {"Location": isslocation, "Status": issStatus } myDB.upsert("issues", newValueDict, controlValueDict) n+=1 #let's update the total count of comics that was found. controlValueStat = {"ComicID": rescan['ComicID']} newValueStat = {"Have": havefiles } myDB.upsert("comics", newValueStat, controlValueStat) logger.info(u"I've found " + str(havefiles) + " / " + str(rescan['Total']) + " issues." ) return
def forceRescan(ComicID): myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() logger.info(u"Now checking files for " + str(rescan['ComicName']) + " (" + str(rescan['ComicYear']) + ") in " + str(rescan['ComicLocation']) ) fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName']) iscnt = rescan['Total'] havefiles = 0 fccnt = int(fc['comiccount']) issnum = 1 fcnew = [] fn = 0 reissues = myDB.action('SELECT * FROM issues WHERE ComicID=?', [ComicID]).fetchall() # if filechecker returns 0 files (it doesn't find any), but some issues have a status of 'Archived' # the loop below won't work...let's adjust :) arcissues = myDB.select("SELECT * FROM issues WHERE ComicID=? and Status='Archived'", [ComicID]) if len(arcissues) > 0: havefiles = len(arcissues) print "have count adjusted to:" + str(len(arcissues)) while (fn < fccnt): haveissue = "no" try: tmpfc = fc['comiclist'][fn] except IndexError: break temploc = tmpfc['ComicFilename'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) if 'annual' not in temploc: fcnew = shlex.split(str(temploc)) fcn = len(fcnew) n = 0 while (n <= iscnt): som = 0 try: reiss = reissues[n] except IndexError: break int_iss = reiss['Int_IssueNumber'] issyear = reiss['IssueDate'][:4] old_status = reiss['Status'] #print "integer_issue:" + str(int_iss) + " ... status: " + str(old_status) while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' #print ("checking word - " + str(fcnew[som])) if ".cbr" in fcnew[som]: fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som]: fcnew[som] = fcnew[som].replace(".cbz", "") if fcnew[som].isdigit(): #print ("digit detected") if int(fcnew[som]) > 0: # fcdigit = fcnew[som].lstrip('0') fcdigit = str(int(fcnew[som])) else: fcdigit = "0" if int(fcdigit) == int_iss: #if issyear in fcnew[som+1]: # print "matched on year:" + str(issyear) #print ("matched...issue: " + str(fcdigit) + " --- " + str(int_iss)) havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) break #else: # if the issue # matches, but there is no year present - still match. # determine a way to match on year if present, or no year (currently). som+=1 if haveissue == "yes": break n+=1 #we have the # of comics, now let's update the db. #even if we couldn't find the physical issue, check the status. #if Archived, increase the 'Have' count. if haveissue == "no": isslocation = "None" if old_status == "Skipped": if mylar.AUTOWANT_ALL: issStatus = "Wanted" else: issStatus = "Skipped" elif old_status == "Archived": havefiles+=1 issStatus = "Archived" elif old_status == "Downloaded": issStatus = "Archived" havefiles+=1 elif old_status == "Wanted": issStatus = "Wanted" else: issStatus = "Skipped" elif haveissue == "yes": issStatus = "Downloaded" controlValueDict = {"IssueID": reiss['IssueID']} newValueDict = {"Location": isslocation, "Status": issStatus } myDB.upsert("issues", newValueDict, controlValueDict) fn+=1 #let's update the total count of comics that was found. controlValueStat = {"ComicID": rescan['ComicID']} newValueStat = {"Have": havefiles } myDB.upsert("comics", newValueStat, controlValueStat) logger.info(u"I've found " + str(havefiles) + " / " + str(rescan['Total']) + " issues." ) #now that we are finished... #adjust for issues that have been marked as Downloaded, but aren't found/don't exist. #do it here, because above loop only cycles though found comics using filechecker. downissues = myDB.action("SELECT * FROM issues WHERE ComicID=? and Status='Downloaded'", [ComicID]).fetchall() if downissues is None: pass else: for down in downissues: #print "downlocation:" + str(down['Location']) comicpath = os.path.join(rescan['ComicLocation'], down['Location']) if os.path.exists(comicpath): pass #print "Issue exists - no need to change status." else: #print "Changing status from Downloaded to Archived - cannot locate file" controlValue = {"IssueID": down['IssueID']} newValue = {"Status": "Archived"} myDB.upsert("issues", newValue, controlValue) return
def forceRescan(ComicID, archive=None): myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() logger.info(u"Now checking files for " + rescan['ComicName'] + " (" + str(rescan['ComicYear']) + ") in " + str(rescan['ComicLocation'])) if archive is None: fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) else: fc = filechecker.listFiles(dir=archive, watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) iscnt = rescan['Total'] havefiles = 0 fccnt = int(fc['comiccount']) issnum = 1 fcnew = [] fn = 0 issuedupechk = [] issueexceptdupechk = [] reissues = myDB.action('SELECT * FROM issues WHERE ComicID=?', [ComicID]).fetchall() while (fn < fccnt): haveissue = "no" issuedupe = "no" try: tmpfc = fc['comiclist'][fn] except IndexError: break temploc = tmpfc['JusttheDigits'].replace('_', ' ') # temploc = tmpfc['ComicFilename'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) logger.fdebug("temploc: " + str(temploc)) if 'annual' not in temploc: #remove the extension here extensions = ('.cbr', '.cbz') if temploc.lower().endswith(extensions): #logger.fdebug("removed extension for issue:" + str(temploc)) temploc = temploc[:-4] deccnt = str(temploc).count('.') if deccnt > 1: #logger.fdebug("decimal counts are :" + str(deccnt)) #if the file is formatted with '.' in place of spaces we need to adjust. #before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE occur = 1 prevstart = 0 digitfound = "no" decimalfound = "no" tempreconstruct = '' while (occur <= deccnt): n = occur start = temploc.find('.') while start >= 0 and n > 1: start = temploc.find('.', start + len('.')) n -= 1 #logger.fdebug("occurance " + str(occur) + " of . at position: " + str(start)) if temploc[prevstart:start].isdigit(): if digitfound == "yes": #logger.fdebug("this is a decimal, assuming decimal issue.") decimalfound = "yes" reconst = "." + temploc[prevstart:start] + " " else: #logger.fdebug("digit detected.") digitfound = "yes" reconst = temploc[prevstart:start] else: reconst = temploc[prevstart:start] + " " #logger.fdebug("word: " + reconst) tempreconstruct = tempreconstruct + reconst #logger.fdebug("tempreconstruct is : " + tempreconstruct) prevstart = (start + 1) occur += 1 #logger.fdebug("word: " + temploc[prevstart:]) tempreconstruct = tempreconstruct + " " + temploc[prevstart:] #logger.fdebug("final filename to use is : " + str(tempreconstruct)) temploc = tempreconstruct #logger.fdebug("checking " + str(temploc)) fcnew = shlex.split(str(temploc)) fcn = len(fcnew) n = 0 while (n <= iscnt): som = 0 try: reiss = reissues[n] except IndexError: break int_iss, iss_except = helpers.decimal_issue( reiss['Issue_Number']) issyear = reiss['IssueDate'][:4] old_status = reiss['Status'] #logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status)) #if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die. #can't distinguish the 5(c2c) to tell it's the issue #... fnd_iss_except = 'None' #print ("Issue, int_iss, iss_except: " + str(reiss['Issue_Number']) + "," + str(int_iss) + "," + str(iss_except)) while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' #logger.fdebug("checking word - " + str(fcnew[som])) if ".cbr" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbz", "") if "(c2c)" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace("(c2c)", " ") get_issue = shlex.split(str(fcnew[som])) if fcnew[som] != " ": fcnew[som] = get_issue[0] if '.' in fcnew[som]: #logger.fdebug("decimal detected...adjusting.") try: i = float(fcnew[som]) except ValueError, TypeError: #not numeric #logger.fdebug("NOT NUMERIC - new word: " + str(fcnew[som])) fcnew[som] = fcnew[som].replace(".", "") else: #numeric pass if fcnew[som].isdigit(): #this won't match on decimal issues - need to fix. #logger.fdebug("digit detected") if int(fcnew[som]) > 0: # fcdigit = fcnew[som].lstrip('0') #fcdigit = str(int(fcnew[som])) fcdigit = int(fcnew[som]) * 1000 if som + 1 < len(fcnew) and 'au' in fcnew[ som + 1].lower(): if len(fcnew[som + 1]) == 2: #if the 'AU' is in 005AU vs 005 AU it will yield different results. fnd_iss_except = 'AU' #logger.info("AU Detected - fnd_iss_except set.") else: #fcdigit = "0" fcdigit = 0 elif "." in fcnew[som]: #this will match on decimal issues IssueChk = fcnew[som] #logger.fdebug("decimal detected...analyzing if issue") isschk_find = IssueChk.find('.') isschk_b4dec = IssueChk[:isschk_find] isschk_decval = IssueChk[isschk_find + 1:] if isschk_b4dec.isdigit(): #logger.fdebug("digit detected prior to decimal.") if isschk_decval.isdigit(): pass #logger.fdebug("digit detected after decimal.") else: #logger.fdebug("not an issue - no digit detected after decimal") break else: #logger.fdebug("not an issue - no digit detected prior to decimal") break #logger.fdebug("IssueNumber: " + str(IssueChk)) #logger.fdebug("..before decimal: " + str(isschk_b4dec)) #logger.fdebug("...after decimal: " + str(isschk_decval)) #--let's make sure we don't wipe out decimal issues ;) if int(isschk_decval) == 0: iss = isschk_b4dec intdec = int(isschk_decval) else: if len(isschk_decval) == 1: iss = isschk_b4dec + "." + isschk_decval intdec = int(isschk_decval) * 10 else: iss = isschk_b4dec + "." + isschk_decval.rstrip( '0') intdec = int(isschk_decval.rstrip('0')) * 10 fcdigit = (int(isschk_b4dec) * 1000) + intdec #logger.fdebug("b4dec: " + str(isschk_b4dec)) #logger.fdebug("decval: " + str(isschk_decval)) #logger.fdebug("intdec: " + str(intdec)) #logger.fdebug("let's compare with this issue value: " + str(fcdigit)) elif 'au' in fcnew[som].lower(): #if AU is part of issue (5AU instead of 5 AU) austart = fcnew[som].lower().find('au') if fcnew[som][:austart].isdigit(): fcdigit = int(fcnew[som][:austart]) * 1000 fnd_iss_except = 'AU' #logger.info("iss_except set to AU") else: # it's a word, skip it. fcdigit = 19283838380101193 #logger.fdebug("fcdigit: " + str(fcdigit)) #logger.fdebug("int_iss: " + str(int_iss)) if "." in str(int_iss): int_iss = helpers.decimal_issue(int_iss) #logger.fdebug("this is the int issue:" + str(int_iss)) #logger.fdebug("this is the fcdigit:" + str(fcdigit)) if int(fcdigit) == int_iss: #logger.fdebug("issue match") #logger.fdebug("fnd_iss_except: " + str(fnd_iss_except)) #logger.fdebug("iss_except: " + str(iss_except)) if str(fnd_iss_except) != 'None' and str( iss_except) == 'AU': if fnd_iss_except.lower() == iss_except.lower(): logger.fdebug("matched for AU") else: logger.fdebug( "this is not an AU match..ignoring result." ) break elif str(fnd_iss_except) == 'None' and str( iss_except) == 'AU': break elif str(fnd_iss_except) == 'AU' and str( iss_except) == 'None': break #if issyear in fcnew[som+1]: # print "matched on year:" + str(issyear) #issuedupechk here. #print ("fcdigit:" + str(fcdigit)) #print ("findiss_except:" + str(fnd_iss_except) + " = iss_except:" + str(iss_except)) #if int(fcdigit) in issuedupechk and str(fnd_iss_except) not in issueexceptdupechk: #str(fnd_iss_except) == str(iss_except): for d in issuedupechk: if int(d['fcdigit']) == int(fcdigit) and d[ 'fnd_iss_except'] == str(fnd_iss_except): logger.fdebug( "duplicate issue detected - not counting this: " + str(tmpfc['ComicFilename'])) issuedupe = "yes" break if issuedupe == "no": logger.fdebug("matched...issue: " + rescan['ComicName'] + "#" + str(reiss['Issue_Number']) + " --- " + str(int_iss)) havefiles += 1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) issSize = str(tmpfc['ComicSize']) logger.fdebug(".......filename: " + str(isslocation)) logger.fdebug(".......filesize: " + str(tmpfc['ComicSize'])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... issuedupechk.append({ 'fcdigit': int(fcdigit), 'fnd_iss_except': fnd_iss_except }) break #else: # if the issue # matches, but there is no year present - still match. # determine a way to match on year if present, or no year (currently). som += 1 if haveissue == "yes": break n += 1 #we have the # of comics, now let's update the db. #even if we couldn't find the physical issue, check the status. #if Archived, increase the 'Have' count. if archive: issStatus = "Archived" if haveissue == "no" and issuedupe == "no": isslocation = "None" if old_status == "Skipped": if mylar.AUTOWANT_ALL: issStatus = "Wanted" else: issStatus = "Skipped" elif old_status == "Archived": havefiles += 1 issStatus = "Archived" elif old_status == "Downloaded": issStatus = "Archived" havefiles += 1 elif old_status == "Wanted": issStatus = "Wanted" else: issStatus = "Skipped" controlValueDict = {"IssueID": reiss['IssueID']} newValueDict = {"Status": issStatus} elif haveissue == "yes": issStatus = "Downloaded" controlValueDict = {"IssueID": reiss['IssueID']} newValueDict = { "Location": isslocation, "ComicSize": issSize, "Status": issStatus } myDB.upsert("issues", newValueDict, controlValueDict) fn += 1
def Process(self): self._log("nzb name: " + str(self.nzb_name), logger.DEBUG) self._log("nzb folder: " + str(self.nzb_folder), logger.DEBUG) logger.fdebug("nzb name: " + str(self.nzb_name)) logger.fdebug("nzb folder: " + str(self.nzb_folder)) if mylar.USE_SABNZBD == 0: logger.fdebug("Not using SABnzbd") elif mylar.USE_SABNZBD != 0 and self.nzb_name == 'Manual Run': logger.fdebug('Not using SABnzbd : Manual Run') else: # if the SAB Directory option is enabled, let's use that folder name and append the jobname. if mylar.SAB_DIRECTORY is not None and mylar.SAB_DIRECTORY is not 'None' and len( mylar.SAB_DIRECTORY) > 4: self.nzb_folder = os.path.join(mylar.SAB_DIRECTORY, self.nzb_name).encode( mylar.SYS_ENCODING) logger.fdebug( 'SABnzbd Download folder option enabled. Directory set to : ' + self.nzb_folder) # -- start. not used. #query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals #http://localhost:8080/sabnzbd/api?mode=set_config§ion=misc&keyword=dirscan_speed&value=5 #querysab = str(mylar.SAB_HOST) + "/api?mode=get_config§ion=misc&output=xml&apikey=" + str(mylar.SAB_APIKEY) #logger.info("querysab_string:" + str(querysab)) #file = urllib2.urlopen(querysab) #data = file.read() #file.close() #dom = parseString(data) #try: # sabreps = dom.getElementsByTagName('replace_spaces')[0].firstChild.wholeText #except: # errorm = dom.getElementsByTagName('error')[0].firstChild.wholeText # logger.error(u"Error detected attempting to retrieve SAB data : " + errorm) # return #sabrepd = dom.getElementsByTagName('replace_dots')[0].firstChild.wholeText #logger.fdebug("SAB Replace Spaces: " + str(sabreps)) #logger.fdebug("SAB Replace Dots: " + str(sabrepd)) # -- end. not used. if mylar.USE_NZBGET == 1: logger.fdebug("Using NZBGET") logger.fdebug("NZB name as passed from NZBGet: " + self.nzb_name) # if the NZBGet Directory option is enabled, let's use that folder name and append the jobname. if mylar.NZBGET_DIRECTORY is not None and mylar.NZBGET_DIRECTORY is not 'None' and len( mylar.NZBGET_DIRECTORY) > 4: self.nzb_folder = os.path.join(mylar.NZBGET_DIRECTORY, self.nzb_name).encode( mylar.SYS_ENCODING) logger.fdebug( 'NZBGET Download folder option enabled. Directory set to : ' + self.nzb_folder) myDB = db.DBConnection() if self.nzb_name == 'Manual Run': logger.fdebug("manual run initiated") #Manual postprocessing on a folder. #use the nzb_folder to determine every file #walk the dir, #once a series name and issue are matched, #write the series/issue/filename to a tuple #when all done, iterate over the tuple until completion... comicseries = myDB.action("SELECT * FROM comics") manual_list = [] if comicseries is None: logger.error( u"No Series in Watchlist - aborting Manual Post Processing. Maybe you should be running Import?" ) return else: ccnt = 0 nm = 0 watchvals = {} for cs in comicseries: watchvals = { "SeriesYear": cs['ComicYear'], "LatestDate": cs['LatestDate'], "ComicVersion": cs['ComicVersion'], "Publisher": cs['ComicPublisher'], "Total": cs['Total'] } watchmatch = filechecker.listFiles(self.nzb_folder, cs['ComicName'], cs['ComicPublisher'], cs['AlternateSearch'], manual=watchvals) if watchmatch['comiccount'] == 0: # is None: nm += 1 continue else: fn = 0 fccnt = int(watchmatch['comiccount']) if len(watchmatch) == 1: continue while (fn < fccnt): try: tmpfc = watchmatch['comiclist'][fn] except IndexError, KeyError: break temploc = tmpfc['JusttheDigits'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) if 'annual' in temploc.lower(): logger.info("annual detected.") annchk = "yes" fcdigit = helpers.issuedigits( re.sub('annual', '', str(temploc.lower())).strip()) issuechk = myDB.action( "SELECT * from annuals WHERE ComicID=? AND Int_IssueNumber=?", [cs['ComicID'], fcdigit]).fetchone() else: fcdigit = helpers.issuedigits(temploc) issuechk = myDB.action( "SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?", [cs['ComicID'], fcdigit]).fetchone() if issuechk is None: logger.info( "No corresponding issue # found for " + str(cs['ComicID'])) else: datematch = "True" if len(watchmatch) > 1 and tmpfc[ 'ComicYear'] is not None: #if the # of matches is more than 1, we need to make sure we get the right series #compare the ReleaseDate for the issue, to the found issue date in the filename. #if ReleaseDate doesn't exist, use IssueDate #if no issue date was found, then ignore. issyr = None if int(issuechk['IssueDate'] [5:7]) == 11 or issuechk[ 'IssueDate'][5:7] == 12: issyr = int(issuechk['IssueDate'][:4]) elif int( issuechk['IssueDate'] [5:7]) == 1 or int( issuechk['IssueDate'][5:7]) == 2: issyr = int(issuechk['IssueDate'][:4]) if issuechk['ReleaseDate'] is not None: if int(issuechk['ReleaseDate'] [:4]) < int(tmpfc['ComicYear']): logger.fdebug( str(issuechk['ReleaseDate']) + ' is before the issue year of ' + str(tmpfc['ComicYear']) + ' that was discovered in the filename' ) datematch = "False" else: if int(issuechk['IssueDate'] [:4]) < int(tmpfc['ComicYear']): logger.fdebug( str(issuechk['IssueDate']) + ' is before the issue year ' + str(tmpfc['ComicYear']) + ' that was discovered in the filename' ) datematch = "False" if datematch == "False" and issyr is not None: logger.fdebug( str(issyr) + ' comparing to ' + str(tmpfc['ComicYear']) + 'rechecking by month-check versus year.' ) datematch == "True" if int(issyr) != int( tmpfc['ComicYear']): logger.fdebug( '[fail] Issue is before the modified issue year of ' + str(issyr)) datematch = "False" else: logger.info("Found matching issue # " + str(fcdigit) + " for ComicID: " + str(cs['ComicID']) + " / IssueID: " + str(issuechk['IssueID'])) if datematch == "True": manual_list.append({ "ComicLocation": tmpfc['ComicLocation'], "ComicID": cs['ComicID'], "IssueID": issuechk['IssueID'], "IssueNumber": issuechk['Issue_Number'], "ComicName": cs['ComicName'] }) else: logger.fdebug( 'Incorrect series - not populating..continuing post-processing' ) #ccnt+=1 fn += 1 logger.fdebug("There are " + str(len(manual_list)) + " files found that match on your watchlist, " + str(nm) + " do not match anything and will be ignored.")
def Process(self): self._log("nzb name: " + str(self.nzb_name), logger.DEBUG) self._log("nzb folder: " + str(self.nzb_folder), logger.DEBUG) logger.fdebug("nzb name: " + str(self.nzb_name)) logger.fdebug("nzb folder: " + str(self.nzb_folder)) if mylar.USE_SABNZBD == 0: logger.fdebug("Not using SABnzbd") elif mylar.USE_SABNZBD != 0 and self.nzb_name == 'Manual Run': logger.fdebug('Not using SABnzbd : Manual Run') else: # if the SAB Directory option is enabled, let's use that folder name and append the jobname. if mylar.SAB_DIRECTORY is not None and mylar.SAB_DIRECTORY is not 'None' and len( mylar.SAB_DIRECTORY) > 4: self.nzb_folder = os.path.join(mylar.SAB_DIRECTORY, self.nzb_name).encode( mylar.SYS_ENCODING) logger.fdebug( 'SABnzbd Download folder option enabled. Directory set to : ' + self.nzb_folder) #lookup nzb_name in nzblog table to get issueid #query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals #http://localhost:8080/sabnzbd/api?mode=set_config§ion=misc&keyword=dirscan_speed&value=5 querysab = str( mylar.SAB_HOST ) + "/api?mode=get_config§ion=misc&output=xml&apikey=" + str( mylar.SAB_APIKEY) #logger.info("querysab_string:" + str(querysab)) file = urllib2.urlopen(querysab) data = file.read() file.close() dom = parseString(data) try: sabreps = dom.getElementsByTagName( 'replace_spaces')[0].firstChild.wholeText except: errorm = dom.getElementsByTagName( 'error')[0].firstChild.wholeText logger.error( u"Error detected attempting to retrieve SAB data : " + errorm) return sabrepd = dom.getElementsByTagName( 'replace_dots')[0].firstChild.wholeText logger.fdebug("SAB Replace Spaces: " + str(sabreps)) logger.fdebug("SAB Replace Dots: " + str(sabrepd)) if mylar.USE_NZBGET == 1: logger.fdebug("Using NZBGET") logger.fdebug("NZB name as passed from NZBGet: " + self.nzb_name) # if the NZBGet Directory option is enabled, let's use that folder name and append the jobname. if mylar.NZBGET_DIRECTORY is not None and mylar.NZBGET_DIRECTORY is not 'None' and len( mylar.NZBGET_DIRECTORY) > 4: self.nzb_folder = os.path.join(mylar.NZBGET_DIRECTORY, self.nzb_name).encode( mylar.SYS_ENCODING) logger.fdebug( 'NZBGET Download folder option enabled. Directory set to : ' + self.nzb_folder) myDB = db.DBConnection() if self.nzb_name == 'Manual Run': logger.fdebug("manual run initiated") #Manual postprocessing on a folder. #use the nzb_folder to determine every file #walk the dir, #once a series name and issue are matched, #write the series/issue/filename to a tuple #when all done, iterate over the tuple until completion... comicseries = myDB.action("SELECT * FROM comics") manual_list = [] if comicseries is None: logger.error( u"No Series in Watchlist - aborting Manual Post Processing. Maybe you should be running Import?" ) return else: ccnt = 0 nm = 0 watchvals = {} for cs in comicseries: watchvals = { "SeriesYear": cs['ComicYear'], "LatestDate": cs['LatestDate'], "ComicVersion": cs['ComicVersion'], "Publisher": cs['ComicPublisher'], "Total": cs['Total'] } watchmatch = filechecker.listFiles(self.nzb_folder, cs['ComicName'], cs['ComicPublisher'], cs['AlternateSearch'], manual=watchvals) if watchmatch['comiccount'] == 0: # is None: nm += 1 continue else: fn = 0 fccnt = int(watchmatch['comiccount']) if len(watchmatch) == 1: continue while (fn < fccnt): try: tmpfc = watchmatch['comiclist'][fn] except IndexError, KeyError: break temploc = tmpfc['JusttheDigits'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) #logger.fdebug("temploc: " + str(temploc)) ww = shlex.split(temploc) lnw = len(ww) wdc = 0 while (wdc < lnw): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' if ".cbr" in ww[wdc].lower(): ww[wdc] = ww[wdc].replace(".cbr", "") elif ".cbz" in ww[wdc].lower(): ww[wdc] = ww[wdc].replace(".cbz", "") if "(c2c)" in ww[wdc].lower(): ww[wdc] = ww[wdc].replace("(c2c)", " ") get_issue = shlex.split(str(ww[wdc])) if ww[wdc] != " ": ww[wdc] = get_issue[0] if '.' in ww[wdc]: #logger.fdebug("decimal detected...adjusting.") try: i = float(ww[wdc]) except ValueError, TypeError: #not numeric #logger.fdebug("NOT NUMERIC - new word: " + str(ww[wdc])) ww[wdc] = ww[wdc].replace(".", "") else: #numeric pass if ww[wdc].isdigit(): if int(ww[wdc]) > 0: if wdc + 1 < len(ww) and 'au' in ww[ wdc + 1].lower(): if len(ww[wdc + 1]) == 2: #if the 'AU' is in 005AU vs 005 AU it will yield different results. ww[wdc] = ww[wdc] + 'AU' ww[wdc + 1] = '93939999919190933' logger.info( "AU Detected seperate from issue - combining and continuing" ) fcdigit = helpers.issuedigits(ww[wdc]) if 'annual' in self.nzb_name.lower(): logger.info("annual detected.") annchk = "yes" issuechk = myDB.action( "SELECT * from annuals WHERE ComicID=? AND Int_IssueNumber=?", [cs['ComicID'], fcdigit]).fetchone() else: issuechk = myDB.action( "SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?", [cs['ComicID'], fcdigit]).fetchone() if issuechk is None: logger.info( "No corresponding issue # found for " + str(cs['ComicID'])) else: datematch = "True" if len(watchmatch) > 1: #if the # of matches is more than 1, we need to make sure we get the right series #compare the ReleaseDate for the issue, to the found issue date in the filename. #if ReleaseDate doesn't exist, use IssueDate #if no issue date was found, then ignore. if issuechk['ReleaseDate'] is not None: if int(issuechk['ReleaseDate'][:4] ) < int(tmpfc['ComicYear']): logger.fdebug( str(issuechk['ReleaseDate'] ) + ' is before the issue year of ' + str(tmpfc['ComicYear']) + ' that was discovered in the filename' ) datematch = "False" else: if int(issuechk['IssueDate'][:4] ) < int(tmpfc['ComicYear']): logger.fdebug( str(issuechk['IssueDate']) + ' is before the issue year ' + str(tmpfc['ComicYear']) + ' that was discovered in the filename' ) datematch = "False" else: logger.info("Found matching issue # " + str(fcdigit) + " for ComicID: " + str(cs['ComicID']) + " / IssueID: " + str(issuechk['IssueID'])) if datematch == "True": manual_list.append({ "ComicLocation": tmpfc['ComicLocation'], "ComicID": cs['ComicID'], "IssueID": issuechk['IssueID'], "IssueNumber": issuechk['Issue_Number'], "ComicName": cs['ComicName'] }) else: logger.fdebug( 'Incorrect series - not populating..continuing post-processing' ) ccnt += 1 #print manual_list wdc += 1 fn += 1 logger.fdebug("There are " + str(len(manual_list)) + " files found that match on your watchlist, " + str(nm) + " do not match anything and will be ignored.")