def forceRescan(ComicID): myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.action("SELECT * FROM comics WHERE ComicID=?", [ComicID]).fetchone() logger.info( u"Now checking files for " + str(rescan["ComicName"]) + " (" + str(rescan["ComicYear"]) + ") in " + str(rescan["ComicLocation"]) ) fc = filechecker.listFiles( dir=rescan["ComicLocation"], watchcomic=rescan["ComicName"], AlternateSearch=rescan["AlternateSearch"] ) iscnt = rescan["Total"] havefiles = 0 fccnt = int(fc["comiccount"]) issnum = 1 fcnew = [] fn = 0 issuedupechk = [] reissues = myDB.action("SELECT * FROM issues WHERE ComicID=?", [ComicID]).fetchall() # if filechecker returns 0 files (it doesn't find any), but some issues have a status of 'Archived' # the loop below won't work...let's adjust :) arcissues = myDB.select("SELECT * FROM issues WHERE ComicID=? and Status='Archived'", [ComicID]) if len(arcissues) > 0: havefiles = len(arcissues) logger.fdebug("Adjusting have total because of this many archive files:" + str(len(arcissues))) while fn < fccnt: haveissue = "no" issuedupe = "no" try: tmpfc = fc["comiclist"][fn] except IndexError: break temploc = tmpfc["ComicFilename"].replace("_", " ") temploc = re.sub("[\#']", "", temploc) # logger.fdebug("temploc: " + str(temploc)) if "annual" not in temploc: fcnew = shlex.split(str(temploc)) fcn = len(fcnew) n = 0 while n <= iscnt: som = 0 try: reiss = reissues[n] except IndexError: break int_iss = helpers.decimal_issue(reiss["Issue_Number"]) issyear = reiss["IssueDate"][:4] old_status = reiss["Status"] # logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status)) # if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die. # can't distinguish the 5(c2c) to tell it's the issue #... while som < fcn: # counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' # logger.fdebug("checking word - " + str(fcnew[som])) if ".cbr" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbz", "") if "(c2c)" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace("(c2c)", " ") get_issue = shlex.split(str(fcnew[som])) if fcnew[som] != " ": fcnew[som] = get_issue[0] if "." in fcnew[som]: # logger.fdebug("decimal detected...adjusting.") try: i = float(fcnew[som]) except ValueError, TypeError: # not numeric fcnew[som] = fcnew[som].replace(".", "") # logger.fdebug("NOT NUMERIC - new word: " + str(fcnew[som])) else: # numeric pass if fcnew[som].isdigit(): # this won't match on decimal issues - need to fix. # logger.fdebug("digit detected") if int(fcnew[som]) > 0: # fcdigit = fcnew[som].lstrip('0') # fcdigit = str(int(fcnew[som])) fcdigit = int(fcnew[som]) * 1000 else: # fcdigit = "0" fcdigit = 0 elif "." in fcnew[som]: # this will match on decimal issues IssueChk = fcnew[som] # logger.fdebug("decimal detected...analyzing if issue") isschk_find = IssueChk.find(".") isschk_b4dec = IssueChk[:isschk_find] isschk_decval = IssueChk[isschk_find + 1 :] if isschk_b4dec.isdigit(): # logger.fdebug("digit detected prior to decimal.") if isschk_decval.isdigit(): pass # logger.fdebug("digit detected after decimal.") else: # logger.fdebug("not an issue - no digit detected after decimal") break else: # logger.fdebug("not an issue - no digit detected prior to decimal") break # logger.fdebug("IssueNumber: " + str(IssueChk)) # logger.fdebug("..before decimal: " + str(isschk_b4dec)) # logger.fdebug("...after decimal: " + str(isschk_decval)) # --let's make sure we don't wipe out decimal issues ;) if int(isschk_decval) == 0: iss = isschk_b4dec intdec = int(isschk_decval) else: if len(isschk_decval) == 1: iss = isschk_b4dec + "." + isschk_decval intdec = int(isschk_decval) * 10 else: iss = isschk_b4dec + "." + isschk_decval.rstrip("0") intdec = int(isschk_decval.rstrip("0")) * 10 fcdigit = (int(isschk_b4dec) * 1000) + intdec # logger.fdebug("b4dec: " + str(isschk_b4dec)) # logger.fdebug("decval: " + str(isschk_decval)) # logger.fdebug("intdec: " + str(intdec)) # logger.fdebug("let's compare with this issue value: " + str(fcdigit)) else: # it's a word, skip it. fcdigit = 1000000 # logger.fdebug("fcdigit: " + str(fcdigit)) # logger.fdebug("int_iss: " + str(int_iss)) if "." in str(int_iss): int_iss = helpers.decimal_issue(int_iss) # logger.fdebug("this is the int issue:" + str(int_iss)) if int(fcdigit) == int_iss: # if issyear in fcnew[som+1]: # print "matched on year:" + str(issyear) # issuedupechk here. if int(fcdigit) in issuedupechk: logger.fdebug( "duplicate issue detected - not counting this: " + str(tmpfc["ComicFilename"]) ) issuedupe = "yes" break logger.fdebug("matched...issue: " + str(rescan["ComicName"]) + " --- " + str(int_iss)) havefiles += 1 haveissue = "yes" isslocation = str(tmpfc["ComicFilename"]) issSize = str(tmpfc["ComicSize"]) logger.fdebug(".......filename: " + str(isslocation)) logger.fdebug(".......filesize: " + str(tmpfc["ComicSize"])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... issuedupechk.append(int(fcdigit)) break # else: # if the issue # matches, but there is no year present - still match. # determine a way to match on year if present, or no year (currently). som += 1 if haveissue == "yes": break n += 1 # we have the # of comics, now let's update the db. # even if we couldn't find the physical issue, check the status. # if Archived, increase the 'Have' count. if haveissue == "no" and issuedupe == "no": isslocation = "None" if old_status == "Skipped": if mylar.AUTOWANT_ALL: issStatus = "Wanted" else: issStatus = "Skipped" elif old_status == "Archived": havefiles += 1 issStatus = "Archived" elif old_status == "Downloaded": issStatus = "Archived" havefiles += 1 elif old_status == "Wanted": issStatus = "Wanted" else: issStatus = "Skipped" controlValueDict = {"IssueID": reiss["IssueID"]} newValueDict = {"Status": issStatus} elif haveissue == "yes": issStatus = "Downloaded" controlValueDict = {"IssueID": reiss["IssueID"]} newValueDict = {"Location": isslocation, "ComicSize": issSize, "Status": issStatus} myDB.upsert("issues", newValueDict, controlValueDict) fn += 1
def forceRescan(ComicID,archive=None): myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() logger.info(u"Now checking files for " + rescan['ComicName'] + " (" + str(rescan['ComicYear']) + ") in " + str(rescan['ComicLocation']) ) if archive is None: fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) else: fc = filechecker.listFiles(dir=archive, watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) iscnt = rescan['Total'] havefiles = 0 fccnt = int(fc['comiccount']) issnum = 1 fcnew = [] fn = 0 issuedupechk = [] issueexceptdupechk = [] reissues = myDB.action('SELECT * FROM issues WHERE ComicID=?', [ComicID]).fetchall() while (fn < fccnt): haveissue = "no" issuedupe = "no" try: tmpfc = fc['comiclist'][fn] except IndexError: break temploc= tmpfc['JusttheDigits'].replace('_', ' ') # temploc = tmpfc['ComicFilename'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) logger.fdebug("temploc: " + str(temploc)) if 'annual' not in temploc: #remove the extension here extensions = ('.cbr','.cbz') if temploc.lower().endswith(extensions): #logger.fdebug("removed extension for issue:" + str(temploc)) temploc = temploc[:-4] deccnt = str(temploc).count('.') if deccnt > 1: #logger.fdebug("decimal counts are :" + str(deccnt)) #if the file is formatted with '.' in place of spaces we need to adjust. #before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE occur=1 prevstart = 0 digitfound = "no" decimalfound = "no" tempreconstruct = '' while (occur <= deccnt): n = occur start = temploc.find('.') while start >=0 and n > 1: start = temploc.find('.', start+len('.')) n-=1 #logger.fdebug("occurance " + str(occur) + " of . at position: " + str(start)) if temploc[prevstart:start].isdigit(): if digitfound == "yes": #logger.fdebug("this is a decimal, assuming decimal issue.") decimalfound = "yes" reconst = "." + temploc[prevstart:start] + " " else: #logger.fdebug("digit detected.") digitfound = "yes" reconst = temploc[prevstart:start] else: reconst = temploc[prevstart:start] + " " #logger.fdebug("word: " + reconst) tempreconstruct = tempreconstruct + reconst #logger.fdebug("tempreconstruct is : " + tempreconstruct) prevstart = (start+1) occur+=1 #logger.fdebug("word: " + temploc[prevstart:]) tempreconstruct = tempreconstruct + " " + temploc[prevstart:] #logger.fdebug("final filename to use is : " + str(tempreconstruct)) temploc = tempreconstruct #logger.fdebug("checking " + str(temploc)) fcnew = shlex.split(str(temploc)) fcn = len(fcnew) n = 0 while (n <= iscnt): som = 0 try: reiss = reissues[n] except IndexError: break int_iss, iss_except = helpers.decimal_issue(reiss['Issue_Number']) issyear = reiss['IssueDate'][:4] old_status = reiss['Status'] #logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status)) #if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die. #can't distinguish the 5(c2c) to tell it's the issue #... fnd_iss_except = 'None' #print ("Issue, int_iss, iss_except: " + str(reiss['Issue_Number']) + "," + str(int_iss) + "," + str(iss_except)) while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' #logger.fdebug("checking word - " + str(fcnew[som])) if ".cbr" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbz", "") if "(c2c)" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace("(c2c)", " ") get_issue = shlex.split(str(fcnew[som])) if fcnew[som] != " ": fcnew[som] = get_issue[0] if '.' in fcnew[som]: #logger.fdebug("decimal detected...adjusting.") try: i = float(fcnew[som]) except ValueError, TypeError: #not numeric #logger.fdebug("NOT NUMERIC - new word: " + str(fcnew[som])) fcnew[som] = fcnew[som].replace(".", "") else: #numeric pass if fcnew[som].isdigit(): #this won't match on decimal issues - need to fix. #logger.fdebug("digit detected") if int(fcnew[som]) > 0: # fcdigit = fcnew[som].lstrip('0') #fcdigit = str(int(fcnew[som])) fcdigit = int(fcnew[som]) * 1000 if som+1 < len(fcnew) and 'au' in fcnew[som+1].lower(): if len(fcnew[som+1]) == 2: #if the 'AU' is in 005AU vs 005 AU it will yield different results. fnd_iss_except = 'AU' #logger.info("AU Detected - fnd_iss_except set.") else: #fcdigit = "0" fcdigit = 0 elif "." in fcnew[som]: #this will match on decimal issues IssueChk = fcnew[som] #logger.fdebug("decimal detected...analyzing if issue") isschk_find = IssueChk.find('.') isschk_b4dec = IssueChk[:isschk_find] isschk_decval = IssueChk[isschk_find+1:] if isschk_b4dec.isdigit(): #logger.fdebug("digit detected prior to decimal.") if isschk_decval.isdigit(): pass #logger.fdebug("digit detected after decimal.") else: #logger.fdebug("not an issue - no digit detected after decimal") break else: #logger.fdebug("not an issue - no digit detected prior to decimal") break #logger.fdebug("IssueNumber: " + str(IssueChk)) #logger.fdebug("..before decimal: " + str(isschk_b4dec)) #logger.fdebug("...after decimal: " + str(isschk_decval)) #--let's make sure we don't wipe out decimal issues ;) if int(isschk_decval) == 0: iss = isschk_b4dec intdec = int(isschk_decval) else: if len(isschk_decval) == 1: iss = isschk_b4dec + "." + isschk_decval intdec = int(isschk_decval) * 10 else: iss = isschk_b4dec + "." + isschk_decval.rstrip('0') intdec = int(isschk_decval.rstrip('0')) * 10 fcdigit = (int(isschk_b4dec) * 1000) + intdec #logger.fdebug("b4dec: " + str(isschk_b4dec)) #logger.fdebug("decval: " + str(isschk_decval)) #logger.fdebug("intdec: " + str(intdec)) #logger.fdebug("let's compare with this issue value: " + str(fcdigit)) elif 'au' in fcnew[som].lower(): #if AU is part of issue (5AU instead of 5 AU) austart = fcnew[som].lower().find('au') if fcnew[som][:austart].isdigit(): fcdigit = int(fcnew[som][:austart]) * 1000 fnd_iss_except = 'AU' #logger.info("iss_except set to AU") else: # it's a word, skip it. fcdigit = 19283838380101193 #logger.fdebug("fcdigit: " + str(fcdigit)) #logger.fdebug("int_iss: " + str(int_iss)) if "." in str(int_iss): int_iss = helpers.decimal_issue(int_iss) #logger.fdebug("this is the int issue:" + str(int_iss)) #logger.fdebug("this is the fcdigit:" + str(fcdigit)) if int(fcdigit) == int_iss: #logger.fdebug("issue match") #logger.fdebug("fnd_iss_except: " + str(fnd_iss_except)) #logger.fdebug("iss_except: " + str(iss_except)) if str(fnd_iss_except) != 'None' and str(iss_except) == 'AU': if fnd_iss_except.lower() == iss_except.lower(): logger.fdebug("matched for AU") else: logger.fdebug("this is not an AU match..ignoring result.") break elif str(fnd_iss_except) == 'None' and str(iss_except) == 'AU':break elif str(fnd_iss_except) == 'AU' and str(iss_except) == 'None':break #if issyear in fcnew[som+1]: # print "matched on year:" + str(issyear) #issuedupechk here. #print ("fcdigit:" + str(fcdigit)) #print ("findiss_except:" + str(fnd_iss_except) + " = iss_except:" + str(iss_except)) #if int(fcdigit) in issuedupechk and str(fnd_iss_except) not in issueexceptdupechk: #str(fnd_iss_except) == str(iss_except): for d in issuedupechk: if int(d['fcdigit']) == int(fcdigit) and d['fnd_iss_except'] == str(fnd_iss_except): logger.fdebug("duplicate issue detected - not counting this: " + str(tmpfc['ComicFilename'])) issuedupe = "yes" break if issuedupe == "no": logger.fdebug("matched...issue: " + rescan['ComicName'] + "#" + str(reiss['Issue_Number']) + " --- " + str(int_iss)) havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) issSize = str(tmpfc['ComicSize']) logger.fdebug(".......filename: " + str(isslocation)) logger.fdebug(".......filesize: " + str(tmpfc['ComicSize'])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... issuedupechk.append({'fcdigit': int(fcdigit), 'fnd_iss_except': fnd_iss_except}) break #else: # if the issue # matches, but there is no year present - still match. # determine a way to match on year if present, or no year (currently). som+=1 if haveissue == "yes": break n+=1 #we have the # of comics, now let's update the db. #even if we couldn't find the physical issue, check the status. #if Archived, increase the 'Have' count. if archive: issStatus = "Archived" if haveissue == "no" and issuedupe == "no": isslocation = "None" if old_status == "Skipped": if mylar.AUTOWANT_ALL: issStatus = "Wanted" else: issStatus = "Skipped" elif old_status == "Archived": havefiles+=1 issStatus = "Archived" elif old_status == "Downloaded": issStatus = "Archived" havefiles+=1 elif old_status == "Wanted": issStatus = "Wanted" else: issStatus = "Skipped" controlValueDict = {"IssueID": reiss['IssueID']} newValueDict = {"Status": issStatus } elif haveissue == "yes": issStatus = "Downloaded" controlValueDict = {"IssueID": reiss['IssueID']} newValueDict = {"Location": isslocation, "ComicSize": issSize, "Status": issStatus } myDB.upsert("issues", newValueDict, controlValueDict) fn+=1
def forceRescan(ComicID,archive=None): myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() logger.info('Now checking files for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') in ' + rescan['ComicLocation'] ) if archive is None: fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) else: fc = filechecker.listFiles(dir=archive, watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) iscnt = rescan['Total'] havefiles = 0 if mylar.ANNUALS_ON: an_cnt = myDB.action("SELECT COUNT(*) FROM annuals WHERE ComicID=?", [ComicID]).fetchall() anncnt = an_cnt[0][0] else: anncnt = 0 fccnt = int(fc['comiccount']) issnum = 1 fcnew = [] fn = 0 issuedupechk = [] annualdupechk = [] issueexceptdupechk = [] reissues = myDB.action('SELECT * FROM issues WHERE ComicID=?', [ComicID]).fetchall() issID_to_ignore = [] issID_to_ignore.append(str(ComicID)) while (fn < fccnt): haveissue = "no" issuedupe = "no" try: tmpfc = fc['comiclist'][fn] except IndexError: logger.fdebug('Unable to properly retrieve a file listing for the given series.') logger.fdebug('Probably because the filenames being scanned are not in a parseable format') if fn == 0: return else: break temploc= tmpfc['JusttheDigits'].replace('_', ' ') # temploc = tmpfc['ComicFilename'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) logger.fdebug('temploc: ' + str(temploc)) if 'annual' not in temploc.lower(): #remove the extension here extensions = ('.cbr','.cbz') if temploc.lower().endswith(extensions): logger.fdebug('removed extension for issue: ' + str(temploc)) temploc = temploc[:-4] # deccnt = str(temploc).count('.') # if deccnt > 1: #logger.fdebug('decimal counts are :' + str(deccnt)) #if the file is formatted with '.' in place of spaces we need to adjust. #before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE # occur=1 # prevstart = 0 # digitfound = "no" # decimalfound = "no" # tempreconstruct = '' # while (occur <= deccnt): # n = occur # start = temploc.find('.') # while start >=0 and n > 1: # start = temploc.find('.', start+len('.')) # n-=1 # #logger.fdebug('occurance ' + str(occur) + ' of . at position: ' + str(start)) # if temploc[prevstart:start].isdigit(): # if digitfound == "yes": # #logger.fdebug('this is a decimal, assuming decimal issue.') # decimalfound = "yes" # reconst = "." + temploc[prevstart:start] + " " # else: # #logger.fdebug('digit detected.') # digitfound = "yes" # reconst = temploc[prevstart:start] # else: # reconst = temploc[prevstart:start] + " " # #logger.fdebug('word: ' + reconst) # tempreconstruct = tempreconstruct + reconst # #logger.fdebug('tempreconstruct is : ' + tempreconstruct) # prevstart = (start+1) # occur+=1 # #logger.fdebug('word: ' + temploc[prevstart:]) # tempreconstruct = tempreconstruct + " " + temploc[prevstart:] # #logger.fdebug('final filename to use is : ' + str(tempreconstruct)) # temploc = tempreconstruct #logger.fdebug("checking " + str(temploc)) #fcnew_b4 = shlex.split(str(temploc)) fcnew_af = re.findall('[^\()]+', temploc) fcnew = shlex.split(fcnew_af[0]) fcn = len(fcnew) n = 0 while (n <= iscnt): som = 0 try: reiss = reissues[n] except IndexError: break # int_iss, iss_except = helpers.decimal_issue(reiss['Issue_Number']) int_iss = helpers.issuedigits(reiss['Issue_Number']) issyear = reiss['IssueDate'][:4] old_status = reiss['Status'] issname = reiss['IssueName'] #logger.fdebug('integer_issue:' + str(int_iss) + ' ... status: ' + str(old_status)) #if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die. #can't distinguish the 5(c2c) to tell it's the issue #... fnd_iss_except = 'None' #print ("Issue, int_iss, iss_except: " + str(reiss['Issue_Number']) + "," + str(int_iss) + "," + str(iss_except)) while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' #logger.fdebug('checking word - ' + str(fcnew[som])) if ".cbr" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbz", "") if "(c2c)" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace("(c2c)", " ") get_issue = shlex.split(str(fcnew[som])) if fcnew[som] != " ": fcnew[som] = get_issue[0] if som+1 < len(fcnew) and len(fcnew[som+1]) == 2: #print "fcnew[som+1]: " + str(fcnew[som+1]) #print "fcnew[som]: " + str(fcnew[som]) if 'au' in fcnew[som+1].lower(): #if the 'AU' is in 005AU vs 005 AU it will yield different results. fcnew[som] = fcnew[som] + 'AU' fcnew[som+1] = '93939999919190933' logger.info('AU Detected seperate from issue - combining and continuing') elif 'ai' in fcnew[som+1].lower(): #if the 'AI' is in 005AI vs 005 AI it will yield different results. fcnew[som] = fcnew[som] + 'AI' fcnew[som+1] = '93939999919190933' logger.info('AI Detected seperate from issue - combining and continuing') #sometimes scanners refuse to use spaces between () and lump the issue right at the start #mylar assumes it's all one word in this case..let's dump the brackets. fcdigit = helpers.issuedigits(fcnew[som]) #logger.fdebug("fcdigit: " + str(fcdigit)) #logger.fdebug("int_iss: " + str(int_iss)) if int(fcdigit) == int_iss: logger.fdebug('issue match - fcdigit: ' + str(fcdigit) + ' ... int_iss: ' + str(int_iss)) if '-' in temploc and temploc.find(reiss['Issue_Number']) > temploc.find('-'): logger.fdebug('I have detected a possible Title in the filename') logger.fdebug('the issue # has occured after the -, so I assume that it is part of the Title') break for d in issuedupechk: if int(d['fcdigit']) == int(fcdigit): logger.fdebug('duplicate issue detected - not counting this: ' + str(tmpfc['ComicFilename'])) logger.fdebug('is a duplicate of ' + d['filename']) logger.fdebug('fcdigit:' + str(fcdigit) + ' === dupedigit: ' + str(d['fcdigit'])) issuedupe = "yes" break if issuedupe == "no": logger.fdebug('matched...issue: ' + rescan['ComicName'] + '#' + str(reiss['Issue_Number']) + ' --- ' + str(int_iss)) havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) issSize = str(tmpfc['ComicSize']) logger.fdebug('.......filename: ' + str(isslocation)) logger.fdebug('.......filesize: ' + str(tmpfc['ComicSize'])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... issuedupechk.append({'fcdigit': int(fcdigit), 'filename': tmpfc['ComicFilename']}) break #else: # if the issue # matches, but there is no year present - still match. # determine a way to match on year if present, or no year (currently). if issuedupe == "yes": logger.fdebug('I should break out here because of a dupe.') break som+=1 if haveissue == "yes" or issuedupe == "yes": break n+=1 else: # annual inclusion here. #logger.fdebug("checking " + str(temploc)) reannuals = myDB.action('SELECT * FROM annuals WHERE ComicID=?', [ComicID]).fetchall() fcnew = shlex.split(str(temploc)) fcn = len(fcnew) n = 0 while (n < anncnt): som = 0 try: reann = reannuals[n] except IndexError: break int_iss, iss_except = helpers.decimal_issue(reann['Issue_Number']) issyear = reann['IssueDate'][:4] old_status = reann['Status'] while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50$ #logger.fdebug('checking word - ' + str(fcnew[som])) if ".cbr" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbz", "") if "(c2c)" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace("(c2c)", " ") get_issue = shlex.split(str(fcnew[som])) if fcnew[som] != " ": fcnew[som] = get_issue[0] if fcnew[som].lower() == 'annual': logger.fdebug('Annual detected.') if fcnew[som+1].isdigit(): ann_iss = fcnew[som+1] logger.fdebug('Annual # ' + str(ann_iss) + ' detected.') fcdigit = helpers.issuedigits(ann_iss) logger.fdebug('fcdigit:' + str(fcdigit)) logger.fdebug('int_iss:' + str(int_iss)) if int(fcdigit) == int_iss: logger.fdebug('annual match - issue : ' + str(int_iss)) for d in annualdupechk: if int(d['fcdigit']) == int(fcdigit): logger.fdebug('duplicate annual issue detected - not counting this: ' + str(tmpfc['ComicFilename'])) issuedupe = "yes" break if issuedupe == "no": logger.fdebug('matched...annual issue: ' + rescan['ComicName'] + '#' + str(reann['Issue_Number']) + ' --- ' + str(int_iss)) havefiles+=1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) issSize = str(tmpfc['ComicSize']) logger.fdebug('.......filename: ' + str(isslocation)) logger.fdebug('.......filesize: ' + str(tmpfc['ComicSize'])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... annualdupechk.append({'fcdigit': int(fcdigit)}) break som+=1 if haveissue == "yes": break n+=1 if issuedupe == "yes": pass else: #we have the # of comics, now let's update the db. #even if we couldn't find the physical issue, check the status. #-- if annuals aren't enabled, this will bugger out. writeit = True if mylar.ANNUALS_ON: if 'annual' in temploc.lower(): iss_id = reann['IssueID'] else: iss_id = reiss['IssueID'] else: if 'annual' in temploc.lower(): logger.fdebug('Annual support not enabled, but annual issue present within directory. Ignoring annual.') writeit = False else: iss_id = reiss['IssueID'] if writeit == True: logger.fdebug('issueID to write to db:' + str(iss_id)) controlValueDict = {"IssueID": iss_id} #if Archived, increase the 'Have' count. #if archive: # issStatus = "Archived" if haveissue == "yes": issStatus = "Downloaded" newValueDict = {"Location": isslocation, "ComicSize": issSize, "Status": issStatus } issID_to_ignore.append(str(iss_id)) if 'annual' in temploc.lower(): myDB.upsert("annuals", newValueDict, controlValueDict) else: myDB.upsert("issues", newValueDict, controlValueDict) fn+=1 logger.fdebug('IssueID to ignore: ' + str(issID_to_ignore)) #here we need to change the status of the ones we DIDN'T FIND above since the loop only hits on FOUND issues. update_iss = [] tmpsql = "SELECT * FROM issues WHERE ComicID=? AND IssueID not in ({seq})".format(seq=','.join(['?']*(len(issID_to_ignore)-1))) chkthis = myDB.action(tmpsql, issID_to_ignore).fetchall() # chkthis = None if chkthis is None: pass else: for chk in chkthis: old_status = chk['Status'] logger.fdebug('old_status:' + str(old_status)) if old_status == "Skipped": if mylar.AUTOWANT_ALL: issStatus = "Wanted" else: issStatus = "Skipped" elif old_status == "Archived": issStatus = "Archived" elif old_status == "Downloaded": issStatus = "Archived" elif old_status == "Wanted": issStatus = "Wanted" elif old_status == "Ignored": issStatus = "Ignored" elif old_status == "Snatched": #this is needed for torrents, or else it'll keep on queuing.. issStatus = "Snatched" else: issStatus = "Skipped" logger.fdebug("new status: " + str(issStatus)) update_iss.append({"IssueID": chk['IssueID'], "Status": issStatus}) if len(update_iss) > 0: i = 0 #do it like this to avoid DB locks... for ui in update_iss: controlValueDict = {"IssueID": ui['IssueID']} newStatusValue = {"Status": ui['Status']} myDB.upsert("issues", newStatusValue, controlValueDict) i+=1 logger.info('Updated the status of ' + str(i) + ' issues for ' + rescan['ComicName'] + ' (' + str(rescan['ComicYear']) + ') that were not found.') logger.info('Total files located: ' + str(havefiles)) foundcount = havefiles arcfiles = 0 # if filechecker returns 0 files (it doesn't find any), but some issues have a status of 'Archived' # the loop below won't work...let's adjust :) arcissues = myDB.action("SELECT count(*) FROM issues WHERE ComicID=? and Status='Archived'", [ComicID]).fetchall() if int(arcissues[0][0]) > 0: arcfiles = arcissues[0][0] havefiles = havefiles + arcfiles logger.fdebug('Adjusting have total to ' + str(havefiles) + ' because of this many archive files:' + str(arcfiles)) ignorecount = 0 if mylar.IGNORE_HAVETOTAL: # if this is enabled, will increase Have total as if in Archived Status ignores = myDB.action("SELECT count(*) FROM issues WHERE ComicID=? AND Status='Ignored'", [ComicID]).fetchall() if int(ignores[0][0]) > 0: ignorecount = ignores[0][0] havefiles = havefiles + ignorecount logger.fdebug('Adjusting have total to ' + str(havefiles) + ' because of this many Ignored files:' + str(ignorecount)) #now that we are finished... #adjust for issues that have been marked as Downloaded, but aren't found/don't exist. #do it here, because above loop only cycles though found comics using filechecker. downissues = myDB.select("SELECT * FROM issues WHERE ComicID=? and Status='Downloaded'", [ComicID]) if downissues is None: pass else: archivedissues = 0 #set this to 0 so it tallies correctly. for down in downissues: #print "downlocation:" + str(down['Location']) #remove special characters from #temploc = rescan['ComicLocation'].replace('_', ' ') #temploc = re.sub('[\#\'\/\.]', '', temploc) #print ("comiclocation: " + str(rescan['ComicLocation'])) #print ("downlocation: " + str(down['Location'])) if down['Location'] is None: logger.fdebug('location does not exist which means file was not downloaded successfully, or was moved.') controlValue = {"IssueID": down['IssueID']} newValue = {"Status": "Archived"} myDB.upsert("issues", newValue, controlValue) archivedissues+=1 pass else: comicpath = os.path.join(rescan['ComicLocation'], down['Location']) if os.path.exists(comicpath): pass #print "Issue exists - no need to change status." else: #print "Changing status from Downloaded to Archived - cannot locate file" controlValue = {"IssueID": down['IssueID']} newValue = {"Status": "Archived"} myDB.upsert("issues", newValue, controlValue) archivedissues+=1 totalarc = arcfiles + archivedissues havefiles = havefiles + archivedissues #arcfiles already tallied in havefiles in above segment logger.fdebug('I have changed the status of ' + str(archivedissues) + ' issues to a status of Archived, as I now cannot locate them in the series directory.') #let's update the total count of comics that was found. controlValueStat = {"ComicID": rescan['ComicID']} newValueStat = {"Have": havefiles } combined_total = rescan['Total'] + anncnt myDB.upsert("comics", newValueStat, controlValueStat) logger.info('I have physically found ' + str(foundcount) + ' issues, ignored ' + str(ignorecount) + ' issues, and accounted for ' + str(totalarc) + ' in an Archived state. Total Issue Count: ' + str(havefiles) + ' / ' + str(combined_total)) return
def forceRescan(ComicID, archive=None): myDB = db.DBConnection() # file check to see if issue exists rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?', [ComicID]).fetchone() logger.info(u"Now checking files for " + rescan['ComicName'] + " (" + str(rescan['ComicYear']) + ") in " + str(rescan['ComicLocation'])) if archive is None: fc = filechecker.listFiles(dir=rescan['ComicLocation'], watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) else: fc = filechecker.listFiles(dir=archive, watchcomic=rescan['ComicName'], AlternateSearch=rescan['AlternateSearch']) iscnt = rescan['Total'] havefiles = 0 fccnt = int(fc['comiccount']) issnum = 1 fcnew = [] fn = 0 issuedupechk = [] issueexceptdupechk = [] reissues = myDB.action('SELECT * FROM issues WHERE ComicID=?', [ComicID]).fetchall() while (fn < fccnt): haveissue = "no" issuedupe = "no" try: tmpfc = fc['comiclist'][fn] except IndexError: break temploc = tmpfc['JusttheDigits'].replace('_', ' ') # temploc = tmpfc['ComicFilename'].replace('_', ' ') temploc = re.sub('[\#\']', '', temploc) logger.fdebug("temploc: " + str(temploc)) if 'annual' not in temploc: #remove the extension here extensions = ('.cbr', '.cbz') if temploc.lower().endswith(extensions): #logger.fdebug("removed extension for issue:" + str(temploc)) temploc = temploc[:-4] deccnt = str(temploc).count('.') if deccnt > 1: #logger.fdebug("decimal counts are :" + str(deccnt)) #if the file is formatted with '.' in place of spaces we need to adjust. #before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE occur = 1 prevstart = 0 digitfound = "no" decimalfound = "no" tempreconstruct = '' while (occur <= deccnt): n = occur start = temploc.find('.') while start >= 0 and n > 1: start = temploc.find('.', start + len('.')) n -= 1 #logger.fdebug("occurance " + str(occur) + " of . at position: " + str(start)) if temploc[prevstart:start].isdigit(): if digitfound == "yes": #logger.fdebug("this is a decimal, assuming decimal issue.") decimalfound = "yes" reconst = "." + temploc[prevstart:start] + " " else: #logger.fdebug("digit detected.") digitfound = "yes" reconst = temploc[prevstart:start] else: reconst = temploc[prevstart:start] + " " #logger.fdebug("word: " + reconst) tempreconstruct = tempreconstruct + reconst #logger.fdebug("tempreconstruct is : " + tempreconstruct) prevstart = (start + 1) occur += 1 #logger.fdebug("word: " + temploc[prevstart:]) tempreconstruct = tempreconstruct + " " + temploc[prevstart:] #logger.fdebug("final filename to use is : " + str(tempreconstruct)) temploc = tempreconstruct #logger.fdebug("checking " + str(temploc)) fcnew = shlex.split(str(temploc)) fcn = len(fcnew) n = 0 while (n <= iscnt): som = 0 try: reiss = reissues[n] except IndexError: break int_iss, iss_except = helpers.decimal_issue( reiss['Issue_Number']) issyear = reiss['IssueDate'][:4] old_status = reiss['Status'] #logger.fdebug("integer_issue:" + str(int_iss) + " ... status: " + str(old_status)) #if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die. #can't distinguish the 5(c2c) to tell it's the issue #... fnd_iss_except = 'None' #print ("Issue, int_iss, iss_except: " + str(reiss['Issue_Number']) + "," + str(int_iss) + "," + str(iss_except)) while (som < fcn): #counts get buggered up when the issue is the last field in the filename - ie. '50.cbr' #logger.fdebug("checking word - " + str(fcnew[som])) if ".cbr" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbr", "") elif ".cbz" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace(".cbz", "") if "(c2c)" in fcnew[som].lower(): fcnew[som] = fcnew[som].replace("(c2c)", " ") get_issue = shlex.split(str(fcnew[som])) if fcnew[som] != " ": fcnew[som] = get_issue[0] if '.' in fcnew[som]: #logger.fdebug("decimal detected...adjusting.") try: i = float(fcnew[som]) except ValueError, TypeError: #not numeric #logger.fdebug("NOT NUMERIC - new word: " + str(fcnew[som])) fcnew[som] = fcnew[som].replace(".", "") else: #numeric pass if fcnew[som].isdigit(): #this won't match on decimal issues - need to fix. #logger.fdebug("digit detected") if int(fcnew[som]) > 0: # fcdigit = fcnew[som].lstrip('0') #fcdigit = str(int(fcnew[som])) fcdigit = int(fcnew[som]) * 1000 if som + 1 < len(fcnew) and 'au' in fcnew[ som + 1].lower(): if len(fcnew[som + 1]) == 2: #if the 'AU' is in 005AU vs 005 AU it will yield different results. fnd_iss_except = 'AU' #logger.info("AU Detected - fnd_iss_except set.") else: #fcdigit = "0" fcdigit = 0 elif "." in fcnew[som]: #this will match on decimal issues IssueChk = fcnew[som] #logger.fdebug("decimal detected...analyzing if issue") isschk_find = IssueChk.find('.') isschk_b4dec = IssueChk[:isschk_find] isschk_decval = IssueChk[isschk_find + 1:] if isschk_b4dec.isdigit(): #logger.fdebug("digit detected prior to decimal.") if isschk_decval.isdigit(): pass #logger.fdebug("digit detected after decimal.") else: #logger.fdebug("not an issue - no digit detected after decimal") break else: #logger.fdebug("not an issue - no digit detected prior to decimal") break #logger.fdebug("IssueNumber: " + str(IssueChk)) #logger.fdebug("..before decimal: " + str(isschk_b4dec)) #logger.fdebug("...after decimal: " + str(isschk_decval)) #--let's make sure we don't wipe out decimal issues ;) if int(isschk_decval) == 0: iss = isschk_b4dec intdec = int(isschk_decval) else: if len(isschk_decval) == 1: iss = isschk_b4dec + "." + isschk_decval intdec = int(isschk_decval) * 10 else: iss = isschk_b4dec + "." + isschk_decval.rstrip( '0') intdec = int(isschk_decval.rstrip('0')) * 10 fcdigit = (int(isschk_b4dec) * 1000) + intdec #logger.fdebug("b4dec: " + str(isschk_b4dec)) #logger.fdebug("decval: " + str(isschk_decval)) #logger.fdebug("intdec: " + str(intdec)) #logger.fdebug("let's compare with this issue value: " + str(fcdigit)) elif 'au' in fcnew[som].lower(): #if AU is part of issue (5AU instead of 5 AU) austart = fcnew[som].lower().find('au') if fcnew[som][:austart].isdigit(): fcdigit = int(fcnew[som][:austart]) * 1000 fnd_iss_except = 'AU' #logger.info("iss_except set to AU") else: # it's a word, skip it. fcdigit = 19283838380101193 #logger.fdebug("fcdigit: " + str(fcdigit)) #logger.fdebug("int_iss: " + str(int_iss)) if "." in str(int_iss): int_iss = helpers.decimal_issue(int_iss) #logger.fdebug("this is the int issue:" + str(int_iss)) #logger.fdebug("this is the fcdigit:" + str(fcdigit)) if int(fcdigit) == int_iss: #logger.fdebug("issue match") #logger.fdebug("fnd_iss_except: " + str(fnd_iss_except)) #logger.fdebug("iss_except: " + str(iss_except)) if str(fnd_iss_except) != 'None' and str( iss_except) == 'AU': if fnd_iss_except.lower() == iss_except.lower(): logger.fdebug("matched for AU") else: logger.fdebug( "this is not an AU match..ignoring result." ) break elif str(fnd_iss_except) == 'None' and str( iss_except) == 'AU': break elif str(fnd_iss_except) == 'AU' and str( iss_except) == 'None': break #if issyear in fcnew[som+1]: # print "matched on year:" + str(issyear) #issuedupechk here. #print ("fcdigit:" + str(fcdigit)) #print ("findiss_except:" + str(fnd_iss_except) + " = iss_except:" + str(iss_except)) #if int(fcdigit) in issuedupechk and str(fnd_iss_except) not in issueexceptdupechk: #str(fnd_iss_except) == str(iss_except): for d in issuedupechk: if int(d['fcdigit']) == int(fcdigit) and d[ 'fnd_iss_except'] == str(fnd_iss_except): logger.fdebug( "duplicate issue detected - not counting this: " + str(tmpfc['ComicFilename'])) issuedupe = "yes" break if issuedupe == "no": logger.fdebug("matched...issue: " + rescan['ComicName'] + "#" + str(reiss['Issue_Number']) + " --- " + str(int_iss)) havefiles += 1 haveissue = "yes" isslocation = str(tmpfc['ComicFilename']) issSize = str(tmpfc['ComicSize']) logger.fdebug(".......filename: " + str(isslocation)) logger.fdebug(".......filesize: " + str(tmpfc['ComicSize'])) # to avoid duplicate issues which screws up the count...let's store the filename issues then # compare earlier... issuedupechk.append({ 'fcdigit': int(fcdigit), 'fnd_iss_except': fnd_iss_except }) break #else: # if the issue # matches, but there is no year present - still match. # determine a way to match on year if present, or no year (currently). som += 1 if haveissue == "yes": break n += 1 #we have the # of comics, now let's update the db. #even if we couldn't find the physical issue, check the status. #if Archived, increase the 'Have' count. if archive: issStatus = "Archived" if haveissue == "no" and issuedupe == "no": isslocation = "None" if old_status == "Skipped": if mylar.AUTOWANT_ALL: issStatus = "Wanted" else: issStatus = "Skipped" elif old_status == "Archived": havefiles += 1 issStatus = "Archived" elif old_status == "Downloaded": issStatus = "Archived" havefiles += 1 elif old_status == "Wanted": issStatus = "Wanted" else: issStatus = "Skipped" controlValueDict = {"IssueID": reiss['IssueID']} newValueDict = {"Status": issStatus} elif haveissue == "yes": issStatus = "Downloaded" controlValueDict = {"IssueID": reiss['IssueID']} newValueDict = { "Location": isslocation, "ComicSize": issSize, "Status": issStatus } myDB.upsert("issues", newValueDict, controlValueDict) fn += 1