def solicit(month, year): #convert to numerics just to ensure this... month = int(month) year = int(year) #print ( "month: " + str(month) ) #print ( "year: " + str(year) ) # in order to gather ALL upcoming - let's start to loop through months going ahead one at a time # until we get a null then break. (Usually not more than 3 months in advance is available) mnloop = 0 upcoming = [] publishers = { 'DC Comics': 'DC Comics', 'DC\'s': 'DC Comics', 'Marvel': 'Marvel Comics', 'Image': 'Image Comics', 'IDW': 'IDW Publishing', 'Dark Horse': 'Dark Horse' } # -- this is no longer needed (testing) # while (mnloop < 5): # if year == 2014: # if len(str(month)) == 1: # month_string = '0' + str(month) # else: # month_string = str(month) # datestring = str(year) + str(month_string) # else: # datestring = str(month) + str(year) # pagelinks = "http://www.comicbookresources.com/tag/solicits" + str(datestring) #using the solicits+datestring leaves out some entries occasionally #should use http://www.comicbookresources.com/tag/solicitations #then just use the logic below but instead of datestring, find the month term and #go ahead up to +5 months. if month > 0: month_start = month month_end = month + 5 #if month_end > 12: # ms = 8, me=13 [(12-8)+(13-12)] = [4 + 1] = 5 # [(12 - ms) + (me - 12)] = number of months (5) monthlist = [] mongr = month_start #we need to build the months we can grab, but the non-numeric way. while (mongr <= month_end): mon = mongr if mon == 13: mon = 1 year += 1 if len(str(mon)) == 1: mon = '0' + str(mon) monthlist.append({ "month": helpers.fullmonth(str(mon)).lower(), "num_month": mon, "year": str(year) }) mongr += 1 logger.info('months: ' + str(monthlist)) pagelinks = "http://www.comicbookresources.com/tag/solicitations" #logger.info('datestring:' + datestring) #logger.info('checking:' + pagelinks) pageresponse = urllib.request.urlopen(pagelinks) soup = BeautifulSoup(pageresponse) cntlinks = soup.findAll('h3') lenlinks = len(cntlinks) #logger.info( str(lenlinks) + ' results' ) publish = [] resultURL = [] resultmonth = [] resultyear = [] x = 0 cnt = 0 while (x < lenlinks): headt = cntlinks[ x] #iterate through the hrefs pulling out only results. if "/?page=article&id=" in str(headt): #print ("titlet: " + str(headt)) headName = headt.findNext(text=True) #print ('headName: ' + headName) if 'Image' in headName: print('IMAGE FOUND') if not all([ 'Marvel' in headName, 'DC' in headName, 'Image' in headName ]) and ('Solicitations' in headName or 'Solicits' in headName): # test for month here (int(month) + 5) if not any( d.get('month', None) == str(headName).lower() for d in monthlist): for mt in monthlist: if mt['month'] in headName.lower(): logger.info('matched on month: ' + str(mt['month'])) logger.info('matched on year: ' + str(mt['year'])) resultmonth.append(mt['num_month']) resultyear.append(mt['year']) pubstart = headName.find('Solicitations') publishchk = False for pub in publishers: if pub in headName[:pubstart]: #print 'publisher:' + str(publishers[pub]) publish.append(publishers[pub]) publishchk = True break if publishchk == False: break #publish.append( headName[:pubstart].strip() ) abc = headt.findAll('a', href=True)[0] ID_som = abc[ 'href'] #first instance will have the right link... resultURL.append(ID_som) #print '(' + str(cnt) + ') [ ' + publish[cnt] + '] Link URL: ' + resultURL[cnt] cnt += 1 else: logger.info('incorrect month - not using.') x += 1 if cnt == 0: return #break # no results means, end it loopthis = (cnt - 1) #this loops through each 'found' solicit page #shipdate = str(month_string) + '-' + str(year) - not needed. while (loopthis >= 0): #print 'loopthis is : ' + str(loopthis) #print 'resultURL is : ' + str(resultURL[loopthis]) shipdate = str(resultmonth[loopthis]) + '-' + str( resultyear[loopthis]) upcoming += populate(resultURL[loopthis], publish[loopthis], shipdate) loopthis -= 1 logger.info(str(len(upcoming)) + ' upcoming issues discovered.') newfl = mylar.CACHE_DIR + "/future-releases.txt" newtxtfile = open(newfl, 'wb') cntr = 1 for row in upcoming: if row['Extra'] is None or row['Extra'] == '': extrarow = 'N/A' else: extrarow = row['Extra'] newtxtfile.write( str(row['Shipdate']) + '\t' + str(row['Publisher']) + '\t' + str(row['Issue']) + '\t' + str(row['Comic']) + '\t' + str(extrarow) + '\tSkipped' + '\t' + str(cntr) + '\n') cntr += 1 newtxtfile.close() logger.fdebug('attempting to populate future upcoming...') mylardb = os.path.join(mylar.DATA_DIR, "mylar.db") connection = sqlite3.connect(str(mylardb)) cursor = connection.cursor() # we should extract the issues that are being watched, but no data is available yet ('Watch For' status) # once we get the data, store it, wipe the existing table, retrieve the new data, populate the data into # the table, recheck the series against the current watchlist and then restore the Watch For data. cursor.executescript('drop table if exists future;') cursor.execute( "CREATE TABLE IF NOT EXISTS future (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, FutureID text, ComicID text);" ) connection.commit() csvfile = open(newfl, "rb") creader = csv.reader(csvfile, delimiter='\t') t = 1 for row in creader: try: #print ("Row: %s" % row) cursor.execute("INSERT INTO future VALUES (?,?,?,?,?,?,?,null);", row) except Exception as e: logger.fdebug("Error - invald arguments...-skipping") pass t += 1 logger.fdebug('successfully added ' + str(t) + ' issues to future upcoming table.') csvfile.close() connection.commit() connection.close() mylar.weeklypull.pullitcheck(futurepull="yes")
myDB.upsert("issues", newValueDict, controlValueDict) except sqlite3.InterfaceError, e: #raise sqlite3.InterfaceError(e) logger.error("Something went wrong - I can't add the issue information into my DB.") myDB.action("DELETE FROM comics WHERE ComicID=?", [comicid]) return n+=1 #figure publish dates here... styear = str(SeriesYear) #if SeriesYear == '0000': # styear = firstdate[:4] if firstdate[5:7] == '00': stmonth = "?" else: stmonth = helpers.fullmonth(firstdate[5:7]) ltyear = re.sub('/s','', latestdate[:4]) if latestdate[5:7] == '00': ltmonth = "?" else: ltmonth = helpers.fullmonth(latestdate[5:7]) #try to determine if it's an 'actively' published comic from above dates #threshold is if it's within a month (<45 days) let's assume it's recent. c_date = datetime.date(int(latestdate[:4]),int(latestdate[5:7]),1) n_date = datetime.date.today() recentchk = (n_date - c_date).days #print ("recentchk: " + str(recentchk)) if recentchk <= 55: lastpubdate = 'Present' else:
def solicit(month, year): #convert to numerics just to ensure this... month = int(month) year = int(year) #print ( "month: " + str(month) ) #print ( "year: " + str(year) ) # in order to gather ALL upcoming - let's start to loop through months going ahead one at a time # until we get a null then break. (Usually not more than 3 months in advance is available) mnloop = 0 upcoming = [] publishers = {'DC Comics':'DC Comics', 'DC\'s': 'DC Comics', 'Marvel':'Marvel Comics', 'Image':'Image Comics', 'IDW':'IDW Publishing', 'Dark Horse':'Dark Horse'} # -- this is no longer needed (testing) # while (mnloop < 5): # if year == 2014: # if len(str(month)) == 1: # month_string = '0' + str(month) # else: # month_string = str(month) # datestring = str(year) + str(month_string) # else: # datestring = str(month) + str(year) # pagelinks = "http://www.comicbookresources.com/tag/solicits" + str(datestring) #using the solicits+datestring leaves out some entries occasionally #should use http://www.comicbookresources.com/tag/solicitations #then just use the logic below but instead of datestring, find the month term and #go ahead up to +5 months. if month > 0: month_start = month month_end = month + 5 #if month_end > 12: # ms = 8, me=13 [(12-8)+(13-12)] = [4 + 1] = 5 # [(12 - ms) + (me - 12)] = number of months (5) monthlist = [] mongr = month_start #we need to build the months we can grab, but the non-numeric way. while (mongr <= month_end): mon = mongr if mon == 13: mon = 1 year +=1 if len(str(mon)) == 1: mon = '0' + str(mon) monthlist.append({"month": helpers.fullmonth(str(mon)).lower(), "num_month": mon, "year": str(year)}) mongr+=1 logger.info('months: ' + str(monthlist)) pagelinks = "http://www.comicbookresources.com/tag/solicitations" #logger.info('datestring:' + datestring) #logger.info('checking:' + pagelinks) pageresponse = urllib2.urlopen ( pagelinks ) soup = BeautifulSoup (pageresponse) cntlinks = soup.findAll('h3') lenlinks = len(cntlinks) #logger.info( str(lenlinks) + ' results' ) publish = [] resultURL = [] resultmonth = [] resultyear = [] x = 0 cnt = 0 while (x < lenlinks): headt = cntlinks[x] #iterate through the hrefs pulling out only results. if "/?page=article&id=" in str(headt): #print ("titlet: " + str(headt)) headName = headt.findNext(text=True) #print ('headName: ' + headName) if 'Image' in headName: print 'IMAGE FOUND' if not all( ['Marvel' in headName, 'DC' in headName, 'Image' in headName] ) and ('Solicitations' in headName or 'Solicits' in headName): # test for month here (int(month) + 5) if not any(d.get('month', None) == str(headName).lower() for d in monthlist): for mt in monthlist: if mt['month'] in headName.lower(): logger.info('matched on month: ' + str(mt['month'])) logger.info('matched on year: ' + str(mt['year'])) resultmonth.append(mt['num_month']) resultyear.append(mt['year']) pubstart = headName.find('Solicitations') publishchk = False for pub in publishers: if pub in headName[:pubstart]: #print 'publisher:' + str(publishers[pub]) publish.append(publishers[pub]) publishchk = True break if publishchk == False: break #publish.append( headName[:pubstart].strip() ) abc = headt.findAll('a', href=True)[0] ID_som = abc['href'] #first instance will have the right link... resultURL.append( ID_som ) #print '(' + str(cnt) + ') [ ' + publish[cnt] + '] Link URL: ' + resultURL[cnt] cnt+=1 else: logger.info('incorrect month - not using.') x+=1 if cnt == 0: return #break # no results means, end it loopthis = (cnt-1) #this loops through each 'found' solicit page #shipdate = str(month_string) + '-' + str(year) - not needed. while ( loopthis >= 0 ): #print 'loopthis is : ' + str(loopthis) #print 'resultURL is : ' + str(resultURL[loopthis]) shipdate = str(resultmonth[loopthis]) + '-' + str(resultyear[loopthis]) upcoming += populate(resultURL[loopthis], publish[loopthis], shipdate) loopthis -=1 logger.info( str(len(upcoming)) + ' upcoming issues discovered.' ) newfl = mylar.CACHE_DIR + "/future-releases.txt" newtxtfile = open(newfl, 'wb') cntr = 1 for row in upcoming: if row['Extra'] is None or row['Extra'] == '': extrarow = 'N/A' else: extrarow = row['Extra'] newtxtfile.write(str(row['Shipdate']) + '\t' + str(row['Publisher']) + '\t' + str(row['Issue']) + '\t' + str(row['Comic']) + '\t' + str(extrarow) + '\tSkipped' + '\t' + str(cntr) + '\n') cntr +=1 newtxtfile.close() logger.fdebug( 'attempting to populate future upcoming...' ) mylardb = os.path.join(mylar.DATA_DIR, "mylar.db") connection = sqlite3.connect(str(mylardb)) cursor = connection.cursor() # we should extract the issues that are being watched, but no data is available yet ('Watch For' status) # once we get the data, store it, wipe the existing table, retrieve the new data, populate the data into # the table, recheck the series against the current watchlist and then restore the Watch For data. cursor.executescript('drop table if exists future;') cursor.execute("CREATE TABLE IF NOT EXISTS future (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, FutureID text, ComicID text);") connection.commit() csvfile = open(newfl, "rb") creader = csv.reader(csvfile, delimiter='\t') t = 1 for row in creader: try: #print ("Row: %s" % row) cursor.execute("INSERT INTO future VALUES (?,?,?,?,?,?,?,null);", row) except Exception, e: logger.fdebug("Error - invald arguments...-skipping") pass t+=1
def Process_next(self,comicid,issueid,issuenumOG,ml=None): annchk = "no" extensions = ('.cbr', '.cbz') snatchedtorrent = False myDB = db.DBConnection() comicnzb = myDB.selectone("SELECT * from comics WHERE comicid=?", [comicid]).fetchone() issuenzb = myDB.selectone("SELECT * from issues WHERE issueid=? AND comicid=? AND ComicName NOT NULL", [issueid,comicid]).fetchone() if ml is not None and mylar.SNATCHEDTORRENT_NOTIFY: snatchnzb = myDB.selectone("SELECT * from snatched WHERE IssueID=? AND ComicID=? AND (provider=? OR provider=?) AND Status='Snatched'", [issueid,comicid,'KAT','CBT']).fetchone() if snatchnzb is None: logger.fdebug('Was not downloaded with Mylar and the usage of torrents. Disabling torrent manual post-processing completion notification.') else: logger.fdebug('Was downloaded from ' + snatchnzb['Provider'] + '. Enabling torrent manual post-processing completion notification.') snatchedtorrent = True logger.fdebug('issueid: ' + str(issueid)) logger.fdebug('issuenumOG: ' + str(issuenumOG)) if issuenzb is None: issuenzb = myDB.selectone("SELECT * from annuals WHERE issueid=? and comicid=?", [issueid,comicid]).fetchone() annchk = "yes" #issueno = str(issuenum).split('.')[0] #new CV API - removed all decimals...here we go AGAIN! issuenum = issuenzb['Issue_Number'] issue_except = 'None' if 'au' in issuenum.lower() and issuenum[:1].isdigit(): issuenum = re.sub("[^0-9]", "", issuenum) issue_except = ' AU' elif 'ai' in issuenum.lower() and issuenum[:1].isdigit(): issuenum = re.sub("[^0-9]", "", issuenum) issue_except = ' AI' elif 'inh' in issuenum.lower() and issuenum[:1].isdigit(): issuenum = re.sub("[^0-9]", "", issuenum) issue_except = '.INH' elif 'now' in issuenum.lower() and issuenum[:1].isdigit(): if '!' in issuenum: issuenum = re.sub('\!', '', issuenum) issuenum = re.sub("[^0-9]", "", issuenum) issue_except = '.NOW' if '.' in issuenum: iss_find = issuenum.find('.') iss_b4dec = issuenum[:iss_find] iss_decval = issuenum[iss_find+1:] if int(iss_decval) == 0: iss = iss_b4dec issdec = int(iss_decval) issueno = str(iss) self._log("Issue Number: " + str(issueno)) logger.fdebug("Issue Number: " + str(issueno)) else: if len(iss_decval) == 1: iss = iss_b4dec + "." + iss_decval issdec = int(iss_decval) * 10 else: iss = iss_b4dec + "." + iss_decval.rstrip('0') issdec = int(iss_decval.rstrip('0')) * 10 issueno = iss_b4dec self._log("Issue Number: " + str(iss)) logger.fdebug("Issue Number: " + str(iss)) else: iss = issuenum issueno = str(iss) # issue zero-suppression here if mylar.ZERO_LEVEL == "0": zeroadd = "" else: if mylar.ZERO_LEVEL_N == "none": zeroadd = "" elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0" elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00" logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N)) if str(len(issueno)) > 1: if int(issueno) < 0: self._log("issue detected is a negative") prettycomiss = '-' + str(zeroadd) + str(abs(issueno)) elif int(issueno) < 10: self._log("issue detected less than 10") if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(zeroadd) + str(iss) else: prettycomiss = str(zeroadd) + str(int(issueno)) else: prettycomiss = str(zeroadd) + str(iss) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss)) elif int(issueno) >= 10 and int(issueno) < 100: self._log("issue detected greater than 10, but less than 100") if mylar.ZERO_LEVEL_N == "none": zeroadd = "" else: zeroadd = "0" if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(zeroadd) + str(iss) else: prettycomiss = str(zeroadd) + str(int(issueno)) else: prettycomiss = str(zeroadd) + str(iss) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss)) else: self._log("issue detected greater than 100") if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(issueno) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss)) else: prettycomiss = str(issueno) self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss)) if annchk == "yes": self._log("Annual detected.") logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss)) issueyear = issuenzb['IssueDate'][:4] self._log("Issue Year: " + str(issueyear)) logger.fdebug("Issue Year : " + str(issueyear)) month = issuenzb['IssueDate'][5:7].replace('-','').strip() month_name = helpers.fullmonth(month) # comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone() publisher = comicnzb['ComicPublisher'] self._log("Publisher: " + publisher) logger.fdebug("Publisher: " + str(publisher)) #we need to un-unicode this to make sure we can write the filenames properly for spec.chars series = comicnzb['ComicName'].encode('ascii', 'ignore').strip() self._log("Series: " + series) logger.fdebug("Series: " + str(series)) seriesyear = comicnzb['ComicYear'] self._log("Year: " + seriesyear) logger.fdebug("Year: " + str(seriesyear)) comlocation = comicnzb['ComicLocation'] self._log("Comic Location: " + comlocation) logger.fdebug("Comic Location: " + str(comlocation)) comversion = comicnzb['ComicVersion'] self._log("Comic Version: " + str(comversion)) logger.fdebug("Comic Version: " + str(comversion)) if comversion is None: comversion = 'None' #if comversion is None, remove it so it doesn't populate with 'None' if comversion == 'None': chunk_f_f = re.sub('\$VolumeN','',mylar.FILE_FORMAT) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) self._log("No version # found for series - tag will not be available for renaming.") logger.fdebug("No version # found for series, removing from filename") logger.fdebug("new format is now: " + str(chunk_file_format)) else: chunk_file_format = mylar.FILE_FORMAT if annchk == "no": chunk_f_f = re.sub('\$Annual','',chunk_file_format) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) logger.fdebug('not an annual - removing from filename paramaters') logger.fdebug('new format: ' + str(chunk_file_format)) else: logger.fdebug('chunk_file_format is: ' + str(chunk_file_format)) if '$Annual' not in chunk_file_format: #if it's an annual, but $Annual isn't specified in file_format, we need to #force it in there, by default in the format of $Annual $Issue prettycomiss = "Annual " + str(prettycomiss) logger.fdebug('prettycomiss: ' + str(prettycomiss)) ofilename = None #if meta-tagging is not enabled, we need to declare the check as being fail #if meta-tagging is enabled, it gets changed just below to a default of pass pcheck = "fail" #tag the meta. if mylar.ENABLE_META: self._log("Metatagging enabled - proceeding...") logger.fdebug("Metatagging enabled - proceeding...") pcheck = "pass" try: import cmtagmylar if ml is None: pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid) else: pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid, manual="yes", filename=ml['ComicLocation']) except ImportError: logger.fdebug("comictaggerlib not found on system. Ensure the ENTIRE lib directory is located within mylar/lib/comictaggerlib/") logger.fdebug("continuing with PostProcessing, but I'm not using metadata.") pcheck = "fail" if pcheck == "fail": self._log("Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging...") logger.fdebug("Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging...") elif pcheck == "unrar error": self._log("This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy.") logger.error("This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy.") return self.log else: otofilename = pcheck self._log("Sucessfully wrote metadata to .cbz - Continuing..") logger.fdebug("Sucessfully wrote metadata to .cbz (" + str(otofilename) + ") - Continuing..") #Run Pre-script if mylar.ENABLE_PRE_SCRIPTS: nzbn = self.nzb_name #original nzb name nzbf = self.nzb_folder #original nzb folder #name, comicyear, comicid , issueid, issueyear, issue, publisher #create the dic and send it. seriesmeta = [] seriesmetadata = {} seriesmeta.append({ 'name': series, 'comicyear': seriesyear, 'comicid': comicid, 'issueid': issueid, 'issueyear': issueyear, 'issue': issuenum, 'publisher': publisher }) seriesmetadata['seriesmeta'] = seriesmeta self._run_pre_scripts(nzbn, nzbf, seriesmetadata ) #rename file and move to new path #nfilename = series + " " + issueno + " (" + seriesyear + ")" file_values = {'$Series': series, '$Issue': prettycomiss, '$Year': issueyear, '$series': series.lower(), '$Publisher': publisher, '$publisher': publisher.lower(), '$VolumeY': 'V' + str(seriesyear), '$VolumeN': comversion, '$monthname': month_name, '$month': month, '$Annual': 'Annual' } #if it's a Manual Run, use the ml['ComicLocation'] for the exact filename. if ml is None: for root, dirnames, filenames in os.walk(self.nzb_folder): for filename in filenames: if filename.lower().endswith(extensions): odir = root ofilename = filename path, ext = os.path.splitext(ofilename) if odir is None: logger.fdebug('no root folder set.') odir = self.nzb_folder logger.fdebug('odir: ' + str(odir)) logger.fdebug('ofilename: ' + str(ofilename)) else: if pcheck == "fail": otofilename = ml['ComicLocation'] logger.fdebug('otofilename:' + str(otofilename)) odir, ofilename = os.path.split(otofilename) logger.fdebug('odir: ' + str(odir)) logger.fdebug('ofilename: ' + str(ofilename)) path, ext = os.path.splitext(ofilename) logger.fdebug('path: ' + str(path)) logger.fdebug('ext:' + str(ext)) if ofilename is None: logger.error(u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that " + str(self.nzb_folder) + " exists and is the correct location.") return self._log("Original Filename: " + ofilename) self._log("Original Extension: " + ext) logger.fdebug("Original Filname: " + str(ofilename)) logger.fdebug("Original Extension: " + str(ext)) if mylar.FILE_FORMAT == '' or not mylar.RENAME_FILES: self._log("Rename Files isn't enabled...keeping original filename.") logger.fdebug("Rename Files isn't enabled - keeping original filename.") #check if extension is in nzb_name - will screw up otherwise if ofilename.lower().endswith(extensions): nfilename = ofilename[:-4] else: nfilename = ofilename else: nfilename = helpers.replace_all(chunk_file_format, file_values) if mylar.REPLACE_SPACES: #mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR) nfilename = re.sub('[\,\:\?]', '', nfilename) nfilename = re.sub('[\/]', '-', nfilename) self._log("New Filename: " + nfilename) logger.fdebug("New Filename: " + str(nfilename)) #src = os.path.join(self.nzb_folder, ofilename) src = os.path.join(odir, ofilename) filechecker.validateAndCreateDirectory(comlocation, True) if mylar.LOWERCASE_FILENAMES: dst = (comlocation + "/" + nfilename + ext).lower() else: dst = comlocation + "/" + nfilename + ext.lower() self._log("Source:" + src) self._log("Destination:" + dst) logger.fdebug("Source: " + str(src)) logger.fdebug("Destination: " + str(dst)) if ml is None: #downtype = for use with updater on history table to set status to 'Downloaded' downtype = 'True' #non-manual run moving/deleting... logger.fdebug('self.nzb_folder: ' + self.nzb_folder) logger.fdebug('odir: ' + str(odir)) logger.fdebug('ofilename:' + str(ofilename)) logger.fdebug('nfilename:' + str(nfilename + ext)) if mylar.RENAME_FILES: if str(ofilename) != str(nfilename + ext): logger.fdebug("Renaming " + os.path.join(odir, str(ofilename)) + " ..to.. " + os.path.join(odir,str(nfilename + ext))) os.rename(os.path.join(odir, str(ofilename)), os.path.join(odir,str(nfilename + ext))) else: logger.fdebug('filename is identical as original, not renaming.') #src = os.path.join(self.nzb_folder, str(nfilename + ext)) src = os.path.join(odir, str(nfilename + ext)) try: shutil.move(src, dst) except (OSError, IOError): self._log("Failed to move directory - check directories and manually re-run.") self._log("Post-Processing ABORTED.") return #tidyup old path try: shutil.rmtree(self.nzb_folder) except (OSError, IOError): self._log("Failed to remove temporary directory - check directory and manually re-run.") self._log("Post-Processing ABORTED.") return self._log("Removed temporary directory : " + str(self.nzb_folder)) else: #downtype = for use with updater on history table to set status to 'Post-Processed' downtype = 'PP' #Manual Run, this is the portion. if mylar.RENAME_FILES: if str(ofilename) != str(nfilename + ext): logger.fdebug("Renaming " + os.path.join(self.nzb_folder, str(ofilename)) + " ..to.. " + os.path.join(self.nzb_folder,str(nfilename + ext))) os.rename(os.path.join(odir, str(ofilename)), os.path.join(odir ,str(nfilename + ext))) else: logger.fdebug('filename is identical as original, not renaming.') src = os.path.join(odir, str(nfilename + ext)) logger.fdebug('odir rename: ' + os.path.join(odir, str(ofilename)) + ' TO ' + os.path.join(odir, str(nfilename + ext))) logger.fdebug('odir src : ' + os.path.join(odir, str(nfilename + ext))) logger.fdebug("Moving " + src + " ... to ... " + dst) try: shutil.move(src, dst) except (OSError, IOError): logger.fdebug("Failed to move directory - check directories and manually re-run.") logger.fdebug("Post-Processing ABORTED.") return logger.fdebug("Successfully moved to : " + dst) #tidyup old path #try: # os.remove(os.path.join(self.nzb_folder, str(ofilename))) # logger.fdebug("Deleting : " + os.path.join(self.nzb_folder, str(ofilename))) #except (OSError, IOError): # logger.fdebug("Failed to remove temporary directory - check directory and manually re-run.") # logger.fdebug("Post-Processing ABORTED.") # return #logger.fdebug("Removed temporary directory : " + str(self.nzb_folder)) #Hopefully set permissions on downloaded file try: permission = int(mylar.CHMOD_FILE, 8) os.umask(0) os.chmod(dst.rstrip(), permission) except OSError: logger.error('Failed to change file permissions. Ensure that the user running Mylar has proper permissions to change permissions in : ' + dst) logger.fdebug('Continuing post-processing but unable to change file permissions in ' + dst) #delete entry from nzblog table myDB.action('DELETE from nzblog WHERE issueid=?', [issueid]) #update snatched table to change status to Downloaded if annchk == "no": updater.foundsearch(comicid, issueid, down=downtype) dispiss = 'issue: ' + str(issuenumOG) else: updater.foundsearch(comicid, issueid, mode='want_ann', down=downtype) dispiss = 'annual issue: ' + str(issuenumOG) #force rescan of files updater.forceRescan(comicid) logger.info(u"Post-Processing completed for: " + series + " " + dispiss ) self._log(u"Post Processing SUCCESSFUL! ") if mylar.WEEKFOLDER: #if enabled, will *copy* the post-processed file to the weeklypull list folder for the given week. weeklypull.weekly_singlecopy(comicid,issuenum,str(nfilename+ext),dst) # retrieve/create the corresponding comic objects if mylar.ENABLE_EXTRA_SCRIPTS: folderp = str(dst) #folder location after move/rename nzbn = self.nzb_name #original nzb name filen = str(nfilename + ext) #new filename #name, comicyear, comicid , issueid, issueyear, issue, publisher #create the dic and send it. seriesmeta = [] seriesmetadata = {} seriesmeta.append({ 'name': series, 'comicyear': seriesyear, 'comicid': comicid, 'issueid': issueid, 'issueyear': issueyear, 'issue': issuenum, 'publisher': publisher }) seriesmetadata['seriesmeta'] = seriesmeta self._run_extra_scripts(nzbn, self.nzb_folder, filen, folderp, seriesmetadata ) if ml is not None: #we only need to return self.log if it's a manual run and it's not a snatched torrent if snatchedtorrent: #manual run + snatched torrent pass else: #manual run + not snatched torrent (or normal manual-run) return self.log if annchk == "no": prline = series + '(' + issueyear + ') - issue #' + issuenumOG else: prline = series + ' Annual (' + issueyear + ') - issue #' + issuenumOG prline2 = 'Mylar has downloaded and post-processed: ' + prline if mylar.PROWL_ENABLED: pushmessage = prline logger.info(u"Prowl request") prowl = notifiers.PROWL() prowl.notify(pushmessage,"Download and Postprocessing completed") if mylar.NMA_ENABLED: nma = notifiers.NMA() nma.notify(prline=prline, prline2=prline2) if mylar.PUSHOVER_ENABLED: logger.info(u"Pushover request") pushover = notifiers.PUSHOVER() pushover.notify(prline, "Download and Post-Processing completed") if mylar.BOXCAR_ENABLED: boxcar = notifiers.BOXCAR() boxcar.notify(prline=prline, prline2=prline2) if mylar.PUSHBULLET_ENABLED: pushbullet = notifiers.PUSHBULLET() pushbullet.notify(prline=prline, prline2=prline2) return self.log
def rename_file(self, ofilename, issue=None, annualize=None, arc=False, file_format=None): #comicname, issue, comicyear=None, issueid=None) comicid = self.comicid # it's coming in unicoded... issueid = self.issueid if file_format is None: file_format = mylar.CONFIG.FILE_FORMAT logger.fdebug(type(comicid)) logger.fdebug(type(issueid)) logger.fdebug('comicid: %s' % comicid) logger.fdebug('issue# as per cv: %s' % issue) logger.fdebug('issueid:' + str(issueid)) if issueid is None: logger.fdebug('annualize is ' + str(annualize)) if arc: #this has to be adjusted to be able to include story arc issues that span multiple arcs chkissue = self.myDB.selectone("SELECT * from storyarcs WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone() else: chkissue = self.myDB.selectone("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone() if all([chkissue is None, annualize is None, not mylar.CONFIG.ANNUALS_ON]): chkissue = self.myDB.selectone("SELECT * from annuals WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone() if chkissue is None: #rechk chkissue against int value of issue # if arc: chkissue = self.myDB.selectone("SELECT * from storyarcs WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone() else: chkissue = self.myDB.selectone("SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone() if all([chkissue is None, annualize == 'yes', mylar.CONFIG.ANNUALS_ON]): chkissue = self.myDB.selectone("SELECT * from annuals WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone() if chkissue is None: logger.error('Invalid Issue_Number - please validate.') return else: logger.info('Int Issue_number compare found. continuing...') issueid = chkissue['IssueID'] else: issueid = chkissue['IssueID'] #use issueid to get publisher, series, year, issue number logger.fdebug('issueid is now : ' + str(issueid)) if arc: issueinfo = self.myDB.selectone("SELECT * from storyarcs WHERE ComicID=? AND IssueID=? AND StoryArc=?", [comicid, issueid, arc]).fetchone() else: issueinfo = self.myDB.selectone("SELECT * from issues WHERE ComicID=? AND IssueID=?", [comicid, issueid]).fetchone() if issueinfo is None: logger.fdebug('not an issue, checking against annuals') issueinfo = self.myDB.selectone("SELECT * from annuals WHERE ComicID=? AND IssueID=?", [comicid, issueid]).fetchone() if issueinfo is None: logger.fdebug('Unable to rename - cannot locate issue id within db') return else: annualize = True if issueinfo is None: logger.fdebug('Unable to rename - cannot locate issue id within db') return #remap the variables to a common factor. if arc: issuenum = issueinfo['IssueNumber'] issuedate = issueinfo['IssueDate'] publisher = issueinfo['IssuePublisher'] series = issueinfo['ComicName'] seriesfilename = series #Alternate FileNaming is not available with story arcs. seriesyear = issueinfo['SeriesYear'] arcdir = helpers.filesafe(issueinfo['StoryArc']) if mylar.CONFIG.REPLACE_SPACES: arcdir = arcdir.replace(' ', mylar.CONFIG.REPLACE_CHAR) if mylar.CONFIG.STORYARCDIR: storyarcd = os.path.join(mylar.CONFIG.DESTINATION_DIR, "StoryArcs", arcdir) logger.fdebug('Story Arc Directory set to : ' + storyarcd) else: logger.fdebug('Story Arc Directory set to : ' + mylar.CONFIG.GRABBAG_DIR) storyarcd = os.path.join(mylar.CONFIG.DESTINATION_DIR, mylar.CONFIG.GRABBAG_DIR) comlocation = storyarcd comversion = None #need to populate this. else: issuenum = issueinfo['Issue_Number'] issuedate = issueinfo['IssueDate'] publisher = self.comic['ComicPublisher'] series = self.comic['ComicName'] if self.comic['AlternateFileName'] is None or self.comic['AlternateFileName'] == 'None': seriesfilename = series else: seriesfilename = self.comic['AlternateFileName'] logger.fdebug('Alternate File Naming has been enabled for this series. Will rename series title to : ' + seriesfilename) seriesyear = self.comic['ComicYear'] comlocation = self.comic['ComicLocation'] comversion = self.comic['ComicVersion'] unicodeissue = issuenum if type(issuenum) == str: vals = {'\xbd':'.5','\xbc':'.25','\xbe':'.75','\u221e':'9999999999','\xe2':'9999999999'} else: vals = {'\xbd':'.5','\xbc':'.25','\xbe':'.75','\\u221e':'9999999999','\xe2':'9999999999'} x = [vals[key] for key in vals if key in issuenum] if x: issuenum = x[0] logger.fdebug('issue number formatted: %s' % issuenum) #comicid = issueinfo['ComicID'] #issueno = str(issuenum).split('.')[0] issue_except = 'None' issue_exceptions = ['AU', 'INH', 'NOW', 'AI', 'MU', 'A', 'B', 'C', 'X', 'O'] valid_spaces = ('.', '-') for issexcept in issue_exceptions: if issexcept.lower() in issuenum.lower(): logger.fdebug('ALPHANUMERIC EXCEPTION : [' + issexcept + ']') v_chk = [v for v in valid_spaces if v in issuenum] if v_chk: iss_space = v_chk[0] logger.fdebug('character space denoted as : ' + iss_space) else: logger.fdebug('character space not denoted.') iss_space = '' # if issexcept == 'INH': # issue_except = '.INH' if issexcept == 'NOW': if '!' in issuenum: issuenum = re.sub('\!', '', issuenum) # issue_except = '.NOW' issue_except = iss_space + issexcept logger.fdebug('issue_except denoted as : ' + issue_except) issuenum = re.sub("[^0-9]", "", issuenum) break # if 'au' in issuenum.lower() and issuenum[:1].isdigit(): # issue_except = ' AU' # elif 'ai' in issuenum.lower() and issuenum[:1].isdigit(): # issuenum = re.sub("[^0-9]", "", issuenum) # issue_except = ' AI' # elif 'inh' in issuenum.lower() and issuenum[:1].isdigit(): # issuenum = re.sub("[^0-9]", "", issuenum) # issue_except = '.INH' # elif 'now' in issuenum.lower() and issuenum[:1].isdigit(): # if '!' in issuenum: issuenum = re.sub('\!', '', issuenum) # issuenum = re.sub("[^0-9]", "", issuenum) # issue_except = '.NOW' if '.' in issuenum: iss_find = issuenum.find('.') iss_b4dec = issuenum[:iss_find] if iss_find == 0: iss_b4dec = '0' iss_decval = issuenum[iss_find +1:] if iss_decval.endswith('.'): iss_decval = iss_decval[:-1] if int(iss_decval) == 0: iss = iss_b4dec issdec = int(iss_decval) issueno = iss else: if len(iss_decval) == 1: iss = iss_b4dec + "." + iss_decval issdec = int(iss_decval) * 10 else: iss = iss_b4dec + "." + iss_decval.rstrip('0') issdec = int(iss_decval.rstrip('0')) * 10 issueno = iss_b4dec else: iss = issuenum issueno = iss # issue zero-suppression here if mylar.CONFIG.ZERO_LEVEL == "0": zeroadd = "" else: if mylar.CONFIG.ZERO_LEVEL_N == "none": zeroadd = "" elif mylar.CONFIG.ZERO_LEVEL_N == "0x": zeroadd = "0" elif mylar.CONFIG.ZERO_LEVEL_N == "00x": zeroadd = "00" logger.fdebug('Zero Suppression set to : ' + str(mylar.CONFIG.ZERO_LEVEL_N)) prettycomiss = None if issueno.isalpha(): logger.fdebug('issue detected as an alpha.') prettycomiss = str(issueno) else: try: x = float(issuenum) #validity check if x < 0: logger.info('I\'ve encountered a negative issue #: %s. Trying to accomodate.' % issueno) prettycomiss = '-' + str(zeroadd) + str(issueno[1:]) elif x == 9999999999: logger.fdebug('Infinity issue found.') issuenum = 'infinity' elif x >= 0: pass else: raise ValueError except ValueError as e: logger.warn('Unable to properly determine issue number [ %s] - you should probably log this on github for help.' % issueno) return if prettycomiss is None and len(str(issueno)) > 0: #if int(issueno) < 0: # self._log("issue detected is a negative") # prettycomiss = '-' + str(zeroadd) + str(abs(issueno)) if int(issueno) < 10: logger.fdebug('issue detected less than 10') if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(zeroadd) + str(iss) else: prettycomiss = str(zeroadd) + str(int(issueno)) else: prettycomiss = str(zeroadd) + str(iss) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '. Issue will be set as : ' + str(prettycomiss)) elif int(issueno) >= 10 and int(issueno) < 100: logger.fdebug('issue detected greater than 10, but less than 100') if mylar.CONFIG.ZERO_LEVEL_N == "none": zeroadd = "" else: zeroadd = "0" if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(zeroadd) + str(iss) else: prettycomiss = str(zeroadd) + str(int(issueno)) else: prettycomiss = str(zeroadd) + str(iss) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '.Issue will be set as : ' + str(prettycomiss)) else: logger.fdebug('issue detected greater than 100') if issuenum == 'infinity': prettycomiss = 'infinity' else: if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(issueno) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '. Issue will be set as : ' + str(prettycomiss)) elif len(str(issueno)) == 0: prettycomiss = str(issueno) logger.fdebug('issue length error - cannot determine length. Defaulting to None: ' + str(prettycomiss)) logger.fdebug('Pretty Comic Issue is : ' + str(prettycomiss)) if mylar.CONFIG.UNICODE_ISSUENUMBER: logger.fdebug('Setting this to Unicode format as requested: %s' % prettycomiss) prettycomiss = unicodeissue issueyear = issuedate[:4] month = issuedate[5:7].replace('-', '').strip() month_name = helpers.fullmonth(month) if month_name is None: month_name = 'None' logger.fdebug('Issue Year : ' + str(issueyear)) logger.fdebug('Publisher: ' + publisher) logger.fdebug('Series: ' + series) logger.fdebug('Year: ' + str(seriesyear)) logger.fdebug('Comic Location: ' + comlocation) if self.comic['Corrected_Type'] is not None: if self.comic['Type'] != self.comic['Corrected_Type']: booktype = self.comic['Corrected_Type'] else: booktype = self.comic['Type'] else: booktype = self.comic['Type'] if booktype == 'Print' or all([booktype != 'Print', mylar.CONFIG.FORMAT_BOOKTYPE is False]): chunk_fb = re.sub('\$Type', '', file_format) chunk_b = re.compile(r'\s+') chunk_file_format = chunk_b.sub(' ', chunk_fb) else: chunk_file_format = file_format if any([comversion is None, booktype != 'Print']): comversion = 'None' #if comversion is None, remove it so it doesn't populate with 'None' if comversion == 'None': chunk_f_f = re.sub('\$VolumeN', '', chunk_file_format) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) logger.fdebug('No version # found for series, removing from filename') logger.fdebug("new format: " + str(chunk_file_format)) if annualize is None: chunk_f_f = re.sub('\$Annual', '', chunk_file_format) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) logger.fdebug('not an annual - removing from filename paramaters') logger.fdebug('new format: ' + str(chunk_file_format)) else: logger.fdebug('chunk_file_format is: ' + str(chunk_file_format)) if mylar.CONFIG.ANNUALS_ON: if 'annual' in series.lower(): if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower(): #if it's an annual, but $annual isn't specified in file_format, we need to #force it in there, by default in the format of $Annual $Issue #prettycomiss = "Annual " + str(prettycomiss) logger.fdebug('[%s][ANNUALS-ON][ANNUAL IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: #because it exists within title, strip it then use formatting tag for placement of wording. chunk_f_f = re.sub('\$Annual', '', chunk_file_format) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) logger.fdebug('[%s][ANNUALS-ON][ANNUAL IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower(): #if it's an annual, but $annual isn't specified in file_format, we need to #force it in there, by default in the format of $Annual $Issue prettycomiss = "Annual %s" % prettycomiss logger.fdebug('[%s][ANNUALS-ON][ANNUAL NOT IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: logger.fdebug('[%s][ANNUALS-ON][ANNUAL NOT IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: #if annuals aren't enabled, then annuals are being tracked as independent series. #annualize will be true since it's an annual in the seriesname. if 'annual' in series.lower(): if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower(): #if it's an annual, but $annual isn't specified in file_format, we need to #force it in there, by default in the format of $Annual $Issue #prettycomiss = "Annual " + str(prettycomiss) logger.fdebug('[%s][ANNUALS-OFF][ANNUAL IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: #because it exists within title, strip it then use formatting tag for placement of wording. chunk_f_f = re.sub('\$Annual', '', chunk_file_format) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) logger.fdebug('[%s][ANNUALS-OFF][ANNUAL IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower(): #if it's an annual, but $annual isn't specified in file_format, we need to #force it in there, by default in the format of $Annual $Issue prettycomiss = "Annual %s" % prettycomiss logger.fdebug('[%s][ANNUALS-OFF][ANNUAL NOT IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: logger.fdebug('[%s][ANNUALS-OFF][ANNUAL NOT IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) logger.fdebug('Annual detected within series title of ' + series + '. Not auto-correcting issue #') seriesfilename = seriesfilename #.encode('ascii', 'ignore').strip() filebad = [':', ',', '/', '?', '!', '\'', '\"', '\*'] #in u_comicname or '/' in u_comicname or ',' in u_comicname or '?' in u_comicname: for dbd in filebad: if dbd in seriesfilename: if any([dbd == '/', dbd == '*']): repthechar = '-' else: repthechar = '' seriesfilename = seriesfilename.replace(dbd, repthechar) logger.fdebug('Altering series name due to filenaming restrictions: ' + seriesfilename) publisher = re.sub('!', '', publisher) file_values = {'$Series': seriesfilename, '$Issue': prettycomiss, '$Year': issueyear, '$series': series.lower(), '$Publisher': publisher, '$publisher': publisher.lower(), '$VolumeY': 'V' + str(seriesyear), '$VolumeN': comversion, '$monthname': month_name, '$month': month, '$Annual': 'Annual', '$Type': booktype } extensions = ('.cbr', '.cbz', '.cb7') if ofilename.lower().endswith(extensions): path, ext = os.path.splitext(ofilename) if file_format == '': logger.fdebug('Rename Files is not enabled - keeping original filename.') #check if extension is in nzb_name - will screw up otherwise if ofilename.lower().endswith(extensions): nfilename = ofilename[:-4] else: nfilename = ofilename else: chunk_file_format = re.sub('[()|[]]', '', chunk_file_format).strip() nfilename = helpers.replace_all(chunk_file_format, file_values) if mylar.CONFIG.REPLACE_SPACES: #mylar.CONFIG.REPLACE_CHAR ...determines what to replace spaces with underscore or dot nfilename = nfilename.replace(' ', mylar.CONFIG.REPLACE_CHAR) nfilename = re.sub('[\,\:]', '', nfilename) + ext.lower() logger.fdebug('New Filename: ' + nfilename) if mylar.CONFIG.LOWERCASE_FILENAMES: nfilename = nfilename.lower() dst = os.path.join(comlocation, nfilename) else: dst = os.path.join(comlocation, nfilename) logger.fdebug('Source: ' + ofilename) logger.fdebug('Destination: ' + dst) rename_this = {"destination_dir": dst, "nfilename": nfilename, "issueid": issueid, "comicid": comicid} return rename_this
logger.error( "Something went wrong - I can't add the issue information into my DB." ) myDB.action("DELETE FROM comics WHERE ComicID=?", [comicid]) return n += 1 #figure publish dates here... styear = str(SeriesYear) #if SeriesYear == '0000': # styear = firstdate[:4] if firstdate[5:7] == '00': stmonth = "?" else: stmonth = helpers.fullmonth(firstdate[5:7]) ltyear = re.sub('/s', '', latestdate[:4]) if latestdate[5:7] == '00': ltmonth = "?" else: ltmonth = helpers.fullmonth(latestdate[5:7]) #try to determine if it's an 'actively' published comic from above dates #threshold is if it's within a month (<45 days) let's assume it's recent. c_date = datetime.date(int(latestdate[:4]), int(latestdate[5:7]), 1) n_date = datetime.date.today() recentchk = (n_date - c_date).days #print ("recentchk: " + str(recentchk)) if recentchk <= 55: lastpubdate = 'Present' else:
def Process_next(self, comicid, issueid, issuenumOG, ml=None): annchk = "no" extensions = ('.cbr', '.cbz') myDB = db.DBConnection() comicnzb = myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone() issuenzb = myDB.action( "SELECT * from issues WHERE issueid=? AND comicid=? AND ComicName NOT NULL", [issueid, comicid]).fetchone() logger.fdebug('issueid: ' + str(issueid)) logger.fdebug('issuenumOG: ' + str(issuenumOG)) if issuenzb is None: issuenzb = myDB.action( "SELECT * from annuals WHERE issueid=? and comicid=?", [issueid, comicid]).fetchone() annchk = "yes" #issueno = str(issuenum).split('.')[0] #new CV API - removed all decimals...here we go AGAIN! issuenum = issuenzb['Issue_Number'] issue_except = 'None' if 'au' in issuenum.lower() and issuenum[:1].isdigit(): issuenum = re.sub("[^0-9]", "", issuenum) issue_except = ' AU' elif 'ai' in issuenum.lower() and issuenum[:1].isdigit(): issuenum = re.sub("[^0-9]", "", issuenum) issue_except = ' AI' elif 'inh' in issuenum.lower() and issuenum[:1].isdigit(): issuenum = re.sub("[^0-9]", "", issuenum) issue_except = '.INH' elif 'now' in issuenum.lower() and issuenum[:1].isdigit(): if '!' in issuenum: issuenum = re.sub('\!', '', issuenum) issuenum = re.sub("[^0-9]", "", issuenum) issue_except = '.NOW' if '.' in issuenum: iss_find = issuenum.find('.') iss_b4dec = issuenum[:iss_find] iss_decval = issuenum[iss_find + 1:] if int(iss_decval) == 0: iss = iss_b4dec issdec = int(iss_decval) issueno = str(iss) self._log("Issue Number: " + str(issueno), logger.DEBUG) logger.fdebug("Issue Number: " + str(issueno)) else: if len(iss_decval) == 1: iss = iss_b4dec + "." + iss_decval issdec = int(iss_decval) * 10 else: iss = iss_b4dec + "." + iss_decval.rstrip('0') issdec = int(iss_decval.rstrip('0')) * 10 issueno = iss_b4dec self._log("Issue Number: " + str(iss), logger.DEBUG) logger.fdebug("Issue Number: " + str(iss)) else: iss = issuenum issueno = str(iss) # issue zero-suppression here if mylar.ZERO_LEVEL == "0": zeroadd = "" else: if mylar.ZERO_LEVEL_N == "none": zeroadd = "" elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0" elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00" logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N)) if str(len(issueno)) > 1: if int(issueno) < 10: self._log("issue detected less than 10", logger.DEBUG) if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(zeroadd) + str(iss) else: prettycomiss = str(zeroadd) + str(int(issueno)) else: prettycomiss = str(zeroadd) + str(iss) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except self._log( "Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG) elif int(issueno) >= 10 and int(issueno) < 100: self._log("issue detected greater than 10, but less than 100", logger.DEBUG) if mylar.ZERO_LEVEL_N == "none": zeroadd = "" else: zeroadd = "0" if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(zeroadd) + str(iss) else: prettycomiss = str(zeroadd) + str(int(issueno)) else: prettycomiss = str(zeroadd) + str(iss) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except self._log( "Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss), logger.DEBUG) else: self._log("issue detected greater than 100", logger.DEBUG) if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(issueno) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except self._log( "Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG) else: prettycomiss = str(issueno) self._log( "issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss), logger.DEBUG) if annchk == "yes": self._log("Annual detected.") logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss)) issueyear = issuenzb['IssueDate'][:4] self._log("Issue Year: " + str(issueyear), logger.DEBUG) logger.fdebug("Issue Year : " + str(issueyear)) month = issuenzb['IssueDate'][5:7].replace('-', '').strip() month_name = helpers.fullmonth(month) # comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone() publisher = comicnzb['ComicPublisher'] self._log("Publisher: " + publisher, logger.DEBUG) logger.fdebug("Publisher: " + str(publisher)) #we need to un-unicode this to make sure we can write the filenames properly for spec.chars series = comicnzb['ComicName'].encode('ascii', 'ignore').strip() self._log("Series: " + series, logger.DEBUG) logger.fdebug("Series: " + str(series)) seriesyear = comicnzb['ComicYear'] self._log("Year: " + seriesyear, logger.DEBUG) logger.fdebug("Year: " + str(seriesyear)) comlocation = comicnzb['ComicLocation'] self._log("Comic Location: " + comlocation, logger.DEBUG) logger.fdebug("Comic Location: " + str(comlocation)) comversion = comicnzb['ComicVersion'] self._log("Comic Version: " + str(comversion), logger.DEBUG) logger.fdebug("Comic Version: " + str(comversion)) if comversion is None: comversion = 'None' #if comversion is None, remove it so it doesn't populate with 'None' if comversion == 'None': chunk_f_f = re.sub('\$VolumeN', '', mylar.FILE_FORMAT) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) self._log( "No version # found for series - tag will not be available for renaming.", logger.DEBUG) logger.fdebug( "No version # found for series, removing from filename") logger.fdebug("new format is now: " + str(chunk_file_format)) else: chunk_file_format = mylar.FILE_FORMAT if annchk == "no": chunk_f_f = re.sub('\$Annual', '', chunk_file_format) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) logger.fdebug('not an annual - removing from filename paramaters') logger.fdebug('new format: ' + str(chunk_file_format)) else: logger.fdebug('chunk_file_format is: ' + str(chunk_file_format)) if '$Annual' not in chunk_file_format: #if it's an annual, but $Annual isn't specified in file_format, we need to #force it in there, by default in the format of $Annual $Issue prettycomiss = "Annual " + str(prettycomiss) logger.fdebug('prettycomiss: ' + str(prettycomiss)) ofilename = None #if meta-tagging is not enabled, we need to declare the check as being fail #if meta-tagging is enabled, it gets changed just below to a default of pass pcheck = "fail" #tag the meta. if mylar.ENABLE_META: self._log("Metatagging enabled - proceeding...") logger.fdebug("Metatagging enabled - proceeding...") pcheck = "pass" try: import cmtagmylar if ml is None: pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid) else: pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid, manual="yes", filename=ml['ComicLocation']) except ImportError: logger.fdebug( "comictaggerlib not found on system. Ensure the ENTIRE lib directory is located within mylar/lib/comictaggerlib/" ) logger.fdebug( "continuing with PostProcessing, but I'm not using metadata." ) pcheck = "fail" if pcheck == "fail": self._log( "Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging..." ) logger.fdebug( "Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging..." ) elif pcheck == "unrar error": self._log( "This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy." ) logger.error( "This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy." ) return self.log else: otofilename = pcheck self._log("Sucessfully wrote metadata to .cbz - Continuing..") logger.fdebug("Sucessfully wrote metadata to .cbz (" + str(otofilename) + ") - Continuing..") #Run Pre-script if mylar.ENABLE_PRE_SCRIPTS: nzbn = self.nzb_name #original nzb name nzbf = self.nzb_folder #original nzb folder #name, comicyear, comicid , issueid, issueyear, issue, publisher #create the dic and send it. seriesmeta = [] seriesmetadata = {} seriesmeta.append({ 'name': series, 'comicyear': seriesyear, 'comicid': comicid, 'issueid': issueid, 'issueyear': issueyear, 'issue': issuenum, 'publisher': publisher }) seriesmetadata['seriesmeta'] = seriesmeta self._run_pre_scripts(nzbn, nzbf, seriesmetadata) #rename file and move to new path #nfilename = series + " " + issueno + " (" + seriesyear + ")" file_values = { '$Series': series, '$Issue': prettycomiss, '$Year': issueyear, '$series': series.lower(), '$Publisher': publisher, '$publisher': publisher.lower(), '$VolumeY': 'V' + str(seriesyear), '$VolumeN': comversion, '$monthname': month_name, '$month': month, '$Annual': 'Annual' } #if it's a Manual Run, use the ml['ComicLocation'] for the exact filename. if ml is None: for root, dirnames, filenames in os.walk(self.nzb_folder): for filename in filenames: if filename.lower().endswith(extensions): ofilename = filename path, ext = os.path.splitext(ofilename) else: if pcheck == "fail": otofilename = ml['ComicLocation'] logger.fdebug('otofilename:' + str(otofilename)) odir, ofilename = os.path.split(otofilename) logger.fdebug('ofilename: ' + str(ofilename)) path, ext = os.path.splitext(ofilename) logger.fdebug('path: ' + str(path)) logger.fdebug('ext:' + str(ext)) if ofilename is None: logger.error( u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that " + str(self.nzb_folder) + " exists and is the correct location.") return self._log("Original Filename: " + ofilename, logger.DEBUG) self._log("Original Extension: " + ext, logger.DEBUG) logger.fdebug("Original Filname: " + str(ofilename)) logger.fdebug("Original Extension: " + str(ext)) if mylar.FILE_FORMAT == '' or not mylar.RENAME_FILES: self._log( "Rename Files isn't enabled...keeping original filename.", logger.DEBUG) logger.fdebug( "Rename Files isn't enabled - keeping original filename.") #check if extension is in nzb_name - will screw up otherwise if ofilename.lower().endswith(extensions): nfilename = ofilename[:-4] else: nfilename = ofilename else: nfilename = helpers.replace_all(chunk_file_format, file_values) if mylar.REPLACE_SPACES: #mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR) nfilename = re.sub('[\,\:\?]', '', nfilename) nfilename = re.sub('[\/]', '-', nfilename) self._log("New Filename: " + nfilename, logger.DEBUG) logger.fdebug("New Filename: " + str(nfilename)) src = os.path.join(self.nzb_folder, ofilename) filechecker.validateAndCreateDirectory(comlocation, True) if mylar.LOWERCASE_FILENAMES: dst = (comlocation + "/" + nfilename + ext).lower() else: dst = comlocation + "/" + nfilename + ext.lower() self._log("Source:" + src, logger.DEBUG) self._log("Destination:" + dst, logger.DEBUG) logger.fdebug("Source: " + str(src)) logger.fdebug("Destination: " + str(dst)) if ml is None: #non-manual run moving/deleting... logger.fdebug('self.nzb_folder: ' + self.nzb_folder) logger.fdebug('ofilename:' + str(ofilename)) logger.fdebug('nfilename:' + str(nfilename + ext)) os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder, str(nfilename + ext))) src = os.path.join(self.nzb_folder, str(nfilename + ext)) try: shutil.move(src, dst) except (OSError, IOError): self._log( "Failed to move directory - check directories and manually re-run.", logger.DEBUG) self._log("Post-Processing ABORTED.", logger.DEBUG) return #tidyup old path try: shutil.rmtree(self.nzb_folder) except (OSError, IOError): self._log( "Failed to remove temporary directory - check directory and manually re-run.", logger.DEBUG) self._log("Post-Processing ABORTED.", logger.DEBUG) return self._log("Removed temporary directory : " + str(self.nzb_folder), logger.DEBUG) else: #Manual Run, this is the portion. logger.fdebug("Renaming " + os.path.join(self.nzb_folder, str(ofilename)) + " ..to.. " + os.path.join(self.nzb_folder, str(nfilename + ext))) os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder, str(nfilename + ext))) src = os.path.join(self.nzb_folder, str(nfilename + ext)) logger.fdebug("Moving " + src + " ... to ... " + dst) try: shutil.move(src, dst) except (OSError, IOError): logger.fdebug( "Failed to move directory - check directories and manually re-run." ) logger.fdebug("Post-Processing ABORTED.") return logger.fdebug("Successfully moved to : " + dst) #tidyup old path #try: # os.remove(os.path.join(self.nzb_folder, str(ofilename))) # logger.fdebug("Deleting : " + os.path.join(self.nzb_folder, str(ofilename))) #except (OSError, IOError): # logger.fdebug("Failed to remove temporary directory - check directory and manually re-run.") # logger.fdebug("Post-Processing ABORTED.") # return #logger.fdebug("Removed temporary directory : " + str(self.nzb_folder)) #delete entry from nzblog table myDB.action('DELETE from nzblog WHERE issueid=?', [issueid]) #update snatched table to change status to Downloaded if annchk == "no": updater.foundsearch(comicid, issueid, down='True') dispiss = 'issue: ' + str(issuenumOG) else: updater.foundsearch(comicid, issueid, mode='want_ann', down='True') dispiss = 'annual issue: ' + str(issuenumOG) #force rescan of files updater.forceRescan(comicid) logger.info(u"Post-Processing completed for: " + series + " " + dispiss) self._log(u"Post Processing SUCCESSFULL! ", logger.DEBUG) # retrieve/create the corresponding comic objects if mylar.ENABLE_EXTRA_SCRIPTS: folderp = str(dst) #folder location after move/rename nzbn = self.nzb_name #original nzb name filen = str(nfilename + ext) #new filename #name, comicyear, comicid , issueid, issueyear, issue, publisher #create the dic and send it. seriesmeta = [] seriesmetadata = {} seriesmeta.append({ 'name': series, 'comicyear': seriesyear, 'comicid': comicid, 'issueid': issueid, 'issueyear': issueyear, 'issue': issuenum, 'publisher': publisher }) seriesmetadata['seriesmeta'] = seriesmeta self._run_extra_scripts(nzbn, self.nzb_folder, filen, folderp, seriesmetadata) if ml is not None: return self.log else: if mylar.PROWL_ENABLED: pushmessage = series + '(' + issueyear + ') - issue #' + issuenumOG logger.info(u"Prowl request") prowl = notifiers.PROWL() prowl.notify(pushmessage, "Download and Postprocessing completed") if mylar.NMA_ENABLED: nma = notifiers.NMA() nma.notify(series, str(issueyear), str(issuenumOG)) if mylar.PUSHOVER_ENABLED: pushmessage = series + ' (' + str( issueyear) + ') - issue #' + str(issuenumOG) logger.info(u"Pushover request") pushover = notifiers.PUSHOVER() pushover.notify(pushmessage, "Download and Post-Processing completed") if mylar.BOXCAR_ENABLED: boxcar = notifiers.BOXCAR() boxcar.notify(series, str(issueyear), str(issuenumOG)) return self.log