def solicit(month, year):
#convert to numerics just to ensure this...
month = int(month)
year = int(year)
#print ( "month: " + str(month) )
#print ( "year: " + str(year) )
# in order to gather ALL upcoming - let's start to loop through months going ahead one at a time
# until we get a null then break. (Usually not more than 3 months in advance is available)
mnloop = 0
upcoming = []
publishers = {
'DC Comics': 'DC Comics',
'DC\'s': 'DC Comics',
'Marvel': 'Marvel Comics',
'Image': 'Image Comics',
'IDW': 'IDW Publishing',
'Dark Horse': 'Dark Horse'
}
# -- this is no longer needed (testing)
# while (mnloop < 5):
# if year == 2014:
# if len(str(month)) == 1:
# month_string = '0' + str(month)
# else:
# month_string = str(month)
# datestring = str(year) + str(month_string)
# else:
# datestring = str(month) + str(year)
# pagelinks = "http://www.comicbookresources.com/tag/solicits" + str(datestring)
#using the solicits+datestring leaves out some entries occasionally
#should use http://www.comicbookresources.com/tag/solicitations
#then just use the logic below but instead of datestring, find the month term and
#go ahead up to +5 months.
if month > 0:
month_start = month
month_end = month + 5
#if month_end > 12:
# ms = 8, me=13 [(12-8)+(13-12)] = [4 + 1] = 5
# [(12 - ms) + (me - 12)] = number of months (5)
monthlist = []
mongr = month_start
#we need to build the months we can grab, but the non-numeric way.
while (mongr <= month_end):
mon = mongr
if mon == 13:
mon = 1
year += 1
if len(str(mon)) == 1:
mon = '0' + str(mon)
monthlist.append({
"month": helpers.fullmonth(str(mon)).lower(),
"num_month": mon,
"year": str(year)
})
mongr += 1
logger.info('months: ' + str(monthlist))
pagelinks = "http://www.comicbookresources.com/tag/solicitations"
#logger.info('datestring:' + datestring)
#logger.info('checking:' + pagelinks)
pageresponse = urllib.request.urlopen(pagelinks)
soup = BeautifulSoup(pageresponse)
cntlinks = soup.findAll('h3')
lenlinks = len(cntlinks)
#logger.info( str(lenlinks) + ' results' )
publish = []
resultURL = []
resultmonth = []
resultyear = []
x = 0
cnt = 0
while (x < lenlinks):
headt = cntlinks[
x] #iterate through the hrefs pulling out only results.
if "/?page=article&id=" in str(headt):
#print ("titlet: " + str(headt))
headName = headt.findNext(text=True)
#print ('headName: ' + headName)
if 'Image' in headName: print('IMAGE FOUND')
if not all([
'Marvel' in headName, 'DC' in headName, 'Image'
in headName
]) and ('Solicitations' in headName or 'Solicits' in headName):
# test for month here (int(month) + 5)
if not any(
d.get('month', None) == str(headName).lower()
for d in monthlist):
for mt in monthlist:
if mt['month'] in headName.lower():
logger.info('matched on month: ' +
str(mt['month']))
logger.info('matched on year: ' +
str(mt['year']))
resultmonth.append(mt['num_month'])
resultyear.append(mt['year'])
pubstart = headName.find('Solicitations')
publishchk = False
for pub in publishers:
if pub in headName[:pubstart]:
#print 'publisher:' + str(publishers[pub])
publish.append(publishers[pub])
publishchk = True
break
if publishchk == False:
break
#publish.append( headName[:pubstart].strip() )
abc = headt.findAll('a', href=True)[0]
ID_som = abc[
'href'] #first instance will have the right link...
resultURL.append(ID_som)
#print '(' + str(cnt) + ') [ ' + publish[cnt] + '] Link URL: ' + resultURL[cnt]
cnt += 1
else:
logger.info('incorrect month - not using.')
x += 1
if cnt == 0:
return #break # no results means, end it
loopthis = (cnt - 1)
#this loops through each 'found' solicit page
#shipdate = str(month_string) + '-' + str(year) - not needed.
while (loopthis >= 0):
#print 'loopthis is : ' + str(loopthis)
#print 'resultURL is : ' + str(resultURL[loopthis])
shipdate = str(resultmonth[loopthis]) + '-' + str(
resultyear[loopthis])
upcoming += populate(resultURL[loopthis], publish[loopthis],
shipdate)
loopthis -= 1
logger.info(str(len(upcoming)) + ' upcoming issues discovered.')
newfl = mylar.CACHE_DIR + "/future-releases.txt"
newtxtfile = open(newfl, 'wb')
cntr = 1
for row in upcoming:
if row['Extra'] is None or row['Extra'] == '':
extrarow = 'N/A'
else:
extrarow = row['Extra']
newtxtfile.write(
str(row['Shipdate']) + '\t' + str(row['Publisher']) + '\t' +
str(row['Issue']) + '\t' + str(row['Comic']) + '\t' +
str(extrarow) + '\tSkipped' + '\t' + str(cntr) + '\n')
cntr += 1
newtxtfile.close()
logger.fdebug('attempting to populate future upcoming...')
mylardb = os.path.join(mylar.DATA_DIR, "mylar.db")
connection = sqlite3.connect(str(mylardb))
cursor = connection.cursor()
# we should extract the issues that are being watched, but no data is available yet ('Watch For' status)
# once we get the data, store it, wipe the existing table, retrieve the new data, populate the data into
# the table, recheck the series against the current watchlist and then restore the Watch For data.
cursor.executescript('drop table if exists future;')
cursor.execute(
"CREATE TABLE IF NOT EXISTS future (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, FutureID text, ComicID text);"
)
connection.commit()
csvfile = open(newfl, "rb")
creader = csv.reader(csvfile, delimiter='\t')
t = 1
for row in creader:
try:
#print ("Row: %s" % row)
cursor.execute("INSERT INTO future VALUES (?,?,?,?,?,?,?,null);",
row)
except Exception as e:
logger.fdebug("Error - invald arguments...-skipping")
pass
t += 1
logger.fdebug('successfully added ' + str(t) +
' issues to future upcoming table.')
csvfile.close()
connection.commit()
connection.close()
mylar.weeklypull.pullitcheck(futurepull="yes")
myDB.upsert("issues", newValueDict, controlValueDict)
except sqlite3.InterfaceError, e:
#raise sqlite3.InterfaceError(e)
logger.error("Something went wrong - I can't add the issue information into my DB.")
myDB.action("DELETE FROM comics WHERE ComicID=?", [comicid])
return
n+=1
#figure publish dates here...
styear = str(SeriesYear)
#if SeriesYear == '0000':
# styear = firstdate[:4]
if firstdate[5:7] == '00':
stmonth = "?"
else:
stmonth = helpers.fullmonth(firstdate[5:7])
ltyear = re.sub('/s','', latestdate[:4])
if latestdate[5:7] == '00':
ltmonth = "?"
else:
ltmonth = helpers.fullmonth(latestdate[5:7])
#try to determine if it's an 'actively' published comic from above dates
#threshold is if it's within a month (<45 days) let's assume it's recent.
c_date = datetime.date(int(latestdate[:4]),int(latestdate[5:7]),1)
n_date = datetime.date.today()
recentchk = (n_date - c_date).days
#print ("recentchk: " + str(recentchk))
if recentchk <= 55:
lastpubdate = 'Present'
else:
def solicit(month, year):
#convert to numerics just to ensure this...
month = int(month)
year = int(year)
#print ( "month: " + str(month) )
#print ( "year: " + str(year) )
# in order to gather ALL upcoming - let's start to loop through months going ahead one at a time
# until we get a null then break. (Usually not more than 3 months in advance is available)
mnloop = 0
upcoming = []
publishers = {'DC Comics':'DC Comics', 'DC\'s': 'DC Comics', 'Marvel':'Marvel Comics', 'Image':'Image Comics', 'IDW':'IDW Publishing', 'Dark Horse':'Dark Horse'}
# -- this is no longer needed (testing)
# while (mnloop < 5):
# if year == 2014:
# if len(str(month)) == 1:
# month_string = '0' + str(month)
# else:
# month_string = str(month)
# datestring = str(year) + str(month_string)
# else:
# datestring = str(month) + str(year)
# pagelinks = "http://www.comicbookresources.com/tag/solicits" + str(datestring)
#using the solicits+datestring leaves out some entries occasionally
#should use http://www.comicbookresources.com/tag/solicitations
#then just use the logic below but instead of datestring, find the month term and
#go ahead up to +5 months.
if month > 0:
month_start = month
month_end = month + 5
#if month_end > 12:
# ms = 8, me=13 [(12-8)+(13-12)] = [4 + 1] = 5
# [(12 - ms) + (me - 12)] = number of months (5)
monthlist = []
mongr = month_start
#we need to build the months we can grab, but the non-numeric way.
while (mongr <= month_end):
mon = mongr
if mon == 13:
mon = 1
year +=1
if len(str(mon)) == 1:
mon = '0' + str(mon)
monthlist.append({"month": helpers.fullmonth(str(mon)).lower(),
"num_month": mon,
"year": str(year)})
mongr+=1
logger.info('months: ' + str(monthlist))
pagelinks = "http://www.comicbookresources.com/tag/solicitations"
#logger.info('datestring:' + datestring)
#logger.info('checking:' + pagelinks)
pageresponse = urllib2.urlopen ( pagelinks )
soup = BeautifulSoup (pageresponse)
cntlinks = soup.findAll('h3')
lenlinks = len(cntlinks)
#logger.info( str(lenlinks) + ' results' )
publish = []
resultURL = []
resultmonth = []
resultyear = []
x = 0
cnt = 0
while (x < lenlinks):
headt = cntlinks[x] #iterate through the hrefs pulling out only results.
if "/?page=article&id=" in str(headt):
#print ("titlet: " + str(headt))
headName = headt.findNext(text=True)
#print ('headName: ' + headName)
if 'Image' in headName: print 'IMAGE FOUND'
if not all( ['Marvel' in headName, 'DC' in headName, 'Image' in headName] ) and ('Solicitations' in headName or 'Solicits' in headName):
# test for month here (int(month) + 5)
if not any(d.get('month', None) == str(headName).lower() for d in monthlist):
for mt in monthlist:
if mt['month'] in headName.lower():
logger.info('matched on month: ' + str(mt['month']))
logger.info('matched on year: ' + str(mt['year']))
resultmonth.append(mt['num_month'])
resultyear.append(mt['year'])
pubstart = headName.find('Solicitations')
publishchk = False
for pub in publishers:
if pub in headName[:pubstart]:
#print 'publisher:' + str(publishers[pub])
publish.append(publishers[pub])
publishchk = True
break
if publishchk == False:
break
#publish.append( headName[:pubstart].strip() )
abc = headt.findAll('a', href=True)[0]
ID_som = abc['href'] #first instance will have the right link...
resultURL.append( ID_som )
#print '(' + str(cnt) + ') [ ' + publish[cnt] + '] Link URL: ' + resultURL[cnt]
cnt+=1
else:
logger.info('incorrect month - not using.')
x+=1
if cnt == 0:
return #break # no results means, end it
loopthis = (cnt-1)
#this loops through each 'found' solicit page
#shipdate = str(month_string) + '-' + str(year) - not needed.
while ( loopthis >= 0 ):
#print 'loopthis is : ' + str(loopthis)
#print 'resultURL is : ' + str(resultURL[loopthis])
shipdate = str(resultmonth[loopthis]) + '-' + str(resultyear[loopthis])
upcoming += populate(resultURL[loopthis], publish[loopthis], shipdate)
loopthis -=1
logger.info( str(len(upcoming)) + ' upcoming issues discovered.' )
newfl = mylar.CACHE_DIR + "/future-releases.txt"
newtxtfile = open(newfl, 'wb')
cntr = 1
for row in upcoming:
if row['Extra'] is None or row['Extra'] == '':
extrarow = 'N/A'
else:
extrarow = row['Extra']
newtxtfile.write(str(row['Shipdate']) + '\t' + str(row['Publisher']) + '\t' + str(row['Issue']) + '\t' + str(row['Comic']) + '\t' + str(extrarow) + '\tSkipped' + '\t' + str(cntr) + '\n')
cntr +=1
newtxtfile.close()
logger.fdebug( 'attempting to populate future upcoming...' )
mylardb = os.path.join(mylar.DATA_DIR, "mylar.db")
connection = sqlite3.connect(str(mylardb))
cursor = connection.cursor()
# we should extract the issues that are being watched, but no data is available yet ('Watch For' status)
# once we get the data, store it, wipe the existing table, retrieve the new data, populate the data into
# the table, recheck the series against the current watchlist and then restore the Watch For data.
cursor.executescript('drop table if exists future;')
cursor.execute("CREATE TABLE IF NOT EXISTS future (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, FutureID text, ComicID text);")
connection.commit()
csvfile = open(newfl, "rb")
creader = csv.reader(csvfile, delimiter='\t')
t = 1
for row in creader:
try:
#print ("Row: %s" % row)
cursor.execute("INSERT INTO future VALUES (?,?,?,?,?,?,?,null);", row)
except Exception, e:
logger.fdebug("Error - invald arguments...-skipping")
pass
t+=1
def Process_next(self,comicid,issueid,issuenumOG,ml=None):
annchk = "no"
extensions = ('.cbr', '.cbz')
snatchedtorrent = False
myDB = db.DBConnection()
comicnzb = myDB.selectone("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
issuenzb = myDB.selectone("SELECT * from issues WHERE issueid=? AND comicid=? AND ComicName NOT NULL", [issueid,comicid]).fetchone()
if ml is not None and mylar.SNATCHEDTORRENT_NOTIFY:
snatchnzb = myDB.selectone("SELECT * from snatched WHERE IssueID=? AND ComicID=? AND (provider=? OR provider=?) AND Status='Snatched'", [issueid,comicid,'KAT','CBT']).fetchone()
if snatchnzb is None:
logger.fdebug('Was not downloaded with Mylar and the usage of torrents. Disabling torrent manual post-processing completion notification.')
else:
logger.fdebug('Was downloaded from ' + snatchnzb['Provider'] + '. Enabling torrent manual post-processing completion notification.')
snatchedtorrent = True
logger.fdebug('issueid: ' + str(issueid))
logger.fdebug('issuenumOG: ' + str(issuenumOG))
if issuenzb is None:
issuenzb = myDB.selectone("SELECT * from annuals WHERE issueid=? and comicid=?", [issueid,comicid]).fetchone()
annchk = "yes"
#issueno = str(issuenum).split('.')[0]
#new CV API - removed all decimals...here we go AGAIN!
issuenum = issuenzb['Issue_Number']
issue_except = 'None'
if 'au' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AU'
elif 'ai' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AI'
elif 'inh' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = '.INH'
elif 'now' in issuenum.lower() and issuenum[:1].isdigit():
if '!' in issuenum: issuenum = re.sub('\!', '', issuenum)
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = '.NOW'
if '.' in issuenum:
iss_find = issuenum.find('.')
iss_b4dec = issuenum[:iss_find]
iss_decval = issuenum[iss_find+1:]
if int(iss_decval) == 0:
iss = iss_b4dec
issdec = int(iss_decval)
issueno = str(iss)
self._log("Issue Number: " + str(issueno))
logger.fdebug("Issue Number: " + str(issueno))
else:
if len(iss_decval) == 1:
iss = iss_b4dec + "." + iss_decval
issdec = int(iss_decval) * 10
else:
iss = iss_b4dec + "." + iss_decval.rstrip('0')
issdec = int(iss_decval.rstrip('0')) * 10
issueno = iss_b4dec
self._log("Issue Number: " + str(iss))
logger.fdebug("Issue Number: " + str(iss))
else:
iss = issuenum
issueno = str(iss)
# issue zero-suppression here
if mylar.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00"
logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N))
if str(len(issueno)) > 1:
if int(issueno) < 0:
self._log("issue detected is a negative")
prettycomiss = '-' + str(zeroadd) + str(abs(issueno))
elif int(issueno) < 10:
self._log("issue detected less than 10")
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss))
elif int(issueno) >= 10 and int(issueno) < 100:
self._log("issue detected greater than 10, but less than 100")
if mylar.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss))
else:
self._log("issue detected greater than 100")
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(issueno)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss))
else:
prettycomiss = str(issueno)
self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss))
if annchk == "yes":
self._log("Annual detected.")
logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss))
issueyear = issuenzb['IssueDate'][:4]
self._log("Issue Year: " + str(issueyear))
logger.fdebug("Issue Year : " + str(issueyear))
month = issuenzb['IssueDate'][5:7].replace('-','').strip()
month_name = helpers.fullmonth(month)
# comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
publisher = comicnzb['ComicPublisher']
self._log("Publisher: " + publisher)
logger.fdebug("Publisher: " + str(publisher))
#we need to un-unicode this to make sure we can write the filenames properly for spec.chars
series = comicnzb['ComicName'].encode('ascii', 'ignore').strip()
self._log("Series: " + series)
logger.fdebug("Series: " + str(series))
seriesyear = comicnzb['ComicYear']
self._log("Year: " + seriesyear)
logger.fdebug("Year: " + str(seriesyear))
comlocation = comicnzb['ComicLocation']
self._log("Comic Location: " + comlocation)
logger.fdebug("Comic Location: " + str(comlocation))
comversion = comicnzb['ComicVersion']
self._log("Comic Version: " + str(comversion))
logger.fdebug("Comic Version: " + str(comversion))
if comversion is None:
comversion = 'None'
#if comversion is None, remove it so it doesn't populate with 'None'
if comversion == 'None':
chunk_f_f = re.sub('\$VolumeN','',mylar.FILE_FORMAT)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
self._log("No version # found for series - tag will not be available for renaming.")
logger.fdebug("No version # found for series, removing from filename")
logger.fdebug("new format is now: " + str(chunk_file_format))
else:
chunk_file_format = mylar.FILE_FORMAT
if annchk == "no":
chunk_f_f = re.sub('\$Annual','',chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('not an annual - removing from filename paramaters')
logger.fdebug('new format: ' + str(chunk_file_format))
else:
logger.fdebug('chunk_file_format is: ' + str(chunk_file_format))
if '$Annual' not in chunk_file_format:
#if it's an annual, but $Annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
prettycomiss = "Annual " + str(prettycomiss)
logger.fdebug('prettycomiss: ' + str(prettycomiss))
ofilename = None
#if meta-tagging is not enabled, we need to declare the check as being fail
#if meta-tagging is enabled, it gets changed just below to a default of pass
pcheck = "fail"
#tag the meta.
if mylar.ENABLE_META:
self._log("Metatagging enabled - proceeding...")
logger.fdebug("Metatagging enabled - proceeding...")
pcheck = "pass"
try:
import cmtagmylar
if ml is None:
pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid)
else:
pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid, manual="yes", filename=ml['ComicLocation'])
except ImportError:
logger.fdebug("comictaggerlib not found on system. Ensure the ENTIRE lib directory is located within mylar/lib/comictaggerlib/")
logger.fdebug("continuing with PostProcessing, but I'm not using metadata.")
pcheck = "fail"
if pcheck == "fail":
self._log("Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging...")
logger.fdebug("Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging...")
elif pcheck == "unrar error":
self._log("This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy.")
logger.error("This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy.")
return self.log
else:
otofilename = pcheck
self._log("Sucessfully wrote metadata to .cbz - Continuing..")
logger.fdebug("Sucessfully wrote metadata to .cbz (" + str(otofilename) + ") - Continuing..")
#Run Pre-script
if mylar.ENABLE_PRE_SCRIPTS:
nzbn = self.nzb_name #original nzb name
nzbf = self.nzb_folder #original nzb folder
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_pre_scripts(nzbn, nzbf, seriesmetadata )
#rename file and move to new path
#nfilename = series + " " + issueno + " (" + seriesyear + ")"
file_values = {'$Series': series,
'$Issue': prettycomiss,
'$Year': issueyear,
'$series': series.lower(),
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion,
'$monthname': month_name,
'$month': month,
'$Annual': 'Annual'
}
#if it's a Manual Run, use the ml['ComicLocation'] for the exact filename.
if ml is None:
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
odir = root
ofilename = filename
path, ext = os.path.splitext(ofilename)
if odir is None:
logger.fdebug('no root folder set.')
odir = self.nzb_folder
logger.fdebug('odir: ' + str(odir))
logger.fdebug('ofilename: ' + str(ofilename))
else:
if pcheck == "fail":
otofilename = ml['ComicLocation']
logger.fdebug('otofilename:' + str(otofilename))
odir, ofilename = os.path.split(otofilename)
logger.fdebug('odir: ' + str(odir))
logger.fdebug('ofilename: ' + str(ofilename))
path, ext = os.path.splitext(ofilename)
logger.fdebug('path: ' + str(path))
logger.fdebug('ext:' + str(ext))
if ofilename is None:
logger.error(u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that " + str(self.nzb_folder) + " exists and is the correct location.")
return
self._log("Original Filename: " + ofilename)
self._log("Original Extension: " + ext)
logger.fdebug("Original Filname: " + str(ofilename))
logger.fdebug("Original Extension: " + str(ext))
if mylar.FILE_FORMAT == '' or not mylar.RENAME_FILES:
self._log("Rename Files isn't enabled...keeping original filename.")
logger.fdebug("Rename Files isn't enabled - keeping original filename.")
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
nfilename = helpers.replace_all(chunk_file_format, file_values)
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
nfilename = re.sub('[\,\:\?]', '', nfilename)
nfilename = re.sub('[\/]', '-', nfilename)
self._log("New Filename: " + nfilename)
logger.fdebug("New Filename: " + str(nfilename))
#src = os.path.join(self.nzb_folder, ofilename)
src = os.path.join(odir, ofilename)
filechecker.validateAndCreateDirectory(comlocation, True)
if mylar.LOWERCASE_FILENAMES:
dst = (comlocation + "/" + nfilename + ext).lower()
else:
dst = comlocation + "/" + nfilename + ext.lower()
self._log("Source:" + src)
self._log("Destination:" + dst)
logger.fdebug("Source: " + str(src))
logger.fdebug("Destination: " + str(dst))
if ml is None:
#downtype = for use with updater on history table to set status to 'Downloaded'
downtype = 'True'
#non-manual run moving/deleting...
logger.fdebug('self.nzb_folder: ' + self.nzb_folder)
logger.fdebug('odir: ' + str(odir))
logger.fdebug('ofilename:' + str(ofilename))
logger.fdebug('nfilename:' + str(nfilename + ext))
if mylar.RENAME_FILES:
if str(ofilename) != str(nfilename + ext):
logger.fdebug("Renaming " + os.path.join(odir, str(ofilename)) + " ..to.. " + os.path.join(odir,str(nfilename + ext)))
os.rename(os.path.join(odir, str(ofilename)), os.path.join(odir,str(nfilename + ext)))
else:
logger.fdebug('filename is identical as original, not renaming.')
#src = os.path.join(self.nzb_folder, str(nfilename + ext))
src = os.path.join(odir, str(nfilename + ext))
try:
shutil.move(src, dst)
except (OSError, IOError):
self._log("Failed to move directory - check directories and manually re-run.")
self._log("Post-Processing ABORTED.")
return
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log("Failed to remove temporary directory - check directory and manually re-run.")
self._log("Post-Processing ABORTED.")
return
self._log("Removed temporary directory : " + str(self.nzb_folder))
else:
#downtype = for use with updater on history table to set status to 'Post-Processed'
downtype = 'PP'
#Manual Run, this is the portion.
if mylar.RENAME_FILES:
if str(ofilename) != str(nfilename + ext):
logger.fdebug("Renaming " + os.path.join(self.nzb_folder, str(ofilename)) + " ..to.. " + os.path.join(self.nzb_folder,str(nfilename + ext)))
os.rename(os.path.join(odir, str(ofilename)), os.path.join(odir ,str(nfilename + ext)))
else:
logger.fdebug('filename is identical as original, not renaming.')
src = os.path.join(odir, str(nfilename + ext))
logger.fdebug('odir rename: ' + os.path.join(odir, str(ofilename)) + ' TO ' + os.path.join(odir, str(nfilename + ext)))
logger.fdebug('odir src : ' + os.path.join(odir, str(nfilename + ext)))
logger.fdebug("Moving " + src + " ... to ... " + dst)
try:
shutil.move(src, dst)
except (OSError, IOError):
logger.fdebug("Failed to move directory - check directories and manually re-run.")
logger.fdebug("Post-Processing ABORTED.")
return
logger.fdebug("Successfully moved to : " + dst)
#tidyup old path
#try:
# os.remove(os.path.join(self.nzb_folder, str(ofilename)))
# logger.fdebug("Deleting : " + os.path.join(self.nzb_folder, str(ofilename)))
#except (OSError, IOError):
# logger.fdebug("Failed to remove temporary directory - check directory and manually re-run.")
# logger.fdebug("Post-Processing ABORTED.")
# return
#logger.fdebug("Removed temporary directory : " + str(self.nzb_folder))
#Hopefully set permissions on downloaded file
try:
permission = int(mylar.CHMOD_FILE, 8)
os.umask(0)
os.chmod(dst.rstrip(), permission)
except OSError:
logger.error('Failed to change file permissions. Ensure that the user running Mylar has proper permissions to change permissions in : ' + dst)
logger.fdebug('Continuing post-processing but unable to change file permissions in ' + dst)
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
#update snatched table to change status to Downloaded
if annchk == "no":
updater.foundsearch(comicid, issueid, down=downtype)
dispiss = 'issue: ' + str(issuenumOG)
else:
updater.foundsearch(comicid, issueid, mode='want_ann', down=downtype)
dispiss = 'annual issue: ' + str(issuenumOG)
#force rescan of files
updater.forceRescan(comicid)
logger.info(u"Post-Processing completed for: " + series + " " + dispiss )
self._log(u"Post Processing SUCCESSFUL! ")
if mylar.WEEKFOLDER:
#if enabled, will *copy* the post-processed file to the weeklypull list folder for the given week.
weeklypull.weekly_singlecopy(comicid,issuenum,str(nfilename+ext),dst)
# retrieve/create the corresponding comic objects
if mylar.ENABLE_EXTRA_SCRIPTS:
folderp = str(dst) #folder location after move/rename
nzbn = self.nzb_name #original nzb name
filen = str(nfilename + ext) #new filename
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_extra_scripts(nzbn, self.nzb_folder, filen, folderp, seriesmetadata )
if ml is not None:
#we only need to return self.log if it's a manual run and it's not a snatched torrent
if snatchedtorrent:
#manual run + snatched torrent
pass
else:
#manual run + not snatched torrent (or normal manual-run)
return self.log
if annchk == "no":
prline = series + '(' + issueyear + ') - issue #' + issuenumOG
else:
prline = series + ' Annual (' + issueyear + ') - issue #' + issuenumOG
prline2 = 'Mylar has downloaded and post-processed: ' + prline
if mylar.PROWL_ENABLED:
pushmessage = prline
logger.info(u"Prowl request")
prowl = notifiers.PROWL()
prowl.notify(pushmessage,"Download and Postprocessing completed")
if mylar.NMA_ENABLED:
nma = notifiers.NMA()
nma.notify(prline=prline, prline2=prline2)
if mylar.PUSHOVER_ENABLED:
logger.info(u"Pushover request")
pushover = notifiers.PUSHOVER()
pushover.notify(prline, "Download and Post-Processing completed")
if mylar.BOXCAR_ENABLED:
boxcar = notifiers.BOXCAR()
boxcar.notify(prline=prline, prline2=prline2)
if mylar.PUSHBULLET_ENABLED:
pushbullet = notifiers.PUSHBULLET()
pushbullet.notify(prline=prline, prline2=prline2)
return self.log
def rename_file(self, ofilename, issue=None, annualize=None, arc=False, file_format=None): #comicname, issue, comicyear=None, issueid=None)
comicid = self.comicid # it's coming in unicoded...
issueid = self.issueid
if file_format is None:
file_format = mylar.CONFIG.FILE_FORMAT
logger.fdebug(type(comicid))
logger.fdebug(type(issueid))
logger.fdebug('comicid: %s' % comicid)
logger.fdebug('issue# as per cv: %s' % issue)
logger.fdebug('issueid:' + str(issueid))
if issueid is None:
logger.fdebug('annualize is ' + str(annualize))
if arc:
#this has to be adjusted to be able to include story arc issues that span multiple arcs
chkissue = self.myDB.selectone("SELECT * from storyarcs WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()
else:
chkissue = self.myDB.selectone("SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()
if all([chkissue is None, annualize is None, not mylar.CONFIG.ANNUALS_ON]):
chkissue = self.myDB.selectone("SELECT * from annuals WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone()
if chkissue is None:
#rechk chkissue against int value of issue #
if arc:
chkissue = self.myDB.selectone("SELECT * from storyarcs WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone()
else:
chkissue = self.myDB.selectone("SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone()
if all([chkissue is None, annualize == 'yes', mylar.CONFIG.ANNUALS_ON]):
chkissue = self.myDB.selectone("SELECT * from annuals WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone()
if chkissue is None:
logger.error('Invalid Issue_Number - please validate.')
return
else:
logger.info('Int Issue_number compare found. continuing...')
issueid = chkissue['IssueID']
else:
issueid = chkissue['IssueID']
#use issueid to get publisher, series, year, issue number
logger.fdebug('issueid is now : ' + str(issueid))
if arc:
issueinfo = self.myDB.selectone("SELECT * from storyarcs WHERE ComicID=? AND IssueID=? AND StoryArc=?", [comicid, issueid, arc]).fetchone()
else:
issueinfo = self.myDB.selectone("SELECT * from issues WHERE ComicID=? AND IssueID=?", [comicid, issueid]).fetchone()
if issueinfo is None:
logger.fdebug('not an issue, checking against annuals')
issueinfo = self.myDB.selectone("SELECT * from annuals WHERE ComicID=? AND IssueID=?", [comicid, issueid]).fetchone()
if issueinfo is None:
logger.fdebug('Unable to rename - cannot locate issue id within db')
return
else:
annualize = True
if issueinfo is None:
logger.fdebug('Unable to rename - cannot locate issue id within db')
return
#remap the variables to a common factor.
if arc:
issuenum = issueinfo['IssueNumber']
issuedate = issueinfo['IssueDate']
publisher = issueinfo['IssuePublisher']
series = issueinfo['ComicName']
seriesfilename = series #Alternate FileNaming is not available with story arcs.
seriesyear = issueinfo['SeriesYear']
arcdir = helpers.filesafe(issueinfo['StoryArc'])
if mylar.CONFIG.REPLACE_SPACES:
arcdir = arcdir.replace(' ', mylar.CONFIG.REPLACE_CHAR)
if mylar.CONFIG.STORYARCDIR:
storyarcd = os.path.join(mylar.CONFIG.DESTINATION_DIR, "StoryArcs", arcdir)
logger.fdebug('Story Arc Directory set to : ' + storyarcd)
else:
logger.fdebug('Story Arc Directory set to : ' + mylar.CONFIG.GRABBAG_DIR)
storyarcd = os.path.join(mylar.CONFIG.DESTINATION_DIR, mylar.CONFIG.GRABBAG_DIR)
comlocation = storyarcd
comversion = None #need to populate this.
else:
issuenum = issueinfo['Issue_Number']
issuedate = issueinfo['IssueDate']
publisher = self.comic['ComicPublisher']
series = self.comic['ComicName']
if self.comic['AlternateFileName'] is None or self.comic['AlternateFileName'] == 'None':
seriesfilename = series
else:
seriesfilename = self.comic['AlternateFileName']
logger.fdebug('Alternate File Naming has been enabled for this series. Will rename series title to : ' + seriesfilename)
seriesyear = self.comic['ComicYear']
comlocation = self.comic['ComicLocation']
comversion = self.comic['ComicVersion']
unicodeissue = issuenum
if type(issuenum) == str:
vals = {'\xbd':'.5','\xbc':'.25','\xbe':'.75','\u221e':'9999999999','\xe2':'9999999999'}
else:
vals = {'\xbd':'.5','\xbc':'.25','\xbe':'.75','\\u221e':'9999999999','\xe2':'9999999999'}
x = [vals[key] for key in vals if key in issuenum]
if x:
issuenum = x[0]
logger.fdebug('issue number formatted: %s' % issuenum)
#comicid = issueinfo['ComicID']
#issueno = str(issuenum).split('.')[0]
issue_except = 'None'
issue_exceptions = ['AU',
'INH',
'NOW',
'AI',
'MU',
'A',
'B',
'C',
'X',
'O']
valid_spaces = ('.', '-')
for issexcept in issue_exceptions:
if issexcept.lower() in issuenum.lower():
logger.fdebug('ALPHANUMERIC EXCEPTION : [' + issexcept + ']')
v_chk = [v for v in valid_spaces if v in issuenum]
if v_chk:
iss_space = v_chk[0]
logger.fdebug('character space denoted as : ' + iss_space)
else:
logger.fdebug('character space not denoted.')
iss_space = ''
# if issexcept == 'INH':
# issue_except = '.INH'
if issexcept == 'NOW':
if '!' in issuenum: issuenum = re.sub('\!', '', issuenum)
# issue_except = '.NOW'
issue_except = iss_space + issexcept
logger.fdebug('issue_except denoted as : ' + issue_except)
issuenum = re.sub("[^0-9]", "", issuenum)
break
# if 'au' in issuenum.lower() and issuenum[:1].isdigit():
# issue_except = ' AU'
# elif 'ai' in issuenum.lower() and issuenum[:1].isdigit():
# issuenum = re.sub("[^0-9]", "", issuenum)
# issue_except = ' AI'
# elif 'inh' in issuenum.lower() and issuenum[:1].isdigit():
# issuenum = re.sub("[^0-9]", "", issuenum)
# issue_except = '.INH'
# elif 'now' in issuenum.lower() and issuenum[:1].isdigit():
# if '!' in issuenum: issuenum = re.sub('\!', '', issuenum)
# issuenum = re.sub("[^0-9]", "", issuenum)
# issue_except = '.NOW'
if '.' in issuenum:
iss_find = issuenum.find('.')
iss_b4dec = issuenum[:iss_find]
if iss_find == 0:
iss_b4dec = '0'
iss_decval = issuenum[iss_find +1:]
if iss_decval.endswith('.'):
iss_decval = iss_decval[:-1]
if int(iss_decval) == 0:
iss = iss_b4dec
issdec = int(iss_decval)
issueno = iss
else:
if len(iss_decval) == 1:
iss = iss_b4dec + "." + iss_decval
issdec = int(iss_decval) * 10
else:
iss = iss_b4dec + "." + iss_decval.rstrip('0')
issdec = int(iss_decval.rstrip('0')) * 10
issueno = iss_b4dec
else:
iss = issuenum
issueno = iss
# issue zero-suppression here
if mylar.CONFIG.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.CONFIG.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.CONFIG.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.CONFIG.ZERO_LEVEL_N == "00x": zeroadd = "00"
logger.fdebug('Zero Suppression set to : ' + str(mylar.CONFIG.ZERO_LEVEL_N))
prettycomiss = None
if issueno.isalpha():
logger.fdebug('issue detected as an alpha.')
prettycomiss = str(issueno)
else:
try:
x = float(issuenum)
#validity check
if x < 0:
logger.info('I\'ve encountered a negative issue #: %s. Trying to accomodate.' % issueno)
prettycomiss = '-' + str(zeroadd) + str(issueno[1:])
elif x == 9999999999:
logger.fdebug('Infinity issue found.')
issuenum = 'infinity'
elif x >= 0:
pass
else:
raise ValueError
except ValueError as e:
logger.warn('Unable to properly determine issue number [ %s] - you should probably log this on github for help.' % issueno)
return
if prettycomiss is None and len(str(issueno)) > 0:
#if int(issueno) < 0:
# self._log("issue detected is a negative")
# prettycomiss = '-' + str(zeroadd) + str(abs(issueno))
if int(issueno) < 10:
logger.fdebug('issue detected less than 10')
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '. Issue will be set as : ' + str(prettycomiss))
elif int(issueno) >= 10 and int(issueno) < 100:
logger.fdebug('issue detected greater than 10, but less than 100')
if mylar.CONFIG.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '.Issue will be set as : ' + str(prettycomiss))
else:
logger.fdebug('issue detected greater than 100')
if issuenum == 'infinity':
prettycomiss = 'infinity'
else:
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(issueno)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '. Issue will be set as : ' + str(prettycomiss))
elif len(str(issueno)) == 0:
prettycomiss = str(issueno)
logger.fdebug('issue length error - cannot determine length. Defaulting to None: ' + str(prettycomiss))
logger.fdebug('Pretty Comic Issue is : ' + str(prettycomiss))
if mylar.CONFIG.UNICODE_ISSUENUMBER:
logger.fdebug('Setting this to Unicode format as requested: %s' % prettycomiss)
prettycomiss = unicodeissue
issueyear = issuedate[:4]
month = issuedate[5:7].replace('-', '').strip()
month_name = helpers.fullmonth(month)
if month_name is None:
month_name = 'None'
logger.fdebug('Issue Year : ' + str(issueyear))
logger.fdebug('Publisher: ' + publisher)
logger.fdebug('Series: ' + series)
logger.fdebug('Year: ' + str(seriesyear))
logger.fdebug('Comic Location: ' + comlocation)
if self.comic['Corrected_Type'] is not None:
if self.comic['Type'] != self.comic['Corrected_Type']:
booktype = self.comic['Corrected_Type']
else:
booktype = self.comic['Type']
else:
booktype = self.comic['Type']
if booktype == 'Print' or all([booktype != 'Print', mylar.CONFIG.FORMAT_BOOKTYPE is False]):
chunk_fb = re.sub('\$Type', '', file_format)
chunk_b = re.compile(r'\s+')
chunk_file_format = chunk_b.sub(' ', chunk_fb)
else:
chunk_file_format = file_format
if any([comversion is None, booktype != 'Print']):
comversion = 'None'
#if comversion is None, remove it so it doesn't populate with 'None'
if comversion == 'None':
chunk_f_f = re.sub('\$VolumeN', '', chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('No version # found for series, removing from filename')
logger.fdebug("new format: " + str(chunk_file_format))
if annualize is None:
chunk_f_f = re.sub('\$Annual', '', chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('not an annual - removing from filename paramaters')
logger.fdebug('new format: ' + str(chunk_file_format))
else:
logger.fdebug('chunk_file_format is: ' + str(chunk_file_format))
if mylar.CONFIG.ANNUALS_ON:
if 'annual' in series.lower():
if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
#if it's an annual, but $annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
#prettycomiss = "Annual " + str(prettycomiss)
logger.fdebug('[%s][ANNUALS-ON][ANNUAL IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
else:
#because it exists within title, strip it then use formatting tag for placement of wording.
chunk_f_f = re.sub('\$Annual', '', chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('[%s][ANNUALS-ON][ANNUAL IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
else:
if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
#if it's an annual, but $annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
prettycomiss = "Annual %s" % prettycomiss
logger.fdebug('[%s][ANNUALS-ON][ANNUAL NOT IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
else:
logger.fdebug('[%s][ANNUALS-ON][ANNUAL NOT IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
else:
#if annuals aren't enabled, then annuals are being tracked as independent series.
#annualize will be true since it's an annual in the seriesname.
if 'annual' in series.lower():
if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
#if it's an annual, but $annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
#prettycomiss = "Annual " + str(prettycomiss)
logger.fdebug('[%s][ANNUALS-OFF][ANNUAL IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
else:
#because it exists within title, strip it then use formatting tag for placement of wording.
chunk_f_f = re.sub('\$Annual', '', chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('[%s][ANNUALS-OFF][ANNUAL IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
else:
if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
#if it's an annual, but $annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
prettycomiss = "Annual %s" % prettycomiss
logger.fdebug('[%s][ANNUALS-OFF][ANNUAL NOT IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
else:
logger.fdebug('[%s][ANNUALS-OFF][ANNUAL NOT IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss))
logger.fdebug('Annual detected within series title of ' + series + '. Not auto-correcting issue #')
seriesfilename = seriesfilename #.encode('ascii', 'ignore').strip()
filebad = [':', ',', '/', '?', '!', '\'', '\"', '\*'] #in u_comicname or '/' in u_comicname or ',' in u_comicname or '?' in u_comicname:
for dbd in filebad:
if dbd in seriesfilename:
if any([dbd == '/', dbd == '*']):
repthechar = '-'
else:
repthechar = ''
seriesfilename = seriesfilename.replace(dbd, repthechar)
logger.fdebug('Altering series name due to filenaming restrictions: ' + seriesfilename)
publisher = re.sub('!', '', publisher)
file_values = {'$Series': seriesfilename,
'$Issue': prettycomiss,
'$Year': issueyear,
'$series': series.lower(),
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion,
'$monthname': month_name,
'$month': month,
'$Annual': 'Annual',
'$Type': booktype
}
extensions = ('.cbr', '.cbz', '.cb7')
if ofilename.lower().endswith(extensions):
path, ext = os.path.splitext(ofilename)
if file_format == '':
logger.fdebug('Rename Files is not enabled - keeping original filename.')
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
chunk_file_format = re.sub('[()|[]]', '', chunk_file_format).strip()
nfilename = helpers.replace_all(chunk_file_format, file_values)
if mylar.CONFIG.REPLACE_SPACES:
#mylar.CONFIG.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.CONFIG.REPLACE_CHAR)
nfilename = re.sub('[\,\:]', '', nfilename) + ext.lower()
logger.fdebug('New Filename: ' + nfilename)
if mylar.CONFIG.LOWERCASE_FILENAMES:
nfilename = nfilename.lower()
dst = os.path.join(comlocation, nfilename)
else:
dst = os.path.join(comlocation, nfilename)
logger.fdebug('Source: ' + ofilename)
logger.fdebug('Destination: ' + dst)
rename_this = {"destination_dir": dst,
"nfilename": nfilename,
"issueid": issueid,
"comicid": comicid}
return rename_this
logger.error(
"Something went wrong - I can't add the issue information into my DB."
)
myDB.action("DELETE FROM comics WHERE ComicID=?",
[comicid])
return
n += 1
#figure publish dates here...
styear = str(SeriesYear)
#if SeriesYear == '0000':
# styear = firstdate[:4]
if firstdate[5:7] == '00':
stmonth = "?"
else:
stmonth = helpers.fullmonth(firstdate[5:7])
ltyear = re.sub('/s', '', latestdate[:4])
if latestdate[5:7] == '00':
ltmonth = "?"
else:
ltmonth = helpers.fullmonth(latestdate[5:7])
#try to determine if it's an 'actively' published comic from above dates
#threshold is if it's within a month (<45 days) let's assume it's recent.
c_date = datetime.date(int(latestdate[:4]), int(latestdate[5:7]), 1)
n_date = datetime.date.today()
recentchk = (n_date - c_date).days
#print ("recentchk: " + str(recentchk))
if recentchk <= 55:
lastpubdate = 'Present'
else:
def Process_next(self, comicid, issueid, issuenumOG, ml=None):
annchk = "no"
extensions = ('.cbr', '.cbz')
myDB = db.DBConnection()
comicnzb = myDB.action("SELECT * from comics WHERE comicid=?",
[comicid]).fetchone()
issuenzb = myDB.action(
"SELECT * from issues WHERE issueid=? AND comicid=? AND ComicName NOT NULL",
[issueid, comicid]).fetchone()
logger.fdebug('issueid: ' + str(issueid))
logger.fdebug('issuenumOG: ' + str(issuenumOG))
if issuenzb is None:
issuenzb = myDB.action(
"SELECT * from annuals WHERE issueid=? and comicid=?",
[issueid, comicid]).fetchone()
annchk = "yes"
#issueno = str(issuenum).split('.')[0]
#new CV API - removed all decimals...here we go AGAIN!
issuenum = issuenzb['Issue_Number']
issue_except = 'None'
if 'au' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AU'
elif 'ai' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AI'
elif 'inh' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = '.INH'
elif 'now' in issuenum.lower() and issuenum[:1].isdigit():
if '!' in issuenum: issuenum = re.sub('\!', '', issuenum)
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = '.NOW'
if '.' in issuenum:
iss_find = issuenum.find('.')
iss_b4dec = issuenum[:iss_find]
iss_decval = issuenum[iss_find + 1:]
if int(iss_decval) == 0:
iss = iss_b4dec
issdec = int(iss_decval)
issueno = str(iss)
self._log("Issue Number: " + str(issueno), logger.DEBUG)
logger.fdebug("Issue Number: " + str(issueno))
else:
if len(iss_decval) == 1:
iss = iss_b4dec + "." + iss_decval
issdec = int(iss_decval) * 10
else:
iss = iss_b4dec + "." + iss_decval.rstrip('0')
issdec = int(iss_decval.rstrip('0')) * 10
issueno = iss_b4dec
self._log("Issue Number: " + str(iss), logger.DEBUG)
logger.fdebug("Issue Number: " + str(iss))
else:
iss = issuenum
issueno = str(iss)
# issue zero-suppression here
if mylar.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00"
logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N))
if str(len(issueno)) > 1:
if int(issueno) < 10:
self._log("issue detected less than 10", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log(
"Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) +
". Issue will be set as : " + str(prettycomiss),
logger.DEBUG)
elif int(issueno) >= 10 and int(issueno) < 100:
self._log("issue detected greater than 10, but less than 100",
logger.DEBUG)
if mylar.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log(
"Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) +
".Issue will be set as : " + str(prettycomiss),
logger.DEBUG)
else:
self._log("issue detected greater than 100", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(issueno)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log(
"Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) +
". Issue will be set as : " + str(prettycomiss),
logger.DEBUG)
else:
prettycomiss = str(issueno)
self._log(
"issue length error - cannot determine length. Defaulting to None: "
+ str(prettycomiss), logger.DEBUG)
if annchk == "yes":
self._log("Annual detected.")
logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss))
issueyear = issuenzb['IssueDate'][:4]
self._log("Issue Year: " + str(issueyear), logger.DEBUG)
logger.fdebug("Issue Year : " + str(issueyear))
month = issuenzb['IssueDate'][5:7].replace('-', '').strip()
month_name = helpers.fullmonth(month)
# comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
publisher = comicnzb['ComicPublisher']
self._log("Publisher: " + publisher, logger.DEBUG)
logger.fdebug("Publisher: " + str(publisher))
#we need to un-unicode this to make sure we can write the filenames properly for spec.chars
series = comicnzb['ComicName'].encode('ascii', 'ignore').strip()
self._log("Series: " + series, logger.DEBUG)
logger.fdebug("Series: " + str(series))
seriesyear = comicnzb['ComicYear']
self._log("Year: " + seriesyear, logger.DEBUG)
logger.fdebug("Year: " + str(seriesyear))
comlocation = comicnzb['ComicLocation']
self._log("Comic Location: " + comlocation, logger.DEBUG)
logger.fdebug("Comic Location: " + str(comlocation))
comversion = comicnzb['ComicVersion']
self._log("Comic Version: " + str(comversion), logger.DEBUG)
logger.fdebug("Comic Version: " + str(comversion))
if comversion is None:
comversion = 'None'
#if comversion is None, remove it so it doesn't populate with 'None'
if comversion == 'None':
chunk_f_f = re.sub('\$VolumeN', '', mylar.FILE_FORMAT)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
self._log(
"No version # found for series - tag will not be available for renaming.",
logger.DEBUG)
logger.fdebug(
"No version # found for series, removing from filename")
logger.fdebug("new format is now: " + str(chunk_file_format))
else:
chunk_file_format = mylar.FILE_FORMAT
if annchk == "no":
chunk_f_f = re.sub('\$Annual', '', chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('not an annual - removing from filename paramaters')
logger.fdebug('new format: ' + str(chunk_file_format))
else:
logger.fdebug('chunk_file_format is: ' + str(chunk_file_format))
if '$Annual' not in chunk_file_format:
#if it's an annual, but $Annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
prettycomiss = "Annual " + str(prettycomiss)
logger.fdebug('prettycomiss: ' + str(prettycomiss))
ofilename = None
#if meta-tagging is not enabled, we need to declare the check as being fail
#if meta-tagging is enabled, it gets changed just below to a default of pass
pcheck = "fail"
#tag the meta.
if mylar.ENABLE_META:
self._log("Metatagging enabled - proceeding...")
logger.fdebug("Metatagging enabled - proceeding...")
pcheck = "pass"
try:
import cmtagmylar
if ml is None:
pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid)
else:
pcheck = cmtagmylar.run(self.nzb_folder,
issueid=issueid,
manual="yes",
filename=ml['ComicLocation'])
except ImportError:
logger.fdebug(
"comictaggerlib not found on system. Ensure the ENTIRE lib directory is located within mylar/lib/comictaggerlib/"
)
logger.fdebug(
"continuing with PostProcessing, but I'm not using metadata."
)
pcheck = "fail"
if pcheck == "fail":
self._log(
"Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging..."
)
logger.fdebug(
"Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging..."
)
elif pcheck == "unrar error":
self._log(
"This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy."
)
logger.error(
"This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy."
)
return self.log
else:
otofilename = pcheck
self._log("Sucessfully wrote metadata to .cbz - Continuing..")
logger.fdebug("Sucessfully wrote metadata to .cbz (" +
str(otofilename) + ") - Continuing..")
#Run Pre-script
if mylar.ENABLE_PRE_SCRIPTS:
nzbn = self.nzb_name #original nzb name
nzbf = self.nzb_folder #original nzb folder
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_pre_scripts(nzbn, nzbf, seriesmetadata)
#rename file and move to new path
#nfilename = series + " " + issueno + " (" + seriesyear + ")"
file_values = {
'$Series': series,
'$Issue': prettycomiss,
'$Year': issueyear,
'$series': series.lower(),
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion,
'$monthname': month_name,
'$month': month,
'$Annual': 'Annual'
}
#if it's a Manual Run, use the ml['ComicLocation'] for the exact filename.
if ml is None:
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
ofilename = filename
path, ext = os.path.splitext(ofilename)
else:
if pcheck == "fail":
otofilename = ml['ComicLocation']
logger.fdebug('otofilename:' + str(otofilename))
odir, ofilename = os.path.split(otofilename)
logger.fdebug('ofilename: ' + str(ofilename))
path, ext = os.path.splitext(ofilename)
logger.fdebug('path: ' + str(path))
logger.fdebug('ext:' + str(ext))
if ofilename is None:
logger.error(
u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that "
+ str(self.nzb_folder) +
" exists and is the correct location.")
return
self._log("Original Filename: " + ofilename, logger.DEBUG)
self._log("Original Extension: " + ext, logger.DEBUG)
logger.fdebug("Original Filname: " + str(ofilename))
logger.fdebug("Original Extension: " + str(ext))
if mylar.FILE_FORMAT == '' or not mylar.RENAME_FILES:
self._log(
"Rename Files isn't enabled...keeping original filename.",
logger.DEBUG)
logger.fdebug(
"Rename Files isn't enabled - keeping original filename.")
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
nfilename = helpers.replace_all(chunk_file_format, file_values)
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
nfilename = re.sub('[\,\:\?]', '', nfilename)
nfilename = re.sub('[\/]', '-', nfilename)
self._log("New Filename: " + nfilename, logger.DEBUG)
logger.fdebug("New Filename: " + str(nfilename))
src = os.path.join(self.nzb_folder, ofilename)
filechecker.validateAndCreateDirectory(comlocation, True)
if mylar.LOWERCASE_FILENAMES:
dst = (comlocation + "/" + nfilename + ext).lower()
else:
dst = comlocation + "/" + nfilename + ext.lower()
self._log("Source:" + src, logger.DEBUG)
self._log("Destination:" + dst, logger.DEBUG)
logger.fdebug("Source: " + str(src))
logger.fdebug("Destination: " + str(dst))
if ml is None:
#non-manual run moving/deleting...
logger.fdebug('self.nzb_folder: ' + self.nzb_folder)
logger.fdebug('ofilename:' + str(ofilename))
logger.fdebug('nfilename:' + str(nfilename + ext))
os.rename(os.path.join(self.nzb_folder, str(ofilename)),
os.path.join(self.nzb_folder, str(nfilename + ext)))
src = os.path.join(self.nzb_folder, str(nfilename + ext))
try:
shutil.move(src, dst)
except (OSError, IOError):
self._log(
"Failed to move directory - check directories and manually re-run.",
logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log(
"Failed to remove temporary directory - check directory and manually re-run.",
logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
self._log("Removed temporary directory : " + str(self.nzb_folder),
logger.DEBUG)
else:
#Manual Run, this is the portion.
logger.fdebug("Renaming " +
os.path.join(self.nzb_folder, str(ofilename)) +
" ..to.. " +
os.path.join(self.nzb_folder, str(nfilename + ext)))
os.rename(os.path.join(self.nzb_folder, str(ofilename)),
os.path.join(self.nzb_folder, str(nfilename + ext)))
src = os.path.join(self.nzb_folder, str(nfilename + ext))
logger.fdebug("Moving " + src + " ... to ... " + dst)
try:
shutil.move(src, dst)
except (OSError, IOError):
logger.fdebug(
"Failed to move directory - check directories and manually re-run."
)
logger.fdebug("Post-Processing ABORTED.")
return
logger.fdebug("Successfully moved to : " + dst)
#tidyup old path
#try:
# os.remove(os.path.join(self.nzb_folder, str(ofilename)))
# logger.fdebug("Deleting : " + os.path.join(self.nzb_folder, str(ofilename)))
#except (OSError, IOError):
# logger.fdebug("Failed to remove temporary directory - check directory and manually re-run.")
# logger.fdebug("Post-Processing ABORTED.")
# return
#logger.fdebug("Removed temporary directory : " + str(self.nzb_folder))
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
#update snatched table to change status to Downloaded
if annchk == "no":
updater.foundsearch(comicid, issueid, down='True')
dispiss = 'issue: ' + str(issuenumOG)
else:
updater.foundsearch(comicid, issueid, mode='want_ann', down='True')
dispiss = 'annual issue: ' + str(issuenumOG)
#force rescan of files
updater.forceRescan(comicid)
logger.info(u"Post-Processing completed for: " + series + " " +
dispiss)
self._log(u"Post Processing SUCCESSFULL! ", logger.DEBUG)
# retrieve/create the corresponding comic objects
if mylar.ENABLE_EXTRA_SCRIPTS:
folderp = str(dst) #folder location after move/rename
nzbn = self.nzb_name #original nzb name
filen = str(nfilename + ext) #new filename
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_extra_scripts(nzbn, self.nzb_folder, filen, folderp,
seriesmetadata)
if ml is not None:
return self.log
else:
if mylar.PROWL_ENABLED:
pushmessage = series + '(' + issueyear + ') - issue #' + issuenumOG
logger.info(u"Prowl request")
prowl = notifiers.PROWL()
prowl.notify(pushmessage,
"Download and Postprocessing completed")
if mylar.NMA_ENABLED:
nma = notifiers.NMA()
nma.notify(series, str(issueyear), str(issuenumOG))
if mylar.PUSHOVER_ENABLED:
pushmessage = series + ' (' + str(
issueyear) + ') - issue #' + str(issuenumOG)
logger.info(u"Pushover request")
pushover = notifiers.PUSHOVER()
pushover.notify(pushmessage,
"Download and Post-Processing completed")
if mylar.BOXCAR_ENABLED:
boxcar = notifiers.BOXCAR()
boxcar.notify(series, str(issueyear), str(issuenumOG))
return self.log
