def markissues(self, action=None, **args):
myDB = db.DBConnection()
issuesToAdd = []
issuestoArchive = []
if action == 'WantedNew':
newaction = 'Wanted'
else:
newaction = action
for IssueID in args:
if IssueID is None: continue
else:
mi = myDB.action("SELECT * FROM issues WHERE IssueID=?",[IssueID]).fetchone()
miyr = myDB.action("SELECT ComicYear FROM comics WHERE ComicID=?", [mi['ComicID']]).fetchone()
if action == 'Downloaded':
if mi['Status'] == "Skipped" or mi['Status'] == "Wanted":
logger.info(u"Cannot change status to %s as comic is not Snatched or Downloaded" % (newaction))
continue
elif action == 'Archived':
logger.info(u"Marking %s %s as %s" % (mi['ComicName'], mi['Issue_Number'], newaction))
#updater.forceRescan(mi['ComicID'])
issuestoArchive.append(IssueID)
elif action == 'Wanted':
logger.info(u"Marking %s %s as %s" % (mi['ComicName'], mi['Issue_Number'], newaction))
issuesToAdd.append(IssueID)
controlValueDict = {"IssueID": IssueID}
newValueDict = {"Status": newaction}
myDB.upsert("issues", newValueDict, controlValueDict)
if len(issuestoArchive) > 0:
updater.forceRescan(mi['ComicID'])
if len(issuesToAdd) > 0:
logger.debug("Marking issues: %s as Wanted" % issuesToAdd)
threading.Thread(target=search.searchIssueIDList, args=[issuesToAdd]).start()
#if IssueID:
raise cherrypy.HTTPRedirect("artistPage?ComicID=%s" % mi['ComicID'])
def archivefiles(comicid, ogdir, ogcname):
myDB = db.DBConnection()
# if move files isn't enabled, let's set all found comics to Archive status :)
result = myDB.select("SELECT * FROM importresults WHERE ComicName=?",
[ogcname])
if result is None:
pass
else:
scandir = []
for res in result:
if any(
[os.path.dirname(res['ComicLocation']) in x for x in scandir]):
pass
else:
scandir.append(os.path.dirname(res['ComicLocation']))
for sdir in scandir:
logger.info(
'Updating issue information and setting status to Archived for location: '
+ sdir)
updater.forceRescan(
comicid,
archive=sdir) #send to rescanner with archive mode turned on
logger.info('Now scanning in files.')
updater.forceRescan(comicid)
return
def archivefiles(comicid, comlocation, imported):
myDB = db.DBConnection()
# if move files isn't enabled, let's set all found comics to Archive status :)
try:
imported = ast.literal_eval(imported)
except Exception as e:
logger.warn('[%s] Error encountered converting import data' % e)
ComicName = imported['ComicName']
impres = imported['filelisting']
if impres is not None:
scandir = []
for impr in impres:
srcimp = impr['comiclocation']
orig_filename = impr['comicfilename']
if not any([
os.path.abspath(os.path.join(srcimp, os.pardir)) == x
for x in scandir
]):
scandir.append(os.path.abspath(os.path.join(srcimp,
os.pardir)))
for sdir in scandir:
logger.info(
'Updating issue information and setting status to Archived for location: '
+ sdir)
updater.forceRescan(
comicid,
archive=sdir) #send to rescanner with archive mode turned on
logger.info('Now scanning in files.')
updater.forceRescan(comicid)
for result in impres:
try:
res = result['import_id']
except:
#if it's an 'older' import that wasn't imported, just make it a basic match so things can move and update properly.
controlValue = {
"ComicFilename": result['comicfilename'],
"SRID": imported['srid']
}
newValue = {"Status": "Imported", "ComicID": comicid}
else:
controlValue = {
"impID": result['import_id'],
"ComicFilename": result['comicfilename']
}
newValue = {
"Status": "Imported",
"SRID": imported['srid'],
"ComicID": comicid
}
myDB.upsert("importresults", newValue, controlValue)
return
def archivefiles(comicid, comlocation, imported):
myDB = db.DBConnection()
# if move files isn't enabled, let's set all found comics to Archive status :)
try:
imported = ast.literal_eval(imported)
except ValueError:
pass
ComicName = imported['ComicName']
impres = imported['filelisting']
if impres is not None:
scandir = []
for impr in impres:
srcimp = impr['comiclocation']
orig_filename = impr['comicfilename']
if not any([os.path.abspath(os.path.join(srcimp, os.pardir)) == x for x in scandir]):
scandir.append(os.path.abspath(os.path.join(srcimp, os.pardir)))
for sdir in scandir:
logger.info('Updating issue information and setting status to Archived for location: ' + sdir)
updater.forceRescan(comicid, archive=sdir) #send to rescanner with archive mode turned on
logger.info('Now scanning in files.')
updater.forceRescan(comicid)
for result in impres:
try:
res = result['import_id']
except:
#if it's an 'older' import that wasn't imported, just make it a basic match so things can move and update properly.
controlValue = {"ComicFilename": result['comicfilename'],
"SRID": imported['srid']}
newValue = {"Status": "Imported",
"ComicID": comicid}
else:
controlValue = {"impID": result['import_id'],
"ComicFilename": result['comicfilename']}
newValue = {"Status": "Imported",
"SRID": imported['srid'],
"ComicID": comicid}
myDB.upsert("importresults", newValue, controlValue)
return
def archivefiles(comicid, ogdir, ogcname):
myDB = db.DBConnection()
# if move files isn't enabled, let's set all found comics to Archive status :)
result = myDB.select("SELECT * FROM importresults WHERE ComicName=?", [ogcname])
if result is None:
pass
else:
scandir = []
for res in result:
if any([os.path.dirname(res['ComicLocation']) in x for x in scandir]):
pass
else:
scandir.append(os.path.dirname(res['ComicLocation']))
for sdir in scandir:
logger.info('Updating issue information and setting status to Archived for location: ' + sdir)
updater.forceRescan(comicid, archive=sdir) #send to rescanner with archive mode turned on
logger.info('Now scanning in files.')
updater.forceRescan(comicid)
return
def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None, queue=None):
if cron and not mylar.LIBRARYSCAN:
return
if not dir:
dir = mylar.CONFIG.COMIC_DIR
if not os.path.isdir(dir):
logger.warn('Cannot find directory: %s. Not scanning' % dir)
return "Fail"
logger.info('Scanning comic directory: %s' % dir)
basedir = dir
comic_list = []
failure_list = []
utter_failure_list = []
comiccnt = 0
extensions = ('cbr','cbz')
cv_location = []
cbz_retry = 0
mylar.IMPORT_STATUS = 'Now attempting to parse files for additional information'
myDB = db.DBConnection()
#mylar.IMPORT_PARSED_COUNT #used to count what #/totalfiles the filename parser is currently on
for r, d, f in os.walk(dir):
for files in f:
mylar.IMPORT_FILES +=1
if any(files.lower().endswith('.' + x.lower()) for x in extensions):
comicpath = os.path.join(r, files)
if mylar.CONFIG.IMP_PATHS is True:
if myDB.select('SELECT * FROM comics JOIN issues WHERE issues.Status="Downloaded" AND ComicLocation=? AND issues.Location=?', [r, files]):
logger.info('Skipped known issue path: %s' % comicpath)
continue
comic = files
if not os.path.exists(comicpath):
logger.fdebug(f'''Comic: {comic} doesn't actually exist - assuming it is a symlink to a nonexistant path.''')
continue
comicsize = os.path.getsize(comicpath)
logger.fdebug('Comic: ' + comic + ' [' + comicpath + '] - ' + str(comicsize) + ' bytes')
try:
t = filechecker.FileChecker(dir=r, file=comic)
results = t.listFiles()
#logger.info(results)
#'type': re.sub('\.','', filetype).strip(),
#'sub': path_list,
#'volume': volume,
#'match_type': match_type,
#'comicfilename': filename,
#'comiclocation': clocation,
#'series_name': series_name,
#'series_volume': issue_volume,
#'series_year': issue_year,
#'justthedigits': issue_number,
#'annualcomicid': annual_comicid,
#'scangroup': scangroup}
if results:
resultline = '[PARSE-' + results['parse_status'].upper() + ']'
resultline += '[SERIES: ' + results['series_name'] + ']'
if results['series_volume'] is not None:
resultline += '[VOLUME: ' + results['series_volume'] + ']'
if results['issue_year'] is not None:
resultline += '[ISSUE YEAR: ' + str(results['issue_year']) + ']'
if results['issue_number'] is not None:
resultline += '[ISSUE #: ' + results['issue_number'] + ']'
logger.fdebug(resultline)
else:
logger.fdebug('[PARSED] FAILURE.')
continue
# We need the unicode path to use for logging, inserting into database
unicode_comic_path = comicpath
if results['parse_status'] == 'success':
comic_list.append({'ComicFilename': comic,
'ComicLocation': comicpath,
'ComicSize': comicsize,
'Unicode_ComicLocation': unicode_comic_path,
'parsedinfo': {'series_name': results['series_name'],
'series_volume': results['series_volume'],
'issue_year': results['issue_year'],
'issue_number': results['issue_number']}
})
comiccnt +=1
mylar.IMPORT_PARSED_COUNT +=1
else:
failure_list.append({'ComicFilename': comic,
'ComicLocation': comicpath,
'ComicSize': comicsize,
'Unicode_ComicLocation': unicode_comic_path,
'parsedinfo': {'series_name': results['series_name'],
'series_volume': results['series_volume'],
'issue_year': results['issue_year'],
'issue_number': results['issue_number']}
})
mylar.IMPORT_FAILURE_COUNT +=1
if comic.endswith('.cbz'):
cbz_retry +=1
except Exception as e:
logger.info('bang')
utter_failure_list.append({'ComicFilename': comic,
'ComicLocation': comicpath,
'ComicSize': comicsize,
'Unicode_ComicLocation': unicode_comic_path,
'parsedinfo': None,
'error': e
})
logger.info('[' + str(e) + '] FAILURE encountered. Logging the error for ' + comic + ' and continuing...')
mylar.IMPORT_FAILURE_COUNT +=1
if comic.endswith('.cbz'):
cbz_retry +=1
continue
if 'cvinfo' in files:
cv_location.append(r)
logger.fdebug('CVINFO found: ' + os.path.join(r))
mylar.IMPORT_TOTALFILES = comiccnt
logger.info('I have successfully discovered & parsed a total of ' + str(comiccnt) + ' files....analyzing now')
logger.info('I have not been able to determine what ' + str(len(failure_list)) + ' files are')
logger.info('However, ' + str(cbz_retry) + ' out of the ' + str(len(failure_list)) + ' files are in a cbz format, which may contain metadata.')
logger.info('[ERRORS] I have encountered ' + str(len(utter_failure_list)) + ' file-scanning errors during the scan, but have recorded the necessary information.')
mylar.IMPORT_STATUS = 'Successfully parsed ' + str(comiccnt) + ' files'
#return queue.put(valreturn)
if len(utter_failure_list) > 0:
logger.fdebug('Failure list: %s' % utter_failure_list)
#let's load in the watchlist to see if we have any matches.
logger.info("loading in the watchlist to see if a series is being watched already...")
watchlist = myDB.select("SELECT * from comics")
ComicName = []
DisplayName = []
ComicYear = []
ComicPublisher = []
ComicTotal = []
ComicID = []
ComicLocation = []
AltName = []
watchcnt = 0
watch_kchoice = []
watchchoice = {}
import_by_comicids = []
import_comicids = {}
for watch in watchlist:
#use the comicname_filesafe to start
watchdisplaycomic = watch['ComicName']
# let's clean up the name, just in case for comparison purposes...
watchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '', watch['ComicName_Filesafe'])
#watchcomic = re.sub('\s+', ' ', str(watchcomic)).strip()
if ' the ' in watchcomic.lower():
#drop the 'the' from the watchcomic title for proper comparisons.
watchcomic = watchcomic[-4:]
alt_chk = "no" # alt-checker flag (default to no)
# account for alternate names as well
if watch['AlternateSearch'] is not None and watch['AlternateSearch'] != 'None':
altcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '', watch['AlternateSearch'])
#altcomic = re.sub('\s+', ' ', str(altcomic)).strip()
AltName.append(altcomic)
alt_chk = "yes" # alt-checker flag
ComicName.append(watchcomic)
DisplayName.append(watchdisplaycomic)
ComicYear.append(watch['ComicYear'])
ComicPublisher.append(watch['ComicPublisher'])
ComicTotal.append(watch['Total'])
ComicID.append(watch['ComicID'])
ComicLocation.append(watch['ComicLocation'])
watchcnt+=1
logger.info("Successfully loaded " + str(watchcnt) + " series from your watchlist.")
ripperlist=['digital-',
'empire',
'dcp']
watchfound = 0
datelist = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
# datemonth = {'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9,'ten':10,'eleven':$
# #search for number as text, and change to numeric
# for numbs in basnumbs:
# #logger.fdebug("numbs:" + str(numbs))
# if numbs in ComicName.lower():
# numconv = basnumbs[numbs]
# #logger.fdebug("numconv: " + str(numconv))
issueid_list = []
cvscanned_loc = None
cvinfo_CID = None
cnt = 0
mylar.IMPORT_STATUS = '[0%] Now parsing individual filenames for metadata if available'
for i in comic_list:
mylar.IMPORT_STATUS = '[' + str(cnt) + '/' + str(comiccnt) + '] Now parsing individual filenames for metadata if available'
logger.fdebug('Analyzing : ' + i['ComicFilename'])
comfilename = i['ComicFilename']
comlocation = i['ComicLocation']
issueinfo = None
#probably need to zero these issue-related metadata to None so we can pick the best option
issuevolume = None
#Make sure cvinfo is checked for FIRST (so that CID can be attached to all files properly thereafter as they're scanned in)
if os.path.dirname(comlocation) in cv_location and os.path.dirname(comlocation) != cvscanned_loc:
#if comfilename == 'cvinfo':
logger.info('comfilename: ' + comfilename)
logger.info('cvscanned_loc: ' + str(cv_location))
logger.info('comlocation: ' + os.path.dirname(comlocation))
#if cvscanned_loc != comlocation:
try:
with open(os.path.join(os.path.dirname(comlocation), 'cvinfo')) as f:
urllink = f.readline()
if urllink:
cid = urllink.strip()
pattern = re.compile(r"^.*?\b(49|4050)-(?P<num>\d{2,})\b.*$", re.I)
match = pattern.match(cid)
if match:
cvinfo_CID = match.group("num")
logger.info('CVINFO file located within directory. Attaching everything in directory that is valid to ComicID: ' + str(cvinfo_CID))
#store the location of the cvinfo so it's applied to the correct directory (since we're scanning multile direcorties usually)
cvscanned_loc = os.path.dirname(comlocation)
else:
logger.error("Could not read cvinfo file properly (or it does not contain any data)")
except (OSError, IOError):
logger.error("Could not read cvinfo file properly (or it does not contain any data)")
#else:
# don't scan in it again if it's already been done initially
# continue
if mylar.CONFIG.IMP_METADATA:
#if read tags is enabled during import, check here.
if i['ComicLocation'].endswith('.cbz'):
logger.fdebug('[IMPORT-CBZ] Metatagging checking enabled.')
logger.info('[IMPORT-CBZ} Attempting to read tags present in filename: ' + i['ComicLocation'])
try:
issueinfo = helpers.IssueDetails(i['ComicLocation'], justinfo=True)
except:
logger.fdebug('[IMPORT-CBZ] Unable to retrieve metadata - possibly doesn\'t exist. Ignoring meta-retrieval')
pass
else:
logger.info('issueinfo: ' + str(issueinfo))
if issueinfo is None or issueinfo['metadata'] is None:
logger.fdebug('[IMPORT-CBZ] No valid metadata contained within filename. Dropping down to parsing the filename itself.')
pass
else:
issuenotes_id = None
logger.info('[IMPORT-CBZ] Successfully retrieved some tags. Lets see what I can figure out.')
comicname = issueinfo['metadata']['series']
if comicname is not None:
logger.fdebug('[IMPORT-CBZ] Series Name: ' + comicname)
as_d = filechecker.FileChecker()
as_dyninfo = as_d.dynamic_replace(comicname)
logger.fdebug('Dynamic-ComicName: ' + as_dyninfo['mod_seriesname'])
else:
logger.fdebug('[IMPORT-CBZ] No series name found within metadata. This is bunk - dropping down to file parsing for usable information.')
issueinfo = None
issue_number = None
if issueinfo is not None:
try:
issueyear = issueinfo['metadata']['year']
except:
issueyear = None
#if the issue number is a non-numeric unicode string, this will screw up along with impID
issue_number = issueinfo['metadata']['issue_number']
if issue_number is not None:
logger.fdebug('[IMPORT-CBZ] Issue Number: ' + issue_number)
else:
issue_number = i['parsed']['issue_number']
if 'annual' in comicname.lower() or 'annual' in comfilename.lower():
if issue_number is None or issue_number == 'None':
logger.info('Annual detected with no issue number present within metadata. Assuming year as issue.')
try:
issue_number = 'Annual ' + str(issueyear)
except:
issue_number = 'Annual ' + i['parsed']['issue_year']
else:
logger.info('Annual detected with issue number present within metadata.')
if 'annual' not in issue_number.lower():
issue_number = 'Annual ' + issue_number
mod_series = re.sub('annual', '', comicname, flags=re.I).strip()
else:
mod_series = comicname
logger.fdebug('issue number SHOULD Be: ' + issue_number)
try:
issuetitle = issueinfo['metadata']['title']
except:
issuetitle = None
try:
issueyear = issueinfo['metadata']['year']
except:
issueyear = None
try:
issuevolume = str(issueinfo['metadata']['volume'])
if all([issuevolume is not None, issuevolume != 'None', not issuevolume.lower().startswith('v')]):
issuevolume = 'v' + str(issuevolume)
if any([issuevolume is None, issuevolume == 'None']):
logger.info('EXCEPT] issue volume is NONE')
issuevolume = None
else:
logger.fdebug('[TRY]issue volume is: ' + str(issuevolume))
except:
logger.fdebug('[EXCEPT]issue volume is: ' + str(issuevolume))
issuevolume = None
if any([comicname is None, comicname == 'None', issue_number is None, issue_number == 'None']):
logger.fdebug('[IMPORT-CBZ] Improperly tagged file as the metatagging is invalid. Ignoring meta and just parsing the filename.')
issueinfo = None
pass
else:
# if used by ComicTagger, Notes field will have the IssueID.
issuenotes = issueinfo['metadata']['notes']
logger.fdebug('[IMPORT-CBZ] Notes: ' + issuenotes)
if issuenotes is not None and issuenotes != 'None':
if 'Issue ID' in issuenotes:
st_find = issuenotes.find('Issue ID')
tmp_issuenotes_id = re.sub("[^0-9]", " ", issuenotes[st_find:]).strip()
if tmp_issuenotes_id.isdigit():
issuenotes_id = tmp_issuenotes_id
logger.fdebug('[IMPORT-CBZ] Successfully retrieved CV IssueID for ' + comicname + ' #' + issue_number + ' [' + str(issuenotes_id) + ']')
elif 'CVDB' in issuenotes:
st_find = issuenotes.find('CVDB')
tmp_issuenotes_id = re.sub("[^0-9]", " ", issuenotes[st_find:]).strip()
if tmp_issuenotes_id.isdigit():
issuenotes_id = tmp_issuenotes_id
logger.fdebug('[IMPORT-CBZ] Successfully retrieved CV IssueID for ' + comicname + ' #' + issue_number + ' [' + str(issuenotes_id) + ']')
else:
logger.fdebug('[IMPORT-CBZ] Unable to retrieve IssueID from meta-tagging. If there is other metadata present I will use that.')
logger.fdebug('[IMPORT-CBZ] Adding ' + comicname + ' to the import-queue!')
#impid = comicname + '-' + str(issueyear) + '-' + str(issue_number) #com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
impid = str(random.randint(1000000,99999999))
logger.fdebug('[IMPORT-CBZ] impid: ' + str(impid))
#make sure we only add in those issueid's which don't already have a comicid attached via the cvinfo scan above (this is for reverse-lookup of issueids)
issuepopulated = False
if cvinfo_CID is None:
if issuenotes_id is None:
logger.info('[IMPORT-CBZ] No ComicID detected where it should be. Bypassing this metadata entry and going the parsing route [' + comfilename + ']')
else:
#we need to store the impid here as well so we can look it up.
issueid_list.append({'issueid': issuenotes_id,
'importinfo': {'impid': impid,
'comicid': None,
'comicname': comicname,
'dynamicname': as_dyninfo['mod_seriesname'],
'comicyear': issueyear,
'issuenumber': issue_number,
'volume': issuevolume,
'comfilename': comfilename,
'comlocation': comlocation}
})
mylar.IMPORT_CID_COUNT +=1
issuepopulated = True
if issuepopulated == False:
if cvscanned_loc == os.path.dirname(comlocation):
cv_cid = cvinfo_CID
logger.fdebug('[IMPORT-CBZ] CVINFO_COMICID attached : ' + str(cv_cid))
else:
cv_cid = None
import_by_comicids.append({
"impid": impid,
"comicid": cv_cid,
"watchmatch": None,
"displayname": mod_series,
"comicname": comicname,
"dynamicname": as_dyninfo['mod_seriesname'],
"comicyear": issueyear,
"issuenumber": issue_number,
"volume": issuevolume,
"issueid": issuenotes_id,
"comfilename": comfilename,
"comlocation": comlocation
})
mylar.IMPORT_CID_COUNT +=1
else:
pass
#logger.fdebug(i['ComicFilename'] + ' is not in a metatagged format (cbz). Bypassing reading of the metatags')
if issueinfo is None:
if i['parsedinfo']['issue_number'] is None:
if 'annual' in i['parsedinfo']['series_name'].lower():
logger.fdebug('Annual detected with no issue number present. Assuming year as issue.')##1 issue')
if i['parsedinfo']['issue_year'] is not None:
issuenumber = 'Annual ' + str(i['parsedinfo']['issue_year'])
else:
issuenumber = 'Annual 1'
else:
issuenumber = i['parsedinfo']['issue_number']
if 'annual' in i['parsedinfo']['series_name'].lower():
mod_series = re.sub('annual', '', i['parsedinfo']['series_name'], flags=re.I).strip()
logger.fdebug('Annual detected with no issue number present. Assuming year as issue.')##1 issue')
if i['parsedinfo']['issue_number'] is not None:
issuenumber = 'Annual ' + str(i['parsedinfo']['issue_number'])
else:
if i['parsedinfo']['issue_year'] is not None:
issuenumber = 'Annual ' + str(i['parsedinfo']['issue_year'])
else:
issuenumber = 'Annual 1'
else:
mod_series = i['parsedinfo']['series_name']
issuenumber = i['parsedinfo']['issue_number']
logger.fdebug('[' + mod_series + '] Adding to the import-queue!')
isd = filechecker.FileChecker()
is_dyninfo = isd.dynamic_replace(mod_series) #helpers.conversion(mod_series))
logger.fdebug('Dynamic-ComicName: ' + is_dyninfo['mod_seriesname'])
#impid = dispname + '-' + str(result_comyear) + '-' + str(comiss) #com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
impid = str(random.randint(1000000,99999999))
logger.fdebug("impid: " + str(impid))
if cvscanned_loc == os.path.dirname(comlocation):
cv_cid = cvinfo_CID
logger.fdebug('CVINFO_COMICID attached : ' + str(cv_cid))
else:
cv_cid = None
if issuevolume is None:
logger.fdebug('issue volume is : ' + str(issuevolume))
if i['parsedinfo']['series_volume'] is None:
issuevolume = None
else:
if str(i['parsedinfo']['series_volume'].lower()).startswith('v'):
issuevolume = i['parsedinfo']['series_volume']
else:
issuevolume = 'v' + str(i['parsedinfo']['series_volume'])
else:
logger.fdebug('issue volume not none : ' + str(issuevolume))
if issuevolume.lower().startswith('v'):
issuevolume = issuevolume
else:
issuevolume = 'v' + str(issuevolume)
logger.fdebug('IssueVolume is : ' + str(issuevolume))
import_by_comicids.append({
"impid": impid,
"comicid": cv_cid,
"issueid": None,
"watchmatch": None, #watchmatch (should be true/false if it already exists on watchlist)
"displayname": mod_series,
"comicname": i['parsedinfo']['series_name'],
"dynamicname": is_dyninfo['mod_seriesname'].lower(),
"comicyear": i['parsedinfo']['issue_year'],
"issuenumber": issuenumber, #issuenumber,
"volume": issuevolume,
"comfilename": comfilename,
"comlocation": comlocation #helpers.conversion(comlocation)
})
cnt+=1
#logger.fdebug('import_by_ids: ' + str(import_by_comicids))
#reverse lookup all of the gathered IssueID's in order to get the related ComicID
reverse_issueids = []
for x in issueid_list:
reverse_issueids.append(x['issueid'])
vals = []
if len(reverse_issueids) > 0:
mylar.IMPORT_STATUS = 'Now Reverse looking up ' + str(len(reverse_issueids)) + ' IssueIDs to get the ComicIDs'
vals = mylar.cv.getComic(None, 'import', comicidlist=reverse_issueids)
#logger.fdebug('vals returned:' + str(vals))
if len(watch_kchoice) > 0:
watchchoice['watchlist'] = watch_kchoice
#logger.fdebug("watchchoice: " + str(watchchoice))
logger.info("I have found " + str(watchfound) + " out of " + str(comiccnt) + " comics for series that are being watched.")
wat = 0
comicids = []
if watchfound > 0:
if mylar.CONFIG.IMP_MOVE:
logger.info('You checked off Move Files...so that\'s what I am going to do')
#check to see if Move Files is enabled.
#if not being moved, set the archive bit.
logger.fdebug('Moving files into appropriate directory')
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comlocation = watch_the_list['ComicLocation']
watch_comicid = watch_the_list['ComicID']
watch_comicname = watch_the_list['ComicName']
watch_comicyear = watch_the_list['ComicYear']
watch_comiciss = watch_the_list['ComicIssue']
logger.fdebug('ComicLocation: ' + watch_comlocation)
orig_comlocation = watch_the_list['OriginalLocation']
orig_filename = watch_the_list['OriginalFilename']
logger.fdebug('Orig. Location: ' + orig_comlocation)
logger.fdebug('Orig. Filename: ' + orig_filename)
#before moving check to see if Rename to Mylar structure is enabled.
if mylar.CONFIG.IMP_RENAME:
logger.fdebug('Renaming files according to configuration details : ' + str(mylar.CONFIG.FILE_FORMAT))
renameit = helpers.rename_param(watch_comicid, watch_comicname, watch_comicyear, watch_comiciss)
nfilename = renameit['nfilename']
dst_path = os.path.join(watch_comlocation, nfilename)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
else:
logger.fdebug('Renaming files not enabled, keeping original filename(s)')
dst_path = os.path.join(watch_comlocation, orig_filename)
#os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
#src = os.path.join(, str(nfilename + ext))
logger.fdebug('I am going to move ' + orig_comlocation + ' to ' + dst_path)
try:
shutil.move(orig_comlocation, dst_path)
except (OSError, IOError):
logger.info("Failed to move directory - check directories and manually re-run.")
wat+=1
else:
# if move files isn't enabled, let's set all found comics to Archive status :)
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comicid = watch_the_list['ComicID']
watch_issue = watch_the_list['ComicIssue']
logger.fdebug('ComicID: ' + str(watch_comicid))
logger.fdebug('Issue#: ' + str(watch_issue))
issuechk = myDB.selectone("SELECT * from issues where ComicID=? AND INT_IssueNumber=?", [watch_comicid, watch_issue]).fetchone()
if issuechk is None:
logger.fdebug('No matching issues for this comic#')
else:
logger.fdebug('...Existing status: ' + str(issuechk['Status']))
control = {"IssueID": issuechk['IssueID']}
values = {"Status": "Archived"}
logger.fdebug('...changing status of ' + str(issuechk['Issue_Number']) + ' to Archived ')
myDB.upsert("issues", values, control)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
wat+=1
if comicids is None: pass
else:
c_upd = len(comicids)
c = 0
while (c < c_upd ):
logger.fdebug('Rescanning.. ' + str(c))
updater.forceRescan(c)
if not len(import_by_comicids):
return "Completed"
if len(import_by_comicids) > 0 or len(vals) > 0:
#import_comicids['comic_info'] = import_by_comicids
#if vals:
# import_comicids['issueid_info'] = vals
#else:
# import_comicids['issueid_info'] = None
if vals:
cvimport_comicids = vals
import_cv_ids = len(vals)
else:
cvimport_comicids = None
import_cv_ids = 0
else:
import_cv_ids = 0
cvimport_comicids = None
return {'import_by_comicids': import_by_comicids,
'import_count': len(import_by_comicids),
'CV_import_comicids': cvimport_comicids,
'import_cv_ids': import_cv_ids,
'issueid_list': issueid_list,
'failure_list': failure_list,
'utter_failure_list': utter_failure_list}
def libraryScan(dir=None,
append=False,
ComicID=None,
ComicName=None,
cron=None):
if cron and not mylar.LIBRARYSCAN:
return
if not dir:
dir = mylar.COMIC_DIR
# If we're appending a dir, it's coming from the post processor which is
# already bytestring
if not append:
dir = dir.encode(mylar.SYS_ENCODING)
if not os.path.isdir(dir):
logger.warn('Cannot find directory: %s. Not scanning' %
dir.decode(mylar.SYS_ENCODING, 'replace'))
return
logger.info('Scanning comic directory: %s' %
dir.decode(mylar.SYS_ENCODING, 'replace'))
basedir = dir
comic_list = []
comiccnt = 0
extensions = ('cbr', 'cbz')
for r, d, f in os.walk(dir):
#for directory in d[:]:
# if directory.startswith("."):
# d.remove(directory)
for files in f:
if any(files.lower().endswith('.' + x.lower())
for x in extensions):
comic = files
comicpath = os.path.join(r, files)
comicsize = os.path.getsize(comicpath)
print "Comic: " + comic
print "Comic Path: " + comicpath
print "Comic Size: " + str(comicsize)
# We need the unicode path to use for logging, inserting into database
unicode_comic_path = comicpath.decode(mylar.SYS_ENCODING,
'replace')
comiccnt += 1
comic_dict = {
'ComicFilename': comic,
'ComicLocation': comicpath,
'ComicSize': comicsize,
'Unicode_ComicLocation': unicode_comic_path
}
comic_list.append(comic_dict)
logger.info("I've found a total of " + str(comiccnt) +
" comics....analyzing now")
logger.info("comiclist: " + str(comic_list))
myDB = db.DBConnection()
#let's load in the watchlist to see if we have any matches.
logger.info(
"loading in the watchlist to see if a series is being watched already..."
)
watchlist = myDB.action("SELECT * from comics")
ComicName = []
ComicYear = []
ComicPublisher = []
ComicTotal = []
ComicID = []
ComicLocation = []
AltName = []
watchcnt = 0
watch_kchoice = []
watchchoice = {}
import_by_comicids = []
import_comicids = {}
for watch in watchlist:
# let's clean up the name, just in case for comparison purposes...
watchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ',
watch['ComicName']).encode('utf-8').strip()
#watchcomic = re.sub('\s+', ' ', str(watchcomic)).strip()
if ' the ' in watchcomic.lower():
#drop the 'the' from the watchcomic title for proper comparisons.
watchcomic = watchcomic[-4:]
alt_chk = "no" # alt-checker flag (default to no)
# account for alternate names as well
if watch['AlternateSearch'] is not None and watch[
'AlternateSearch'] is not 'None':
altcomic = re.sub(
'[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ',
watch['AlternateSearch']).encode('utf-8').strip()
#altcomic = re.sub('\s+', ' ', str(altcomic)).strip()
AltName.append(altcomic)
alt_chk = "yes" # alt-checker flag
ComicName.append(watchcomic)
ComicYear.append(watch['ComicYear'])
ComicPublisher.append(watch['ComicPublisher'])
ComicTotal.append(watch['Total'])
ComicID.append(watch['ComicID'])
ComicLocation.append(watch['ComicLocation'])
watchcnt += 1
logger.info("Successfully loaded " + str(watchcnt) +
" series from your watchlist.")
ripperlist = ['digital-', 'empire', 'dcp']
watchfound = 0
datelist = [
'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct',
'nov', 'dec'
]
# datemonth = {'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9,'ten':10,'eleven':$
# #search for number as text, and change to numeric
# for numbs in basnumbs:
# #print ("numbs:" + str(numbs))
# if numbs in ComicName.lower():
# numconv = basnumbs[numbs]
# #print ("numconv: " + str(numconv))
for i in comic_list:
print i['ComicFilename']
comfilename = i['ComicFilename']
comlocation = i['ComicLocation']
#let's clean up the filename for matching purposes
cfilename = re.sub('[\_\#\,\/\:\;\-\!\$\%\&\+\'\?\@]', ' ',
comfilename)
#cfilename = re.sub('\s', '_', str(cfilename))
#versioning - remove it
subsplit = cfilename.replace('_', ' ').split()
volno = None
volyr = None
for subit in subsplit:
if subit[0].lower() == 'v':
vfull = 0
if subit[1:].isdigit():
#if in format v1, v2009 etc...
if len(subit) > 3:
# if it's greater than 3 in length, then the format is Vyyyy
vfull = 1 # add on 1 character length to account for extra space
cfilename = re.sub(subit, '', cfilename)
volno = re.sub("[^0-9]", " ", subit)
elif subit.lower()[:3] == 'vol':
#if in format vol.2013 etc
#because the '.' in Vol. gets removed, let's loop thru again after the Vol hit to remove it entirely
logger.fdebug('volume indicator detected as version #:' +
str(subit))
cfilename = re.sub(subit, '', cfilename)
cfilename = " ".join(cfilename.split())
volyr = re.sub("[^0-9]", " ", subit).strip()
logger.fdebug('volume year set as : ' + str(volyr))
cm_cn = 0
#we need to track the counter to make sure we are comparing the right array parts
#this takes care of the brackets :)
m = re.findall('[^()]+', cfilename)
lenm = len(m)
logger.fdebug("there are " + str(lenm) + " words.")
cnt = 0
yearmatch = "false"
foundonwatch = "False"
issue = 999999
while (cnt < lenm):
if m[cnt] is None: break
if m[cnt] == ' ':
pass
else:
logger.fdebug(str(cnt) + ". Bracket Word: " + m[cnt])
if cnt == 0:
comic_andiss = m[cnt]
logger.fdebug("Comic: " + comic_andiss)
# if it's not in the standard format this will bork.
# let's try to accomodate (somehow).
# first remove the extension (if any)
extensions = ('cbr', 'cbz')
if comic_andiss.lower().endswith(extensions):
comic_andiss = comic_andiss[:-4]
logger.fdebug("removed extension from filename.")
#now we have to break up the string regardless of formatting.
#let's force the spaces.
comic_andiss = re.sub('_', ' ', comic_andiss)
cs = comic_andiss.split()
cs_len = len(cs)
cn = ''
ydetected = 'no'
idetected = 'no'
decimaldetect = 'no'
for i in reversed(xrange(len(cs))):
#start at the end.
logger.fdebug("word: " + str(cs[i]))
#assume once we find issue - everything prior is the actual title
#idetected = no will ignore everything so it will assume all title
if cs[i][:-2] == '19' or cs[
i][:-2] == '20' and idetected == 'no':
logger.fdebug("year detected: " + str(cs[i]))
ydetected = 'yes'
result_comyear = cs[i]
elif cs[i].isdigit(
) and idetected == 'no' or '.' in cs[i]:
issue = cs[i]
logger.fdebug("issue detected : " + str(issue))
idetected = 'yes'
if '.' in cs[i]:
#make sure it's a number on either side of decimal and assume decimal issue.
decst = cs[i].find('.')
dec_st = cs[i][:decst]
dec_en = cs[i][decst + 1:]
logger.fdebug("st: " + str(dec_st))
logger.fdebug("en: " + str(dec_en))
if dec_st.isdigit() and dec_en.isdigit():
logger.fdebug(
"decimal issue detected...adjusting.")
issue = dec_st + "." + dec_en
logger.fdebug("issue detected: " +
str(issue))
idetected = 'yes'
else:
logger.fdebug(
"false decimal represent. Chunking to extra word."
)
cn = cn + cs[i] + " "
break
elif '\#' in cs[i] or decimaldetect == 'yes':
logger.fdebug("issue detected: " + str(cs[i]))
idetected = 'yes'
else:
cn = cn + cs[i] + " "
if ydetected == 'no':
#assume no year given in filename...
result_comyear = "0000"
logger.fdebug("cm?: " + str(cn))
if issue is not '999999':
comiss = issue
else:
logger.ERROR(
"Invalid Issue number (none present) for " +
comfilename)
break
cnsplit = cn.split()
cname = ''
findcn = 0
while (findcn < len(cnsplit)):
cname = cname + cs[findcn] + " "
findcn += 1
cname = cname[:len(cname) - 1] # drop the end space...
print("assuming name is : " + cname)
com_NAME = cname
print("com_NAME : " + com_NAME)
yearmatch = "True"
else:
logger.fdebug('checking ' + m[cnt])
# we're assuming that the year is in brackets (and it should be damnit)
if m[cnt][:-2] == '19' or m[cnt][:-2] == '20':
print("year detected: " + str(m[cnt]))
ydetected = 'yes'
result_comyear = m[cnt]
elif m[cnt][:3].lower() in datelist:
logger.fdebug(
'possible issue date format given - verifying')
#if the date of the issue is given as (Jan 2010) or (January 2010) let's adjust.
#keeping in mind that ',' and '.' are already stripped from the string
if m[cnt][-4:].isdigit():
ydetected = 'yes'
result_comyear = m[cnt][-4:]
logger.fdebug('Valid Issue year of ' +
str(result_comyear) +
'detected in format of ' +
str(m[cnt]))
cnt += 1
splitit = []
watchcomic_split = []
logger.fdebug("filename comic and issue: " + comic_andiss)
#changed this from '' to ' '
comic_iss_b4 = re.sub('[\-\:\,]', ' ', comic_andiss)
comic_iss = comic_iss_b4.replace('.', ' ')
comic_iss = re.sub('[\s+]', ' ', comic_iss).strip()
logger.fdebug("adjusted comic and issue: " + str(comic_iss))
#remove 'the' from here for proper comparisons.
if ' the ' in comic_iss.lower():
comic_iss = comic_iss[-4:]
splitit = comic_iss.split(None)
logger.fdebug("adjusting from: " + str(comic_iss_b4) + " to: " +
str(comic_iss))
#bmm = re.findall('v\d', comic_iss)
#if len(bmm) > 0: splitst = len(splitit) - 2
#else: splitst = len(splitit) - 1
#-----
#here we cycle through the Watchlist looking for a match.
while (cm_cn < watchcnt):
#setup the watchlist
comname = ComicName[cm_cn]
print("watch_comic:" + comname)
comyear = ComicYear[cm_cn]
compub = ComicPublisher[cm_cn]
comtotal = ComicTotal[cm_cn]
comicid = ComicID[cm_cn]
watch_location = ComicLocation[cm_cn]
# if splitit[(len(splitit)-1)].isdigit():
# #compares - if the last digit and second last digit are #'s seperated by spaces assume decimal
# comic_iss = splitit[(len(splitit)-1)]
# splitst = len(splitit) - 1
# if splitit[(len(splitit)-2)].isdigit():
# # for series that have a digit at the end, it screws up the logistics.
# i = 1
# chg_comic = splitit[0]
# while (i < (len(splitit)-1)):
# chg_comic = chg_comic + " " + splitit[i]
# i+=1
# logger.fdebug("chg_comic:" + str(chg_comic))
# if chg_comic.upper() == comname.upper():
# logger.fdebug("series contains numerics...adjusting..")
# else:
# changeup = "." + splitit[(len(splitit)-1)]
# logger.fdebug("changeup to decimal: " + str(changeup))
# comic_iss = splitit[(len(splitit)-2)] + "." + comic_iss
# splitst = len(splitit) - 2
# else:
# if the nzb name doesn't follow the series-issue-year format even closely..ignore nzb
# logger.fdebug("invalid naming format of filename detected - cannot properly determine issue")
# continue
# make sure that things like - in watchcomic are accounted for when comparing to nzb.
# there shouldn't be an issue in the comic now, so let's just assume it's all gravy.
splitst = len(splitit)
watchcomic_split = helpers.cleanName(comname)
watchcomic_split = re.sub('[\-\:\,\.]', ' ',
watchcomic_split).split(None)
logger.fdebug(
str(splitit) + " file series word count: " + str(splitst))
logger.fdebug(
str(watchcomic_split) + " watchlist word count: " +
str(len(watchcomic_split)))
if (splitst) != len(watchcomic_split):
logger.fdebug("incorrect comic lengths...not a match")
# if str(splitit[0]).lower() == "the":
# logger.fdebug("THE word detected...attempting to adjust pattern matching")
# splitit[0] = splitit[4:]
else:
logger.fdebug("length match..proceeding")
n = 0
scount = 0
logger.fdebug("search-length: " + str(splitst))
logger.fdebug("Watchlist-length: " +
str(len(watchcomic_split)))
while (n <= (splitst) - 1):
logger.fdebug("splitit: " + str(splitit[n]))
if n < (splitst) and n < len(watchcomic_split):
logger.fdebug(
str(n) + " Comparing: " +
str(watchcomic_split[n]) + " .to. " +
str(splitit[n]))
if '+' in watchcomic_split[n]:
watchcomic_split[n] = re.sub(
'+', '', str(watchcomic_split[n]))
if str(watchcomic_split[n].lower()) in str(
splitit[n].lower()) and len(
watchcomic_split[n]) >= len(splitit[n]):
logger.fdebug("word matched on : " +
str(splitit[n]))
scount += 1
#elif ':' in splitit[n] or '-' in splitit[n]:
# splitrep = splitit[n].replace('-', '')
# print ("non-character keyword...skipped on " + splitit[n])
elif str(splitit[n]).lower().startswith('v'):
logger.fdebug("possible versioning..checking")
#we hit a versioning # - account for it
if splitit[n][1:].isdigit():
comicversion = str(splitit[n])
logger.fdebug("version found: " +
str(comicversion))
else:
logger.fdebug("Comic / Issue section")
if splitit[n].isdigit():
logger.fdebug("issue detected")
#comiss = splitit[n]
# comicNAMER = n - 1
# com_NAME = splitit[0]
# cmnam = 1
# while (cmnam <= comicNAMER):
# com_NAME = str(com_NAME) + " " + str(splitit[cmnam])
# cmnam+=1
# logger.fdebug("comic: " + str(com_NAME))
else:
logger.fdebug("non-match for: " + str(splitit[n]))
pass
n += 1
#set the match threshold to 80% (for now)
# if it's less than 80% consider it a non-match and discard.
#splitit has to splitit-1 because last position is issue.
wordcnt = int(scount)
logger.fdebug("scount:" + str(wordcnt))
totalcnt = int(splitst)
logger.fdebug("splitit-len:" + str(totalcnt))
spercent = (wordcnt / totalcnt) * 100
logger.fdebug("we got " + str(spercent) + " percent.")
if int(spercent) >= 80:
logger.fdebug("it's a go captain... - we matched " +
str(spercent) + "%!")
logger.fdebug("this should be a match!")
# if '.' in comic_iss:
# comisschk_find = comic_iss.find('.')
# comisschk_b4dec = comic_iss[:comisschk_find]
# comisschk_decval = comic_iss[comisschk_find+1:]
# logger.fdebug("Found IssueNumber: " + str(comic_iss))
# logger.fdebug("..before decimal: " + str(comisschk_b4dec))
# logger.fdebug("...after decimal: " + str(comisschk_decval))
# #--let's make sure we don't wipe out decimal issues ;)
# if int(comisschk_decval) == 0:
# ciss = comisschk_b4dec
# cintdec = int(comisschk_decval)
# else:
# if len(comisschk_decval) == 1:
# ciss = comisschk_b4dec + "." + comisschk_decval
# cintdec = int(comisschk_decval) * 10
# else:
# ciss = comisschk_b4dec + "." + comisschk_decval.rstrip('0')
# cintdec = int(comisschk_decval.rstrip('0')) * 10
# comintIss = (int(comisschk_b4dec) * 1000) + cintdec
# else:
# comintIss = int(comic_iss) * 1000
logger.fdebug("issue we found for is : " + str(comiss))
#set the year to the series we just found ;)
result_comyear = comyear
#issue comparison now as well
logger.info(u"Found " + comname + " (" + str(comyear) +
") issue: " + str(comiss))
# watchfound+=1
watchmatch = str(comicid)
# watch_kchoice.append({
# "ComicID": str(comicid),
# "ComicName": str(comname),
# "ComicYear": str(comyear),
# "ComicIssue": str(int(comic_iss)),
# "ComicLocation": str(watch_location),
# "OriginalLocation" : str(comlocation),
# "OriginalFilename" : str(comfilename)
# })
foundonwatch = "True"
break
elif int(spercent) < 80:
logger.fdebug("failure - we only got " + str(spercent) +
"% right!")
cm_cn += 1
if foundonwatch == "False":
watchmatch = None
#---if it's not a match - send it to the importer.
n = 0
# print ("comic_andiss : " + str(comic_andiss))
# csplit = comic_andiss.split(None)
# while ( n <= (len(csplit)-1) ):
# print ("csplit:" + str(csplit[n]))
# if csplit[n].isdigit():
# logger.fdebug("issue detected")
# comiss = splitit[n]
# logger.fdebug("issue # : " + str(comiss))
# comicNAMER = n - 1
# com_NAME = csplit[0]
# cmnam = 1
# while (cmnam <= comicNAMER):
# com_NAME = str(com_NAME) + " " + str(csplit[cmnam])
# cmnam+=1
# logger.fdebug("comic: " + str(com_NAME))
# n+=1
if volyr is None:
if result_comyear is None:
result_comyear = '0000' #no year in filename basically.
else:
if result_comyear is None:
result_comyear = volyr
if volno is None:
if volyr is None:
vol_label = None
else:
vol_label = volyr
else:
vol_label = volno
print("adding " + com_NAME + " to the import-queue!")
impid = com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
print("impid: " + str(impid))
import_by_comicids.append({
"impid":
impid,
"watchmatch":
watchmatch,
"comicname":
com_NAME,
"comicyear":
result_comyear,
"volume":
vol_label,
"comfilename":
comfilename,
"comlocation":
comlocation.decode(mylar.SYS_ENCODING)
})
if len(watch_kchoice) > 0:
watchchoice['watchlist'] = watch_kchoice
print("watchchoice: " + str(watchchoice))
logger.info("I have found " + str(watchfound) + " out of " +
str(comiccnt) +
" comics for series that are being watched.")
wat = 0
comicids = []
if watchfound > 0:
if mylar.IMP_MOVE:
logger.info(
"You checked off Move Files...so that's what I'm going to do"
)
#check to see if Move Files is enabled.
#if not being moved, set the archive bit.
print("Moving files into appropriate directory")
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comlocation = watch_the_list['ComicLocation']
watch_comicid = watch_the_list['ComicID']
watch_comicname = watch_the_list['ComicName']
watch_comicyear = watch_the_list['ComicYear']
watch_comiciss = watch_the_list['ComicIssue']
print("ComicLocation: " + str(watch_comlocation))
orig_comlocation = watch_the_list['OriginalLocation']
orig_filename = watch_the_list['OriginalFilename']
print("Orig. Location: " + str(orig_comlocation))
print("Orig. Filename: " + str(orig_filename))
#before moving check to see if Rename to Mylar structure is enabled.
if mylar.IMP_RENAME:
print(
"Renaming files according to configuration details : "
+ str(mylar.FILE_FORMAT))
renameit = helpers.rename_param(
watch_comicid, watch_comicname, watch_comicyear,
watch_comiciss)
nfilename = renameit['nfilename']
dst_path = os.path.join(watch_comlocation, nfilename)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
else:
print(
"Renaming files not enabled, keeping original filename(s)"
)
dst_path = os.path.join(watch_comlocation,
orig_filename)
#os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
#src = os.path.join(, str(nfilename + ext))
print("I'm going to move " + str(orig_comlocation) +
" to .." + str(dst_path))
try:
shutil.move(orig_comlocation, dst_path)
except (OSError, IOError):
logger.info(
"Failed to move directory - check directories and manually re-run."
)
wat += 1
else:
# if move files isn't enabled, let's set all found comics to Archive status :)
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comicid = watch_the_list['ComicID']
watch_issue = watch_the_list['ComicIssue']
print("ComicID: " + str(watch_comicid))
print("Issue#: " + str(watch_issue))
issuechk = myDB.action(
"SELECT * from issues where ComicID=? AND INT_IssueNumber=?",
[watch_comicid, watch_issue]).fetchone()
if issuechk is None:
print("no matching issues for this comic#")
else:
print("...Existing status: " + str(issuechk['Status']))
control = {"IssueID": issuechk['IssueID']}
values = {"Status": "Archived"}
print("...changing status of " +
str(issuechk['Issue_Number']) + " to Archived ")
myDB.upsert("issues", values, control)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
wat += 1
if comicids is None: pass
else:
c_upd = len(comicids)
c = 0
while (c < c_upd):
print("Rescanning.. " + str(c))
updater.forceRescan(c)
if not len(import_by_comicids):
return "Completed"
if len(import_by_comicids) > 0:
import_comicids['comic_info'] = import_by_comicids
print("import comicids: " + str(import_by_comicids))
return import_comicids, len(import_by_comicids)
def Process(self):
self._log("nzb name: " + str(self.nzb_name), logger.DEBUG)
self._log("nzb folder: " + str(self.nzb_folder), logger.DEBUG)
logger.fdebug("nzb name: " + str(self.nzb_name))
logger.fdebug("nzb folder: " + str(self.nzb_folder))
if mylar.USE_SABNZBD==0:
logger.fdebug("Not using SABNzbd")
else:
# if the SAB Directory option is enabled, let's use that folder name and append the jobname.
if mylar.SAB_DIRECTORY is not None and mylar.SAB_DIRECTORY is not 'None' and len(mylar.SAB_DIRECTORY) > 4:
self.nzb_folder = os.path.join(mylar.SAB_DIRECTORY, self.nzb_name).encode(mylar.SYS_ENCODING)
#lookup nzb_name in nzblog table to get issueid
#query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals
#http://localhost:8080/sabnzbd/api?mode=set_config§ion=misc&keyword=dirscan_speed&value=5
querysab = str(mylar.SAB_HOST) + "/api?mode=get_config§ion=misc&output=xml&apikey=" + str(mylar.SAB_APIKEY)
#logger.info("querysab_string:" + str(querysab))
file = urllib2.urlopen(querysab)
data = file.read()
file.close()
dom = parseString(data)
sabreps = dom.getElementsByTagName('replace_spaces')[0].firstChild.wholeText
sabrepd = dom.getElementsByTagName('replace_dots')[0].firstChild.wholeText
logger.fdebug("SAB Replace Spaces: " + str(sabreps))
logger.fdebug("SAB Replace Dots: " + str(sabrepd))
if mylar.USE_NZBGET==1:
logger.fdebug("Using NZBGET")
logger.fdebug("NZB name as passed from NZBGet: " + self.nzb_name)
myDB = db.DBConnection()
nzbname = self.nzb_name
#remove extensions from nzb_name if they somehow got through (Experimental most likely)
extensions = ('.cbr', '.cbz')
if nzbname.lower().endswith(extensions):
fd, ext = os.path.splitext(nzbname)
self._log("Removed extension from nzb: " + ext, logger.DEBUG)
nzbname = re.sub(str(ext), '', str(nzbname))
#replace spaces
nzbname = re.sub(' ', '.', str(nzbname))
nzbname = re.sub('[\,\:\?]', '', str(nzbname))
nzbname = re.sub('[\&]', 'and', str(nzbname))
logger.fdebug("After conversions, nzbname is : " + str(nzbname))
# if mylar.USE_NZBGET==1:
# nzbname=self.nzb_name
self._log("nzbname: " + str(nzbname), logger.DEBUG)
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
self._log("Failure - could not initially locate nzbfile in my database to rename.", logger.DEBUG)
logger.fdebug("Failure - could not locate nzbfile initially.")
# if failed on spaces, change it all to decimals and try again.
nzbname = re.sub('_', '.', str(nzbname))
self._log("trying again with this nzbname: " + str(nzbname), logger.DEBUG)
logger.fdebug("trying again with nzbname of : " + str(nzbname))
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
logger.error(u"Unable to locate downloaded file to rename. PostProcessing aborted.")
return
else:
self._log("I corrected and found the nzb as : " + str(nzbname))
logger.fdebug("auto-corrected and found the nzb as : " + str(nzbname))
issueid = nzbiss['IssueID']
else:
issueid = nzbiss['IssueID']
print "issueid:" + str(issueid)
#use issueid to get publisher, series, year, issue number
issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone()
if helpers.is_number(issueid):
sandwich = int(issuenzb['IssueID'])
else:
#if it's non-numeric, it contains a 'G' at the beginning indicating it's a multi-volume
#using GCD data. Set sandwich to 1 so it will bypass and continue post-processing.
sandwich = 1
if issuenzb is None or sandwich >= 900000:
# this has no issueID, therefore it's a one-off or a manual post-proc.
# At this point, let's just drop it into the Comic Location folder and forget about it..
self._log("One-off mode enabled for Post-Processing. All I'm doing is moving the file untouched into the Grab-bag directory.", logger.DEBUG)
logger.info("One-off mode enabled for Post-Processing. Will move into Grab-bag directory.")
self._log("Grab-Bag Directory set to : " + mylar.GRABBAG_DIR, logger.DEBUG)
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
ofilename = filename
path, ext = os.path.splitext(ofilename)
if mylar.GRABBAG_DIR:
grdst = mylar.GRABBAG_DIR
else:
grdst = mylar.DESTINATION_DIR
grab_dst = os.path.join(grdst, ofilename)
self._log("Destination Path : " + grab_dst, logger.DEBUG)
grab_src = os.path.join(self.nzb_folder, ofilename)
self._log("Source Path : " + grab_src, logger.DEBUG)
logger.info("Moving " + str(ofilename) + " into grab-bag directory : " + str(grdst))
try:
shutil.move(grab_src, grab_dst)
except (OSError, IOError):
self.log("Failed to move directory - check directories and manually re-run.", logger.DEBUG)
logger.debug("Failed to move directory - check directories and manually re-run.")
return self.log
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log("Failed to remove temporary directory.", logger.DEBUG)
logger.debug("Failed to remove temporary directory - check directory and manually re-run.")
return self.log
logger.debug("Removed temporary directory : " + str(self.nzb_folder))
self._log("Removed temporary directory : " + self.nzb_folder, logger.DEBUG)
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
return self.log
comicid = issuenzb['ComicID']
issuenumOG = issuenzb['Issue_Number']
#issueno = str(issuenum).split('.')[0]
#new CV API - removed all decimals...here we go AGAIN!
issuenum = issuenumOG
issue_except = 'None'
if 'au' in issuenum.lower():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AU'
if '.' in issuenum:
iss_find = issuenum.find('.')
iss_b4dec = issuenum[:iss_find]
iss_decval = issuenum[iss_find+1:]
if int(iss_decval) == 0:
iss = iss_b4dec
issdec = int(iss_decval)
issueno = str(iss)
self._log("Issue Number: " + str(issueno), logger.DEBUG)
logger.fdebug("Issue Number: " + str(issueno))
else:
if len(iss_decval) == 1:
iss = iss_b4dec + "." + iss_decval
issdec = int(iss_decval) * 10
else:
iss = iss_b4dec + "." + iss_decval.rstrip('0')
issdec = int(iss_decval.rstrip('0')) * 10
issueno = iss_b4dec
self._log("Issue Number: " + str(iss), logger.DEBUG)
logger.fdebug("Issue Number: " + str(iss))
else:
iss = issuenum
issueno = str(iss)
# issue zero-suppression here
if mylar.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00"
logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N))
if str(len(issueno)) > 1:
if int(issueno) < 10:
self._log("issue detected less than 10", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG)
elif int(issueno) >= 10 and int(issueno) < 100:
self._log("issue detected greater than 10, but less than 100", logger.DEBUG)
if mylar.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss), logger.DEBUG)
else:
self._log("issue detected greater than 100", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(issueno)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG)
else:
prettycomiss = str(issueno)
self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss), logger.DEBUG)
logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss))
issueyear = issuenzb['IssueDate'][:4]
self._log("Issue Year: " + str(issueyear), logger.DEBUG)
logger.fdebug("Issue Year : " + str(issueyear))
comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
publisher = comicnzb['ComicPublisher']
self._log("Publisher: " + publisher, logger.DEBUG)
logger.fdebug("Publisher: " + str(publisher))
#we need to un-unicode this to make sure we can write the filenames properly for spec.chars
series = comicnzb['ComicName'].encode('ascii', 'ignore').strip()
self._log("Series: " + series, logger.DEBUG)
logger.fdebug("Series: " + str(series))
seriesyear = comicnzb['ComicYear']
self._log("Year: " + seriesyear, logger.DEBUG)
logger.fdebug("Year: " + str(seriesyear))
comlocation = comicnzb['ComicLocation']
self._log("Comic Location: " + comlocation, logger.DEBUG)
logger.fdebug("Comic Location: " + str(comlocation))
comversion = comicnzb['ComicVersion']
self._log("Comic Version: " + str(comversion), logger.DEBUG)
logger.fdebug("Comic Version: " + str(comversion))
if comversion is None:
comversion = 'None'
#if comversion is None, remove it so it doesn't populate with 'None'
if comversion == 'None':
chunk_f_f = re.sub('\$VolumeN','',mylar.FILE_FORMAT)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
self._log("No version # found for series - tag will not be available for renaming.", logger.DEBUG)
logger.fdebug("No version # found for series, removing from filename")
logger.fdebug("new format is now: " + str(chunk_file_format))
else:
chunk_file_format = mylar.FILE_FORMAT
#Run Pre-script
if mylar.ENABLE_PRE_SCRIPTS:
nzbn = self.nzb_name #original nzb name
nzbf = self.nzb_folder #original nzb folder
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_pre_scripts(nzbn, nzbf, seriesmetadata )
#rename file and move to new path
#nfilename = series + " " + issueno + " (" + seriesyear + ")"
file_values = {'$Series': series,
'$Issue': prettycomiss,
'$Year': issueyear,
'$series': series.lower(),
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion
}
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
ofilename = filename
path, ext = os.path.splitext(ofilename)
self._log("Original Filename: " + ofilename, logger.DEBUG)
self._log("Original Extension: " + ext, logger.DEBUG)
logger.fdebug("Original Filname: " + str(ofilename))
logger.fdebug("Original Extension: " + str(ext))
if mylar.FILE_FORMAT == '' or not mylar.RENAME_FILES:
self._log("Rename Files isn't enabled...keeping original filename.", logger.DEBUG)
logger.fdebug("Rename Files isn't enabled - keeping original filename.")
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
nfilename = helpers.replace_all(chunk_file_format, file_values)
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
nfilename = re.sub('[\,\:\?]', '', nfilename)
self._log("New Filename: " + nfilename, logger.DEBUG)
logger.fdebug("New Filename: " + str(nfilename))
src = os.path.join(self.nzb_folder, ofilename)
filechecker.validateAndCreateDirectory(comlocation, True)
if mylar.LOWERCASE_FILENAMES:
dst = (comlocation + "/" + nfilename + ext).lower()
else:
dst = comlocation + "/" + nfilename + ext.lower()
self._log("Source:" + src, logger.DEBUG)
self._log("Destination:" + dst, logger.DEBUG)
logger.fdebug("Source: " + str(src))
logger.fdebug("Destination: " + str(dst))
os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
src = os.path.join(self.nzb_folder, str(nfilename + ext))
try:
shutil.move(src, dst)
except (OSError, IOError):
self._log("Failed to move directory - check directories and manually re-run.", logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log("Failed to remove temporary directory - check directory and manually re-run.", logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
self._log("Removed temporary directory : " + str(self.nzb_folder), logger.DEBUG)
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
#force rescan of files
updater.forceRescan(comicid)
logger.info(u"Post-Processing completed for: " + series + " issue: " + str(issuenumOG) )
self._log(u"Post Processing SUCCESSFULL! ", logger.DEBUG)
if mylar.PROWL_ENABLED:
pushmessage = series + '(' + issueyear + ') - issue #' + issuenumOG
logger.info(u"Prowl request")
prowl = notifiers.PROWL()
prowl.notify(pushmessage,"Download and Postprocessing completed")
if mylar.NMA_ENABLED:
nma = notifiers.NMA()
nma.notify(series, str(issueyear), str(issuenumOG))
if mylar.PUSHOVER_ENABLED:
pushmessage = series + ' (' + str(issueyear) + ') - issue #' + str(issuenumOG)
logger.info(u"Pushover request")
pushover = notifiers.PUSHOVER()
pushover.notify(pushmessage, "Download and Post-Processing completed")
# retrieve/create the corresponding comic objects
if mylar.ENABLE_EXTRA_SCRIPTS:
folderp = str(dst) #folder location after move/rename
nzbn = self.nzb_name #original nzb name
filen = str(nfilename + ext) #new filename
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_extra_scripts(nzbname, self.nzb_folder, filen, folderp, seriesmetadata )
return self.log
else:
logger.fdebug('...Existing status: ' + str(issuechk['Status']))
control = {"IssueID": issuechk['IssueID']}
values = {"Status": "Archived"}
logger.fdebug('...changing status of ' + str(issuechk['Issue_Number']) + ' to Archived ')
myDB.upsert("issues", values, control)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
wat+=1
if comicids is None: pass
else:
c_upd = len(comicids)
c = 0
while (c < c_upd ):
logger.fdebug('Rescanning.. ' + str(c))
updater.forceRescan(c)
if not len(import_by_comicids):
return "Completed"
if len(import_by_comicids) > 0 or len(vals) > 0:
#import_comicids['comic_info'] = import_by_comicids
#if vals:
# import_comicids['issueid_info'] = vals
#else:
# import_comicids['issueid_info'] = None
if vals:
cvimport_comicids = vals
import_cv_ids = len(vals)
else:
cvimport_comicids = None
import_cv_ids = 0
def Process_next(self, comicid, issueid, issuenumOG, ml=None):
annchk = "no"
extensions = ('.cbr', '.cbz')
myDB = db.DBConnection()
comicnzb = myDB.action("SELECT * from comics WHERE comicid=?",
[comicid]).fetchone()
issuenzb = myDB.action(
"SELECT * from issues WHERE issueid=? AND comicid=? AND ComicName NOT NULL",
[issueid, comicid]).fetchone()
logger.fdebug('issueid: ' + str(issueid))
logger.fdebug('issuenumOG: ' + str(issuenumOG))
if issuenzb is None:
issuenzb = myDB.action(
"SELECT * from annuals WHERE issueid=? and comicid=?",
[issueid, comicid]).fetchone()
annchk = "yes"
#issueno = str(issuenum).split('.')[0]
#new CV API - removed all decimals...here we go AGAIN!
issuenum = issuenzb['Issue_Number']
issue_except = 'None'
if 'au' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AU'
elif 'ai' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AI'
elif 'inh' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = '.INH'
elif 'now' in issuenum.lower() and issuenum[:1].isdigit():
if '!' in issuenum: issuenum = re.sub('\!', '', issuenum)
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = '.NOW'
if '.' in issuenum:
iss_find = issuenum.find('.')
iss_b4dec = issuenum[:iss_find]
iss_decval = issuenum[iss_find + 1:]
if int(iss_decval) == 0:
iss = iss_b4dec
issdec = int(iss_decval)
issueno = str(iss)
self._log("Issue Number: " + str(issueno), logger.DEBUG)
logger.fdebug("Issue Number: " + str(issueno))
else:
if len(iss_decval) == 1:
iss = iss_b4dec + "." + iss_decval
issdec = int(iss_decval) * 10
else:
iss = iss_b4dec + "." + iss_decval.rstrip('0')
issdec = int(iss_decval.rstrip('0')) * 10
issueno = iss_b4dec
self._log("Issue Number: " + str(iss), logger.DEBUG)
logger.fdebug("Issue Number: " + str(iss))
else:
iss = issuenum
issueno = str(iss)
# issue zero-suppression here
if mylar.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00"
logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N))
if str(len(issueno)) > 1:
if int(issueno) < 10:
self._log("issue detected less than 10", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log(
"Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) +
". Issue will be set as : " + str(prettycomiss),
logger.DEBUG)
elif int(issueno) >= 10 and int(issueno) < 100:
self._log("issue detected greater than 10, but less than 100",
logger.DEBUG)
if mylar.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log(
"Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) +
".Issue will be set as : " + str(prettycomiss),
logger.DEBUG)
else:
self._log("issue detected greater than 100", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(issueno)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log(
"Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) +
". Issue will be set as : " + str(prettycomiss),
logger.DEBUG)
else:
prettycomiss = str(issueno)
self._log(
"issue length error - cannot determine length. Defaulting to None: "
+ str(prettycomiss), logger.DEBUG)
if annchk == "yes":
self._log("Annual detected.")
logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss))
issueyear = issuenzb['IssueDate'][:4]
self._log("Issue Year: " + str(issueyear), logger.DEBUG)
logger.fdebug("Issue Year : " + str(issueyear))
month = issuenzb['IssueDate'][5:7].replace('-', '').strip()
month_name = helpers.fullmonth(month)
# comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
publisher = comicnzb['ComicPublisher']
self._log("Publisher: " + publisher, logger.DEBUG)
logger.fdebug("Publisher: " + str(publisher))
#we need to un-unicode this to make sure we can write the filenames properly for spec.chars
series = comicnzb['ComicName'].encode('ascii', 'ignore').strip()
self._log("Series: " + series, logger.DEBUG)
logger.fdebug("Series: " + str(series))
seriesyear = comicnzb['ComicYear']
self._log("Year: " + seriesyear, logger.DEBUG)
logger.fdebug("Year: " + str(seriesyear))
comlocation = comicnzb['ComicLocation']
self._log("Comic Location: " + comlocation, logger.DEBUG)
logger.fdebug("Comic Location: " + str(comlocation))
comversion = comicnzb['ComicVersion']
self._log("Comic Version: " + str(comversion), logger.DEBUG)
logger.fdebug("Comic Version: " + str(comversion))
if comversion is None:
comversion = 'None'
#if comversion is None, remove it so it doesn't populate with 'None'
if comversion == 'None':
chunk_f_f = re.sub('\$VolumeN', '', mylar.FILE_FORMAT)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
self._log(
"No version # found for series - tag will not be available for renaming.",
logger.DEBUG)
logger.fdebug(
"No version # found for series, removing from filename")
logger.fdebug("new format is now: " + str(chunk_file_format))
else:
chunk_file_format = mylar.FILE_FORMAT
if annchk == "no":
chunk_f_f = re.sub('\$Annual', '', chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('not an annual - removing from filename paramaters')
logger.fdebug('new format: ' + str(chunk_file_format))
else:
logger.fdebug('chunk_file_format is: ' + str(chunk_file_format))
if '$Annual' not in chunk_file_format:
#if it's an annual, but $Annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
prettycomiss = "Annual " + str(prettycomiss)
logger.fdebug('prettycomiss: ' + str(prettycomiss))
ofilename = None
#if meta-tagging is not enabled, we need to declare the check as being fail
#if meta-tagging is enabled, it gets changed just below to a default of pass
pcheck = "fail"
#tag the meta.
if mylar.ENABLE_META:
self._log("Metatagging enabled - proceeding...")
logger.fdebug("Metatagging enabled - proceeding...")
pcheck = "pass"
try:
import cmtagmylar
if ml is None:
pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid)
else:
pcheck = cmtagmylar.run(self.nzb_folder,
issueid=issueid,
manual="yes",
filename=ml['ComicLocation'])
except ImportError:
logger.fdebug(
"comictaggerlib not found on system. Ensure the ENTIRE lib directory is located within mylar/lib/comictaggerlib/"
)
logger.fdebug(
"continuing with PostProcessing, but I'm not using metadata."
)
pcheck = "fail"
if pcheck == "fail":
self._log(
"Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging..."
)
logger.fdebug(
"Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging..."
)
elif pcheck == "unrar error":
self._log(
"This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy."
)
logger.error(
"This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy."
)
return self.log
else:
otofilename = pcheck
self._log("Sucessfully wrote metadata to .cbz - Continuing..")
logger.fdebug("Sucessfully wrote metadata to .cbz (" +
str(otofilename) + ") - Continuing..")
#Run Pre-script
if mylar.ENABLE_PRE_SCRIPTS:
nzbn = self.nzb_name #original nzb name
nzbf = self.nzb_folder #original nzb folder
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_pre_scripts(nzbn, nzbf, seriesmetadata)
#rename file and move to new path
#nfilename = series + " " + issueno + " (" + seriesyear + ")"
file_values = {
'$Series': series,
'$Issue': prettycomiss,
'$Year': issueyear,
'$series': series.lower(),
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion,
'$monthname': month_name,
'$month': month,
'$Annual': 'Annual'
}
#if it's a Manual Run, use the ml['ComicLocation'] for the exact filename.
if ml is None:
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
ofilename = filename
path, ext = os.path.splitext(ofilename)
else:
if pcheck == "fail":
otofilename = ml['ComicLocation']
logger.fdebug('otofilename:' + str(otofilename))
odir, ofilename = os.path.split(otofilename)
logger.fdebug('ofilename: ' + str(ofilename))
path, ext = os.path.splitext(ofilename)
logger.fdebug('path: ' + str(path))
logger.fdebug('ext:' + str(ext))
if ofilename is None:
logger.error(
u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that "
+ str(self.nzb_folder) +
" exists and is the correct location.")
return
self._log("Original Filename: " + ofilename, logger.DEBUG)
self._log("Original Extension: " + ext, logger.DEBUG)
logger.fdebug("Original Filname: " + str(ofilename))
logger.fdebug("Original Extension: " + str(ext))
if mylar.FILE_FORMAT == '' or not mylar.RENAME_FILES:
self._log(
"Rename Files isn't enabled...keeping original filename.",
logger.DEBUG)
logger.fdebug(
"Rename Files isn't enabled - keeping original filename.")
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
nfilename = helpers.replace_all(chunk_file_format, file_values)
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
nfilename = re.sub('[\,\:\?]', '', nfilename)
nfilename = re.sub('[\/]', '-', nfilename)
self._log("New Filename: " + nfilename, logger.DEBUG)
logger.fdebug("New Filename: " + str(nfilename))
src = os.path.join(self.nzb_folder, ofilename)
filechecker.validateAndCreateDirectory(comlocation, True)
if mylar.LOWERCASE_FILENAMES:
dst = (comlocation + "/" + nfilename + ext).lower()
else:
dst = comlocation + "/" + nfilename + ext.lower()
self._log("Source:" + src, logger.DEBUG)
self._log("Destination:" + dst, logger.DEBUG)
logger.fdebug("Source: " + str(src))
logger.fdebug("Destination: " + str(dst))
if ml is None:
#non-manual run moving/deleting...
logger.fdebug('self.nzb_folder: ' + self.nzb_folder)
logger.fdebug('ofilename:' + str(ofilename))
logger.fdebug('nfilename:' + str(nfilename + ext))
os.rename(os.path.join(self.nzb_folder, str(ofilename)),
os.path.join(self.nzb_folder, str(nfilename + ext)))
src = os.path.join(self.nzb_folder, str(nfilename + ext))
try:
shutil.move(src, dst)
except (OSError, IOError):
self._log(
"Failed to move directory - check directories and manually re-run.",
logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log(
"Failed to remove temporary directory - check directory and manually re-run.",
logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
self._log("Removed temporary directory : " + str(self.nzb_folder),
logger.DEBUG)
else:
#Manual Run, this is the portion.
logger.fdebug("Renaming " +
os.path.join(self.nzb_folder, str(ofilename)) +
" ..to.. " +
os.path.join(self.nzb_folder, str(nfilename + ext)))
os.rename(os.path.join(self.nzb_folder, str(ofilename)),
os.path.join(self.nzb_folder, str(nfilename + ext)))
src = os.path.join(self.nzb_folder, str(nfilename + ext))
logger.fdebug("Moving " + src + " ... to ... " + dst)
try:
shutil.move(src, dst)
except (OSError, IOError):
logger.fdebug(
"Failed to move directory - check directories and manually re-run."
)
logger.fdebug("Post-Processing ABORTED.")
return
logger.fdebug("Successfully moved to : " + dst)
#tidyup old path
#try:
# os.remove(os.path.join(self.nzb_folder, str(ofilename)))
# logger.fdebug("Deleting : " + os.path.join(self.nzb_folder, str(ofilename)))
#except (OSError, IOError):
# logger.fdebug("Failed to remove temporary directory - check directory and manually re-run.")
# logger.fdebug("Post-Processing ABORTED.")
# return
#logger.fdebug("Removed temporary directory : " + str(self.nzb_folder))
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
#update snatched table to change status to Downloaded
if annchk == "no":
updater.foundsearch(comicid, issueid, down='True')
dispiss = 'issue: ' + str(issuenumOG)
else:
updater.foundsearch(comicid, issueid, mode='want_ann', down='True')
dispiss = 'annual issue: ' + str(issuenumOG)
#force rescan of files
updater.forceRescan(comicid)
logger.info(u"Post-Processing completed for: " + series + " " +
dispiss)
self._log(u"Post Processing SUCCESSFULL! ", logger.DEBUG)
# retrieve/create the corresponding comic objects
if mylar.ENABLE_EXTRA_SCRIPTS:
folderp = str(dst) #folder location after move/rename
nzbn = self.nzb_name #original nzb name
filen = str(nfilename + ext) #new filename
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_extra_scripts(nzbn, self.nzb_folder, filen, folderp,
seriesmetadata)
if ml is not None:
return self.log
else:
if mylar.PROWL_ENABLED:
pushmessage = series + '(' + issueyear + ') - issue #' + issuenumOG
logger.info(u"Prowl request")
prowl = notifiers.PROWL()
prowl.notify(pushmessage,
"Download and Postprocessing completed")
if mylar.NMA_ENABLED:
nma = notifiers.NMA()
nma.notify(series, str(issueyear), str(issuenumOG))
if mylar.PUSHOVER_ENABLED:
pushmessage = series + ' (' + str(
issueyear) + ') - issue #' + str(issuenumOG)
logger.info(u"Pushover request")
pushover = notifiers.PUSHOVER()
pushover.notify(pushmessage,
"Download and Post-Processing completed")
if mylar.BOXCAR_ENABLED:
boxcar = notifiers.BOXCAR()
boxcar.notify(series, str(issueyear), str(issuenumOG))
return self.log
#move the files...if imported is not empty (meaning it's not from the mass importer.)
if imported is None or imported == 'None':
pass
else:
if mylar.IMP_MOVE:
logger.info("Mass import - Move files")
moveit.movefiles(comicid, comlocation, ogcname)
else:
logger.info(
"Mass import - Moving not Enabled. Setting Archived Status for import."
)
moveit.archivefiles(comicid, ogcname)
#check for existing files...
updater.forceRescan(comicid)
if pullupd is None:
# lets' check the pullist for anything at this time as well since we're here.
# do this for only Present comics....
if mylar.AUTOWANT_UPCOMING and lastpubdate == 'Present': #and 'Present' in gcdinfo['resultPublished']:
print("latestissue: #" + str(latestiss))
chkstats = myDB.action(
"SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?",
[comicid, str(latestiss)]).fetchone()
print chkstats['Status']
if chkstats['Status'] == 'Skipped' or chkstats[
'Status'] == 'Wanted' or chkstats['Status'] == 'Snatched':
logger.info(
u"Checking this week's pullist for new issues of " +
comic['ComicName'])
def GCDimport(gcomicid, pullupd=None,imported=None,ogcname=None):
# this is for importing via GCD only and not using CV.
# used when volume spanning is discovered for a Comic (and can't be added using CV).
# Issue Counts are wrong (and can't be added).
# because Comicvine ComicID and GCD ComicID could be identical at some random point, let's distinguish.
# CV = comicid, GCD = gcomicid :) (ie. CV=2740, GCD=G3719)
gcdcomicid = gcomicid
myDB = db.DBConnection()
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ComicID": gcdcomicid}
comic = myDB.action('SELECT ComicName, ComicYear, Total, ComicPublished, ComicImage, ComicLocation, ComicPublisher FROM comics WHERE ComicID=?', [gcomicid]).fetchone()
ComicName = comic[0]
ComicYear = comic[1]
ComicIssues = comic[2]
ComicPublished = comic[3]
comlocation = comic[5]
ComicPublisher = comic[6]
#ComicImage = comic[4]
#print ("Comic:" + str(ComicName))
newValueDict = {"Status": "Loading"}
myDB.upsert("comics", newValueDict, controlValueDict)
# we need to lookup the info for the requested ComicID in full now
#comic = cv.getComic(comicid,'comic')
if not comic:
logger.warn("Error fetching comic. ID for : " + gcdcomicid)
if dbcomic is None:
newValueDict = {"ComicName": "Fetch failed, try refreshing. (%s)" % (gcdcomicid),
"Status": "Active"}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("comics", newValueDict, controlValueDict)
return
#run the re-sortorder here in order to properly display the page
if pullupd is None:
helpers.ComicSort(comicorder=mylar.COMICSORT, imported=gcomicid)
if ComicName.startswith('The '):
sortname = ComicName[4:]
else:
sortname = ComicName
logger.info(u"Now adding/updating: " + ComicName)
#--Now that we know ComicName, let's try some scraping
#--Start
# gcd will return issue details (most importantly publishing date)
comicid = gcomicid[1:]
resultURL = "/series/" + str(comicid) + "/"
gcdinfo=parseit.GCDdetails(comseries=None, resultURL=resultURL, vari_loop=0, ComicID=gcdcomicid, TotalIssues=ComicIssues, issvariation=None, resultPublished=None)
if gcdinfo == "No Match":
logger.warn("No matching result found for " + ComicName + " (" + ComicYear + ")" )
updater.no_searchresults(gcomicid)
nomatch = "true"
return nomatch
logger.info(u"Sucessfully retrieved details for " + ComicName )
# print ("Series Published" + parseit.resultPublished)
#--End
ComicImage = gcdinfo['ComicImage']
#comic book location on machine
# setup default location here
if comlocation is None:
# let's remove the non-standard characters here.
u_comicnm = ComicName
u_comicname = u_comicnm.encode('ascii', 'ignore').strip()
if ':' in u_comicname or '/' in u_comicname or ',' in u_comicname:
comicdir = u_comicname
if ':' in comicdir:
comicdir = comicdir.replace(':','')
if '/' in comicdir:
comicdir = comicdir.replace('/','-')
if ',' in comicdir:
comicdir = comicdir.replace(',','')
else: comicdir = u_comicname
series = comicdir
publisher = ComicPublisher
year = ComicYear
#do work to generate folder path
values = {'$Series': series,
'$Publisher': publisher,
'$Year': year,
'$series': series.lower(),
'$publisher': publisher.lower(),
'$Volume': year
}
if mylar.FOLDER_FORMAT == '':
comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + comic['ComicYear'] + ")"
else:
comlocation = mylar.DESTINATION_DIR + "/" + helpers.replace_all(mylar.FOLDER_FORMAT, values)
#comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + ComicYear + ")"
if mylar.DESTINATION_DIR == "":
logger.error(u"There is no general directory specified - please specify in Config/Post-Processing.")
return
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
comlocation = comlocation.replace(' ', mylar.REPLACE_CHAR)
#if it doesn't exist - create it (otherwise will bugger up later on)
if os.path.isdir(str(comlocation)):
logger.info(u"Directory (" + str(comlocation) + ") already exists! Continuing...")
else:
#print ("Directory doesn't exist!")
#try:
# os.makedirs(str(comlocation))
# logger.info(u"Directory successfully created at: " + str(comlocation))
#except OSError:
# logger.error(u"Could not create comicdir : " + str(comlocation))
filechecker.validateAndCreateDirectory(comlocation, True)
comicIssues = gcdinfo['totalissues']
#let's download the image...
if os.path.exists(mylar.CACHE_DIR):pass
else:
#let's make the dir.
try:
os.makedirs(str(mylar.CACHE_DIR))
logger.info(u"Cache Directory successfully created at: " + str(mylar.CACHE_DIR))
except OSError:
logger.error(u"Could not create cache dir : " + str(mylar.CACHE_DIR))
coverfile = os.path.join(mylar.CACHE_DIR, str(gcomicid) + ".jpg")
#try:
urllib.urlretrieve(str(ComicImage), str(coverfile))
try:
with open(str(coverfile)) as f:
ComicImage = os.path.join('cache',str(gcomicid) + ".jpg")
#this is for Firefox when outside the LAN...it works, but I don't know how to implement it
#without breaking the normal flow for inside the LAN (above)
#ComicImage = "http://" + str(mylar.HTTP_HOST) + ":" + str(mylar.HTTP_PORT) + "/cache/" + str(comi$
logger.info(u"Sucessfully retrieved cover for " + ComicName)
#if the comic cover local is checked, save a cover.jpg to the series folder.
if mylar.COMIC_COVER_LOCAL:
comiclocal = os.path.join(str(comlocation) + "/cover.jpg")
shutil.copy(ComicImage,comiclocal)
except IOError as e:
logger.error(u"Unable to save cover locally at this time.")
#if comic['ComicVersion'].isdigit():
# comicVol = "v" + comic['ComicVersion']
#else:
# comicVol = None
controlValueDict = {"ComicID": gcomicid}
newValueDict = {"ComicName": ComicName,
"ComicSortName": sortname,
"ComicYear": ComicYear,
"Total": comicIssues,
"ComicLocation": comlocation,
#"ComicVersion": comicVol,
"ComicImage": ComicImage,
#"ComicPublisher": comic['ComicPublisher'],
#"ComicPublished": comicPublished,
"DateAdded": helpers.today(),
"Status": "Loading"}
myDB.upsert("comics", newValueDict, controlValueDict)
#comicsort here...
#run the re-sortorder here in order to properly display the page
if pullupd is None:
helpers.ComicSort(sequence='update')
logger.info(u"Sucessfully retrieved issue details for " + ComicName )
n = 0
iscnt = int(comicIssues)
issnum = []
issname = []
issdate = []
int_issnum = []
#let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :)
latestiss = "0"
latestdate = "0000-00-00"
#print ("total issues:" + str(iscnt))
#---removed NEW code here---
logger.info(u"Now adding/updating issues for " + ComicName)
bb = 0
while (bb <= iscnt):
#---NEW.code
try:
gcdval = gcdinfo['gcdchoice'][bb]
#print ("gcdval: " + str(gcdval))
except IndexError:
#account for gcd variation here
if gcdinfo['gcdvariation'] == 'gcd':
#print ("gcd-variation accounted for.")
issdate = '0000-00-00'
int_issnum = int ( issis / 1000 )
break
if 'nn' in str(gcdval['GCDIssue']):
#no number detected - GN, TP or the like
logger.warn(u"Non Series detected (Graphic Novel, etc) - cannot proceed at this time.")
updater.no_searchresults(comicid)
return
elif '.' in str(gcdval['GCDIssue']):
issst = str(gcdval['GCDIssue']).find('.')
issb4dec = str(gcdval['GCDIssue'])[:issst]
#if the length of decimal is only 1 digit, assume it's a tenth
decis = str(gcdval['GCDIssue'])[issst+1:]
if len(decis) == 1:
decisval = int(decis) * 10
issaftdec = str(decisval)
if len(decis) == 2:
decisval = int(decis)
issaftdec = str(decisval)
if int(issaftdec) == 0: issaftdec = "00"
gcd_issue = issb4dec + "." + issaftdec
gcdis = (int(issb4dec) * 1000) + decisval
else:
gcdis = int(str(gcdval['GCDIssue'])) * 1000
gcd_issue = str(gcdval['GCDIssue'])
#get the latest issue / date using the date.
int_issnum = int( gcdis / 1000 )
issdate = str(gcdval['GCDDate'])
issid = "G" + str(gcdval['IssueID'])
if gcdval['GCDDate'] > latestdate:
latestiss = str(gcd_issue)
latestdate = str(gcdval['GCDDate'])
#print("(" + str(bb) + ") IssueID: " + str(issid) + " IssueNo: " + str(gcd_issue) + " Date" + str(issdate) )
#---END.NEW.
# check if the issue already exists
iss_exists = myDB.action('SELECT * from issues WHERE IssueID=?', [issid]).fetchone()
# Only change the status & add DateAdded if the issue is not already in the database
if iss_exists is None:
newValueDict['DateAdded'] = helpers.today()
#adjust for inconsistencies in GCD date format - some dates have ? which borks up things.
if "?" in str(issdate):
issdate = "0000-00-00"
controlValueDict = {"IssueID": issid}
newValueDict = {"ComicID": gcomicid,
"ComicName": ComicName,
"Issue_Number": gcd_issue,
"IssueDate": issdate,
"Int_IssueNumber": int_issnum
}
#print ("issueid:" + str(controlValueDict))
#print ("values:" + str(newValueDict))
if mylar.AUTOWANT_ALL:
newValueDict['Status'] = "Wanted"
elif issdate > helpers.today() and mylar.AUTOWANT_UPCOMING:
newValueDict['Status'] = "Wanted"
else:
newValueDict['Status'] = "Skipped"
if iss_exists:
#print ("Existing status : " + str(iss_exists['Status']))
newValueDict['Status'] = iss_exists['Status']
myDB.upsert("issues", newValueDict, controlValueDict)
bb+=1
# logger.debug(u"Updating comic cache for " + ComicName)
# cache.getThumb(ComicID=issue['issueid'])
# logger.debug(u"Updating cache for: " + ComicName)
# cache.getThumb(ComicIDcomicid)
controlValueStat = {"ComicID": gcomicid}
newValueStat = {"Status": "Active",
"LatestIssue": latestiss,
"LatestDate": latestdate,
"LastUpdated": helpers.now()
}
myDB.upsert("comics", newValueStat, controlValueStat)
if mylar.CVINFO:
if not os.path.exists(comlocation + "/cvinfo"):
with open(comlocation + "/cvinfo","w") as text_file:
text_file.write("http://www.comicvine.com/volume/49-" + str(comicid))
logger.info(u"Updating complete for: " + ComicName)
#move the files...if imported is not empty (meaning it's not from the mass importer.)
if imported is None or imported == 'None':
pass
else:
if mylar.IMP_MOVE:
logger.info("Mass import - Move files")
moveit.movefiles(gcomicid,comlocation,ogcname)
else:
logger.info("Mass import - Moving not Enabled. Setting Archived Status for import.")
moveit.archivefiles(gcomicid,ogcname)
#check for existing files...
updater.forceRescan(gcomicid)
if pullupd is None:
# lets' check the pullist for anyting at this time as well since we're here.
if mylar.AUTOWANT_UPCOMING and 'Present' in ComicPublished:
logger.info(u"Checking this week's pullist for new issues of " + ComicName)
updater.newpullcheck(comic['ComicName'], gcomicid)
#here we grab issues that have been marked as wanted above...
results = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [gcomicid])
if results:
logger.info(u"Attempting to grab wanted issues for : " + ComicName)
for result in results:
foundNZB = "none"
if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL or mylar.NEWZNAB or mylar.NZBX) and (mylar.SAB_HOST):
foundNZB = search.searchforissue(result['IssueID'])
if foundNZB == "yes":
updater.foundsearch(result['ComicID'], result['IssueID'])
else: logger.info(u"No issues marked as wanted for " + ComicName)
logger.info(u"Finished grabbing what I could.")
def addComictoDB(comicid):
# Putting this here to get around the circular import. Will try to use this to update images at later date.
from mylar import cache
myDB = db.DBConnection()
# myDB.action('DELETE from blacklist WHERE ComicID=?', [comicid])
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ComicID": comicid}
dbcomic = myDB.action('SELECT * FROM comics WHERE ComicID=?', [comicid]).fetchone()
if dbcomic is None:
newValueDict = {"ComicName": "Comic ID: %s" % (comicid),
"Status": "Loading"}
else:
newValueDict = {"Status": "Loading"}
myDB.upsert("comics", newValueDict, controlValueDict)
# we need to lookup the info for the requested ComicID in full now
comic = cv.getComic(comicid,'comic')
if not comic:
logger.warn("Error fetching comic. ID for : " + comicid)
if dbcomic is None:
newValueDict = {"ComicName": "Fetch failed, try refreshing. (%s)" % (comicid),
"Status": "Active"}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("comics", newValueDict, controlValueDict)
return
if comic['ComicName'].startswith('The '):
sortname = comic['ComicName'][4:]
else:
sortname = comic['ComicName']
logger.info(u"Now adding/updating: " + comic['ComicName'])
#--Now that we know ComicName, let's try some scraping
#--Start
# gcd will return issue details (most importantly publishing date)
gcdinfo=parseit.GCDScraper(comic['ComicName'], comic['ComicYear'], comic['ComicIssues'], comicid)
if gcdinfo == "No Match":
logger.warn("No matching result found for " + comic['ComicName'] + " (" + comic['ComicYear'] + ")" )
updater.no_searchresults(comicid)
return
logger.info(u"Sucessfully retrieved details for " + comic['ComicName'] )
# print ("Series Published" + parseit.resultPublished)
#--End
#comic book location on machine
# setup default location here
if ':' in comic['ComicName']:
comicdir = comic['ComicName'].replace(':','')
else: comicdir = comic['ComicName']
comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + comic['ComicYear'] + ")"
if mylar.DESTINATION_DIR == "":
logger.error(u"There is no general directory specified - please specify in Config/Post-Processing.")
return
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
comlocation = comlocation.replace(' ', mylar.REPLACE_CHAR)
#if it doesn't exist - create it (otherwise will bugger up later on)
if os.path.isdir(str(comlocation)):
logger.info(u"Directory (" + str(comlocation) + ") already exists! Continuing...")
else:
#print ("Directory doesn't exist!")
try:
os.makedirs(str(comlocation))
logger.info(u"Directory successfully created at: " + str(comlocation))
except OSError.e:
if e.errno != errno.EEXIST:
raise
#print ("root dir for series: " + comlocation)
#try to account for CV not updating new issues as fast as GCD
#seems CV doesn't update total counts
#comicIssues = gcdinfo['totalissues']
if gcdinfo['gcdvariation'] == "cv":
comicIssues = str(int(comic['ComicIssues']) + 1)
else:
comicIssues = comic['ComicIssues']
controlValueDict = {"ComicID": comicid}
newValueDict = {"ComicName": comic['ComicName'],
"ComicSortName": sortname,
"ComicYear": comic['ComicYear'],
"ComicImage": comic['ComicImage'],
"Total": comicIssues,
"ComicLocation": comlocation,
"ComicPublisher": comic['ComicPublisher'],
"ComicPublished": parseit.resultPublished,
"DateAdded": helpers.today(),
"Status": "Loading"}
myDB.upsert("comics", newValueDict, controlValueDict)
issued = cv.getComic(comicid,'issue')
logger.info(u"Sucessfully retrieved issue details for " + comic['ComicName'] )
n = 0
iscnt = int(comicIssues)
issid = []
issnum = []
issname = []
issdate = []
int_issnum = []
#let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :)
latestiss = "0"
latestdate = "0000-00-00"
#print ("total issues:" + str(iscnt))
#---removed NEW code here---
logger.info(u"Now adding/updating issues for" + comic['ComicName'])
# file check to see if issue exists
logger.info(u"Checking directory for existing issues.")
#fc = filechecker.listFiles(dir=comlocation, watchcomic=comic['ComicName'])
#havefiles = 0
#fccnt = int(fc['comiccount'])
#logger.info(u"Found " + str(fccnt) + "/" + str(iscnt) + " issues of " + comic['ComicName'] + "...verifying")
#fcnew = []
while (n <= iscnt):
#---NEW.code
try:
firstval = issued['issuechoice'][n]
except IndexError:
break
cleanname = helpers.cleanName(firstval['Issue_Name'])
issid = str(firstval['Issue_ID'])
issnum = str(firstval['Issue_Number'])
issname = cleanname
if '.' in str(issnum):
issn_st = str(issnum).find('.')
issn_b4dec = str(issnum)[:issn_st]
#if the length of decimal is only 1 digit, assume it's a tenth
dec_is = str(issnum)[issn_st + 1:]
if len(dec_is) == 1:
dec_nisval = int(dec_is) * 10
iss_naftdec = str(dec_nisval)
if len(dec_is) == 2:
dec_nisval = int(dec_is)
iss_naftdec = str(dec_nisval)
iss_issue = issn_b4dec + "." + iss_naftdec
issis = (int(issn_b4dec) * 1000) + dec_nisval
else: issis = int(issnum) * 1000
bb = 0
while (bb <= iscnt):
try:
gcdval = gcdinfo['gcdchoice'][bb]
except IndexError:
#account for gcd variation here
if gcdinfo['gcdvariation'] == 'gcd':
print ("gcd-variation accounted for.")
issdate = '0000-00-00'
int_issnum = int ( issis / 1000 )
break
if 'nn' in str(gcdval['GCDIssue']):
#no number detected - GN, TP or the like
logger.warn(u"Non Series detected (Graphic Novel, etc) - cannot proceed at this time.")
updater.no_searchresults(comicid)
return
elif '.' in str(gcdval['GCDIssue']):
issst = str(gcdval['GCDIssue']).find('.')
issb4dec = str(gcdval['GCDIssue'])[:issst]
#if the length of decimal is only 1 digit, assume it's a tenth
decis = str(gcdval['GCDIssue'])[issst+1:]
if len(decis) == 1:
decisval = int(decis) * 10
issaftdec = str(decisval)
if len(decis) == 2:
decisval = int(decis)
issaftdec = str(decisval)
gcd_issue = issb4dec + "." + issaftdec
gcdis = (int(issb4dec) * 1000) + decisval
else:
gcdis = int(str(gcdval['GCDIssue'])) * 1000
if gcdis == issis:
issdate = str(gcdval['GCDDate'])
int_issnum = int( gcdis / 1000 )
#get the latest issue / date using the date.
if gcdval['GCDDate'] > latestdate:
latestiss = str(issnum)
latestdate = str(gcdval['GCDDate'])
break
#bb = iscnt
bb+=1
#print("(" + str(n) + ") IssueID: " + str(issid) + " IssueNo: " + str(issnum) + " Date" + str(issdate))
#---END.NEW.
# check if the issue already exists
iss_exists = myDB.select('SELECT * from issues WHERE IssueID=?', [issid])
# Only change the status & add DateAdded if the issue is not already in the database
if not len(iss_exists):
newValueDict['DateAdded'] = helpers.today()
controlValueDict = {"IssueID": issid}
newValueDict = {"ComicID": comicid,
"ComicName": comic['ComicName'],
"IssueName": issname,
"Issue_Number": issnum,
"IssueDate": issdate,
"Int_IssueNumber": int_issnum
}
if mylar.AUTOWANT_ALL:
newValueDict['Status'] = "Wanted"
#elif release_dict['releasedate'] > helpers.today() and mylar.AUTOWANT_UPCOMING:
# newValueDict['Status'] = "Wanted"
else:
newValueDict['Status'] = "Skipped"
myDB.upsert("issues", newValueDict, controlValueDict)
n+=1
# logger.debug(u"Updating comic cache for " + comic['ComicName'])
# cache.getThumb(ComicID=issue['issueid'])
# logger.debug(u"Updating cache for: " + comic['ComicName'])
# cache.getThumb(ComicIDcomicid)
#check for existing files...
updater.forceRescan(comicid)
controlValueStat = {"ComicID": comicid}
newValueStat = {"Status": "Active",
"LatestIssue": latestiss,
"LatestDate": latestdate
}
myDB.upsert("comics", newValueStat, controlValueStat)
logger.info(u"Updating complete for: " + comic['ComicName'])
#here we grab issues that have been marked as wanted above...
results = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [comicid])
if results:
logger.info(u"Attempting to grab wanted issues for : " + comic['ComicName'])
for result in results:
foundNZB = "none"
if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL) and (mylar.SAB_HOST):
foundNZB = search.searchforissue(result['IssueID'])
if foundNZB == "yes":
updater.foundsearch(result['ComicID'], result['IssueID'])
else: logger.info(u"No issues marked as wanted for " + comic['ComicName'])
logger.info(u"Finished grabbing what I could.")
def Process(self):
self._log("nzb name: " + str(self.nzb_name), logger.DEBUG)
self._log("nzb folder: " + str(self.nzb_folder), logger.DEBUG)
logger.fdebug("nzb name: " + str(self.nzb_name))
logger.fdebug("nzb folder: " + str(self.nzb_folder))
#lookup nzb_name in nzblog table to get issueid
#query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals
#http://localhost:8080/sabnzbd/api?mode=set_config§ion=misc&keyword=dirscan_speed&value=5
querysab = str(mylar.SAB_HOST) + "/api?mode=get_config§ion=misc&output=xml&apikey=" + str(mylar.SAB_APIKEY)
#logger.info("querysab_string:" + str(querysab))
file = urllib2.urlopen(querysab)
data = file.read()
file.close()
dom = parseString(data)
sabreps = dom.getElementsByTagName('replace_spaces')[0].firstChild.wholeText
sabrepd = dom.getElementsByTagName('replace_dots')[0].firstChild.wholeText
logger.fdebug("SAB Replace Spaces: " + str(sabreps))
logger.fdebug("SAB Replace Dots: " + str(sabrepd))
myDB = db.DBConnection()
nzbname = self.nzb_name
#remove extensions from nzb_name if they somehow got through (Experimental most likely)
extensions = ('.cbr', '.cbz')
if nzbname.lower().endswith(extensions):
fd, ext = os.path.splitext(nzbname)
self._log("Removed extension from nzb: " + ext, logger.DEBUG)
nzbname = re.sub(str(ext), '', str(nzbname))
#replace spaces
nzbname = re.sub(' ', '.', str(nzbname))
nzbname = re.sub('[\,\:]', '', str(nzbname))
logger.fdebug("After conversions, nzbname is : " + str(nzbname))
self._log("nzbname: " + str(nzbname), logger.DEBUG)
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
self._log("Failure - could not initially locate nzbfile in my database to rename.", logger.DEBUG)
logger.fdebug("Failure - could not locate nzbfile initially.")
# if failed on spaces, change it all to decimals and try again.
nzbname = re.sub('_', '.', str(nzbname))
self._log("trying again with this nzbname: " + str(nzbname), logger.DEBUG)
logger.fdebug("trying again with nzbname of : " + str(nzbname))
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
logger.error(u"Unable to locate downloaded file to rename. PostProcessing aborted.")
return
else:
self._log("I corrected and found the nzb as : " + str(nzbname))
logger.fdebug("auto-corrected and found the nzb as : " + str(nzbname))
issueid = nzbiss['IssueID']
else:
issueid = nzbiss['IssueID']
#use issueid to get publisher, series, year, issue number
issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone()
comicid = issuenzb['ComicID']
issuenum = issuenzb['Issue_Number']
#issueno = str(issuenum).split('.')[0]
iss_find = issuenum.find('.')
iss_b4dec = issuenum[:iss_find]
iss_decval = issuenum[iss_find+1:]
if int(iss_decval) == 0:
iss = iss_b4dec
issdec = int(iss_decval)
issueno = str(iss)
self._log("Issue Number: " + str(issueno), logger.DEBUG)
logger.fdebug("Issue Number: " + str(issueno))
else:
if len(iss_decval) == 1:
iss = iss_b4dec + "." + iss_decval
issdec = int(iss_decval) * 10
else:
iss = iss_b4dec + "." + iss_decval.rstrip('0')
issdec = int(iss_decval.rstrip('0')) * 10
issueno = iss_b4dec
self._log("Issue Number: " + str(iss), logger.DEBUG)
logger.fdebug("Issue Number: " + str(iss))
# issue zero-suppression here
if mylar.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00"
logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N))
if str(len(issueno)) > 1:
if int(issueno) < 10:
self._log("issue detected less than 10", logger.DEBUG)
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG)
elif int(issueno) >= 10 and int(issueno) < 100:
self._log("issue detected greater than 10, but less than 100", logger.DEBUG)
if mylar.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss), logger.DEBUG)
else:
self._log("issue detected greater than 100", logger.DEBUG)
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(issueno)
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG)
else:
prettycomiss = str(issueno)
self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss), logger.DEBUG)
logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss))
issueyear = issuenzb['IssueDate'][:4]
self._log("Issue Year: " + str(issueyear), logger.DEBUG)
logger.fdebug("Issue Year : " + str(issueyear))
comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
publisher = comicnzb['ComicPublisher']
self._log("Publisher: " + publisher, logger.DEBUG)
logger.fdebug("Publisher: " + str(publisher))
series = comicnzb['ComicName']
self._log("Series: " + series, logger.DEBUG)
logger.fdebug("Series: " + str(series))
seriesyear = comicnzb['ComicYear']
self._log("Year: " + seriesyear, logger.DEBUG)
logger.fdebug("Year: " + str(seriesyear))
comlocation = comicnzb['ComicLocation']
self._log("Comic Location: " + comlocation, logger.DEBUG)
logger.fdebug("Comic Location: " + str(comlocation))
#Run Pre-script
if mylar.ENABLE_PRE_SCRIPTS:
nzbn = self.nzb_name #original nzb name
nzbf = self.nzb_folder #original nzb folder
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_pre_scripts(nzbn, nzbf, seriesmetadata )
#rename file and move to new path
#nfilename = series + " " + issueno + " (" + seriesyear + ")"
file_values = {'$Series': series,
'$Issue': prettycomiss,
'$Year': issueyear,
'$series': series.lower(),
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$Volume': seriesyear
}
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
ofilename = filename
path, ext = os.path.splitext(ofilename)
self._log("Original Filename: " + ofilename, logger.DEBUG)
self._log("Original Extension: " + ext, logger.DEBUG)
logger.fdebug("Original Filname: " + str(ofilename))
logger.fdebug("Original Extension: " + str(ext))
if mylar.FILE_FORMAT == '':
self._log("Rename Files isn't enabled...keeping original filename.", logger.DEBUG)
logger.fdebug("Rename Files isn't enabled - keeping original filename.")
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
nfilename = helpers.replace_all(mylar.FILE_FORMAT, file_values)
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
nfilename = re.sub('[\,\:]', '', nfilename)
self._log("New Filename: " + nfilename, logger.DEBUG)
logger.fdebug("New Filename: " + str(nfilename))
src = os.path.join(self.nzb_folder, ofilename)
if mylar.LOWERCASE_FILENAMES:
dst = (comlocation + "/" + nfilename + ext).lower()
else:
dst = comlocation + "/" + nfilename + ext.lower()
self._log("Source:" + src, logger.DEBUG)
self._log("Destination:" + dst, logger.DEBUG)
logger.fdebug("Source: " + str(src))
logger.fdebug("Destination: " + str(dst))
os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
src = os.path.join(self.nzb_folder, str(nfilename + ext))
try:
shutil.move(src, dst)
except (OSError, IOError):
self._log("Failed to move directory - check directories and manually re-run.", logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log("Failed to remove temporary directory - check directory and manually re-run.", logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
self._log("Removed temporary directory : " + str(self.nzb_folder), logger.DEBUG)
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
#force rescan of files
updater.forceRescan(comicid)
logger.info(u"Post-Processing completed for: " + series + " issue: " + str(issuenum) )
self._log(u"Post Processing SUCCESSFULL! ", logger.DEBUG)
# retrieve/create the corresponding comic objects
if mylar.ENABLE_EXTRA_SCRIPTS:
folderp = str(dst) #folder location after move/rename
nzbn = self.nzb_name #original nzb name
filen = str(nfilename + ext) #new filename
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_extra_scripts(nzbname, self.nzb_folder, filen, folderp, seriesmetadata )
return self.log
control = {"IssueID": issuechk['IssueID']}
values = {"Status": "Archived"}
logger.fdebug('...changing status of ' +
str(issuechk['Issue_Number']) +
' to Archived ')
myDB.upsert("issues", values, control)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
wat += 1
if comicids is None: pass
else:
c_upd = len(comicids)
c = 0
while (c < c_upd):
logger.fdebug('Rescanning.. ' + str(c))
updater.forceRescan(c)
if not len(import_by_comicids):
return "Completed"
if len(import_by_comicids) > 0 or len(vals) > 0:
#import_comicids['comic_info'] = import_by_comicids
#if vals:
# import_comicids['issueid_info'] = vals
#else:
# import_comicids['issueid_info'] = None
if vals:
cvimport_comicids = vals
import_cv_ids = len(vals)
else:
cvimport_comicids = None
import_cv_ids = 0
def Process(self):
self._log("nzb name: " + str(self.nzb_name), logger.DEBUG)
self._log("nzb folder: " + str(self.nzb_folder), logger.DEBUG)
logger.fdebug("nzb name: " + str(self.nzb_name))
logger.fdebug("nzb folder: " + str(self.nzb_folder))
if mylar.USE_SABNZBD==0:
logger.fdebug("Not using SABNzbd")
else:
# if the SAB Directory option is enabled, let's use that folder name and append the jobname.
if mylar.SAB_DIRECTORY is not None and mylar.SAB_DIRECTORY is not 'None' and len(mylar.SAB_DIRECTORY) > 4:
self.nzb_folder = os.path.join(mylar.SAB_DIRECTORY, self.nzb_name).encode(mylar.SYS_ENCODING)
#lookup nzb_name in nzblog table to get issueid
#query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals
#http://localhost:8080/sabnzbd/api?mode=set_config§ion=misc&keyword=dirscan_speed&value=5
querysab = str(mylar.SAB_HOST) + "/api?mode=get_config§ion=misc&output=xml&apikey=" + str(mylar.SAB_APIKEY)
#logger.info("querysab_string:" + str(querysab))
file = urllib2.urlopen(querysab)
data = file.read()
file.close()
dom = parseString(data)
try:
sabreps = dom.getElementsByTagName('replace_spaces')[0].firstChild.wholeText
except:
errorm = dom.getElementsByTagName('error')[0].firstChild.wholeText
logger.error(u"Error detected attempting to retrieve SAB data : " + errorm)
return
sabrepd = dom.getElementsByTagName('replace_dots')[0].firstChild.wholeText
logger.fdebug("SAB Replace Spaces: " + str(sabreps))
logger.fdebug("SAB Replace Dots: " + str(sabrepd))
if mylar.USE_NZBGET==1:
logger.fdebug("Using NZBGET")
logger.fdebug("NZB name as passed from NZBGet: " + self.nzb_name)
myDB = db.DBConnection()
nzbname = self.nzb_name
#remove extensions from nzb_name if they somehow got through (Experimental most likely)
extensions = ('.cbr', '.cbz')
if nzbname.lower().endswith(extensions):
fd, ext = os.path.splitext(nzbname)
self._log("Removed extension from nzb: " + ext, logger.DEBUG)
nzbname = re.sub(str(ext), '', str(nzbname))
#replace spaces
nzbname = re.sub(' ', '.', str(nzbname))
nzbname = re.sub('[\,\:\?]', '', str(nzbname))
nzbname = re.sub('[\&]', 'and', str(nzbname))
logger.fdebug("After conversions, nzbname is : " + str(nzbname))
# if mylar.USE_NZBGET==1:
# nzbname=self.nzb_name
self._log("nzbname: " + str(nzbname), logger.DEBUG)
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
self._log("Failure - could not initially locate nzbfile in my database to rename.", logger.DEBUG)
logger.fdebug("Failure - could not locate nzbfile initially.")
# if failed on spaces, change it all to decimals and try again.
nzbname = re.sub('_', '.', str(nzbname))
self._log("trying again with this nzbname: " + str(nzbname), logger.DEBUG)
logger.fdebug("trying again with nzbname of : " + str(nzbname))
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
logger.error(u"Unable to locate downloaded file to rename. PostProcessing aborted.")
return
else:
self._log("I corrected and found the nzb as : " + str(nzbname))
logger.fdebug("auto-corrected and found the nzb as : " + str(nzbname))
issueid = nzbiss['IssueID']
else:
issueid = nzbiss['IssueID']
logger.fdebug("issueid:" + str(issueid))
sarc = nzbiss['SARC']
#use issueid to get publisher, series, year, issue number
issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone()
if issuenzb is not None:
if helpers.is_number(issueid):
sandwich = int(issuenzb['IssueID'])
else:
#if it's non-numeric, it contains a 'G' at the beginning indicating it's a multi-volume
#using GCD data. Set sandwich to 1 so it will bypass and continue post-processing.
if 'S' in issueid:
sandwich = issueid
elif 'G' in issueid:
sandwich = 1
if helpers.is_number(sandwich):
if sandwich < 900000:
# if sandwich is less than 900000 it's a normal watchlist download. Bypass.
pass
else:
if issuenzb is None or 'S' in sandwich or int(sandwich) >= 900000:
# this has no issueID, therefore it's a one-off or a manual post-proc.
# At this point, let's just drop it into the Comic Location folder and forget about it..
if 'S' in sandwich:
self._log("One-off STORYARC mode enabled for Post-Processing for " + str(sarc))
logger.info("One-off STORYARC mode enabled for Post-Processing for " + str(sarc))
if mylar.STORYARCDIR:
storyarcd = os.path.join(mylar.DESTINATION_DIR, "StoryArcs", sarc)
self._log("StoryArc Directory set to : " + storyarcd, logger.DEBUG)
else:
self._log("Grab-Bag Directory set to : " + mylar.GRABBAG_DIR, logger.DEBUG)
else:
self._log("One-off mode enabled for Post-Processing. All I'm doing is moving the file untouched into the Grab-bag directory.", logger.DEBUG)
logger.info("One-off mode enabled for Post-Processing. Will move into Grab-bag directory.")
self._log("Grab-Bag Directory set to : " + mylar.GRABBAG_DIR, logger.DEBUG)
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
ofilename = filename
path, ext = os.path.splitext(ofilename)
if 'S' in sandwich:
if mylar.STORYARCDIR:
grdst = storyarcd
else:
grdst = mylar.DESTINATION_DIR
else:
if mylar.GRABBAG_DIR:
grdst = mylar.GRABBAG_DIR
else:
grdst = mylar.DESTINATION_DIR
filechecker.validateAndCreateDirectory(grdst, True)
grab_dst = os.path.join(grdst, ofilename)
self._log("Destination Path : " + grab_dst, logger.DEBUG)
logger.info("Destination Path : " + grab_dst)
grab_src = os.path.join(self.nzb_folder, ofilename)
self._log("Source Path : " + grab_src, logger.DEBUG)
logger.info("Source Path : " + grab_src)
logger.info("Moving " + str(ofilename) + " into directory : " + str(grdst))
try:
shutil.move(grab_src, grab_dst)
except (OSError, IOError):
self._log("Failed to move directory - check directories and manually re-run.", logger.DEBUG)
logger.debug("Failed to move directory - check directories and manually re-run.")
return
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log("Failed to remove temporary directory.", logger.DEBUG)
logger.debug("Failed to remove temporary directory - check directory and manually re-run.")
return
logger.debug("Removed temporary directory : " + str(self.nzb_folder))
self._log("Removed temporary directory : " + self.nzb_folder, logger.DEBUG)
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
if 'S' in issueid:
issuearcid = re.sub('S', '', issueid)
logger.info("IssueArcID is : " + str(issuearcid))
ctrlVal = {"IssueArcID": issuearcid}
newVal = {"Status": "Downloaded",
"Location": grab_dst }
myDB.upsert("readinglist",newVal,ctrlVal)
logger.info("updated status to Downloaded")
return self.log
comicid = issuenzb['ComicID']
issuenumOG = issuenzb['Issue_Number']
#issueno = str(issuenum).split('.')[0]
#new CV API - removed all decimals...here we go AGAIN!
issuenum = issuenumOG
issue_except = 'None'
if 'au' in issuenum.lower():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AU'
if '.' in issuenum:
iss_find = issuenum.find('.')
iss_b4dec = issuenum[:iss_find]
iss_decval = issuenum[iss_find+1:]
if int(iss_decval) == 0:
iss = iss_b4dec
issdec = int(iss_decval)
issueno = str(iss)
self._log("Issue Number: " + str(issueno), logger.DEBUG)
logger.fdebug("Issue Number: " + str(issueno))
else:
if len(iss_decval) == 1:
iss = iss_b4dec + "." + iss_decval
issdec = int(iss_decval) * 10
else:
iss = iss_b4dec + "." + iss_decval.rstrip('0')
issdec = int(iss_decval.rstrip('0')) * 10
issueno = iss_b4dec
self._log("Issue Number: " + str(iss), logger.DEBUG)
logger.fdebug("Issue Number: " + str(iss))
else:
iss = issuenum
issueno = str(iss)
# issue zero-suppression here
if mylar.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00"
logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N))
if str(len(issueno)) > 1:
if int(issueno) < 10:
self._log("issue detected less than 10", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG)
elif int(issueno) >= 10 and int(issueno) < 100:
self._log("issue detected greater than 10, but less than 100", logger.DEBUG)
if mylar.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss), logger.DEBUG)
else:
self._log("issue detected greater than 100", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(issueno)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG)
else:
prettycomiss = str(issueno)
self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss), logger.DEBUG)
logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss))
issueyear = issuenzb['IssueDate'][:4]
self._log("Issue Year: " + str(issueyear), logger.DEBUG)
logger.fdebug("Issue Year : " + str(issueyear))
comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
publisher = comicnzb['ComicPublisher']
self._log("Publisher: " + publisher, logger.DEBUG)
logger.fdebug("Publisher: " + str(publisher))
#we need to un-unicode this to make sure we can write the filenames properly for spec.chars
series = comicnzb['ComicName'].encode('ascii', 'ignore').strip()
self._log("Series: " + series, logger.DEBUG)
logger.fdebug("Series: " + str(series))
seriesyear = comicnzb['ComicYear']
self._log("Year: " + seriesyear, logger.DEBUG)
logger.fdebug("Year: " + str(seriesyear))
comlocation = comicnzb['ComicLocation']
self._log("Comic Location: " + comlocation, logger.DEBUG)
logger.fdebug("Comic Location: " + str(comlocation))
comversion = comicnzb['ComicVersion']
self._log("Comic Version: " + str(comversion), logger.DEBUG)
logger.fdebug("Comic Version: " + str(comversion))
if comversion is None:
comversion = 'None'
#if comversion is None, remove it so it doesn't populate with 'None'
if comversion == 'None':
chunk_f_f = re.sub('\$VolumeN','',mylar.FILE_FORMAT)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
self._log("No version # found for series - tag will not be available for renaming.", logger.DEBUG)
logger.fdebug("No version # found for series, removing from filename")
logger.fdebug("new format is now: " + str(chunk_file_format))
else:
chunk_file_format = mylar.FILE_FORMAT
#Run Pre-script
if mylar.ENABLE_PRE_SCRIPTS:
nzbn = self.nzb_name #original nzb name
nzbf = self.nzb_folder #original nzb folder
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_pre_scripts(nzbn, nzbf, seriesmetadata )
#rename file and move to new path
#nfilename = series + " " + issueno + " (" + seriesyear + ")"
file_values = {'$Series': series,
'$Issue': prettycomiss,
'$Year': issueyear,
'$series': series.lower(),
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion
}
ofilename = None
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
ofilename = filename
path, ext = os.path.splitext(ofilename)
if ofilename is None:
logger.error(u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that " + str(self.nzb_folder) + " exists and is the correct location.")
return
self._log("Original Filename: " + ofilename, logger.DEBUG)
self._log("Original Extension: " + ext, logger.DEBUG)
logger.fdebug("Original Filname: " + str(ofilename))
logger.fdebug("Original Extension: " + str(ext))
if mylar.FILE_FORMAT == '' or not mylar.RENAME_FILES:
self._log("Rename Files isn't enabled...keeping original filename.", logger.DEBUG)
logger.fdebug("Rename Files isn't enabled - keeping original filename.")
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
nfilename = helpers.replace_all(chunk_file_format, file_values)
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
nfilename = re.sub('[\,\:\?]', '', nfilename)
self._log("New Filename: " + nfilename, logger.DEBUG)
logger.fdebug("New Filename: " + str(nfilename))
src = os.path.join(self.nzb_folder, ofilename)
filechecker.validateAndCreateDirectory(comlocation, True)
if mylar.LOWERCASE_FILENAMES:
dst = (comlocation + "/" + nfilename + ext).lower()
else:
dst = comlocation + "/" + nfilename + ext.lower()
self._log("Source:" + src, logger.DEBUG)
self._log("Destination:" + dst, logger.DEBUG)
logger.fdebug("Source: " + str(src))
logger.fdebug("Destination: " + str(dst))
os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
src = os.path.join(self.nzb_folder, str(nfilename + ext))
try:
shutil.move(src, dst)
except (OSError, IOError):
self._log("Failed to move directory - check directories and manually re-run.", logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log("Failed to remove temporary directory - check directory and manually re-run.", logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
self._log("Removed temporary directory : " + str(self.nzb_folder), logger.DEBUG)
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
#update snatched table to change status to Downloaded
updater.foundsearch(comicid, issueid, down='True')
#force rescan of files
updater.forceRescan(comicid)
logger.info(u"Post-Processing completed for: " + series + " issue: " + str(issuenumOG) )
self._log(u"Post Processing SUCCESSFULL! ", logger.DEBUG)
if mylar.PROWL_ENABLED:
pushmessage = series + '(' + issueyear + ') - issue #' + issuenumOG
logger.info(u"Prowl request")
prowl = notifiers.PROWL()
prowl.notify(pushmessage,"Download and Postprocessing completed")
if mylar.NMA_ENABLED:
nma = notifiers.NMA()
nma.notify(series, str(issueyear), str(issuenumOG))
if mylar.PUSHOVER_ENABLED:
pushmessage = series + ' (' + str(issueyear) + ') - issue #' + str(issuenumOG)
logger.info(u"Pushover request")
pushover = notifiers.PUSHOVER()
pushover.notify(pushmessage, "Download and Post-Processing completed")
# retrieve/create the corresponding comic objects
if mylar.ENABLE_EXTRA_SCRIPTS:
folderp = str(dst) #folder location after move/rename
nzbn = self.nzb_name #original nzb name
filen = str(nfilename + ext) #new filename
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_extra_scripts(nzbname, self.nzb_folder, filen, folderp, seriesmetadata )
return self.log
def libraryScan(dir=None,
append=False,
ComicID=None,
ComicName=None,
cron=None):
if cron and not mylar.LIBRARYSCAN:
return
if not dir:
dir = mylar.COMIC_DIR
# If we're appending a dir, it's coming from the post processor which is
# already bytestring
if not append:
dir = dir.encode(mylar.SYS_ENCODING)
if not os.path.isdir(dir):
logger.warn('Cannot find directory: %s. Not scanning' %
dir.decode(mylar.SYS_ENCODING, 'replace'))
return
logger.info('Scanning comic directory: %s' %
dir.decode(mylar.SYS_ENCODING, 'replace'))
basedir = dir
comic_list = []
comiccnt = 0
extensions = ('cbr', 'cbz')
cv_location = []
for r, d, f in os.walk(dir):
for files in f:
if 'cvinfo' in files:
cv_location.append(r)
logger.fdebug('CVINFO found: ' + os.path.join(r))
if any(files.lower().endswith('.' + x.lower())
for x in extensions):
comic = files
comicpath = os.path.join(r, files)
comicsize = os.path.getsize(comicpath)
logger.fdebug('Comic: ' + comic + ' [' + comicpath + '] - ' +
str(comicsize) + ' bytes')
comiccnt += 1
# We need the unicode path to use for logging, inserting into database
unicode_comic_path = comicpath.decode(mylar.SYS_ENCODING,
'replace')
comic_dict = {
'ComicFilename': comic,
'ComicLocation': comicpath,
'ComicSize': comicsize,
'Unicode_ComicLocation': unicode_comic_path
}
comic_list.append(comic_dict)
logger.info("I've found a total of " + str(comiccnt) +
" comics....analyzing now")
#logger.info("comiclist: " + str(comic_list))
myDB = db.DBConnection()
#let's load in the watchlist to see if we have any matches.
logger.info(
"loading in the watchlist to see if a series is being watched already..."
)
watchlist = myDB.select("SELECT * from comics")
ComicName = []
DisplayName = []
ComicYear = []
ComicPublisher = []
ComicTotal = []
ComicID = []
ComicLocation = []
AltName = []
watchcnt = 0
watch_kchoice = []
watchchoice = {}
import_by_comicids = []
import_comicids = {}
for watch in watchlist:
#use the comicname_filesafe to start
watchdisplaycomic = watch['ComicName'].encode('utf-8').strip(
) #re.sub('[\_\#\,\/\:\;\!\$\%\&\+\'\?\@]', ' ', watch['ComicName']).encode('utf-8').strip()
# let's clean up the name, just in case for comparison purposes...
watchcomic = re.sub(
'[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '',
watch['ComicName_Filesafe']).encode('utf-8').strip()
#watchcomic = re.sub('\s+', ' ', str(watchcomic)).strip()
if ' the ' in watchcomic.lower():
#drop the 'the' from the watchcomic title for proper comparisons.
watchcomic = watchcomic[-4:]
alt_chk = "no" # alt-checker flag (default to no)
# account for alternate names as well
if watch['AlternateSearch'] is not None and watch[
'AlternateSearch'] is not 'None':
altcomic = re.sub(
'[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '',
watch['AlternateSearch']).encode('utf-8').strip()
#altcomic = re.sub('\s+', ' ', str(altcomic)).strip()
AltName.append(altcomic)
alt_chk = "yes" # alt-checker flag
ComicName.append(watchcomic)
DisplayName.append(watchdisplaycomic)
ComicYear.append(watch['ComicYear'])
ComicPublisher.append(watch['ComicPublisher'])
ComicTotal.append(watch['Total'])
ComicID.append(watch['ComicID'])
ComicLocation.append(watch['ComicLocation'])
watchcnt += 1
logger.info("Successfully loaded " + str(watchcnt) +
" series from your watchlist.")
ripperlist = ['digital-', 'empire', 'dcp']
watchfound = 0
datelist = [
'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct',
'nov', 'dec'
]
# datemonth = {'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9,'ten':10,'eleven':$
# #search for number as text, and change to numeric
# for numbs in basnumbs:
# #logger.fdebug("numbs:" + str(numbs))
# if numbs in ComicName.lower():
# numconv = basnumbs[numbs]
# #logger.fdebug("numconv: " + str(numconv))
issueid_list = []
cvscanned_loc = None
cvinfo_CID = None
for i in comic_list:
logger.fdebug('Analyzing : ' + i['ComicFilename'])
comfilename = i['ComicFilename']
comlocation = i['ComicLocation']
issueinfo = None
#Make sure cvinfo is checked for FIRST (so that CID can be attached to all files properly thereafter as they're scanned in)
if os.path.dirname(comlocation) in cv_location and os.path.dirname(
comlocation) != cvscanned_loc:
#if comfilename == 'cvinfo':
logger.info('comfilename: ' + comfilename)
logger.info('cvscanned_loc: ' + str(cv_location))
logger.info('comlocation: ' + os.path.dirname(comlocation))
#if cvscanned_loc != comlocation:
try:
with open(os.path.join(os.path.dirname(comlocation),
'cvinfo')) as f:
urllink = f.readline()
print 'urllink: ' + str(urllink)
if urllink:
cid = urllink.split('/')
if '4050-' in cid[-2]:
cvinfo_CID = re.sub('4050-', '', cid[-2]).strip()
logger.info(
'CVINFO file located within directory. Attaching everything in directory that is valid to ComicID: '
+ str(cvinfo_CID))
#store the location of the cvinfo so it's applied to the correct directory (since we're scanning multile direcorties usually)
cvscanned_loc = os.path.dirname(comlocation)
else:
logger.error(
"Could not read cvinfo file properly (or it does not contain any data)"
)
except (OSError, IOError):
logger.error(
"Could not read cvinfo file properly (or it does not contain any data)"
)
#else:
# don't scan in it again if it's already been done initially
# continue
if mylar.IMP_METADATA:
logger.info('metatagging checking enabled.')
#if read tags is enabled during import, check here.
if i['ComicLocation'].endswith('.cbz'):
logger.info('Attempting to read tags present in filename: ' +
i['ComicLocation'])
issueinfo = helpers.IssueDetails(i['ComicLocation'])
if issueinfo is None:
pass
else:
issuenotes_id = None
logger.info(
'Successfully retrieved some tags. Lets see what I can figure out.'
)
comicname = issueinfo[0]['series']
logger.fdebug('Series Name: ' + comicname)
issue_number = issueinfo[0]['issue_number']
logger.fdebug('Issue Number: ' + str(issue_number))
issuetitle = issueinfo[0]['title']
logger.fdebug('Issue Title: ' + issuetitle)
issueyear = issueinfo[0]['year']
logger.fdebug('Issue Year: ' + str(issueyear))
try:
issuevolume = issueinfo[0]['volume']
except:
issuevolume = None
# if used by ComicTagger, Notes field will have the IssueID.
issuenotes = issueinfo[0]['notes']
logger.fdebug('Notes: ' + issuenotes)
if issuenotes is not None:
if 'Issue ID' in issuenotes:
st_find = issuenotes.find('Issue ID')
tmp_issuenotes_id = re.sub(
"[^0-9]", " ", issuenotes[st_find:]).strip()
if tmp_issuenotes_id.isdigit():
issuenotes_id = tmp_issuenotes_id
logger.fdebug(
'Successfully retrieved CV IssueID for ' +
comicname + ' #' + str(issue_number) +
' [' + str(issuenotes_id) + ']')
elif 'CVDB' in issuenotes:
st_find = issuenotes.find('CVDB')
tmp_issuenotes_id = re.sub(
"[^0-9]", " ", issuenotes[st_find:]).strip()
if tmp_issuenotes_id.isdigit():
issuenotes_id = tmp_issuenotes_id
logger.fdebug(
'Successfully retrieved CV IssueID for ' +
comicname + ' #' + str(issue_number) +
' [' + str(issuenotes_id) + ']')
else:
logger.fdebug(
'Unable to retrieve IssueID from meta-tagging. If there is other metadata present I will use that.'
)
logger.fdebug("adding " + comicname +
" to the import-queue!")
impid = comicname + '-' + str(issueyear) + '-' + str(
issue_number
) #com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
logger.fdebug("impid: " + str(impid))
#make sure we only add in those issueid's which don't already have a comicid attached via the cvinfo scan above (this is for reverse-lookup of issueids)
if cvinfo_CID is None:
issueid_list.append(issuenotes_id)
if cvscanned_loc == os.path.dirname(comlocation):
cv_cid = cvinfo_CID
logger.info('CVINFO_COMICID attached : ' + str(cv_cid))
else:
cv_cid = None
import_by_comicids.append({
"impid":
impid,
"comicid":
cv_cid,
"watchmatch":
None,
"displayname":
helpers.cleanName(comicname),
"comicname":
comicname, #com_NAME,
"comicyear":
issueyear,
"volume":
issuevolume,
"issueid":
issuenotes_id,
"comfilename":
comfilename,
"comlocation":
comlocation.decode(mylar.SYS_ENCODING)
})
else:
logger.info(
i['ComicLocation'] +
' is not in a metatagged format (cbz). Bypassing reading of the metatags'
)
if issueinfo is None:
#let's clean up the filename for matching purposes
cfilename = re.sub('[\_\#\,\/\:\;\-\!\$\%\&\+\'\?\@]', ' ',
comfilename)
#cfilename = re.sub('\s', '_', str(cfilename))
d_filename = re.sub('[\_\#\,\/\;\!\$\%\&\?\@]', ' ', comfilename)
d_filename = re.sub('[\:\-\+\']', '#', d_filename)
#strip extraspaces
d_filename = re.sub('\s+', ' ', d_filename)
cfilename = re.sub('\s+', ' ', cfilename)
#versioning - remove it
subsplit = cfilename.replace('_', ' ').split()
volno = None
volyr = None
for subit in subsplit:
if subit[0].lower() == 'v':
vfull = 0
if subit[1:].isdigit():
#if in format v1, v2009 etc...
if len(subit) > 3:
# if it's greater than 3 in length, then the format is Vyyyy
vfull = 1 # add on 1 character length to account for extra space
cfilename = re.sub(subit, '', cfilename)
d_filename = re.sub(subit, '', d_filename)
volno = re.sub("[^0-9]", " ", subit)
elif subit.lower()[:3] == 'vol':
#if in format vol.2013 etc
#because the '.' in Vol. gets removed, let's loop thru again after the Vol hit to remove it entirely
logger.fdebug(
'volume indicator detected as version #:' +
str(subit))
cfilename = re.sub(subit, '', cfilename)
cfilename = " ".join(cfilename.split())
d_filename = re.sub(subit, '', d_filename)
d_filename = " ".join(d_filename.split())
volyr = re.sub("[^0-9]", " ", subit).strip()
logger.fdebug('volume year set as : ' + str(volyr))
cm_cn = 0
#we need to track the counter to make sure we are comparing the right array parts
#this takes care of the brackets :)
m = re.findall('[^()]+', d_filename) #cfilename)
lenm = len(m)
logger.fdebug("there are " + str(lenm) + " words.")
cnt = 0
yearmatch = "false"
foundonwatch = "False"
issue = 999999
while (cnt < lenm):
if m[cnt] is None: break
if m[cnt] == ' ':
pass
else:
logger.fdebug(str(cnt) + ". Bracket Word: " + m[cnt])
if cnt == 0:
comic_andiss = m[cnt]
logger.fdebug("Comic: " + comic_andiss)
# if it's not in the standard format this will bork.
# let's try to accomodate (somehow).
# first remove the extension (if any)
extensions = ('cbr', 'cbz')
if comic_andiss.lower().endswith(extensions):
comic_andiss = comic_andiss[:-4]
logger.fdebug("removed extension from filename.")
#now we have to break up the string regardless of formatting.
#let's force the spaces.
comic_andiss = re.sub('_', ' ', comic_andiss)
cs = comic_andiss.split()
cs_len = len(cs)
cn = ''
ydetected = 'no'
idetected = 'no'
decimaldetect = 'no'
for i in reversed(xrange(len(cs))):
#start at the end.
logger.fdebug("word: " + str(cs[i]))
#assume once we find issue - everything prior is the actual title
#idetected = no will ignore everything so it will assume all title
if cs[i][:-2] == '19' or cs[
i][:-2] == '20' and idetected == 'no':
logger.fdebug("year detected: " + str(cs[i]))
ydetected = 'yes'
result_comyear = cs[i]
elif cs[i].isdigit(
) and idetected == 'no' or '.' in cs[i]:
if '.' in cs[i]:
#make sure it's a number on either side of decimal and assume decimal issue.
decst = cs[i].find('.')
dec_st = cs[i][:decst]
dec_en = cs[i][decst + 1:]
logger.fdebug("st: " + str(dec_st))
logger.fdebug("en: " + str(dec_en))
if dec_st.isdigit() and dec_en.isdigit():
logger.fdebug(
"decimal issue detected...adjusting."
)
issue = dec_st + "." + dec_en
logger.fdebug("issue detected: " +
str(issue))
idetected = 'yes'
else:
logger.fdebug(
"false decimal represent. Chunking to extra word."
)
cn = cn + cs[i] + " "
#break
else:
issue = cs[i]
logger.fdebug("issue detected : " +
str(issue))
idetected = 'yes'
elif '\#' in cs[i] or decimaldetect == 'yes':
logger.fdebug("issue detected: " + str(cs[i]))
idetected = 'yes'
else:
cn = cn + cs[i] + " "
if ydetected == 'no':
#assume no year given in filename...
result_comyear = "0000"
logger.fdebug("cm?: " + str(cn))
if issue is not '999999':
comiss = issue
else:
logger.ERROR(
"Invalid Issue number (none present) for " +
comfilename)
break
cnsplit = cn.split()
cname = ''
findcn = 0
while (findcn < len(cnsplit)):
cname = cname + cs[findcn] + " "
findcn += 1
cname = cname[:len(cname) - 1] # drop the end space...
logger.fdebug('assuming name is : ' + cname)
com_NAME = cname
logger.fdebug('com_NAME : ' + com_NAME)
yearmatch = "True"
else:
logger.fdebug('checking ' + m[cnt])
# we're assuming that the year is in brackets (and it should be damnit)
if m[cnt][:-2] == '19' or m[cnt][:-2] == '20':
logger.fdebug('year detected: ' + str(m[cnt]))
ydetected = 'yes'
result_comyear = m[cnt]
elif m[cnt][:3].lower() in datelist:
logger.fdebug(
'possible issue date format given - verifying')
#if the date of the issue is given as (Jan 2010) or (January 2010) let's adjust.
#keeping in mind that ',' and '.' are already stripped from the string
if m[cnt][-4:].isdigit():
ydetected = 'yes'
result_comyear = m[cnt][-4:]
logger.fdebug('Valid Issue year of ' +
str(result_comyear) +
'detected in format of ' +
str(m[cnt]))
cnt += 1
displength = len(cname)
logger.fdebug('cname length : ' + str(displength) + ' --- ' +
str(cname))
logger.fdebug('d_filename is : ' + d_filename)
charcount = d_filename.count('#')
logger.fdebug('charcount is : ' + str(charcount))
if charcount > 0:
logger.fdebug('entering loop')
for i, m in enumerate(re.finditer('\#', d_filename)):
if m.end() <= displength:
logger.fdebug(comfilename[m.start():m.end()])
# find occurance in c_filename, then replace into d_filname so special characters are brought across
newchar = comfilename[m.start():m.end()]
logger.fdebug('newchar:' + str(newchar))
d_filename = d_filename[:m.start()] + str(
newchar) + d_filename[m.end():]
logger.fdebug('d_filename:' + str(d_filename))
dispname = d_filename[:displength]
logger.fdebug('dispname : ' + dispname)
splitit = []
watchcomic_split = []
logger.fdebug("filename comic and issue: " + comic_andiss)
#changed this from '' to ' '
comic_iss_b4 = re.sub('[\-\:\,]', ' ', comic_andiss)
comic_iss = comic_iss_b4.replace('.', ' ')
comic_iss = re.sub('[\s+]', ' ', comic_iss).strip()
logger.fdebug("adjusted comic and issue: " + str(comic_iss))
#remove 'the' from here for proper comparisons.
if ' the ' in comic_iss.lower():
comic_iss = re.sub('\\bthe\\b', '', comic_iss).strip()
splitit = comic_iss.split(None)
logger.fdebug("adjusting from: " + str(comic_iss_b4) + " to: " +
str(comic_iss))
#here we cycle through the Watchlist looking for a match.
while (cm_cn < watchcnt):
#setup the watchlist
comname = ComicName[cm_cn]
comyear = ComicYear[cm_cn]
compub = ComicPublisher[cm_cn]
comtotal = ComicTotal[cm_cn]
comicid = ComicID[cm_cn]
watch_location = ComicLocation[cm_cn]
# there shouldn't be an issue in the comic now, so let's just assume it's all gravy.
splitst = len(splitit)
watchcomic_split = helpers.cleanName(comname)
watchcomic_split = re.sub('[\-\:\,\.]', ' ',
watchcomic_split).split(None)
logger.fdebug(
str(splitit) + " file series word count: " + str(splitst))
logger.fdebug(
str(watchcomic_split) + " watchlist word count: " +
str(len(watchcomic_split)))
if (splitst) != len(watchcomic_split):
logger.fdebug("incorrect comic lengths...not a match")
# if str(splitit[0]).lower() == "the":
# logger.fdebug("THE word detected...attempting to adjust pattern matching")
# splitit[0] = splitit[4:]
else:
logger.fdebug("length match..proceeding")
n = 0
scount = 0
logger.fdebug("search-length: " + str(splitst))
logger.fdebug("Watchlist-length: " +
str(len(watchcomic_split)))
while (n <= (splitst) - 1):
logger.fdebug("splitit: " + str(splitit[n]))
if n < (splitst) and n < len(watchcomic_split):
logger.fdebug(
str(n) + " Comparing: " +
str(watchcomic_split[n]) + " .to. " +
str(splitit[n]))
if '+' in watchcomic_split[n]:
watchcomic_split[n] = re.sub(
'+', '', str(watchcomic_split[n]))
if str(watchcomic_split[n].lower()) in str(
splitit[n].lower()
) and len(watchcomic_split[n]) >= len(splitit[n]):
logger.fdebug("word matched on : " +
str(splitit[n]))
scount += 1
#elif ':' in splitit[n] or '-' in splitit[n]:
# splitrep = splitit[n].replace('-', '')
# logger.fdebug("non-character keyword...skipped on " + splitit[n])
elif str(splitit[n]).lower().startswith('v'):
logger.fdebug("possible versioning..checking")
#we hit a versioning # - account for it
if splitit[n][1:].isdigit():
comicversion = str(splitit[n])
logger.fdebug("version found: " +
str(comicversion))
else:
logger.fdebug("Comic / Issue section")
if splitit[n].isdigit():
logger.fdebug("issue detected")
else:
logger.fdebug("non-match for: " +
str(splitit[n]))
pass
n += 1
#set the match threshold to 80% (for now)
# if it's less than 80% consider it a non-match and discard.
#splitit has to splitit-1 because last position is issue.
wordcnt = int(scount)
logger.fdebug("scount:" + str(wordcnt))
totalcnt = int(splitst)
logger.fdebug("splitit-len:" + str(totalcnt))
spercent = (wordcnt / totalcnt) * 100
logger.fdebug("we got " + str(spercent) + " percent.")
if int(spercent) >= 80:
logger.fdebug("it's a go captain... - we matched " +
str(spercent) + "%!")
logger.fdebug("this should be a match!")
logger.fdebug("issue we found for is : " + str(comiss))
#set the year to the series we just found ;)
result_comyear = comyear
#issue comparison now as well
logger.info(u"Found " + comname + " (" + str(comyear) +
") issue: " + str(comiss))
watchmatch = str(comicid)
dispname = DisplayName[cm_cn]
foundonwatch = "True"
break
elif int(spercent) < 80:
logger.fdebug("failure - we only got " +
str(spercent) + "% right!")
cm_cn += 1
if foundonwatch == "False":
watchmatch = None
#---if it's not a match - send it to the importer.
n = 0
if volyr is None:
if result_comyear is None:
result_comyear = '0000' #no year in filename basically.
else:
if result_comyear is None:
result_comyear = volyr
if volno is None:
if volyr is None:
vol_label = None
else:
vol_label = volyr
else:
vol_label = volno
logger.fdebug("adding " + com_NAME + " to the import-queue!")
impid = dispname + '-' + str(result_comyear) + '-' + str(
comiss
) #com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
logger.fdebug("impid: " + str(impid))
if cvscanned_loc == os.path.dirname(comlocation):
cv_cid = cvinfo_CID
logger.info('CVINFO_COMICID attached : ' + str(cv_cid))
else:
cv_cid = None
import_by_comicids.append({
"impid":
impid,
"comicid":
cv_cid,
"issueid":
None,
"watchmatch":
watchmatch,
"displayname":
dispname,
"comicname":
dispname, #com_NAME,
"comicyear":
result_comyear,
"volume":
vol_label,
"comfilename":
comfilename,
"comlocation":
comlocation.decode(mylar.SYS_ENCODING)
})
#logger.fdebug('import_by_ids: ' + str(import_by_comicids))
#reverse lookup all of the gathered IssueID's in order to get the related ComicID
vals = mylar.cv.getComic(None, 'import', comicidlist=issueid_list)
logger.fdebug('vals returned:' + str(vals))
if len(watch_kchoice) > 0:
watchchoice['watchlist'] = watch_kchoice
#logger.fdebug("watchchoice: " + str(watchchoice))
logger.info("I have found " + str(watchfound) + " out of " +
str(comiccnt) +
" comics for series that are being watched.")
wat = 0
comicids = []
if watchfound > 0:
if mylar.IMP_MOVE:
logger.info(
'You checked off Move Files...so that\'s what I am going to do'
)
#check to see if Move Files is enabled.
#if not being moved, set the archive bit.
logger.fdebug('Moving files into appropriate directory')
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comlocation = watch_the_list['ComicLocation']
watch_comicid = watch_the_list['ComicID']
watch_comicname = watch_the_list['ComicName']
watch_comicyear = watch_the_list['ComicYear']
watch_comiciss = watch_the_list['ComicIssue']
logger.fdebug('ComicLocation: ' + watch_comlocation)
orig_comlocation = watch_the_list['OriginalLocation']
orig_filename = watch_the_list['OriginalFilename']
logger.fdebug('Orig. Location: ' + orig_comlocation)
logger.fdebug('Orig. Filename: ' + orig_filename)
#before moving check to see if Rename to Mylar structure is enabled.
if mylar.IMP_RENAME:
logger.fdebug(
'Renaming files according to configuration details : '
+ str(mylar.FILE_FORMAT))
renameit = helpers.rename_param(
watch_comicid, watch_comicname, watch_comicyear,
watch_comiciss)
nfilename = renameit['nfilename']
dst_path = os.path.join(watch_comlocation, nfilename)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
else:
logger.fdebug(
'Renaming files not enabled, keeping original filename(s)'
)
dst_path = os.path.join(watch_comlocation,
orig_filename)
#os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
#src = os.path.join(, str(nfilename + ext))
logger.fdebug('I am going to move ' + orig_comlocation +
' to ' + dst_path)
try:
shutil.move(orig_comlocation, dst_path)
except (OSError, IOError):
logger.info(
"Failed to move directory - check directories and manually re-run."
)
wat += 1
else:
# if move files isn't enabled, let's set all found comics to Archive status :)
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comicid = watch_the_list['ComicID']
watch_issue = watch_the_list['ComicIssue']
logger.fdebug('ComicID: ' + str(watch_comicid))
logger.fdebug('Issue#: ' + str(watch_issue))
issuechk = myDB.selectone(
"SELECT * from issues where ComicID=? AND INT_IssueNumber=?",
[watch_comicid, watch_issue]).fetchone()
if issuechk is None:
logger.fdebug('No matching issues for this comic#')
else:
logger.fdebug('...Existing status: ' +
str(issuechk['Status']))
control = {"IssueID": issuechk['IssueID']}
values = {"Status": "Archived"}
logger.fdebug('...changing status of ' +
str(issuechk['Issue_Number']) +
' to Archived ')
myDB.upsert("issues", values, control)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
wat += 1
if comicids is None: pass
else:
c_upd = len(comicids)
c = 0
while (c < c_upd):
logger.fdebug('Rescanning.. ' + str(c))
updater.forceRescan(c)
if not len(import_by_comicids):
return "Completed"
if len(import_by_comicids) > 0:
import_comicids['comic_info'] = import_by_comicids
if vals:
import_comicids['issueid_info'] = vals
else:
import_comicids['issueid_info'] = None
logger.fdebug('import comicids: ' + str(import_by_comicids))
return import_comicids, len(import_by_comicids)
def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None):
if cron and not mylar.LIBRARYSCAN:
return
if not dir:
dir = mylar.COMIC_DIR
# If we're appending a dir, it's coming from the post processor which is
# already bytestring
if not append:
dir = dir.encode(mylar.SYS_ENCODING)
if not os.path.isdir(dir):
logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(mylar.SYS_ENCODING, 'replace'))
return
logger.info('Scanning comic directory: %s' % dir.decode(mylar.SYS_ENCODING, 'replace'))
basedir = dir
comic_list = []
comiccnt = 0
extensions = ('cbr','cbz')
for r,d,f in os.walk(dir):
#for directory in d[:]:
# if directory.startswith("."):
# d.remove(directory)
for files in f:
if any(files.lower().endswith('.' + x.lower()) for x in extensions):
comic = files
comicpath = os.path.join(r, files)
comicsize = os.path.getsize(comicpath)
print "Comic: " + comic
print "Comic Path: " + comicpath
print "Comic Size: " + str(comicsize)
# We need the unicode path to use for logging, inserting into database
unicode_comic_path = comicpath.decode(mylar.SYS_ENCODING, 'replace')
comiccnt+=1
comic_dict = { 'ComicFilename': comic,
'ComicLocation': comicpath,
'ComicSize': comicsize,
'Unicode_ComicLocation': unicode_comic_path }
comic_list.append(comic_dict)
logger.info("I've found a total of " + str(comiccnt) + " comics....analyzing now")
logger.info("comiclist: " + str(comic_list))
myDB = db.DBConnection()
#let's load in the watchlist to see if we have any matches.
logger.info("loading in the watchlist to see if a series is being watched already...")
watchlist = myDB.action("SELECT * from comics")
ComicName = []
ComicYear = []
ComicPublisher = []
ComicTotal = []
ComicID = []
ComicLocation = []
AltName = []
watchcnt = 0
watch_kchoice = []
watchchoice = {}
import_by_comicids = []
import_comicids = {}
for watch in watchlist:
# let's clean up the name, just in case for comparison purposes...
watchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ', watch['ComicName']).encode('utf-8').strip()
#watchcomic = re.sub('\s+', ' ', str(watchcomic)).strip()
if ' the ' in watchcomic.lower():
#drop the 'the' from the watchcomic title for proper comparisons.
watchcomic = watchcomic[-4:]
alt_chk = "no" # alt-checker flag (default to no)
# account for alternate names as well
if watch['AlternateSearch'] is not None and watch['AlternateSearch'] is not 'None':
altcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ', watch['AlternateSearch']).encode('utf-8').strip()
#altcomic = re.sub('\s+', ' ', str(altcomic)).strip()
AltName.append(altcomic)
alt_chk = "yes" # alt-checker flag
ComicName.append(watchcomic)
ComicYear.append(watch['ComicYear'])
ComicPublisher.append(watch['ComicPublisher'])
ComicTotal.append(watch['Total'])
ComicID.append(watch['ComicID'])
ComicLocation.append(watch['ComicLocation'])
watchcnt+=1
logger.info("Successfully loaded " + str(watchcnt) + " series from your watchlist.")
ripperlist=['digital-',
'empire',
'dcp']
watchfound = 0
for i in comic_list:
print i['ComicFilename']
comfilename = i['ComicFilename']
comlocation = i['ComicLocation']
#let's clean up the filename for matching purposes
cfilename = re.sub('[\_\#\,\/\:\;\-\!\$\%\&\+\'\?\@]', ' ', comfilename)
#cfilename = re.sub('\s', '_', str(cfilename))
cm_cn = 0
#we need to track the counter to make sure we are comparing the right array parts
#this takes care of the brackets :)
m = re.findall('[^()]+', cfilename)
lenm = len(m)
print ("there are " + str(lenm) + " words.")
cnt = 0
yearmatch = "false"
foundonwatch = "False"
issue = 999999
while (cnt < lenm):
if m[cnt] is None: break
if m[cnt] == ' ':
pass
else:
logger.fdebug(str(cnt) + ". Bracket Word: " + m[cnt])
if cnt == 0:
comic_andiss = m[cnt]
logger.fdebug("Comic: " + comic_andiss)
# if it's not in the standard format this will bork.
# let's try to accomodate (somehow).
# first remove the extension (if any)
extensions = ('cbr', 'cbz')
if comic_andiss.lower().endswith(extensions):
comic_andiss = comic_andiss[:-4]
print ("removed extension from filename.")
#now we have to break up the string regardless of formatting.
#let's force the spaces.
comic_andiss = re.sub('_', ' ', comic_andiss)
cs = comic_andiss.split()
cs_len = len(cs)
cn = ''
ydetected = 'no'
idetected = 'no'
decimaldetect = 'no'
for i in reversed(xrange(len(cs))):
#start at the end.
print ("word: " + str(cs[i]))
#assume once we find issue - everything prior is the actual title
#idetected = no will ignore everything so it will assume all title
if cs[i][:-2] == '19' or cs[i][:-2] == '20' and idetected == 'no':
print ("year detected: " + str(cs[i]))
ydetected = 'yes'
result_comyear = cs[i]
elif cs[i].isdigit() and idetected == 'no' or '.' in cs[i]:
issue = cs[i]
print ("issue detected : " + str(issue))
idetected = 'yes'
if '.' in cs[i]:
#make sure it's a number on either side of decimal and assume decimal issue.
decst = cs[i].find('.')
dec_st = cs[i][:decst]
dec_en = cs[i][decst+1:]
print ("st: " + str(dec_st))
print ("en: " + str(dec_en))
if dec_st.isdigit() and dec_en.isdigit():
print ("decimal issue detected...adjusting.")
issue = dec_st + "." + dec_en
print ("issue detected: " + str(issue))
idetected = 'yes'
else:
print ("false decimal represent. Chunking to extra word.")
cn = cn + cs[i] + " "
break
elif '\#' in cs[i] or decimaldetect == 'yes':
print ("issue detected: " + str(cs[i]))
idetected = 'yes'
else: cn = cn + cs[i] + " "
if ydetected == 'no':
#assume no year given in filename...
result_comyear = "0000"
print ("cm?: " + str(cn))
if issue is not '999999':
comiss = issue
else:
logger.ERROR("Invalid Issue number (none present) for " + comfilename)
break
cnsplit = cn.split()
cname = ''
findcn = 0
while (findcn < len(cnsplit)):
cname = cname + cs[findcn] + " "
findcn+=1
cname = cname[:len(cname)-1] # drop the end space...
print ("assuming name is : " + cname)
com_NAME = cname
print ("com_NAME : " + com_NAME)
yearmatch = "True"
else:
# we're assuming that the year is in brackets (and it should be damnit)
if m[cnt][:-2] == '19' or m[cnt][:-2] == '20':
print ("year detected: " + str(m[cnt]))
ydetected = 'yes'
result_comyear = m[cnt]
cnt+=1
splitit = []
watchcomic_split = []
logger.fdebug("filename comic and issue: " + cfilename)
#changed this from '' to ' '
comic_iss_b4 = re.sub('[\-\:\,]', ' ', com_NAME)
comic_iss = comic_iss_b4.replace('.',' ')
logger.fdebug("adjusted comic and issue: " + str(comic_iss))
#remove 'the' from here for proper comparisons.
if ' the ' in comic_iss.lower():
comic_iss = comic_iss[-4:]
splitit = comic_iss.split(None)
logger.fdebug("adjusting from: " + str(comic_iss_b4) + " to: " + str(comic_iss))
#bmm = re.findall('v\d', comic_iss)
#if len(bmm) > 0: splitst = len(splitit) - 2
#else: splitst = len(splitit) - 1
#-----
#here we cycle through the Watchlist looking for a match.
while (cm_cn < watchcnt):
#setup the watchlist
comname = ComicName[cm_cn]
print ("watch_comic:" + comname)
comyear = ComicYear[cm_cn]
compub = ComicPublisher[cm_cn]
comtotal = ComicTotal[cm_cn]
comicid = ComicID[cm_cn]
watch_location = ComicLocation[cm_cn]
# if splitit[(len(splitit)-1)].isdigit():
# #compares - if the last digit and second last digit are #'s seperated by spaces assume decimal
# comic_iss = splitit[(len(splitit)-1)]
# splitst = len(splitit) - 1
# if splitit[(len(splitit)-2)].isdigit():
# # for series that have a digit at the end, it screws up the logistics.
# i = 1
# chg_comic = splitit[0]
# while (i < (len(splitit)-1)):
# chg_comic = chg_comic + " " + splitit[i]
# i+=1
# logger.fdebug("chg_comic:" + str(chg_comic))
# if chg_comic.upper() == comname.upper():
# logger.fdebug("series contains numerics...adjusting..")
# else:
# changeup = "." + splitit[(len(splitit)-1)]
# logger.fdebug("changeup to decimal: " + str(changeup))
# comic_iss = splitit[(len(splitit)-2)] + "." + comic_iss
# splitst = len(splitit) - 2
# else:
# if the nzb name doesn't follow the series-issue-year format even closely..ignore nzb
# logger.fdebug("invalid naming format of filename detected - cannot properly determine issue")
# continue
# make sure that things like - in watchcomic are accounted for when comparing to nzb.
# there shouldn't be an issue in the comic now, so let's just assume it's all gravy.
splitst = len(splitit)
watchcomic_split = helpers.cleanName(comname)
watchcomic_split = re.sub('[\-\:\,\.]', ' ', watchcomic_split).split(None)
logger.fdebug(str(splitit) + " file series word count: " + str(splitst))
logger.fdebug(str(watchcomic_split) + " watchlist word count: " + str(len(watchcomic_split)))
if (splitst) != len(watchcomic_split):
logger.fdebug("incorrect comic lengths...not a match")
# if str(splitit[0]).lower() == "the":
# logger.fdebug("THE word detected...attempting to adjust pattern matching")
# splitit[0] = splitit[4:]
else:
logger.fdebug("length match..proceeding")
n = 0
scount = 0
logger.fdebug("search-length: " + str(splitst))
logger.fdebug("Watchlist-length: " + str(len(watchcomic_split)))
while ( n <= (splitst)-1 ):
logger.fdebug("splitit: " + str(splitit[n]))
if n < (splitst) and n < len(watchcomic_split):
logger.fdebug(str(n) + " Comparing: " + str(watchcomic_split[n]) + " .to. " + str(splitit[n]))
if '+' in watchcomic_split[n]:
watchcomic_split[n] = re.sub('+', '', str(watchcomic_split[n]))
if str(watchcomic_split[n].lower()) in str(splitit[n].lower()) and len(watchcomic_split[n]) >= len(splitit[n]):
logger.fdebug("word matched on : " + str(splitit[n]))
scount+=1
#elif ':' in splitit[n] or '-' in splitit[n]:
# splitrep = splitit[n].replace('-', '')
# print ("non-character keyword...skipped on " + splitit[n])
elif str(splitit[n]).lower().startswith('v'):
logger.fdebug("possible versioning..checking")
#we hit a versioning # - account for it
if splitit[n][1:].isdigit():
comicversion = str(splitit[n])
logger.fdebug("version found: " + str(comicversion))
else:
logger.fdebug("Comic / Issue section")
if splitit[n].isdigit():
logger.fdebug("issue detected")
#comiss = splitit[n]
# comicNAMER = n - 1
# com_NAME = splitit[0]
# cmnam = 1
# while (cmnam <= comicNAMER):
# com_NAME = str(com_NAME) + " " + str(splitit[cmnam])
# cmnam+=1
# logger.fdebug("comic: " + str(com_NAME))
else:
logger.fdebug("non-match for: "+ str(splitit[n]))
pass
n+=1
#set the match threshold to 80% (for now)
# if it's less than 80% consider it a non-match and discard.
#splitit has to splitit-1 because last position is issue.
wordcnt = int(scount)
logger.fdebug("scount:" + str(wordcnt))
totalcnt = int(splitst)
logger.fdebug("splitit-len:" + str(totalcnt))
spercent = (wordcnt/totalcnt) * 100
logger.fdebug("we got " + str(spercent) + " percent.")
if int(spercent) >= 80:
logger.fdebug("it's a go captain... - we matched " + str(spercent) + "%!")
logger.fdebug("this should be a match!")
# if '.' in comic_iss:
# comisschk_find = comic_iss.find('.')
# comisschk_b4dec = comic_iss[:comisschk_find]
# comisschk_decval = comic_iss[comisschk_find+1:]
# logger.fdebug("Found IssueNumber: " + str(comic_iss))
# logger.fdebug("..before decimal: " + str(comisschk_b4dec))
# logger.fdebug("...after decimal: " + str(comisschk_decval))
# #--let's make sure we don't wipe out decimal issues ;)
# if int(comisschk_decval) == 0:
# ciss = comisschk_b4dec
# cintdec = int(comisschk_decval)
# else:
# if len(comisschk_decval) == 1:
# ciss = comisschk_b4dec + "." + comisschk_decval
# cintdec = int(comisschk_decval) * 10
# else:
# ciss = comisschk_b4dec + "." + comisschk_decval.rstrip('0')
# cintdec = int(comisschk_decval.rstrip('0')) * 10
# comintIss = (int(comisschk_b4dec) * 1000) + cintdec
# else:
# comintIss = int(comic_iss) * 1000
logger.fdebug("issue we found for is : " + str(comiss))
#set the year to the series we just found ;)
result_comyear = comyear
#issue comparison now as well
logger.info(u"Found " + comname + " (" + str(comyear) + ") issue: " + str(comiss))
# watchfound+=1
watchmatch = str(comicid)
# watch_kchoice.append({
# "ComicID": str(comicid),
# "ComicName": str(comname),
# "ComicYear": str(comyear),
# "ComicIssue": str(int(comic_iss)),
# "ComicLocation": str(watch_location),
# "OriginalLocation" : str(comlocation),
# "OriginalFilename" : str(comfilename)
# })
foundonwatch = "True"
break
elif int(spercent) < 80:
logger.fdebug("failure - we only got " + str(spercent) + "% right!")
cm_cn+=1
if foundonwatch == "False":
watchmatch = None
#---if it's not a match - send it to the importer.
n = 0
# print ("comic_andiss : " + str(comic_andiss))
# csplit = comic_andiss.split(None)
# while ( n <= (len(csplit)-1) ):
# print ("csplit:" + str(csplit[n]))
# if csplit[n].isdigit():
# logger.fdebug("issue detected")
# comiss = splitit[n]
# logger.fdebug("issue # : " + str(comiss))
# comicNAMER = n - 1
# com_NAME = csplit[0]
# cmnam = 1
# while (cmnam <= comicNAMER):
# com_NAME = str(com_NAME) + " " + str(csplit[cmnam])
# cmnam+=1
# logger.fdebug("comic: " + str(com_NAME))
# n+=1
if result_comyear is None: result_comyear = '0000' #no year in filename basically.
print ("adding " + com_NAME + " to the import-queue!")
impid = com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
print ("impid: " + str(impid))
import_by_comicids.append({
"impid": impid,
"watchmatch": watchmatch,
"comicname" : com_NAME,
"comicyear" : result_comyear,
"comfilename" : comfilename,
"comlocation" : comlocation.decode(mylar.SYS_ENCODING)
})
if len(watch_kchoice) > 0:
watchchoice['watchlist'] = watch_kchoice
print ("watchchoice: " + str(watchchoice))
logger.info("I have found " + str(watchfound) + " out of " + str(comiccnt) + " comics for series that are being watched.")
wat = 0
comicids = []
if watchfound > 0:
if mylar.IMP_MOVE:
logger.info("You checked off Move Files...so that's what I'm going to do")
#check to see if Move Files is enabled.
#if not being moved, set the archive bit.
print("Moving files into appropriate directory")
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comlocation = watch_the_list['ComicLocation']
watch_comicid = watch_the_list['ComicID']
watch_comicname = watch_the_list['ComicName']
watch_comicyear = watch_the_list['ComicYear']
watch_comiciss = watch_the_list['ComicIssue']
print ("ComicLocation: " + str(watch_comlocation))
orig_comlocation = watch_the_list['OriginalLocation']
orig_filename = watch_the_list['OriginalFilename']
print ("Orig. Location: " + str(orig_comlocation))
print ("Orig. Filename: " + str(orig_filename))
#before moving check to see if Rename to Mylar structure is enabled.
if mylar.IMP_RENAME:
print("Renaming files according to configuration details : " + str(mylar.FILE_FORMAT))
renameit = helpers.rename_param(watch_comicid, watch_comicname, watch_comicyear, watch_comiciss)
nfilename = renameit['nfilename']
dst_path = os.path.join(watch_comlocation,nfilename)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
else:
print("Renaming files not enabled, keeping original filename(s)")
dst_path = os.path.join(watch_comlocation,orig_filename)
#os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
#src = os.path.join(, str(nfilename + ext))
print ("I'm going to move " + str(orig_comlocation) + " to .." + str(dst_path))
try:
shutil.move(orig_comlocation, dst_path)
except (OSError, IOError):
logger.info("Failed to move directory - check directories and manually re-run.")
wat+=1
else:
# if move files isn't enabled, let's set all found comics to Archive status :)
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comicid = watch_the_list['ComicID']
watch_issue = watch_the_list['ComicIssue']
print ("ComicID: " + str(watch_comicid))
print ("Issue#: " + str(watch_issue))
issuechk = myDB.action("SELECT * from issues where ComicID=? AND INT_IssueNumber=?", [watch_comicid, watch_issue]).fetchone()
if issuechk is None:
print ("no matching issues for this comic#")
else:
print("...Existing status: " + str(issuechk['Status']))
control = {"IssueID": issuechk['IssueID']}
values = { "Status": "Archived"}
print ("...changing status of " + str(issuechk['Issue_Number']) + " to Archived ")
myDB.upsert("issues", values, control)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
wat+=1
if comicids is None: pass
else:
c_upd = len(comicids)
c = 0
while (c < c_upd ):
print ("Rescanning.. " + str(c))
updater.forceRescan(c)
if not len(import_by_comicids):
return "Completed"
if len(import_by_comicids) > 0:
import_comicids['comic_info'] = import_by_comicids
print ("import comicids: " + str(import_by_comicids))
return import_comicids, len(import_by_comicids)
def GCDimport(gcomicid, pullupd=None, imported=None, ogcname=None):
# this is for importing via GCD only and not using CV.
# used when volume spanning is discovered for a Comic (and can't be added using CV).
# Issue Counts are wrong (and can't be added).
# because Comicvine ComicID and GCD ComicID could be identical at some random point, let's distinguish.
# CV = comicid, GCD = gcomicid :) (ie. CV=2740, GCD=G3719)
gcdcomicid = gcomicid
myDB = db.DBConnection()
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ComicID": gcdcomicid}
comic = myDB.action(
'SELECT ComicName, ComicYear, Total, ComicPublished, ComicImage, ComicLocation, ComicPublisher FROM comics WHERE ComicID=?',
[gcomicid]).fetchone()
ComicName = comic[0]
ComicYear = comic[1]
ComicIssues = comic[2]
ComicPublished = comic[3]
comlocation = comic[5]
ComicPublisher = comic[6]
#ComicImage = comic[4]
#print ("Comic:" + str(ComicName))
newValueDict = {"Status": "Loading"}
myDB.upsert("comics", newValueDict, controlValueDict)
# we need to lookup the info for the requested ComicID in full now
#comic = cv.getComic(comicid,'comic')
if not comic:
logger.warn("Error fetching comic. ID for : " + gcdcomicid)
if dbcomic is None:
newValueDict = {
"ComicName":
"Fetch failed, try refreshing. (%s)" % (gcdcomicid),
"Status": "Active"
}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("comics", newValueDict, controlValueDict)
return
#run the re-sortorder here in order to properly display the page
if pullupd is None:
helpers.ComicSort(comicorder=mylar.COMICSORT, imported=gcomicid)
if ComicName.startswith('The '):
sortname = ComicName[4:]
else:
sortname = ComicName
logger.info(u"Now adding/updating: " + ComicName)
#--Now that we know ComicName, let's try some scraping
#--Start
# gcd will return issue details (most importantly publishing date)
comicid = gcomicid[1:]
resultURL = "/series/" + str(comicid) + "/"
gcdinfo = parseit.GCDdetails(comseries=None,
resultURL=resultURL,
vari_loop=0,
ComicID=gcdcomicid,
TotalIssues=ComicIssues,
issvariation=None,
resultPublished=None)
if gcdinfo == "No Match":
logger.warn("No matching result found for " + ComicName + " (" +
ComicYear + ")")
updater.no_searchresults(gcomicid)
nomatch = "true"
return nomatch
logger.info(u"Sucessfully retrieved details for " + ComicName)
# print ("Series Published" + parseit.resultPublished)
#--End
ComicImage = gcdinfo['ComicImage']
#comic book location on machine
# setup default location here
if comlocation is None:
# let's remove the non-standard characters here.
u_comicnm = ComicName
u_comicname = u_comicnm.encode('ascii', 'ignore').strip()
if ':' in u_comicname or '/' in u_comicname or ',' in u_comicname:
comicdir = u_comicname
if ':' in comicdir:
comicdir = comicdir.replace(':', '')
if '/' in comicdir:
comicdir = comicdir.replace('/', '-')
if ',' in comicdir:
comicdir = comicdir.replace(',', '')
else:
comicdir = u_comicname
series = comicdir
publisher = ComicPublisher
year = ComicYear
#do work to generate folder path
values = {
'$Series': series,
'$Publisher': publisher,
'$Year': year,
'$series': series.lower(),
'$publisher': publisher.lower(),
'$Volume': year
}
if mylar.FOLDER_FORMAT == '':
comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + comic[
'ComicYear'] + ")"
else:
comlocation = mylar.DESTINATION_DIR + "/" + helpers.replace_all(
mylar.FOLDER_FORMAT, values)
#comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + ComicYear + ")"
if mylar.DESTINATION_DIR == "":
logger.error(
u"There is no general directory specified - please specify in Config/Post-Processing."
)
return
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
comlocation = comlocation.replace(' ', mylar.REPLACE_CHAR)
#if it doesn't exist - create it (otherwise will bugger up later on)
if os.path.isdir(str(comlocation)):
logger.info(u"Directory (" + str(comlocation) +
") already exists! Continuing...")
else:
#print ("Directory doesn't exist!")
#try:
# os.makedirs(str(comlocation))
# logger.info(u"Directory successfully created at: " + str(comlocation))
#except OSError:
# logger.error(u"Could not create comicdir : " + str(comlocation))
filechecker.validateAndCreateDirectory(comlocation, True)
comicIssues = gcdinfo['totalissues']
#let's download the image...
if os.path.exists(mylar.CACHE_DIR): pass
else:
#let's make the dir.
try:
os.makedirs(str(mylar.CACHE_DIR))
logger.info(u"Cache Directory successfully created at: " +
str(mylar.CACHE_DIR))
except OSError:
logger.error(u"Could not create cache dir : " +
str(mylar.CACHE_DIR))
coverfile = os.path.join(mylar.CACHE_DIR, str(gcomicid) + ".jpg")
#try:
urllib.urlretrieve(str(ComicImage), str(coverfile))
try:
with open(str(coverfile)) as f:
ComicImage = os.path.join('cache', str(gcomicid) + ".jpg")
#this is for Firefox when outside the LAN...it works, but I don't know how to implement it
#without breaking the normal flow for inside the LAN (above)
#ComicImage = "http://" + str(mylar.HTTP_HOST) + ":" + str(mylar.HTTP_PORT) + "/cache/" + str(comi$
logger.info(u"Sucessfully retrieved cover for " + ComicName)
#if the comic cover local is checked, save a cover.jpg to the series folder.
if mylar.COMIC_COVER_LOCAL:
comiclocal = os.path.join(str(comlocation) + "/cover.jpg")
shutil.copy(ComicImage, comiclocal)
except IOError as e:
logger.error(u"Unable to save cover locally at this time.")
#if comic['ComicVersion'].isdigit():
# comicVol = "v" + comic['ComicVersion']
#else:
# comicVol = None
controlValueDict = {"ComicID": gcomicid}
newValueDict = {
"ComicName": ComicName,
"ComicSortName": sortname,
"ComicYear": ComicYear,
"Total": comicIssues,
"ComicLocation": comlocation,
#"ComicVersion": comicVol,
"ComicImage": ComicImage,
#"ComicPublisher": comic['ComicPublisher'],
#"ComicPublished": comicPublished,
"DateAdded": helpers.today(),
"Status": "Loading"
}
myDB.upsert("comics", newValueDict, controlValueDict)
#comicsort here...
#run the re-sortorder here in order to properly display the page
if pullupd is None:
helpers.ComicSort(sequence='update')
logger.info(u"Sucessfully retrieved issue details for " + ComicName)
n = 0
iscnt = int(comicIssues)
issnum = []
issname = []
issdate = []
int_issnum = []
#let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :)
latestiss = "0"
latestdate = "0000-00-00"
#print ("total issues:" + str(iscnt))
#---removed NEW code here---
logger.info(u"Now adding/updating issues for " + ComicName)
bb = 0
while (bb <= iscnt):
#---NEW.code
try:
gcdval = gcdinfo['gcdchoice'][bb]
#print ("gcdval: " + str(gcdval))
except IndexError:
#account for gcd variation here
if gcdinfo['gcdvariation'] == 'gcd':
#print ("gcd-variation accounted for.")
issdate = '0000-00-00'
int_issnum = int(issis / 1000)
break
if 'nn' in str(gcdval['GCDIssue']):
#no number detected - GN, TP or the like
logger.warn(
u"Non Series detected (Graphic Novel, etc) - cannot proceed at this time."
)
updater.no_searchresults(comicid)
return
elif '.' in str(gcdval['GCDIssue']):
issst = str(gcdval['GCDIssue']).find('.')
issb4dec = str(gcdval['GCDIssue'])[:issst]
#if the length of decimal is only 1 digit, assume it's a tenth
decis = str(gcdval['GCDIssue'])[issst + 1:]
if len(decis) == 1:
decisval = int(decis) * 10
issaftdec = str(decisval)
if len(decis) == 2:
decisval = int(decis)
issaftdec = str(decisval)
if int(issaftdec) == 0: issaftdec = "00"
gcd_issue = issb4dec + "." + issaftdec
gcdis = (int(issb4dec) * 1000) + decisval
else:
gcdis = int(str(gcdval['GCDIssue'])) * 1000
gcd_issue = str(gcdval['GCDIssue'])
#get the latest issue / date using the date.
int_issnum = int(gcdis / 1000)
issdate = str(gcdval['GCDDate'])
issid = "G" + str(gcdval['IssueID'])
if gcdval['GCDDate'] > latestdate:
latestiss = str(gcd_issue)
latestdate = str(gcdval['GCDDate'])
#print("(" + str(bb) + ") IssueID: " + str(issid) + " IssueNo: " + str(gcd_issue) + " Date" + str(issdate) )
#---END.NEW.
# check if the issue already exists
iss_exists = myDB.action('SELECT * from issues WHERE IssueID=?',
[issid]).fetchone()
# Only change the status & add DateAdded if the issue is not already in the database
if iss_exists is None:
newValueDict['DateAdded'] = helpers.today()
#adjust for inconsistencies in GCD date format - some dates have ? which borks up things.
if "?" in str(issdate):
issdate = "0000-00-00"
controlValueDict = {"IssueID": issid}
newValueDict = {
"ComicID": gcomicid,
"ComicName": ComicName,
"Issue_Number": gcd_issue,
"IssueDate": issdate,
"Int_IssueNumber": int_issnum
}
#print ("issueid:" + str(controlValueDict))
#print ("values:" + str(newValueDict))
if mylar.AUTOWANT_ALL:
newValueDict['Status'] = "Wanted"
elif issdate > helpers.today() and mylar.AUTOWANT_UPCOMING:
newValueDict['Status'] = "Wanted"
else:
newValueDict['Status'] = "Skipped"
if iss_exists:
#print ("Existing status : " + str(iss_exists['Status']))
newValueDict['Status'] = iss_exists['Status']
myDB.upsert("issues", newValueDict, controlValueDict)
bb += 1
# logger.debug(u"Updating comic cache for " + ComicName)
# cache.getThumb(ComicID=issue['issueid'])
# logger.debug(u"Updating cache for: " + ComicName)
# cache.getThumb(ComicIDcomicid)
controlValueStat = {"ComicID": gcomicid}
newValueStat = {
"Status": "Active",
"LatestIssue": latestiss,
"LatestDate": latestdate,
"LastUpdated": helpers.now()
}
myDB.upsert("comics", newValueStat, controlValueStat)
if mylar.CVINFO:
if not os.path.exists(comlocation + "/cvinfo"):
with open(comlocation + "/cvinfo", "w") as text_file:
text_file.write("http://www.comicvine.com/volume/49-" +
str(comicid))
logger.info(u"Updating complete for: " + ComicName)
#move the files...if imported is not empty (meaning it's not from the mass importer.)
if imported is None or imported == 'None':
pass
else:
if mylar.IMP_MOVE:
logger.info("Mass import - Move files")
moveit.movefiles(gcomicid, comlocation, ogcname)
else:
logger.info(
"Mass import - Moving not Enabled. Setting Archived Status for import."
)
moveit.archivefiles(gcomicid, ogcname)
#check for existing files...
updater.forceRescan(gcomicid)
if pullupd is None:
# lets' check the pullist for anyting at this time as well since we're here.
if mylar.AUTOWANT_UPCOMING and 'Present' in ComicPublished:
logger.info(u"Checking this week's pullist for new issues of " +
ComicName)
updater.newpullcheck(comic['ComicName'], gcomicid)
#here we grab issues that have been marked as wanted above...
results = myDB.select(
"SELECT * FROM issues where ComicID=? AND Status='Wanted'",
[gcomicid])
if results:
logger.info(u"Attempting to grab wanted issues for : " + ComicName)
for result in results:
foundNZB = "none"
if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL
or mylar.NEWZNAB or mylar.NZBX) and (mylar.SAB_HOST):
foundNZB = search.searchforissue(result['IssueID'])
if foundNZB == "yes":
updater.foundsearch(result['ComicID'],
result['IssueID'])
else:
logger.info(u"No issues marked as wanted for " + ComicName)
logger.info(u"Finished grabbing what I could.")
def Process_next(self,comicid,issueid,issuenumOG,ml=None):
annchk = "no"
extensions = ('.cbr', '.cbz')
snatchedtorrent = False
myDB = db.DBConnection()
comicnzb = myDB.selectone("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
issuenzb = myDB.selectone("SELECT * from issues WHERE issueid=? AND comicid=? AND ComicName NOT NULL", [issueid,comicid]).fetchone()
if ml is not None and mylar.SNATCHEDTORRENT_NOTIFY:
snatchnzb = myDB.selectone("SELECT * from snatched WHERE IssueID=? AND ComicID=? AND (provider=? OR provider=?) AND Status='Snatched'", [issueid,comicid,'KAT','CBT']).fetchone()
if snatchnzb is None:
logger.fdebug('Was not downloaded with Mylar and the usage of torrents. Disabling torrent manual post-processing completion notification.')
else:
logger.fdebug('Was downloaded from ' + snatchnzb['Provider'] + '. Enabling torrent manual post-processing completion notification.')
snatchedtorrent = True
logger.fdebug('issueid: ' + str(issueid))
logger.fdebug('issuenumOG: ' + str(issuenumOG))
if issuenzb is None:
issuenzb = myDB.selectone("SELECT * from annuals WHERE issueid=? and comicid=?", [issueid,comicid]).fetchone()
annchk = "yes"
#issueno = str(issuenum).split('.')[0]
#new CV API - removed all decimals...here we go AGAIN!
issuenum = issuenzb['Issue_Number']
issue_except = 'None'
if 'au' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AU'
elif 'ai' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AI'
elif 'inh' in issuenum.lower() and issuenum[:1].isdigit():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = '.INH'
elif 'now' in issuenum.lower() and issuenum[:1].isdigit():
if '!' in issuenum: issuenum = re.sub('\!', '', issuenum)
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = '.NOW'
if '.' in issuenum:
iss_find = issuenum.find('.')
iss_b4dec = issuenum[:iss_find]
iss_decval = issuenum[iss_find+1:]
if int(iss_decval) == 0:
iss = iss_b4dec
issdec = int(iss_decval)
issueno = str(iss)
self._log("Issue Number: " + str(issueno))
logger.fdebug("Issue Number: " + str(issueno))
else:
if len(iss_decval) == 1:
iss = iss_b4dec + "." + iss_decval
issdec = int(iss_decval) * 10
else:
iss = iss_b4dec + "." + iss_decval.rstrip('0')
issdec = int(iss_decval.rstrip('0')) * 10
issueno = iss_b4dec
self._log("Issue Number: " + str(iss))
logger.fdebug("Issue Number: " + str(iss))
else:
iss = issuenum
issueno = str(iss)
# issue zero-suppression here
if mylar.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00"
logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N))
if str(len(issueno)) > 1:
if int(issueno) < 0:
self._log("issue detected is a negative")
prettycomiss = '-' + str(zeroadd) + str(abs(issueno))
elif int(issueno) < 10:
self._log("issue detected less than 10")
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss))
elif int(issueno) >= 10 and int(issueno) < 100:
self._log("issue detected greater than 10, but less than 100")
if mylar.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss))
else:
self._log("issue detected greater than 100")
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(issueno)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss))
else:
prettycomiss = str(issueno)
self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss))
if annchk == "yes":
self._log("Annual detected.")
logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss))
issueyear = issuenzb['IssueDate'][:4]
self._log("Issue Year: " + str(issueyear))
logger.fdebug("Issue Year : " + str(issueyear))
month = issuenzb['IssueDate'][5:7].replace('-','').strip()
month_name = helpers.fullmonth(month)
# comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
publisher = comicnzb['ComicPublisher']
self._log("Publisher: " + publisher)
logger.fdebug("Publisher: " + str(publisher))
#we need to un-unicode this to make sure we can write the filenames properly for spec.chars
series = comicnzb['ComicName'].encode('ascii', 'ignore').strip()
self._log("Series: " + series)
logger.fdebug("Series: " + str(series))
seriesyear = comicnzb['ComicYear']
self._log("Year: " + seriesyear)
logger.fdebug("Year: " + str(seriesyear))
comlocation = comicnzb['ComicLocation']
self._log("Comic Location: " + comlocation)
logger.fdebug("Comic Location: " + str(comlocation))
comversion = comicnzb['ComicVersion']
self._log("Comic Version: " + str(comversion))
logger.fdebug("Comic Version: " + str(comversion))
if comversion is None:
comversion = 'None'
#if comversion is None, remove it so it doesn't populate with 'None'
if comversion == 'None':
chunk_f_f = re.sub('\$VolumeN','',mylar.FILE_FORMAT)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
self._log("No version # found for series - tag will not be available for renaming.")
logger.fdebug("No version # found for series, removing from filename")
logger.fdebug("new format is now: " + str(chunk_file_format))
else:
chunk_file_format = mylar.FILE_FORMAT
if annchk == "no":
chunk_f_f = re.sub('\$Annual','',chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('not an annual - removing from filename paramaters')
logger.fdebug('new format: ' + str(chunk_file_format))
else:
logger.fdebug('chunk_file_format is: ' + str(chunk_file_format))
if '$Annual' not in chunk_file_format:
#if it's an annual, but $Annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
prettycomiss = "Annual " + str(prettycomiss)
logger.fdebug('prettycomiss: ' + str(prettycomiss))
ofilename = None
#if meta-tagging is not enabled, we need to declare the check as being fail
#if meta-tagging is enabled, it gets changed just below to a default of pass
pcheck = "fail"
#tag the meta.
if mylar.ENABLE_META:
self._log("Metatagging enabled - proceeding...")
logger.fdebug("Metatagging enabled - proceeding...")
pcheck = "pass"
try:
import cmtagmylar
if ml is None:
pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid)
else:
pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid, manual="yes", filename=ml['ComicLocation'])
except ImportError:
logger.fdebug("comictaggerlib not found on system. Ensure the ENTIRE lib directory is located within mylar/lib/comictaggerlib/")
logger.fdebug("continuing with PostProcessing, but I'm not using metadata.")
pcheck = "fail"
if pcheck == "fail":
self._log("Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging...")
logger.fdebug("Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging...")
elif pcheck == "unrar error":
self._log("This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy.")
logger.error("This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy.")
return self.log
else:
otofilename = pcheck
self._log("Sucessfully wrote metadata to .cbz - Continuing..")
logger.fdebug("Sucessfully wrote metadata to .cbz (" + str(otofilename) + ") - Continuing..")
#Run Pre-script
if mylar.ENABLE_PRE_SCRIPTS:
nzbn = self.nzb_name #original nzb name
nzbf = self.nzb_folder #original nzb folder
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_pre_scripts(nzbn, nzbf, seriesmetadata )
#rename file and move to new path
#nfilename = series + " " + issueno + " (" + seriesyear + ")"
file_values = {'$Series': series,
'$Issue': prettycomiss,
'$Year': issueyear,
'$series': series.lower(),
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion,
'$monthname': month_name,
'$month': month,
'$Annual': 'Annual'
}
#if it's a Manual Run, use the ml['ComicLocation'] for the exact filename.
if ml is None:
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
odir = root
ofilename = filename
path, ext = os.path.splitext(ofilename)
if odir is None:
logger.fdebug('no root folder set.')
odir = self.nzb_folder
logger.fdebug('odir: ' + str(odir))
logger.fdebug('ofilename: ' + str(ofilename))
else:
if pcheck == "fail":
otofilename = ml['ComicLocation']
logger.fdebug('otofilename:' + str(otofilename))
odir, ofilename = os.path.split(otofilename)
logger.fdebug('odir: ' + str(odir))
logger.fdebug('ofilename: ' + str(ofilename))
path, ext = os.path.splitext(ofilename)
logger.fdebug('path: ' + str(path))
logger.fdebug('ext:' + str(ext))
if ofilename is None:
logger.error(u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that " + str(self.nzb_folder) + " exists and is the correct location.")
return
self._log("Original Filename: " + ofilename)
self._log("Original Extension: " + ext)
logger.fdebug("Original Filname: " + str(ofilename))
logger.fdebug("Original Extension: " + str(ext))
if mylar.FILE_FORMAT == '' or not mylar.RENAME_FILES:
self._log("Rename Files isn't enabled...keeping original filename.")
logger.fdebug("Rename Files isn't enabled - keeping original filename.")
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
nfilename = helpers.replace_all(chunk_file_format, file_values)
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
nfilename = re.sub('[\,\:\?]', '', nfilename)
nfilename = re.sub('[\/]', '-', nfilename)
self._log("New Filename: " + nfilename)
logger.fdebug("New Filename: " + str(nfilename))
#src = os.path.join(self.nzb_folder, ofilename)
src = os.path.join(odir, ofilename)
filechecker.validateAndCreateDirectory(comlocation, True)
if mylar.LOWERCASE_FILENAMES:
dst = (comlocation + "/" + nfilename + ext).lower()
else:
dst = comlocation + "/" + nfilename + ext.lower()
self._log("Source:" + src)
self._log("Destination:" + dst)
logger.fdebug("Source: " + str(src))
logger.fdebug("Destination: " + str(dst))
if ml is None:
#downtype = for use with updater on history table to set status to 'Downloaded'
downtype = 'True'
#non-manual run moving/deleting...
logger.fdebug('self.nzb_folder: ' + self.nzb_folder)
logger.fdebug('odir: ' + str(odir))
logger.fdebug('ofilename:' + str(ofilename))
logger.fdebug('nfilename:' + str(nfilename + ext))
if mylar.RENAME_FILES:
if str(ofilename) != str(nfilename + ext):
logger.fdebug("Renaming " + os.path.join(odir, str(ofilename)) + " ..to.. " + os.path.join(odir,str(nfilename + ext)))
os.rename(os.path.join(odir, str(ofilename)), os.path.join(odir,str(nfilename + ext)))
else:
logger.fdebug('filename is identical as original, not renaming.')
#src = os.path.join(self.nzb_folder, str(nfilename + ext))
src = os.path.join(odir, str(nfilename + ext))
try:
shutil.move(src, dst)
except (OSError, IOError):
self._log("Failed to move directory - check directories and manually re-run.")
self._log("Post-Processing ABORTED.")
return
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log("Failed to remove temporary directory - check directory and manually re-run.")
self._log("Post-Processing ABORTED.")
return
self._log("Removed temporary directory : " + str(self.nzb_folder))
else:
#downtype = for use with updater on history table to set status to 'Post-Processed'
downtype = 'PP'
#Manual Run, this is the portion.
if mylar.RENAME_FILES:
if str(ofilename) != str(nfilename + ext):
logger.fdebug("Renaming " + os.path.join(self.nzb_folder, str(ofilename)) + " ..to.. " + os.path.join(self.nzb_folder,str(nfilename + ext)))
os.rename(os.path.join(odir, str(ofilename)), os.path.join(odir ,str(nfilename + ext)))
else:
logger.fdebug('filename is identical as original, not renaming.')
src = os.path.join(odir, str(nfilename + ext))
logger.fdebug('odir rename: ' + os.path.join(odir, str(ofilename)) + ' TO ' + os.path.join(odir, str(nfilename + ext)))
logger.fdebug('odir src : ' + os.path.join(odir, str(nfilename + ext)))
logger.fdebug("Moving " + src + " ... to ... " + dst)
try:
shutil.move(src, dst)
except (OSError, IOError):
logger.fdebug("Failed to move directory - check directories and manually re-run.")
logger.fdebug("Post-Processing ABORTED.")
return
logger.fdebug("Successfully moved to : " + dst)
#tidyup old path
#try:
# os.remove(os.path.join(self.nzb_folder, str(ofilename)))
# logger.fdebug("Deleting : " + os.path.join(self.nzb_folder, str(ofilename)))
#except (OSError, IOError):
# logger.fdebug("Failed to remove temporary directory - check directory and manually re-run.")
# logger.fdebug("Post-Processing ABORTED.")
# return
#logger.fdebug("Removed temporary directory : " + str(self.nzb_folder))
#Hopefully set permissions on downloaded file
try:
permission = int(mylar.CHMOD_FILE, 8)
os.umask(0)
os.chmod(dst.rstrip(), permission)
except OSError:
logger.error('Failed to change file permissions. Ensure that the user running Mylar has proper permissions to change permissions in : ' + dst)
logger.fdebug('Continuing post-processing but unable to change file permissions in ' + dst)
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
#update snatched table to change status to Downloaded
if annchk == "no":
updater.foundsearch(comicid, issueid, down=downtype)
dispiss = 'issue: ' + str(issuenumOG)
else:
updater.foundsearch(comicid, issueid, mode='want_ann', down=downtype)
dispiss = 'annual issue: ' + str(issuenumOG)
#force rescan of files
updater.forceRescan(comicid)
logger.info(u"Post-Processing completed for: " + series + " " + dispiss )
self._log(u"Post Processing SUCCESSFUL! ")
if mylar.WEEKFOLDER:
#if enabled, will *copy* the post-processed file to the weeklypull list folder for the given week.
weeklypull.weekly_singlecopy(comicid,issuenum,str(nfilename+ext),dst)
# retrieve/create the corresponding comic objects
if mylar.ENABLE_EXTRA_SCRIPTS:
folderp = str(dst) #folder location after move/rename
nzbn = self.nzb_name #original nzb name
filen = str(nfilename + ext) #new filename
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_extra_scripts(nzbn, self.nzb_folder, filen, folderp, seriesmetadata )
if ml is not None:
#we only need to return self.log if it's a manual run and it's not a snatched torrent
if snatchedtorrent:
#manual run + snatched torrent
pass
else:
#manual run + not snatched torrent (or normal manual-run)
return self.log
if annchk == "no":
prline = series + '(' + issueyear + ') - issue #' + issuenumOG
else:
prline = series + ' Annual (' + issueyear + ') - issue #' + issuenumOG
prline2 = 'Mylar has downloaded and post-processed: ' + prline
if mylar.PROWL_ENABLED:
pushmessage = prline
logger.info(u"Prowl request")
prowl = notifiers.PROWL()
prowl.notify(pushmessage,"Download and Postprocessing completed")
if mylar.NMA_ENABLED:
nma = notifiers.NMA()
nma.notify(prline=prline, prline2=prline2)
if mylar.PUSHOVER_ENABLED:
logger.info(u"Pushover request")
pushover = notifiers.PUSHOVER()
pushover.notify(prline, "Download and Post-Processing completed")
if mylar.BOXCAR_ENABLED:
boxcar = notifiers.BOXCAR()
boxcar.notify(prline=prline, prline2=prline2)
if mylar.PUSHBULLET_ENABLED:
pushbullet = notifiers.PUSHBULLET()
pushbullet.notify(prline=prline, prline2=prline2)
return self.log
def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None, queue=None):
if cron and not mylar.LIBRARYSCAN:
return
if not dir:
dir = mylar.COMIC_DIR
# If we're appending a dir, it's coming from the post processor which is
# already bytestring
if not append:
dir = dir.encode(mylar.SYS_ENCODING)
if not os.path.isdir(dir):
logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(mylar.SYS_ENCODING, 'replace'))
return
logger.info('Scanning comic directory: %s' % dir.decode(mylar.SYS_ENCODING, 'replace'))
basedir = dir
comic_list = []
failure_list = []
comiccnt = 0
extensions = ('cbr','cbz')
cv_location = []
cbz_retry = 0
mylar.IMPORT_STATUS = 'Now attempting to parse files for additional information'
#mylar.IMPORT_PARSED_COUNT #used to count what #/totalfiles the filename parser is currently on
for r, d, f in os.walk(dir):
for files in f:
mylar.IMPORT_FILES +=1
if 'cvinfo' in files:
cv_location.append(r)
logger.fdebug('CVINFO found: ' + os.path.join(r))
if any(files.lower().endswith('.' + x.lower()) for x in extensions):
comic = files
comicpath = os.path.join(r, files)
comicsize = os.path.getsize(comicpath)
t = filechecker.FileChecker(dir=r, file=comic)
results = t.listFiles()
#logger.info(results)
#'type': re.sub('\.','', filetype).strip(),
#'sub': path_list,
#'volume': volume,
#'match_type': match_type,
#'comicfilename': filename,
#'comiclocation': clocation,
#'series_name': series_name,
#'series_volume': issue_volume,
#'series_year': issue_year,
#'justthedigits': issue_number,
#'annualcomicid': annual_comicid,
#'scangroup': scangroup}
logger.fdebug('Comic: ' + comic + ' [' + comicpath + '] - ' + str(comicsize) + ' bytes')
if results:
resultline = '[PARSE-' + results['parse_status'].upper() + ']'
resultline += '[SERIES: ' + results['series_name'] + ']'
if results['series_volume'] is not None:
resultline += '[VOLUME: ' + results['series_volume'] + ']'
if results['issue_year'] is not None:
resultline += '[ISSUE YEAR: ' + str(results['issue_year']) + ']'
if results['issue_number'] is not None:
resultline += '[ISSUE #: ' + results['issue_number'] + ']'
logger.fdebug(resultline)
else:
logger.fdebug('[PARSED] FAILURE.')
continue
# We need the unicode path to use for logging, inserting into database
unicode_comic_path = comicpath.decode(mylar.SYS_ENCODING, 'replace')
if results['parse_status'] == 'success':
comic_list.append({'ComicFilename': comic,
'ComicLocation': comicpath,
'ComicSize': comicsize,
'Unicode_ComicLocation': unicode_comic_path,
'parsedinfo': {'series_name': results['series_name'],
'series_volume': results['series_volume'],
'issue_year': results['issue_year'],
'issue_number': results['issue_number']}
})
comiccnt +=1
mylar.IMPORT_PARSED_COUNT +=1
else:
failure_list.append({'ComicFilename': comic,
'ComicLocation': comicpath,
'ComicSize': comicsize,
'Unicode_ComicLocation': unicode_comic_path,
'parsedinfo': {'series_name': results['series_name'],
'series_volume': results['series_volume'],
'issue_year': results['issue_year'],
'issue_number': results['issue_number']}
})
mylar.IMPORT_FAILURE_COUNT +=1
if comic.endswith('.cbz'):
cbz_retry +=1
mylar.IMPORT_TOTALFILES = comiccnt
logger.info('I have successfully discovered & parsed a total of ' + str(comiccnt) + ' files....analyzing now')
logger.info('I have not been able to determine what ' + str(len(failure_list)) + ' files are')
logger.info('However, ' + str(cbz_retry) + ' files are in a cbz format, which may contain metadata.')
mylar.IMPORT_STATUS = 'Successfully parsed ' + str(comiccnt) + ' files'
#return queue.put(valreturn)
myDB = db.DBConnection()
#let's load in the watchlist to see if we have any matches.
logger.info("loading in the watchlist to see if a series is being watched already...")
watchlist = myDB.select("SELECT * from comics")
ComicName = []
DisplayName = []
ComicYear = []
ComicPublisher = []
ComicTotal = []
ComicID = []
ComicLocation = []
AltName = []
watchcnt = 0
watch_kchoice = []
watchchoice = {}
import_by_comicids = []
import_comicids = {}
for watch in watchlist:
#use the comicname_filesafe to start
watchdisplaycomic = watch['ComicName'].encode('utf-8').strip() #re.sub('[\_\#\,\/\:\;\!\$\%\&\+\'\?\@]', ' ', watch['ComicName']).encode('utf-8').strip()
# let's clean up the name, just in case for comparison purposes...
watchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '', watch['ComicName_Filesafe']).encode('utf-8').strip()
#watchcomic = re.sub('\s+', ' ', str(watchcomic)).strip()
if ' the ' in watchcomic.lower():
#drop the 'the' from the watchcomic title for proper comparisons.
watchcomic = watchcomic[-4:]
alt_chk = "no" # alt-checker flag (default to no)
# account for alternate names as well
if watch['AlternateSearch'] is not None and watch['AlternateSearch'] is not 'None':
altcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', '', watch['AlternateSearch']).encode('utf-8').strip()
#altcomic = re.sub('\s+', ' ', str(altcomic)).strip()
AltName.append(altcomic)
alt_chk = "yes" # alt-checker flag
ComicName.append(watchcomic)
DisplayName.append(watchdisplaycomic)
ComicYear.append(watch['ComicYear'])
ComicPublisher.append(watch['ComicPublisher'])
ComicTotal.append(watch['Total'])
ComicID.append(watch['ComicID'])
ComicLocation.append(watch['ComicLocation'])
watchcnt+=1
logger.info("Successfully loaded " + str(watchcnt) + " series from your watchlist.")
ripperlist=['digital-',
'empire',
'dcp']
watchfound = 0
datelist = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
# datemonth = {'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9,'ten':10,'eleven':$
# #search for number as text, and change to numeric
# for numbs in basnumbs:
# #logger.fdebug("numbs:" + str(numbs))
# if numbs in ComicName.lower():
# numconv = basnumbs[numbs]
# #logger.fdebug("numconv: " + str(numconv))
issueid_list = []
cvscanned_loc = None
cvinfo_CID = None
cnt = 0
mylar.IMPORT_STATUS = '[0%] Now parsing individual filenames for metadata if available'
for i in comic_list:
mylar.IMPORT_STATUS = '[' + str(cnt) + '/' + str(comiccnt) + '] Now parsing individual filenames for metadata if available'
logger.fdebug('Analyzing : ' + i['ComicFilename'])
comfilename = i['ComicFilename']
comlocation = i['ComicLocation']
issueinfo = None
#probably need to zero these issue-related metadata to None so we can pick the best option
issuevolume = None
#Make sure cvinfo is checked for FIRST (so that CID can be attached to all files properly thereafter as they're scanned in)
if os.path.dirname(comlocation) in cv_location and os.path.dirname(comlocation) != cvscanned_loc:
#if comfilename == 'cvinfo':
logger.info('comfilename: ' + comfilename)
logger.info('cvscanned_loc: ' + str(cv_location))
logger.info('comlocation: ' + os.path.dirname(comlocation))
#if cvscanned_loc != comlocation:
try:
with open(os.path.join(os.path.dirname(comlocation), 'cvinfo')) as f:
urllink = f.readline()
if urllink:
cid = urllink.split('/')
if '4050-' in cid[-2]:
cvinfo_CID = re.sub('4050-', '', cid[-2]).strip()
logger.info('CVINFO file located within directory. Attaching everything in directory that is valid to ComicID: ' + str(cvinfo_CID))
#store the location of the cvinfo so it's applied to the correct directory (since we're scanning multile direcorties usually)
cvscanned_loc = os.path.dirname(comlocation)
else:
logger.error("Could not read cvinfo file properly (or it does not contain any data)")
except (OSError, IOError):
logger.error("Could not read cvinfo file properly (or it does not contain any data)")
#else:
# don't scan in it again if it's already been done initially
# continue
if mylar.IMP_METADATA:
#if read tags is enabled during import, check here.
if i['ComicLocation'].endswith('.cbz'):
logger.fdebug('[IMPORT-CBZ] Metatagging checking enabled.')
logger.info('[IMPORT-CBZ} Attempting to read tags present in filename: ' + i['ComicLocation'])
issueinfo = helpers.IssueDetails(i['ComicLocation'])
logger.info('issueinfo: ' + str(issueinfo))
if issueinfo is None:
logger.fdebug('[IMPORT-CBZ] No valid metadata contained within filename. Dropping down to parsing the filename itself.')
pass
else:
issuenotes_id = None
logger.info('[IMPORT-CBZ] Successfully retrieved some tags. Lets see what I can figure out.')
comicname = issueinfo[0]['series']
if comicname is not None:
logger.fdebug('[IMPORT-CBZ] Series Name: ' + comicname)
as_d = filechecker.FileChecker(watchcomic=comicname.decode('utf-8'))
as_dyninfo = as_d.dynamic_replace(comicname)
logger.fdebug('Dynamic-ComicName: ' + as_dyninfo['mod_seriesname'])
else:
logger.fdebug('[IMPORT-CBZ] No series name found within metadata. This is bunk - dropping down to file parsing for usable information.')
issueinfo = None
issue_number = None
if issueinfo is not None:
try:
issueyear = issueinfo[0]['year']
except:
issueyear = None
#if the issue number is a non-numeric unicode string, this will screw up along with impID
issue_number = issueinfo[0]['issue_number']
if issue_number is not None:
logger.fdebug('[IMPORT-CBZ] Issue Number: ' + issue_number)
else:
issue_number = i['parsed']['issue_number']
if 'annual' in comicname.lower() or 'annual' in comfilename.lower():
if issue_number is None or issue_number == 'None':
logger.info('Annual detected with no issue number present within metadata. Assuming year as issue.')
try:
issue_number = 'Annual ' + str(issueyear)
except:
issue_number = 'Annual ' + i['parsed']['issue_year']
else:
logger.info('Annual detected with issue number present within metadata.')
if 'annual' not in issue_number.lower():
issue_number = 'Annual ' + issue_number
mod_series = re.sub('annual', '', comicname, flags=re.I).strip()
else:
mod_series = comicname
logger.fdebug('issue number SHOULD Be: ' + issue_number)
try:
issuetitle = issueinfo[0]['title']
except:
issuetitle = None
try:
issueyear = issueinfo[0]['year']
except:
issueyear = None
try:
issuevolume = str(issueinfo[0]['volume'])
if all([issuevolume is not None, issuevolume != 'None']) and not issuevolume.lower().startswith('v'):
issuevolume = 'v' + str(issuevolume)
logger.fdebug('[TRY]issue volume is: ' + str(issuevolume))
except:
logger.fdebug('[EXCEPT]issue volume is: ' + str(issuevolume))
issuevolume = None
if any([comicname is None, comicname == 'None', issue_number is None, issue_number == 'None']):
logger.fdebug('[IMPORT-CBZ] Improperly tagged file as the metatagging is invalid. Ignoring meta and just parsing the filename.')
issueinfo = None
pass
else:
# if used by ComicTagger, Notes field will have the IssueID.
issuenotes = issueinfo[0]['notes']
logger.fdebug('[IMPORT-CBZ] Notes: ' + issuenotes)
if issuenotes is not None and issuenotes != 'None':
if 'Issue ID' in issuenotes:
st_find = issuenotes.find('Issue ID')
tmp_issuenotes_id = re.sub("[^0-9]", " ", issuenotes[st_find:]).strip()
if tmp_issuenotes_id.isdigit():
issuenotes_id = tmp_issuenotes_id
logger.fdebug('[IMPORT-CBZ] Successfully retrieved CV IssueID for ' + comicname + ' #' + issue_number + ' [' + str(issuenotes_id) + ']')
elif 'CVDB' in issuenotes:
st_find = issuenotes.find('CVDB')
tmp_issuenotes_id = re.sub("[^0-9]", " ", issuenotes[st_find:]).strip()
if tmp_issuenotes_id.isdigit():
issuenotes_id = tmp_issuenotes_id
logger.fdebug('[IMPORT-CBZ] Successfully retrieved CV IssueID for ' + comicname + ' #' + issue_number + ' [' + str(issuenotes_id) + ']')
else:
logger.fdebug('[IMPORT-CBZ] Unable to retrieve IssueID from meta-tagging. If there is other metadata present I will use that.')
logger.fdebug('[IMPORT-CBZ] Adding ' + comicname + ' to the import-queue!')
#impid = comicname + '-' + str(issueyear) + '-' + str(issue_number) #com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
impid = str(random.randint(1000000,99999999))
logger.fdebug('[IMPORT-CBZ] impid: ' + str(impid))
#make sure we only add in those issueid's which don't already have a comicid attached via the cvinfo scan above (this is for reverse-lookup of issueids)
issuepopulated = False
if cvinfo_CID is None:
if issuenotes_id is None:
logger.info('[IMPORT-CBZ] No ComicID detected where it should be. Bypassing this metadata entry and going the parsing route [' + comfilename + ']')
else:
#we need to store the impid here as well so we can look it up.
issueid_list.append({'issueid': issuenotes_id,
'importinfo': {'impid': impid,
'comicid': None,
'comicname': comicname,
'dynamicname': as_dyninfo['mod_seriesname'],
'comicyear': issueyear,
'issuenumber': issue_number,
'volume': issuevolume,
'comfilename': comfilename,
'comlocation': comlocation.decode(mylar.SYS_ENCODING)}
})
mylar.IMPORT_CID_COUNT +=1
issuepopulated = True
if issuepopulated == False:
if cvscanned_loc == os.path.dirname(comlocation):
cv_cid = cvinfo_CID
logger.fdebug('[IMPORT-CBZ] CVINFO_COMICID attached : ' + str(cv_cid))
else:
cv_cid = None
import_by_comicids.append({
"impid": impid,
"comicid": cv_cid,
"watchmatch": None,
"displayname": mod_series,
"comicname": comicname,
"dynamicname": as_dyninfo['mod_seriesname'],
"comicyear": issueyear,
"issuenumber": issue_number,
"volume": issuevolume,
"issueid": issuenotes_id,
"comfilename": comfilename,
"comlocation": comlocation.decode(mylar.SYS_ENCODING)
})
mylar.IMPORT_CID_COUNT +=1
else:
pass
#logger.fdebug(i['ComicFilename'] + ' is not in a metatagged format (cbz). Bypassing reading of the metatags')
if issueinfo is None:
if i['parsedinfo']['issue_number'] is None:
if 'annual' in i['parsedinfo']['series_name'].lower():
logger.fdebug('Annual detected with no issue number present. Assuming year as issue.')##1 issue')
if i['parsedinfo']['issue_year'] is not None:
issuenumber = 'Annual ' + str(i['parsedinfo']['issue_year'])
else:
issuenumber = 'Annual 1'
else:
issuenumber = i['parsedinfo']['issue_number']
if 'annual' in i['parsedinfo']['series_name'].lower():
mod_series = re.sub('annual', '', i['parsedinfo']['series_name'], flags=re.I).strip()
logger.fdebug('Annual detected with no issue number present. Assuming year as issue.')##1 issue')
if i['parsedinfo']['issue_number'] is not None:
issuenumber = 'Annual ' + str(i['parsedinfo']['issue_number'])
else:
if i['parsedinfo']['issue_year'] is not None:
issuenumber = 'Annual ' + str(i['parsedinfo']['issue_year'])
else:
issuenumber = 'Annual 1'
else:
mod_series = i['parsedinfo']['series_name']
issuenumber = i['parsedinfo']['issue_number']
logger.fdebug('[' + mod_series + '] Adding to the import-queue!')
isd = filechecker.FileChecker(watchcomic=mod_series.decode('utf-8'))
is_dyninfo = isd.dynamic_replace(mod_series)
logger.fdebug('Dynamic-ComicName: ' + is_dyninfo['mod_seriesname'])
#impid = dispname + '-' + str(result_comyear) + '-' + str(comiss) #com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
impid = str(random.randint(1000000,99999999))
logger.fdebug("impid: " + str(impid))
if cvscanned_loc == os.path.dirname(comlocation):
cv_cid = cvinfo_CID
logger.fdebug('CVINFO_COMICID attached : ' + str(cv_cid))
else:
cv_cid = None
if issuevolume is None:
logger.fdebug('issue volume is : ' + str(issuevolume))
if i['parsedinfo']['series_volume'] is None:
issuevolume = None
else:
if str(i['parsedinfo']['series_volume'].lower()).startswith('v'):
issuevolume = i['parsedinfo']['series_volume']
else:
issuevolume = 'v' + str(i['parsedinfo']['series_volume'])
else:
logger.fdebug('issue volume not none : ' + str(issuevolume))
if issuevolume.lower().startswith('v'):
issuevolume = issuevolume
else:
issuevolume = 'v' + str(issuevolume)
logger.fdebug('IssueVolume is : ' + str(issuevolume))
import_by_comicids.append({
"impid": impid,
"comicid": cv_cid,
"issueid": None,
"watchmatch": None, #watchmatch (should be true/false if it already exists on watchlist)
"displayname": mod_series,
"comicname": i['parsedinfo']['series_name'],
"dynamicname": is_dyninfo['mod_seriesname'].lower(),
"comicyear": i['parsedinfo']['issue_year'],
"issuenumber": issuenumber, #issuenumber,
"volume": issuevolume,
"comfilename": comfilename,
"comlocation": comlocation.decode(mylar.SYS_ENCODING)
})
cnt+=1
#logger.fdebug('import_by_ids: ' + str(import_by_comicids))
#reverse lookup all of the gathered IssueID's in order to get the related ComicID
reverse_issueids = []
for x in issueid_list:
reverse_issueids.append(x['issueid'])
vals = None
if len(reverse_issueids) > 0:
mylar.IMPORT_STATUS = 'Now Reverse looking up ' + str(len(reverse_issueids)) + ' IssueIDs to get the ComicIDs'
vals = mylar.cv.getComic(None, 'import', comicidlist=reverse_issueids)
#logger.fdebug('vals returned:' + str(vals))
if len(watch_kchoice) > 0:
watchchoice['watchlist'] = watch_kchoice
#logger.fdebug("watchchoice: " + str(watchchoice))
logger.info("I have found " + str(watchfound) + " out of " + str(comiccnt) + " comics for series that are being watched.")
wat = 0
comicids = []
if watchfound > 0:
if mylar.IMP_MOVE:
logger.info('You checked off Move Files...so that\'s what I am going to do')
#check to see if Move Files is enabled.
#if not being moved, set the archive bit.
logger.fdebug('Moving files into appropriate directory')
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comlocation = watch_the_list['ComicLocation']
watch_comicid = watch_the_list['ComicID']
watch_comicname = watch_the_list['ComicName']
watch_comicyear = watch_the_list['ComicYear']
watch_comiciss = watch_the_list['ComicIssue']
logger.fdebug('ComicLocation: ' + watch_comlocation)
orig_comlocation = watch_the_list['OriginalLocation']
orig_filename = watch_the_list['OriginalFilename']
logger.fdebug('Orig. Location: ' + orig_comlocation)
logger.fdebug('Orig. Filename: ' + orig_filename)
#before moving check to see if Rename to Mylar structure is enabled.
if mylar.IMP_RENAME:
logger.fdebug('Renaming files according to configuration details : ' + str(mylar.FILE_FORMAT))
renameit = helpers.rename_param(watch_comicid, watch_comicname, watch_comicyear, watch_comiciss)
nfilename = renameit['nfilename']
dst_path = os.path.join(watch_comlocation, nfilename)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
else:
logger.fdebug('Renaming files not enabled, keeping original filename(s)')
dst_path = os.path.join(watch_comlocation, orig_filename)
#os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
#src = os.path.join(, str(nfilename + ext))
logger.fdebug('I am going to move ' + orig_comlocation + ' to ' + dst_path)
try:
shutil.move(orig_comlocation, dst_path)
except (OSError, IOError):
logger.info("Failed to move directory - check directories and manually re-run.")
wat+=1
else:
# if move files isn't enabled, let's set all found comics to Archive status :)
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comicid = watch_the_list['ComicID']
watch_issue = watch_the_list['ComicIssue']
logger.fdebug('ComicID: ' + str(watch_comicid))
logger.fdebug('Issue#: ' + str(watch_issue))
issuechk = myDB.selectone("SELECT * from issues where ComicID=? AND INT_IssueNumber=?", [watch_comicid, watch_issue]).fetchone()
if issuechk is None:
logger.fdebug('No matching issues for this comic#')
else:
logger.fdebug('...Existing status: ' + str(issuechk['Status']))
control = {"IssueID": issuechk['IssueID']}
values = {"Status": "Archived"}
logger.fdebug('...changing status of ' + str(issuechk['Issue_Number']) + ' to Archived ')
myDB.upsert("issues", values, control)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
wat+=1
if comicids is None: pass
else:
c_upd = len(comicids)
c = 0
while (c < c_upd ):
logger.fdebug('Rescanning.. ' + str(c))
updater.forceRescan(c)
if not len(import_by_comicids):
return "Completed"
if len(import_by_comicids) > 0 or len(vals) > 0:
#import_comicids['comic_info'] = import_by_comicids
#if vals:
# import_comicids['issueid_info'] = vals
#else:
# import_comicids['issueid_info'] = None
if vals:
cvimport_comicids = vals
import_cv_ids = len(vals)
else:
cvimport_comicids = None
import_cv_ids = 0
else:
import_cv_ids = 0
return {'import_by_comicids': import_by_comicids,
'import_count': len(import_by_comicids),
'CV_import_comicids': cvimport_comicids,
'import_cv_ids': import_cv_ids,
'issueid_list': issueid_list,
'failure_list': failure_list}
logger.info(u"Updating complete for: " + comic['ComicName'])
#move the files...if imported is not empty (meaning it's not from the mass importer.)
if imported is None or imported == 'None':
pass
else:
if mylar.IMP_MOVE:
logger.info("Mass import - Move files")
moveit.movefiles(comicid,comlocation,ogcname)
else:
logger.info("Mass import - Moving not Enabled. Setting Archived Status for import.")
moveit.archivefiles(comicid,ogcname)
#check for existing files...
updater.forceRescan(comicid)
if pullupd is None:
# lets' check the pullist for anything at this time as well since we're here.
# do this for only Present comics....
if mylar.AUTOWANT_UPCOMING and lastpubdate == 'Present': #and 'Present' in gcdinfo['resultPublished']:
print ("latestissue: #" + str(latestiss))
chkstats = myDB.action("SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?", [comicid,str(latestiss)]).fetchone()
print chkstats['Status']
if chkstats['Status'] == 'Skipped' or chkstats['Status'] == 'Wanted' or chkstats['Status'] == 'Snatched':
logger.info(u"Checking this week's pullist for new issues of " + comic['ComicName'])
updater.newpullcheck(comic['ComicName'], comicid)
#here we grab issues that have been marked as wanted above...
results = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [comicid])
def Process_next(self,comicid,issueid,issuenumOG,ml=None):
annchk = "no"
extensions = ('.cbr', '.cbz')
myDB = db.DBConnection()
comicnzb = myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
issuenzb = myDB.action("SELECT * from issues WHERE issueid=? AND comicid=? AND ComicName NOT NULL", [issueid,comicid]).fetchone()
print "issueid: " + str(issueid)
print "issuenumOG: " + str(issuenumOG)
if issuenzb is None:
print "chk1"
issuenzb = myDB.action("SELECT * from annuals WHERE issueid=? and comicid=?", [issueid,comicid]).fetchone()
print "chk2"
annchk = "yes"
print issuenzb
#issueno = str(issuenum).split('.')[0]
#new CV API - removed all decimals...here we go AGAIN!
issuenum = issuenzb['Issue_Number']
issue_except = 'None'
if 'au' in issuenum.lower():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AU'
elif 'ai' in issuenum.lower():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AI'
if '.' in issuenum:
iss_find = issuenum.find('.')
iss_b4dec = issuenum[:iss_find]
iss_decval = issuenum[iss_find+1:]
if int(iss_decval) == 0:
iss = iss_b4dec
issdec = int(iss_decval)
issueno = str(iss)
self._log("Issue Number: " + str(issueno), logger.DEBUG)
logger.fdebug("Issue Number: " + str(issueno))
else:
if len(iss_decval) == 1:
iss = iss_b4dec + "." + iss_decval
issdec = int(iss_decval) * 10
else:
iss = iss_b4dec + "." + iss_decval.rstrip('0')
issdec = int(iss_decval.rstrip('0')) * 10
issueno = iss_b4dec
self._log("Issue Number: " + str(iss), logger.DEBUG)
logger.fdebug("Issue Number: " + str(iss))
else:
iss = issuenum
issueno = str(iss)
# issue zero-suppression here
if mylar.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00"
logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N))
if str(len(issueno)) > 1:
if int(issueno) < 10:
self._log("issue detected less than 10", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG)
elif int(issueno) >= 10 and int(issueno) < 100:
self._log("issue detected greater than 10, but less than 100", logger.DEBUG)
if mylar.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss), logger.DEBUG)
else:
self._log("issue detected greater than 100", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(issueno)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG)
else:
prettycomiss = str(issueno)
self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss), logger.DEBUG)
if annchk == "yes":
prettycomiss = "Annual " + str(prettycomiss)
self._log("Annual detected.")
logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss))
issueyear = issuenzb['IssueDate'][:4]
self._log("Issue Year: " + str(issueyear), logger.DEBUG)
logger.fdebug("Issue Year : " + str(issueyear))
# comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
publisher = comicnzb['ComicPublisher']
self._log("Publisher: " + publisher, logger.DEBUG)
logger.fdebug("Publisher: " + str(publisher))
#we need to un-unicode this to make sure we can write the filenames properly for spec.chars
series = comicnzb['ComicName'].encode('ascii', 'ignore').strip()
self._log("Series: " + series, logger.DEBUG)
logger.fdebug("Series: " + str(series))
seriesyear = comicnzb['ComicYear']
self._log("Year: " + seriesyear, logger.DEBUG)
logger.fdebug("Year: " + str(seriesyear))
comlocation = comicnzb['ComicLocation']
self._log("Comic Location: " + comlocation, logger.DEBUG)
logger.fdebug("Comic Location: " + str(comlocation))
comversion = comicnzb['ComicVersion']
self._log("Comic Version: " + str(comversion), logger.DEBUG)
logger.fdebug("Comic Version: " + str(comversion))
if comversion is None:
comversion = 'None'
#if comversion is None, remove it so it doesn't populate with 'None'
if comversion == 'None':
chunk_f_f = re.sub('\$VolumeN','',mylar.FILE_FORMAT)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
self._log("No version # found for series - tag will not be available for renaming.", logger.DEBUG)
logger.fdebug("No version # found for series, removing from filename")
logger.fdebug("new format is now: " + str(chunk_file_format))
else:
chunk_file_format = mylar.FILE_FORMAT
ofilename = None
#if meta-tagging is not enabled, we need to declare the check as being fail
#if meta-tagging is enabled, it gets changed just below to a default of pass
pcheck = "fail"
#tag the meta.
if mylar.ENABLE_META:
self._log("Metatagging enabled - proceeding...")
logger.fdebug("Metatagging enabled - proceeding...")
pcheck = "pass"
try:
import cmtagmylar
if ml is None:
pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid)
else:
pcheck = cmtagmylar.run(self.nzb_folder, issueid=issueid, manual="yes", filename=ml['ComicLocation'])
except ImportError:
logger.fdebug("comictaggerlib not found on system. Ensure the ENTIRE lib directory is located within mylar/lib/comictaggerlib/")
logger.fdebug("continuing with PostProcessing, but I'm not using metadata.")
pcheck = "fail"
if pcheck == "fail":
self._log("Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging...")
logger.fdebug("Unable to write metadata successfully - check mylar.log file. Attempting to continue without tagging...")
elif pcheck == "unrar error":
self._log("This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy.")
logger.error("This is a corrupt archive - whether CRC errors or it's incomplete. Marking as BAD, and retrying a different copy.")
return self.log
else:
otofilename = pcheck
self._log("Sucessfully wrote metadata to .cbz - Continuing..")
logger.fdebug("Sucessfully wrote metadata to .cbz (" + str(otofilename) + ") - Continuing..")
#Run Pre-script
if mylar.ENABLE_PRE_SCRIPTS:
nzbn = self.nzb_name #original nzb name
nzbf = self.nzb_folder #original nzb folder
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_pre_scripts(nzbn, nzbf, seriesmetadata )
#rename file and move to new path
#nfilename = series + " " + issueno + " (" + seriesyear + ")"
file_values = {'$Series': series,
'$Issue': prettycomiss,
'$Year': issueyear,
'$series': series.lower(),
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion
}
#if it's a Manual Run, use the ml['ComicLocation'] for the exact filename.
if ml is None:
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
ofilename = filename
path, ext = os.path.splitext(ofilename)
else:
if pcheck == "fail":
otofilename = ml['ComicLocation']
print "otofilename:" + str(otofilename)
odir, ofilename = os.path.split(otofilename)
print "ofilename: " + str(ofilename)
path, ext = os.path.splitext(ofilename)
print "path: " + str(path)
print "ext:" + str(ext)
if ofilename is None:
logger.error(u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that " + str(self.nzb_folder) + " exists and is the correct location.")
return
self._log("Original Filename: " + ofilename, logger.DEBUG)
self._log("Original Extension: " + ext, logger.DEBUG)
logger.fdebug("Original Filname: " + str(ofilename))
logger.fdebug("Original Extension: " + str(ext))
if mylar.FILE_FORMAT == '' or not mylar.RENAME_FILES:
self._log("Rename Files isn't enabled...keeping original filename.", logger.DEBUG)
logger.fdebug("Rename Files isn't enabled - keeping original filename.")
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
nfilename = helpers.replace_all(chunk_file_format, file_values)
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
nfilename = re.sub('[\,\:\?]', '', nfilename)
self._log("New Filename: " + nfilename, logger.DEBUG)
logger.fdebug("New Filename: " + str(nfilename))
src = os.path.join(self.nzb_folder, ofilename)
filechecker.validateAndCreateDirectory(comlocation, True)
if mylar.LOWERCASE_FILENAMES:
dst = (comlocation + "/" + nfilename + ext).lower()
else:
dst = comlocation + "/" + nfilename + ext.lower()
self._log("Source:" + src, logger.DEBUG)
self._log("Destination:" + dst, logger.DEBUG)
logger.fdebug("Source: " + str(src))
logger.fdebug("Destination: " + str(dst))
if ml is None:
#non-manual run moving/deleting...
os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
src = os.path.join(self.nzb_folder, str(nfilename + ext))
try:
shutil.move(src, dst)
except (OSError, IOError):
self._log("Failed to move directory - check directories and manually re-run.", logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log("Failed to remove temporary directory - check directory and manually re-run.", logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
self._log("Removed temporary directory : " + str(self.nzb_folder), logger.DEBUG)
else:
#Manual Run, this is the portion.
logger.fdebug("Renaming " + os.path.join(self.nzb_folder, str(ofilename)) + " ..to.. " + os.path.join(self.nzb_folder,str(nfilename + ext)))
os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
src = os.path.join(self.nzb_folder, str(nfilename + ext))
logger.fdebug("Moving " + src + " ... to ... " + dst)
try:
shutil.move(src, dst)
except (OSError, IOError):
logger.fdebug("Failed to move directory - check directories and manually re-run.")
logger.fdebug("Post-Processing ABORTED.")
return
logger.fdebug("Successfully moved to : " + dst)
#tidyup old path
#try:
# os.remove(os.path.join(self.nzb_folder, str(ofilename)))
# logger.fdebug("Deleting : " + os.path.join(self.nzb_folder, str(ofilename)))
#except (OSError, IOError):
# logger.fdebug("Failed to remove temporary directory - check directory and manually re-run.")
# logger.fdebug("Post-Processing ABORTED.")
# return
#logger.fdebug("Removed temporary directory : " + str(self.nzb_folder))
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
#update snatched table to change status to Downloaded
if annchk == "no":
updater.foundsearch(comicid, issueid, down='True')
else:
updater.foundsearch(comicid, issueid, mode='want_ann', down='True')
#force rescan of files
updater.forceRescan(comicid)
logger.info(u"Post-Processing completed for: " + series + " issue: " + str(issuenumOG) )
self._log(u"Post Processing SUCCESSFULL! ", logger.DEBUG)
if ml is not None:
return
else:
if mylar.PROWL_ENABLED:
pushmessage = series + '(' + issueyear + ') - issue #' + issuenumOG
logger.info(u"Prowl request")
prowl = notifiers.PROWL()
prowl.notify(pushmessage,"Download and Postprocessing completed")
if mylar.NMA_ENABLED:
nma = notifiers.NMA()
nma.notify(series, str(issueyear), str(issuenumOG))
if mylar.PUSHOVER_ENABLED:
pushmessage = series + ' (' + str(issueyear) + ') - issue #' + str(issuenumOG)
logger.info(u"Pushover request")
pushover = notifiers.PUSHOVER()
pushover.notify(pushmessage, "Download and Post-Processing completed")
if mylar.BOXCAR_ENABLED:
boxcar = notifiers.BOXCAR()
boxcar.notify(series, str(issueyear), str(issuenumOG))
# retrieve/create the corresponding comic objects
if mylar.ENABLE_EXTRA_SCRIPTS:
folderp = str(dst) #folder location after move/rename
nzbn = self.nzb_name #original nzb name
filen = str(nfilename + ext) #new filename
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_extra_scripts(nzbname, self.nzb_folder, filen, folderp, seriesmetadata )
return self.log
def GCDimport(gcomicid):
# this is for importing via GCD only and not using CV.
# used when volume spanning is discovered for a Comic (and can't be added using CV).
# Issue Counts are wrong (and can't be added).
# because Comicvine ComicID and GCD ComicID could be identical at some random point, let's distinguish.
# CV = comicid, GCD = gcomicid :) (ie. CV=2740, GCD=G3719)
gcdcomicid = gcomicid
myDB = db.DBConnection()
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ComicID": gcdcomicid}
comic = myDB.action(
"SELECT ComicName, ComicYear, Total, ComicPublished, ComicImage, ComicLocation FROM comics WHERE ComicID=?",
[gcomicid],
).fetchone()
ComicName = comic[0]
ComicYear = comic[1]
ComicIssues = comic[2]
comlocation = comic[5]
# ComicImage = comic[4]
# print ("Comic:" + str(ComicName))
newValueDict = {"Status": "Loading"}
myDB.upsert("comics", newValueDict, controlValueDict)
# we need to lookup the info for the requested ComicID in full now
# comic = cv.getComic(comicid,'comic')
if not comic:
logger.warn("Error fetching comic. ID for : " + gcdcomicid)
if dbcomic is None:
newValueDict = {"ComicName": "Fetch failed, try refreshing. (%s)" % (gcdcomicid), "Status": "Active"}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("comics", newValueDict, controlValueDict)
return
if ComicName.startswith("The "):
sortname = ComicName[4:]
else:
sortname = ComicName
logger.info(u"Now adding/updating: " + ComicName)
# --Now that we know ComicName, let's try some scraping
# --Start
# gcd will return issue details (most importantly publishing date)
comicid = gcomicid[1:]
resultURL = "/series/" + str(comicid) + "/"
gcdinfo = parseit.GCDdetails(
comseries=None,
resultURL=resultURL,
vari_loop=0,
ComicID=gcdcomicid,
TotalIssues=ComicIssues,
issvariation=None,
resultPublished=None,
)
if gcdinfo == "No Match":
logger.warn("No matching result found for " + ComicName + " (" + ComicYear + ")")
updater.no_searchresults(gcomicid)
nomatch = "true"
return nomatch
logger.info(u"Sucessfully retrieved details for " + ComicName)
# print ("Series Published" + parseit.resultPublished)
# --End
ComicImage = gcdinfo["ComicImage"]
# comic book location on machine
# setup default location here
if comlocation is None:
if ":" in ComicName or "/" in ComicName or "," in ComicName:
comicdir = ComicName
if ":" in comicdir:
comicdir = comicdir.replace(":", "")
if "/" in comicdir:
comicdir = comicdir.replace("/", "-")
if "," in comicdir:
comicdir = comicdir.replace(",", "")
else:
comicdir = ComicName
comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + ComicYear + ")"
if mylar.DESTINATION_DIR == "":
logger.error(u"There is no general directory specified - please specify in Config/Post-Processing.")
return
if mylar.REPLACE_SPACES:
# mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
comlocation = comlocation.replace(" ", mylar.REPLACE_CHAR)
# if it doesn't exist - create it (otherwise will bugger up later on)
if os.path.isdir(str(comlocation)):
logger.info(u"Directory (" + str(comlocation) + ") already exists! Continuing...")
else:
# print ("Directory doesn't exist!")
try:
os.makedirs(str(comlocation))
logger.info(u"Directory successfully created at: " + str(comlocation))
except OSError:
logger.error(u"Could not create comicdir : " + str(comlocation))
comicIssues = gcdinfo["totalissues"]
# let's download the image...
if os.path.exists(mylar.CACHE_DIR):
pass
else:
# let's make the dir.
try:
os.makedirs(str(mylar.CACHE_DIR))
logger.info(u"Cache Directory successfully created at: " + str(mylar.CACHE_DIR))
except OSError:
logger.error(u"Could not create cache dir : " + str(mylar.CACHE_DIR))
coverfile = mylar.CACHE_DIR + "/" + str(gcomicid) + ".jpg"
urllib.urlretrieve(str(ComicImage), str(coverfile))
try:
with open(str(coverfile)) as f:
ComicImage = "cache/" + str(gcomicid) + ".jpg"
logger.info(u"Sucessfully retrieved cover for " + str(ComicName))
except IOError as e:
logger.error(u"Unable to save cover locally at this time.")
controlValueDict = {"ComicID": gcomicid}
newValueDict = {
"ComicName": ComicName,
"ComicSortName": sortname,
"ComicYear": ComicYear,
"Total": comicIssues,
"ComicLocation": comlocation,
"ComicImage": ComicImage,
# "ComicPublisher": comic['ComicPublisher'],
# "ComicPublished": comicPublished,
"DateAdded": helpers.today(),
"Status": "Loading",
}
myDB.upsert("comics", newValueDict, controlValueDict)
logger.info(u"Sucessfully retrieved issue details for " + ComicName)
n = 0
iscnt = int(comicIssues)
issnum = []
issname = []
issdate = []
int_issnum = []
# let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :)
latestiss = "0"
latestdate = "0000-00-00"
# print ("total issues:" + str(iscnt))
# ---removed NEW code here---
logger.info(u"Now adding/updating issues for " + ComicName)
bb = 0
while bb <= iscnt:
# ---NEW.code
try:
gcdval = gcdinfo["gcdchoice"][bb]
# print ("gcdval: " + str(gcdval))
except IndexError:
# account for gcd variation here
if gcdinfo["gcdvariation"] == "gcd":
# print ("gcd-variation accounted for.")
issdate = "0000-00-00"
int_issnum = int(issis / 1000)
break
if "nn" in str(gcdval["GCDIssue"]):
# no number detected - GN, TP or the like
logger.warn(u"Non Series detected (Graphic Novel, etc) - cannot proceed at this time.")
updater.no_searchresults(comicid)
return
elif "." in str(gcdval["GCDIssue"]):
issst = str(gcdval["GCDIssue"]).find(".")
issb4dec = str(gcdval["GCDIssue"])[:issst]
# if the length of decimal is only 1 digit, assume it's a tenth
decis = str(gcdval["GCDIssue"])[issst + 1 :]
if len(decis) == 1:
decisval = int(decis) * 10
issaftdec = str(decisval)
if len(decis) == 2:
decisval = int(decis)
issaftdec = str(decisval)
if int(issaftdec) == 0:
issaftdec = "00"
gcd_issue = issb4dec + "." + issaftdec
gcdis = (int(issb4dec) * 1000) + decisval
else:
gcdis = int(str(gcdval["GCDIssue"])) * 1000
gcd_issue = str(gcdval["GCDIssue"])
# get the latest issue / date using the date.
int_issnum = int(gcdis / 1000)
issdate = str(gcdval["GCDDate"])
issid = "G" + str(gcdval["IssueID"])
if gcdval["GCDDate"] > latestdate:
latestiss = str(gcd_issue)
latestdate = str(gcdval["GCDDate"])
# print("(" + str(bb) + ") IssueID: " + str(issid) + " IssueNo: " + str(gcd_issue) + " Date" + str(issdate) )
# ---END.NEW.
# check if the issue already exists
iss_exists = myDB.action("SELECT * from issues WHERE IssueID=?", [issid]).fetchone()
# Only change the status & add DateAdded if the issue is not already in the database
if iss_exists is None:
newValueDict["DateAdded"] = helpers.today()
# adjust for inconsistencies in GCD date format - some dates have ? which borks up things.
if "?" in str(issdate):
issdate = "0000-00-00"
controlValueDict = {"IssueID": issid}
newValueDict = {
"ComicID": gcomicid,
"ComicName": ComicName,
"Issue_Number": gcd_issue,
"IssueDate": issdate,
"Int_IssueNumber": int_issnum,
}
# print ("issueid:" + str(controlValueDict))
# print ("values:" + str(newValueDict))
if mylar.AUTOWANT_ALL:
newValueDict["Status"] = "Wanted"
# elif release_dict['releasedate'] > helpers.today() and mylar.AUTOWANT_UPCOMING:
# newValueDict['Status'] = "Wanted"
else:
newValueDict["Status"] = "Skipped"
if iss_exists:
# print ("Existing status : " + str(iss_exists['Status']))
newValueDict["Status"] = iss_exists["Status"]
myDB.upsert("issues", newValueDict, controlValueDict)
bb += 1
# logger.debug(u"Updating comic cache for " + ComicName)
# cache.getThumb(ComicID=issue['issueid'])
# logger.debug(u"Updating cache for: " + ComicName)
# cache.getThumb(ComicIDcomicid)
# check for existing files...
updater.forceRescan(gcomicid)
controlValueStat = {"ComicID": gcomicid}
newValueStat = {
"Status": "Active",
"LatestIssue": latestiss,
"LatestDate": latestdate,
"LastUpdated": helpers.now(),
}
myDB.upsert("comics", newValueStat, controlValueStat)
logger.info(u"Updating complete for: " + ComicName)
# lets' check the pullist for anyting at this time as well since we're here.
if mylar.AUTOWANT_UPCOMING:
logger.info(u"Checking this week's pullist for new issues of " + str(ComicName))
updater.newpullcheck()
# here we grab issues that have been marked as wanted above...
results = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [gcomicid])
if results:
logger.info(u"Attempting to grab wanted issues for : " + ComicName)
for result in results:
foundNZB = "none"
if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL) and (mylar.SAB_HOST):
foundNZB = search.searchforissue(result["IssueID"])
if foundNZB == "yes":
updater.foundsearch(result["ComicID"], result["IssueID"])
else:
logger.info(u"No issues marked as wanted for " + ComicName)
logger.info(u"Finished grabbing what I could.")
def addComictoDB(comicid, mismatch=None):
# Putting this here to get around the circular import. Will try to use this to update images at later date.
from mylar import cache
myDB = db.DBConnection()
# We need the current minimal info in the database instantly
# so we don't throw a 500 error when we redirect to the artistPage
controlValueDict = {"ComicID": comicid}
dbcomic = myDB.action("SELECT * FROM comics WHERE ComicID=?", [comicid]).fetchone()
if dbcomic is None:
newValueDict = {"ComicName": "Comic ID: %s" % (comicid), "Status": "Loading"}
comlocation = None
else:
newValueDict = {"Status": "Loading"}
comlocation = dbcomic["ComicLocation"]
myDB.upsert("comics", newValueDict, controlValueDict)
# we need to lookup the info for the requested ComicID in full now
comic = cv.getComic(comicid, "comic")
# comic = myDB.action('SELECT * FROM comics WHERE ComicID=?', [comicid]).fetchone()
if not comic:
logger.warn("Error fetching comic. ID for : " + comicid)
if dbcomic is None:
newValueDict = {"ComicName": "Fetch failed, try refreshing. (%s)" % (comicid), "Status": "Active"}
else:
newValueDict = {"Status": "Active"}
myDB.upsert("comics", newValueDict, controlValueDict)
return
if comic["ComicName"].startswith("The "):
sortname = comic["ComicName"][4:]
else:
sortname = comic["ComicName"]
logger.info(u"Now adding/updating: " + comic["ComicName"])
# --Now that we know ComicName, let's try some scraping
# --Start
# gcd will return issue details (most importantly publishing date)
if mismatch == "no" or mismatch is None:
gcdinfo = parseit.GCDScraper(comic["ComicName"], comic["ComicYear"], comic["ComicIssues"], comicid)
mismatch_com = "no"
if gcdinfo == "No Match":
updater.no_searchresults(comicid)
nomatch = "true"
logger.info(
u"There was an error when trying to add " + comic["ComicName"] + " (" + comic["ComicYear"] + ")"
)
return nomatch
else:
mismatch_com = "yes"
# print ("gcdinfo:" + str(gcdinfo))
elif mismatch == "yes":
CV_EXcomicid = myDB.action("SELECT * from exceptions WHERE ComicID=?", [comicid]).fetchone()
if CV_EXcomicid["variloop"] is None:
pass
else:
vari_loop = CV_EXcomicid["variloop"]
NewComicID = CV_EXcomicid["NewComicID"]
gcomicid = CV_EXcomicid["GComicID"]
resultURL = "/series/" + str(NewComicID) + "/"
# print ("variloop" + str(CV_EXcomicid['variloop']))
# if vari_loop == '99':
gcdinfo = parseit.GCDdetails(
comseries=None,
resultURL=resultURL,
vari_loop=0,
ComicID=comicid,
TotalIssues=0,
issvariation="no",
resultPublished=None,
)
logger.info(u"Sucessfully retrieved details for " + comic["ComicName"])
# print ("Series Published" + parseit.resultPublished)
# comic book location on machine
# setup default location here
if comlocation is None:
if ":" in comic["ComicName"] or "/" in comic["ComicName"] or "," in comic["ComicName"]:
comicdir = comic["ComicName"]
if ":" in comicdir:
comicdir = comicdir.replace(":", "")
if "/" in comicdir:
comicdir = comicdir.replace("/", "-")
if "," in comicdir:
comicdir = comicdir.replace(",", "")
else:
comicdir = comic["ComicName"]
series = comicdir
publisher = comic["ComicPublisher"]
year = comic["ComicYear"]
# do work to generate folder path
values = {"$Series": series, "$Publisher": publisher, "$Year": year}
# print mylar.FOLDER_FORMAT
# print 'working dir:'
# print helpers.replace_all(mylar.FOLDER_FORMAT, values)
if mylar.FOLDER_FORMAT == "":
comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + comic["ComicYear"] + ")"
else:
comlocation = mylar.DESTINATION_DIR + "/" + helpers.replace_all(mylar.FOLDER_FORMAT, values)
# comlocation = mylar.DESTINATION_DIR + "/" + comicdir + " (" + comic['ComicYear'] + ")"
if mylar.DESTINATION_DIR == "":
logger.error(u"There is no general directory specified - please specify in Config/Post-Processing.")
return
if mylar.REPLACE_SPACES:
# mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
comlocation = comlocation.replace(" ", mylar.REPLACE_CHAR)
# if it doesn't exist - create it (otherwise will bugger up later on)
if os.path.isdir(str(comlocation)):
logger.info(u"Directory (" + str(comlocation) + ") already exists! Continuing...")
else:
# print ("Directory doesn't exist!")
try:
os.makedirs(str(comlocation))
logger.info(u"Directory successfully created at: " + str(comlocation))
except OSError:
logger.error(u"Could not create comicdir : " + str(comlocation))
# try to account for CV not updating new issues as fast as GCD
# seems CV doesn't update total counts
# comicIssues = gcdinfo['totalissues']
if gcdinfo["gcdvariation"] == "cv":
comicIssues = str(int(comic["ComicIssues"]) + 1)
else:
comicIssues = comic["ComicIssues"]
# let's download the image...
if os.path.exists(mylar.CACHE_DIR):
pass
else:
# let's make the dir.
try:
os.makedirs(str(mylar.CACHE_DIR))
logger.info(u"Cache Directory successfully created at: " + str(mylar.CACHE_DIR))
except OSError:
logger.error("Could not create cache dir. Check permissions of cache dir: " + str(mylar.CACHE_DIR))
coverfile = mylar.CACHE_DIR + "/" + str(comicid) + ".jpg"
# try:
urllib.urlretrieve(str(comic["ComicImage"]), str(coverfile))
try:
with open(str(coverfile)) as f:
ComicImage = "cache/" + str(comicid) + ".jpg"
logger.info(u"Sucessfully retrieved cover for " + str(comic["ComicName"]))
except IOError as e:
logger.error(u"Unable to save cover locally at this time.")
controlValueDict = {"ComicID": comicid}
newValueDict = {
"ComicName": comic["ComicName"],
"ComicSortName": sortname,
"ComicYear": comic["ComicYear"],
"ComicImage": ComicImage,
"Total": comicIssues,
"ComicLocation": comlocation,
"ComicPublisher": comic["ComicPublisher"],
"ComicPublished": gcdinfo["resultPublished"],
"DateAdded": helpers.today(),
"Status": "Loading",
}
myDB.upsert("comics", newValueDict, controlValueDict)
issued = cv.getComic(comicid, "issue")
logger.info(u"Sucessfully retrieved issue details for " + comic["ComicName"])
n = 0
iscnt = int(comicIssues)
issid = []
issnum = []
issname = []
issdate = []
int_issnum = []
# let's start issue #'s at 0 -- thanks to DC for the new 52 reboot! :)
latestiss = "0"
latestdate = "0000-00-00"
# print ("total issues:" + str(iscnt))
# ---removed NEW code here---
logger.info(u"Now adding/updating issues for " + comic["ComicName"])
# file check to see if issue exists
logger.info(u"Checking directory for existing issues.")
# fc = filechecker.listFiles(dir=comlocation, watchcomic=comic['ComicName'])
# havefiles = 0
# fccnt = int(fc['comiccount'])
# logger.info(u"Found " + str(fccnt) + "/" + str(iscnt) + " issues of " + comic['ComicName'] + "...verifying")
# fcnew = []
while n <= iscnt:
# ---NEW.code
try:
firstval = issued["issuechoice"][n]
except IndexError:
break
cleanname = helpers.cleanName(firstval["Issue_Name"])
issid = str(firstval["Issue_ID"])
issnum = str(firstval["Issue_Number"])
issname = cleanname
if "." in str(issnum):
issn_st = str(issnum).find(".")
issn_b4dec = str(issnum)[:issn_st]
# if the length of decimal is only 1 digit, assume it's a tenth
dec_is = str(issnum)[issn_st + 1 :]
if len(dec_is) == 1:
dec_nisval = int(dec_is) * 10
iss_naftdec = str(dec_nisval)
if len(dec_is) == 2:
dec_nisval = int(dec_is)
iss_naftdec = str(dec_nisval)
iss_issue = issn_b4dec + "." + iss_naftdec
issis = (int(issn_b4dec) * 1000) + dec_nisval
else:
issis = int(issnum) * 1000
bb = 0
while bb <= iscnt:
try:
gcdval = gcdinfo["gcdchoice"][bb]
except IndexError:
# account for gcd variation here
if gcdinfo["gcdvariation"] == "gcd":
# print ("gcd-variation accounted for.")
issdate = "0000-00-00"
int_issnum = int(issis / 1000)
break
if "nn" in str(gcdval["GCDIssue"]):
# no number detected - GN, TP or the like
logger.warn(u"Non Series detected (Graphic Novel, etc) - cannot proceed at this time.")
updater.no_searchresults(comicid)
return
elif "." in str(gcdval["GCDIssue"]):
# print ("g-issue:" + str(gcdval['GCDIssue']))
issst = str(gcdval["GCDIssue"]).find(".")
# print ("issst:" + str(issst))
issb4dec = str(gcdval["GCDIssue"])[:issst]
# print ("issb4dec:" + str(issb4dec))
# if the length of decimal is only 1 digit, assume it's a tenth
decis = str(gcdval["GCDIssue"])[issst + 1 :]
# print ("decis:" + str(decis))
if len(decis) == 1:
decisval = int(decis) * 10
issaftdec = str(decisval)
if len(decis) == 2:
decisval = int(decis)
issaftdec = str(decisval)
gcd_issue = issb4dec + "." + issaftdec
# print ("gcd_issue:" + str(gcd_issue))
gcdis = (int(issb4dec) * 1000) + decisval
else:
gcdis = int(str(gcdval["GCDIssue"])) * 1000
if gcdis == issis:
issdate = str(gcdval["GCDDate"])
int_issnum = int(gcdis / 1000)
# get the latest issue / date using the date.
if gcdval["GCDDate"] > latestdate:
latestiss = str(issnum)
latestdate = str(gcdval["GCDDate"])
break
# bb = iscnt
bb += 1
# print("(" + str(n) + ") IssueID: " + str(issid) + " IssueNo: " + str(issnum) + " Date" + str(issdate))
# ---END.NEW.
# check if the issue already exists
iss_exists = myDB.action("SELECT * from issues WHERE IssueID=?", [issid]).fetchone()
# Only change the status & add DateAdded if the issue is already in the database
if iss_exists is None:
newValueDict["DateAdded"] = helpers.today()
controlValueDict = {"IssueID": issid}
newValueDict = {
"ComicID": comicid,
"ComicName": comic["ComicName"],
"IssueName": issname,
"Issue_Number": issnum,
"IssueDate": issdate,
"Int_IssueNumber": int_issnum,
}
if mylar.AUTOWANT_ALL:
newValueDict["Status"] = "Wanted"
# elif release_dict['releasedate'] > helpers.today() and mylar.AUTOWANT_UPCOMING:
# newValueDict['Status'] = "Wanted"
else:
newValueDict["Status"] = "Skipped"
if iss_exists:
# print ("Existing status : " + str(iss_exists['Status']))
newValueDict["Status"] = iss_exists["Status"]
myDB.upsert("issues", newValueDict, controlValueDict)
n += 1
# logger.debug(u"Updating comic cache for " + comic['ComicName'])
# cache.getThumb(ComicID=issue['issueid'])
# logger.debug(u"Updating cache for: " + comic['ComicName'])
# cache.getThumb(ComicIDcomicid)
# check for existing files...
updater.forceRescan(comicid)
controlValueStat = {"ComicID": comicid}
newValueStat = {
"Status": "Active",
"LatestIssue": latestiss,
"LatestDate": latestdate,
"LastUpdated": helpers.now(),
}
myDB.upsert("comics", newValueStat, controlValueStat)
logger.info(u"Updating complete for: " + comic["ComicName"])
# lets' check the pullist for anyting at this time as well since we're here.
if mylar.AUTOWANT_UPCOMING:
logger.info(u"Checking this week's pullist for new issues of " + str(comic["ComicName"]))
updater.newpullcheck()
# here we grab issues that have been marked as wanted above...
results = myDB.select("SELECT * FROM issues where ComicID=? AND Status='Wanted'", [comicid])
if results:
logger.info(u"Attempting to grab wanted issues for : " + comic["ComicName"])
for result in results:
foundNZB = "none"
if (mylar.NZBSU or mylar.DOGNZB or mylar.EXPERIMENTAL) and (mylar.SAB_HOST):
foundNZB = search.searchforissue(result["IssueID"])
if foundNZB == "yes":
updater.foundsearch(result["ComicID"], result["IssueID"])
else:
logger.info(u"No issues marked as wanted for " + comic["ComicName"])
logger.info(u"Finished grabbing what I could.")
def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None):
if cron and not mylar.LIBRARYSCAN:
return
if not dir:
dir = mylar.COMIC_DIR
# If we're appending a dir, it's coming from the post processor which is
# already bytestring
if not append:
dir = dir.encode(mylar.SYS_ENCODING)
if not os.path.isdir(dir):
logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(mylar.SYS_ENCODING, 'replace'))
return
logger.info('Scanning comic directory: %s' % dir.decode(mylar.SYS_ENCODING, 'replace'))
basedir = dir
comic_list = []
comiccnt = 0
extensions = ('cbr','cbz')
for r,d,f in os.walk(dir):
#for directory in d[:]:
# if directory.startswith("."):
# d.remove(directory)
for files in f:
if any(files.lower().endswith('.' + x.lower()) for x in extensions):
comic = files
comicpath = os.path.join(r, files)
comicsize = os.path.getsize(comicpath)
print "Comic: " + comic
print "Comic Path: " + comicpath
print "Comic Size: " + str(comicsize)
# We need the unicode path to use for logging, inserting into database
unicode_comic_path = comicpath.decode(mylar.SYS_ENCODING, 'replace')
comiccnt+=1
comic_dict = { 'ComicFilename': comic,
'ComicLocation': comicpath,
'ComicSize': comicsize,
'Unicode_ComicLocation': unicode_comic_path }
comic_list.append(comic_dict)
logger.info("I've found a total of " + str(comiccnt) + " comics....analyzing now")
logger.info("comiclist: " + str(comic_list))
myDB = db.DBConnection()
#let's load in the watchlist to see if we have any matches.
logger.info("loading in the watchlist to see if a series is being watched already...")
watchlist = myDB.select("SELECT * from comics")
ComicName = []
DisplayName = []
ComicYear = []
ComicPublisher = []
ComicTotal = []
ComicID = []
ComicLocation = []
AltName = []
watchcnt = 0
watch_kchoice = []
watchchoice = {}
import_by_comicids = []
import_comicids = {}
for watch in watchlist:
watchdisplaycomic = re.sub('[\_\#\,\/\:\;\!\$\%\&\+\'\?\@]', ' ', watch['ComicName']).encode('utf-8').strip()
# let's clean up the name, just in case for comparison purposes...
watchcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ', watch['ComicName']).encode('utf-8').strip()
#watchcomic = re.sub('\s+', ' ', str(watchcomic)).strip()
if ' the ' in watchcomic.lower():
#drop the 'the' from the watchcomic title for proper comparisons.
watchcomic = watchcomic[-4:]
alt_chk = "no" # alt-checker flag (default to no)
# account for alternate names as well
if watch['AlternateSearch'] is not None and watch['AlternateSearch'] is not 'None':
altcomic = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ', watch['AlternateSearch']).encode('utf-8').strip()
#altcomic = re.sub('\s+', ' ', str(altcomic)).strip()
AltName.append(altcomic)
alt_chk = "yes" # alt-checker flag
ComicName.append(watchcomic)
DisplayName.append(watchdisplaycomic)
ComicYear.append(watch['ComicYear'])
ComicPublisher.append(watch['ComicPublisher'])
ComicTotal.append(watch['Total'])
ComicID.append(watch['ComicID'])
ComicLocation.append(watch['ComicLocation'])
watchcnt+=1
logger.info("Successfully loaded " + str(watchcnt) + " series from your watchlist.")
ripperlist=['digital-',
'empire',
'dcp']
watchfound = 0
datelist = ['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec']
# datemonth = {'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9,'ten':10,'eleven':$
# #search for number as text, and change to numeric
# for numbs in basnumbs:
# #print ("numbs:" + str(numbs))
# if numbs in ComicName.lower():
# numconv = basnumbs[numbs]
# #print ("numconv: " + str(numconv))
for i in comic_list:
print i['ComicFilename']
comfilename = i['ComicFilename']
comlocation = i['ComicLocation']
#let's clean up the filename for matching purposes
cfilename = re.sub('[\_\#\,\/\:\;\-\!\$\%\&\+\'\?\@]', ' ', comfilename)
#cfilename = re.sub('\s', '_', str(cfilename))
d_filename = re.sub('[\_\#\,\/\;\!\$\%\&\?\@]', ' ', comfilename)
d_filename = re.sub('[\:\-\+\']', '#', d_filename)
#versioning - remove it
subsplit = cfilename.replace('_', ' ').split()
volno = None
volyr = None
for subit in subsplit:
if subit[0].lower() == 'v':
vfull = 0
if subit[1:].isdigit():
#if in format v1, v2009 etc...
if len(subit) > 3:
# if it's greater than 3 in length, then the format is Vyyyy
vfull = 1 # add on 1 character length to account for extra space
cfilename = re.sub(subit, '', cfilename)
d_filename = re.sub(subit, '', d_filename)
volno = re.sub("[^0-9]", " ", subit)
elif subit.lower()[:3] == 'vol':
#if in format vol.2013 etc
#because the '.' in Vol. gets removed, let's loop thru again after the Vol hit to remove it entirely
logger.fdebug('volume indicator detected as version #:' + str(subit))
cfilename = re.sub(subit, '', cfilename)
cfilename = " ".join(cfilename.split())
d_filename = re.sub(subit, '', d_filename)
d_filename = " ".join(d_filename.split())
volyr = re.sub("[^0-9]", " ", subit).strip()
logger.fdebug('volume year set as : ' + str(volyr))
cm_cn = 0
#we need to track the counter to make sure we are comparing the right array parts
#this takes care of the brackets :)
m = re.findall('[^()]+', cfilename)
lenm = len(m)
logger.fdebug("there are " + str(lenm) + " words.")
cnt = 0
yearmatch = "false"
foundonwatch = "False"
issue = 999999
while (cnt < lenm):
if m[cnt] is None: break
if m[cnt] == ' ':
pass
else:
logger.fdebug(str(cnt) + ". Bracket Word: " + m[cnt])
if cnt == 0:
comic_andiss = m[cnt]
logger.fdebug("Comic: " + comic_andiss)
# if it's not in the standard format this will bork.
# let's try to accomodate (somehow).
# first remove the extension (if any)
extensions = ('cbr', 'cbz')
if comic_andiss.lower().endswith(extensions):
comic_andiss = comic_andiss[:-4]
logger.fdebug("removed extension from filename.")
#now we have to break up the string regardless of formatting.
#let's force the spaces.
comic_andiss = re.sub('_', ' ', comic_andiss)
cs = comic_andiss.split()
cs_len = len(cs)
cn = ''
ydetected = 'no'
idetected = 'no'
decimaldetect = 'no'
for i in reversed(xrange(len(cs))):
#start at the end.
logger.fdebug("word: " + str(cs[i]))
#assume once we find issue - everything prior is the actual title
#idetected = no will ignore everything so it will assume all title
if cs[i][:-2] == '19' or cs[i][:-2] == '20' and idetected == 'no':
logger.fdebug("year detected: " + str(cs[i]))
ydetected = 'yes'
result_comyear = cs[i]
elif cs[i].isdigit() and idetected == 'no' or '.' in cs[i]:
issue = cs[i]
logger.fdebug("issue detected : " + str(issue))
idetected = 'yes'
if '.' in cs[i]:
#make sure it's a number on either side of decimal and assume decimal issue.
decst = cs[i].find('.')
dec_st = cs[i][:decst]
dec_en = cs[i][decst+1:]
logger.fdebug("st: " + str(dec_st))
logger.fdebug("en: " + str(dec_en))
if dec_st.isdigit() and dec_en.isdigit():
logger.fdebug("decimal issue detected...adjusting.")
issue = dec_st + "." + dec_en
logger.fdebug("issue detected: " + str(issue))
idetected = 'yes'
else:
logger.fdebug("false decimal represent. Chunking to extra word.")
cn = cn + cs[i] + " "
break
elif '\#' in cs[i] or decimaldetect == 'yes':
logger.fdebug("issue detected: " + str(cs[i]))
idetected = 'yes'
else: cn = cn + cs[i] + " "
if ydetected == 'no':
#assume no year given in filename...
result_comyear = "0000"
logger.fdebug("cm?: " + str(cn))
if issue is not '999999':
comiss = issue
else:
logger.ERROR("Invalid Issue number (none present) for " + comfilename)
break
cnsplit = cn.split()
cname = ''
findcn = 0
while (findcn < len(cnsplit)):
cname = cname + cs[findcn] + " "
findcn+=1
cname = cname[:len(cname)-1] # drop the end space...
print ("assuming name is : " + cname)
com_NAME = cname
print ("com_NAME : " + com_NAME)
yearmatch = "True"
else:
logger.fdebug('checking ' + m[cnt])
# we're assuming that the year is in brackets (and it should be damnit)
if m[cnt][:-2] == '19' or m[cnt][:-2] == '20':
print ("year detected: " + str(m[cnt]))
ydetected = 'yes'
result_comyear = m[cnt]
elif m[cnt][:3].lower() in datelist:
logger.fdebug('possible issue date format given - verifying')
#if the date of the issue is given as (Jan 2010) or (January 2010) let's adjust.
#keeping in mind that ',' and '.' are already stripped from the string
if m[cnt][-4:].isdigit():
ydetected = 'yes'
result_comyear = m[cnt][-4:]
logger.fdebug('Valid Issue year of ' + str(result_comyear) + 'detected in format of ' + str(m[cnt]))
cnt+=1
displength = len(cname)
print 'd_filename is : ' + d_filename
charcount = d_filename.count('#')
print ('charcount is : ' + str(charcount))
if charcount > 0:
print ('entering loop')
for i,m in enumerate(re.finditer('\#', d_filename)):
if m.end() <= displength:
print comfilename[m.start():m.end()]
# find occurance in c_filename, then replace into d_filname so special characters are brought across
newchar = comfilename[m.start():m.end()]
print 'newchar:' + str(newchar)
d_filename = d_filename[:m.start()] + str(newchar) + d_filename[m.end():]
print 'd_filename:' + str(d_filename)
dispname = d_filename[:displength]
print ('dispname : ' + dispname)
splitit = []
watchcomic_split = []
logger.fdebug("filename comic and issue: " + comic_andiss)
#changed this from '' to ' '
comic_iss_b4 = re.sub('[\-\:\,]', ' ', comic_andiss)
comic_iss = comic_iss_b4.replace('.',' ')
comic_iss = re.sub('[\s+]', ' ', comic_iss).strip()
logger.fdebug("adjusted comic and issue: " + str(comic_iss))
#remove 'the' from here for proper comparisons.
if ' the ' in comic_iss.lower():
comic_iss = comic_iss[-4:]
splitit = comic_iss.split(None)
logger.fdebug("adjusting from: " + str(comic_iss_b4) + " to: " + str(comic_iss))
#here we cycle through the Watchlist looking for a match.
while (cm_cn < watchcnt):
#setup the watchlist
comname = ComicName[cm_cn]
print ("watch_comic:" + comname)
comyear = ComicYear[cm_cn]
compub = ComicPublisher[cm_cn]
comtotal = ComicTotal[cm_cn]
comicid = ComicID[cm_cn]
watch_location = ComicLocation[cm_cn]
# there shouldn't be an issue in the comic now, so let's just assume it's all gravy.
splitst = len(splitit)
watchcomic_split = helpers.cleanName(comname)
watchcomic_split = re.sub('[\-\:\,\.]', ' ', watchcomic_split).split(None)
logger.fdebug(str(splitit) + " file series word count: " + str(splitst))
logger.fdebug(str(watchcomic_split) + " watchlist word count: " + str(len(watchcomic_split)))
if (splitst) != len(watchcomic_split):
logger.fdebug("incorrect comic lengths...not a match")
# if str(splitit[0]).lower() == "the":
# logger.fdebug("THE word detected...attempting to adjust pattern matching")
# splitit[0] = splitit[4:]
else:
logger.fdebug("length match..proceeding")
n = 0
scount = 0
logger.fdebug("search-length: " + str(splitst))
logger.fdebug("Watchlist-length: " + str(len(watchcomic_split)))
while ( n <= (splitst)-1 ):
logger.fdebug("splitit: " + str(splitit[n]))
if n < (splitst) and n < len(watchcomic_split):
logger.fdebug(str(n) + " Comparing: " + str(watchcomic_split[n]) + " .to. " + str(splitit[n]))
if '+' in watchcomic_split[n]:
watchcomic_split[n] = re.sub('+', '', str(watchcomic_split[n]))
if str(watchcomic_split[n].lower()) in str(splitit[n].lower()) and len(watchcomic_split[n]) >= len(splitit[n]):
logger.fdebug("word matched on : " + str(splitit[n]))
scount+=1
#elif ':' in splitit[n] or '-' in splitit[n]:
# splitrep = splitit[n].replace('-', '')
# print ("non-character keyword...skipped on " + splitit[n])
elif str(splitit[n]).lower().startswith('v'):
logger.fdebug("possible versioning..checking")
#we hit a versioning # - account for it
if splitit[n][1:].isdigit():
comicversion = str(splitit[n])
logger.fdebug("version found: " + str(comicversion))
else:
logger.fdebug("Comic / Issue section")
if splitit[n].isdigit():
logger.fdebug("issue detected")
else:
logger.fdebug("non-match for: "+ str(splitit[n]))
pass
n+=1
#set the match threshold to 80% (for now)
# if it's less than 80% consider it a non-match and discard.
#splitit has to splitit-1 because last position is issue.
wordcnt = int(scount)
logger.fdebug("scount:" + str(wordcnt))
totalcnt = int(splitst)
logger.fdebug("splitit-len:" + str(totalcnt))
spercent = (wordcnt/totalcnt) * 100
logger.fdebug("we got " + str(spercent) + " percent.")
if int(spercent) >= 80:
logger.fdebug("it's a go captain... - we matched " + str(spercent) + "%!")
logger.fdebug("this should be a match!")
logger.fdebug("issue we found for is : " + str(comiss))
#set the year to the series we just found ;)
result_comyear = comyear
#issue comparison now as well
logger.info(u"Found " + comname + " (" + str(comyear) + ") issue: " + str(comiss))
watchmatch = str(comicid)
dispname = DisplayName[cm_cn]
foundonwatch = "True"
break
elif int(spercent) < 80:
logger.fdebug("failure - we only got " + str(spercent) + "% right!")
cm_cn+=1
if foundonwatch == "False":
watchmatch = None
#---if it's not a match - send it to the importer.
n = 0
if volyr is None:
if result_comyear is None:
result_comyear = '0000' #no year in filename basically.
else:
if result_comyear is None:
result_comyear = volyr
if volno is None:
if volyr is None:
vol_label = None
else:
vol_label = volyr
else:
vol_label = volno
print ("adding " + com_NAME + " to the import-queue!")
impid = com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
print ("impid: " + str(impid))
import_by_comicids.append({
"impid" : impid,
"watchmatch" : watchmatch,
"displayname" : dispname,
"comicname" : com_NAME,
"comicyear" : result_comyear,
"volume" : vol_label,
"comfilename" : comfilename,
"comlocation" : comlocation.decode(mylar.SYS_ENCODING)
})
if len(watch_kchoice) > 0:
watchchoice['watchlist'] = watch_kchoice
print ("watchchoice: " + str(watchchoice))
logger.info("I have found " + str(watchfound) + " out of " + str(comiccnt) + " comics for series that are being watched.")
wat = 0
comicids = []
if watchfound > 0:
if mylar.IMP_MOVE:
logger.info("You checked off Move Files...so that's what I'm going to do")
#check to see if Move Files is enabled.
#if not being moved, set the archive bit.
print("Moving files into appropriate directory")
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comlocation = watch_the_list['ComicLocation']
watch_comicid = watch_the_list['ComicID']
watch_comicname = watch_the_list['ComicName']
watch_comicyear = watch_the_list['ComicYear']
watch_comiciss = watch_the_list['ComicIssue']
print ("ComicLocation: " + str(watch_comlocation))
orig_comlocation = watch_the_list['OriginalLocation']
orig_filename = watch_the_list['OriginalFilename']
print ("Orig. Location: " + str(orig_comlocation))
print ("Orig. Filename: " + str(orig_filename))
#before moving check to see if Rename to Mylar structure is enabled.
if mylar.IMP_RENAME:
print("Renaming files according to configuration details : " + str(mylar.FILE_FORMAT))
renameit = helpers.rename_param(watch_comicid, watch_comicname, watch_comicyear, watch_comiciss)
nfilename = renameit['nfilename']
dst_path = os.path.join(watch_comlocation,nfilename)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
else:
print("Renaming files not enabled, keeping original filename(s)")
dst_path = os.path.join(watch_comlocation,orig_filename)
#os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
#src = os.path.join(, str(nfilename + ext))
print ("I'm going to move " + str(orig_comlocation) + " to .." + str(dst_path))
try:
shutil.move(orig_comlocation, dst_path)
except (OSError, IOError):
logger.info("Failed to move directory - check directories and manually re-run.")
wat+=1
else:
# if move files isn't enabled, let's set all found comics to Archive status :)
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comicid = watch_the_list['ComicID']
watch_issue = watch_the_list['ComicIssue']
print ("ComicID: " + str(watch_comicid))
print ("Issue#: " + str(watch_issue))
issuechk = myDB.selectone("SELECT * from issues where ComicID=? AND INT_IssueNumber=?", [watch_comicid, watch_issue]).fetchone()
if issuechk is None:
print ("no matching issues for this comic#")
else:
print("...Existing status: " + str(issuechk['Status']))
control = {"IssueID": issuechk['IssueID']}
values = { "Status": "Archived"}
print ("...changing status of " + str(issuechk['Issue_Number']) + " to Archived ")
myDB.upsert("issues", values, control)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
wat+=1
if comicids is None: pass
else:
c_upd = len(comicids)
c = 0
while (c < c_upd ):
print ("Rescanning.. " + str(c))
updater.forceRescan(c)
if not len(import_by_comicids):
return "Completed"
if len(import_by_comicids) > 0:
import_comicids['comic_info'] = import_by_comicids
print ("import comicids: " + str(import_by_comicids))
return import_comicids, len(import_by_comicids)
def PostProcess(nzb_name, nzb_folder):
log2screen = ""
log2screen = log2screen + "Nzb Name:" + nzb_name + "\n"
log2screen = log2screen + "Nzb Folder:" + nzb_folder + "\n"
#lookup nzb_name in nzblog table to get issueid
myDB = db.DBConnection()
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzb_name]).fetchone()
if nzbiss is None:
log2screen = log2screen + "Epic failure - could not locate file to rename." + "\n"
logger.error(u"Unable to locate downloaded file to rename. PostProcessing aborted.")
return
else:
issueid = nzbiss['IssueID']
#log2screen = log2screen + "IssueID: " + issueid + "\n"
#use issueid to get publisher, series, year, issue number
issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone()
comicid = issuenzb['ComicID']
#log2screen = log2screen + "ComicID: " + comicid + "\n"
issuenum = issuenzb['Issue_Number']
issueno = str(issuenum).split('.')[0]
log2screen = log2screen + "Issue Number: " + str(issueno) + "\n"
# issue zero-suppression here
if mylar.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00"
if str(len(issueno)) > 1:
if int(issueno) < 10:
log2screen = log2screen + "issue detected less than 10" + "\n"
prettycomiss = str(zeroadd) + str(int(issueno))
log2screen = log2screen + "Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss) + "\n"
elif int(issueno) >= 10 and int(issueno) < 100:
log2screen = log2screen + "issue detected greater than 10, but less than 100" + "\n"
if mylar.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
prettycomiss = str(zeroadd) + str(int(issueno))
log2screen = log2screen + "Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss) + "\n"
else:
log2screen = log2screen + "issue detected greater than 100" + "\n"
prettycomiss = str(issueno)
log2screen = log2screen + "Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss) + "\n"
else:
prettycomiss = str(issueno)
log2screen = log2screen + "issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss) + "\n"
issueyear = issuenzb['IssueDate'][:4]
log2screen = log2screen + "Issue Year: " + str(issueyear) + "\n"
comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
publisher = comicnzb['ComicPublisher']
log2screen = log2screen + "Publisher: " + publisher + "\n"
series = comicnzb['ComicName']
log2screen = log2screen + "Series: " + series + "\n"
seriesyear = comicnzb['ComicYear']
log2screen = log2screen + "Year: " + seriesyear + "\n"
comlocation = comicnzb['ComicLocation']
log2screen = log2screen + "Comic Location: " + comlocation + "\n"
#---move to importer.py
#get output path format
# if ':' in series:
# series = series.replace(':','')
#do work to generate folder path
# values = {'$Series': series,
# '$Publisher': publisher,
# '$Year': seriesyear
# }
# comlocation = mylar.DESTINATION_DIR + "/" + helpers.replace_all(mylar.FOLDER_FORMAT, values)
#last perform space replace
# if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
# comlocation = comlocation.replace(' ', mylar.REPLACE_CHAR)
# log2screen = log2screen + "Final Location: " + comlocation + "\n"
#---
#rename file and move to new path
#nfilename = series + " " + issueno + " (" + seriesyear + ")"
file_values = {'$Series': series,
'$Issue': prettycomiss,
'$Year': issueyear
}
extensions = ('.cbr', '.cbz')
for root, dirnames, filenames in os.walk(nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
ofilename = filename
path, ext = os.path.splitext(ofilename)
log2screen = log2screen + "Original Filename: " + ofilename + "\n"
log2screen = log2screen + "Original Extension: " + ext + "\n"
if mylar.FILE_FORMAT == '':
log2screen = log2screen + "Rename Files isn't enabled...keeping original filename." + "\n"
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
nfilename = helpers.replace_all(mylar.FILE_FORMAT, file_values)
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
#TODO - sort issue numbering 12.00 should be 12
log2screen = log2screen + "New Filename: " + nfilename + "\n"
src = nzb_folder + "/" + ofilename
dst = comlocation + "/" + nfilename + ext
log2screen = log2screen + "Source:" + src + "\n"
log2screen = log2screen + "Destination:" + dst + "\n"
os.rename(nzb_folder + "/" + ofilename, nzb_folder + "/" + nfilename + ext)
src = nzb_folder + "/" + nfilename + ext
try:
shutil.move(src, dst)
except (OSError, IOError):
log2screen = log2screen + "Failed to move directory - check directories and manually re-run." + "\n"
log2screen = log2screen + "Post-Processing ABORTED." + "\n"
return log2screen
#tidyup old path
try:
shutil.rmtree(nzb_folder)
except (OSError, IOError):
log2screen = log2screen + "Failed to remove temporary directory - check directory and manually re-run." + "\n"
log2screen = log2screen + "Post-Processing ABORTED." + "\n"
return log2screen
log2screen = log2screen + "Removed temporary directory : " + str(nzb_folder) + "\n"
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
#force rescan of files
updater.forceRescan(comicid)
logger.info(u"Post-Processing completed for: " + series + " issue: " + str(issuenum) )
log2screen = log2screen + "Post Processing SUCCESSFULL!" + "\n"
#print log2screen
return log2screen
