def weekly_update(ComicName,IssueNumber,CStatus,CID,futurepull=None):
logger.fdebug('weekly_update of table : ' + str(ComicName) + ' #:' + str(IssueNumber))
logger.fdebug('weekly_update of table : ' + str(CStatus))
# here we update status of weekly table...
# added Issue to stop false hits on series' that have multiple releases in a week
# added CStatus to update status flags on Pullist screen
myDB = db.DBConnection()
if futurepull is None:
issuecheck = myDB.action("SELECT * FROM weekly WHERE COMIC=? AND ISSUE=?", [ComicName,IssueNumber]).fetchone()
else:
issuecheck = myDB.action("SELECT * FROM future WHERE COMIC=? AND ISSUE=?", [ComicName,IssueNumber]).fetchone()
if issuecheck is not None:
controlValue = { "COMIC": str(ComicName),
"ISSUE": str(IssueNumber)}
if CStatus:
newValue = {"STATUS": CStatus,
"ComicID": CID}
else:
if mylar.AUTOWANT_UPCOMING:
newValue = {"STATUS": "Wanted"}
else:
newValue = {"STATUS": "Skipped"}
if futurepull is None:
myDB.upsert("weekly", newValue, controlValue)
else:
if issuecheck['ComicID'] is not None:
newValue = {"STATUS": "Wanted",
"ComicID": issuecheck['ComicID']}
myDB.upsert("future", newValue, controlValue)
def storyarcinfo(xmlid):
comicLibrary = listLibrary()
arcinfo = {}
if mylar.COMICVINE_API == 'None' or mylar.COMICVINE_API is None or mylar.COMICVINE_API == mylar.DEFAULT_CVAPI:
logger.warn('You have not specified your own ComicVine API key - alot of things will be limited. Get your own @ http://api.comicvine.com.')
comicapi = mylar.DEFAULT_CVAPI
else:
comicapi = mylar.COMICVINE_API
#respawn to the exact id for the story arc and count the # of issues present.
ARCPULL_URL = mylar.CVURL + 'story_arc/4045-' + str(xmlid) + '/?api_key=' + str(comicapi) + '&field_list=issues,name,first_appeared_in_issue,deck,image&format=xml&offset=0'
logger.fdebug('arcpull_url:' + str(ARCPULL_URL))
if mylar.CVAPI_COUNT == 0 or mylar.CVAPI_COUNT >= mylar.CVAPI_MAX:
chkit = cvapi_check()
if chkit == False:
return 'apireached'
try:
file = urllib2.urlopen(ARCPULL_URL)
except urllib2.HTTPError, err:
logger.error('err : ' + str(err))
logger.error('There was a major problem retrieving data from ComicVine - on their end.')
return
def rssdbupdate(feeddata, i, type):
rsschktime = 15
myDB = db.DBConnection()
# let's add the entries into the db so as to save on searches
# also to build up the ID's ;)
for dataval in feeddata:
if type == "torrent":
# we just store the torrent ID's now.
if dataval["site"] == "32P":
newlink = dataval["link"]
else:
# store the hash/id from KAT
newlink = os.path.basename(re.sub(".torrent", "", dataval["link"][: dataval["link"].find("?title")]))
newVal = {"Link": newlink, "Pubdate": dataval["pubdate"], "Site": dataval["site"], "Size": dataval["size"]}
ctrlVal = {"Title": dataval["title"]}
else:
newlink = dataval["Link"]
newVal = {"Link": newlink, "Pubdate": dataval["Pubdate"], "Site": dataval["Site"], "Size": dataval["Size"]}
ctrlVal = {"Title": dataval["Title"]}
myDB.upsert("rssdb", newVal, ctrlVal)
logger.fdebug("Completed adding new data to RSS DB. Next add in " + str(mylar.RSS_CHECKINTERVAL) + " minutes")
return
def writeconfig(self, values=None):
logger.fdebug("Writing configuration to file")
self.provider_sequence()
config.set('Newznab', 'extra_newznabs', ', '.join(self.write_extras(self.EXTRA_NEWZNABS)))
config.set('Torznab', 'extra_torznabs', ', '.join(self.write_extras(self.EXTRA_TORZNABS)))
###this should be moved elsewhere...
if type(self.BLACKLISTED_PUBLISHERS) != list:
if self.BLACKLISTED_PUBLISHERS is None:
bp = 'None'
else:
bp = ', '.join(self.write_extras(self.BLACKLISTED_PUBLISHERS))
config.set('CV', 'blacklisted_publishers', bp)
else:
config.set('CV', 'blacklisted_publishers', ', '.join(self.BLACKLISTED_PUBLISHERS))
###
config.set('General', 'dynamic_update', str(self.DYNAMIC_UPDATE))
if values is not None:
self.process_kwargs(values)
try:
with codecs.open(self._config_file, encoding='utf8', mode='w+') as configfile:
config.write(configfile)
logger.fdebug('Configuration written to disk.')
except IOError as e:
logger.warn("Error writing configuration file: %s", e)
def comic_config(self, com_location, ComicID, alt_search=None, fuzzy_year=None):
myDB = db.DBConnection()
#--- this is for multipe search terms............
#--- works, just need to redo search.py to accomodate multiple search terms
# ffs_alt = []
# if '+' in alt_search:
#find first +
# ffs = alt_search.find('+')
# ffs_alt.append(alt_search[:ffs])
# ffs_alt_st = str(ffs_alt[0])
# print("ffs_alt: " + str(ffs_alt[0]))
# split the entire string by the delimter +
# ffs_test = alt_search.split('+')
# if len(ffs_test) > 0:
# print("ffs_test names: " + str(len(ffs_test)))
# ffs_count = len(ffs_test)
# n=1
# while (n < ffs_count):
# ffs_alt.append(ffs_test[n])
# print("adding : " + str(ffs_test[n]))
#print("ffs_alt : " + str(ffs_alt))
# ffs_alt_st = str(ffs_alt_st) + "..." + str(ffs_test[n])
# n+=1
# asearch = ffs_alt
# else:
# asearch = alt_search
asearch = str(alt_search)
controlValueDict = {'ComicID': ComicID}
newValues = {"ComicLocation": com_location }
#"QUALalt_vers": qual_altvers,
#"QUALScanner": qual_scanner,
#"QUALtype": qual_type,
#"QUALquality": qual_quality
#}
if asearch is not None:
if asearch == '':
newValues['AlternateSearch'] = "None"
else:
newValues['AlternateSearch'] = str(asearch)
if fuzzy_year is None:
newValues['UseFuzzy'] = "0"
else:
newValues['UseFuzzy'] = str(fuzzy_year)
#force the check/creation of directory com_location here
if os.path.isdir(str(com_location)):
logger.info(u"Validating Directory (" + str(com_location) + "). Already exists! Continuing...")
else:
logger.fdebug("Updated Directory doesn't exist! - attempting to create now.")
try:
os.makedirs(str(com_location))
logger.info(u"Directory successfully created at: " + str(com_location))
except OSError:
logger.error(u"Could not create comicdir : " + str(com_location))
myDB.upsert("comics", newValues, controlValueDict)
raise cherrypy.HTTPRedirect("artistPage?ComicID=%s" % ComicID)
def notify(self, prline=None, prline2=None, sent_to=None, snatched_nzb=None, force=False, module=None, snline=None):
"""
Sends a boxcar notification based on the provided info or SB config
title: The title of the notification to send
message: The message string to send
force: If True then the notification will be sent even if Boxcar is disabled in the config
"""
if module is None:
module = ''
module += '[NOTIFIER]'
if not mylar.BOXCAR_ENABLED and not force:
logger.fdebug(module + ' Notification for Boxcar not enabled, skipping this notification.')
return False
# if no username was given then use the one from the config
if snatched_nzb:
title = snline
message = "Mylar has snatched: " + snatched_nzb + " and has sent it to " + sent_to
else:
title = prline
message = prline2
logger.info(module + ' Sending notification to Boxcar2')
self._sendBoxcar(message, title, module)
return True
def __init__(self, un, pw, session_path=None):
'''
Params:
un: account username (required)
pw: account password (required)
session_path: the path to the actual file you want to persist your cookies in
If blank, saves to $HOME/.32p_cookies.dat
'''
self.module = '[32P-AUTHENTICATION]'
try:
self.ses = cfscrape.create_scraper()
except Exception as e:
logger.error(self.module + " Can't create session with cfscrape")
self.session_path = session_path if session_path is not None else os.path.join(mylar.CACHE_DIR, ".32p_cookies.dat")
self.ses.cookies = LWPCookieJar(self.session_path)
if not os.path.exists(self.session_path):
logger.fdebug(self.module + ' Session cookie does not exist. Signing in and Creating.')
self.ses.cookies.save()
else:
logger.fdebug(self.module + ' Session cookie found. Attempting to load...')
self.ses.cookies.load(ignore_discard=True)
self.un = un
self.pw = pw
self.authkey = None
self.passkey = None
self.uid = None
self.inkdrops = None
def torsend2client(seriesname, linkit, site):
logger.info('matched on ' + str(seriesname))
filename = re.sub('[\'\!\@\#\$\%\:\;\/\\=\?\.]', '',seriesname)
if site == 'ComicBT':
logger.info(linkit)
linkit = str(linkit) + '&passkey=' + str(mylar.CBT_PASSKEY)
if linkit[-7:] != "torrent":
filename += ".torrent"
request = urllib2.Request(linkit)
request.add_header('User-Agent', str(mylar.USER_AGENT))
if mylar.TORRENT_LOCAL and mylar.LOCAL_WATCHDIR is not None:
filepath = os.path.join(mylar.LOCAL_WATCHDIR, filename)
logger.fdebug('filename for torrent set to : ' + filepath)
elif mylar.TORRENT_SEEDBOX and mylar.SEEDBOX_WATCHDIR is not None:
filepath = os.path.join(mylar.CACHE_DIR, filename)
logger.fdebug('filename for torrent set to : ' + filepath)
else:
logger.error('No Local Watch Directory or Seedbox Watch Directory specified. Set it and try again.')
return "fail"
try:
opener = helpers.urlretrieve(urllib2.urlopen(request), filepath)
except Exception, e:
logger.warn('Error fetching data from %s: %s' % (site, e))
return "fail"
def storyarcinfo(xmlid):
comicLibrary = listLibrary()
arcinfo = {}
if mylar.COMICVINE_API == 'None' or mylar.COMICVINE_API is None or mylar.COMICVINE_API == mylar.DEFAULT_CVAPI:
logger.warn('You have not specified your own ComicVine API key - alot of things will be limited. Get your own @ http://api.comicvine.com.')
comicapi = mylar.DEFAULT_CVAPI
else:
comicapi = mylar.COMICVINE_API
#respawn to the exact id for the story arc and count the # of issues present.
ARCPULL_URL = mylar.CVURL + 'story_arc/4045-' + str(xmlid) + '/?api_key=' + str(comicapi) + '&field_list=issues,name,first_appeared_in_issue,deck,image&format=xml&offset=0'
logger.fdebug('arcpull_url:' + str(ARCPULL_URL))
#new CV API restriction - one api request / second.
if mylar.CVAPI_RATE is None or mylar.CVAPI_RATE < 2:
time.sleep(2)
else:
time.sleep(mylar.CVAPI_RATE)
#download the file:
payload = None
verify = False
try:
r = requests.get(ARCPULL_URL, params=payload, verify=verify, headers=mylar.CV_HEADERS)
except Exception, e:
logger.warn('Error fetching data from ComicVine: %s' % (e))
return
def rssdbupdate(feeddata, i, type):
rsschktime = 15
myDB = db.DBConnection()
#let's add the entries into the db so as to save on searches
#also to build up the ID's ;)
for dataval in feeddata:
if type == 'torrent':
#we just store the torrent ID's now.
newVal = {"Link": dataval['link'],
"Pubdate": dataval['pubdate'],
"Site": dataval['site'],
"Size": dataval['size']}
ctrlVal = {"Title": dataval['title']}
else:
newlink = dataval['Link']
newVal = {"Link": newlink,
"Pubdate": dataval['Pubdate'],
"Site": dataval['Site'],
"Size": dataval['Size']}
ctrlVal = {"Title": dataval['Title']}
myDB.upsert("rssdb", newVal, ctrlVal)
logger.fdebug('Completed adding new data to RSS DB. Next add in ' + str(mylar.RSS_CHECKINTERVAL) + ' minutes')
return
def notify(self, ComicName=None, Year=None, Issue=None, sent_to=None, snatched_nzb=None, username=None, force=False):
"""
Sends a boxcar notification based on the provided info or SB config
title: The title of the notification to send
message: The message string to send
username: The username to send the notification to (optional, defaults to the username in the config)
force: If True then the notification will be sent even if Boxcar is disabled in the config
"""
if not mylar.BOXCAR_ENABLED and not force:
logger.fdebug("Notification for Boxcar not enabled, skipping this notification")
return False
# if no username was given then use the one from the config
if not username:
username = mylar.BOXCAR_USERNAME
if snatched_nzb:
title = "Mylar. Sucessfully Snatched!"
message = "Mylar has snatched: " + snatched_nzb + " and has sent it to " + sent_to
else:
title = "Mylar. Successfully Downloaded & Post-Processed!"
message = "Mylar has downloaded and postprocessed: " + ComicName + ' (' + Year + ') #' + Issue
logger.info("Sending notification to Boxcar")
self._sendBoxcar(message, title, username)
return True
def notify(self, ComicName=None, Year=None, Issue=None, sent_to=None, snatched_nzb=None, force=False, module=None):
"""
Sends a boxcar notification based on the provided info or SB config
title: The title of the notification to send
message: The message string to send
force: If True then the notification will be sent even if Boxcar is disabled in the config
"""
if module is None:
module = ''
module += '[NOTIFIER]'
if not mylar.BOXCAR_ENABLED and not force:
logger.fdebug(module + ' Notification for Boxcar not enabled, skipping this notification.')
return False
# if no username was given then use the one from the config
if snatched_nzb:
title = "Mylar. Sucessfully Snatched!"
message = "Mylar has snatched: " + snatched_nzb + " and has sent it to " + sent_to
else:
title = "Mylar. Successfully Downloaded & Post-Processed!"
message = "Mylar has downloaded and postprocessed: " + ComicName + ' (' + Year + ') #' + Issue
logger.info(module + ' Sending notification to Boxcar2')
self._sendBoxcar(message, title, module)
return True
def login(self):
'''
This is generally the only method you'll want to call, as it handles testing test_skey_valid() before
trying test_login().
Returns: True (success) / False (failure)
Side effects: Methods called will handle saving the cookies to disk, and setting
self.authkey, self.passkey, and self.uid
'''
if (self.test_skey_valid()):
logger.fdebug('%s Session key-based login was good.' % self.module)
self.method = 'Session Cookie retrieved OK.'
return {'ses': self.ses,
'status': True}
if (self.test_login()):
logger.fdebug('%s Credential-based login was good.' % self.module)
self.method = 'Credential-based login OK.'
return {'ses': self.ses,
'status': True}
logger.warn('%s Both session key and credential-based logins failed.' % self.module)
self.method = 'Both session key & credential login failed.'
return {'ses': self.ses,
'status': False}
def movefiles(comicid, comlocation, imported):
#comlocation is destination
#comicid is used for rename
files_moved = []
try:
imported = ast.literal_eval(imported)
except ValueError:
pass
myDB = db.DBConnection()
logger.fdebug('comlocation is : ' + comlocation)
logger.fdebug('original comicname is : ' + imported['ComicName'])
impres = imported['filelisting']
if impres is not None:
for impr in impres:
srcimp = impr['comiclocation']
orig_filename = impr['comicfilename']
#before moving check to see if Rename to Mylar structure is enabled.
if mylar.IMP_RENAME and mylar.FILE_FORMAT != '':
logger.fdebug("Renaming files according to configuration details : " + str(mylar.FILE_FORMAT))
renameit = helpers.rename_param(comicid, imported['ComicName'], impr['issuenumber'], orig_filename)
nfilename = renameit['nfilename']
dstimp = os.path.join(comlocation, nfilename)
else:
logger.fdebug("Renaming files not enabled, keeping original filename(s)")
dstimp = os.path.join(comlocation, orig_filename)
logger.info("moving " + srcimp + " ... to " + dstimp)
try:
shutil.move(srcimp, dstimp)
files_moved.append({'srid': imported['srid'],
'filename': impr['comicfilename']})
except (OSError, IOError):
logger.error("Failed to move files - check directories and manually re-run.")
logger.fdebug("all files moved.")
#now that it's moved / renamed ... we remove it from importResults or mark as completed.
if len(files_moved) > 0:
logger.info('files_moved: ' + str(files_moved))
for result in files_moved:
try:
res = result['import_id']
except:
#if it's an 'older' import that wasn't imported, just make it a basic match so things can move and update properly.
controlValue = {"ComicFilename": result['filename'],
"SRID": result['srid']}
newValue = {"Status": "Imported",
"ComicID": comicid}
else:
controlValue = {"impID": result['import_id'],
"ComicFilename": result['filename']}
newValue = {"Status": "Imported",
"SRID": result['srid'],
"ComicID": comicid}
myDB.upsert("importresults", newValue, controlValue)
return
def downloadfile(self, payload, filepath):
url = 'https://32pag.es/torrents.php'
try:
r = self.session.get(url, params=payload, verify=True, stream=True, allow_redirects=True)
except Exception as e:
logger.error('%s [%s] Could not POST URL %s' % ('[32P-DOWNLOADER]', e, url))
return False
if str(r.status_code) != '200':
logger.warn('Unable to download torrent from 32P [Status Code returned: %s]' % r.status_code)
if str(r.status_code) == '404':
logger.warn('[32P-CACHED_ENTRY] Entry found in 32P cache - incorrect. Torrent has probably been merged into a pack, or another series id. Removing from cache.')
self.delete_cache_entry(payload['id'])
else:
logger.fdebug('content: %s' % r.content)
return False
with open(filepath, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
return True
def sendtohome(sftp, remotepath, filelist, transport):
fhost = mylar.CONFIG.TAB_HOST.find(':')
host = mylar.CONFIG.TAB_HOST[:fhost]
port = int(mylar.CONFIG.TAB_HOST[fhost +1:])
successlist = []
filestotal = len(filelist)
for files in filelist:
tempfile = files['filename']
issid = files['issueid']
logger.fdebug('Checking filename for problematic characters: ' + tempfile)
#we need to make the required directory(ies)/subdirectories before the get will work.
if u'\xb4' in files['filename']:
# right quotation
logger.fdebug('detected abnormal character in filename')
filename = tempfile.replace('0xb4', '\'')
if u'\xbd' in files['filename']:
# 1/2 character
filename = tempfile.replace('0xbd', 'half')
if u'\uff1a' in files['filename']:
#some unknown character
filename = tempfile.replace('\0ff1a', '-')
#now we encode the structure to ascii so we can write directories/filenames without error.
filename = tempfile.encode('ascii', 'ignore')
remdir = remotepath
if mylar.CONFIG.MAINTAINSERIESFOLDER == 1:
# Get folder path of issue
comicdir = os.path.split(files['filepath'])[0]
# Isolate comic folder name
comicdir = os.path.split(comicdir)[1]
logger.info('Checking for Comic Folder: ' + comicdir)
chkdir = os.path.join(remdir, comicdir)
try:
sftp.stat(chkdir)
except IOError, e:
logger.info('Comic Folder does not Exist, creating ' + chkdir )
try:
sftp.mkdir(chkdir)
except :
# Fallback to default behavior
logger.info('Could not create Comic Folder, adding to device root')
else :
remdir = chkdir
else :
remdir = chkdir
localsend = files['filepath']
logger.info('Sending : ' + localsend)
remotesend = os.path.join(remdir, filename)
logger.info('To : ' + remotesend)
try:
sftp.stat(remotesend)
except IOError, e:
if e[0] == 2:
filechk = False
def rssdbupdate(feeddata,i,type):
rsschktime = 15
myDB = db.DBConnection()
#let's add the entries into the db so as to save on searches
#also to build up the ID's ;)
x = 1
while x <= i:
try:
dataval = feeddata[x]
except IndexError:
logger.fdebug('reached the end of populating. Exiting the process.')
break
#print "populating : " + str(dataval)
#remove passkey so it doesn't end up in db
if type == 'torrent':
newlink = dataval['Link'][:(dataval['Link'].find('&passkey'))]
newVal = {"Link": newlink,
"Pubdate": dataval['Pubdate'],
"Site": dataval['Site']}
else:
newlink = dataval['Link']
newVal = {"Link": newlink,
"Pubdate": dataval['Pubdate'],
"Site": dataval['Site'],
"Size": dataval['Size']}
ctrlVal = {"Title": dataval['Title']}
myDB.upsert("rssdb", newVal,ctrlVal)
x+=1
logger.fdebug('Completed adding new data to RSS DB. Next add in ' + str(mylar.RSS_CHECKINTERVAL) + ' minutes')
return
def get_the_hash(self, filepath):
# Open torrent file
torrent_file = open(filepath, "rb")
metainfo = bencode.decode(torrent_file.read())
info = metainfo['info']
thehash = hashlib.sha1(bencode.encode(info)).hexdigest().upper()
logger.fdebug('Hash: ' + thehash)
return thehash
def valid_skey_attempt(self, skey):
'''
Not generally the proper method to call - call test_key_valid()
instead - which calls this method.
Attempts to fetch data via an ajax method that will fail if not
authorized. The parameter skey should be set to the string
value of the cookie named session.
Returns: True on success, False on failure. Side Effects: Sets
self.uid, self,authkey and self.passkey
'''
u = '''https://32pag.es/ajax.php'''
params = {'action': 'index'}
testcookie = dict(session=skey)
try:
r = self.ses.get(u, params=params, timeout=60, allow_redirects=False, cookies=testcookie)
except Exception as e:
logger.error('Got an exception [%s] trying to GET to: %s' % (e,u))
self.error = {'status':'error', 'message':'exception trying to retrieve site'}
return False
if r.status_code != 200:
if r.status_code == 302:
newloc = r.headers.get('Location', '')
logger.warn('Got redirect from the POST-ajax action=login GET: %s' % newloc)
self.error = {'status':'redirect-error', 'message':'got redirect from POST-ajax login action : ' + newloc}
else:
logger.error('Got bad status code in the POST-ajax action=login GET: %s' % r.status_code)
self.error = {'status':'bad status code', 'message':'bad status code received in the POST-ajax login action :' + str(r.status_code)}
return False
try:
j = r.json()
except:
logger.warn('Error - response from session-based skey check was not JSON: %s' % r.text)
return False
self.uid = j['response']['id']
self.authkey = j['response']['authkey']
self.passkey = pk = j['response']['passkey']
try:
d = self.ses.get('https://32pag.es/ajax.php', params={'action': 'user_inkdrops'}, verify=True, allow_redirects=True)
except Exception as e:
logger.error('Unable to retreive Inkdrop total : %s' % e)
else:
try:
self.inkdrops = d.json()
except:
logger.error('Inkdrop result did not return valid JSON, unable to verify response')
else:
logger.fdebug('inkdrops: %s' % self.inkdrops)
return True
def _parse_feed(site, url, verify):
logger.fdebug('[RSS] Fetching items from ' + site)
payload = None
headers = {'User-Agent': str(mylar.USER_AGENT)}
try:
r = requests.get(url, params=payload, verify=verify, headers=headers)
except Exception, e:
logger.warn('Error fetching RSS Feed Data from %s: %s' % (site, e))
return
def importIT(self):
#set startup...
if len(self.comiclist) > 0:
self.sql_attach()
query = "DELETE FROM maintenance"
self.db.execute(query)
query = "INSERT INTO maintenance (id, mode, total, status) VALUES (%s,'%s',%s,'%s')" % ('1', self.mode, len(self.comiclist), "running")
self.db.execute(query)
self.sql_close()
logger.info('[MAINTENANCE-MODE][%s] Found %s series in previous db. Preparing to migrate into existing db.' % (self.mode.upper(), len(self.comiclist)))
count = 1
for x in self.comiclist:
logger.info('[MAINTENANCE-MODE][%s] [%s/%s] now attempting to add %s to watchlist...' % (self.mode.upper(), count, len(self.comiclist), x['ComicID']))
try:
self.sql_attach()
self.db.execute("UPDATE maintenance SET progress=?, total=?, current=? WHERE id='1'", (count, len(self.comiclist), re.sub('4050-', '', x['ComicID'].strip())))
self.sql_close()
except Exception as e:
logger.warn('[ERROR] %s' % e)
maintenance_info = importer.addComictoDB(re.sub('4050-', '', x['ComicID']).strip(), calledfrom='maintenance')
try:
logger.info('MAINTENANCE: %s' % maintenance_info)
if maintenance_info['status'] == 'complete':
logger.fdebug('[MAINTENANCE-MODE][%s] Successfully added %s [%s] to watchlist.' % (self.mode.upper(), maintenance_info['comicname'], maintenance_info['year']))
else:
logger.fdebug('[MAINTENANCE-MODE][%s] Unable to add %s [%s] to watchlist.' % (self.mode.upper(), maintenance_info['comicname'], maintenance_info['year']))
raise IOError
self.maintenance_success.append(x)
try:
self.sql_attach()
self.db.execute("UPDATE maintenance SET progress=?, last_comicid=?, last_series=?, last_seriesyear=? WHERE id='1'", (count, re.sub('4050-', '', x['ComicID'].strip()), maintenance_info['comicname'], maintenance_info['year']))
self.sql_close()
except Exception as e:
logger.warn('[ERROR] %s' % e)
except IOError as e:
logger.warn('[MAINTENANCE-MODE][%s] Unable to add series to watchlist: %s' % (self.mode.upper(), e))
self.maintenance_fail.append(x)
count+=1
else:
logger.warn('[MAINTENANCE-MODE][%s] Unable to locate any series in db. This is probably a FATAL error and an unrecoverable db.' % self.mode.upper())
return
logger.info('[MAINTENANCE-MODE][%s] Successfully imported %s series into existing db.' % (self.mode.upper(), len(self.maintenance_success)))
if len(self.maintenance_fail) > 0:
logger.info('[MAINTENANCE-MODE][%s] Failed to import %s series into existing db: %s' % (self.mode.upper(), len(self.maintenance_success), self.maintenance_fail))
try:
self.sql_attach()
self.db.execute("UPDATE maintenance SET status=? WHERE id='1'", ["completed"])
self.sql_close()
except Exception as e:
logger.warn('[ERROR] %s' % e)
def _deliverFile(self, **kwargs):
logger.fdebug("_deliverFile: kwargs: %s" % kwargs)
if 'file' not in kwargs:
self.data = self._error_with_message('No file provided')
elif 'filename' not in kwargs:
self.data = self._error_with_message('No filename provided')
else:
#logger.fdebug("file name: %s" % str(kwargs['file'])
self.filename = os.path.split(str(kwargs['file']))[1]
self.file = str(kwargs['file'])
return
def addtoreadlist(self):
annualize = False
myDB = db.DBConnection()
readlist = myDB.selectone("SELECT * from issues where IssueID=?", [self.IssueID]).fetchone()
if readlist is None:
logger.fdebug(self.module + ' Checking against annuals..')
readlist = myDB.selectone("SELECT * from annuals where IssueID=?", [self.IssueID]).fetchone()
if readlist is None:
logger.error(self.module + ' Cannot locate IssueID - aborting..')
return
else:
annualize = True
comicinfo = myDB.selectone("SELECT * from comics where ComicID=?", [readlist['ComicID']]).fetchone()
logger.info(self.module + ' Attempting to add issueid ' + readlist['IssueID'])
if comicinfo is None:
logger.info(self.module + ' Issue not located on your current watchlist. I should probably check story-arcs but I do not have that capability just yet.')
else:
locpath = None
if mylar.MULTIPLE_DEST_DIRS is not None and mylar.MULTIPLE_DEST_DIRS != 'None' and os.path.join(mylar.MULTIPLE_DEST_DIRS, os.path.basename(comicinfo['ComicLocation'])) != comicinfo['ComicLocation']:
pathdir = os.path.join(mylar.MULTIPLE_DEST_DIRS, os.path.basename(comicinfo['ComicLocation']))
if os.path.exists(os.path.join(pathdir, readlist['Location'])):
locpath = os.path.join(pathdir, readlist['Location'])
else:
if os.path.exists(os.path.join(comicinfo['ComicLocation'], readlist['Location'])):
locpath = os.path.join(comicinfo['ComicLocation'], readlist['Location'])
else:
if os.path.exists(os.path.join(comicinfo['ComicLocation'], readlist['Location'])):
locpath = os.path.join(comicinfo['ComicLocation'], readlist['Location'])
if not locpath is None:
comicissue = readlist['Issue_Number']
comicname = comicinfo['ComicName']
dspinfo = comicname + ' #' + comicissue
if annualize:
if mylar.ANNUALS_ON:
comicissue = 'Annual ' + readlist['Issue_Number']
dspinfo = comicname + ' Annual #' + readlist['Issue_Number']
else:
comicname = comicinfo['ComicName'] + ' Annual'
dspinfo = comicname + ' #' + comicissue
ctrlval = {"IssueID": self.IssueID}
newval = {"DateAdded": helpers.today(),
"Status": "Added",
"ComicID": readlist['ComicID'],
"Issue_Number": comicissue,
"IssueDate": readlist['IssueDate'],
"SeriesYear": comicinfo['ComicYear'],
"ComicName": comicname,
"Location": locpath}
myDB.upsert("readlist", newval, ctrlval)
logger.info(self.module + ' Added ' + dspinfo + ' to the Reading list.')
return
def notify(self, event, message=None, snatched_nzb=None, prov=None, sent_to=None, module=None):
if module is None:
module = ''
module += '[NOTIFIER]'
if snatched_nzb:
if snatched_nzb[-1] == '\.':
snatched_nzb = snatched_nzb[:-1]
message = "Mylar has snatched: " + snatched_nzb + " from " + prov + " and " + sent_to
data = {'token': mylar.CONFIG.PUSHOVER_APIKEY,
'user': mylar.CONFIG.PUSHOVER_USERKEY,
'message': message.encode("utf-8"),
'title': event,
'priority': mylar.CONFIG.PUSHOVER_PRIORITY}
if all([self.device is not None, self.device != 'None']):
data.update({'device': self.device})
r = self._session.post(self.PUSHOVER_URL, data=data, verify=True)
if r.status_code == 200:
try:
response = r.json()
if 'devices' in response and self.test is True:
logger.fdebug('%s Available devices: %s' % (module, response))
if any([self.device is None, self.device == 'None']):
self.device = 'all available devices'
r = self._session.post('https://api.pushover.net/1/messages.json', data=data, verify=True)
if r.status_code == 200:
logger.info('%s PushOver notifications sent to %s.' % (module, self.device))
elif r.status_code >=400 and r.status_code < 500:
logger.error('%s PushOver request failed to %s: %s' % (module, self.device, r.content))
return False
else:
logger.error('%s PushOver notification failed serverside.' % module)
return False
else:
logger.info('%s PushOver notifications sent.' % module)
except Exception as e:
logger.warn('%s[ERROR] - %s' % (module, e))
return False
else:
return True
elif r.status_code >= 400 and r.status_code < 500:
logger.error('%s PushOver request failed: %s' % (module, r.content))
return False
else:
logger.error('%s PushOver notification failed serverside.' % module)
return False
def dbUpdate():
myDB = db.DBConnection()
activecomics = myDB.select('SELECT ComicID, ComicName from comics WHERE Status="Active" or Status="Loading" order by LastUpdated ASC')
logger.info('Starting update for %i active comics' % len(activecomics))
for comic in activecomics:
comicid = comic[0]
mismatch = "no"
if not mylar.CV_ONLY or comicid[:1] == "G":
CV_EXcomicid = myDB.action("SELECT * from exceptions WHERE ComicID=?", [comicid]).fetchone()
if CV_EXcomicid is None: pass
else:
if CV_EXcomicid['variloop'] == '99':
mismatch = "yes"
if comicid[:1] == "G":
mylar.importer.GCDimport(comicid)
else:
mylar.importer.addComictoDB(comicid,mismatch)
else:
if mylar.CV_ONETIMER == 1:
logger.fdebug("CV_OneTimer option enabled...")
#in order to update to JUST CV_ONLY, we need to delete the issues for a given series so it's a clean refresh.
logger.fdebug("Gathering the status of all issues for the series.")
issues = myDB.select('SELECT * FROM issues WHERE ComicID=?', [comicid])
#store the issues' status for a given comicid, after deleting and readding, flip the status back to what it is currently.
logger.fdebug("Deleting all issue data.")
myDB.select('DELETE FROM issues WHERE ComicID=?', [comicid])
logger.fdebug("Refreshing the series and pulling in new data using only CV.")
mylar.importer.addComictoDB(comicid,mismatch)
issues_new = myDB.select('SELECT * FROM issues WHERE ComicID=?', [comicid])
icount = 0
logger.fdebug("Attempting to put the Status' back how they were.")
for issue in issues:
for issuenew in issues_new:
if issuenew['IssueID'] == issue['IssueID'] and issuenew['Status'] != issue['Status']:
#change the status to the previous status
ctrlVAL = {'IssueID': issue['IssueID']}
newVAL = {'Status': issue['Status']}
myDB.upsert("Issues", newVAL, ctrlVAL)
icount+=1
break
logger.info("In converting data to CV only, I changed the status of " + str(icount) + " issues.")
mylar.CV_ONETIMER = 0
else:
mylar.importer.addComictoDB(comicid,mismatch)
time.sleep(5) #pause for 5 secs so dont hammer CV and get 500 error
logger.info('Update complete')
def GCDaddComic(self, comicid, comicname=None, comicyear=None, comicissues=None, comiccover=None, comicpublisher=None):
#since we already know most of the info, let's add it to the db so we can reference it later.
myDB = db.DBConnection()
gcomicid = "G" + str(comicid)
comicyear_len = comicyear.find(' ', 2)
comyear = comicyear[comicyear_len+1:comicyear_len+5]
if comyear.isdigit():
logger.fdebug("Series year set to : " + str(comyear))
else:
logger.fdebug("Invalid Series year detected - trying to adjust from " + str(comyear))
#comicyear_len above will trap wrong year if it's 10 October 2010 - etc ( 2000 AD)...
find_comicyear = comicyear.split()
for i in find_comicyear:
if len(i) == 4:
logger.fdebug("Series year detected as : " + str(i))
comyear = str(i)
continue
logger.fdebug("Series year set to: " + str(comyear))
controlValueDict = { 'ComicID': gcomicid }
newValueDict = {'ComicName': comicname,
'ComicYear': comyear,
'ComicPublished': comicyear,
'ComicPublisher': comicpublisher,
'ComicImage': comiccover,
'Total' : comicissues }
myDB.upsert("comics", newValueDict, controlValueDict)
threading.Thread(target=importer.GCDimport, args=[gcomicid]).start()
raise cherrypy.HTTPRedirect("artistPage?ComicID=%s" % gcomicid)
def checkthis(datecheck,datestatus,usedate):
logger.fdebug('Now checking date comparison using an issue store date of ' + str(datecheck))
logger.fdebug('Using a compare date (usedate) of ' + str(usedate))
logger.fdebug('Status of ' + str(datestatus))
if int(datecheck) >= int(usedate):
logger.fdebug('Store Date falls within acceptable range - series MATCH')
valid_check = True
elif int(datecheck) < int(usedate):
logger.fdebug('The issue date of issue was on ' + str(datecheck) + ' which is prior to ' + str(usedate))
valid_check = False
return valid_check
def weekly_singlecopy(comicid, issuenum, file, path):
myDB = db.DBConnection()
try:
pull_date = myDB.selectone("SELECT SHIPDATE from weekly").fetchone()
if (pull_date is None):
pulldate = '00000000'
else:
pulldate = pull_date['SHIPDATE']
logger.fdebug(u"Weekly pull list detected as : " + str(pulldate))
except (sqlite3.OperationalError, TypeError),msg:
logger.info(u"Error determining current weekly pull-list date - you should refresh the pull-list manually probably.")
return
def markFailed(self):
#use this to forcibly mark a single issue as being Failed (ie. if a search result is sent to a client, but the result
#ends up passing in a 404 or something that makes it so that the download can't be initiated).
module = '[FAILED-DOWNLOAD]'
myDB = db.DBConnection()
logger.info(module + ' Marking as a Failed Download.')
logger.fdebug(module + 'nzb_name: ' + self.nzb_name)
logger.fdebug(module + 'issueid: ' + str(self.issueid))
logger.fdebug(module + 'nzb_id: ' + str(self.id))
logger.fdebug(module + 'prov: ' + self.prov)
if 'annual' in self.nzb_name.lower():
logger.info(module + ' Annual detected.')
annchk = "yes"
issuenzb = myDB.selectone("SELECT * from annuals WHERE IssueID=? AND ComicName NOT NULL", [self.issueid]).fetchone()
else:
issuenzb = myDB.selectone("SELECT * from issues WHERE IssueID=? AND ComicName NOT NULL", [self.issueid]).fetchone()
ctrlVal = {"IssueID": self.issueid}
Vals = {"Status": 'Failed'}
myDB.upsert("issues", Vals, ctrlVal)
ctrlVal = {"ID": self.id,
"Provider": self.prov,
"NZBName": self.nzb_name}
Vals = {"Status": 'Failed',
"ComicName": issuenzb['ComicName'],
"Issue_Number": issuenzb['Issue_Number']}
myDB.upsert("failed", Vals, ctrlVal)
logger.info(module + ' Successfully marked as Failed.')
def _send(self, data, module):
r = self._session.post(self.NMA_URL, data=data)
logger.fdebug('[NMA] Status code returned: ' + str(r.status_code))
if r.status_code == 200:
logger.info(module + ' NotifyMyAndroid notifications sent.')
return True
elif r.status_code >= 400 and r.status_code < 500:
logger.error(module + ' NotifyMyAndroid request failed: %s' % r.content)
return False
else:
logger.error(module + ' NotifyMyAndroid notification failed serverside.')
return False
csvfile = open(newfl, "rb")
creader = csv.reader(csvfile, delimiter='\t')
t = 1
for row in creader:
try:
#print ("Row: %s" % row)
cursor.execute("INSERT INTO future VALUES (?,?,?,?,?,?,?,null);",
row)
except Exception, e:
logger.fdebug("Error - invald arguments...-skipping")
pass
t += 1
logger.fdebug('successfully added ' + str(t) +
' issues to future upcoming table.')
csvfile.close()
connection.commit()
connection.close()
mylar.weeklypull.pullitcheck(futurepull="yes")
#.end
def populate(link, publisher, shipdate):
#this is the secondary url call to populate
input = 'http://www.comicbookresources.com/' + link
#print 'checking ' + str(input)
response = urllib2.urlopen(input)
soup = BeautifulSoup(response)
abc = soup.findAll('p')
def run(dirName,
nzbName=None,
issueid=None,
comversion=None,
manual=None,
filename=None,
module=None,
manualmeta=False):
if module is None:
module = ''
module += '[META-TAGGER]'
logger.fdebug(module + ' dirName:' + dirName)
# 2015-11-23: Recent CV API changes restrict the rate-limit to 1 api request / second.
# ComicTagger has to be included now with the install as a timer had to be added to allow for the 1/second rule.
comictagger_cmd = os.path.join(mylar.CMTAGGER_PATH, 'comictagger.py')
logger.fdebug(
'ComicTagger Path location for internal comictagger.py set to : ' +
comictagger_cmd)
# Force mylar to use cmtagger_path = mylar.PROG_DIR to force the use of the included lib.
logger.fdebug(module + ' Filename is : ' + filename)
filepath = filename
og_filepath = filepath
try:
filename = os.path.split(filename)[1] # just the filename itself
except:
logger.warn(
'Unable to detect filename within directory - I am aborting the tagging. You best check things out.'
)
return "fail"
#make use of temporary file location in order to post-process this to ensure that things don't get hammered when converting
new_filepath = None
new_folder = None
try:
import tempfile
logger.fdebug('Filepath: %s' % filepath)
logger.fdebug('Filename: %s' % filename)
new_folder = tempfile.mkdtemp(
prefix='mylar_', dir=mylar.CONFIG.CACHE_DIR) #prefix, suffix, dir
logger.fdebug('New_Folder: %s' % new_folder)
new_filepath = os.path.join(new_folder, filename)
logger.fdebug('New_Filepath: %s' % new_filepath)
if mylar.CONFIG.FILE_OPTS == 'copy' and manualmeta == False:
shutil.copy(filepath, new_filepath)
else:
shutil.copy(filepath, new_filepath)
filepath = new_filepath
except Exception as e:
logger.warn('%s Unexpected Error: %s [%s]' %
(module, sys.exc_info()[0], e))
logger.warn(
module +
' Unable to create temporary directory to perform meta-tagging. Processing without metatagging.'
)
tidyup(og_filepath, new_filepath, new_folder, manualmeta)
return "fail"
## Sets up other directories ##
scriptname = os.path.basename(sys.argv[0])
downloadpath = os.path.abspath(dirName)
sabnzbdscriptpath = os.path.dirname(sys.argv[0])
comicpath = new_folder
logger.fdebug(module + ' Paths / Locations:')
logger.fdebug(module + ' scriptname : ' + scriptname)
logger.fdebug(module + ' downloadpath : ' + downloadpath)
logger.fdebug(module + ' sabnzbdscriptpath : ' + sabnzbdscriptpath)
logger.fdebug(module + ' comicpath : ' + comicpath)
logger.fdebug(module + ' Running the ComicTagger Add-on for Mylar')
##set up default comictagger options here.
#used for cbr - to - cbz conversion
#depending on copy/move - eitehr we retain the rar or we don't.
if mylar.CONFIG.FILE_OPTS == 'move':
cbr2cbzoptions = ["-e", "--delete-rar"]
else:
cbr2cbzoptions = ["-e"]
tagoptions = ["-s"]
if mylar.CONFIG.CMTAG_VOLUME:
if mylar.CONFIG.CMTAG_START_YEAR_AS_VOLUME:
comversion = str(comversion)
else:
if any(
[comversion is None, comversion == '', comversion == 'None']):
comversion = '1'
comversion = re.sub('[^0-9]', '', comversion).strip()
cvers = 'volume=' + str(comversion)
else:
cvers = "volume="
tagoptions.extend(["-m", cvers])
try:
#from comictaggerlib import ctversion
ct_check = subprocess.check_output(
[sys.executable, comictagger_cmd, "--version"],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
#logger.warn(module + "[WARNING] "command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output))
logger.warn(
module +
'[WARNING] Make sure that you are using the comictagger included with Mylar.'
)
tidyup(filepath, new_filepath, new_folder, manualmeta)
return "fail"
logger.info('ct_check: %s' % ct_check)
ctend = str(ct_check).find(':')
ct_version = re.sub("[^0-9]", "", str(ct_check)[:ctend])
from pkg_resources import parse_version
if parse_version(ct_version) >= parse_version('1.3.1'):
if any([
mylar.CONFIG.COMICVINE_API == 'None',
mylar.CONFIG.COMICVINE_API is None
]):
logger.fdebug(
'%s ComicTagger v.%s being used - no personal ComicVine API Key supplied. Take your chances.'
% (module, ct_version))
use_cvapi = "False"
else:
logger.fdebug(
'%s ComicTagger v.%s being used - using personal ComicVine API key supplied via mylar.'
% (module, ct_version))
use_cvapi = "True"
tagoptions.extend(["--cv-api-key", mylar.CONFIG.COMICVINE_API])
else:
logger.fdebug(
'%s ComicTagger v.ct_version being used - personal ComicVine API key not supported in this version. Good luck.'
% (module, ct_version))
use_cvapi = "False"
i = 1
tagcnt = 0
if mylar.CONFIG.CT_TAG_CR:
tagcnt = 1
logger.fdebug(module + ' CR Tagging enabled.')
if mylar.CONFIG.CT_TAG_CBL:
if not mylar.CONFIG.CT_TAG_CR:
i = 2 #set the tag to start at cbl and end without doing another tagging.
tagcnt = 2
logger.fdebug(module + ' CBL Tagging enabled.')
if tagcnt == 0:
logger.warn(
module +
' You have metatagging enabled, but you have not selected the type(s) of metadata to write. Please fix and re-run manually'
)
tidyup(filepath, new_filepath, new_folder, manualmeta)
return "fail"
#if it's a cbz file - check if no-overwrite existing tags is enabled / disabled in config.
if filename.endswith('.cbz'):
if mylar.CONFIG.CT_CBZ_OVERWRITE:
logger.fdebug(
module + ' Will modify existing tag blocks even if it exists.')
else:
logger.fdebug(
module +
' Will NOT modify existing tag blocks even if they exist already.'
)
tagoptions.extend(["--nooverwrite"])
if issueid is None:
tagoptions.extend(["-f", "-o"])
else:
tagoptions.extend(["-o", "--id", issueid])
original_tagoptions = tagoptions
og_tagtype = None
initial_ctrun = True
error_remove = False
while (i <= tagcnt):
if initial_ctrun:
f_tagoptions = cbr2cbzoptions
f_tagoptions.extend([filepath])
else:
if i == 1:
tagtype = 'cr' # CR meta-tagging cycle.
tagdisp = 'ComicRack tagging'
elif i == 2:
tagtype = 'cbl' # Cbl meta-tagging cycle
tagdisp = 'Comicbooklover tagging'
f_tagoptions = original_tagoptions
if og_tagtype is not None:
for index, item in enumerate(f_tagoptions):
if item == og_tagtype:
f_tagoptions[index] = tagtype
else:
f_tagoptions.extend(["--type", tagtype, filepath])
og_tagtype = tagtype
logger.info(module + ' ' + tagdisp +
' meta-tagging processing started.')
currentScriptName = [sys.executable, comictagger_cmd]
script_cmd = currentScriptName + f_tagoptions
if initial_ctrun:
logger.fdebug('%s Enabling ComicTagger script with options: %s' %
(module, f_tagoptions))
script_cmdlog = script_cmd
else:
logger.fdebug('%s Enabling ComicTagger script with options: %s' %
(module,
re.sub(
f_tagoptions[f_tagoptions.index(
mylar.CONFIG.COMICVINE_API)], 'REDACTED',
str(f_tagoptions))))
# generate a safe command line string to execute the script and provide all the parameters
script_cmdlog = re.sub(
f_tagoptions[f_tagoptions.index(mylar.CONFIG.COMICVINE_API)],
'REDACTED', str(script_cmd))
logger.fdebug(module + ' Executing command: ' + str(script_cmdlog))
logger.fdebug(module + ' Absolute path to script: ' + script_cmd[0])
try:
# use subprocess to run the command and capture output
p = subprocess.Popen(script_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = p.communicate()
logger.info(out)
logger.info(err)
if out is not None:
out = out.decode('utf-8')
if err is not None:
err = err.decode('utf-8')
if initial_ctrun and 'exported successfully' in out:
logger.fdebug(module + '[COMIC-TAGGER] : ' + str(out))
#Archive exported successfully to: X-Men v4 008 (2014) (Digital) (Nahga-Empire).cbz (Original deleted)
if 'Error deleting' in filepath:
tf1 = out.find('exported successfully to: ')
tmpfilename = out[tf1 + len('exported successfully to: '
):].strip()
error_remove = True
else:
tmpfilename = re.sub('Archive exported successfully to: ',
'', out.rstrip())
if mylar.CONFIG.FILE_OPTS == 'move':
tmpfilename = re.sub('\(Original deleted\)', '',
tmpfilename).strip()
tmpf = tmpfilename #.decode('utf-8')
filepath = os.path.join(comicpath, tmpf)
if filename.lower() != tmpf.lower() and tmpf.endswith(
'(1).cbz'):
logger.fdebug(
'New filename [%s] is named incorrectly due to duplication during metatagging - Making sure it\'s named correctly [%s].'
% (tmpf, filename))
tmpfilename = filename
filepath_new = os.path.join(comicpath, tmpfilename)
try:
os.rename(filepath, filepath_new)
filepath = filepath_new
except:
logger.warn(
'%s unable to rename file to accomodate metatagging cbz to the same filename'
% module)
if not os.path.isfile(filepath):
logger.fdebug(module + 'Trying utf-8 conversion.')
tmpf = tmpfilename.encode('utf-8')
filepath = os.path.join(comicpath, tmpf)
if not os.path.isfile(filepath):
logger.fdebug(module + 'Trying latin-1 conversion.')
tmpf = tmpfilename.encode('Latin-1')
filepath = os.path.join(comicpath, tmpf)
logger.fdebug(module +
'[COMIC-TAGGER][CBR-TO-CBZ] New filename: ' +
filepath)
initial_ctrun = False
elif initial_ctrun and 'Archive is not a RAR' in out:
logger.fdebug('%s Output: %s' % (module, out))
logger.warn(module +
'[COMIC-TAGGER] file is not in a RAR format: ' +
filename)
initial_ctrun = False
elif initial_ctrun:
initial_ctrun = False
if 'file is not expected size' in out:
logger.fdebug('%s Output: %s' % (module, out))
tidyup(og_filepath, new_filepath, new_folder, manualmeta)
return 'corrupt'
else:
logger.warn(
module +
'[COMIC-TAGGER][CBR-TO-CBZ] Failed to convert cbr to cbz - check permissions on folder : '
+ mylar.CONFIG.CACHE_DIR +
' and/or the location where Mylar is trying to tag the files from.'
)
tidyup(og_filepath, new_filepath, new_folder, manualmeta)
return 'fail'
elif 'Cannot find' in out:
logger.fdebug('%s Output: %s' % (module, out))
logger.warn(module + '[COMIC-TAGGER] Unable to locate file: ' +
filename)
file_error = 'file not found||' + filename
return file_error
elif 'not a comic archive!' in out:
logger.fdebug('%s Output: %s' % (module, out))
logger.warn(module + '[COMIC-TAGGER] Unable to locate file: ' +
filename)
file_error = 'file not found||' + filename
return file_error
else:
logger.info(module + '[COMIC-TAGGER] Successfully wrote ' +
tagdisp + ' [' + filepath + ']')
i += 1
except OSError as e:
logger.warn(
module +
'[COMIC-TAGGER] Unable to run comictagger with the options provided: '
+ re.sub(
f_tagoptions[f_tagoptions.index(
mylar.CONFIG.COMICVINE_API)], 'REDACTED',
str(script_cmd)))
tidyup(filepath, new_filepath, new_folder, manualmeta)
return "fail"
if mylar.CONFIG.CBR2CBZ_ONLY and initial_ctrun == False:
break
return filepath
def libraryScan(dir=None,
append=False,
ComicID=None,
ComicName=None,
cron=None):
if cron and not mylar.LIBRARYSCAN:
return
if not dir:
dir = mylar.COMIC_DIR
# If we're appending a dir, it's coming from the post processor which is
# already bytestring
if not append:
dir = dir.encode(mylar.SYS_ENCODING)
if not os.path.isdir(dir):
logger.warn('Cannot find directory: %s. Not scanning' %
dir.decode(mylar.SYS_ENCODING, 'replace'))
return
logger.info('Scanning comic directory: %s' %
dir.decode(mylar.SYS_ENCODING, 'replace'))
basedir = dir
comic_list = []
comiccnt = 0
extensions = ('cbr', 'cbz')
for r, d, f in os.walk(dir):
#for directory in d[:]:
# if directory.startswith("."):
# d.remove(directory)
for files in f:
if any(files.lower().endswith('.' + x.lower())
for x in extensions):
comic = files
comicpath = os.path.join(r, files)
comicsize = os.path.getsize(comicpath)
print "Comic: " + comic
print "Comic Path: " + comicpath
print "Comic Size: " + str(comicsize)
# We need the unicode path to use for logging, inserting into database
unicode_comic_path = comicpath.decode(mylar.SYS_ENCODING,
'replace')
comiccnt += 1
comic_dict = {
'ComicFilename': comic,
'ComicLocation': comicpath,
'ComicSize': comicsize,
'Unicode_ComicLocation': unicode_comic_path
}
comic_list.append(comic_dict)
logger.info("I've found a total of " + str(comiccnt) +
" comics....analyzing now")
logger.info("comiclist: " + str(comic_list))
myDB = db.DBConnection()
#let's load in the watchlist to see if we have any matches.
logger.info(
"loading in the watchlist to see if a series is being watched already..."
)
watchlist = myDB.select("SELECT * from comics")
ComicName = []
DisplayName = []
ComicYear = []
ComicPublisher = []
ComicTotal = []
ComicID = []
ComicLocation = []
AltName = []
watchcnt = 0
watch_kchoice = []
watchchoice = {}
import_by_comicids = []
import_comicids = {}
for watch in watchlist:
#use the comicname_filesafe to start
watchdisplaycomic = re.sub('[\_\#\,\/\:\;\!\$\%\&\+\'\?\@]', ' ',
watch['ComicName']).encode('utf-8').strip()
# let's clean up the name, just in case for comparison purposes...
watchcomic = re.sub(
'[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ',
watch['ComicName_Filesafe']).encode('utf-8').strip()
#watchcomic = re.sub('\s+', ' ', str(watchcomic)).strip()
if ' the ' in watchcomic.lower():
#drop the 'the' from the watchcomic title for proper comparisons.
watchcomic = watchcomic[-4:]
alt_chk = "no" # alt-checker flag (default to no)
# account for alternate names as well
if watch['AlternateSearch'] is not None and watch[
'AlternateSearch'] is not 'None':
altcomic = re.sub(
'[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ',
watch['AlternateSearch']).encode('utf-8').strip()
#altcomic = re.sub('\s+', ' ', str(altcomic)).strip()
AltName.append(altcomic)
alt_chk = "yes" # alt-checker flag
ComicName.append(watchcomic)
DisplayName.append(watchdisplaycomic)
ComicYear.append(watch['ComicYear'])
ComicPublisher.append(watch['ComicPublisher'])
ComicTotal.append(watch['Total'])
ComicID.append(watch['ComicID'])
ComicLocation.append(watch['ComicLocation'])
watchcnt += 1
logger.info("Successfully loaded " + str(watchcnt) +
" series from your watchlist.")
ripperlist = ['digital-', 'empire', 'dcp']
watchfound = 0
datelist = [
'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct',
'nov', 'dec'
]
# datemonth = {'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9,'ten':10,'eleven':$
# #search for number as text, and change to numeric
# for numbs in basnumbs:
# #print ("numbs:" + str(numbs))
# if numbs in ComicName.lower():
# numconv = basnumbs[numbs]
# #print ("numconv: " + str(numconv))
for i in comic_list:
print i['ComicFilename']
#if mylar.IMP_METADATA:
#logger.info('metatagging checking enabled.')
#if read tags is enabled during import, check here.
#if i['ComicLocation'].endswith('.cbz'):
# logger.info('Attempting to read tags present in filename: ' + str(i['ComicLocation']))
# issueinfo = helpers.IssueDetails(i['ComicLocation'])
# if issueinfo is None:
# pass
# else:
# logger.info('Successfully retrieved some tags. Lets see what I can figure out.')
# comicname = issueinfo[0]['series']
# logger.fdebug('Series Name: ' + comicname)
# issue_number = issueinfo[0]['issue_number']
# logger.fdebug('Issue Number: ' + str(issue_number))
# issuetitle = issueinfo[0]['title']
# logger.fdebug('Issue Title: ' + issuetitle)
# issueyear = issueinfo[0]['year']
# logger.fdebug('Issue Year: ' + str(issueyear))
# # if used by ComicTagger, Notes field will have the IssueID.
# issuenotes = issueinfo[0]['notes']
# logger.fdebug('Notes: ' + issuenotes)
comfilename = i['ComicFilename']
comlocation = i['ComicLocation']
#let's clean up the filename for matching purposes
cfilename = re.sub('[\_\#\,\/\:\;\-\!\$\%\&\+\'\?\@]', ' ',
comfilename)
#cfilename = re.sub('\s', '_', str(cfilename))
d_filename = re.sub('[\_\#\,\/\;\!\$\%\&\?\@]', ' ', comfilename)
d_filename = re.sub('[\:\-\+\']', '#', d_filename)
#strip extraspaces
d_filename = re.sub('\s+', ' ', d_filename)
cfilename = re.sub('\s+', ' ', cfilename)
#versioning - remove it
subsplit = cfilename.replace('_', ' ').split()
volno = None
volyr = None
for subit in subsplit:
if subit[0].lower() == 'v':
vfull = 0
if subit[1:].isdigit():
#if in format v1, v2009 etc...
if len(subit) > 3:
# if it's greater than 3 in length, then the format is Vyyyy
vfull = 1 # add on 1 character length to account for extra space
cfilename = re.sub(subit, '', cfilename)
d_filename = re.sub(subit, '', d_filename)
volno = re.sub("[^0-9]", " ", subit)
elif subit.lower()[:3] == 'vol':
#if in format vol.2013 etc
#because the '.' in Vol. gets removed, let's loop thru again after the Vol hit to remove it entirely
logger.fdebug('volume indicator detected as version #:' +
str(subit))
cfilename = re.sub(subit, '', cfilename)
cfilename = " ".join(cfilename.split())
d_filename = re.sub(subit, '', d_filename)
d_filename = " ".join(d_filename.split())
volyr = re.sub("[^0-9]", " ", subit).strip()
logger.fdebug('volume year set as : ' + str(volyr))
cm_cn = 0
#we need to track the counter to make sure we are comparing the right array parts
#this takes care of the brackets :)
m = re.findall('[^()]+', cfilename)
lenm = len(m)
logger.fdebug("there are " + str(lenm) + " words.")
cnt = 0
yearmatch = "false"
foundonwatch = "False"
issue = 999999
while (cnt < lenm):
if m[cnt] is None: break
if m[cnt] == ' ':
pass
else:
logger.fdebug(str(cnt) + ". Bracket Word: " + m[cnt])
if cnt == 0:
comic_andiss = m[cnt]
logger.fdebug("Comic: " + comic_andiss)
# if it's not in the standard format this will bork.
# let's try to accomodate (somehow).
# first remove the extension (if any)
extensions = ('cbr', 'cbz')
if comic_andiss.lower().endswith(extensions):
comic_andiss = comic_andiss[:-4]
logger.fdebug("removed extension from filename.")
#now we have to break up the string regardless of formatting.
#let's force the spaces.
comic_andiss = re.sub('_', ' ', comic_andiss)
cs = comic_andiss.split()
cs_len = len(cs)
cn = ''
ydetected = 'no'
idetected = 'no'
decimaldetect = 'no'
for i in reversed(xrange(len(cs))):
#start at the end.
logger.fdebug("word: " + str(cs[i]))
#assume once we find issue - everything prior is the actual title
#idetected = no will ignore everything so it will assume all title
if cs[i][:-2] == '19' or cs[
i][:-2] == '20' and idetected == 'no':
logger.fdebug("year detected: " + str(cs[i]))
ydetected = 'yes'
result_comyear = cs[i]
elif cs[i].isdigit(
) and idetected == 'no' or '.' in cs[i]:
if '.' in cs[i]:
#make sure it's a number on either side of decimal and assume decimal issue.
decst = cs[i].find('.')
dec_st = cs[i][:decst]
dec_en = cs[i][decst + 1:]
logger.fdebug("st: " + str(dec_st))
logger.fdebug("en: " + str(dec_en))
if dec_st.isdigit() and dec_en.isdigit():
logger.fdebug(
"decimal issue detected...adjusting.")
issue = dec_st + "." + dec_en
logger.fdebug("issue detected: " +
str(issue))
idetected = 'yes'
else:
logger.fdebug(
"false decimal represent. Chunking to extra word."
)
cn = cn + cs[i] + " "
break
issue = cs[i]
logger.fdebug("issue detected : " + str(issue))
idetected = 'yes'
elif '\#' in cs[i] or decimaldetect == 'yes':
logger.fdebug("issue detected: " + str(cs[i]))
idetected = 'yes'
else:
cn = cn + cs[i] + " "
if ydetected == 'no':
#assume no year given in filename...
result_comyear = "0000"
logger.fdebug("cm?: " + str(cn))
if issue is not '999999':
comiss = issue
else:
logger.ERROR(
"Invalid Issue number (none present) for " +
comfilename)
break
cnsplit = cn.split()
cname = ''
findcn = 0
while (findcn < len(cnsplit)):
cname = cname + cs[findcn] + " "
findcn += 1
cname = cname[:len(cname) - 1] # drop the end space...
print("assuming name is : " + cname)
com_NAME = cname
print("com_NAME : " + com_NAME)
yearmatch = "True"
else:
logger.fdebug('checking ' + m[cnt])
# we're assuming that the year is in brackets (and it should be damnit)
if m[cnt][:-2] == '19' or m[cnt][:-2] == '20':
print("year detected: " + str(m[cnt]))
ydetected = 'yes'
result_comyear = m[cnt]
elif m[cnt][:3].lower() in datelist:
logger.fdebug(
'possible issue date format given - verifying')
#if the date of the issue is given as (Jan 2010) or (January 2010) let's adjust.
#keeping in mind that ',' and '.' are already stripped from the string
if m[cnt][-4:].isdigit():
ydetected = 'yes'
result_comyear = m[cnt][-4:]
logger.fdebug('Valid Issue year of ' +
str(result_comyear) +
'detected in format of ' +
str(m[cnt]))
cnt += 1
displength = len(cname)
print 'd_filename is : ' + d_filename
charcount = d_filename.count('#')
print('charcount is : ' + str(charcount))
if charcount > 0:
print('entering loop')
for i, m in enumerate(re.finditer('\#', d_filename)):
if m.end() <= displength:
print comfilename[m.start():m.end()]
# find occurance in c_filename, then replace into d_filname so special characters are brought across
newchar = comfilename[m.start():m.end()]
print 'newchar:' + str(newchar)
d_filename = d_filename[:m.start()] + str(
newchar) + d_filename[m.end():]
print 'd_filename:' + str(d_filename)
dispname = d_filename[:displength]
print('dispname : ' + dispname)
splitit = []
watchcomic_split = []
logger.fdebug("filename comic and issue: " + comic_andiss)
#changed this from '' to ' '
comic_iss_b4 = re.sub('[\-\:\,]', ' ', comic_andiss)
comic_iss = comic_iss_b4.replace('.', ' ')
comic_iss = re.sub('[\s+]', ' ', comic_iss).strip()
logger.fdebug("adjusted comic and issue: " + str(comic_iss))
#remove 'the' from here for proper comparisons.
if ' the ' in comic_iss.lower():
comic_iss = re.sub('\\bthe\\b', '', comic_iss).strip()
splitit = comic_iss.split(None)
logger.fdebug("adjusting from: " + str(comic_iss_b4) + " to: " +
str(comic_iss))
#here we cycle through the Watchlist looking for a match.
while (cm_cn < watchcnt):
#setup the watchlist
comname = ComicName[cm_cn]
comyear = ComicYear[cm_cn]
compub = ComicPublisher[cm_cn]
comtotal = ComicTotal[cm_cn]
comicid = ComicID[cm_cn]
watch_location = ComicLocation[cm_cn]
# there shouldn't be an issue in the comic now, so let's just assume it's all gravy.
splitst = len(splitit)
watchcomic_split = helpers.cleanName(comname)
watchcomic_split = re.sub('[\-\:\,\.]', ' ',
watchcomic_split).split(None)
logger.fdebug(
str(splitit) + " file series word count: " + str(splitst))
logger.fdebug(
str(watchcomic_split) + " watchlist word count: " +
str(len(watchcomic_split)))
if (splitst) != len(watchcomic_split):
logger.fdebug("incorrect comic lengths...not a match")
# if str(splitit[0]).lower() == "the":
# logger.fdebug("THE word detected...attempting to adjust pattern matching")
# splitit[0] = splitit[4:]
else:
logger.fdebug("length match..proceeding")
n = 0
scount = 0
logger.fdebug("search-length: " + str(splitst))
logger.fdebug("Watchlist-length: " +
str(len(watchcomic_split)))
while (n <= (splitst) - 1):
logger.fdebug("splitit: " + str(splitit[n]))
if n < (splitst) and n < len(watchcomic_split):
logger.fdebug(
str(n) + " Comparing: " +
str(watchcomic_split[n]) + " .to. " +
str(splitit[n]))
if '+' in watchcomic_split[n]:
watchcomic_split[n] = re.sub(
'+', '', str(watchcomic_split[n]))
if str(watchcomic_split[n].lower()) in str(
splitit[n].lower()) and len(
watchcomic_split[n]) >= len(splitit[n]):
logger.fdebug("word matched on : " +
str(splitit[n]))
scount += 1
#elif ':' in splitit[n] or '-' in splitit[n]:
# splitrep = splitit[n].replace('-', '')
# print ("non-character keyword...skipped on " + splitit[n])
elif str(splitit[n]).lower().startswith('v'):
logger.fdebug("possible versioning..checking")
#we hit a versioning # - account for it
if splitit[n][1:].isdigit():
comicversion = str(splitit[n])
logger.fdebug("version found: " +
str(comicversion))
else:
logger.fdebug("Comic / Issue section")
if splitit[n].isdigit():
logger.fdebug("issue detected")
else:
logger.fdebug("non-match for: " + str(splitit[n]))
pass
n += 1
#set the match threshold to 80% (for now)
# if it's less than 80% consider it a non-match and discard.
#splitit has to splitit-1 because last position is issue.
wordcnt = int(scount)
logger.fdebug("scount:" + str(wordcnt))
totalcnt = int(splitst)
logger.fdebug("splitit-len:" + str(totalcnt))
spercent = (wordcnt / totalcnt) * 100
logger.fdebug("we got " + str(spercent) + " percent.")
if int(spercent) >= 80:
logger.fdebug("it's a go captain... - we matched " +
str(spercent) + "%!")
logger.fdebug("this should be a match!")
logger.fdebug("issue we found for is : " + str(comiss))
#set the year to the series we just found ;)
result_comyear = comyear
#issue comparison now as well
logger.info(u"Found " + comname + " (" + str(comyear) +
") issue: " + str(comiss))
watchmatch = str(comicid)
dispname = DisplayName[cm_cn]
foundonwatch = "True"
break
elif int(spercent) < 80:
logger.fdebug("failure - we only got " + str(spercent) +
"% right!")
cm_cn += 1
if foundonwatch == "False":
watchmatch = None
#---if it's not a match - send it to the importer.
n = 0
if volyr is None:
if result_comyear is None:
result_comyear = '0000' #no year in filename basically.
else:
if result_comyear is None:
result_comyear = volyr
if volno is None:
if volyr is None:
vol_label = None
else:
vol_label = volyr
else:
vol_label = volno
print("adding " + com_NAME + " to the import-queue!")
impid = com_NAME + "-" + str(result_comyear) + "-" + str(comiss)
print("impid: " + str(impid))
import_by_comicids.append({
"impid":
impid,
"watchmatch":
watchmatch,
"displayname":
dispname,
"comicname":
com_NAME,
"comicyear":
result_comyear,
"volume":
vol_label,
"comfilename":
comfilename,
"comlocation":
comlocation.decode(mylar.SYS_ENCODING)
})
if len(watch_kchoice) > 0:
watchchoice['watchlist'] = watch_kchoice
print("watchchoice: " + str(watchchoice))
logger.info("I have found " + str(watchfound) + " out of " +
str(comiccnt) +
" comics for series that are being watched.")
wat = 0
comicids = []
if watchfound > 0:
if mylar.IMP_MOVE:
logger.info(
"You checked off Move Files...so that's what I'm going to do"
)
#check to see if Move Files is enabled.
#if not being moved, set the archive bit.
print("Moving files into appropriate directory")
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comlocation = watch_the_list['ComicLocation']
watch_comicid = watch_the_list['ComicID']
watch_comicname = watch_the_list['ComicName']
watch_comicyear = watch_the_list['ComicYear']
watch_comiciss = watch_the_list['ComicIssue']
print("ComicLocation: " + str(watch_comlocation))
orig_comlocation = watch_the_list['OriginalLocation']
orig_filename = watch_the_list['OriginalFilename']
print("Orig. Location: " + str(orig_comlocation))
print("Orig. Filename: " + str(orig_filename))
#before moving check to see if Rename to Mylar structure is enabled.
if mylar.IMP_RENAME:
print(
"Renaming files according to configuration details : "
+ str(mylar.FILE_FORMAT))
renameit = helpers.rename_param(
watch_comicid, watch_comicname, watch_comicyear,
watch_comiciss)
nfilename = renameit['nfilename']
dst_path = os.path.join(watch_comlocation, nfilename)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
else:
print(
"Renaming files not enabled, keeping original filename(s)"
)
dst_path = os.path.join(watch_comlocation,
orig_filename)
#os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
#src = os.path.join(, str(nfilename + ext))
print("I'm going to move " + str(orig_comlocation) +
" to .." + str(dst_path))
try:
shutil.move(orig_comlocation, dst_path)
except (OSError, IOError):
logger.info(
"Failed to move directory - check directories and manually re-run."
)
wat += 1
else:
# if move files isn't enabled, let's set all found comics to Archive status :)
while (wat < watchfound):
watch_the_list = watchchoice['watchlist'][wat]
watch_comicid = watch_the_list['ComicID']
watch_issue = watch_the_list['ComicIssue']
print("ComicID: " + str(watch_comicid))
print("Issue#: " + str(watch_issue))
issuechk = myDB.selectone(
"SELECT * from issues where ComicID=? AND INT_IssueNumber=?",
[watch_comicid, watch_issue]).fetchone()
if issuechk is None:
print("no matching issues for this comic#")
else:
print("...Existing status: " + str(issuechk['Status']))
control = {"IssueID": issuechk['IssueID']}
values = {"Status": "Archived"}
print("...changing status of " +
str(issuechk['Issue_Number']) + " to Archived ")
myDB.upsert("issues", values, control)
if str(watch_comicid) not in comicids:
comicids.append(watch_comicid)
wat += 1
if comicids is None: pass
else:
c_upd = len(comicids)
c = 0
while (c < c_upd):
print("Rescanning.. " + str(c))
updater.forceRescan(c)
if not len(import_by_comicids):
return "Completed"
if len(import_by_comicids) > 0:
import_comicids['comic_info'] = import_by_comicids
print("import comicids: " + str(import_by_comicids))
return import_comicids, len(import_by_comicids)
def folder_create(self, booktype=None, update_loc=None):
# dictionary needs to passed called comic with
# {'ComicPublisher', 'CorrectedType, 'Type', 'ComicYear', 'ComicName', 'ComicVersion'}
# or pass in comicid value from __init__
# setup default location here
if update_loc is not None:
comic_location = update_loc['temppath']
enforce_format = update_loc['tempff']
folder_format = update_loc['tempformat']
comicid = update_loc['comicid']
else:
comic_location = mylar.CONFIG.DESTINATION_DIR
enforce_format = False
folder_format = mylar.CONFIG.FOLDER_FORMAT
if folder_format is None:
folder_format = '$Series ($Year)'
if mylar.OS_DETECT == 'Windows':
if '/' in folder_format:
folder_format = re.sub('/', '\\', folder_format).strip()
else:
if '\\' in folder_format:
folder_format = folder_format.replace('\\', '/').strip()
u_comicnm = self.comic['ComicName']
# let's remove the non-standard characters here that will break filenaming / searching.
comicname_filesafe = helpers.filesafe(u_comicnm)
comicdir = comicname_filesafe
series = comicdir
if series[-1:] == '.':
series[:-1]
publisher = re.sub('!', '',
self.comic['ComicPublisher']) # thanks Boom!
publisher = helpers.filesafe(publisher)
if booktype is not None:
if self.comic['Corrected_Type'] is not None:
if self.comic['Corrected_Type'] != booktype:
booktype = booktype
else:
booktype = self.comic['Corrected_Type']
else:
booktype = booktype
else:
booktype = self.comic['Type']
if any([booktype is None, booktype == 'None', booktype == 'Print'
]) or all([
booktype != 'Print', mylar.CONFIG.FORMAT_BOOKTYPE is False
]):
chunk_fb = re.sub('\$Type', '', folder_format)
chunk_b = re.compile(r'\s+')
chunk_folder_format = chunk_b.sub(' ', chunk_fb)
if booktype != 'Print':
booktype = 'None'
else:
chunk_folder_format = folder_format
if any([self.comic['ComicVersion'] is None, booktype != 'Print']):
comicVol = 'None'
else:
comicVol = self.comic['ComicVersion']
#if comversion is None, remove it so it doesn't populate with 'None'
if comicVol == 'None':
chunk_f_f = re.sub('\$VolumeN', '', chunk_folder_format)
chunk_f = re.compile(r'\s+')
chunk_folder_format = chunk_f.sub(' ', chunk_f_f)
chunk_folder_format = re.sub("[()|[]]", '',
chunk_folder_format).strip()
ccf = chunk_folder_format.find('/ ')
if ccf != -1:
chunk_folder_format = chunk_folder_format[:ccf +
1] + chunk_folder_format[
ccf + 2:]
ccf = chunk_folder_format.find('\ ')
if ccf != -1:
chunk_folder_format = chunk_folder_format[:ccf +
1] + chunk_folder_format[
ccf + 2:]
#do work to generate folder path
values = {
'$Series': series,
'$Publisher': publisher,
'$Year': self.comic['ComicYear'],
'$series': series.lower(),
'$publisher': publisher.lower(),
'$VolumeY': 'V' + self.comic['ComicYear'],
'$VolumeN': comicVol.upper(),
'$Annual': 'Annual',
'$Type': booktype
}
if update_loc is not None:
#set the paths here with the seperator removed allowing for cross-platform altering.
ccdir = pathlib.PurePath(comic_location)
ddir = pathlib.PurePath(mylar.CONFIG.DESTINATION_DIR)
dlc = pathlib.PurePath(self.comic['ComicLocation'])
path_convert = True
i = 0
bb = []
while i < len(dlc.parts):
try:
if dlc.parts[i] == ddir.parts[i]:
i += 1
continue
else:
bb.append(dlc.parts[i])
i += 1 #print('d.parts: %s' % ccdir.parts[i])
except IndexError:
bb.append(dlc.parts[i])
i += 1
bb_tuple = pathlib.PurePath(os.path.sep.join(bb))
try:
com_base = pathlib.PurePath(dlc).relative_to(ddir)
except ValueError as e:
#if the original path is not located in the same path as the ComicLocation (destination_dir).
#this can happen when manually altered to a new path, or thru various changes to the ComicLocation path over time.
#ie. ValueError: '/mnt/Comics/Death of Wolverine The Logan Legacy-(2014)' does not start with '/mnt/mediavg/Comics/Comics-2'
dir_fix = []
dir_parts = pathlib.PurePath(dlc).parts
for dp in dir_parts:
try:
if self.comic['ComicYear'] is not None:
if self.comic['ComicYear'] in dp:
break
if self.comic['ComicName'] is not None:
if self.comic['ComicName'] in dp:
break
if self.comic['ComicPublisher'] is not None:
if self.comic['ComicPublisher'] in dp:
break
if self.comic['ComicVersion'] is not None:
if self.comic['ComicVersion'] in dp:
break
dir_fix.append(dp)
except:
pass
if len(dir_fix) > 0:
spath = ''
t = 0
while (t < len(dir_parts)):
newpath = os.path.join(spath, dir_parts[t])
t += 1
com_base = newpath
#path_convert = False
#print('com_base: %s' % com_base)
#detect comiclocation path based on OS so that the path seperators are correct
#have to figure out how to determine OS of original location...
if mylar.OS_DETECT == 'Windows':
p_path = pathlib.PureWindowsPath(ccdir)
else:
p_path = pathlib.PurePosixPath(ccdir)
if enforce_format is True:
first = helpers.replace_all(chunk_folder_format, values)
if mylar.CONFIG.REPLACE_SPACES:
#mylar.CONFIG.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
first = first.replace(' ', mylar.CONFIG.REPLACE_CHAR)
comlocation = str(p_path.joinpath(first))
else:
comlocation = str(p_path.joinpath(com_base))
return {
'comlocation': comlocation,
'path_convert': path_convert,
'comicid': comicid
}
else:
ddir = pathlib.PurePath(mylar.CONFIG.DESTINATION_DIR)
i = 0
bb = []
while i < len(ddir.parts):
try:
bb.append(ddir.parts[i])
i += 1
except IndexError:
break
bb2 = bb[0]
bb.pop(0)
bb_tuple = pathlib.PurePath(os.path.sep.join(bb))
logger.fdebug('bb_tuple: %s' % bb_tuple)
if mylar.OS_DETECT == 'Windows':
p_path = pathlib.PureWindowsPath(
pathlib.PurePath(bb2).joinpath(bb_tuple))
else:
p_path = pathlib.PurePosixPath(
pathlib.PurePath(bb2).joinpath(bb_tuple))
logger.fdebug('p_path: %s' % p_path)
first = helpers.replace_all(chunk_folder_format, values)
logger.fdebug('first-1: %s' % first)
if mylar.CONFIG.REPLACE_SPACES:
first = first.replace(' ', mylar.CONFIG.REPLACE_CHAR)
logger.fdebug('first-2: %s' % first)
comlocation = str(p_path.joinpath(first))
logger.fdebug('comlocation: %s' % comlocation)
#try:
# if folder_format == '':
# #comlocation = pathlib.PurePath(comiclocation).joinpath(comicdir, '(%s)') % comic['SeriesYear']
# comlocation = os.path.join(comic_location, comicdir, " (" + comic['SeriesYear'] + ")")
# else:
#except TypeError as e:
# if comic_location is None:
# logger.error('[ERROR] %s' % e)
# logger.error('No Comic Location specified. This NEEDS to be set before anything can be added successfully.')
# return
# else:
# logger.error('[ERROR] %s' % e)
# return
#except Exception as e:
# logger.error('[ERROR] %s' % e)
# logger.error('Cannot determine Comic Location path properly. Check your Comic Location and Folder Format for any errors.')
# return
if comlocation == "":
logger.error(
'There is no Comic Location Path specified - please specify one in Config/Web Interface.'
)
return
return {'comlocation': comlocation, 'subpath': bb_tuple}
def historycheck(self, nzbinfo):
nzbid = nzbinfo['NZBID']
history = self.server.history(True)
found = False
destdir = None
double_pp = False
hq = [hs for hs in history if hs['NZBID'] == nzbid
] # and ('SUCCESS' in hs['Status'] or ('COPY' in hs['Status']))]
if len(hq) > 0:
logger.fdebug(
'found matching completed item in history. Job has a status of %s'
% hq[0]['Status'])
if len(hq[0]['ScriptStatuses']) > 0:
for x in hq[0]['ScriptStatuses']:
if 'comicrn' in x['Name'].lower():
double_pp = True
break
if all([len(hq[0]['Parameters']) > 0, double_pp is False]):
for x in hq[0]['Parameters']:
if all(
['comicrn' in x['Name'].lower(), x['Value'] == 'yes']):
double_pp = True
break
if double_pp is True:
logger.warn(
'ComicRN has been detected as being active for this category & download. Completed Download Handling will NOT be performed due to this.'
)
logger.warn(
'Either disable Completed Download Handling for NZBGet within Mylar, or remove ComicRN from your category script in NZBGet.'
)
return {'status': 'double-pp', 'failed': False}
if all([
'SUCCESS' in hq[0]['Status'],
(hq[0]['FileSizeMB'] * .95) <= hq[0]['DownloadedSizeMB'] <=
(hq[0]['FileSizeMB'] * 1.05)
]):
logger.fdebug('%s has final file size of %sMB' %
(hq[0]['Name'], hq[0]['DownloadedSizeMB']))
if os.path.isdir(hq[0]['DestDir']):
destdir = hq[0]['DestDir']
logger.fdebug('location found @ %s' % destdir)
elif mylar.CONFIG.NZBGET_DIRECTORY is None:
logger.fdebug(
'Unable to locate path (%s) on the machine that is running Mylar. If Mylar and nzbget are on separate machines, you need to set a directory location that is accessible to both'
% hq[0]['DestDir'])
return {'status': 'file not found', 'failed': False}
elif all([
'COPY' in hq[0]['Status'],
int(hq[0]['FileSizeMB']) > 0,
hq[0]['DeleteStatus'] == 'COPY'
]):
if hq[0]['Deleted'] is False:
config = self.server.config()
cDestDir = None
for x in config:
if x['Name'] == 'TempDir':
cTempDir = x['Value']
elif x['Name'] == 'DestDir':
cDestDir = x['Value']
if cDestDir is not None:
break
if cTempDir in hq[0]['DestDir']:
destdir2 = re.sub(cTempDir, cDestDir,
hq[0]['DestDir']).strip()
if not destdir2.endswith(os.sep):
destdir2 = destdir2 + os.sep
destdir = os.path.join(destdir2, hq[0]['Name'])
logger.fdebug('NZBGET Destination dir set to: %s' %
destdir)
else:
history_del = self.server.editqueue(
'HistoryMarkBad', '', hq[0]['NZBID'])
if history_del is False:
logger.fdebug(
'[NZBGET] Unable to delete item (%s)from history so I can redownload a clean copy.'
% hq[0]['NZBName'])
return {'status': 'failure', 'failed': False}
else:
logger.fdebug(
'[NZBGET] Successfully removed prior item (%s) from history. Attempting to get another version.'
% hq[0]['NZBName'])
return {
'status':
True,
'name':
re.sub('.nzb', '', hq[0]['NZBName']).strip(),
'location':
os.path.abspath(
os.path.join(hq[0]['DestDir'], os.pardir)),
'failed':
True,
'issueid':
nzbinfo['issueid'],
'comicid':
nzbinfo['comicid'],
'apicall':
True,
'ddl':
False,
'download_info':
nzbinfo['download_info']
}
elif 'FAILURE' in hq[0]['Status']:
logger.warn(
'item is in a %s status. Considering this a FAILED attempted download for NZBID %s - %s'
% (hq[0]['Status'], hq[0]['NZBID'], hq[0]['NZBName']))
return {
'status':
True,
'name':
re.sub('.nzb', '', hq[0]['NZBName']).strip(),
'location':
os.path.abspath(os.path.join(hq[0]['DestDir'], os.pardir)),
'failed':
True,
'issueid':
nzbinfo['issueid'],
'comicid':
nzbinfo['comicid'],
'apicall':
True,
'ddl':
False,
'download_info':
nzbinfo['download_info']
}
else:
logger.warn(
'no file found where it should be @ %s - is there another script that moves things after completion ?'
% hq[0]['DestDir'])
return {'status': 'file not found', 'failed': False}
if mylar.CONFIG.NZBGET_DIRECTORY is not None:
logger.fdebug('DestDir being passed to cdh conversion is: %s' %
hq[0]['DestDir'])
cdh = cdh_mapping.CDH_MAP(hq[0]['DestDir'],
nzbget=True,
nzbget_server=self.server)
destdir = cdh.the_sequence()
if destdir is not None:
logger.fdebug(
'NZBGet Destination folder Successfully set via config to: %s'
% destdir)
else:
logger.fdebug(
'Unable to locate path (%s) on the machine that is running Mylar. If Mylar and nzbget are on separate machines, you need to set a directory location that is accessible to both'
% hq[0]['DestDir'])
return {'status': 'file not found', 'failed': False}
if destdir is not None:
return {
'status': True,
'name': re.sub('.nzb', '', hq[0]['Name']).strip(),
'location': destdir,
'failed': False,
'issueid': nzbinfo['issueid'],
'comicid': nzbinfo['comicid'],
'apicall': True,
'ddl': False,
'download_info': nzbinfo['download_info']
}
else:
logger.warn('Could not find completed NZBID %s in history' % nzbid)
return {'status': False}
def Process(self):
module = '[FAILED-DOWNLOAD]'
myDB = db.DBConnection()
if self.nzb_name and self.nzb_folder:
self._log('Failed download has been detected: ' + self.nzb_name + ' in ' + self.nzb_folder)
#since this has already been passed through the search module, which holds the IssueID in the nzblog,
#let's find the matching nzbname and pass it the IssueID in order to mark it as Failed and then return
#to the search module and continue trucking along.
nzbname = self.nzb_name
#remove extensions from nzb_name if they somehow got through (Experimental most likely)
extensions = ('.cbr', '.cbz')
if nzbname.lower().endswith(extensions):
fd, ext = os.path.splitext(nzbname)
self._log("Removed extension from nzb: " + ext)
nzbname = re.sub(str(ext), '', str(nzbname))
#replace spaces
nzbname = re.sub(' ', '.', str(nzbname))
nzbname = re.sub('[\,\:\?\'\(\)]', '', str(nzbname))
nzbname = re.sub('[\&]', 'and', str(nzbname))
nzbname = re.sub('_', '.', str(nzbname))
logger.fdebug(module + ' After conversions, nzbname is : ' + str(nzbname))
self._log("nzbname: " + str(nzbname))
nzbiss = myDB.selectone("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
self._log("Failure - could not initially locate nzbfile in my database to rename.")
logger.fdebug(module + ' Failure - could not locate nzbfile initially')
# if failed on spaces, change it all to decimals and try again.
nzbname = re.sub('_', '.', str(nzbname))
self._log("trying again with this nzbname: " + str(nzbname))
logger.fdebug(module + ' Trying to locate nzbfile again with nzbname of : ' + str(nzbname))
nzbiss = myDB.selectone("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
logger.error(module + ' Unable to locate downloaded file to rename. PostProcessing aborted.')
self._log('Unable to locate downloaded file to rename. PostProcessing aborted.')
self.valreturn.append({"self.log": self.log,
"mode": 'stop'})
return self.queue.put(self.valreturn)
else:
self._log("I corrected and found the nzb as : " + str(nzbname))
logger.fdebug(module + ' Auto-corrected and found the nzb as : ' + str(nzbname))
issueid = nzbiss['IssueID']
else:
issueid = nzbiss['IssueID']
logger.fdebug(module + ' Issueid: ' + str(issueid))
sarc = nzbiss['SARC']
#use issueid to get publisher, series, year, issue number
else:
issueid = self.issueid
nzbiss = myDB.selectone("SELECT * from nzblog WHERE IssueID=?", [issueid]).fetchone()
if nzbiss is None:
logger.info(module + ' Cannot locate corresponding record in download history. This will be implemented soon.')
self.valreturn.append({"self.log": self.log,
"mode": 'stop'})
return self.queue.put(self.valreturn)
nzbname = nzbiss['NZBName']
# find the provider.
self.prov = nzbiss['PROVIDER']
logger.info(module + ' Provider: ' + self.prov)
# grab the id.
self.id = nzbiss['ID']
logger.info(module + ' ID: ' + self.id)
annchk = "no"
if 'annual' in nzbname.lower():
logger.info(module + ' Annual detected.')
annchk = "yes"
issuenzb = myDB.selectone("SELECT * from annuals WHERE IssueID=? AND ComicName NOT NULL", [issueid]).fetchone()
else:
issuenzb = myDB.selectone("SELECT * from issues WHERE IssueID=? AND ComicName NOT NULL", [issueid]).fetchone()
if issuenzb is not None:
logger.info(module + ' issuenzb found.')
if helpers.is_number(issueid):
sandwich = int(issuenzb['IssueID'])
else:
logger.info(module + ' issuenzb not found.')
#if it's non-numeric, it contains a 'G' at the beginning indicating it's a multi-volume
#using GCD data. Set sandwich to 1 so it will bypass and continue post-processing.
if 'S' in issueid:
sandwich = issueid
elif 'G' in issueid or '-' in issueid:
sandwich = 1
try:
if helpers.is_number(sandwich):
if sandwich < 900000:
# if sandwich is less than 900000 it's a normal watchlist download. Bypass.
pass
else:
logger.info('Failed download handling for story-arcs and one-off\'s are not supported yet. Be patient!')
self._log(' Unable to locate downloaded file to rename. PostProcessing aborted.')
self.valreturn.append({"self.log": self.log,
"mode": 'stop'})
return self.queue.put(self.valreturn)
except NameError:
logger.info('sandwich was not defined. Post-processing aborted...')
self.valreturn.append({"self.log": self.log,
"mode": 'stop'})
return self.queue.put(self.valreturn)
comicid = issuenzb['ComicID']
issuenumOG = issuenzb['Issue_Number']
logger.info(module + ' Successfully detected as : ' + issuenzb['ComicName'] + ' issue: ' + str(issuenzb['Issue_Number']) + ' that was downloaded using ' + self.prov)
self._log('Successfully detected as : ' + issuenzb['ComicName'] + ' issue: ' + str(issuenzb['Issue_Number']) + ' downloaded using ' + self.prov)
logger.info(module + ' Marking as a Failed Download.')
self._log('Marking as a Failed Download.')
ctrlVal = {"IssueID": issueid}
Vals = {"Status": 'Failed'}
myDB.upsert("issues", Vals, ctrlVal)
ctrlVal = {"ID": self.id,
"Provider": self.prov,
"NZBName": nzbname}
Vals = {"Status": 'Failed',
"ComicName": issuenzb['ComicName'],
"Issue_Number": issuenzb['Issue_Number'],
"IssueID": issueid,
"ComicID": comicid,
"DateFailed": helpers.now()}
myDB.upsert("failed", Vals, ctrlVal)
logger.info(module + ' Successfully marked as Failed.')
self._log('Successfully marked as Failed.')
if mylar.CONFIG.FAILED_AUTO:
logger.info(module + ' Sending back to search to see if we can find something that will not fail.')
self._log('Sending back to search to see if we can find something better that will not fail.')
self.valreturn.append({"self.log": self.log,
"mode": 'retry',
"issueid": issueid,
"comicid": comicid,
"comicname": issuenzb['ComicName'],
"issuenumber": issuenzb['Issue_Number'],
"annchk": annchk})
return self.queue.put(self.valreturn)
else:
logger.info(module + ' Stopping search here as automatic handling of failed downloads is not enabled *hint*')
self._log('Stopping search here as automatic handling of failed downloads is not enabled *hint*')
self.valreturn.append({"self.log": self.log,
"mode": 'stop'})
return self.queue.put(self.valreturn)
def dbUpdate(ComicIDList=None):
myDB = db.DBConnection()
#print "comicidlist:" + str(ComicIDList)
if ComicIDList is None:
comiclist = myDB.select(
'SELECT ComicID, ComicName from comics WHERE Status="Active" or Status="Loading" order by LastUpdated ASC'
)
else:
comiclist = ComicIDList
logger.info('Starting update for %i active comics' % len(comiclist))
for comic in comiclist:
if ComicIDList is None:
comicid = comic[0]
else:
comicid = comic
#print "comicid:" + str(comicid)
mismatch = "no"
if not mylar.CV_ONLY or comicid[:1] == "G":
CV_EXcomicid = myDB.action(
"SELECT * from exceptions WHERE ComicID=?",
[comicid]).fetchone()
if CV_EXcomicid is None: pass
else:
if CV_EXcomicid['variloop'] == '99':
mismatch = "yes"
if comicid[:1] == "G":
mylar.importer.GCDimport(comicid)
else:
mylar.importer.addComictoDB(comicid, mismatch)
else:
if mylar.CV_ONETIMER == 1:
logger.fdebug('CV_OneTimer option enabled...')
#in order to update to JUST CV_ONLY, we need to delete the issues for a given series so it's a clean refresh.
logger.fdebug(
'Gathering the status of all issues for the series.')
issues = myDB.select('SELECT * FROM issues WHERE ComicID=?',
[comicid])
if mylar.ANNUALS_ON:
issues += myDB.select(
'SELECT * FROM annuals WHERE ComicID=?', [comicid])
#store the issues' status for a given comicid, after deleting and readding, flip the status back to what it is currently.
logger.fdebug('Deleting all issue data.')
myDB.select('DELETE FROM issues WHERE ComicID=?', [comicid])
myDB.select('DELETE FROM annuals WHERE ComicID=?', [comicid])
logger.fdebug(
'Refreshing the series and pulling in new data using only CV.'
)
mylar.importer.addComictoDB(comicid,
mismatch,
calledfrom='dbupdate')
issues_new = myDB.select(
'SELECT * FROM issues WHERE ComicID=?', [comicid])
annuals = []
ann_list = []
if mylar.ANNUALS_ON:
annuals_list = myDB.select(
'SELECT * FROM annuals WHERE ComicID=?', [comicid])
ann_list += annuals_list
issues_new += annuals_list
icount = 0
logger.fdebug(
'Attempting to put the Statuses back how they were.')
for issue in issues:
for issuenew in issues_new:
if issuenew['IssueID'] == issue['IssueID'] and issuenew[
'Status'] != issue['Status']:
#if the status is now Downloaded, keep status.
logger.info('existing status: ' +
str(issuenew['Status']))
logger.info('new status: ' + str(issue['Status']))
if issuenew['Status'] == 'Downloaded' or issue[
'Status'] == 'Snatched':
break
#change the status to the previous status
ctrlVAL = {'IssueID': issue['IssueID']}
newVAL = {'Status': issue['Status']}
if any(d['IssueID'] == str(issue['IssueID'])
for d in ann_list):
logger.fdebug('annual detected for ' +
str(issue['IssueID']) + ' #: ' +
str(issue['Issue_Number']))
myDB.upsert("Annuals", newVAL, ctrlVAL)
else:
myDB.upsert("Issues", newVAL, ctrlVAL)
icount += 1
break
logger.info(
'In converting data to CV only, I changed the status of ' +
str(icount) + ' issues.')
mylar.CV_ONETIMER = 0
else:
mylar.importer.addComictoDB(comicid, mismatch)
#check global skipped2wanted status here
#if mylar.GLOBAL_SKIPPED2WANTED:
# logger.fdebug('Global change for ' + str(comicid) + ' - Marking all issues not present as Wanted.')
# mylar.webserve.skipped2wanted(comicid,True)
time.sleep(5) #pause for 5 secs so dont hammer CV and get 500 error
logger.info('Update complete')
def upcoming_update(ComicID,
ComicName,
IssueNumber,
IssueDate,
forcecheck=None,
futurepull=None,
altissuenumber=None):
# here we add to upcoming table...
myDB = db.DBConnection()
dspComicName = ComicName #to make sure that the word 'annual' will be displayed on screen
if 'annual' in ComicName.lower():
adjComicName = re.sub("\\bannual\\b", "",
ComicName.lower()) # for use with comparisons.
logger.fdebug('annual detected - adjusting name to : ' + adjComicName)
else:
adjComicName = ComicName
controlValue = {"ComicID": ComicID}
newValue = {
"ComicName": adjComicName,
"IssueNumber": str(IssueNumber),
"DisplayComicName": dspComicName,
"IssueDate": str(IssueDate)
}
#let's refresh the series here just to make sure if an issue is available/not.
mismatch = "no"
CV_EXcomicid = myDB.action("SELECT * from exceptions WHERE ComicID=?",
[ComicID]).fetchone()
if CV_EXcomicid is None: pass
else:
if CV_EXcomicid['variloop'] == '99':
mismatch = "yes"
lastupdatechk = myDB.action("SELECT * FROM comics WHERE ComicID=?",
[ComicID]).fetchone()
if lastupdatechk is None:
pullupd = "yes"
else:
c_date = lastupdatechk['LastUpdated']
if c_date is None:
logger.error(
lastupdatechk['ComicName'] +
' failed during a previous add /refresh. Please either delete and readd the series, or try a refresh of the series.'
)
return
c_obj_date = datetime.datetime.strptime(c_date, "%Y-%m-%d %H:%M:%S")
n_date = datetime.datetime.now()
absdiff = abs(n_date - c_obj_date)
hours = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 3600.0
# no need to hammer the refresh
# let's check it every 5 hours (or more)
#pullupd = "yes"
if 'annual' in ComicName.lower():
if mylar.ANNUALS_ON:
issuechk = myDB.action(
"SELECT * FROM annuals WHERE ComicID=? AND Issue_Number=?",
[ComicID, IssueNumber]).fetchone()
else:
logger.fdebug(
'Annual detected, but annuals not enabled. Ignoring result.')
return
else:
issuechk = myDB.action(
"SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?",
[ComicID, IssueNumber]).fetchone()
if issuechk is None and altissuenumber is not None:
logger.info('altissuenumber is : ' + str(altissuenumber))
issuechk = myDB.action(
"SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?",
[ComicID, helpers.issuedigits(altissuenumber)]).fetchone()
if issuechk is None:
if futurepull is None:
logger.fdebug(
adjComicName + ' Issue: ' + str(IssueNumber) +
' not present in listings to mark for download...updating comic and adding to Upcoming Wanted Releases.'
)
# we need to either decrease the total issue count, OR indicate that an issue is upcoming.
upco_results = myDB.action(
"SELECT COUNT(*) FROM UPCOMING WHERE ComicID=?",
[ComicID]).fetchall()
upco_iss = upco_results[0][0]
#logger.info("upco_iss: " + str(upco_iss))
if int(upco_iss) > 0:
#logger.info("There is " + str(upco_iss) + " of " + str(ComicName) + " that's not accounted for")
newKey = {"ComicID": ComicID}
newVal = {"not_updated_db": str(upco_iss)}
myDB.upsert("comics", newVal, newKey)
elif int(upco_iss) <= 0 and lastupdatechk['not_updated_db']:
#if not_updated_db has a value, and upco_iss is > 0, let's zero it back out cause it's updated now.
newKey = {"ComicID": ComicID}
newVal = {"not_updated_db": ""}
myDB.upsert("comics", newVal, newKey)
if hours > 5 or forcecheck == 'yes':
pullupd = "yes"
logger.fdebug('Now Refreshing comic ' + ComicName +
' to make sure it is up-to-date')
if ComicID[:1] == "G":
mylar.importer.GCDimport(ComicID, pullupd)
else:
mylar.importer.addComictoDB(ComicID, mismatch, pullupd)
else:
logger.fdebug(
'It has not been longer than 5 hours since we last did this...we will wait so we do not hammer things.'
)
return
else:
# if futurepull is not None, let's just update the status and ComicID
# NOTE: THIS IS CREATING EMPTY ENTRIES IN THE FUTURE TABLE. ???
nKey = {"ComicID": ComicID}
nVal = {"Status": "Wanted"}
myDB.upsert("future", nVal, nKey)
if issuechk is not None:
if issuechk['Issue_Number'] == IssueNumber or issuechk[
'Issue_Number'] == altissuenumber:
logger.fdebug(
'Comic series already up-to-date ... no need to refresh at this time.'
)
logger.fdebug('Available to be marked for download - checking...' +
adjComicName + ' Issue: ' +
str(issuechk['Issue_Number']))
logger.fdebug('...Existing status: ' + str(issuechk['Status']))
control = {"IssueID": issuechk['IssueID']}
newValue['IssueID'] = issuechk['IssueID']
if issuechk['Status'] == "Snatched":
values = {"Status": "Snatched"}
newValue['Status'] = "Snatched"
elif issuechk['Status'] == "Downloaded":
values = {"Status": "Downloaded"}
newValue['Status'] = "Downloaded"
#if the status is Downloaded and it's on the pullist - let's mark it so everyone can bask in the glory
elif issuechk['Status'] == "Wanted":
values = {"Status": "Wanted"}
newValue['Status'] = "Wanted"
elif issuechk['Status'] == "Archived":
values = {"Status": "Archived"}
newValue['Status'] = "Archived"
else:
values = {"Status": "Skipped"}
newValue['Status'] = "Skipped"
#was in wrong place :(
else:
logger.fdebug(
'Issues do not match for some reason...weekly new issue: ' +
str(IssueNumber))
return
if mylar.AUTOWANT_UPCOMING:
#for issues not in db - to be added to Upcoming table.
if issuechk is None:
newValue['Status'] = "Wanted"
logger.fdebug(
'...Changing Status to Wanted and throwing it in the Upcoming section since it is not published yet.'
)
#this works for issues existing in DB...
elif issuechk['Status'] == "Skipped":
newValue['Status'] = "Wanted"
values = {"Status": "Wanted"}
logger.fdebug('...New status of Wanted')
elif issuechk['Status'] == "Wanted":
logger.fdebug('...Status already Wanted .. not changing.')
else:
logger.fdebug(
'...Already have issue - keeping existing status of : ' +
str(issuechk['Status']))
if issuechk is None:
myDB.upsert("upcoming", newValue, controlValue)
else:
logger.fdebug('--attempt to find errant adds to Wanted list')
logger.fdebug('UpcomingNewValue: ' + str(newValue))
logger.fdebug('UpcomingcontrolValue: ' + str(controlValue))
if issuechk['IssueDate'] == '0000-00-00' and newValue[
'IssueDate'] != '0000-00-00':
logger.fdebug(
'Found a 0000-00-00 issue - force updating series to try and get it proper.'
)
dateVal = {
"IssueDate": newValue['IssueDate'],
"ComicName": issuechk['ComicName'],
"Status": newValue['Status'],
"IssueNumber": issuechk['Issue_Number']
}
logger.fdebug('updating date in upcoming table to : ' +
str(newValue['IssueDate']))
logger.fdebug('ComicID:' + str(controlValue))
myDB.upsert("upcoming", dateVal, controlValue)
logger.fdebug('Temporarily putting the Issue Date for ' +
str(issuechk['Issue_Number']) + ' to ' +
str(newValue['IssueDate']))
values = {"IssueDate": newValue['IssueDate']}
#if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd='yes')
#else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd='yes')
if 'annual' in ComicName.lower():
myDB.upsert("annuals", values, control)
else:
myDB.upsert("issues", values, control)
if issuechk['Status'] == 'Downloaded' or issuechk[
'Status'] == 'Archived' or issuechk['Status'] == 'Snatched':
logger.fdebug('updating Pull-list to reflect status.')
downstats = {
"Status": issuechk['Status'],
"ComicID": issuechk['ComicID']
}
return downstats
def Process(self):
self._log("nzb name: " + str(self.nzb_name), logger.DEBUG)
self._log("nzb folder: " + str(self.nzb_folder), logger.DEBUG)
logger.fdebug("nzb name: " + str(self.nzb_name))
logger.fdebug("nzb folder: " + str(self.nzb_folder))
if mylar.USE_SABNZBD==0:
logger.fdebug("Not using SABNzbd")
else:
# if the SAB Directory option is enabled, let's use that folder name and append the jobname.
if mylar.SAB_DIRECTORY is not None and mylar.SAB_DIRECTORY is not 'None' and len(mylar.SAB_DIRECTORY) > 4:
self.nzb_folder = os.path.join(mylar.SAB_DIRECTORY, self.nzb_name).encode(mylar.SYS_ENCODING)
#lookup nzb_name in nzblog table to get issueid
#query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals
#http://localhost:8080/sabnzbd/api?mode=set_config§ion=misc&keyword=dirscan_speed&value=5
querysab = str(mylar.SAB_HOST) + "/api?mode=get_config§ion=misc&output=xml&apikey=" + str(mylar.SAB_APIKEY)
#logger.info("querysab_string:" + str(querysab))
file = urllib2.urlopen(querysab)
data = file.read()
file.close()
dom = parseString(data)
try:
sabreps = dom.getElementsByTagName('replace_spaces')[0].firstChild.wholeText
except:
errorm = dom.getElementsByTagName('error')[0].firstChild.wholeText
logger.error(u"Error detected attempting to retrieve SAB data : " + errorm)
return
sabrepd = dom.getElementsByTagName('replace_dots')[0].firstChild.wholeText
logger.fdebug("SAB Replace Spaces: " + str(sabreps))
logger.fdebug("SAB Replace Dots: " + str(sabrepd))
if mylar.USE_NZBGET==1:
logger.fdebug("Using NZBGET")
logger.fdebug("NZB name as passed from NZBGet: " + self.nzb_name)
myDB = db.DBConnection()
nzbname = self.nzb_name
#remove extensions from nzb_name if they somehow got through (Experimental most likely)
extensions = ('.cbr', '.cbz')
if nzbname.lower().endswith(extensions):
fd, ext = os.path.splitext(nzbname)
self._log("Removed extension from nzb: " + ext, logger.DEBUG)
nzbname = re.sub(str(ext), '', str(nzbname))
#replace spaces
nzbname = re.sub(' ', '.', str(nzbname))
nzbname = re.sub('[\,\:\?]', '', str(nzbname))
nzbname = re.sub('[\&]', 'and', str(nzbname))
logger.fdebug("After conversions, nzbname is : " + str(nzbname))
# if mylar.USE_NZBGET==1:
# nzbname=self.nzb_name
self._log("nzbname: " + str(nzbname), logger.DEBUG)
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
self._log("Failure - could not initially locate nzbfile in my database to rename.", logger.DEBUG)
logger.fdebug("Failure - could not locate nzbfile initially.")
# if failed on spaces, change it all to decimals and try again.
nzbname = re.sub('_', '.', str(nzbname))
self._log("trying again with this nzbname: " + str(nzbname), logger.DEBUG)
logger.fdebug("trying again with nzbname of : " + str(nzbname))
nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone()
if nzbiss is None:
logger.error(u"Unable to locate downloaded file to rename. PostProcessing aborted.")
return
else:
self._log("I corrected and found the nzb as : " + str(nzbname))
logger.fdebug("auto-corrected and found the nzb as : " + str(nzbname))
issueid = nzbiss['IssueID']
else:
issueid = nzbiss['IssueID']
logger.fdebug("issueid:" + str(issueid))
sarc = nzbiss['SARC']
#use issueid to get publisher, series, year, issue number
issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone()
if issuenzb is not None:
if helpers.is_number(issueid):
sandwich = int(issuenzb['IssueID'])
else:
#if it's non-numeric, it contains a 'G' at the beginning indicating it's a multi-volume
#using GCD data. Set sandwich to 1 so it will bypass and continue post-processing.
if 'S' in issueid:
sandwich = issueid
elif 'G' in issueid:
sandwich = 1
if helpers.is_number(sandwich):
if sandwich < 900000:
# if sandwich is less than 900000 it's a normal watchlist download. Bypass.
pass
else:
if issuenzb is None or 'S' in sandwich or int(sandwich) >= 900000:
# this has no issueID, therefore it's a one-off or a manual post-proc.
# At this point, let's just drop it into the Comic Location folder and forget about it..
if 'S' in sandwich:
self._log("One-off STORYARC mode enabled for Post-Processing for " + str(sarc))
logger.info("One-off STORYARC mode enabled for Post-Processing for " + str(sarc))
if mylar.STORYARCDIR:
storyarcd = os.path.join(mylar.DESTINATION_DIR, "StoryArcs", sarc)
self._log("StoryArc Directory set to : " + storyarcd, logger.DEBUG)
else:
self._log("Grab-Bag Directory set to : " + mylar.GRABBAG_DIR, logger.DEBUG)
else:
self._log("One-off mode enabled for Post-Processing. All I'm doing is moving the file untouched into the Grab-bag directory.", logger.DEBUG)
logger.info("One-off mode enabled for Post-Processing. Will move into Grab-bag directory.")
self._log("Grab-Bag Directory set to : " + mylar.GRABBAG_DIR, logger.DEBUG)
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
ofilename = filename
path, ext = os.path.splitext(ofilename)
if 'S' in sandwich:
if mylar.STORYARCDIR:
grdst = storyarcd
else:
grdst = mylar.DESTINATION_DIR
else:
if mylar.GRABBAG_DIR:
grdst = mylar.GRABBAG_DIR
else:
grdst = mylar.DESTINATION_DIR
filechecker.validateAndCreateDirectory(grdst, True)
grab_dst = os.path.join(grdst, ofilename)
self._log("Destination Path : " + grab_dst, logger.DEBUG)
logger.info("Destination Path : " + grab_dst)
grab_src = os.path.join(self.nzb_folder, ofilename)
self._log("Source Path : " + grab_src, logger.DEBUG)
logger.info("Source Path : " + grab_src)
logger.info("Moving " + str(ofilename) + " into directory : " + str(grdst))
try:
shutil.move(grab_src, grab_dst)
except (OSError, IOError):
self._log("Failed to move directory - check directories and manually re-run.", logger.DEBUG)
logger.debug("Failed to move directory - check directories and manually re-run.")
return
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log("Failed to remove temporary directory.", logger.DEBUG)
logger.debug("Failed to remove temporary directory - check directory and manually re-run.")
return
logger.debug("Removed temporary directory : " + str(self.nzb_folder))
self._log("Removed temporary directory : " + self.nzb_folder, logger.DEBUG)
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
if 'S' in issueid:
issuearcid = re.sub('S', '', issueid)
logger.info("IssueArcID is : " + str(issuearcid))
ctrlVal = {"IssueArcID": issuearcid}
newVal = {"Status": "Downloaded",
"Location": grab_dst }
myDB.upsert("readinglist",newVal,ctrlVal)
logger.info("updated status to Downloaded")
return self.log
comicid = issuenzb['ComicID']
issuenumOG = issuenzb['Issue_Number']
#issueno = str(issuenum).split('.')[0]
#new CV API - removed all decimals...here we go AGAIN!
issuenum = issuenumOG
issue_except = 'None'
if 'au' in issuenum.lower():
issuenum = re.sub("[^0-9]", "", issuenum)
issue_except = ' AU'
if '.' in issuenum:
iss_find = issuenum.find('.')
iss_b4dec = issuenum[:iss_find]
iss_decval = issuenum[iss_find+1:]
if int(iss_decval) == 0:
iss = iss_b4dec
issdec = int(iss_decval)
issueno = str(iss)
self._log("Issue Number: " + str(issueno), logger.DEBUG)
logger.fdebug("Issue Number: " + str(issueno))
else:
if len(iss_decval) == 1:
iss = iss_b4dec + "." + iss_decval
issdec = int(iss_decval) * 10
else:
iss = iss_b4dec + "." + iss_decval.rstrip('0')
issdec = int(iss_decval.rstrip('0')) * 10
issueno = iss_b4dec
self._log("Issue Number: " + str(iss), logger.DEBUG)
logger.fdebug("Issue Number: " + str(iss))
else:
iss = issuenum
issueno = str(iss)
# issue zero-suppression here
if mylar.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00"
logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N))
if str(len(issueno)) > 1:
if int(issueno) < 10:
self._log("issue detected less than 10", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG)
elif int(issueno) >= 10 and int(issueno) < 100:
self._log("issue detected greater than 10, but less than 100", logger.DEBUG)
if mylar.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss), logger.DEBUG)
else:
self._log("issue detected greater than 100", logger.DEBUG)
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(issueno)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG)
else:
prettycomiss = str(issueno)
self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss), logger.DEBUG)
logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss))
issueyear = issuenzb['IssueDate'][:4]
self._log("Issue Year: " + str(issueyear), logger.DEBUG)
logger.fdebug("Issue Year : " + str(issueyear))
comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone()
publisher = comicnzb['ComicPublisher']
self._log("Publisher: " + publisher, logger.DEBUG)
logger.fdebug("Publisher: " + str(publisher))
#we need to un-unicode this to make sure we can write the filenames properly for spec.chars
series = comicnzb['ComicName'].encode('ascii', 'ignore').strip()
self._log("Series: " + series, logger.DEBUG)
logger.fdebug("Series: " + str(series))
seriesyear = comicnzb['ComicYear']
self._log("Year: " + seriesyear, logger.DEBUG)
logger.fdebug("Year: " + str(seriesyear))
comlocation = comicnzb['ComicLocation']
self._log("Comic Location: " + comlocation, logger.DEBUG)
logger.fdebug("Comic Location: " + str(comlocation))
comversion = comicnzb['ComicVersion']
self._log("Comic Version: " + str(comversion), logger.DEBUG)
logger.fdebug("Comic Version: " + str(comversion))
if comversion is None:
comversion = 'None'
#if comversion is None, remove it so it doesn't populate with 'None'
if comversion == 'None':
chunk_f_f = re.sub('\$VolumeN','',mylar.FILE_FORMAT)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
self._log("No version # found for series - tag will not be available for renaming.", logger.DEBUG)
logger.fdebug("No version # found for series, removing from filename")
logger.fdebug("new format is now: " + str(chunk_file_format))
else:
chunk_file_format = mylar.FILE_FORMAT
#Run Pre-script
if mylar.ENABLE_PRE_SCRIPTS:
nzbn = self.nzb_name #original nzb name
nzbf = self.nzb_folder #original nzb folder
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_pre_scripts(nzbn, nzbf, seriesmetadata )
#rename file and move to new path
#nfilename = series + " " + issueno + " (" + seriesyear + ")"
file_values = {'$Series': series,
'$Issue': prettycomiss,
'$Year': issueyear,
'$series': series.lower(),
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion
}
ofilename = None
for root, dirnames, filenames in os.walk(self.nzb_folder):
for filename in filenames:
if filename.lower().endswith(extensions):
ofilename = filename
path, ext = os.path.splitext(ofilename)
if ofilename is None:
logger.error(u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that " + str(self.nzb_folder) + " exists and is the correct location.")
return
self._log("Original Filename: " + ofilename, logger.DEBUG)
self._log("Original Extension: " + ext, logger.DEBUG)
logger.fdebug("Original Filname: " + str(ofilename))
logger.fdebug("Original Extension: " + str(ext))
if mylar.FILE_FORMAT == '' or not mylar.RENAME_FILES:
self._log("Rename Files isn't enabled...keeping original filename.", logger.DEBUG)
logger.fdebug("Rename Files isn't enabled - keeping original filename.")
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
nfilename = helpers.replace_all(chunk_file_format, file_values)
if mylar.REPLACE_SPACES:
#mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR)
nfilename = re.sub('[\,\:\?]', '', nfilename)
self._log("New Filename: " + nfilename, logger.DEBUG)
logger.fdebug("New Filename: " + str(nfilename))
src = os.path.join(self.nzb_folder, ofilename)
filechecker.validateAndCreateDirectory(comlocation, True)
if mylar.LOWERCASE_FILENAMES:
dst = (comlocation + "/" + nfilename + ext).lower()
else:
dst = comlocation + "/" + nfilename + ext.lower()
self._log("Source:" + src, logger.DEBUG)
self._log("Destination:" + dst, logger.DEBUG)
logger.fdebug("Source: " + str(src))
logger.fdebug("Destination: " + str(dst))
os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext)))
src = os.path.join(self.nzb_folder, str(nfilename + ext))
try:
shutil.move(src, dst)
except (OSError, IOError):
self._log("Failed to move directory - check directories and manually re-run.", logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
#tidyup old path
try:
shutil.rmtree(self.nzb_folder)
except (OSError, IOError):
self._log("Failed to remove temporary directory - check directory and manually re-run.", logger.DEBUG)
self._log("Post-Processing ABORTED.", logger.DEBUG)
return
self._log("Removed temporary directory : " + str(self.nzb_folder), logger.DEBUG)
#delete entry from nzblog table
myDB.action('DELETE from nzblog WHERE issueid=?', [issueid])
#update snatched table to change status to Downloaded
updater.foundsearch(comicid, issueid, down='True')
#force rescan of files
updater.forceRescan(comicid)
logger.info(u"Post-Processing completed for: " + series + " issue: " + str(issuenumOG) )
self._log(u"Post Processing SUCCESSFULL! ", logger.DEBUG)
if mylar.PROWL_ENABLED:
pushmessage = series + '(' + issueyear + ') - issue #' + issuenumOG
logger.info(u"Prowl request")
prowl = notifiers.PROWL()
prowl.notify(pushmessage,"Download and Postprocessing completed")
if mylar.NMA_ENABLED:
nma = notifiers.NMA()
nma.notify(series, str(issueyear), str(issuenumOG))
if mylar.PUSHOVER_ENABLED:
pushmessage = series + ' (' + str(issueyear) + ') - issue #' + str(issuenumOG)
logger.info(u"Pushover request")
pushover = notifiers.PUSHOVER()
pushover.notify(pushmessage, "Download and Post-Processing completed")
# retrieve/create the corresponding comic objects
if mylar.ENABLE_EXTRA_SCRIPTS:
folderp = str(dst) #folder location after move/rename
nzbn = self.nzb_name #original nzb name
filen = str(nfilename + ext) #new filename
#name, comicyear, comicid , issueid, issueyear, issue, publisher
#create the dic and send it.
seriesmeta = []
seriesmetadata = {}
seriesmeta.append({
'name': series,
'comicyear': seriesyear,
'comicid': comicid,
'issueid': issueid,
'issueyear': issueyear,
'issue': issuenum,
'publisher': publisher
})
seriesmetadata['seriesmeta'] = seriesmeta
self._run_extra_scripts(nzbname, self.nzb_folder, filen, folderp, seriesmetadata )
return self.log
def torsend2client(seriesname, issue, seriesyear, linkit, site):
logger.info('matched on ' + str(seriesname))
filename = re.sub('[\'\!\@\#\$\%\:\;\/\\=\?\.]', '', seriesname)
filename = re.sub(' ', '_', filename)
filename += "_" + str(issue) + "_" + str(seriesyear)
if site == 'CBT':
logger.info(linkit)
linkit = str(linkit) + '&passkey=' + str(mylar.CBT_PASSKEY)
if linkit[-7:] != "torrent": # and site != "KAT":
filename += ".torrent"
if mylar.TORRENT_LOCAL and mylar.LOCAL_WATCHDIR is not None:
filepath = os.path.join(mylar.LOCAL_WATCHDIR, filename)
logger.fdebug('filename for torrent set to : ' + filepath)
elif mylar.TORRENT_SEEDBOX and mylar.SEEDBOX_WATCHDIR is not None:
filepath = os.path.join(mylar.CACHE_DIR, filename)
logger.fdebug('filename for torrent set to : ' + filepath)
else:
logger.error(
'No Local Watch Directory or Seedbox Watch Directory specified. Set it and try again.'
)
return "fail"
try:
request = urllib2.Request(linkit)
#request.add_header('User-Agent', str(mylar.USER_AGENT))
request.add_header('Accept-encoding', 'gzip')
if site == 'KAT':
stfind = linkit.find('?')
kat_referrer = linkit[:stfind]
request.add_header('Referer', kat_referrer)
logger.fdebug('KAT Referer set to :' + kat_referrer)
# response = helpers.urlretrieve(urllib2.urlopen(request), filepath)
response = urllib2.urlopen(request)
logger.fdebug('retrieved response.')
if site == 'KAT':
if response.info(
)['content-encoding'] == 'gzip': #.get('Content-Encoding') == 'gzip':
logger.fdebug('gzip detected')
buf = StringIO(response.read())
logger.fdebug('gzip buffered')
f = gzip.GzipFile(fileobj=buf)
logger.fdebug('gzip filed.')
torrent = f.read()
logger.fdebug('gzip read.')
else:
torrent = response.read()
except Exception, e:
logger.warn('Error fetching data from %s: %s' % (site, e))
return "fail"
def torrents(pickfeed=None, seriesname=None, issue=None):
if pickfeed is None:
pickfeed = 1
#else:
# print "pickfeed is " + str(pickfeed)
passkey = mylar.CBT_PASSKEY
srchterm = None
if seriesname:
srchterm = re.sub(' ', '%20', seriesname)
if issue:
srchterm += '%20' + str(issue)
if mylar.KAT_PROXY:
if mylar.KAT_PROXY.endswith('/'):
kat_url = mylar.KAT_PROXY
else:
kat_url = mylar.KAT_PROXY + '/'
else:
kat_url = 'http://kat.ph/'
if pickfeed == 'KAT':
#we need to cycle through both categories (comics & other) - so we loop.
loopit = 2
else:
loopit = 1
lp = 0
title = []
link = []
description = []
seriestitle = []
i = 0
feeddata = []
myDB = db.DBConnection()
torthekat = []
katinfo = {}
while (lp < loopit):
if lp == 0 and loopit == 2:
pickfeed = '2'
elif lp == 1 and loopit == 2:
pickfeed = '5'
feedtype = None
if pickfeed == "1": # cbt rss feed based on followlist
feed = "http://comicbt.com/rss.php?action=browse&passkey=" + str(
passkey) + "&type=dl"
feedtype = ' from the New Releases RSS Feed for comics'
elif pickfeed == "2" and srchterm is not None: # kat.ph search
feed = kat_url + "usearch/" + str(
srchterm) + "%20category%3Acomics%20seeds%3A" + str(
mylar.MINSEEDS) + "/?rss=1"
elif pickfeed == "3": # kat.ph rss feed
feed = kat_url + "usearch/category%3Acomics%20seeds%3A" + str(
mylar.MINSEEDS) + "/?rss=1"
feedtype = ' from the New Releases RSS Feed for comics'
elif pickfeed == "4": #cbt follow link
feed = "http://comicbt.com/rss.php?action=follow&passkey=" + str(
passkey) + "&type=dl"
feedtype = ' from your CBT Followlist RSS Feed'
elif pickfeed == "5" and srchterm is not None: # kat.ph search (category:other since some 0-day comics initially get thrown there until categorized)
feed = kat_url + "usearch/" + str(
srchterm) + "%20category%3Aother%20seeds%3A1/?rss=1"
elif pickfeed == "6": # kat.ph rss feed (category:other so that we can get them quicker if need-be)
feed = kat_url + "usearch/.cbr%20category%3Aother%20seeds%3A" + str(
mylar.MINSEEDS) + "/?rss=1"
feedtype = ' from the New Releases for category Other RSS Feed that contain comics'
elif pickfeed == "7": # cbt series link
# seriespage = "http://comicbt.com/series.php?passkey=" + str(passkey)
feed = "http://comicbt.com/rss.php?action=series&series=" + str(
seriesno) + "&passkey=" + str(passkey)
else:
logger.error('invalid pickfeed denoted...')
return
#print 'feed URL: ' + str(feed)
if pickfeed == "7": # we need to get the series # first
seriesSearch(seriespage, seriesname)
feedme = feedparser.parse(feed)
if pickfeed == "3" or pickfeed == "6" or pickfeed == "2" or pickfeed == "5":
picksite = 'KAT'
elif pickfeed == "1" or pickfeed == "4":
picksite = 'CBT'
for entry in feedme['entries']:
if pickfeed == "3" or pickfeed == "6":
tmpsz = feedme.entries[i].enclosures[0]
feeddata.append({
'site': picksite,
'title': feedme.entries[i].title,
'link': tmpsz['url'],
'pubdate': feedme.entries[i].updated,
'size': tmpsz['length']
})
#print ("Site: KAT")
#print ("Title: " + str(feedme.entries[i].title))
#print ("Link: " + str(tmpsz['url']))
#print ("pubdate: " + str(feedme.entries[i].updated))
#print ("size: " + str(tmpsz['length']))
elif pickfeed == "2" or pickfeed == "5":
tmpsz = feedme.entries[i].enclosures[0]
torthekat.append({
'site': picksite,
'title': feedme.entries[i].title,
'link': tmpsz['url'],
'pubdate': feedme.entries[i].updated,
'size': tmpsz['length']
})
#print ("Site: KAT")
#print ("Title: " + str(feedme.entries[i].title))
#print ("Link: " + str(tmpsz['url']))
#print ("pubdate: " + str(feedme.entries[i].updated))
#print ("size: " + str(tmpsz['length']))
elif pickfeed == "1" or pickfeed == "4":
if pickfeed == "1":
tmpdesc = feedme.entries[i].description
#break it down to get the Size since it's available on THIS CBT feed only.
sizestart = tmpdesc.find('Size:')
sizeend = tmpdesc.find('Leechers:')
sizestart += 5 # to get to the end of the word 'Size:'
tmpsize = tmpdesc[sizestart:sizeend].strip()
fdigits = re.sub("[^0123456789\.]", "", tmpsize).strip()
if '.' in fdigits:
decfind = fdigits.find('.')
wholenum = fdigits[:decfind]
decnum = fdigits[decfind + 1:]
else:
wholenum = fdigits
decnum = 0
if 'MB' in tmpsize:
wholebytes = int(wholenum) * 1048576
wholedecimal = (int(decnum) * 1048576) / 100
justdigits = wholebytes + wholedecimal
else:
#it's 'GB' then
wholebytes = (int(wholenum) * 1024) * 1048576
wholedecimal = ((int(decnum) * 1024) * 1048576) / 100
justdigits = wholebytes + wholedecimal
#Get the # of seeders.
seedstart = tmpdesc.find('Seeders:')
seedend = tmpdesc.find('Added:')
seedstart += 8 # to get to the end of the word 'Seeders:'
tmpseed = tmpdesc[seedstart:seedend].strip()
seeddigits = re.sub("[^0123456789\.]", "", tmpseed).strip()
else:
justdigits = None #size not available in follow-list rss feed
seeddigits = 0 #number of seeders not available in follow-list rss feed
if int(mylar.MINSEEDS) >= int(seeddigits):
feeddata.append({
'site': picksite,
'title': feedme.entries[i].title,
'link': feedme.entries[i].link,
'pubdate': feedme.entries[i].updated,
'size': justdigits
})
#print ("Site: CBT")
#print ("Title: " + str(feeddata[i]['Title']))
#print ("Link: " + str(feeddata[i]['Link']))
#print ("pubdate: " + str(feeddata[i]['Pubdate']))
i += 1
if feedtype is None:
logger.fdebug('[' + picksite + '] there were ' + str(i) +
' results..')
else:
logger.fdebug('[' + picksite + '] there were ' + str(i) +
' results ' + feedtype)
lp += 1
if not seriesname:
rssdbupdate(feeddata, i, 'torrent')
else:
katinfo['entries'] = torthekat
return katinfo
return
def historycheck(self, nzbinfo, roundtwo=False):
sendresponse = nzbinfo['nzo_id']
hist_params = {
'mode': 'history',
'category': mylar.CONFIG.SAB_CATEGORY,
'failed': 0,
'output': 'json',
'apikey': mylar.CONFIG.SAB_APIKEY
}
sab_check = None
if mylar.CONFIG.SAB_VERSION is None:
try:
sc = mylar.webserve.WebInterface()
sab_check = sc.SABtest(sabhost=mylar.CONFIG.SAB_HOST,
sabusername=mylar.CONFIG.SAB_USERNAME,
sabpassword=mylar.CONFIG.SAB_PASSWORD,
sabapikey=mylar.CONFIG.SAB_APIKEY)
except Exception as e:
logger.warn(
'[SABNZBD-VERSION-TEST] Exception encountered trying to retrieve SABnzbd version: %s. Setting history length to last 200 items.'
% e)
hist_params['limit'] = 200
sab_check = 'some value'
else:
sab_check = None
if sab_check is None:
#set min_sab to 3.2.0 since 3.2.0 beta 1 has the api call for history search by nzo_id
try:
sab_minimum_version = '3.2.0'
min_sab = re.sub('[^0-9]', '', sab_minimum_version)
sab_vers = mylar.CONFIG.SAB_VERSION
if 'beta' in sab_vers:
sab_vers = re.sub('[^0-9]', '', sab_vers)
if len(sab_vers) > 3:
sab_vers = sab_vers[:
-1] # remove beta value entirely...
if parse_version(sab_vers) >= parse_version(min_sab):
logger.fdebug(
'SABnzbd version is higher than 3.2.0. Querying history based on nzo_id directly.'
)
hist_params['nzo_ids'] = sendresponse
else:
logger.fdebug(
'SABnzbd version is less than 3.2.0. Querying history based on history size of 200.'
)
hist_params['limit'] = 200
except Exception as e:
logger.warn(
'[SABNZBD-VERSION-CHECK] Exception encountered trying to compare installed version [%s] to [%s]. Setting history length to last 200 items. (error: %s)'
% (mylar.CONFIG.SAB_VERSION, sab_minimum_version, e))
hist_params['limit'] = 200
hist = requests.get(self.sab_url, params=hist_params, verify=False)
historyresponse = hist.json()
#logger.info(historyresponse)
histqueue = historyresponse['history']
found = {'status': False}
nzo_exists = False
try:
for hq in histqueue['slots']:
logger.fdebug('nzo_id: %s --- %s [%s]' %
(hq['nzo_id'], sendresponse, hq['status']))
if hq['nzo_id'] == sendresponse and any([
hq['status'] == 'Completed', hq['status'] == 'Running',
'comicrn' in hq['script'].lower()
]):
nzo_exists = True
logger.info(
'found matching completed item in history. Job has a status of %s'
% hq['status'])
if 'comicrn' in hq['script'].lower():
logger.warn(
'ComicRN has been detected as being active for this category & download. Completed Download Handling will NOT be performed due to this.'
)
logger.warn(
'Either disable Completed Download Handling for SABnzbd within Mylar, or remove ComicRN from your category script in SABnzbd.'
)
return {'status': 'double-pp', 'failed': False}
if os.path.isfile(hq['storage']):
logger.fdebug('location found @ %s' % hq['storage'])
found = {
'status':
True,
'name':
ntpath.basename(
hq['storage']
), #os.pathre.sub('.nzb', '', hq['nzb_name']).strip(),
'location':
os.path.abspath(
os.path.join(hq['storage'], os.pardir)),
'failed':
False,
'issueid':
nzbinfo['issueid'],
'comicid':
nzbinfo['comicid'],
'apicall':
True,
'ddl':
False
}
break
else:
logger.error(
'no file found where it should be @ %s - is there another script that moves things after completion ?'
% hq['storage'])
return {'status': 'file not found', 'failed': False}
elif hq['nzo_id'] == sendresponse and hq['status'] == 'Failed':
nzo_exists = True
#get the stage / error message and see what we can do
stage = hq['stage_log']
logger.fdebug('stage: %s' % (stage, ))
for x in stage:
if 'Failed' in x['actions'] and any(
[x['name'] == 'Unpack', x['name'] == 'Repair']):
if 'moving' in x['actions']:
logger.warn(
'There was a failure in SABnzbd during the unpack/repair phase that caused a failure: %s'
% x['actions'])
else:
logger.warn(
'Failure occured during the Unpack/Repair phase of SABnzbd. This is probably a bad file: %s'
% x['actions'])
if mylar.FAILED_DOWNLOAD_HANDLING is True:
found = {
'status':
True,
'name':
re.sub('.nzb', '',
hq['nzb_name']).strip(),
'location':
os.path.abspath(
os.path.join(
hq['storage'], os.pardir)),
'failed':
True,
'issueid':
sendresponse['issueid'],
'comicid':
sendresponse['comicid'],
'apicall':
True,
'ddl':
False
}
break
if found['status'] is False:
return {'status': 'failed_in_sab', 'failed': False}
else:
break
elif hq['nzo_id'] == sendresponse:
nzo_exists = True
logger.fdebug(
'nzo_id: %s found while processing queue in an unhandled status: %s'
% (hq['nzo_id'], hq['status']))
if hq['status'] == 'Queued' and roundtwo is False:
time.sleep(4)
return self.historycheck(nzbinfo, roundtwo=True)
else:
return {
'failed': False,
'status':
'unhandled status of: %s' % (hq['status'])
}
if not nzo_exists:
logger.error(
'Cannot find nzb %s in the queue. Was it removed?' %
sendresponse)
time.sleep(5)
if roundtwo is False:
return self.historycheck(nzbinfo, roundtwo=True)
else:
return {'status': 'nzb removed', 'failed': False}
except Exception as e:
logger.warn('error %s' % e)
return {'status': False, 'failed': False}
return found
def torrentdbsearch(seriesname, issue, comicid=None, nzbprov=None):
myDB = db.DBConnection()
seriesname_alt = None
if comicid is None or comicid == 'None':
pass
else:
logger.fdebug('ComicID: ' + str(comicid))
snm = myDB.selectone("SELECT * FROM comics WHERE comicid=?",
[comicid]).fetchone()
if snm is None:
logger.fdebug('Invalid ComicID of ' + str(comicid) +
'. Aborting search.')
return
else:
seriesname = snm['ComicName']
seriesname_alt = snm['AlternateSearch']
#remove 'and' and 'the':
tsearch_rem1 = re.sub("\\band\\b", "%", seriesname.lower())
tsearch_rem2 = re.sub("\\bthe\\b", "%", tsearch_rem1.lower())
tsearch_removed = re.sub('\s+', ' ', tsearch_rem2)
tsearch_seriesname = re.sub('[\'\!\@\#\$\%\:\-\;\/\\=\?\&\.\s]', '%',
tsearch_removed)
tsearch = tsearch_seriesname + "%"
logger.fdebug('tsearch : ' + str(tsearch))
AS_Alt = []
tresults = []
if mylar.ENABLE_CBT:
tresults = myDB.select(
"SELECT * FROM rssdb WHERE Title like ? AND Site='CBT'", [tsearch])
if mylar.ENABLE_KAT:
tresults += myDB.select(
"SELECT * FROM rssdb WHERE Title like ? AND Site='KAT'", [tsearch])
logger.fdebug('seriesname_alt:' + str(seriesname_alt))
if seriesname_alt is None or seriesname_alt == 'None':
if tresults is None:
logger.fdebug('no Alternate name given. Aborting search.')
return "no results"
else:
chkthealt = seriesname_alt.split('##')
if chkthealt == 0:
AS_Alternate = seriesname_alt
AS_Alt.append(seriesname_alt)
for calt in chkthealt:
AS_Alter = re.sub('##', '', calt)
u_altsearchcomic = AS_Alter.encode('ascii', 'ignore').strip()
AS_Altrem = re.sub("\\band\\b", "", u_altsearchcomic.lower())
AS_Altrem = re.sub("\\bthe\\b", "", AS_Altrem.lower())
AS_Alternate = re.sub('[\_\#\,\/\:\;\.\-\!\$\%\+\'\&\?\@\s]', '%',
AS_Altrem)
AS_Altrem_mod = re.sub('[\&]', ' ', AS_Altrem)
AS_formatrem_seriesname = re.sub('[\'\!\@\#\$\%\:\;\/\\=\?\.]', '',
AS_Altrem_mod)
AS_formatrem_seriesname = re.sub('\s+', ' ',
AS_formatrem_seriesname)
if AS_formatrem_seriesname[:1] == ' ':
AS_formatrem_seriesname = AS_formatrem_seriesname[1:]
AS_Alt.append(AS_formatrem_seriesname)
AS_Alternate += '%'
if mylar.ENABLE_CBT:
#print "AS_Alternate:" + str(AS_Alternate)
tresults += myDB.select(
"SELECT * FROM rssdb WHERE Title like ? AND Site='CBT'",
[AS_Alternate])
if mylar.ENABLE_KAT:
tresults += myDB.select(
"SELECT * FROM rssdb WHERE Title like ? AND Site='KAT'",
[AS_Alternate])
if tresults is None:
logger.fdebug('torrent search returned no results for ' + seriesname)
return "no results"
extensions = ('cbr', 'cbz')
tortheinfo = []
torinfo = {}
for tor in tresults:
torsplit = tor['Title'].split('/')
logger.fdebug('tor-Title: ' + tor['Title'])
logger.fdebug('there are ' + str(len(torsplit)) +
' sections in this title')
i = 0
if nzbprov is not None:
if nzbprov != tor['Site']:
logger.fdebug('this is a result from ' + str(tor['Site']) +
', not the site I am looking for of ' +
str(nzbprov))
continue
#0 holds the title/issue and format-type.
while (i < len(torsplit)):
#we'll rebuild the string here so that it's formatted accordingly to be passed back to the parser.
logger.fdebug('section(' + str(i) + '): ' + str(torsplit[i]))
#remove extensions
titletemp = torsplit[i]
titletemp = re.sub('cbr', '', str(titletemp))
titletemp = re.sub('cbz', '', str(titletemp))
titletemp = re.sub('none', '', str(titletemp))
if i == 0:
rebuiltline = str(titletemp)
else:
rebuiltline = rebuiltline + ' (' + str(titletemp) + ')'
i += 1
logger.fdebug('rebuiltline is :' + str(rebuiltline))
seriesname_mod = seriesname
foundname_mod = torsplit[0]
seriesname_mod = re.sub("\\band\\b", " ", seriesname_mod.lower())
foundname_mod = re.sub("\\band\\b", " ", foundname_mod.lower())
seriesname_mod = re.sub("\\bthe\\b", " ", seriesname_mod.lower())
foundname_mod = re.sub("\\bthe\\b", " ", foundname_mod.lower())
seriesname_mod = re.sub('[\&]', ' ', seriesname_mod)
foundname_mod = re.sub('[\&]', ' ', foundname_mod)
formatrem_seriesname = re.sub('[\'\!\@\#\$\%\:\;\=\?\.]', '',
seriesname_mod)
formatrem_seriesname = re.sub('[\-]', ' ', formatrem_seriesname)
formatrem_seriesname = re.sub(
'[\/]', ' ', formatrem_seriesname
) #not necessary since seriesname in a torrent file won't have /
formatrem_seriesname = re.sub('\s+', ' ', formatrem_seriesname)
if formatrem_seriesname[:1] == ' ':
formatrem_seriesname = formatrem_seriesname[1:]
formatrem_torsplit = re.sub('[\'\!\@\#\$\%\:\;\\=\?\.]', '',
foundname_mod)
formatrem_torsplit = re.sub(
'[\-]', ' ', formatrem_torsplit
) #we replace the - with space so we'll get hits if differnces
formatrem_torsplit = re.sub(
'[\/]', ' ', formatrem_torsplit
) #not necessary since if has a /, should be removed in above line
formatrem_torsplit = re.sub('\s+', ' ', formatrem_torsplit)
logger.fdebug(
str(len(formatrem_torsplit)) + ' - formatrem_torsplit : ' +
formatrem_torsplit.lower())
logger.fdebug(
str(len(formatrem_seriesname)) + ' - formatrem_seriesname :' +
formatrem_seriesname.lower())
if formatrem_seriesname.lower() in formatrem_torsplit.lower() or any(
x.lower() in formatrem_torsplit.lower() for x in AS_Alt):
logger.fdebug('matched to : ' + tor['Title'])
logger.fdebug('matched on series title: ' + seriesname)
titleend = formatrem_torsplit[len(formatrem_seriesname):]
titleend = re.sub('\-', '',
titleend) #remove the '-' which is unnecessary
#remove extensions
titleend = re.sub('cbr', '', str(titleend))
titleend = re.sub('cbz', '', str(titleend))
titleend = re.sub('none', '', str(titleend))
logger.fdebug('titleend: ' + str(titleend))
sptitle = titleend.split()
extra = ''
# for sp in sptitle:
# if 'v' in sp.lower() and sp[1:].isdigit():
# volumeadd = sp
# elif 'vol' in sp.lower() and sp[3:].isdigit():
# volumeadd = sp
# #if sp.isdigit():
# #print("issue # detected : " + str(issue))
# elif helpers.issuedigits(issue.rstrip()) == helpers.issuedigits(sp.rstrip()):
# logger.fdebug("Issue matched for : " + str(issue))
#the title on CBT has a mix-mash of crap...ignore everything after cbz/cbr to cleanit
ctitle = tor['Title'].find('cbr')
if ctitle == 0:
ctitle = tor['Title'].find('cbz')
if ctitle == 0:
ctitle = tor['Title'].find('none')
if ctitle == 0:
logger.fdebug(
'cannot determine title properly - ignoring for now.'
)
continue
cttitle = tor['Title'][:ctitle]
#print("change title to : " + str(cttitle))
# if extra == '':
tortheinfo.append({
'title': rebuiltline, #cttitle,
'link': tor['Link'],
'pubdate': tor['Pubdate'],
'site': tor['Site'],
'length': tor['Size']
})
# continue
# #torsend2client(formatrem_seriesname,tor['Link'])
# else:
# logger.fdebug("extra info given as :" + str(extra))
# logger.fdebug("extra information confirmed as a match")
# logger.fdebug("queuing link: " + str(tor['Link']))
# tortheinfo.append({
# 'title': cttitle, #tor['Title'],
# 'link': tor['Link'],
# 'pubdate': tor['Pubdate'],
# 'site': tor['Site'],
# 'length': tor['Size']
# })
# logger.fdebug("entered info.")
# continue
#torsend2client(formatrem_seriesname,tor['Link'])
#else:
# logger.fdebug("invalid issue#: " + str(sp))
# #extra = str(extra) + " " + str(sp)
# else:
# logger.fdebug("word detected - assuming continuation of title: " + str(sp))
# extra = str(extra) + " " + str(sp)
torinfo['entries'] = tortheinfo
return torinfo
def foundsearch(ComicID,
IssueID,
mode=None,
down=None,
provider=None,
SARC=None,
IssueArcID=None):
# When doing a Force Search (Wanted tab), the resulting search calls this to update.
# this is all redudant code that forceRescan already does.
# should be redone at some point so that instead of rescanning entire
# series directory, it just scans for the issue it just downloaded and
# and change the status to Snatched accordingly. It is not to increment the have count
# at this stage as it's not downloaded - just the .nzb has been snatched and sent to SAB.
myDB = db.DBConnection()
logger.info('comicid: ' + str(ComicID))
logger.info('issueid: ' + str(IssueID))
if mode != 'story_arc':
comic = myDB.action('SELECT * FROM comics WHERE ComicID=?',
[ComicID]).fetchone()
ComicName = comic['ComicName']
if mode == 'want_ann':
issue = myDB.action('SELECT * FROM annuals WHERE IssueID=?',
[IssueID]).fetchone()
else:
issue = myDB.action('SELECT * FROM issues WHERE IssueID=?',
[IssueID]).fetchone()
CYear = issue['IssueDate'][:4]
else:
issue = myDB.action('SELECT * FROM readinglist WHERE IssueArcID=?',
[IssueArcID]).fetchone()
ComicName = issue['ComicName']
CYear = issue['IssueYEAR']
if down is None:
# update the status to Snatched (so it won't keep on re-downloading!)
logger.fdebug('updating status to snatched')
logger.fdebug('provider is ' + provider)
newValue = {"Status": "Snatched"}
if mode == 'story_arc':
cValue = {"IssueArcID": IssueArcID}
snatchedupdate = {"IssueArcID": IssueArcID}
myDB.upsert("readinglist", newValue, cValue)
# update the snatched DB
snatchedupdate = {
"IssueID": IssueArcID,
"Status": "Snatched",
"Provider": provider
}
else:
if mode == 'want_ann':
controlValue = {"IssueID": IssueID}
myDB.upsert("annuals", newValue, controlValue)
else:
controlValue = {"IssueID": IssueID}
myDB.upsert("issues", newValue, controlValue)
# update the snatched DB
snatchedupdate = {
"IssueID": IssueID,
"Status": "Snatched",
"Provider": provider
}
if mode == 'story_arc':
IssueNum = issue['IssueNumber']
newsnatchValues = {
"ComicName": ComicName,
"ComicID": 'None',
"Issue_Number": IssueNum,
"DateAdded": helpers.now(),
"Status": "Snatched"
}
else:
if mode == 'want_ann':
IssueNum = "Annual " + issue['Issue_Number']
else:
IssueNum = issue['Issue_Number']
newsnatchValues = {
"ComicName": ComicName,
"ComicID": ComicID,
"Issue_Number": IssueNum,
"DateAdded": helpers.now(),
"Status": "Snatched"
}
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
logger.info("updated the snatched.")
else:
logger.info("updating the downloaded.")
if mode == 'want_ann':
IssueNum = "Annual " + issue['Issue_Number']
elif mode == 'story_arc':
IssueNum = issue['IssueNumber']
IssueID = IssueArcID
else:
IssueNum = issue['Issue_Number']
snatchedupdate = {
"IssueID": IssueID,
"Status": "Downloaded",
"Provider": provider
}
newsnatchValues = {
"ComicName": ComicName,
"ComicID": ComicID,
"Issue_Number": IssueNum,
"DateAdded": helpers.now(),
"Status": "Downloaded"
}
myDB.upsert("snatched", newsnatchValues, snatchedupdate)
if mode == 'story_arc':
cValue = {"IssueArcID": IssueArcID}
nValue = {"Status": "Downloaded"}
myDB.upsert("readinglist", nValue, cValue)
else:
controlValue = {"IssueID": IssueID}
newValue = {"Status": "Downloaded"}
myDB.upsert("issues", newValue, controlValue)
#print ("finished updating snatched db.")
logger.info('Updating now complete for ' + ComicName + ' issue: ' +
str(IssueNum))
return
def sender(self, filename, test=False):
if mylar.CONFIG.NZBGET_PRIORITY:
if any([
mylar.CONFIG.NZBGET_PRIORITY == 'Default',
mylar.CONFIG.NZBGET_PRIORITY == 'Normal'
]):
nzbgetpriority = 0
elif mylar.CONFIG.NZBGET_PRIORITY == 'Low':
nzbgetpriority = -50
elif mylar.CONFIG.NZBGET_PRIORITY == 'High':
nzbgetpriority = 50
elif mylar.CONFIG.NZBGET_PRIORITY == 'Very High':
nzbgetpriority = 100
elif mylar.CONFIG.NZBGET_PRIORITY == 'Force':
nzbgetpriority = 900
#there's no priority for "paused", so set "Very Low" and deal with that later...
elif mylar.CONFIG.NZBGET_PRIORITY == 'Paused':
nzbgetpriority = -100
else:
#if nzbget priority isn't selected, default to Normal (0)
nzbgetpriority = 0
with open(filename, 'rb') as in_file:
nzbcontent = in_file.read()
nzbcontent64 = standard_b64encode(nzbcontent).decode('utf-8')
try:
logger.fdebug('sending now to %s' % self.display_url)
if mylar.CONFIG.NZBGET_CATEGORY is None:
nzb_category = ''
else:
nzb_category = mylar.CONFIG.NZBGET_CATEGORY
sendresponse = self.server.append(filename, nzbcontent64,
nzb_category, nzbgetpriority,
False, False, '', 0, 'SCORE')
except http.client.socket.error as e:
nzb_url = re.sub(mylar.CONFIG.NZBGET_PASSWORD, 'REDACTED', str(e))
logger.error(
'Please check your NZBget host and port (if it is running). Tested against: %s'
% nzb_url)
return {'status': False}
except xmlrpc.client.ProtocolError as e:
logger.info(e, )
if e.errmsg == "Unauthorized":
err = re.sub(mylar.CONFIG.NZBGET_PASSWORD, 'REDACTED',
e.errmsg)
logger.error('Unauthorized username / password provided: %s' %
err)
return {'status': False}
else:
err = "Protocol Error: %s" % re.sub(
mylar.CONFIG.NZBGET_PASSWORD, 'REDACTED', e.errmsg)
logger.error('Protocol error returned: %s' % err)
return {'status': False}
except Exception as e:
logger.warn(
'uh-oh: %s' %
re.sub(mylar.CONFIG.NZBGET_PASSWORD, 'REDACTED', str(e)))
return {'status': False}
else:
if sendresponse <= 0:
logger.warn(
'Invalid response received after sending to NZBGet: %s' %
sendresponse)
return {'status': False}
else:
#sendresponse is the NZBID that we use to track the progress....
return {'status': True, 'NZBID': sendresponse}
def forceRescan(ComicID, archive=None):
myDB = db.DBConnection()
# file check to see if issue exists
rescan = myDB.action('SELECT * FROM comics WHERE ComicID=?',
[ComicID]).fetchone()
logger.info('Now checking files for ' + rescan['ComicName'] + ' (' +
str(rescan['ComicYear']) + ') in ' + rescan['ComicLocation'])
if archive is None:
fc = filechecker.listFiles(dir=rescan['ComicLocation'],
watchcomic=rescan['ComicName'],
Publisher=rescan['ComicPublisher'],
AlternateSearch=rescan['AlternateSearch'])
else:
fc = filechecker.listFiles(dir=archive,
watchcomic=rescan['ComicName'],
Publisher=rescan['ComicPublisher'],
AlternateSearch=rescan['AlternateSearch'])
iscnt = rescan['Total']
havefiles = 0
if mylar.ANNUALS_ON:
an_cnt = myDB.action("SELECT COUNT(*) FROM annuals WHERE ComicID=?",
[ComicID]).fetchall()
anncnt = an_cnt[0][0]
else:
anncnt = 0
fccnt = int(fc['comiccount'])
issnum = 1
fcnew = []
fn = 0
issuedupechk = []
annualdupechk = []
issueexceptdupechk = []
reissues = myDB.action('SELECT * FROM issues WHERE ComicID=?',
[ComicID]).fetchall()
issID_to_ignore = []
issID_to_ignore.append(str(ComicID))
while (fn < fccnt):
haveissue = "no"
issuedupe = "no"
try:
tmpfc = fc['comiclist'][fn]
except IndexError:
logger.fdebug(
'Unable to properly retrieve a file listing for the given series.'
)
logger.fdebug(
'Probably because the filenames being scanned are not in a parseable format'
)
if fn == 0:
return
else:
break
temploc = tmpfc['JusttheDigits'].replace('_', ' ')
# temploc = tmpfc['ComicFilename'].replace('_', ' ')
temploc = re.sub('[\#\']', '', temploc)
logger.fdebug('temploc: ' + str(temploc))
if 'annual' not in temploc.lower():
#remove the extension here
extensions = ('.cbr', '.cbz')
if temploc.lower().endswith(extensions):
logger.fdebug('removed extension for issue: ' + str(temploc))
temploc = temploc[:-4]
# deccnt = str(temploc).count('.')
# if deccnt > 1:
#logger.fdebug('decimal counts are :' + str(deccnt))
#if the file is formatted with '.' in place of spaces we need to adjust.
#before replacing - check to see if digits on either side of decimal and if yes, DON'T REMOVE
# occur=1
# prevstart = 0
# digitfound = "no"
# decimalfound = "no"
# tempreconstruct = ''
# while (occur <= deccnt):
# n = occur
# start = temploc.find('.')
# while start >=0 and n > 1:
# start = temploc.find('.', start+len('.'))
# n-=1
# #logger.fdebug('occurance ' + str(occur) + ' of . at position: ' + str(start))
# if temploc[prevstart:start].isdigit():
# if digitfound == "yes":
# #logger.fdebug('this is a decimal, assuming decimal issue.')
# decimalfound = "yes"
# reconst = "." + temploc[prevstart:start] + " "
# else:
# #logger.fdebug('digit detected.')
# digitfound = "yes"
# reconst = temploc[prevstart:start]
# else:
# reconst = temploc[prevstart:start] + " "
# #logger.fdebug('word: ' + reconst)
# tempreconstruct = tempreconstruct + reconst
# #logger.fdebug('tempreconstruct is : ' + tempreconstruct)
# prevstart = (start+1)
# occur+=1
# #logger.fdebug('word: ' + temploc[prevstart:])
# tempreconstruct = tempreconstruct + " " + temploc[prevstart:]
# #logger.fdebug('final filename to use is : ' + str(tempreconstruct))
# temploc = tempreconstruct
#logger.fdebug("checking " + str(temploc))
#fcnew_b4 = shlex.split(str(temploc))
fcnew_af = re.findall('[^\()]+', temploc)
fcnew = shlex.split(fcnew_af[0])
fcn = len(fcnew)
n = 0
while (n <= iscnt):
som = 0
try:
reiss = reissues[n]
except IndexError:
break
# int_iss, iss_except = helpers.decimal_issue(reiss['Issue_Number'])
int_iss = helpers.issuedigits(reiss['Issue_Number'])
issyear = reiss['IssueDate'][:4]
old_status = reiss['Status']
issname = reiss['IssueName']
#logger.fdebug('integer_issue:' + str(int_iss) + ' ... status: ' + str(old_status))
#if comic in format of "SomeSeries 5(c2c)(2013).cbr" whatever...it'll die.
#can't distinguish the 5(c2c) to tell it's the issue #...
fnd_iss_except = 'None'
#print ("Issue, int_iss, iss_except: " + str(reiss['Issue_Number']) + "," + str(int_iss) + "," + str(iss_except))
while (som < fcn):
#counts get buggered up when the issue is the last field in the filename - ie. '50.cbr'
#logger.fdebug('checking word - ' + str(fcnew[som]))
if ".cbr" in fcnew[som].lower():
fcnew[som] = fcnew[som].replace(".cbr", "")
elif ".cbz" in fcnew[som].lower():
fcnew[som] = fcnew[som].replace(".cbz", "")
if "(c2c)" in fcnew[som].lower():
fcnew[som] = fcnew[som].replace("(c2c)", " ")
get_issue = shlex.split(str(fcnew[som]))
if fcnew[som] != " ":
fcnew[som] = get_issue[0]
if som + 1 < len(fcnew) and len(fcnew[som + 1]) == 2:
#print "fcnew[som+1]: " + str(fcnew[som+1])
#print "fcnew[som]: " + str(fcnew[som])
if 'au' in fcnew[som + 1].lower():
#if the 'AU' is in 005AU vs 005 AU it will yield different results.
fcnew[som] = fcnew[som] + 'AU'
fcnew[som + 1] = '93939999919190933'
logger.info(
'AU Detected seperate from issue - combining and continuing'
)
elif 'ai' in fcnew[som + 1].lower():
#if the 'AI' is in 005AI vs 005 AI it will yield different results.
fcnew[som] = fcnew[som] + 'AI'
fcnew[som + 1] = '93939999919190933'
logger.info(
'AI Detected seperate from issue - combining and continuing'
)
#sometimes scanners refuse to use spaces between () and lump the issue right at the start
#mylar assumes it's all one word in this case..let's dump the brackets.
fcdigit = helpers.issuedigits(fcnew[som])
#logger.fdebug("fcdigit: " + str(fcdigit))
#logger.fdebug("int_iss: " + str(int_iss))
if int(fcdigit) == int_iss:
logger.fdebug('issue match - fcdigit: ' +
str(fcdigit) + ' ... int_iss: ' +
str(int_iss))
if '-' in temploc and temploc.find(
reiss['Issue_Number']) > temploc.find('-'):
logger.fdebug(
'I have detected a possible Title in the filename'
)
logger.fdebug(
'the issue # has occured after the -, so I assume that it is part of the Title'
)
break
for d in issuedupechk:
if int(d['fcdigit']) == int(fcdigit):
logger.fdebug(
'duplicate issue detected - not counting this: '
+ str(tmpfc['ComicFilename']))
logger.fdebug('is a duplicate of ' +
d['filename'])
logger.fdebug('fcdigit:' + str(fcdigit) +
' === dupedigit: ' +
str(d['fcdigit']))
issuedupe = "yes"
break
if issuedupe == "no":
logger.fdebug('matched...issue: ' +
rescan['ComicName'] + '#' +
str(reiss['Issue_Number']) +
' --- ' + str(int_iss))
havefiles += 1
haveissue = "yes"
isslocation = str(tmpfc['ComicFilename'])
issSize = str(tmpfc['ComicSize'])
logger.fdebug('.......filename: ' +
str(isslocation))
logger.fdebug('.......filesize: ' +
str(tmpfc['ComicSize']))
# to avoid duplicate issues which screws up the count...let's store the filename issues then
# compare earlier...
issuedupechk.append({
'fcdigit':
int(fcdigit),
'filename':
tmpfc['ComicFilename']
})
break
#else:
# if the issue # matches, but there is no year present - still match.
# determine a way to match on year if present, or no year (currently).
if issuedupe == "yes":
logger.fdebug(
'I should break out here because of a dupe.')
break
som += 1
if haveissue == "yes" or issuedupe == "yes": break
n += 1
else:
# annual inclusion here.
#logger.fdebug("checking " + str(temploc))
reannuals = myDB.action('SELECT * FROM annuals WHERE ComicID=?',
[ComicID]).fetchall()
fcnew = shlex.split(str(temploc))
fcn = len(fcnew)
n = 0
reann = None
while (n < anncnt):
som = 0
try:
reann = reannuals[n]
except IndexError:
break
int_iss, iss_except = helpers.decimal_issue(
reann['Issue_Number'])
issyear = reann['IssueDate'][:4]
old_status = reann['Status']
while (som < fcn):
#counts get buggered up when the issue is the last field in the filename - ie. '50$
#logger.fdebug('checking word - ' + str(fcnew[som]))
if ".cbr" in fcnew[som].lower():
fcnew[som] = fcnew[som].replace(".cbr", "")
elif ".cbz" in fcnew[som].lower():
fcnew[som] = fcnew[som].replace(".cbz", "")
if "(c2c)" in fcnew[som].lower():
fcnew[som] = fcnew[som].replace("(c2c)", " ")
get_issue = shlex.split(str(fcnew[som]))
if fcnew[som] != " ":
fcnew[som] = get_issue[0]
if fcnew[som].lower() == 'annual':
logger.fdebug('Annual detected.')
if fcnew[som + 1].isdigit():
ann_iss = fcnew[som + 1]
logger.fdebug('Annual # ' + str(ann_iss) +
' detected.')
fcdigit = helpers.issuedigits(ann_iss)
logger.fdebug('fcdigit:' + str(fcdigit))
logger.fdebug('int_iss:' + str(int_iss))
if int(fcdigit) == int_iss:
logger.fdebug('annual match - issue : ' + str(int_iss))
for d in annualdupechk:
if int(d['fcdigit']) == int(fcdigit):
logger.fdebug(
'duplicate annual issue detected - not counting this: '
+ str(tmpfc['ComicFilename']))
issuedupe = "yes"
break
if issuedupe == "no":
logger.fdebug('matched...annual issue: ' +
rescan['ComicName'] + '#' +
str(reann['Issue_Number']) +
' --- ' + str(int_iss))
havefiles += 1
haveissue = "yes"
isslocation = str(tmpfc['ComicFilename'])
issSize = str(tmpfc['ComicSize'])
logger.fdebug('.......filename: ' +
str(isslocation))
logger.fdebug('.......filesize: ' +
str(tmpfc['ComicSize']))
# to avoid duplicate issues which screws up the count...let's store the filename issues then
# compare earlier...
annualdupechk.append({'fcdigit': int(fcdigit)})
break
som += 1
if haveissue == "yes": break
n += 1
if issuedupe == "yes": pass
else:
#we have the # of comics, now let's update the db.
#even if we couldn't find the physical issue, check the status.
#-- if annuals aren't enabled, this will bugger out.
writeit = True
if mylar.ANNUALS_ON:
if 'annual' in temploc.lower():
if reann is None:
logger.fdebug(
'Annual present in location, but series does not have any annuals attached to it - Ignoring'
)
writeit = False
else:
iss_id = reann['IssueID']
else:
iss_id = reiss['IssueID']
else:
if 'annual' in temploc.lower():
logger.fdebug(
'Annual support not enabled, but annual issue present within directory. Ignoring annual.'
)
writeit = False
else:
iss_id = reiss['IssueID']
if writeit == True:
logger.fdebug('issueID to write to db:' + str(iss_id))
controlValueDict = {"IssueID": iss_id}
#if Archived, increase the 'Have' count.
#if archive:
# issStatus = "Archived"
if haveissue == "yes":
issStatus = "Downloaded"
newValueDict = {
"Location": isslocation,
"ComicSize": issSize,
"Status": issStatus
}
issID_to_ignore.append(str(iss_id))
if 'annual' in temploc.lower():
myDB.upsert("annuals", newValueDict, controlValueDict)
else:
myDB.upsert("issues", newValueDict, controlValueDict)
fn += 1
logger.fdebug('IssueID to ignore: ' + str(issID_to_ignore))
#here we need to change the status of the ones we DIDN'T FIND above since the loop only hits on FOUND issues.
update_iss = []
tmpsql = "SELECT * FROM issues WHERE ComicID=? AND IssueID not in ({seq})".format(
seq=','.join(['?'] * (len(issID_to_ignore) - 1)))
chkthis = myDB.action(tmpsql, issID_to_ignore).fetchall()
# chkthis = None
if chkthis is None:
pass
else:
for chk in chkthis:
old_status = chk['Status']
#logger.fdebug('old_status:' + str(old_status))
if old_status == "Skipped":
if mylar.AUTOWANT_ALL:
issStatus = "Wanted"
else:
issStatus = "Skipped"
elif old_status == "Archived":
issStatus = "Archived"
elif old_status == "Downloaded":
issStatus = "Archived"
elif old_status == "Wanted":
issStatus = "Wanted"
elif old_status == "Ignored":
issStatus = "Ignored"
elif old_status == "Snatched": #this is needed for torrents, or else it'll keep on queuing..
issStatus = "Snatched"
else:
issStatus = "Skipped"
#logger.fdebug("new status: " + str(issStatus))
update_iss.append({"IssueID": chk['IssueID'], "Status": issStatus})
if len(update_iss) > 0:
i = 0
#do it like this to avoid DB locks...
for ui in update_iss:
controlValueDict = {"IssueID": ui['IssueID']}
newStatusValue = {"Status": ui['Status']}
myDB.upsert("issues", newStatusValue, controlValueDict)
i += 1
logger.info('Updated the status of ' + str(i) + ' issues for ' +
rescan['ComicName'] + ' (' + str(rescan['ComicYear']) +
') that were not found.')
logger.info('Total files located: ' + str(havefiles))
foundcount = havefiles
arcfiles = 0
arcanns = 0
# if filechecker returns 0 files (it doesn't find any), but some issues have a status of 'Archived'
# the loop below won't work...let's adjust :)
arcissues = myDB.action(
"SELECT count(*) FROM issues WHERE ComicID=? and Status='Archived'",
[ComicID]).fetchall()
if int(arcissues[0][0]) > 0:
arcfiles = arcissues[0][0]
arcannuals = myDB.action(
"SELECT count(*) FROM annuals WHERE ComicID=? and Status='Archived'",
[ComicID]).fetchall()
if int(arcissues[0][0]) > 0:
arcanns = arcannuals[0][0]
if arcfiles > 0 and arcanns > 0:
arcfiles = arcfiles + arcanns
havefiles = havefiles + arcfiles
logger.fdebug('Adjusting have total to ' + str(havefiles) +
' because of this many archive files:' + str(arcfiles))
ignorecount = 0
if mylar.IGNORE_HAVETOTAL: # if this is enabled, will increase Have total as if in Archived Status
ignores = myDB.action(
"SELECT count(*) FROM issues WHERE ComicID=? AND Status='Ignored'",
[ComicID]).fetchall()
if int(ignores[0][0]) > 0:
ignorecount = ignores[0][0]
havefiles = havefiles + ignorecount
logger.fdebug('Adjusting have total to ' + str(havefiles) +
' because of this many Ignored files:' +
str(ignorecount))
#now that we are finished...
#adjust for issues that have been marked as Downloaded, but aren't found/don't exist.
#do it here, because above loop only cycles though found comics using filechecker.
downissues = myDB.select(
"SELECT * FROM issues WHERE ComicID=? and Status='Downloaded'",
[ComicID])
downissues += myDB.select(
"SELECT * FROM annuals WHERE ComicID=? and Status='Downloaded'",
[ComicID])
if downissues is None:
pass
else:
archivedissues = 0 #set this to 0 so it tallies correctly.
for down in downissues:
#print "downlocation:" + str(down['Location'])
#remove special characters from
#temploc = rescan['ComicLocation'].replace('_', ' ')
#temploc = re.sub('[\#\'\/\.]', '', temploc)
#print ("comiclocation: " + str(rescan['ComicLocation']))
#print ("downlocation: " + str(down['Location']))
if down['Location'] is None:
logger.fdebug(
'location does not exist which means file was not downloaded successfully, or was moved.'
)
controlValue = {"IssueID": down['IssueID']}
newValue = {"Status": "Archived"}
myDB.upsert("issues", newValue, controlValue)
archivedissues += 1
pass
else:
comicpath = os.path.join(rescan['ComicLocation'],
down['Location'])
if os.path.exists(comicpath):
pass
#print "Issue exists - no need to change status."
else:
#print "Changing status from Downloaded to Archived - cannot locate file"
controlValue = {"IssueID": down['IssueID']}
newValue = {"Status": "Archived"}
myDB.upsert("issues", newValue, controlValue)
archivedissues += 1
totalarc = arcfiles + archivedissues
havefiles = havefiles + archivedissues #arcfiles already tallied in havefiles in above segment
logger.fdebug(
'I have changed the status of ' + str(archivedissues) +
' issues to a status of Archived, as I now cannot locate them in the series directory.'
)
#let's update the total count of comics that was found.
controlValueStat = {"ComicID": rescan['ComicID']}
newValueStat = {"Have": havefiles}
combined_total = rescan['Total'] + anncnt
myDB.upsert("comics", newValueStat, controlValueStat)
logger.info('I have physically found ' + str(foundcount) +
' issues, ignored ' + str(ignorecount) +
' issues, and accounted for ' + str(totalarc) +
' in an Archived state. Total Issue Count: ' + str(havefiles) +
' / ' + str(combined_total))
return
def processor(self, nzbinfo):
nzbid = nzbinfo['NZBID']
try:
logger.fdebug(
'Now checking the active queue of nzbget for the download')
queueinfo = self.server.listgroups()
except Exception as e:
logger.warn(
'Error attempting to retrieve active queue listing: %s' % e)
return {'status': False}
else:
logger.fdebug('valid queue result returned. Analyzing...')
queuedl = [qu for qu in queueinfo if qu['NZBID'] == nzbid]
if len(queuedl) == 0:
logger.warn(
'Unable to locate NZBID %s in active queue. Could it be finished already ?'
% nzbid)
return self.historycheck(nzbinfo)
stat = False
double_pp = False
double_type = None
while stat is False:
time.sleep(10)
queueinfo = self.server.listgroups()
queuedl = [qu for qu in queueinfo if qu['NZBID'] == nzbid]
if len(queuedl) == 0:
logger.fdebug(
'Item is no longer in active queue. It should be finished by my calculations'
)
stat = True
else:
if 'comicrn' in queuedl[0]['PostInfoText'].lower():
double_pp = True
double_type = 'ComicRN'
elif 'nzbtomylar' in queuedl[0]['PostInfoText'].lower():
double_pp = True
double_type = 'nzbToMylar'
if all([
len(queuedl[0]['ScriptStatuses']) > 0,
double_pp is False
]):
for x in queuedl[0]['ScriptStatuses']:
if 'comicrn' in x['Name'].lower():
double_pp = True
double_type = 'ComicRN'
break
elif 'nzbtomylar' in x['Name'].lower():
double_pp = True
double_type = 'nzbToMylar'
break
if all([
len(queuedl[0]['Parameters']) > 0,
double_pp is False
]):
for x in queuedl[0]['Parameters']:
if all([
'comicrn' in x['Name'].lower(),
x['Value'] == 'yes'
]):
double_pp = True
double_type = 'ComicRN'
break
elif all([
'nzbtomylar' in x['Name'].lower(),
x['Value'] == 'yes'
]):
double_pp = True
double_type = 'nzbToMylar'
break
if double_pp is True:
logger.warn(
'%s has been detected as being active for this category & download. Completed Download Handling will NOT be performed due to this.'
% double_type)
logger.warn(
'Either disable Completed Download Handling for NZBGet within Mylar, or remove %s from your category script in NZBGet.'
% double_type)
return {'status': 'double-pp', 'failed': False}
logger.fdebug('status: %s' % queuedl[0]['Status'])
logger.fdebug('name: %s' % queuedl[0]['NZBName'])
logger.fdebug('FileSize: %sMB' % queuedl[0]['FileSizeMB'])
logger.fdebug('Download Left: %sMB' %
queuedl[0]['RemainingSizeMB'])
logger.fdebug('health: %s' % (queuedl[0]['Health'] / 10))
logger.fdebug('destination: %s' % queuedl[0]['DestDir'])
logger.fdebug('File has now downloaded!')
time.sleep(
5
) #wait some seconds so shit can get written to history properly
return self.historycheck(nzbinfo)
def storyarcinfo(xmlid):
comicLibrary = listStoryArcs()
arcinfo = {}
if mylar.CONFIG.COMICVINE_API == 'None' or mylar.CONFIG.COMICVINE_API is None:
logger.warn('You have not specified your own ComicVine API key - this is a requirement. Get your own @ http://api.comicvine.com.')
return
else:
comicapi = mylar.CONFIG.COMICVINE_API
#respawn to the exact id for the story arc and count the # of issues present.
ARCPULL_URL = mylar.CVURL + 'story_arc/4045-' + str(xmlid) + '/?api_key=' + str(comicapi) + '&field_list=issues,publisher,name,first_appeared_in_issue,deck,image&format=xml&offset=0'
#logger.fdebug('arcpull_url:' + str(ARCPULL_URL))
#new CV API restriction - one api request / second.
if mylar.CONFIG.CVAPI_RATE is None or mylar.CONFIG.CVAPI_RATE < 2:
time.sleep(2)
else:
time.sleep(mylar.CONFIG.CVAPI_RATE)
#download the file:
payload = None
try:
r = requests.get(ARCPULL_URL, params=payload, verify=mylar.CONFIG.CV_VERIFY, headers=mylar.CV_HEADERS)
except Exception as e:
logger.warn('While parsing data from ComicVine, got exception: %s' % e)
return
try:
arcdom = parseString(r.content)
except ExpatError:
if '<title>Abnormal Traffic Detected' in r.content:
logger.error('ComicVine has banned this server\'s IP address because it exceeded the API rate limit.')
else:
logger.warn('While parsing data from ComicVine, got exception: %s for data: %s' % (e, r.content))
return
except Exception as e:
logger.warn('While parsing data from ComicVine, got exception: %s for data: %s' % (e, r.content))
return
try:
logger.fdebug('story_arc ascension')
issuedom = arcdom.getElementsByTagName('issue')
issuecount = len( issuedom ) #arcdom.getElementsByTagName('issue') )
isc = 0
arclist = ''
ordernum = 1
for isd in issuedom:
zeline = isd.getElementsByTagName('id')
isdlen = len( zeline )
isb = 0
while ( isb < isdlen):
if isc == 0:
arclist = str(zeline[isb].firstChild.wholeText).strip() + ',' + str(ordernum)
else:
arclist += '|' + str(zeline[isb].firstChild.wholeText).strip() + ',' + str(ordernum)
ordernum+=1
isb+=1
isc+=1
except:
logger.fdebug('unable to retrive issue count - nullifying value.')
issuecount = 0
try:
firstid = None
arcyear = None
fid = len ( arcdom.getElementsByTagName('id') )
fi = 0
while (fi < fid):
if arcdom.getElementsByTagName('id')[fi].parentNode.nodeName == 'first_appeared_in_issue':
if not arcdom.getElementsByTagName('id')[fi].firstChild.wholeText == xmlid:
logger.fdebug('hit it.')
firstid = arcdom.getElementsByTagName('id')[fi].firstChild.wholeText
break # - dont' break out here as we want to gather ALL the issue ID's since it's here
fi+=1
logger.fdebug('firstid: ' + str(firstid))
if firstid is not None:
firstdom = cv.pulldetails(comicid=None, type='firstissue', issueid=firstid)
logger.fdebug('success')
arcyear = cv.Getissue(firstid,firstdom,'firstissue')
except:
logger.fdebug('Unable to retrieve first issue details. Not caclulating at this time.')
try:
xmlimage = arcdom.getElementsByTagName('super_url')[0].firstChild.wholeText
except:
xmlimage = "cache/blankcover.jpg"
try:
xmldesc = arcdom.getElementsByTagName('desc')[0].firstChild.wholeText
except:
xmldesc = "None"
try:
xmlpub = arcdom.getElementsByTagName('publisher')[0].firstChild.wholeText
except:
xmlpub = "None"
try:
xmldeck = arcdom.getElementsByTagName('deck')[0].firstChild.wholeText
except:
xmldeck = "None"
if xmlid in comicLibrary:
haveit = comicLibrary[xmlid]
else:
haveit = "No"
arcinfo = {
#'name': xmlTag, #theese four are passed into it only when it's a new add
#'url': xmlurl, #needs to be modified for refreshing to work completely.
#'publisher': xmlpub,
'comicyear': arcyear,
'comicid': xmlid,
'issues': issuecount,
'comicimage': xmlimage,
'description': xmldesc,
'deck': xmldeck,
'arclist': arclist,
'haveit': haveit,
'publisher': xmlpub
}
return arcinfo
def rename_file(self,
ofilename,
issue=None,
annualize=None,
arc=False,
file_format=None
): #comicname, issue, comicyear=None, issueid=None)
comicid = self.comicid # it's coming in unicoded...
issueid = self.issueid
if file_format is None:
file_format = mylar.CONFIG.FILE_FORMAT
logger.fdebug(type(comicid))
logger.fdebug(type(issueid))
logger.fdebug('comicid: %s' % comicid)
logger.fdebug('issue# as per cv: %s' % issue)
logger.fdebug('issueid:' + str(issueid))
if issueid is None:
logger.fdebug('annualize is ' + str(annualize))
if arc:
#this has to be adjusted to be able to include story arc issues that span multiple arcs
chkissue = self.myDB.selectone(
"SELECT * from storyarcs WHERE ComicID=? AND Issue_Number=?",
[comicid, issue]).fetchone()
else:
chkissue = self.myDB.selectone(
"SELECT * from issues WHERE ComicID=? AND Issue_Number=?",
[comicid, issue]).fetchone()
if all([
chkissue is None, annualize is None,
not mylar.CONFIG.ANNUALS_ON
]):
chkissue = self.myDB.selectone(
"SELECT * from annuals WHERE ComicID=? AND Issue_Number=?",
[comicid, issue]).fetchone()
if chkissue is None:
#rechk chkissue against int value of issue #
if arc:
chkissue = self.myDB.selectone(
"SELECT * from storyarcs WHERE ComicID=? AND Int_IssueNumber=?",
[comicid, issuedigits(issue)]).fetchone()
else:
chkissue = self.myDB.selectone(
"SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?",
[comicid, issuedigits(issue)]).fetchone()
if all([
chkissue is None, annualize == 'yes',
mylar.CONFIG.ANNUALS_ON
]):
chkissue = self.myDB.selectone(
"SELECT * from annuals WHERE ComicID=? AND Int_IssueNumber=?",
[comicid, issuedigits(issue)]).fetchone()
if chkissue is None:
logger.error('Invalid Issue_Number - please validate.')
return
else:
logger.info(
'Int Issue_number compare found. continuing...')
issueid = chkissue['IssueID']
else:
issueid = chkissue['IssueID']
#use issueid to get publisher, series, year, issue number
logger.fdebug('issueid is now : ' + str(issueid))
if arc:
issueinfo = self.myDB.selectone(
"SELECT * from storyarcs WHERE ComicID=? AND IssueID=? AND StoryArc=?",
[comicid, issueid, arc]).fetchone()
else:
issueinfo = self.myDB.selectone(
"SELECT * from issues WHERE ComicID=? AND IssueID=?",
[comicid, issueid]).fetchone()
if issueinfo is None:
logger.fdebug('not an issue, checking against annuals')
issueinfo = self.myDB.selectone(
"SELECT * from annuals WHERE ComicID=? AND IssueID=?",
[comicid, issueid]).fetchone()
if issueinfo is None:
logger.fdebug(
'Unable to rename - cannot locate issue id within db')
return
else:
annualize = True
if issueinfo is None:
logger.fdebug(
'Unable to rename - cannot locate issue id within db')
return
#remap the variables to a common factor.
if arc:
issuenum = issueinfo['IssueNumber']
issuedate = issueinfo['IssueDate']
publisher = issueinfo['IssuePublisher']
series = issueinfo['ComicName']
seriesfilename = series #Alternate FileNaming is not available with story arcs.
seriesyear = issueinfo['SeriesYear']
arcdir = helpers.filesafe(issueinfo['StoryArc'])
if mylar.CONFIG.REPLACE_SPACES:
arcdir = arcdir.replace(' ', mylar.CONFIG.REPLACE_CHAR)
if mylar.CONFIG.STORYARCDIR:
storyarcd = os.path.join(mylar.CONFIG.DESTINATION_DIR,
"StoryArcs", arcdir)
logger.fdebug('Story Arc Directory set to : ' + storyarcd)
else:
logger.fdebug('Story Arc Directory set to : ' +
mylar.CONFIG.GRABBAG_DIR)
storyarcd = os.path.join(mylar.CONFIG.DESTINATION_DIR,
mylar.CONFIG.GRABBAG_DIR)
comlocation = storyarcd
comversion = None #need to populate this.
else:
issuenum = issueinfo['Issue_Number']
issuedate = issueinfo['IssueDate']
publisher = self.comic['ComicPublisher']
series = self.comic['ComicName']
if self.comic['AlternateFileName'] is None or self.comic[
'AlternateFileName'] == 'None':
seriesfilename = series
else:
seriesfilename = self.comic['AlternateFileName']
logger.fdebug(
'Alternate File Naming has been enabled for this series. Will rename series title to : '
+ seriesfilename)
seriesyear = self.comic['ComicYear']
comlocation = self.comic['ComicLocation']
comversion = self.comic['ComicVersion']
unicodeissue = issuenum
if type(issuenum) == str:
vals = {
'\xbd': '.5',
'\xbc': '.25',
'\xbe': '.75',
'\u221e': '9999999999',
'\xe2': '9999999999'
}
else:
vals = {
'\xbd': '.5',
'\xbc': '.25',
'\xbe': '.75',
'\\u221e': '9999999999',
'\xe2': '9999999999'
}
x = [vals[key] for key in vals if key in issuenum]
if x:
issuenum = x[0]
logger.fdebug('issue number formatted: %s' % issuenum)
#comicid = issueinfo['ComicID']
#issueno = str(issuenum).split('.')[0]
issue_except = 'None'
issue_exceptions = [
'AU', 'INH', 'NOW', 'AI', 'MU', 'A', 'B', 'C', 'X', 'O'
]
valid_spaces = ('.', '-')
for issexcept in issue_exceptions:
if issexcept.lower() in issuenum.lower():
logger.fdebug('ALPHANUMERIC EXCEPTION : [' + issexcept + ']')
v_chk = [v for v in valid_spaces if v in issuenum]
if v_chk:
iss_space = v_chk[0]
logger.fdebug('character space denoted as : ' + iss_space)
else:
logger.fdebug('character space not denoted.')
iss_space = ''
# if issexcept == 'INH':
# issue_except = '.INH'
if issexcept == 'NOW':
if '!' in issuenum: issuenum = re.sub('\!', '', issuenum)
# issue_except = '.NOW'
issue_except = iss_space + issexcept
logger.fdebug('issue_except denoted as : ' + issue_except)
issuenum = re.sub("[^0-9]", "", issuenum)
break
# if 'au' in issuenum.lower() and issuenum[:1].isdigit():
# issue_except = ' AU'
# elif 'ai' in issuenum.lower() and issuenum[:1].isdigit():
# issuenum = re.sub("[^0-9]", "", issuenum)
# issue_except = ' AI'
# elif 'inh' in issuenum.lower() and issuenum[:1].isdigit():
# issuenum = re.sub("[^0-9]", "", issuenum)
# issue_except = '.INH'
# elif 'now' in issuenum.lower() and issuenum[:1].isdigit():
# if '!' in issuenum: issuenum = re.sub('\!', '', issuenum)
# issuenum = re.sub("[^0-9]", "", issuenum)
# issue_except = '.NOW'
if '.' in issuenum:
iss_find = issuenum.find('.')
iss_b4dec = issuenum[:iss_find]
if iss_find == 0:
iss_b4dec = '0'
iss_decval = issuenum[iss_find + 1:]
if iss_decval.endswith('.'):
iss_decval = iss_decval[:-1]
if int(iss_decval) == 0:
iss = iss_b4dec
issdec = int(iss_decval)
issueno = iss
else:
if len(iss_decval) == 1:
iss = iss_b4dec + "." + iss_decval
issdec = int(iss_decval) * 10
else:
iss = iss_b4dec + "." + iss_decval.rstrip('0')
issdec = int(iss_decval.rstrip('0')) * 10
issueno = iss_b4dec
else:
iss = issuenum
issueno = iss
# issue zero-suppression here
if mylar.CONFIG.ZERO_LEVEL == "0":
zeroadd = ""
else:
if mylar.CONFIG.ZERO_LEVEL_N == "none": zeroadd = ""
elif mylar.CONFIG.ZERO_LEVEL_N == "0x": zeroadd = "0"
elif mylar.CONFIG.ZERO_LEVEL_N == "00x": zeroadd = "00"
logger.fdebug('Zero Suppression set to : ' +
str(mylar.CONFIG.ZERO_LEVEL_N))
prettycomiss = None
if issueno.isalpha():
logger.fdebug('issue detected as an alpha.')
prettycomiss = str(issueno)
else:
try:
x = float(issuenum)
#validity check
if x < 0:
logger.info(
'I\'ve encountered a negative issue #: %s. Trying to accomodate.'
% issueno)
prettycomiss = '-' + str(zeroadd) + str(issueno[1:])
elif x == 9999999999:
logger.fdebug('Infinity issue found.')
issuenum = 'infinity'
elif x >= 0:
pass
else:
raise ValueError
except ValueError as e:
logger.warn(
'Unable to properly determine issue number [ %s] - you should probably log this on github for help.'
% issueno)
return
if prettycomiss is None and len(str(issueno)) > 0:
#if int(issueno) < 0:
# self._log("issue detected is a negative")
# prettycomiss = '-' + str(zeroadd) + str(abs(issueno))
if int(issueno) < 10:
logger.fdebug('issue detected less than 10')
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
logger.fdebug('Zero level supplement set to ' +
str(mylar.CONFIG.ZERO_LEVEL_N) +
'. Issue will be set as : ' + str(prettycomiss))
elif int(issueno) >= 10 and int(issueno) < 100:
logger.fdebug(
'issue detected greater than 10, but less than 100')
if mylar.CONFIG.ZERO_LEVEL_N == "none":
zeroadd = ""
else:
zeroadd = "0"
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(zeroadd) + str(iss)
else:
prettycomiss = str(zeroadd) + str(int(issueno))
else:
prettycomiss = str(zeroadd) + str(iss)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
logger.fdebug('Zero level supplement set to ' +
str(mylar.CONFIG.ZERO_LEVEL_N) +
'.Issue will be set as : ' + str(prettycomiss))
else:
logger.fdebug('issue detected greater than 100')
if issuenum == 'infinity':
prettycomiss = 'infinity'
else:
if '.' in iss:
if int(iss_decval) > 0:
issueno = str(iss)
prettycomiss = str(issueno)
if issue_except != 'None':
prettycomiss = str(prettycomiss) + issue_except
logger.fdebug('Zero level supplement set to ' +
str(mylar.CONFIG.ZERO_LEVEL_N) +
'. Issue will be set as : ' + str(prettycomiss))
elif len(str(issueno)) == 0:
prettycomiss = str(issueno)
logger.fdebug(
'issue length error - cannot determine length. Defaulting to None: '
+ str(prettycomiss))
logger.fdebug('Pretty Comic Issue is : ' + str(prettycomiss))
if mylar.CONFIG.UNICODE_ISSUENUMBER:
logger.fdebug('Setting this to Unicode format as requested: %s' %
prettycomiss)
prettycomiss = unicodeissue
issueyear = issuedate[:4]
month = issuedate[5:7].replace('-', '').strip()
month_name = helpers.fullmonth(month)
if month_name is None:
month_name = 'None'
logger.fdebug('Issue Year : ' + str(issueyear))
logger.fdebug('Publisher: ' + publisher)
logger.fdebug('Series: ' + series)
logger.fdebug('Year: ' + str(seriesyear))
logger.fdebug('Comic Location: ' + comlocation)
if self.comic['Corrected_Type'] is not None:
if self.comic['Type'] != self.comic['Corrected_Type']:
booktype = self.comic['Corrected_Type']
else:
booktype = self.comic['Type']
else:
booktype = self.comic['Type']
if booktype == 'Print' or all(
[booktype != 'Print', mylar.CONFIG.FORMAT_BOOKTYPE is False]):
chunk_fb = re.sub('\$Type', '', file_format)
chunk_b = re.compile(r'\s+')
chunk_file_format = chunk_b.sub(' ', chunk_fb)
else:
chunk_file_format = file_format
if any([comversion is None, booktype != 'Print']):
comversion = 'None'
#if comversion is None, remove it so it doesn't populate with 'None'
if comversion == 'None':
chunk_f_f = re.sub('\$VolumeN', '', chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug(
'No version # found for series, removing from filename')
logger.fdebug("new format: " + str(chunk_file_format))
if annualize is None:
chunk_f_f = re.sub('\$Annual', '', chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug('not an annual - removing from filename paramaters')
logger.fdebug('new format: ' + str(chunk_file_format))
else:
logger.fdebug('chunk_file_format is: ' + str(chunk_file_format))
if mylar.CONFIG.ANNUALS_ON:
if 'annual' in series.lower():
if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
#if it's an annual, but $annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
#prettycomiss = "Annual " + str(prettycomiss)
logger.fdebug(
'[%s][ANNUALS-ON][ANNUAL IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s'
% (series, prettycomiss))
else:
#because it exists within title, strip it then use formatting tag for placement of wording.
chunk_f_f = re.sub('\$Annual', '', chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug(
'[%s][ANNUALS-ON][ANNUAL IN SERIES][ANNUAL FORMAT] prettycomiss: %s'
% (series, prettycomiss))
else:
if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
#if it's an annual, but $annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
prettycomiss = "Annual %s" % prettycomiss
logger.fdebug(
'[%s][ANNUALS-ON][ANNUAL NOT IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s'
% (series, prettycomiss))
else:
logger.fdebug(
'[%s][ANNUALS-ON][ANNUAL NOT IN SERIES][ANNUAL FORMAT] prettycomiss: %s'
% (series, prettycomiss))
else:
#if annuals aren't enabled, then annuals are being tracked as independent series.
#annualize will be true since it's an annual in the seriesname.
if 'annual' in series.lower():
if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
#if it's an annual, but $annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
#prettycomiss = "Annual " + str(prettycomiss)
logger.fdebug(
'[%s][ANNUALS-OFF][ANNUAL IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s'
% (series, prettycomiss))
else:
#because it exists within title, strip it then use formatting tag for placement of wording.
chunk_f_f = re.sub('\$Annual', '', chunk_file_format)
chunk_f = re.compile(r'\s+')
chunk_file_format = chunk_f.sub(' ', chunk_f_f)
logger.fdebug(
'[%s][ANNUALS-OFF][ANNUAL IN SERIES][ANNUAL FORMAT] prettycomiss: %s'
% (series, prettycomiss))
else:
if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower():
#if it's an annual, but $annual isn't specified in file_format, we need to
#force it in there, by default in the format of $Annual $Issue
prettycomiss = "Annual %s" % prettycomiss
logger.fdebug(
'[%s][ANNUALS-OFF][ANNUAL NOT IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s'
% (series, prettycomiss))
else:
logger.fdebug(
'[%s][ANNUALS-OFF][ANNUAL NOT IN SERIES][ANNUAL FORMAT] prettycomiss: %s'
% (series, prettycomiss))
logger.fdebug('Annual detected within series title of ' +
series + '. Not auto-correcting issue #')
seriesfilename = seriesfilename #.encode('ascii', 'ignore').strip()
filebad = [
':', ',', '/', '?', '!', '\'', '\"', '\*'
] #in u_comicname or '/' in u_comicname or ',' in u_comicname or '?' in u_comicname:
for dbd in filebad:
if dbd in seriesfilename:
if any([dbd == '/', dbd == '*']):
repthechar = '-'
else:
repthechar = ''
seriesfilename = seriesfilename.replace(dbd, repthechar)
logger.fdebug(
'Altering series name due to filenaming restrictions: ' +
seriesfilename)
publisher = re.sub('!', '', publisher)
file_values = {
'$Series': seriesfilename,
'$Issue': prettycomiss,
'$Year': issueyear,
'$series': series.lower(),
'$Publisher': publisher,
'$publisher': publisher.lower(),
'$VolumeY': 'V' + str(seriesyear),
'$VolumeN': comversion,
'$monthname': month_name,
'$month': month,
'$Annual': 'Annual',
'$Type': booktype
}
extensions = ('.cbr', '.cbz', '.cb7')
if ofilename.lower().endswith(extensions):
path, ext = os.path.splitext(ofilename)
if file_format == '':
logger.fdebug(
'Rename Files is not enabled - keeping original filename.')
#check if extension is in nzb_name - will screw up otherwise
if ofilename.lower().endswith(extensions):
nfilename = ofilename[:-4]
else:
nfilename = ofilename
else:
chunk_file_format = re.sub('[()|[]]', '',
chunk_file_format).strip()
nfilename = helpers.replace_all(chunk_file_format, file_values)
if mylar.CONFIG.REPLACE_SPACES:
#mylar.CONFIG.REPLACE_CHAR ...determines what to replace spaces with underscore or dot
nfilename = nfilename.replace(' ', mylar.CONFIG.REPLACE_CHAR)
nfilename = re.sub('[\,\:]', '', nfilename) + ext.lower()
logger.fdebug('New Filename: ' + nfilename)
if mylar.CONFIG.LOWERCASE_FILENAMES:
nfilename = nfilename.lower()
dst = os.path.join(comlocation, nfilename)
else:
dst = os.path.join(comlocation, nfilename)
logger.fdebug('Source: ' + ofilename)
logger.fdebug('Destination: ' + dst)
rename_this = {
"destination_dir": dst,
"nfilename": nfilename,
"issueid": issueid,
"comicid": comicid
}
return rename_this
def findComic(name, mode, issue, limityear=None, type=None):
#with mb_lock:
comicResults = None
comicLibrary = listLibrary()
comiclist = []
arcinfolist = []
commons = ['and', 'the', '&', '-']
for x in commons:
cnt = 0
for m in re.finditer(x, name.lower()):
cnt +=1
tehstart = m.start()
tehend = m.end()
if any([x == 'the', x == 'and']):
if len(name) == tehend:
tehend =-1
if not all([tehstart == 0, name[tehend] == ' ']) or not all([tehstart != 0, name[tehstart-1] == ' ', name[tehend] == ' ']):
continue
else:
name = name.replace(x, ' ', cnt)
originalname = name
if '+' in name:
name = re.sub('\+', 'PLUS', name)
pattern = re.compile(r'\w+', re.UNICODE)
name = pattern.findall(name)
if '+' in originalname:
y = []
for x in name:
y.append(re.sub("PLUS", "%2B", x))
name = y
if limityear is None: limityear = 'None'
comicquery = name
if mylar.CONFIG.COMICVINE_API == 'None' or mylar.CONFIG.COMICVINE_API is None:
logger.warn('You have not specified your own ComicVine API key - this is a requirement. Get your own @ http://api.comicvine.com.')
return
else:
comicapi = mylar.CONFIG.COMICVINE_API
if type is None:
type = 'volume'
#let's find out how many results we get from the query...
searched = pullsearch(comicapi, comicquery, 0, type)
if searched is None:
return False
totalResults = searched.getElementsByTagName('number_of_total_results')[0].firstChild.wholeText
logger.fdebug("there are " + str(totalResults) + " search results...")
if not totalResults:
return False
if int(totalResults) > 1000:
logger.warn('Search returned more than 1000 hits [' + str(totalResults) + ']. Only displaying first 1000 results - use more specifics or the exact ComicID if required.')
totalResults = 1000
countResults = 0
while (countResults < int(totalResults)):
#logger.fdebug("querying " + str(countResults))
if countResults > 0:
offsetcount = countResults
searched = pullsearch(comicapi, comicquery, offsetcount, type)
comicResults = searched.getElementsByTagName(type)
body = ''
n = 0
if not comicResults:
break
for result in comicResults:
#retrieve the first xml tag (<tag>data</tag>)
#that the parser finds with name tagName:
arclist = []
if type == 'story_arc':
#call cv.py here to find out issue count in story arc
try:
logger.fdebug('story_arc ascension')
names = len(result.getElementsByTagName('name'))
n = 0
logger.fdebug('length: ' + str(names))
xmlpub = None #set this incase the publisher field isn't populated in the xml
while (n < names):
logger.fdebug(result.getElementsByTagName('name')[n].parentNode.nodeName)
if result.getElementsByTagName('name')[n].parentNode.nodeName == 'story_arc':
logger.fdebug('yes')
try:
xmlTag = result.getElementsByTagName('name')[n].firstChild.wholeText
xmlTag = xmlTag.rstrip()
logger.fdebug('name: ' + xmlTag)
except:
logger.error('There was a problem retrieving the given data from ComicVine. Ensure that www.comicvine.com is accessible.')
return
elif result.getElementsByTagName('name')[n].parentNode.nodeName == 'publisher':
logger.fdebug('publisher check.')
xmlpub = result.getElementsByTagName('name')[n].firstChild.wholeText
n+=1
except:
logger.warn('error retrieving story arc search results.')
return
siteurl = len(result.getElementsByTagName('site_detail_url'))
s = 0
logger.fdebug('length: ' + str(names))
xmlurl = None
while (s < siteurl):
logger.fdebug(result.getElementsByTagName('site_detail_url')[s].parentNode.nodeName)
if result.getElementsByTagName('site_detail_url')[s].parentNode.nodeName == 'story_arc':
try:
xmlurl = result.getElementsByTagName('site_detail_url')[s].firstChild.wholeText
except:
logger.error('There was a problem retrieving the given data from ComicVine. Ensure that www.comicvine.com is accessible.')
return
s+=1
xmlid = result.getElementsByTagName('id')[0].firstChild.wholeText
if xmlid is not None:
arcinfolist = storyarcinfo(xmlid)
logger.info('[IMAGE] : ' + arcinfolist['comicimage'])
comiclist.append({
'name': xmlTag,
'comicyear': arcinfolist['comicyear'],
'comicid': xmlid,
'cvarcid': xmlid,
'url': xmlurl,
'issues': arcinfolist['issues'],
'comicimage': arcinfolist['comicimage'],
'publisher': xmlpub,
'description': arcinfolist['description'],
'deck': arcinfolist['deck'],
'arclist': arcinfolist['arclist'],
'haveit': arcinfolist['haveit']
})
else:
comiclist.append({
'name': xmlTag,
'comicyear': arcyear,
'comicid': xmlid,
'url': xmlurl,
'issues': issuecount,
'comicimage': xmlimage,
'publisher': xmlpub,
'description': xmldesc,
'deck': xmldeck,
'arclist': arclist,
'haveit': haveit
})
logger.fdebug('IssueID\'s that are a part of ' + xmlTag + ' : ' + str(arclist))
else:
xmlcnt = result.getElementsByTagName('count_of_issues')[0].firstChild.wholeText
#here we can determine what called us, and either start gathering all issues or just limited ones.
if issue is not None and str(issue).isdigit():
#this gets buggered up with NEW/ONGOING series because the db hasn't been updated
#to reflect the proper count. Drop it by 1 to make sure.
limiter = int(issue) - 1
else: limiter = 0
#get the first issue # (for auto-magick calcs)
iss_len = len(result.getElementsByTagName('name'))
i=0
xmlfirst = '1'
xmllast = None
try:
while (i < iss_len):
if result.getElementsByTagName('name')[i].parentNode.nodeName == 'first_issue':
xmlfirst = result.getElementsByTagName('issue_number')[i].firstChild.wholeText
if '\xbd' in xmlfirst:
xmlfirst = '1' #if the first issue is 1/2, just assume 1 for logistics
elif result.getElementsByTagName('name')[i].parentNode.nodeName == 'last_issue':
xmllast = result.getElementsByTagName('issue_number')[i].firstChild.wholeText
if all([xmllast is not None, xmlfirst is not None]):
break
i+=1
except:
xmlfirst = '1'
if all([xmlfirst == xmllast, xmlfirst.isdigit(), xmlcnt == '0']):
xmlcnt = '1'
#logger.info('There are : ' + str(xmlcnt) + ' issues in this series.')
#logger.info('The first issue started at # ' + str(xmlfirst))
try:
d = decimal.Decimal(xmlfirst)
except Exception as e:
d = 1 # assume 1st issue as #1 if it can't be parsed.
if d < 1:
cnt_numerical = int(xmlcnt) + 1
else:
cnt_numerical = int(xmlcnt) + int(math.ceil(d)) # (of issues + start of first issue = numerical range)
#logger.info('The maximum issue number should be roughly # ' + str(cnt_numerical))
#logger.info('The limiter (issue max that we know of) is # ' + str(limiter))
if cnt_numerical >= limiter:
cnl = len (result.getElementsByTagName('name'))
cl = 0
xmlTag = 'None'
xml_lastissueid = 'None'
while (cl < cnl):
if result.getElementsByTagName('name')[cl].parentNode.nodeName == 'volume':
xmlTag = result.getElementsByTagName('name')[cl].firstChild.wholeText
#break
#if result.getElementsByTagName('name')[cl].parentNode.nodeName == 'image':
# xmlimage = result.getElementsByTagName('super_url')[0].firstChild.wholeText
if result.getElementsByTagName('name')[cl].parentNode.nodeName == 'last_issue':
xml_lastissueid = result.getElementsByTagName('id')[cl].firstChild.wholeText
cl+=1
try:
xmlimage = result.getElementsByTagName('super_url')[0].firstChild.wholeText
except Exception:
try:
xmlimage = result.getElementsByTagName('small_url')[0].firstChild.wholeText
except Exception:
xmlimage = "cache/blankcover.jpg"
if (result.getElementsByTagName('start_year')[0].firstChild) is not None:
xmlYr = result.getElementsByTagName('start_year')[0].firstChild.wholeText
else: xmlYr = "0000"
yearRange = []
tmpYr = re.sub('\?', '', xmlYr)
if tmpYr.isdigit():
yearRange.append(tmpYr)
tmpyearRange = int(xmlcnt) / 12
if float(tmpyearRange): tmpyearRange +1
possible_years = int(tmpYr) + tmpyearRange
for i in range(int(tmpYr), int(possible_years),1):
if not any(int(x) == int(i) for x in yearRange):
yearRange.append(str(i))
logger.fdebug('[RESULT][' + str(limityear) + '] ComicName:' + xmlTag + ' -- ' + str(xmlYr) + ' [Series years: ' + str(yearRange) + ']')
if tmpYr != xmlYr:
xmlYr = tmpYr
if any([v in limityear for v in yearRange]) or limityear == 'None':
xmlurl = result.getElementsByTagName('site_detail_url')[0].firstChild.wholeText
idl = len (result.getElementsByTagName('id'))
idt = 0
xmlid = None
while (idt < idl):
if result.getElementsByTagName('id')[idt].parentNode.nodeName == 'volume':
xmlid = result.getElementsByTagName('id')[idt].firstChild.wholeText
break
idt+=1
if xmlid is None:
logger.error('Unable to figure out the comicid - skipping this : ' + str(xmlurl))
continue
publishers = result.getElementsByTagName('publisher')
if len(publishers) > 0:
pubnames = publishers[0].getElementsByTagName('name')
if len(pubnames) >0:
xmlpub = pubnames[0].firstChild.wholeText
else:
xmlpub = "Unknown"
else:
xmlpub = "Unknown"
#ignore specific publishers on a global scale here.
if mylar.CONFIG.IGNORED_PUBLISHERS is not None and any([x for x in mylar.CONFIG.IGNORED_PUBLISHERS if x.lower() == xmlpub.lower()]):
logger.fdebug('Ignored publisher [%s]. Ignoring this result.' % xmlpub)
continue
try:
xmldesc = result.getElementsByTagName('description')[0].firstChild.wholeText
except:
xmldesc = "None"
#this is needed to display brief synopsis for each series on search results page.
try:
xmldeck = result.getElementsByTagName('deck')[0].firstChild.wholeText
except:
xmldeck = "None"
xmltype = None
if xmldeck != 'None':
if any(['print' in xmldeck.lower(), 'digital' in xmldeck.lower(), 'paperback' in xmldeck.lower(), 'one shot' in re.sub('-', '', xmldeck.lower()).strip(), 'hardcover' in xmldeck.lower()]):
if all(['print' in xmldeck.lower(), 'reprint' not in xmldeck.lower()]):
xmltype = 'Print'
elif 'digital' in xmldeck.lower():
xmltype = 'Digital'
elif 'paperback' in xmldeck.lower():
xmltype = 'TPB'
elif 'hardcover' in xmldeck.lower():
xmltype = 'HC'
elif 'oneshot' in re.sub('-', '', xmldeck.lower()).strip():
xmltype = 'One-Shot'
else:
xmltype = 'Print'
if xmldesc != 'None' and xmltype is None:
if 'print' in xmldesc[:60].lower() and all(['print edition can be found' not in xmldesc.lower(), 'reprints' not in xmldesc.lower()]):
xmltype = 'Print'
elif 'digital' in xmldesc[:60].lower() and 'digital edition can be found' not in xmldesc.lower():
xmltype = 'Digital'
elif all(['paperback' in xmldesc[:60].lower(), 'paperback can be found' not in xmldesc.lower()]) or 'collects' in xmldesc[:60].lower():
xmltype = 'TPB'
elif 'hardcover' in xmldesc[:60].lower() and 'hardcover can be found' not in xmldesc.lower():
xmltype = 'HC'
elif any(['one-shot' in xmldesc[:60].lower(), 'one shot' in xmldesc[:60].lower()]) and any(['can be found' not in xmldesc.lower(), 'following the' not in xmldesc.lower()]):
i = 0
xmltype = 'One-Shot'
avoidwords = ['preceding', 'after the special', 'following the']
while i < 2:
if i == 0:
cbd = 'one-shot'
elif i == 1:
cbd = 'one shot'
tmp1 = xmldesc[:60].lower().find(cbd)
if tmp1 != -1:
for x in avoidwords:
tmp2 = xmldesc[:tmp1].lower().find(x)
if tmp2 != -1:
xmltype = 'Print'
i = 3
break
i+=1
else:
xmltype = 'Print'
if xmlid in comicLibrary:
haveit = comicLibrary[xmlid]
else:
haveit = "No"
comiclist.append({
'name': xmlTag,
'comicyear': xmlYr,
'comicid': xmlid,
'url': xmlurl,
'issues': xmlcnt,
'comicimage': xmlimage,
'publisher': xmlpub,
'description': xmldesc,
'deck': xmldeck,
'type': xmltype,
'haveit': haveit,
'lastissueid': xml_lastissueid,
'seriesrange': yearRange # returning additional information about series run polled from CV
})
#logger.fdebug('year: %s - constraint met: %s [%s] --- 4050-%s' % (xmlYr,xmlTag,xmlYr,xmlid))
else:
#logger.fdebug('year: ' + str(xmlYr) + ' - contraint not met. Has to be within ' + str(limityear))
pass
n+=1
#search results are limited to 100 and by pagination now...let's account for this.
countResults = countResults + 100
return comiclist
logger.fdebug('gzip buffered')
f = gzip.GzipFile(fileobj=buf)
logger.fdebug('gzip filed.')
torrent = f.read()
logger.fdebug('gzip read.')
else:
torrent = response.read()
except Exception, e:
logger.warn('Error fetching data from %s: %s' % (site, e))
return "fail"
with open(filepath, 'wb') as the_file:
the_file.write(torrent)
logger.fdebug("saved.")
#logger.fdebug('torrent file saved as : ' + str(filepath))
if mylar.TORRENT_LOCAL:
return "pass"
#remote_file = urllib2.urlopen(linkit)
#if linkit[-7:] != "torrent":
# filename += ".torrent"
#local_file = open('%s' % (os.path.join(mylar.CACHE_DIR,filename)), 'w')
#local_file.write(remote_file.read())
#local_file.close()
#remote_file.close()
elif mylar.TORRENT_SEEDBOX:
tssh = ftpsshup.putfile(filepath, filename)
return tssh
def test_notify(self):
module = '[TEST-NOTIFIER]'
try:
r = self._session.get(self.TEST_NMA_URL,
params={'apikey': self.apikey},
verify=True)
except requests.exceptions.RequestException as e:
logger.error(
module + '[' + str(e) +
'] Unable to send via NMA. Aborting test notification - something is probably wrong...'
)
return {'status': False, 'message': str(e)}
logger.fdebug('[NMA] Status code returned: ' + str(r.status_code))
if r.status_code == 200:
from xml.dom.minidom import parseString
dom = parseString(r.content)
try:
success_info = dom.getElementsByTagName('success')
success_code = success_info[0].getAttribute('code')
except:
error_info = dom.getElementsByTagName('error')
error_code = error_info[0].getAttribute('code')
error_message = error_info[0].childNodes[0].nodeValue
logger.info(module + '[' + str(error_code) + '] ' +
error_message)
return {
'status': False,
'message': '[' + str(error_code) + '] ' + error_message
}
else:
logger.info(
module + '[' + str(success_code) +
'] NotifyMyAndroid apikey valid. Testing notification service with it.'
)
elif r.status_code >= 400 and r.status_code < 500:
logger.error(module +
' NotifyMyAndroid request failed: %s' % r.content)
return {
'status': False,
'message':
'Unable to send request to NMA - check your connection.'
}
else:
logger.error(module +
' NotifyMyAndroid notification failed serverside.')
return {
'status': False,
'message': 'Internal Server Error. Try again later.'
}
event = 'Test Message'
description = 'ZOMG Lazors PewPewPew!'
data = {
'apikey': self.apikey,
'application': 'Mylar',
'event': event.encode('utf-8'),
'description': description.encode('utf-8'),
'priority': 2
}
return self._send(data, '[NOTIFIER]')
def newpull():
pagelinks = "http://www.previewsworld.com/Home/1/1/71/952"
pageresponse = urllib2.urlopen(pagelinks)
soup = BeautifulSoup(pageresponse)
getthedate = soup.findAll("div", {"class": "Headline"})[0]
#the date will be in the FIRST ahref
try:
getdate_link = getthedate('a')[0]
newdates = getdate_link.findNext(text=True).strip()
except IndexError:
newdates = getthedate.findNext(text=True).strip()
logger.fdebug('New Releases date detected as : ' +
re.sub('New Releases For', '', newdates).strip())
cntlinks = soup.findAll('tr')
lenlinks = len(cntlinks)
publish = []
resultURL = []
resultmonth = []
resultyear = []
x = 0
cnt = 0
endthis = False
pull_list = []
publishers = {
'914': 'DARK HORSE COMICS',
'915': 'DC COMICS',
'916': 'IDW PUBLISHING',
'917': 'IMAGE COMICS',
'918': 'MARVEL COMICS',
'952': 'COMICS & GRAPHIC NOVELS'
}
while (x < lenlinks):
headt = cntlinks[
x] #iterate through the hrefs pulling out only results.
if 'STK669382' in str(headt):
x += 1
continue
elif '?stockItemID=' in str(headt):
#914 - Dark Horse Comics
#915 - DC Comics
#916 - IDW Publishing
#917 - Image Comics
#918 - Marvel Comics
#952 - Comics & Graphic Novels
# - Magazines
findurl_link = headt.findAll('a', href=True)[0]
urlID = findurl_link.findNext(text=True)
issue_link = findurl_link['href']
issue_lk = issue_link.find('?stockItemID=')
if issue_lk == -1:
continue
#headName = headt.findNext(text=True)
publisher_id = issue_link[issue_lk - 3:issue_lk]
for pub in publishers:
if pub == publisher_id:
isspublisher = publishers[pub]
#logger.fdebug('publisher:' + str(isspublisher))
found_iss = headt.findAll('td')
if "Home/1/1/71/920" in issue_link:
#logger.fdebug('Ignoring - menu option.')
return
if "PREVIEWS" in headt:
#logger.fdebug('Ignoring: ' + found_iss[0])
break
if "MAGAZINES" in headt:
#logger.fdebug('End.')
endthis = True
break
if len(found_iss) > 0:
pull_list.append({
"iss_url":
found_iss[0],
"name":
found_iss[1].findNext(text=True),
"price":
found_iss[2],
"publisher":
isspublisher,
"ID":
urlID
})
if endthis == True: break
x += 1
logger.fdebug(
'Saving new pull-list information into local file for subsequent merge'
)
except_file = os.path.join(mylar.CACHE_DIR, 'newreleases.txt')
try:
csvfile = open(str(except_file), 'rb')
csvfile.close()
except (OSError, IOError):
logger.fdebug('file does not exist - continuing.')
else:
logger.fdebug('file exists - removing.')
os.remove(except_file)
oldpub = None
breakhtml = {"<td>", "<tr>", "</td>", "</tr>"}
with open(str(except_file), 'wb') as f:
f.write('%s\n' % (newdates))
for pl in pull_list:
if pl['publisher'] == oldpub:
exceptln = str(pl['ID']) + "\t" + str(pl['name']) + "\t" + str(
pl['price'])
else:
exceptln = pl['publisher'] + "\n" + str(pl['ID']) + "\t" + str(
pl['name']) + "\t" + str(pl['price'])
for lb in breakhtml:
exceptln = re.sub(lb, '', exceptln).strip()
exceptline = exceptln.decode('utf-8', 'ignore')
f.write('%s\n' % (exceptline.encode('ascii', 'replace').strip()))
oldpub = pl['publisher']
def extract_image(location, single=False, imquality=None, comicname=None):
#location = full path to the cbr/cbz (filename included in path)
#single = should be set to True so that a single file can have the coverfile
# extracted and have the cover location returned to the calling function
#imquality = the calling function ('notif' for notifications will initiate a resize image before saving the cover)
if PIL_Found is False:
return
cover = "notfound"
pic_extensions = ('.jpg', '.png', '.webp')
issue_ends = ('1', '0')
modtime = os.path.getmtime(location)
low_infile = 999999999999999999
low_num = 1000
local_filename = os.path.join(mylar.CONFIG.CACHE_DIR, 'temp_notif')
cb_filename = None
cb_filenames = []
metadata = None
if single is True:
location_in, dir_opt = open_archive(location)
try:
cntr = 0
newlencnt = 0
newlen = 0
newlist = []
for infile in location_in.infolist():
cntr += 1
basename = os.path.basename(infile.filename)
if infile.filename == 'ComicInfo.xml':
logger.fdebug('Extracting ComicInfo.xml to display.')
metadata = location_in.read(infile.filename)
if cover == 'found':
break
filename, extension = os.path.splitext(basename)
tmp_infile = re.sub("[^0-9]", "", filename).strip()
lenfile = len(infile.filename)
if any([
tmp_infile == '', not getattr(infile, dir_opt), 'zzz'
in filename.lower(), 'logo' in filename.lower()
]) or ((comicname is not None) and all([
comicname.lower().startswith('z'),
filename.lower().startswith('z')
])):
continue
if all([
infile.filename.lower().endswith(pic_extensions),
int(tmp_infile) < int(low_infile)
]):
#logger.info('cntr: %s / infolist: %s' % (cntr, len(location_in.infolist())) )
#get the length of the filename, compare it to others. scanner ones are always different named than the other 98% of the files.
if lenfile >= newlen:
newlen = lenfile
newlencnt += 1
newlist.append({
'length': lenfile,
'filename': infile.filename,
'tmp_infile': tmp_infile
})
#logger.info('newlen: %s / newlencnt: %s' % (newlen, newlencnt))
if newlencnt > 0 and lenfile >= newlen:
#logger.info('setting it to : %s' % infile.filename)
low_infile = tmp_infile
low_infile_name = infile.filename
elif any([
'00a' in infile.filename, '00b' in infile.filename,
'00c' in infile.filename, '00d' in infile.filename,
'00e' in infile.filename, '00fc'
in infile.filename.lower()
]) and infile.filename.endswith(
pic_extensions) and cover == "notfound":
if cntr == 0:
altlist = ('00a', '00b', '00c', '00d', '00e', '00fc')
for alt in altlist:
if alt in infile.filename.lower():
cb_filename = infile.filename
cover = "found"
#logger.fdebug('[%s] cover found:%s' % (alt, infile.filename))
break
elif all([
tmp_infile.endswith(issue_ends),
infile.filename.lower().endswith(pic_extensions),
int(tmp_infile) < int(low_infile), cover == 'notfound'
]):
cb_filenames.append(infile.filename)
if cover != "found" and any(
[len(cb_filenames) > 0, low_infile != 9999999999999]):
logger.fdebug(
'Invalid naming sequence for jpgs discovered. Attempting to find the lowest sequence and will use as cover (it might not work). Currently : %s'
% (low_infile_name))
# based on newlist - if issue doesn't end in 0 & 1, take the lowest numeric of the most common length of filenames within the rar
if not any(
[low_infile.endswith('0'),
low_infile.endswith('1')]):
from collections import Counter
cnt = Counter([t['length'] for t in newlist])
#logger.info('cnt: %s' % (cnt,)) #cnt: Counter({15: 23, 20: 1})
tmpst = 999999999
cntkey = max(cnt.items(), key=itemgetter(1))[0]
#logger.info('cntkey: %s' % cntkey)
for x in newlist:
if x['length'] == cntkey and int(
x['tmp_infile']) < tmpst:
tmpst = int(x['tmp_infile'])
cb_filename = x['filename']
logger.fdebug('SETTING cb_filename set to : %s' %
cb_filename)
else:
cb_filename = low_infile_name
cover = "found"
except Exception as e:
logger.error(
'[ERROR] Unable to properly retrieve the cover. It\'s probably best to re-tag this file : %s'
% e)
return
logger.fdebug('cb_filename set to : %s' % cb_filename)
if extension is not None:
ComicImage = local_filename + extension
try:
insidefile = location_in.getinfo(cb_filename)
img = Image.open(BytesIO(location_in.read(insidefile)))
imdata = scale_image(img, "JPEG", 600)
try:
ComicImage = str(base64.b64encode(imdata), 'utf-8')
RawImage = imdata
except Exception as e:
ComicImage = str(base64.b64encode(imdata + "==="), 'utf-8')
RawImage = imdata + "==="
except Exception as e:
logger.warn('[WARNING] Unable to resize existing image: %s' %
e)
else:
ComicImage = local_filename
return {
'ComicImage': ComicImage,
'metadata': metadata,
'rawImage': RawImage
}
def solicit(month, year):
#convert to numerics just to ensure this...
month = int(month)
year = int(year)
#print ( "month: " + str(month) )
#print ( "year: " + str(year) )
# in order to gather ALL upcoming - let's start to loop through months going ahead one at a time
# until we get a null then break. (Usually not more than 3 months in advance is available)
mnloop = 0
upcoming = []
publishers = {
'DC Comics': 'DC Comics',
'DC\'s': 'DC Comics',
'Marvel': 'Marvel Comics',
'Image': 'Image Comics',
'IDW': 'IDW Publishing',
'Dark Horse': 'Dark Horse'
}
# -- this is no longer needed (testing)
# while (mnloop < 5):
# if year == 2014:
# if len(str(month)) == 1:
# month_string = '0' + str(month)
# else:
# month_string = str(month)
# datestring = str(year) + str(month_string)
# else:
# datestring = str(month) + str(year)
# pagelinks = "http://www.comicbookresources.com/tag/solicits" + str(datestring)
#using the solicits+datestring leaves out some entries occasionally
#should use http://www.comicbookresources.com/tag/solicitations
#then just use the logic below but instead of datestring, find the month term and
#go ahead up to +5 months.
if month > 0:
month_start = month
month_end = month + 5
#if month_end > 12:
# ms = 8, me=13 [(12-8)+(13-12)] = [4 + 1] = 5
# [(12 - ms) + (me - 12)] = number of months (5)
monthlist = []
mongr = month_start
#we need to build the months we can grab, but the non-numeric way.
while (mongr <= month_end):
mon = mongr
if mon == 13:
mon = 1
year += 1
if len(str(mon)) == 1:
mon = '0' + str(mon)
monthlist.append({
"month": helpers.fullmonth(str(mon)).lower(),
"num_month": mon,
"year": str(year)
})
mongr += 1
logger.info('months: ' + str(monthlist))
pagelinks = "http://www.comicbookresources.com/tag/solicitations"
#logger.info('datestring:' + datestring)
#logger.info('checking:' + pagelinks)
pageresponse = urllib2.urlopen(pagelinks)
soup = BeautifulSoup(pageresponse)
cntlinks = soup.findAll('h3')
lenlinks = len(cntlinks)
#logger.info( str(lenlinks) + ' results' )
publish = []
resultURL = []
resultmonth = []
resultyear = []
x = 0
cnt = 0
while (x < lenlinks):
headt = cntlinks[
x] #iterate through the hrefs pulling out only results.
if "/?page=article&id=" in str(headt):
#print ("titlet: " + str(headt))
headName = headt.findNext(text=True)
#print ('headName: ' + headName)
if 'Image' in headName: print 'IMAGE FOUND'
if not all([
'Marvel' in headName, 'DC' in headName, 'Image'
in headName
]) and ('Solicitations' in headName or 'Solicits' in headName):
# test for month here (int(month) + 5)
if not any(
d.get('month', None) == str(headName).lower()
for d in monthlist):
for mt in monthlist:
if mt['month'] in headName.lower():
logger.info('matched on month: ' +
str(mt['month']))
logger.info('matched on year: ' +
str(mt['year']))
resultmonth.append(mt['num_month'])
resultyear.append(mt['year'])
pubstart = headName.find('Solicitations')
publishchk = False
for pub in publishers:
if pub in headName[:pubstart]:
#print 'publisher:' + str(publishers[pub])
publish.append(publishers[pub])
publishchk = True
break
if publishchk == False:
break
#publish.append( headName[:pubstart].strip() )
abc = headt.findAll('a', href=True)[0]
ID_som = abc[
'href'] #first instance will have the right link...
resultURL.append(ID_som)
#print '(' + str(cnt) + ') [ ' + publish[cnt] + '] Link URL: ' + resultURL[cnt]
cnt += 1
else:
logger.info('incorrect month - not using.')
x += 1
if cnt == 0:
return #break # no results means, end it
loopthis = (cnt - 1)
#this loops through each 'found' solicit page
#shipdate = str(month_string) + '-' + str(year) - not needed.
while (loopthis >= 0):
#print 'loopthis is : ' + str(loopthis)
#print 'resultURL is : ' + str(resultURL[loopthis])
shipdate = str(resultmonth[loopthis]) + '-' + str(
resultyear[loopthis])
upcoming += populate(resultURL[loopthis], publish[loopthis],
shipdate)
loopthis -= 1
logger.info(str(len(upcoming)) + ' upcoming issues discovered.')
newfl = mylar.CACHE_DIR + "/future-releases.txt"
newtxtfile = open(newfl, 'wb')
cntr = 1
for row in upcoming:
if row['Extra'] is None or row['Extra'] == '':
extrarow = 'N/A'
else:
extrarow = row['Extra']
newtxtfile.write(
str(row['Shipdate']) + '\t' + str(row['Publisher']) + '\t' +
str(row['Issue']) + '\t' + str(row['Comic']) + '\t' +
str(extrarow) + '\tSkipped' + '\t' + str(cntr) + '\n')
cntr += 1
newtxtfile.close()
logger.fdebug('attempting to populate future upcoming...')
mylardb = os.path.join(mylar.DATA_DIR, "mylar.db")
connection = sqlite3.connect(str(mylardb))
cursor = connection.cursor()
# we should extract the issues that are being watched, but no data is available yet ('Watch For' status)
# once we get the data, store it, wipe the existing table, retrieve the new data, populate the data into
# the table, recheck the series against the current watchlist and then restore the Watch For data.
cursor.executescript('drop table if exists future;')
cursor.execute(
"CREATE TABLE IF NOT EXISTS future (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, FutureID text, ComicID text);"
)
connection.commit()
csvfile = open(newfl, "rb")
creader = csv.reader(csvfile, delimiter='\t')
t = 1
for row in creader:
try:
#print ("Row: %s" % row)
cursor.execute("INSERT INTO future VALUES (?,?,?,?,?,?,?,null);",
row)
except Exception, e:
logger.fdebug("Error - invald arguments...-skipping")
pass
t += 1
def extract_image(location, single=False, imquality=None):
#location = full path to the cbr/cbz (filename included in path)
#single = should be set to True so that a single file can have the coverfile
# extracted and have the cover location returned to the calling function
#imquality = the calling function ('notif' for notifications will initiate a resize image before saving the cover)
if PIL_Found is False:
return
cover = "notfound"
pic_extensions = ('.jpg', '.png', '.webp')
issue_ends = ('1', '0')
modtime = os.path.getmtime(location)
low_infile = 9999999999999
low_num = 1000
local_filename = os.path.join(mylar.CONFIG.CACHE_DIR, 'temp_notif')
cb_filename = None
cb_filenames = []
metadata = None
if single is True:
if location.endswith(".cbz"):
location_in = zipfile.ZipFile(location)
dir_opt = 'is_dir'
actual_ext = '.cbz'
else:
try:
location_in = rarfile.RarFile(location)
dir_opt = 'isdir'
actual_ext = '.cbr'
except rarfile.BadRarFile as e:
logger.warn('[WARNING] %s: %s' % (location, e))
try:
logger.info(
'Trying to see if this is a zip renamed as a rar: %s' %
(location))
location_in = zipfile.ZipFile(location)
dir_opt = 'is_dir'
actual_ext = '.cbz'
except Exception as e:
logger.warn('[EXCEPTION] %s' % e)
return
except:
logger.warn('[EXCEPTION]: %s' % sys.exec_info()[0])
return
try:
for infile in location_in.infolist():
basename = os.path.basename(infile.filename)
if infile.filename == 'ComicInfo.xml':
logger.fdebug('Extracting ComicInfo.xml to display.')
metadata = location_in.read(infile.filename)
if cover == 'found':
break
filename, extension = os.path.splitext(basename)
tmp_infile = re.sub("[^0-9]", "", filename).strip()
if any([
tmp_infile == '', not getattr(infile, dir_opt), 'zzz'
in filename
]):
continue
#logger.fdebug('[%s]issue_ends: %s' % (tmp_infile, tmp_infile.endswith(issue_ends)))
#logger.fdebug('ext_ends: %s' % infile.filename.lower().endswith(pic_extensions))
#logger.fdebug('(%s) < (%s) == %s' % (int(tmp_infile), int(low_infile), int(tmp_infile)<int(low_infile)))
#logger.fdebug('is_dir == %s' % (not getattr(infile, dir_opt)))
if all([
infile.filename.lower().endswith(pic_extensions),
int(tmp_infile) < int(low_infile)
]):
low_infile = tmp_infile
low_infile_name = infile.filename
elif any([
'00a' in infile.filename, '00b' in infile.filename,
'00c' in infile.filename, '00d' in infile.filename,
'00e' in infile.filename, '00fc'
in infile.filename.lower()
]) and infile.filename.endswith(
pic_extensions) and cover == "notfound":
altlist = ('00a', '00b', '00c', '00d', '00e', '00fc')
for alt in altlist:
if alt in infile.filename.lower():
cb_filename = infile.filename
cover = "found"
#logger.fdebug('[%s] cover found:%s' % (alt, infile.filename))
break
elif all([
tmp_infile.endswith(issue_ends),
infile.filename.lower().endswith(pic_extensions),
int(tmp_infile) < int(low_infile), cover == 'notfound'
]):
cb_filenames.append(infile.filename)
#logger.fdebug('filename set to: %s' % infile.filename)
#low_infile_name = infile.filename
#low_infile = tmp_infile
if cover != "found" and any(
[len(cb_filenames) > 0, low_infile != 9999999999999]):
logger.fdebug(
'Invalid naming sequence for jpgs discovered. Attempting to find the lowest sequence and will use as cover (it might not work). Currently : %s'
% (low_infile_name))
cb_filename = low_infile_name
cover = "found"
except Exception as e:
logger.error(
'[ERROR] Unable to properly retrieve the cover. It\'s probably best to re-tag this file : %s'
% e)
return
logger.fdebug('cb_filename set to : %s' % cb_filename)
if extension is not None:
ComicImage = local_filename + extension
try:
insidefile = location_in.getinfo(cb_filename)
img = Image.open(BytesIO(location_in.read(insidefile)))
wpercent = (600 / float(img.size[0]))
hsize = int((float(img.size[1]) * float(wpercent)))
img = img.resize((600, hsize), Image.ANTIALIAS)
output = BytesIO()
img.save(output, format="JPEG")
try:
ComicImage = str(base64.b64encode(output.getvalue()),
'utf-8')
except Exception as e:
ComicImage = str(
base64.b64encode(output.getvalue() + "==="), 'utf-8')
output.close()
except Exception as e:
logger.warn('[WARNING] Unable to resize existing image: %s' %
e)
else:
ComicImage = local_filename
return {'ComicImage': ComicImage, 'metadata': metadata}
def processor(self):
sendresponse = self.params['nzo_id']
try:
logger.fdebug('sending now to %s' % self.sab_url)
tmp_apikey = self.params['queue'].pop('apikey')
logger.fdebug('parameters set to %s' % self.params)
self.params['queue']['apikey'] = tmp_apikey
time.sleep(
5
) #pause 5 seconds before monitoring just so it hits the queue
h = requests.get(self.sab_url,
params=self.params['queue'],
verify=False)
except Exception as e:
logger.fdebug('uh-oh: %s' % e)
return self.historycheck(self.params)
else:
queueresponse = h.json()
logger.fdebug('successfully queried the queue for status')
try:
queueinfo = queueresponse['queue']
#logger.fdebug('queue: %s' % queueinfo)
logger.fdebug('Queue status : %s' % queueinfo['status'])
logger.fdebug('Queue mbleft : %s' % queueinfo['mbleft'])
if str(queueinfo['status']) == 'Paused':
logger.warn(
'[WARNING] SABnzbd has the active queue Paused. CDH will not work in this state.'
)
return {'status': 'queue_paused', 'failed': False}
while any([
str(queueinfo['status']) == 'Downloading',
str(queueinfo['status']) == 'Idle'
]) and float(queueinfo['mbleft']) > 0:
#if 'comicrn' in queueinfo['script'].lower():
# logger.warn('ComicRN has been detected as being active for this category & download. Completed Download Handling will NOT be performed due to this.')
# logger.warn('Either disable Completed Download Handling for SABnzbd within Mylar, or remove ComicRN from your category script in SABnzbd.')
# return {'status': 'double-pp', 'failed': False}
#logger.fdebug('queue_params: %s' % self.params['queue'])
queue_resp = requests.get(self.sab_url,
params=self.params['queue'],
verify=False)
queueresp = queue_resp.json()
queueinfo = queueresp['queue']
logger.fdebug('status: %s' % queueinfo['status'])
logger.fdebug('mbleft: %s' % queueinfo['mbleft'])
logger.fdebug('timeleft: %s' % queueinfo['timeleft'])
logger.fdebug('eta: %s' % queueinfo['eta'])
time.sleep(5)
except Exception as e:
logger.warn('error: %s' % e)
logger.info('File has now downloaded!')
return self.historycheck(self.params)
def the_sequence(self):
if self.sab is True:
self.completedir()
self.cats()
cat_dir = self.cat['dir']
cat_name = self.cat['name']
if cat_dir is None:
logger.fdebug(
'[CDH MAPPING] No category defined - using %s as the base download folder with no job folder creation'
% self.cdir)
self.basedir = self.cdir
else:
if cat_dir.endswith('*'):
logger.fdebug(
'[CDH MAPPING][%s] category defined - no job folder creation defined - using %s as base download folder'
% (cat_name, cat_dir))
self.basedir = cat_dir[:-1]
else:
logger.fdebug(
'[CDH MAPPING][%s] category defined - job folder creation defined - using %s as based download folder with sub folder creation'
% (cat_name, cat_dir))
self.basedir = cat_dir
self.subdir = True
else:
#query nzbget for categories and if path is different
self.send_nzbget()
cat_dir = self.cat['dir']
cat_name = self.cat['name']
if cat_dir is None:
logger.fdebug(
'[CDH MAPPING] No category defined - using %s as the base download folder with no job folder creation'
% self.cdir)
self.basedir = self.cdir
else:
if self.subdir is False:
logger.fdebug(
'[CDH MAPPING][%s] category defined - no job folder creation defined - using %s as base download folder'
% (cat_name, self.cdir))
self.basedir = self.cdir
else:
logger.fdebug(
'[CDH MAPPING][%s] category defined - job folder creation defined - using %s as based download folder with sub folder creation'
% (cat_name, cat_dir))
self.basedir = cat_dir
logger.fdebug('[CDH MAPPING] Base directory for downloads set to: %s' %
(self.basedir))
logger.fdebug('[CDH MAPPING] Downloaded file @: %s' % self.storage)
logger.fdebug('[CDH MAPPING] Destination root directory @: %s' %
(self.sab_dir))
if self.subdir is False:
maindir = self.storage.parents[0]
file = self.storage.name
else:
maindir = self.storage.parents[1]
file = self.storage.relative_to(maindir)
final_dst = self.sab_dir.joinpath(file)
return final_dst
def nzbdbsearch(seriesname,
issue,
comicid=None,
nzbprov=None,
searchYear=None,
ComicVersion=None):
myDB = db.DBConnection()
seriesname_alt = None
if comicid is None or comicid == 'None':
pass
else:
snm = myDB.selectone("SELECT * FROM comics WHERE comicid=?",
[comicid]).fetchone()
if snm is None:
logger.info('Invalid ComicID of ' + str(comicid) +
'. Aborting search.')
return
else:
seriesname = snm['ComicName']
seriesname_alt = snm['AlternateSearch']
nsearch_seriesname = re.sub('[\'\!\@\#\$\%\:\;\/\\=\?\.\-\s]', '%',
seriesname)
formatrem_seriesname = re.sub('[\'\!\@\#\$\%\:\;\/\\=\?\.]', '',
seriesname)
nsearch = '%' + nsearch_seriesname + "%"
nresults = myDB.select("SELECT * FROM rssdb WHERE Title like ? AND Site=?",
[nsearch, nzbprov])
if nresults is None:
logger.fdebug('nzb search returned no results for ' + seriesname)
if seriesname_alt is None:
logger.fdebug('no nzb Alternate name given. Aborting search.')
return "no results"
else:
chkthealt = seriesname_alt.split('##')
if chkthealt == 0:
AS_Alternate = AlternateSearch
for calt in chkthealt:
AS_Alternate = re.sub('##', '', calt)
nresults += myDB.select(
"SELECT * FROM rssdb WHERE Title like ? AND Site=?",
[AS_Alternate, nzbprov])
if nresults is None:
logger.fdebug('nzb alternate name search returned no results.')
return "no results"
nzbtheinfo = []
nzbinfo = {}
if nzbprov == 'experimental':
except_list = [
'releases', 'gold line', 'distribution', '0-day', '0 day'
]
if ComicVersion:
ComVersChk = re.sub("[^0-9]", "", ComicVersion)
if ComVersChk == '':
ComVersChk = 0
else:
ComVersChk = 0
else:
ComVersChk = 0
for results in nresults:
title = results['Title']
#logger.fdebug("titlesplit: " + str(title.split("\"")))
splitTitle = title.split("\"")
noYear = 'False'
for subs in splitTitle:
#logger.fdebug(subs)
if len(subs) > 10 and not any(d in subs.lower()
for d in except_list):
if ComVersChk == 0:
noYear = 'False'
if ComVersChk != 0 and searchYear not in subs:
noYear = 'True'
noYearline = subs
if searchYear in subs and noYear == 'True':
#this would occur on the next check in the line, if year exists and
#the noYear check in the first check came back valid append it
subs = noYearline + ' (' + searchYear + ')'
noYear = 'False'
if noYear == 'False':
nzbtheinfo.append({
'title':
subs,
'link':
re.sub('\/release\/', '/download/',
results['Link']),
'pubdate':
str(results['PubDate']),
'site':
str(results['Site']),
'length':
str(results['Size'])
})
else:
for nzb in nresults:
# no need to parse here, just compile and throw it back ....
nzbtheinfo.append({
'title': nzb['Title'],
'link': nzb['Link'],
'pubdate': nzb['Pubdate'],
'site': nzb['Site'],
'length': nzb['Size']
})
#logger.fdebug("entered info for " + nzb['Title'])
nzbinfo['entries'] = nzbtheinfo
return nzbinfo
def nzbs(provider=None):
nzbprovider = []
nzbp = 0
if mylar.NZBSU == 1:
nzbprovider.append('nzb.su')
nzbp += 1
if mylar.DOGNZB == 1:
nzbprovider.append('dognzb')
nzbp += 1
# --------
# Xperimental
if mylar.EXPERIMENTAL == 1:
nzbprovider.append('experimental')
nzbp += 1
newznabs = 0
newznab_hosts = []
if mylar.NEWZNAB == 1:
for newznab_host in mylar.EXTRA_NEWZNABS:
if newznab_host[4] == '1' or newznab_host[4] == 1:
newznab_hosts.append(newznab_host)
nzbprovider.append('newznab')
newznabs += 1
logger.fdebug('newznab name:' + str(newznab_host[0]) +
' - enabled: ' + str(newznab_host[4]))
# --------
providercount = int(nzbp + newznabs)
logger.fdebug('there are : ' + str(providercount) +
' nzb RSS search providers you have enabled.')
nzbpr = providercount - 1
if nzbpr < 0:
nzbpr == 0
feeddata = []
feedthis = []
ft = 0
totNum = 0
nonexp = "no"
while (nzbpr >= 0):
if nzbprovider[nzbpr] == 'experimental':
feed = feedparser.parse(
"http://nzbindex.nl/rss/alt.binaries.comics.dcp/?sort=agedesc&max=50&more=1"
)
totNum = len(feed.entries)
site = 'experimental'
keyPair = {}
regList = []
entries = []
mres = {}
countUp = 0
i = 0
for entry in feed['entries']:
tmpsz = feed.entries[i].enclosures[0]
feeddata.append({
'Site': site,
'Title': feed.entries[i].title,
'Link': tmpsz['url'], #feed.entries[i].link,
'Pubdate': feed.entries[i].updated,
'Size': tmpsz['length']
})
# print ("Site:" + str(site))
# print ("Title:" + str(feed.entries[i].title))
# print ("Link:" + str(feed.entries[i].link))
# print ("Pubdate:" + str(feed.entries[i].updated))
# print ("Size:" + str(tmpsz['length']))
i += 1
logger.info(str(i) + ' results from Experimental feed indexed.')
nzbpr -= 1
else:
if nzbprovider[nzbpr] == 'newznab':
for newznab_host in newznab_hosts:
if newznab_host[3] is None:
newznabuid = '1'
newznabcat = '7030'
else:
if '#' not in newznab_host[3]:
newznabuid = newznab_host[3]
newznabcat = '7030'
else:
newzst = newznab_host[3].find('#')
newznabuid = newznab_host[3][:newzst]
newznabcat = newznab_host[3][newzst + 1:]
feed = newznab_host[1].rstrip() + '/rss?t=' + str(
newznabcat) + '&dl=1&i=' + str(
newznabuid) + '&r=' + newznab_host[2].rstrip()
feedme = feedparser.parse(feed)
site = newznab_host[0].rstrip()
feedthis.append({"feed": feedme, "site": site})
totNum += len(feedme.entries)
ft += 1
nonexp = "yes"
nzbpr -= 1
elif nzbprovider[nzbpr] == 'nzb.su':
if mylar.NZBSU_UID is None:
mylar.NZBSU_UID = '1'
feed = 'http://api.nzb.su/rss?t=7030&dl=1&i=' + mylar.NZBSU_UID + '&r=' + mylar.NZBSU_APIKEY
feedme = feedparser.parse(feed)
site = nzbprovider[nzbpr]
feedthis.append({"feed": feedme, "site": site})
totNum += len(feedme.entries)
ft += 1
nonexp = "yes"
nzbpr -= 1
elif nzbprovider[nzbpr] == 'dognzb':
if mylar.DOGNZB_UID is None:
mylar.DOGNZB_UID = '1'
feed = 'https://dognzb.cr/rss.cfm?r=' + mylar.DOGNZB_APIKEY + '&t=7030'
feedme = feedparser.parse(feed)
site = nzbprovider[nzbpr]
ft += 1
nonexp = "yes"
feedthis.append({"feed": feedme, "site": site})
totNum += len(feedme.entries)
nzbpr -= 1
i = 0
if nonexp == "yes":
#print str(ft) + " sites checked. There are " + str(totNum) + " entries to be updated."
#print feedme
for ft in feedthis:
sitei = 0
site = ft['site']
logger.fdebug(str(site) + " now being updated...")
#logger.fdebug('feedthis:' + str(ft))
for entry in ft['feed'].entries:
if site == 'dognzb':
#because the rss of dog doesn't carry the enclosure item, we'll use the newznab size value
tmpsz = 0
#for attr in entry['newznab:attrib']:
# if attr('@name') == 'size':
# tmpsz = attr['@value']
# logger.fdebug('size retrieved as ' + str(tmpsz))
# break
feeddata.append({
'Site': site,
'Title': entry.title, #ft['feed'].entries[i].title,
'Link': entry.link, #ft['feed'].entries[i].link,
'Pubdate':
entry.updated, #ft['feed'].entries[i].updated,
'Size': tmpsz
})
else:
#this should work for all newznabs (nzb.su included)
#only difference is the size of the file between this and above (which is probably the same)
tmpsz = entry.enclosures[
0] #ft['feed'].entries[i].enclosures[0]
feeddata.append({
'Site': site,
'Title': entry.title, #ft['feed'].entries[i].title,
'Link': entry.link, #ft['feed'].entries[i].link,
'Pubdate':
entry.updated, #ft['feed'].entries[i].updated,
'Size': tmpsz['length']
})
#logger.fdebug("Site: " + str(feeddata[i]['Site']))
#logger.fdebug("Title: " + str(feeddata[i]['Title']))
#logger.fdebug("Link: " + str(feeddata[i]['Link']))
#logger.fdebug("pubdate: " + str(feeddata[i]['Pubdate']))
#logger.fdebug("size: " + str(feeddata[i]['Size']))
sitei += 1
logger.info('[' + str(site) + '] ' + str(sitei) +
' entries indexed.')
i += sitei
if i > 0:
logger.info(
'[RSS] ' + str(i) +
' entries have been indexed and are now going to be stored for caching.'
)
rssdbupdate(feeddata, i, 'usenet')
return
