def notify(self, event, message=None, snatched_nzb=None, prov=None, sent_to=None, module=None): if not mylar.PUSHOVER_ENABLED: return if module is None: module = '' module += '[NOTIFIER]' if snatched_nzb: if snatched_nzb[-1] == '\.': snatched_nzb = snatched_nzb[:-1] message = "Mylar has snatched: " + snatched_nzb + " from " + prov + " and has sent it to " + sent_to data = {'token': mylar.PUSHOVER_APIKEY, 'user': mylar.PUSHOVER_USERKEY, 'message': message.encode("utf-8"), 'title': event, 'priority': mylar.PUSHOVER_PRIORITY} r = self._session.post(self.PUSHOVER_URL, data=data, verify=True) if r.status_code == 200: logger.info(module + ' PushOver notifications sent.') return True elif r.status_code >= 400 and r.status_code < 500: logger.error(module + ' PushOver request failed: %s' % r.content) return False else: logger.error(module + ' PushOver notification failed serverside.') return False
def sendfiles(filelist): try: import paramiko except ImportError: logger.fdebug('paramiko not found on system. Please install manually in order to use seedbox option') logger.fdebug('get it at https://github.com/paramiko/paramiko') logger.fdebug('to install: python setup.py install') logger.fdebug('aborting send.') return fhost = mylar.CONFIG.TAB_HOST.find(':') host = mylar.CONFIG.TAB_HOST[:fhost] port = int(mylar.CONFIG.TAB_HOST[fhost +1:]) logger.fdebug('Destination: ' + host) logger.fdebug('Using SSH port : ' + str(port)) transport = paramiko.Transport((host, port)) password = mylar.CONFIG.TAB_PASS username = mylar.CONFIG.TAB_USER transport.connect(username = username, password = password) sftp = paramiko.SFTPClient.from_transport(transport) remotepath = mylar.CONFIG.TAB_DIRECTORY logger.fdebug('remote path set to ' + remotepath) if len(filelist) > 0: logger.info('Initiating send for ' + str(len(filelist)) + ' files...') return sendtohome(sftp, remotepath, filelist, transport)
def shutdown(restart=False, update=False): cherrypy.engine.exit() halt() if not restart and not update: logger.info('Mylar is shutting down...') if update: logger.info('Mylar is updating...') try: versioncheck.update() except Exception as e: logger.warn('Mylar failed to update: %s. Restarting.' % e) if CREATEPID: logger.info('Removing pidfile %s' % PIDFILE) os.remove(PIDFILE) if restart: logger.info('Mylar is restarting...') popen_list = [sys.executable, FULL_PATH] popen_list += ARGS # if '--nolaunch' not in popen_list: # popen_list += ['--nolaunch'] logger.info('Restarting Mylar with ' + str(popen_list)) subprocess.Popen(popen_list, cwd=os.getcwd()) os._exit(0)
def torsend2client(seriesname, linkit, site): logger.info('matched on ' + str(seriesname)) filename = re.sub('[\'\!\@\#\$\%\:\;\/\\=\?\.]', '',seriesname) if site == 'ComicBT': logger.info(linkit) linkit = str(linkit) + '&passkey=' + str(mylar.CBT_PASSKEY) if linkit[-7:] != "torrent": filename += ".torrent" request = urllib2.Request(linkit) request.add_header('User-Agent', str(mylar.USER_AGENT)) if mylar.TORRENT_LOCAL and mylar.LOCAL_WATCHDIR is not None: filepath = os.path.join(mylar.LOCAL_WATCHDIR, filename) logger.fdebug('filename for torrent set to : ' + filepath) elif mylar.TORRENT_SEEDBOX and mylar.SEEDBOX_WATCHDIR is not None: filepath = os.path.join(mylar.CACHE_DIR, filename) logger.fdebug('filename for torrent set to : ' + filepath) else: logger.error('No Local Watch Directory or Seedbox Watch Directory specified. Set it and try again.') return "fail" try: opener = helpers.urlretrieve(urllib2.urlopen(request), filepath) except Exception, e: logger.warn('Error fetching data from %s: %s' % (site, e)) return "fail"
def notify(self, ComicName=None, Year=None, Issue=None, sent_to=None, snatched_nzb=None, username=None, force=False): """ Sends a boxcar notification based on the provided info or SB config title: The title of the notification to send message: The message string to send username: The username to send the notification to (optional, defaults to the username in the config) force: If True then the notification will be sent even if Boxcar is disabled in the config """ if not mylar.BOXCAR_ENABLED and not force: logger.fdebug("Notification for Boxcar not enabled, skipping this notification") return False # if no username was given then use the one from the config if not username: username = mylar.BOXCAR_USERNAME if snatched_nzb: title = "Mylar. Sucessfully Snatched!" message = "Mylar has snatched: " + snatched_nzb + " and has sent it to " + sent_to else: title = "Mylar. Successfully Downloaded & Post-Processed!" message = "Mylar has downloaded and postprocessed: " + ComicName + ' (' + Year + ') #' + Issue logger.info("Sending notification to Boxcar") self._sendBoxcar(message, title, username) return True
def addfile(self, filepath=None, filename=None, bytes=None): params = {'action': 'add-file', 'token': self.token} try: d = open(filepath, 'rb') tordata = d.read() d.close() except: logger.warn('Unable to load torrent file. Aborting at this time.') return 'fail' files = {'torrent_file': tordata} try: r = requests.post(url=self.utorrent_url, auth=self.auth, cookies=self.cookies, params=params, files=files) except requests.exceptions.RequestException as err: logger.debug('URL: ' + str(self.utorrent_url)) logger.debug('Error sending to uTorrent Client. uTorrent responded with error: ' + str(err)) return 'fail' # (to-do) verify the hash in order to ensure it's loaded here if str(r.status_code) == '200': logger.info('Successfully added torrent to uTorrent client.') hash = self.calculate_torrent_hash(data=tordata) if mylar.UTORRENT_LABEL: try: self.setlabel(hash) except: logger.warn('Unable to set label for torrent.') return hash else: return 'fail'
def notify(self, ComicName=None, Year=None, Issue=None, sent_to=None, snatched_nzb=None, force=False, module=None): """ Sends a boxcar notification based on the provided info or SB config title: The title of the notification to send message: The message string to send force: If True then the notification will be sent even if Boxcar is disabled in the config """ if module is None: module = '' module += '[NOTIFIER]' if not mylar.BOXCAR_ENABLED and not force: logger.fdebug(module + ' Notification for Boxcar not enabled, skipping this notification.') return False # if no username was given then use the one from the config if snatched_nzb: title = "Mylar. Sucessfully Snatched!" message = "Mylar has snatched: " + snatched_nzb + " and has sent it to " + sent_to else: title = "Mylar. Successfully Downloaded & Post-Processed!" message = "Mylar has downloaded and postprocessed: " + ComicName + ' (' + Year + ') #' + Issue logger.info(module + ' Sending notification to Boxcar2') self._sendBoxcar(message, title, module) return True
def markasRead(self): myDB = db.DBConnection() if self.IssueID: issue = myDB.selectone('SELECT * from readlist WHERE IssueID=?', [self.IssueID]).fetchone() if issue['Status'] == 'Read': NewVal = {"Status": "Added"} else: NewVal = {"Status": "Read"} NewVal['StatusChange'] = helpers.today() CtrlVal = {"IssueID": self.IssueID} myDB.upsert("readlist", NewVal, CtrlVal) logger.info(self.module + ' Marked ' + issue['ComicName'] + ' #' + str(issue['Issue_Number']) + ' as Read.') elif self.IssueArcID: issue = myDB.selectone('SELECT * from readinglist WHERE IssueArcID=?', [self.IssueArcID]).fetchone() if issue['Status'] == 'Read': NewVal = {"Status": "Added"} else: NewVal = {"Status": "Read"} NewVal['StatusChange'] = helpers.today() CtrlVal = {"IssueArcID": self.IssueArcID} myDB.upsert("readinglist", NewVal, CtrlVal) logger.info(self.module + ' Marked ' + issue['ComicName'] + ' #' + str(issue['IssueNumber']) + ' as Read.') return
def downloadfile(self, payload, filepath): url = 'https://32pag.es/torrents.php' try: r = self.session.get(url, params=payload, verify=True, stream=True, allow_redirects=True) except Exception as e: logger.error('%s [%s] Could not POST URL %s' % ('[32P-DOWNLOADER]', e, url)) return False if str(r.status_code) != '200': logger.warn('Unable to download torrent from 32P [Status Code returned: %s]' % r.status_code) if str(r.status_code) == '404' and site == '32P': logger.warn('[32P-CACHED_ENTRY] Entry found in 32P cache - incorrect. Torrent has probably been merged into a pack, or another series id. Removing from cache.') helpers.delete_cache_entry(linkit) else: logger.info('content: %s' % r.content) return False with open(filepath, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) f.flush() return True
def notify(self, text, attachment_text, snatched_nzb=None, prov=None, sent_to=None, module=None): if module is None: module = '' module += '[NOTIFIER]' if all([sent_to is not None, prov is not None]): attachment_text += ' from %s and %s' % (prov, sent_to) elif sent_to is None: attachment_text += ' from %s' % prov else: pass payload = { # "text": text, # "attachments": [ # { # "color": "#36a64f", # "text": attachment_text # } # ] # FIX: #1861 move notif from attachment to msg body - bbq "text": attachment_text } try: response = requests.post(self.webhook_url, json=payload, verify=True) except Exception, e: logger.info(module + u'Slack notify failed: ' + str(e))
def comic_config(self, com_location, ComicID, alt_search=None, fuzzy_year=None): myDB = db.DBConnection() #--- this is for multipe search terms............ #--- works, just need to redo search.py to accomodate multiple search terms # ffs_alt = [] # if '+' in alt_search: #find first + # ffs = alt_search.find('+') # ffs_alt.append(alt_search[:ffs]) # ffs_alt_st = str(ffs_alt[0]) # print("ffs_alt: " + str(ffs_alt[0])) # split the entire string by the delimter + # ffs_test = alt_search.split('+') # if len(ffs_test) > 0: # print("ffs_test names: " + str(len(ffs_test))) # ffs_count = len(ffs_test) # n=1 # while (n < ffs_count): # ffs_alt.append(ffs_test[n]) # print("adding : " + str(ffs_test[n])) #print("ffs_alt : " + str(ffs_alt)) # ffs_alt_st = str(ffs_alt_st) + "..." + str(ffs_test[n]) # n+=1 # asearch = ffs_alt # else: # asearch = alt_search asearch = str(alt_search) controlValueDict = {'ComicID': ComicID} newValues = {"ComicLocation": com_location } #"QUALalt_vers": qual_altvers, #"QUALScanner": qual_scanner, #"QUALtype": qual_type, #"QUALquality": qual_quality #} if asearch is not None: if asearch == '': newValues['AlternateSearch'] = "None" else: newValues['AlternateSearch'] = str(asearch) if fuzzy_year is None: newValues['UseFuzzy'] = "0" else: newValues['UseFuzzy'] = str(fuzzy_year) #force the check/creation of directory com_location here if os.path.isdir(str(com_location)): logger.info(u"Validating Directory (" + str(com_location) + "). Already exists! Continuing...") else: logger.fdebug("Updated Directory doesn't exist! - attempting to create now.") try: os.makedirs(str(com_location)) logger.info(u"Directory successfully created at: " + str(com_location)) except OSError: logger.error(u"Could not create comicdir : " + str(com_location)) myDB.upsert("comics", newValues, controlValueDict) raise cherrypy.HTTPRedirect("artistPage?ComicID=%s" % ComicID)
def deleteArtist(self, ComicID): myDB = db.DBConnection() comic = myDB.action('SELECT * from comics WHERE ComicID=?', [ComicID]).fetchone() logger.info(u"Deleting all traces of Comic: " + comic['ComicName']) myDB.action('DELETE from comics WHERE ComicID=?', [ComicID]) myDB.action('DELETE from issues WHERE ComicID=?', [ComicID]) raise cherrypy.HTTPRedirect("home")
def versionload(): mylar.CURRENT_VERSION, mylar.CONFIG.GIT_BRANCH = getVersion() if mylar.CURRENT_VERSION is not None: hash = mylar.CURRENT_VERSION[:7] else: hash = "unknown" if mylar.CONFIG.GIT_BRANCH == 'master': vers = 'M' elif mylar.CONFIG.GIT_BRANCH == 'development': vers = 'D' else: vers = 'NONE' mylar.USER_AGENT = 'Mylar/' +str(hash) +'(' +vers +') +http://www.github.com/evilhero/mylar/' logger.info('Version information: %s [%s]' % (mylar.CONFIG.GIT_BRANCH, mylar.CURRENT_VERSION)) if mylar.CONFIG.CHECK_GITHUB_ON_STARTUP: try: mylar.LATEST_VERSION = checkGithub() #(CURRENT_VERSION) except: mylar.LATEST_VERSION = mylar.CURRENT_VERSION else: mylar.LATEST_VERSION = mylar.CURRENT_VERSION if mylar.CONFIG.AUTO_UPDATE: if mylar.CURRENT_VERSION != mylar.LATEST_VERSION and mylar.INSTALL_TYPE != 'win' and mylar.COMMITS_BEHIND > 0: logger.info('Auto-updating has been enabled. Attempting to auto-update.') mylar.SIGNAL = 'update'
def sendtohome(sftp, remotepath, filelist, transport): fhost = mylar.CONFIG.TAB_HOST.find(':') host = mylar.CONFIG.TAB_HOST[:fhost] port = int(mylar.CONFIG.TAB_HOST[fhost +1:]) successlist = [] filestotal = len(filelist) for files in filelist: tempfile = files['filename'] issid = files['issueid'] logger.fdebug('Checking filename for problematic characters: ' + tempfile) #we need to make the required directory(ies)/subdirectories before the get will work. if u'\xb4' in files['filename']: # right quotation logger.fdebug('detected abnormal character in filename') filename = tempfile.replace('0xb4', '\'') if u'\xbd' in files['filename']: # 1/2 character filename = tempfile.replace('0xbd', 'half') if u'\uff1a' in files['filename']: #some unknown character filename = tempfile.replace('\0ff1a', '-') #now we encode the structure to ascii so we can write directories/filenames without error. filename = tempfile.encode('ascii', 'ignore') remdir = remotepath if mylar.CONFIG.MAINTAINSERIESFOLDER == 1: # Get folder path of issue comicdir = os.path.split(files['filepath'])[0] # Isolate comic folder name comicdir = os.path.split(comicdir)[1] logger.info('Checking for Comic Folder: ' + comicdir) chkdir = os.path.join(remdir, comicdir) try: sftp.stat(chkdir) except IOError, e: logger.info('Comic Folder does not Exist, creating ' + chkdir ) try: sftp.mkdir(chkdir) except : # Fallback to default behavior logger.info('Could not create Comic Folder, adding to device root') else : remdir = chkdir else : remdir = chkdir localsend = files['filepath'] logger.info('Sending : ' + localsend) remotesend = os.path.join(remdir, filename) logger.info('To : ' + remotesend) try: sftp.stat(remotesend) except IOError, e: if e[0] == 2: filechk = False
def resumeArtist(self, ComicID): logger.info(u"Resuming comic: " + ComicID) myDB = db.DBConnection() controlValueDict = {'ComicID': ComicID} newValueDict = {'Status': 'Active'} myDB.upsert("comics", newValueDict, controlValueDict) raise cherrypy.HTTPRedirect("artistPage?ComicID=%s" % ComicID)
def pullsearch(comicapi,comicquery,offset,explicit,type): u_comicquery = urllib.quote(comicquery.encode('utf-8').strip()) u_comicquery = u_comicquery.replace(" ", "%20") if explicit == 'all' or explicit == 'loose': PULLURL = mylar.CVURL + 'search?api_key=' + str(comicapi) + '&resources=' + str(type) + '&query=' + u_comicquery + '&field_list=id,name,start_year,site_detail_url,count_of_issues,image,publisher,description&format=xml&page=' + str(offset) else: # 02/22/2014 use the volume filter label to get the right results. # add the 's' to the end of type to pluralize the caption (it's needed) if type == 'story_arc': logger.info('redefining.') u_comicquery = re.sub("%20AND%20", "%20", u_comicquery) PULLURL = mylar.CVURL + str(type) + 's?api_key=' + str(comicapi) + '&filter=name:' + u_comicquery + '&field_list=id,name,start_year,site_detail_url,count_of_issues,image,publisher,description&format=xml&offset=' + str(offset) # 2012/22/02 - CVAPI flipped back to offset instead of page #all these imports are standard on most modern python implementations #CV API Check here. if mylar.CVAPI_COUNT == 0 or mylar.CVAPI_COUNT >= mylar.CVAPI_MAX: cvapi_check() #download the file: try: file = urllib2.urlopen(PULLURL) except urllib2.HTTPError, err: logger.error('err : ' + str(err)) logger.error("There was a major problem retrieving data from ComicVine - on their end. You'll have to try again later most likely.") return
def __init__(self, reauthenticate=False, searchterm=None, test=False): self.module = '[32P-AUTHENTICATION]' self.url = 'https://32pag.es/user.php?action=notify' self.headers = {'Content-type': 'application/x-www-form-urlencoded', 'Accept-Charset': 'utf-8', 'User-Agent': 'Mozilla/5.0'} self.error = None self.method = None lses = self.LoginSession(mylar.USERNAME_32P, mylar.PASSWORD_32P) if not lses.login(): if not self.test: logger.error(self.module + ' [LOGIN FAILED] Disabling 32P provider until login error(s) can be fixed in order to avoid temporary bans.') return "disable" else: if self.error: return self.error #rtnmsg else: return self.method else: logger.info(self.module + '[LOGIN SUCCESS] Now preparing for the use of 32P keyed authentication...') self.authkey = lses.authkey self.passkey = lses.passkey self.uid = lses.uid self.reauthenticate = reauthenticate self.searchterm = searchterm self.test = test self.publisher_list = {'Entertainment', 'Press', 'Comics', 'Publishing', 'Comix', 'Studios!'}
def runAction(self): while True: currentTime = datetime.datetime.now() if currentTime - self.lastRun > self.cycleTime: self.lastRun = currentTime try: if not self.silent: logger.fdebug("Starting new thread: " + self.threadName) if self.delay: logger.info('delaying startup thread for ' + str(self.delay) + ' seconds to avoid locks.') time.sleep(self.delay) self.action.run() except Exception, e: logger.fdebug("Exception generated in thread " + self.threadName + ": %s" % e) logger.fdebug(repr(traceback.format_exc())) if self.abort: self.abort = False self.thread = None return time.sleep(1)
def notify(self, ComicName=None, Year=None, Issue=None, snatched_nzb=None, sent_to=None): apikey = self.apikey priority = self.priority if snatched_nzb: event = snatched_nzb + " snatched!" description = "Mylar has snatched: " + snatched_nzb + " and has sent it to " + sent_to else: event = ComicName + " (" + Year + ") - Issue #" + Issue + " complete!" description = "Mylar has downloaded and postprocessed: " + ComicName + " (" + Year + ") #" + Issue data = { "apikey": apikey, "application": "Mylar", "event": event, "description": description, "priority": priority, } logger.info("Sending notification request to NotifyMyAndroid") request = self._send(data) if not request: logger.warn("Error sending notification request to NotifyMyAndroid")
def movefiles(comicid, comlocation, imported): #comlocation is destination #comicid is used for rename files_moved = [] try: imported = ast.literal_eval(imported) except ValueError: pass myDB = db.DBConnection() logger.fdebug('comlocation is : ' + comlocation) logger.fdebug('original comicname is : ' + imported['ComicName']) impres = imported['filelisting'] if impres is not None: for impr in impres: srcimp = impr['comiclocation'] orig_filename = impr['comicfilename'] #before moving check to see if Rename to Mylar structure is enabled. if mylar.IMP_RENAME and mylar.FILE_FORMAT != '': logger.fdebug("Renaming files according to configuration details : " + str(mylar.FILE_FORMAT)) renameit = helpers.rename_param(comicid, imported['ComicName'], impr['issuenumber'], orig_filename) nfilename = renameit['nfilename'] dstimp = os.path.join(comlocation, nfilename) else: logger.fdebug("Renaming files not enabled, keeping original filename(s)") dstimp = os.path.join(comlocation, orig_filename) logger.info("moving " + srcimp + " ... to " + dstimp) try: shutil.move(srcimp, dstimp) files_moved.append({'srid': imported['srid'], 'filename': impr['comicfilename']}) except (OSError, IOError): logger.error("Failed to move files - check directories and manually re-run.") logger.fdebug("all files moved.") #now that it's moved / renamed ... we remove it from importResults or mark as completed. if len(files_moved) > 0: logger.info('files_moved: ' + str(files_moved)) for result in files_moved: try: res = result['import_id'] except: #if it's an 'older' import that wasn't imported, just make it a basic match so things can move and update properly. controlValue = {"ComicFilename": result['filename'], "SRID": result['srid']} newValue = {"Status": "Imported", "ComicID": comicid} else: controlValue = {"impID": result['import_id'], "ComicFilename": result['filename']} newValue = {"Status": "Imported", "SRID": result['srid'], "ComicID": comicid} myDB.upsert("importresults", newValue, controlValue) return
def notify(self, prline=None, prline2=None, sent_to=None, snatched_nzb=None, force=False, module=None, snline=None): """ Sends a boxcar notification based on the provided info or SB config title: The title of the notification to send message: The message string to send force: If True then the notification will be sent even if Boxcar is disabled in the config """ if module is None: module = '' module += '[NOTIFIER]' if not mylar.BOXCAR_ENABLED and not force: logger.fdebug(module + ' Notification for Boxcar not enabled, skipping this notification.') return False # if no username was given then use the one from the config if snatched_nzb: title = snline message = "Mylar has snatched: " + snatched_nzb + " and has sent it to " + sent_to else: title = prline message = prline2 logger.info(module + ' Sending notification to Boxcar2') self._sendBoxcar(message, title, module) return True
def run(self): logger.info('[VersionCheck] Checking for new release on Github.') helpers.job_management(write=True, job='Check Version', current_run=helpers.utctimestamp(), status='Running') mylar.VERSION_STATUS = 'Running' versioncheck.checkGithub() helpers.job_management(write=True, job='Check Version', last_run_completed=helpers.utctimestamp(), status='Waiting') mylar.VERSION_STATUS = 'Waiting'
def run(self): import PostProcessor, logger #monitor a selected folder for 'snatched' files that haven't been processed logger.info('Checking folder ' + mylar.CHECK_FOLDER + ' for newly snatched downloads') PostProcess = PostProcessor.PostProcessor('Manual Run', mylar.CHECK_FOLDER) result = PostProcess.Process() logger.info('Finished checking for newly snatched downloads')
def halt(): global _INITIALIZED, started with INIT_LOCK: if _INITIALIZED: logger.info('Shutting down the background schedulers...') SCHED.shutdown(wait=False) if NZBPOOL is not None: logger.info('Terminating the nzb auto-complete thread.') try: NZBPOOL.join(10) logger.info('Joined pool for termination - successful') except KeyboardInterrupt: NZB_QUEUE.put('exit') NZBPOOL.join(5) except AssertionError: os._exit(0) if SNPOOL is not None: logger.info('Terminating the auto-snatch thread.') try: SNPOOL.join(10) logger.info('Joined pool for termination - successful') except KeyboardInterrupt: SNATCHED_QUEUE.put('exit') SNPOOL.join(5) except AssertionError: os._exit(0) _INITIALIZED = False
def start(): global __INITIALIZED__, started if __INITIALIZED__: # Start our scheduled background tasks #from mylar import updater, searcher, librarysync, postprocessor from mylar import updater, search, weeklypull SCHED.add_interval_job(updater.dbUpdate, hours=48) SCHED.add_interval_job(search.searchforissue, minutes=SEARCH_INTERVAL) #SCHED.add_interval_job(librarysync.libraryScan, minutes=LIBRARYSCAN_INTERVAL) #weekly pull list gets messed up if it's not populated first, so let's populate it then set the scheduler. logger.info("Checking for existance of Weekly Comic listing...") PULLNEW = 'no' #reset the indicator here. threading.Thread(target=weeklypull.pullit).start() #now the scheduler (check every 24 hours) SCHED.add_interval_job(weeklypull.pullit, hours=24) #let's do a run at the Wanted issues here (on startup) if enabled. if NZB_STARTUP_SEARCH: threading.Thread(target=search.searchforissue).start() if CHECK_GITHUB: SCHED.add_interval_job(versioncheck.checkGithub, minutes=CHECK_GITHUB_INTERVAL) #SCHED.add_interval_job(postprocessor.checkFolder, minutes=DOWNLOAD_SCAN_INTERVAL) SCHED.start() started = True
def get_the_hash(self, filepath): # Open torrent file torrent_file = open(filepath, "rb") metainfo = bencode.decode(torrent_file.read()) info = metainfo['info'] thehash = hashlib.sha1(bencode.encode(info)).hexdigest().upper() logger.info('Hash: ' + thehash) return thehash
def unqueueissue(self, IssueID, ComicID): myDB = db.DBConnection() issue = myDB.action('SELECT * FROM issues WHERE IssueID=?', [IssueID]).fetchone() logger.info(u"Marking " + issue['ComicName'] + " issue # " + issue['Issue_Number'] + " as skipped...") controlValueDict = {'IssueID': IssueID} newValueDict = {'Status': 'Skipped'} myDB.upsert("issues", newValueDict, controlValueDict) raise cherrypy.HTTPRedirect("artistPage?ComicID=%s" % ComicID)
def run(self): logger.info('[SEARCH] Running Search for Wanted.') helpers.job_management(write=True, job='Auto-Search', current_run=helpers.utctimestamp(), status='Running') mylar.SEARCH_STATUS = 'Running' mylar.search.searchforissue() helpers.job_management(write=True, job='Auto-Search', last_run_completed=helpers.utctimestamp(), status='Waiting') mylar.SEARCH_STATUS = 'Waiting'
def run(self): logger.info('[WEEKLY] Checking Weekly Pull-list for new releases/updates') helpers.job_management(write=True, job='Weekly Pullist', current_run=helpers.utctimestamp(), status='Running') mylar.WEEKLY_STATUS = 'Running' weeklypull.pullit() weeklypull.future_check() helpers.job_management(write=True, job='Weekly Pullist', last_run_completed=helpers.utctimestamp(), status='Waiting') mylar.WEEKLY_STATUS = 'Waiting'
def setlabel(self, hash): params = {'token': self.token, 'action': 'setprops', 'hash': hash, 's': 'label', 'v': str(mylar.UTORRENT_LABEL)} r = requests.post(url=self.utorrent_url, auth=self.auth, cookies=self.cookies, params=params) if str(r.status_code) == '200': logger.info('label ' + str(mylar.UTORRENT_LABEL) + ' successfully applied') else: logger.info('Unable to label torrent') return
def test_notify(self): module = '[TEST-NOTIFIER]' try: r = self._session.get(self.TEST_NMA_URL, params={'apikey': self.apikey}, verify=True) except requests.exceptions.RequestException as e: logger.error( module + '[' + str(e) + '] Unable to send via NMA. Aborting test notification - something is probably wrong...' ) return {'status': False, 'message': str(e)} logger.fdebug('[NMA] Status code returned: ' + str(r.status_code)) if r.status_code == 200: from xml.dom.minidom import parseString dom = parseString(r.content) try: success_info = dom.getElementsByTagName('success') success_code = success_info[0].getAttribute('code') except: error_info = dom.getElementsByTagName('error') error_code = error_info[0].getAttribute('code') error_message = error_info[0].childNodes[0].nodeValue logger.info(module + '[' + str(error_code) + '] ' + error_message) return { 'status': False, 'message': '[' + str(error_code) + '] ' + error_message } else: logger.info( module + '[' + str(success_code) + '] NotifyMyAndroid apikey valid. Testing notification service with it.' ) elif r.status_code >= 400 and r.status_code < 500: logger.error(module + ' NotifyMyAndroid request failed: %s' % r.content) return { 'status': False, 'message': 'Unable to send request to NMA - check your connection.' } else: logger.error(module + ' NotifyMyAndroid notification failed serverside.') return { 'status': False, 'message': 'Internal Server Error. Try again later.' } event = 'Test Message' description = 'ZOMG Lazors PewPewPew!' data = { 'apikey': self.apikey, 'application': 'Mylar', 'event': event.encode('utf-8'), 'description': description.encode('utf-8'), 'priority': 2 } return self._send(data, '[NOTIFIER]')
def update(): if mylar.INSTALL_TYPE == 'win': logger.info('Windows .exe updating not supported yet.') pass elif mylar.INSTALL_TYPE == 'git': output, err = runGit('pull origin ' + mylar.CONFIG.GIT_BRANCH) if output is None: logger.error('Couldn\'t download latest version') return for line in output.split('\n'): if 'Already up-to-date.' in line: logger.info('No update available, not updating') logger.info('Output: ' + str(output)) elif line.endswith('Aborting.'): logger.error('Unable to update from git: ' + line) logger.info('Output: ' + str(output)) else: tar_download_url = 'https://github.com/%s/mylar/tarball/%s' % ( mylar.CONFIG.GIT_USER, mylar.CONFIG.GIT_BRANCH) update_dir = os.path.join(mylar.PROG_DIR, 'update') version_path = os.path.join(mylar.PROG_DIR, 'version.txt') try: logger.info('Downloading update from: ' + tar_download_url) response = requests.get(tar_download_url, verify=True, stream=True) except (IOError, urllib2.URLError): logger.error("Unable to retrieve new version from " + tar_download_url + ", can't update") return #try sanitizing the name here... download_name = mylar.CONFIG.GIT_BRANCH + '-github' #data.geturl().split('/')[-1].split('?')[0] tar_download_path = os.path.join(mylar.PROG_DIR, download_name) # Save tar to disk with open(tar_download_path, 'wb') as f: for chunk in response.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) f.flush() # Extract the tar to update folder logger.info('Extracting file' + tar_download_path) tar = tarfile.open(tar_download_path) tar.extractall(update_dir) tar.close() # Delete the tar.gz logger.info('Deleting file' + tar_download_path) os.remove(tar_download_path) # Find update dir name update_dir_contents = [ x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x)) ] if len(update_dir_contents) != 1: logger.error(u"Invalid update data, update failed: " + str(update_dir_contents)) return content_dir = os.path.join(update_dir, update_dir_contents[0]) # walk temp folder and move files to main folder for dirname, dirnames, filenames in os.walk(content_dir): dirname = dirname[len(content_dir) + 1:] for curfile in filenames: old_path = os.path.join(content_dir, dirname, curfile) new_path = os.path.join(mylar.PROG_DIR, dirname, curfile) if os.path.isfile(new_path): os.remove(new_path) os.renames(old_path, new_path) # Update version.txt try: with open(version_path, 'w') as f: f.write(str(mylar.LATEST_VERSION)) except IOError as e: logger.error( "Unable to write current version to version.txt, update not complete: %s" % ex(e)) return
def historycheck(self, nzbinfo, roundtwo=False): sendresponse = nzbinfo['nzo_id'] hist_params = { 'mode': 'history', 'category': mylar.CONFIG.SAB_CATEGORY, 'failed': 0, 'output': 'json', 'apikey': mylar.CONFIG.SAB_APIKEY } sab_check = None if mylar.CONFIG.SAB_VERSION is None: try: sc = mylar.webserve.WebInterface() sab_check = sc.SABtest(sabhost=mylar.CONFIG.SAB_HOST, sabusername=mylar.CONFIG.SAB_USERNAME, sabpassword=mylar.CONFIG.SAB_PASSWORD, sabapikey=mylar.CONFIG.SAB_APIKEY) except Exception as e: logger.warn( '[SABNZBD-VERSION-TEST] Exception encountered trying to retrieve SABnzbd version: %s. Setting history length to last 200 items.' % e) hist_params['limit'] = 200 sab_check = 'some value' else: sab_check = None if sab_check is None: #set min_sab to 3.2.0 since 3.2.0 beta 1 has the api call for history search by nzo_id try: sab_minimum_version = '3.2.0' min_sab = re.sub('[^0-9]', '', sab_minimum_version) sab_vers = mylar.CONFIG.SAB_VERSION if 'beta' in sab_vers: sab_vers = re.sub('[^0-9]', '', sab_vers) if len(sab_vers) > 3: sab_vers = sab_vers[: -1] # remove beta value entirely... if parse_version(sab_vers) >= parse_version(min_sab): logger.fdebug( 'SABnzbd version is higher than 3.2.0. Querying history based on nzo_id directly.' ) hist_params['nzo_ids'] = sendresponse else: logger.fdebug( 'SABnzbd version is less than 3.2.0. Querying history based on history size of 200.' ) hist_params['limit'] = 200 except Exception as e: logger.warn( '[SABNZBD-VERSION-CHECK] Exception encountered trying to compare installed version [%s] to [%s]. Setting history length to last 200 items. (error: %s)' % (mylar.CONFIG.SAB_VERSION, sab_minimum_version, e)) hist_params['limit'] = 200 hist = requests.get(self.sab_url, params=hist_params, verify=False) historyresponse = hist.json() #logger.info(historyresponse) histqueue = historyresponse['history'] found = {'status': False} nzo_exists = False try: for hq in histqueue['slots']: logger.fdebug('nzo_id: %s --- %s [%s]' % (hq['nzo_id'], sendresponse, hq['status'])) if hq['nzo_id'] == sendresponse and any([ hq['status'] == 'Completed', hq['status'] == 'Running', 'comicrn' in hq['script'].lower() ]): nzo_exists = True logger.info( 'found matching completed item in history. Job has a status of %s' % hq['status']) if 'comicrn' in hq['script'].lower(): logger.warn( 'ComicRN has been detected as being active for this category & download. Completed Download Handling will NOT be performed due to this.' ) logger.warn( 'Either disable Completed Download Handling for SABnzbd within Mylar, or remove ComicRN from your category script in SABnzbd.' ) return {'status': 'double-pp', 'failed': False} if os.path.isfile(hq['storage']): logger.fdebug('location found @ %s' % hq['storage']) found = { 'status': True, 'name': ntpath.basename( hq['storage'] ), #os.pathre.sub('.nzb', '', hq['nzb_name']).strip(), 'location': os.path.abspath( os.path.join(hq['storage'], os.pardir)), 'failed': False, 'issueid': nzbinfo['issueid'], 'comicid': nzbinfo['comicid'], 'apicall': True, 'ddl': False } break else: logger.error( 'no file found where it should be @ %s - is there another script that moves things after completion ?' % hq['storage']) return {'status': 'file not found', 'failed': False} elif hq['nzo_id'] == sendresponse and hq['status'] == 'Failed': nzo_exists = True #get the stage / error message and see what we can do stage = hq['stage_log'] logger.fdebug('stage: %s' % (stage, )) for x in stage: if 'Failed' in x['actions'] and any( [x['name'] == 'Unpack', x['name'] == 'Repair']): if 'moving' in x['actions']: logger.warn( 'There was a failure in SABnzbd during the unpack/repair phase that caused a failure: %s' % x['actions']) else: logger.warn( 'Failure occured during the Unpack/Repair phase of SABnzbd. This is probably a bad file: %s' % x['actions']) if mylar.FAILED_DOWNLOAD_HANDLING is True: found = { 'status': True, 'name': re.sub('.nzb', '', hq['nzb_name']).strip(), 'location': os.path.abspath( os.path.join( hq['storage'], os.pardir)), 'failed': True, 'issueid': sendresponse['issueid'], 'comicid': sendresponse['comicid'], 'apicall': True, 'ddl': False } break if found['status'] is False: return {'status': 'failed_in_sab', 'failed': False} else: break elif hq['nzo_id'] == sendresponse: nzo_exists = True logger.fdebug( 'nzo_id: %s found while processing queue in an unhandled status: %s' % (hq['nzo_id'], hq['status'])) if hq['status'] == 'Queued' and roundtwo is False: time.sleep(4) return self.historycheck(nzbinfo, roundtwo=True) else: return { 'failed': False, 'status': 'unhandled status of: %s' % (hq['status']) } if not nzo_exists: logger.error( 'Cannot find nzb %s in the queue. Was it removed?' % sendresponse) time.sleep(5) if roundtwo is False: return self.historycheck(nzbinfo, roundtwo=True) else: return {'status': 'nzb removed', 'failed': False} except Exception as e: logger.warn('error %s' % e) return {'status': False, 'failed': False} return found
def load_torrent(self, filepath): if not filepath.startswith('magnet'): logger.info('filepath to torrent file set to : ' + filepath) if self.client._is_authenticated is True: logger.info('Checking if Torrent Exists!') if filepath.startswith('magnet'): torrent_hash = re.findall("urn:btih:([\w]{32,40})", filepath)[0] if len(torrent_hash) == 32: torrent_hash = b16encode(b32decode(torrent_hash)).lower() hash = torrent_hash.upper() logger.debug('Magnet (load_torrent) initiating') else: hash = self.get_the_hash(filepath) logger.debug('FileName (load_torrent): ' + str(os.path.basename(filepath))) logger.debug('Torrent Hash (load_torrent): "' + hash + '"') #Check if torrent already added if self.find_torrent(hash): logger.info('load_torrent: Torrent already exists!') return {'status': False} #should set something here to denote that it's already loaded, and then the failed download checker not run so it doesn't download #multiple copies of the same issues that's already downloaded else: logger.info('Torrent not added yet, trying to add it now!') if filepath.startswith('magnet'): try: tid = self.client.download_from_link(filepath, category=str(mylar.CONFIG.QBITTORRENT_LABEL)) except Exception as e: logger.debug('Torrent not added') return {'status': False} else: logger.debug('Successfully submitted for add as a magnet. Verifying item is now on client.') else: try: torrent_content = open(filepath, 'rb') tid = self.client.download_from_file(torrent_content, category=str(mylar.CONFIG.QBITTORRENT_LABEL)) except Exception as e: logger.debug('Torrent not added') return {'status': False} else: logger.debug('Successfully submitted for add via file. Verifying item is now on client.') if mylar.CONFIG.QBITTORRENT_STARTONLOAD: logger.info('attempting to start') startit = self.client.force_start(hash) logger.info('startit returned:' + str(startit)) else: logger.info('attempting to pause torrent incase it starts') try: startit = self.client.pause(hash) logger.info('startit paused:' + str(startit)) except: logger.warn('Unable to pause torrent - possibly already paused?') try: time.sleep(5) # wait 5 in case it's not populated yet. tinfo = self.get_torrent(hash) except Exception as e: logger.warn('Torrent was not added! Please check logs') return {'status': False} else: logger.info('Torrent successfully added!') filelist = self.client.get_torrent_files(hash) #logger.info(filelist) if len(filelist) == 1: to_name = filelist[0]['name'] else: to_name = tinfo['save_path'] torrent_info = {'hash': hash, 'files': filelist, 'name': to_name, 'total_filesize': tinfo['total_size'], 'folder': tinfo['save_path'], 'time_started': tinfo['addition_date'], 'label': mylar.CONFIG.QBITTORRENT_LABEL, 'status': True} #logger.info(torrent_info) return torrent_info
def populate(link, publisher, shipdate): #this is the secondary url call to populate input = 'http://www.comicbookresources.com/' + link #print 'checking ' + str(input) response = urllib2.urlopen(input) soup = BeautifulSoup(response) abc = soup.findAll('p') lenabc = len(abc) i = 0 resultName = [] resultID = [] resultURL = [] matched = "no" upcome = [] get_next = False prev_chk = False while (i < lenabc): titlet = abc[i] #iterate through the p pulling out only results. titlet_next = titlet.findNext(text=True) #print ("titlet: " + str(titlet)) if "/prev_img.php?pid" in str(titlet) and titlet_next is None: #solicits in 03-2014 have seperated <p> tags, so we need to take the subsequent <p>, not the initial. prev_chk = False get_next = True i += 1 continue elif titlet_next is not None: #logger.fdebug('non seperated <p> tags - taking next text.') get_next = False prev_chk = True elif "/news/preview2.php" in str(titlet): prev_chk = True get_next = False elif get_next == True: prev_chk = True else: prev_chk = False get_next = False if prev_chk == True: tempName = titlet.findNext(text=True) if not any([ ' TPB' in tempName, 'HC' in tempName, 'GN-TPB' in tempName, 'for $1' in tempName.lower(), 'subscription variant' in tempName.lower(), 'poster' in tempName.lower() ]): if '#' in tempName[:50]: #tempName = tempName.replace(u'.',u"'") tempName = tempName.encode('ascii', 'replace') #.decode('utf-8') if '???' in tempName: tempName = tempName.replace('???', ' ') stissue = tempName.find('#') endissue = tempName.find(' ', stissue) if tempName[ stissue + 1] == ' ': #if issue has space between # and number, adjust. endissue = tempName.find(' ', stissue + 2) if endissue == -1: endissue = len(tempName) issue = tempName[stissue:endissue].lstrip(' ') if ':' in issue: issue = re.sub(':', '', issue).rstrip() exinfo = tempName[endissue:].lstrip(' ') issue1 = None issue2 = None if '-' in issue: #print ('multiple issues detected. Splitting.') ststart = issue.find('-') issue1 = issue[:ststart] issue2 = '#' + str(issue[ststart + 1:]) if '&' in exinfo: #print ('multiple issues detected. Splitting.') ststart = exinfo.find('&') issue1 = issue # this detects fine issue2 = '#' + str(exinfo[ststart + 1:]) if '& ' in issue2: issue2 = re.sub("&\\b", "", issue2) exinfo = exinfo.replace( exinfo[ststart + 1:len(issue2)], '').strip() if exinfo == '&': exinfo = 'N/A' comic = tempName[:stissue].strip() if 'for \$1' in comic: exinfo = 'for $1' comic = comic.replace('for \$1\:', '').lstrip() issuedate = shipdate if 'on sale' in str(titlet).lower(): onsale_start = str(titlet).lower().find('on sale') + 8 onsale_end = str(titlet).lower().find( '<br>', onsale_start) thedate = str(titlet)[onsale_start:onsale_end] m = None basemonths = { 'january': '1', 'jan': '1', 'february': '2', 'feb': '2', 'march': '3', 'mar': '3', 'april': '4', 'apr': '4', 'may': '5', 'june': '6', 'july': '7', 'august': '8', 'aug': '8', 'september': '9', 'sept': '9', 'october': '10', 'oct': '10', 'november': '11', 'nov': '11', 'december': '12', 'dec': '12' } for month in basemonths: if month in thedate.lower(): m = basemonths[month] monthname = month break if m is not None: theday = len( month ) + 1 # account for space between month & day thedaystart = thedate[theday:( theday + 2)].strip() # day numeric won't exceed 2 if len(str(thedaystart)) == 1: thedaystart = '0' + str(thedaystart) if len(str(m)) == 1: m = '0' + str(m) thedate = shipdate[-4:] + '-' + str(m) + '-' + str( thedaystart) logger.info('[' + comic + '] On sale :' + str(thedate)) exinfo += ' [' + str(thedate) + ']' issuedate = thedate if issue1: upcome.append({ 'Shipdate': issuedate, 'Publisher': publisher.upper(), 'Issue': re.sub('#', '', issue1).lstrip(), 'Comic': comic.upper(), 'Extra': exinfo.upper() }) #print ('Comic: ' + comic) #print('issue#: ' + re.sub('#', '', issue1)) #print ('extra info: ' + exinfo) if issue2: upcome.append({ 'Shipdate': issuedate, 'Publisher': publisher.upper(), 'Issue': re.sub('#', '', issue2).lstrip(), 'Comic': comic.upper(), 'Extra': exinfo.upper() }) #print ('Comic: ' + comic) #print('issue#: ' + re.sub('#', '', issue2)) #print ('extra info: ' + exinfo) else: upcome.append({ 'Shipdate': issuedate, 'Publisher': publisher.upper(), 'Issue': re.sub('#', '', issue).lstrip(), 'Comic': comic.upper(), 'Extra': exinfo.upper() }) #print ('Comic: ' + comic) #print ('issue#: ' + re.sub('#', '', issue)) #print ('extra info: ' + exinfo) else: pass #print ('no issue # to retrieve.') i += 1 return upcome
def findComic(name, mode, issue, limityear=None, type=None): #with mb_lock: comicResults = None comicLibrary = listLibrary() comiclist = [] arcinfolist = [] commons = ['and', 'the', '&', '-'] for x in commons: cnt = 0 for m in re.finditer(x, name.lower()): cnt += 1 tehstart = m.start() tehend = m.end() if any([x == 'the', x == 'and']): if len(name) == tehend: tehend = -1 if not all([tehstart == 0, name[tehend] == ' ']) or not all([ tehstart != 0, name[tehstart - 1] == ' ', name[tehend] == ' ' ]): continue else: name = name.replace(x, ' ', cnt) originalname = name if '+' in name: name = re.sub('\+', 'PLUS', name) pattern = re.compile(r'\w+', re.UNICODE) name = pattern.findall(name) if '+' in originalname: y = [] for x in name: y.append(re.sub("PLUS", "%2B", x)) name = y if limityear is None: limityear = 'None' comicquery = name if mylar.CONFIG.COMICVINE_API == 'None' or mylar.CONFIG.COMICVINE_API is None: logger.warn( 'You have not specified your own ComicVine API key - this is a requirement. Get your own @ http://api.comicvine.com.' ) return else: comicapi = mylar.CONFIG.COMICVINE_API if type is None: type = 'volume' #let's find out how many results we get from the query... searched = pullsearch(comicapi, comicquery, 0, type) if searched is None: return False totalResults = searched.getElementsByTagName( 'number_of_total_results')[0].firstChild.wholeText logger.fdebug("there are " + str(totalResults) + " search results...") if not totalResults: return False if int(totalResults) > 1000: logger.warn( 'Search returned more than 1000 hits [' + str(totalResults) + ']. Only displaying first 1000 results - use more specifics or the exact ComicID if required.' ) totalResults = 1000 countResults = 0 while (countResults < int(totalResults)): #logger.fdebug("querying " + str(countResults)) if countResults > 0: offsetcount = countResults searched = pullsearch(comicapi, comicquery, offsetcount, type) comicResults = searched.getElementsByTagName(type) body = '' n = 0 if not comicResults: break for result in comicResults: #retrieve the first xml tag (<tag>data</tag>) #that the parser finds with name tagName: arclist = [] if type == 'story_arc': #call cv.py here to find out issue count in story arc try: logger.fdebug('story_arc ascension') names = len(result.getElementsByTagName('name')) n = 0 logger.fdebug('length: ' + str(names)) xmlpub = None #set this incase the publisher field isn't populated in the xml while (n < names): logger.fdebug( result.getElementsByTagName('name') [n].parentNode.nodeName) if result.getElementsByTagName( 'name')[n].parentNode.nodeName == 'story_arc': logger.fdebug('yes') try: xmlTag = result.getElementsByTagName( 'name')[n].firstChild.wholeText xmlTag = xmlTag.rstrip() logger.fdebug('name: ' + xmlTag) except: logger.error( 'There was a problem retrieving the given data from ComicVine. Ensure that www.comicvine.com is accessible.' ) return elif result.getElementsByTagName( 'name')[n].parentNode.nodeName == 'publisher': logger.fdebug('publisher check.') xmlpub = result.getElementsByTagName( 'name')[n].firstChild.wholeText n += 1 except: logger.warn('error retrieving story arc search results.') return siteurl = len(result.getElementsByTagName('site_detail_url')) s = 0 logger.fdebug('length: ' + str(names)) xmlurl = None while (s < siteurl): logger.fdebug( result.getElementsByTagName('site_detail_url') [s].parentNode.nodeName) if result.getElementsByTagName('site_detail_url')[ s].parentNode.nodeName == 'story_arc': try: xmlurl = result.getElementsByTagName( 'site_detail_url')[s].firstChild.wholeText except: logger.error( 'There was a problem retrieving the given data from ComicVine. Ensure that www.comicvine.com is accessible.' ) return s += 1 xmlid = result.getElementsByTagName( 'id')[0].firstChild.wholeText if xmlid is not None: arcinfolist = storyarcinfo(xmlid) logger.info('[IMAGE] : ' + arcinfolist['comicimage']) comiclist.append({ 'name': xmlTag, 'comicyear': arcinfolist['comicyear'], 'comicid': xmlid, 'cvarcid': xmlid, 'url': xmlurl, 'issues': arcinfolist['issues'], 'comicimage': arcinfolist['comicimage'], 'publisher': xmlpub, 'description': arcinfolist['description'], 'deck': arcinfolist['deck'], 'arclist': arcinfolist['arclist'], 'haveit': arcinfolist['haveit'] }) else: comiclist.append({ 'name': xmlTag, 'comicyear': arcyear, 'comicid': xmlid, 'url': xmlurl, 'issues': issuecount, 'comicimage': xmlimage, 'publisher': xmlpub, 'description': xmldesc, 'deck': xmldeck, 'arclist': arclist, 'haveit': haveit }) logger.fdebug('IssueID\'s that are a part of ' + xmlTag + ' : ' + str(arclist)) else: xmlcnt = result.getElementsByTagName( 'count_of_issues')[0].firstChild.wholeText #here we can determine what called us, and either start gathering all issues or just limited ones. if issue is not None and str(issue).isdigit(): #this gets buggered up with NEW/ONGOING series because the db hasn't been updated #to reflect the proper count. Drop it by 1 to make sure. limiter = int(issue) - 1 else: limiter = 0 #get the first issue # (for auto-magick calcs) iss_len = len(result.getElementsByTagName('name')) i = 0 xmlfirst = '1' xmllast = None try: while (i < iss_len): if result.getElementsByTagName('name')[ i].parentNode.nodeName == 'first_issue': xmlfirst = result.getElementsByTagName( 'issue_number')[i].firstChild.wholeText if '\xbd' in xmlfirst: xmlfirst = '1' #if the first issue is 1/2, just assume 1 for logistics elif result.getElementsByTagName( 'name')[i].parentNode.nodeName == 'last_issue': xmllast = result.getElementsByTagName( 'issue_number')[i].firstChild.wholeText if all([xmllast is not None, xmlfirst is not None]): break i += 1 except: xmlfirst = '1' if all( [xmlfirst == xmllast, xmlfirst.isdigit(), xmlcnt == '0']): xmlcnt = '1' #logger.info('There are : ' + str(xmlcnt) + ' issues in this series.') #logger.info('The first issue started at # ' + str(xmlfirst)) d = decimal.Decimal(xmlfirst) if d < 1: cnt_numerical = int(xmlcnt) + 1 else: cnt_numerical = int(xmlcnt) + int( math.ceil(d) ) # (of issues + start of first issue = numerical range) #logger.info('The maximum issue number should be roughly # ' + str(cnt_numerical)) #logger.info('The limiter (issue max that we know of) is # ' + str(limiter)) if cnt_numerical >= limiter: cnl = len(result.getElementsByTagName('name')) cl = 0 xmlTag = 'None' xmlimage = "cache/blankcover.jpg" xml_lastissueid = 'None' while (cl < cnl): if result.getElementsByTagName( 'name')[cl].parentNode.nodeName == 'volume': xmlTag = result.getElementsByTagName( 'name')[cl].firstChild.wholeText #break if result.getElementsByTagName( 'name')[cl].parentNode.nodeName == 'image': xmlimage = result.getElementsByTagName( 'super_url')[0].firstChild.wholeText if result.getElementsByTagName('name')[ cl].parentNode.nodeName == 'last_issue': xml_lastissueid = result.getElementsByTagName( 'id')[cl].firstChild.wholeText cl += 1 if (result.getElementsByTagName('start_year')[0].firstChild ) is not None: xmlYr = result.getElementsByTagName( 'start_year')[0].firstChild.wholeText else: xmlYr = "0000" yearRange = [] tmpYr = re.sub('\?', '', xmlYr) if tmpYr.isdigit(): yearRange.append(tmpYr) tmpyearRange = int(xmlcnt) / 12 if float(tmpyearRange): tmpyearRange + 1 possible_years = int(tmpYr) + tmpyearRange for i in range(int(tmpYr), int(possible_years), 1): if not any(int(x) == int(i) for x in yearRange): yearRange.append(str(i)) logger.fdebug('[RESULT][' + str(limityear) + '] ComicName:' + xmlTag + ' -- ' + str(xmlYr) + ' [Series years: ' + str(yearRange) + ']') if tmpYr != xmlYr: xmlYr = tmpYr if any([v in limityear for v in yearRange]) or limityear == 'None': xmlurl = result.getElementsByTagName( 'site_detail_url')[0].firstChild.wholeText idl = len(result.getElementsByTagName('id')) idt = 0 xmlid = None while (idt < idl): if result.getElementsByTagName( 'id')[idt].parentNode.nodeName == 'volume': xmlid = result.getElementsByTagName( 'id')[idt].firstChild.wholeText break idt += 1 if xmlid is None: logger.error( 'Unable to figure out the comicid - skipping this : ' + str(xmlurl)) continue publishers = result.getElementsByTagName('publisher') if len(publishers) > 0: pubnames = publishers[0].getElementsByTagName( 'name') if len(pubnames) > 0: xmlpub = pubnames[0].firstChild.wholeText else: xmlpub = "Unknown" else: xmlpub = "Unknown" #ignore specific publishers on a global scale here. if mylar.CONFIG.BLACKLISTED_PUBLISHERS is not None and any( [ x for x in mylar.CONFIG.BLACKLISTED_PUBLISHERS if x.lower() == xmlpub.lower() ]): logger.fdebug('Blacklisted publisher [' + xmlpub + ']. Ignoring this result.') continue try: xmldesc = result.getElementsByTagName( 'description')[0].firstChild.wholeText except: xmldesc = "None" #this is needed to display brief synopsis for each series on search results page. try: xmldeck = result.getElementsByTagName( 'deck')[0].firstChild.wholeText except: xmldeck = "None" xmltype = None if xmldeck != 'None': if any([ 'print' in xmldeck.lower(), 'digital' in xmldeck.lower(), 'paperback' in xmldeck.lower(), 'one shot' in re.sub('-', '', xmldeck.lower()).strip(), 'hardcover' in xmldeck.lower() ]): if all([ 'print' in xmldeck.lower(), 'reprint' not in xmldeck.lower() ]): xmltype = 'Print' elif 'digital' in xmldeck.lower(): xmltype = 'Digital' elif 'paperback' in xmldeck.lower(): xmltype = 'TPB' elif 'hardcover' in xmldeck.lower(): xmltype = 'HC' elif 'oneshot' in re.sub( '-', '', xmldeck.lower()).strip(): xmltype = 'One-Shot' else: xmltype = 'Print' if xmldesc != 'None' and xmltype is None: if 'print' in xmldesc[:60].lower() and all([ 'print edition can be found' not in xmldesc.lower(), 'reprints' not in xmldesc.lower() ]): xmltype = 'Print' elif 'digital' in xmldesc[:60].lower( ) and 'digital edition can be found' not in xmldesc.lower( ): xmltype = 'Digital' elif all([ 'paperback' in xmldesc[:60].lower(), 'paperback can be found' not in xmldesc.lower() ]) or 'collects' in xmldesc[:60].lower(): xmltype = 'TPB' elif 'hardcover' in xmldesc[:60].lower( ) and 'hardcover can be found' not in xmldesc.lower( ): xmltype = 'HC' elif any([ 'one-shot' in xmldesc[:60].lower(), 'one shot' in xmldesc[:60].lower() ]) and any([ 'can be found' not in xmldesc.lower(), 'following the' not in xmldesc.lower() ]): i = 0 xmltype = 'One-Shot' avoidwords = [ 'preceding', 'after the special', 'following the' ] while i < 2: if i == 0: cbd = 'one-shot' elif i == 1: cbd = 'one shot' tmp1 = xmldesc[:60].lower().find(cbd) if tmp1 != -1: for x in avoidwords: tmp2 = xmldesc[:tmp1].lower().find( x) if tmp2 != -1: xmltype = 'Print' i = 3 break i += 1 else: xmltype = 'Print' if xmlid in comicLibrary: haveit = comicLibrary[xmlid] else: haveit = "No" comiclist.append({ 'name': xmlTag, 'comicyear': xmlYr, 'comicid': xmlid, 'url': xmlurl, 'issues': xmlcnt, 'comicimage': xmlimage, 'publisher': xmlpub, 'description': xmldesc, 'deck': xmldeck, 'type': xmltype, 'haveit': haveit, 'lastissueid': xml_lastissueid, 'seriesrange': yearRange # returning additional information about series run polled from CV }) #logger.fdebug('year: %s - constraint met: %s [%s] --- 4050-%s' % (xmlYr,xmlTag,xmlYr,xmlid)) else: #logger.fdebug('year: ' + str(xmlYr) + ' - contraint not met. Has to be within ' + str(limityear)) pass n += 1 #search results are limited to 100 and by pagination now...let's account for this. countResults = countResults + 100 return comiclist
def libraryScan(dir=None, append=False, ComicID=None, ComicName=None, cron=None): if cron and not mylar.LIBRARYSCAN: return if not dir: dir = mylar.COMIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring if not append: dir = dir.encode(mylar.SYS_ENCODING) if not os.path.isdir(dir): logger.warn('Cannot find directory: %s. Not scanning' % dir.decode(mylar.SYS_ENCODING, 'replace')) return logger.info('Scanning comic directory: %s' % dir.decode(mylar.SYS_ENCODING, 'replace')) basedir = dir comic_list = [] comiccnt = 0 extensions = ('cbr', 'cbz') for r, d, f in os.walk(dir): #for directory in d[:]: # if directory.startswith("."): # d.remove(directory) for files in f: if any(files.lower().endswith('.' + x.lower()) for x in extensions): comic = files comicpath = os.path.join(r, files) comicsize = os.path.getsize(comicpath) print "Comic: " + comic print "Comic Path: " + comicpath print "Comic Size: " + str(comicsize) # We need the unicode path to use for logging, inserting into database unicode_comic_path = comicpath.decode(mylar.SYS_ENCODING, 'replace') comiccnt += 1 comic_dict = { 'ComicFilename': comic, 'ComicLocation': comicpath, 'ComicSize': comicsize, 'Unicode_ComicLocation': unicode_comic_path } comic_list.append(comic_dict) logger.info("I've found a total of " + str(comiccnt) + " comics....analyzing now") logger.info("comiclist: " + str(comic_list)) myDB = db.DBConnection() #let's load in the watchlist to see if we have any matches. logger.info( "loading in the watchlist to see if a series is being watched already..." ) watchlist = myDB.select("SELECT * from comics") ComicName = [] DisplayName = [] ComicYear = [] ComicPublisher = [] ComicTotal = [] ComicID = [] ComicLocation = [] AltName = [] watchcnt = 0 watch_kchoice = [] watchchoice = {} import_by_comicids = [] import_comicids = {} for watch in watchlist: #use the comicname_filesafe to start watchdisplaycomic = re.sub('[\_\#\,\/\:\;\!\$\%\&\+\'\?\@]', ' ', watch['ComicName']).encode('utf-8').strip() # let's clean up the name, just in case for comparison purposes... watchcomic = re.sub( '[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ', watch['ComicName_Filesafe']).encode('utf-8').strip() #watchcomic = re.sub('\s+', ' ', str(watchcomic)).strip() if ' the ' in watchcomic.lower(): #drop the 'the' from the watchcomic title for proper comparisons. watchcomic = watchcomic[-4:] alt_chk = "no" # alt-checker flag (default to no) # account for alternate names as well if watch['AlternateSearch'] is not None and watch[ 'AlternateSearch'] is not 'None': altcomic = re.sub( '[\_\#\,\/\:\;\.\-\!\$\%\&\+\'\?\@]', ' ', watch['AlternateSearch']).encode('utf-8').strip() #altcomic = re.sub('\s+', ' ', str(altcomic)).strip() AltName.append(altcomic) alt_chk = "yes" # alt-checker flag ComicName.append(watchcomic) DisplayName.append(watchdisplaycomic) ComicYear.append(watch['ComicYear']) ComicPublisher.append(watch['ComicPublisher']) ComicTotal.append(watch['Total']) ComicID.append(watch['ComicID']) ComicLocation.append(watch['ComicLocation']) watchcnt += 1 logger.info("Successfully loaded " + str(watchcnt) + " series from your watchlist.") ripperlist = ['digital-', 'empire', 'dcp'] watchfound = 0 datelist = [ 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec' ] # datemonth = {'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9,'ten':10,'eleven':$ # #search for number as text, and change to numeric # for numbs in basnumbs: # #print ("numbs:" + str(numbs)) # if numbs in ComicName.lower(): # numconv = basnumbs[numbs] # #print ("numconv: " + str(numconv)) for i in comic_list: print i['ComicFilename'] #if mylar.IMP_METADATA: #logger.info('metatagging checking enabled.') #if read tags is enabled during import, check here. #if i['ComicLocation'].endswith('.cbz'): # logger.info('Attempting to read tags present in filename: ' + str(i['ComicLocation'])) # issueinfo = helpers.IssueDetails(i['ComicLocation']) # if issueinfo is None: # pass # else: # logger.info('Successfully retrieved some tags. Lets see what I can figure out.') # comicname = issueinfo[0]['series'] # logger.fdebug('Series Name: ' + comicname) # issue_number = issueinfo[0]['issue_number'] # logger.fdebug('Issue Number: ' + str(issue_number)) # issuetitle = issueinfo[0]['title'] # logger.fdebug('Issue Title: ' + issuetitle) # issueyear = issueinfo[0]['year'] # logger.fdebug('Issue Year: ' + str(issueyear)) # # if used by ComicTagger, Notes field will have the IssueID. # issuenotes = issueinfo[0]['notes'] # logger.fdebug('Notes: ' + issuenotes) comfilename = i['ComicFilename'] comlocation = i['ComicLocation'] #let's clean up the filename for matching purposes cfilename = re.sub('[\_\#\,\/\:\;\-\!\$\%\&\+\'\?\@]', ' ', comfilename) #cfilename = re.sub('\s', '_', str(cfilename)) d_filename = re.sub('[\_\#\,\/\;\!\$\%\&\?\@]', ' ', comfilename) d_filename = re.sub('[\:\-\+\']', '#', d_filename) #strip extraspaces d_filename = re.sub('\s+', ' ', d_filename) cfilename = re.sub('\s+', ' ', cfilename) #versioning - remove it subsplit = cfilename.replace('_', ' ').split() volno = None volyr = None for subit in subsplit: if subit[0].lower() == 'v': vfull = 0 if subit[1:].isdigit(): #if in format v1, v2009 etc... if len(subit) > 3: # if it's greater than 3 in length, then the format is Vyyyy vfull = 1 # add on 1 character length to account for extra space cfilename = re.sub(subit, '', cfilename) d_filename = re.sub(subit, '', d_filename) volno = re.sub("[^0-9]", " ", subit) elif subit.lower()[:3] == 'vol': #if in format vol.2013 etc #because the '.' in Vol. gets removed, let's loop thru again after the Vol hit to remove it entirely logger.fdebug('volume indicator detected as version #:' + str(subit)) cfilename = re.sub(subit, '', cfilename) cfilename = " ".join(cfilename.split()) d_filename = re.sub(subit, '', d_filename) d_filename = " ".join(d_filename.split()) volyr = re.sub("[^0-9]", " ", subit).strip() logger.fdebug('volume year set as : ' + str(volyr)) cm_cn = 0 #we need to track the counter to make sure we are comparing the right array parts #this takes care of the brackets :) m = re.findall('[^()]+', cfilename) lenm = len(m) logger.fdebug("there are " + str(lenm) + " words.") cnt = 0 yearmatch = "false" foundonwatch = "False" issue = 999999 while (cnt < lenm): if m[cnt] is None: break if m[cnt] == ' ': pass else: logger.fdebug(str(cnt) + ". Bracket Word: " + m[cnt]) if cnt == 0: comic_andiss = m[cnt] logger.fdebug("Comic: " + comic_andiss) # if it's not in the standard format this will bork. # let's try to accomodate (somehow). # first remove the extension (if any) extensions = ('cbr', 'cbz') if comic_andiss.lower().endswith(extensions): comic_andiss = comic_andiss[:-4] logger.fdebug("removed extension from filename.") #now we have to break up the string regardless of formatting. #let's force the spaces. comic_andiss = re.sub('_', ' ', comic_andiss) cs = comic_andiss.split() cs_len = len(cs) cn = '' ydetected = 'no' idetected = 'no' decimaldetect = 'no' for i in reversed(xrange(len(cs))): #start at the end. logger.fdebug("word: " + str(cs[i])) #assume once we find issue - everything prior is the actual title #idetected = no will ignore everything so it will assume all title if cs[i][:-2] == '19' or cs[ i][:-2] == '20' and idetected == 'no': logger.fdebug("year detected: " + str(cs[i])) ydetected = 'yes' result_comyear = cs[i] elif cs[i].isdigit( ) and idetected == 'no' or '.' in cs[i]: if '.' in cs[i]: #make sure it's a number on either side of decimal and assume decimal issue. decst = cs[i].find('.') dec_st = cs[i][:decst] dec_en = cs[i][decst + 1:] logger.fdebug("st: " + str(dec_st)) logger.fdebug("en: " + str(dec_en)) if dec_st.isdigit() and dec_en.isdigit(): logger.fdebug( "decimal issue detected...adjusting.") issue = dec_st + "." + dec_en logger.fdebug("issue detected: " + str(issue)) idetected = 'yes' else: logger.fdebug( "false decimal represent. Chunking to extra word." ) cn = cn + cs[i] + " " break issue = cs[i] logger.fdebug("issue detected : " + str(issue)) idetected = 'yes' elif '\#' in cs[i] or decimaldetect == 'yes': logger.fdebug("issue detected: " + str(cs[i])) idetected = 'yes' else: cn = cn + cs[i] + " " if ydetected == 'no': #assume no year given in filename... result_comyear = "0000" logger.fdebug("cm?: " + str(cn)) if issue is not '999999': comiss = issue else: logger.ERROR( "Invalid Issue number (none present) for " + comfilename) break cnsplit = cn.split() cname = '' findcn = 0 while (findcn < len(cnsplit)): cname = cname + cs[findcn] + " " findcn += 1 cname = cname[:len(cname) - 1] # drop the end space... print("assuming name is : " + cname) com_NAME = cname print("com_NAME : " + com_NAME) yearmatch = "True" else: logger.fdebug('checking ' + m[cnt]) # we're assuming that the year is in brackets (and it should be damnit) if m[cnt][:-2] == '19' or m[cnt][:-2] == '20': print("year detected: " + str(m[cnt])) ydetected = 'yes' result_comyear = m[cnt] elif m[cnt][:3].lower() in datelist: logger.fdebug( 'possible issue date format given - verifying') #if the date of the issue is given as (Jan 2010) or (January 2010) let's adjust. #keeping in mind that ',' and '.' are already stripped from the string if m[cnt][-4:].isdigit(): ydetected = 'yes' result_comyear = m[cnt][-4:] logger.fdebug('Valid Issue year of ' + str(result_comyear) + 'detected in format of ' + str(m[cnt])) cnt += 1 displength = len(cname) print 'd_filename is : ' + d_filename charcount = d_filename.count('#') print('charcount is : ' + str(charcount)) if charcount > 0: print('entering loop') for i, m in enumerate(re.finditer('\#', d_filename)): if m.end() <= displength: print comfilename[m.start():m.end()] # find occurance in c_filename, then replace into d_filname so special characters are brought across newchar = comfilename[m.start():m.end()] print 'newchar:' + str(newchar) d_filename = d_filename[:m.start()] + str( newchar) + d_filename[m.end():] print 'd_filename:' + str(d_filename) dispname = d_filename[:displength] print('dispname : ' + dispname) splitit = [] watchcomic_split = [] logger.fdebug("filename comic and issue: " + comic_andiss) #changed this from '' to ' ' comic_iss_b4 = re.sub('[\-\:\,]', ' ', comic_andiss) comic_iss = comic_iss_b4.replace('.', ' ') comic_iss = re.sub('[\s+]', ' ', comic_iss).strip() logger.fdebug("adjusted comic and issue: " + str(comic_iss)) #remove 'the' from here for proper comparisons. if ' the ' in comic_iss.lower(): comic_iss = re.sub('\\bthe\\b', '', comic_iss).strip() splitit = comic_iss.split(None) logger.fdebug("adjusting from: " + str(comic_iss_b4) + " to: " + str(comic_iss)) #here we cycle through the Watchlist looking for a match. while (cm_cn < watchcnt): #setup the watchlist comname = ComicName[cm_cn] comyear = ComicYear[cm_cn] compub = ComicPublisher[cm_cn] comtotal = ComicTotal[cm_cn] comicid = ComicID[cm_cn] watch_location = ComicLocation[cm_cn] # there shouldn't be an issue in the comic now, so let's just assume it's all gravy. splitst = len(splitit) watchcomic_split = helpers.cleanName(comname) watchcomic_split = re.sub('[\-\:\,\.]', ' ', watchcomic_split).split(None) logger.fdebug( str(splitit) + " file series word count: " + str(splitst)) logger.fdebug( str(watchcomic_split) + " watchlist word count: " + str(len(watchcomic_split))) if (splitst) != len(watchcomic_split): logger.fdebug("incorrect comic lengths...not a match") # if str(splitit[0]).lower() == "the": # logger.fdebug("THE word detected...attempting to adjust pattern matching") # splitit[0] = splitit[4:] else: logger.fdebug("length match..proceeding") n = 0 scount = 0 logger.fdebug("search-length: " + str(splitst)) logger.fdebug("Watchlist-length: " + str(len(watchcomic_split))) while (n <= (splitst) - 1): logger.fdebug("splitit: " + str(splitit[n])) if n < (splitst) and n < len(watchcomic_split): logger.fdebug( str(n) + " Comparing: " + str(watchcomic_split[n]) + " .to. " + str(splitit[n])) if '+' in watchcomic_split[n]: watchcomic_split[n] = re.sub( '+', '', str(watchcomic_split[n])) if str(watchcomic_split[n].lower()) in str( splitit[n].lower()) and len( watchcomic_split[n]) >= len(splitit[n]): logger.fdebug("word matched on : " + str(splitit[n])) scount += 1 #elif ':' in splitit[n] or '-' in splitit[n]: # splitrep = splitit[n].replace('-', '') # print ("non-character keyword...skipped on " + splitit[n]) elif str(splitit[n]).lower().startswith('v'): logger.fdebug("possible versioning..checking") #we hit a versioning # - account for it if splitit[n][1:].isdigit(): comicversion = str(splitit[n]) logger.fdebug("version found: " + str(comicversion)) else: logger.fdebug("Comic / Issue section") if splitit[n].isdigit(): logger.fdebug("issue detected") else: logger.fdebug("non-match for: " + str(splitit[n])) pass n += 1 #set the match threshold to 80% (for now) # if it's less than 80% consider it a non-match and discard. #splitit has to splitit-1 because last position is issue. wordcnt = int(scount) logger.fdebug("scount:" + str(wordcnt)) totalcnt = int(splitst) logger.fdebug("splitit-len:" + str(totalcnt)) spercent = (wordcnt / totalcnt) * 100 logger.fdebug("we got " + str(spercent) + " percent.") if int(spercent) >= 80: logger.fdebug("it's a go captain... - we matched " + str(spercent) + "%!") logger.fdebug("this should be a match!") logger.fdebug("issue we found for is : " + str(comiss)) #set the year to the series we just found ;) result_comyear = comyear #issue comparison now as well logger.info(u"Found " + comname + " (" + str(comyear) + ") issue: " + str(comiss)) watchmatch = str(comicid) dispname = DisplayName[cm_cn] foundonwatch = "True" break elif int(spercent) < 80: logger.fdebug("failure - we only got " + str(spercent) + "% right!") cm_cn += 1 if foundonwatch == "False": watchmatch = None #---if it's not a match - send it to the importer. n = 0 if volyr is None: if result_comyear is None: result_comyear = '0000' #no year in filename basically. else: if result_comyear is None: result_comyear = volyr if volno is None: if volyr is None: vol_label = None else: vol_label = volyr else: vol_label = volno print("adding " + com_NAME + " to the import-queue!") impid = com_NAME + "-" + str(result_comyear) + "-" + str(comiss) print("impid: " + str(impid)) import_by_comicids.append({ "impid": impid, "watchmatch": watchmatch, "displayname": dispname, "comicname": com_NAME, "comicyear": result_comyear, "volume": vol_label, "comfilename": comfilename, "comlocation": comlocation.decode(mylar.SYS_ENCODING) }) if len(watch_kchoice) > 0: watchchoice['watchlist'] = watch_kchoice print("watchchoice: " + str(watchchoice)) logger.info("I have found " + str(watchfound) + " out of " + str(comiccnt) + " comics for series that are being watched.") wat = 0 comicids = [] if watchfound > 0: if mylar.IMP_MOVE: logger.info( "You checked off Move Files...so that's what I'm going to do" ) #check to see if Move Files is enabled. #if not being moved, set the archive bit. print("Moving files into appropriate directory") while (wat < watchfound): watch_the_list = watchchoice['watchlist'][wat] watch_comlocation = watch_the_list['ComicLocation'] watch_comicid = watch_the_list['ComicID'] watch_comicname = watch_the_list['ComicName'] watch_comicyear = watch_the_list['ComicYear'] watch_comiciss = watch_the_list['ComicIssue'] print("ComicLocation: " + str(watch_comlocation)) orig_comlocation = watch_the_list['OriginalLocation'] orig_filename = watch_the_list['OriginalFilename'] print("Orig. Location: " + str(orig_comlocation)) print("Orig. Filename: " + str(orig_filename)) #before moving check to see if Rename to Mylar structure is enabled. if mylar.IMP_RENAME: print( "Renaming files according to configuration details : " + str(mylar.FILE_FORMAT)) renameit = helpers.rename_param( watch_comicid, watch_comicname, watch_comicyear, watch_comiciss) nfilename = renameit['nfilename'] dst_path = os.path.join(watch_comlocation, nfilename) if str(watch_comicid) not in comicids: comicids.append(watch_comicid) else: print( "Renaming files not enabled, keeping original filename(s)" ) dst_path = os.path.join(watch_comlocation, orig_filename) #os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext))) #src = os.path.join(, str(nfilename + ext)) print("I'm going to move " + str(orig_comlocation) + " to .." + str(dst_path)) try: shutil.move(orig_comlocation, dst_path) except (OSError, IOError): logger.info( "Failed to move directory - check directories and manually re-run." ) wat += 1 else: # if move files isn't enabled, let's set all found comics to Archive status :) while (wat < watchfound): watch_the_list = watchchoice['watchlist'][wat] watch_comicid = watch_the_list['ComicID'] watch_issue = watch_the_list['ComicIssue'] print("ComicID: " + str(watch_comicid)) print("Issue#: " + str(watch_issue)) issuechk = myDB.selectone( "SELECT * from issues where ComicID=? AND INT_IssueNumber=?", [watch_comicid, watch_issue]).fetchone() if issuechk is None: print("no matching issues for this comic#") else: print("...Existing status: " + str(issuechk['Status'])) control = {"IssueID": issuechk['IssueID']} values = {"Status": "Archived"} print("...changing status of " + str(issuechk['Issue_Number']) + " to Archived ") myDB.upsert("issues", values, control) if str(watch_comicid) not in comicids: comicids.append(watch_comicid) wat += 1 if comicids is None: pass else: c_upd = len(comicids) c = 0 while (c < c_upd): print("Rescanning.. " + str(c)) updater.forceRescan(c) if not len(import_by_comicids): return "Completed" if len(import_by_comicids) > 0: import_comicids['comic_info'] = import_by_comicids print("import comicids: " + str(import_by_comicids)) return import_comicids, len(import_by_comicids)
def upcoming_update(ComicID, ComicName, IssueNumber, IssueDate, forcecheck=None, futurepull=None, altissuenumber=None): # here we add to upcoming table... myDB = db.DBConnection() dspComicName = ComicName #to make sure that the word 'annual' will be displayed on screen if 'annual' in ComicName.lower(): adjComicName = re.sub("\\bannual\\b", "", ComicName.lower()) # for use with comparisons. logger.fdebug('annual detected - adjusting name to : ' + adjComicName) else: adjComicName = ComicName controlValue = {"ComicID": ComicID} newValue = { "ComicName": adjComicName, "IssueNumber": str(IssueNumber), "DisplayComicName": dspComicName, "IssueDate": str(IssueDate) } #let's refresh the series here just to make sure if an issue is available/not. mismatch = "no" CV_EXcomicid = myDB.action("SELECT * from exceptions WHERE ComicID=?", [ComicID]).fetchone() if CV_EXcomicid is None: pass else: if CV_EXcomicid['variloop'] == '99': mismatch = "yes" lastupdatechk = myDB.action("SELECT * FROM comics WHERE ComicID=?", [ComicID]).fetchone() if lastupdatechk is None: pullupd = "yes" else: c_date = lastupdatechk['LastUpdated'] if c_date is None: logger.error( lastupdatechk['ComicName'] + ' failed during a previous add /refresh. Please either delete and readd the series, or try a refresh of the series.' ) return c_obj_date = datetime.datetime.strptime(c_date, "%Y-%m-%d %H:%M:%S") n_date = datetime.datetime.now() absdiff = abs(n_date - c_obj_date) hours = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 3600.0 # no need to hammer the refresh # let's check it every 5 hours (or more) #pullupd = "yes" if 'annual' in ComicName.lower(): if mylar.ANNUALS_ON: issuechk = myDB.action( "SELECT * FROM annuals WHERE ComicID=? AND Issue_Number=?", [ComicID, IssueNumber]).fetchone() else: logger.fdebug( 'Annual detected, but annuals not enabled. Ignoring result.') return else: issuechk = myDB.action( "SELECT * FROM issues WHERE ComicID=? AND Issue_Number=?", [ComicID, IssueNumber]).fetchone() if issuechk is None and altissuenumber is not None: logger.info('altissuenumber is : ' + str(altissuenumber)) issuechk = myDB.action( "SELECT * FROM issues WHERE ComicID=? AND Int_IssueNumber=?", [ComicID, helpers.issuedigits(altissuenumber)]).fetchone() if issuechk is None: if futurepull is None: logger.fdebug( adjComicName + ' Issue: ' + str(IssueNumber) + ' not present in listings to mark for download...updating comic and adding to Upcoming Wanted Releases.' ) # we need to either decrease the total issue count, OR indicate that an issue is upcoming. upco_results = myDB.action( "SELECT COUNT(*) FROM UPCOMING WHERE ComicID=?", [ComicID]).fetchall() upco_iss = upco_results[0][0] #logger.info("upco_iss: " + str(upco_iss)) if int(upco_iss) > 0: #logger.info("There is " + str(upco_iss) + " of " + str(ComicName) + " that's not accounted for") newKey = {"ComicID": ComicID} newVal = {"not_updated_db": str(upco_iss)} myDB.upsert("comics", newVal, newKey) elif int(upco_iss) <= 0 and lastupdatechk['not_updated_db']: #if not_updated_db has a value, and upco_iss is > 0, let's zero it back out cause it's updated now. newKey = {"ComicID": ComicID} newVal = {"not_updated_db": ""} myDB.upsert("comics", newVal, newKey) if hours > 5 or forcecheck == 'yes': pullupd = "yes" logger.fdebug('Now Refreshing comic ' + ComicName + ' to make sure it is up-to-date') if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID, pullupd) else: mylar.importer.addComictoDB(ComicID, mismatch, pullupd) else: logger.fdebug( 'It has not been longer than 5 hours since we last did this...we will wait so we do not hammer things.' ) return else: # if futurepull is not None, let's just update the status and ComicID # NOTE: THIS IS CREATING EMPTY ENTRIES IN THE FUTURE TABLE. ??? nKey = {"ComicID": ComicID} nVal = {"Status": "Wanted"} myDB.upsert("future", nVal, nKey) if issuechk is not None: if issuechk['Issue_Number'] == IssueNumber or issuechk[ 'Issue_Number'] == altissuenumber: logger.fdebug( 'Comic series already up-to-date ... no need to refresh at this time.' ) logger.fdebug('Available to be marked for download - checking...' + adjComicName + ' Issue: ' + str(issuechk['Issue_Number'])) logger.fdebug('...Existing status: ' + str(issuechk['Status'])) control = {"IssueID": issuechk['IssueID']} newValue['IssueID'] = issuechk['IssueID'] if issuechk['Status'] == "Snatched": values = {"Status": "Snatched"} newValue['Status'] = "Snatched" elif issuechk['Status'] == "Downloaded": values = {"Status": "Downloaded"} newValue['Status'] = "Downloaded" #if the status is Downloaded and it's on the pullist - let's mark it so everyone can bask in the glory elif issuechk['Status'] == "Wanted": values = {"Status": "Wanted"} newValue['Status'] = "Wanted" elif issuechk['Status'] == "Archived": values = {"Status": "Archived"} newValue['Status'] = "Archived" else: values = {"Status": "Skipped"} newValue['Status'] = "Skipped" #was in wrong place :( else: logger.fdebug( 'Issues do not match for some reason...weekly new issue: ' + str(IssueNumber)) return if mylar.AUTOWANT_UPCOMING: #for issues not in db - to be added to Upcoming table. if issuechk is None: newValue['Status'] = "Wanted" logger.fdebug( '...Changing Status to Wanted and throwing it in the Upcoming section since it is not published yet.' ) #this works for issues existing in DB... elif issuechk['Status'] == "Skipped": newValue['Status'] = "Wanted" values = {"Status": "Wanted"} logger.fdebug('...New status of Wanted') elif issuechk['Status'] == "Wanted": logger.fdebug('...Status already Wanted .. not changing.') else: logger.fdebug( '...Already have issue - keeping existing status of : ' + str(issuechk['Status'])) if issuechk is None: myDB.upsert("upcoming", newValue, controlValue) else: logger.fdebug('--attempt to find errant adds to Wanted list') logger.fdebug('UpcomingNewValue: ' + str(newValue)) logger.fdebug('UpcomingcontrolValue: ' + str(controlValue)) if issuechk['IssueDate'] == '0000-00-00' and newValue[ 'IssueDate'] != '0000-00-00': logger.fdebug( 'Found a 0000-00-00 issue - force updating series to try and get it proper.' ) dateVal = { "IssueDate": newValue['IssueDate'], "ComicName": issuechk['ComicName'], "Status": newValue['Status'], "IssueNumber": issuechk['Issue_Number'] } logger.fdebug('updating date in upcoming table to : ' + str(newValue['IssueDate'])) logger.fdebug('ComicID:' + str(controlValue)) myDB.upsert("upcoming", dateVal, controlValue) logger.fdebug('Temporarily putting the Issue Date for ' + str(issuechk['Issue_Number']) + ' to ' + str(newValue['IssueDate'])) values = {"IssueDate": newValue['IssueDate']} #if ComicID[:1] == "G": mylar.importer.GCDimport(ComicID,pullupd='yes') #else: mylar.importer.addComictoDB(ComicID,mismatch,pullupd='yes') if 'annual' in ComicName.lower(): myDB.upsert("annuals", values, control) else: myDB.upsert("issues", values, control) if issuechk['Status'] == 'Downloaded' or issuechk[ 'Status'] == 'Archived' or issuechk['Status'] == 'Snatched': logger.fdebug('updating Pull-list to reflect status.') downstats = { "Status": issuechk['Status'], "ComicID": issuechk['ComicID'] } return downstats
def Process(self): self._log("nzb name: " + str(self.nzb_name), logger.DEBUG) self._log("nzb folder: " + str(self.nzb_folder), logger.DEBUG) logger.fdebug("nzb name: " + str(self.nzb_name)) logger.fdebug("nzb folder: " + str(self.nzb_folder)) if mylar.USE_SABNZBD==0: logger.fdebug("Not using SABNzbd") else: # if the SAB Directory option is enabled, let's use that folder name and append the jobname. if mylar.SAB_DIRECTORY is not None and mylar.SAB_DIRECTORY is not 'None' and len(mylar.SAB_DIRECTORY) > 4: self.nzb_folder = os.path.join(mylar.SAB_DIRECTORY, self.nzb_name).encode(mylar.SYS_ENCODING) #lookup nzb_name in nzblog table to get issueid #query SAB to find out if Replace Spaces enabled / not as well as Replace Decimals #http://localhost:8080/sabnzbd/api?mode=set_config§ion=misc&keyword=dirscan_speed&value=5 querysab = str(mylar.SAB_HOST) + "/api?mode=get_config§ion=misc&output=xml&apikey=" + str(mylar.SAB_APIKEY) #logger.info("querysab_string:" + str(querysab)) file = urllib2.urlopen(querysab) data = file.read() file.close() dom = parseString(data) try: sabreps = dom.getElementsByTagName('replace_spaces')[0].firstChild.wholeText except: errorm = dom.getElementsByTagName('error')[0].firstChild.wholeText logger.error(u"Error detected attempting to retrieve SAB data : " + errorm) return sabrepd = dom.getElementsByTagName('replace_dots')[0].firstChild.wholeText logger.fdebug("SAB Replace Spaces: " + str(sabreps)) logger.fdebug("SAB Replace Dots: " + str(sabrepd)) if mylar.USE_NZBGET==1: logger.fdebug("Using NZBGET") logger.fdebug("NZB name as passed from NZBGet: " + self.nzb_name) myDB = db.DBConnection() nzbname = self.nzb_name #remove extensions from nzb_name if they somehow got through (Experimental most likely) extensions = ('.cbr', '.cbz') if nzbname.lower().endswith(extensions): fd, ext = os.path.splitext(nzbname) self._log("Removed extension from nzb: " + ext, logger.DEBUG) nzbname = re.sub(str(ext), '', str(nzbname)) #replace spaces nzbname = re.sub(' ', '.', str(nzbname)) nzbname = re.sub('[\,\:\?]', '', str(nzbname)) nzbname = re.sub('[\&]', 'and', str(nzbname)) logger.fdebug("After conversions, nzbname is : " + str(nzbname)) # if mylar.USE_NZBGET==1: # nzbname=self.nzb_name self._log("nzbname: " + str(nzbname), logger.DEBUG) nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone() if nzbiss is None: self._log("Failure - could not initially locate nzbfile in my database to rename.", logger.DEBUG) logger.fdebug("Failure - could not locate nzbfile initially.") # if failed on spaces, change it all to decimals and try again. nzbname = re.sub('_', '.', str(nzbname)) self._log("trying again with this nzbname: " + str(nzbname), logger.DEBUG) logger.fdebug("trying again with nzbname of : " + str(nzbname)) nzbiss = myDB.action("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone() if nzbiss is None: logger.error(u"Unable to locate downloaded file to rename. PostProcessing aborted.") return else: self._log("I corrected and found the nzb as : " + str(nzbname)) logger.fdebug("auto-corrected and found the nzb as : " + str(nzbname)) issueid = nzbiss['IssueID'] else: issueid = nzbiss['IssueID'] logger.fdebug("issueid:" + str(issueid)) sarc = nzbiss['SARC'] #use issueid to get publisher, series, year, issue number issuenzb = myDB.action("SELECT * from issues WHERE issueid=?", [issueid]).fetchone() if issuenzb is not None: if helpers.is_number(issueid): sandwich = int(issuenzb['IssueID']) else: #if it's non-numeric, it contains a 'G' at the beginning indicating it's a multi-volume #using GCD data. Set sandwich to 1 so it will bypass and continue post-processing. if 'S' in issueid: sandwich = issueid elif 'G' in issueid: sandwich = 1 if helpers.is_number(sandwich): if sandwich < 900000: # if sandwich is less than 900000 it's a normal watchlist download. Bypass. pass else: if issuenzb is None or 'S' in sandwich or int(sandwich) >= 900000: # this has no issueID, therefore it's a one-off or a manual post-proc. # At this point, let's just drop it into the Comic Location folder and forget about it.. if 'S' in sandwich: self._log("One-off STORYARC mode enabled for Post-Processing for " + str(sarc)) logger.info("One-off STORYARC mode enabled for Post-Processing for " + str(sarc)) if mylar.STORYARCDIR: storyarcd = os.path.join(mylar.DESTINATION_DIR, "StoryArcs", sarc) self._log("StoryArc Directory set to : " + storyarcd, logger.DEBUG) else: self._log("Grab-Bag Directory set to : " + mylar.GRABBAG_DIR, logger.DEBUG) else: self._log("One-off mode enabled for Post-Processing. All I'm doing is moving the file untouched into the Grab-bag directory.", logger.DEBUG) logger.info("One-off mode enabled for Post-Processing. Will move into Grab-bag directory.") self._log("Grab-Bag Directory set to : " + mylar.GRABBAG_DIR, logger.DEBUG) for root, dirnames, filenames in os.walk(self.nzb_folder): for filename in filenames: if filename.lower().endswith(extensions): ofilename = filename path, ext = os.path.splitext(ofilename) if 'S' in sandwich: if mylar.STORYARCDIR: grdst = storyarcd else: grdst = mylar.DESTINATION_DIR else: if mylar.GRABBAG_DIR: grdst = mylar.GRABBAG_DIR else: grdst = mylar.DESTINATION_DIR filechecker.validateAndCreateDirectory(grdst, True) grab_dst = os.path.join(grdst, ofilename) self._log("Destination Path : " + grab_dst, logger.DEBUG) logger.info("Destination Path : " + grab_dst) grab_src = os.path.join(self.nzb_folder, ofilename) self._log("Source Path : " + grab_src, logger.DEBUG) logger.info("Source Path : " + grab_src) logger.info("Moving " + str(ofilename) + " into directory : " + str(grdst)) try: shutil.move(grab_src, grab_dst) except (OSError, IOError): self._log("Failed to move directory - check directories and manually re-run.", logger.DEBUG) logger.debug("Failed to move directory - check directories and manually re-run.") return #tidyup old path try: shutil.rmtree(self.nzb_folder) except (OSError, IOError): self._log("Failed to remove temporary directory.", logger.DEBUG) logger.debug("Failed to remove temporary directory - check directory and manually re-run.") return logger.debug("Removed temporary directory : " + str(self.nzb_folder)) self._log("Removed temporary directory : " + self.nzb_folder, logger.DEBUG) #delete entry from nzblog table myDB.action('DELETE from nzblog WHERE issueid=?', [issueid]) if 'S' in issueid: issuearcid = re.sub('S', '', issueid) logger.info("IssueArcID is : " + str(issuearcid)) ctrlVal = {"IssueArcID": issuearcid} newVal = {"Status": "Downloaded", "Location": grab_dst } myDB.upsert("readinglist",newVal,ctrlVal) logger.info("updated status to Downloaded") return self.log comicid = issuenzb['ComicID'] issuenumOG = issuenzb['Issue_Number'] #issueno = str(issuenum).split('.')[0] #new CV API - removed all decimals...here we go AGAIN! issuenum = issuenumOG issue_except = 'None' if 'au' in issuenum.lower(): issuenum = re.sub("[^0-9]", "", issuenum) issue_except = ' AU' if '.' in issuenum: iss_find = issuenum.find('.') iss_b4dec = issuenum[:iss_find] iss_decval = issuenum[iss_find+1:] if int(iss_decval) == 0: iss = iss_b4dec issdec = int(iss_decval) issueno = str(iss) self._log("Issue Number: " + str(issueno), logger.DEBUG) logger.fdebug("Issue Number: " + str(issueno)) else: if len(iss_decval) == 1: iss = iss_b4dec + "." + iss_decval issdec = int(iss_decval) * 10 else: iss = iss_b4dec + "." + iss_decval.rstrip('0') issdec = int(iss_decval.rstrip('0')) * 10 issueno = iss_b4dec self._log("Issue Number: " + str(iss), logger.DEBUG) logger.fdebug("Issue Number: " + str(iss)) else: iss = issuenum issueno = str(iss) # issue zero-suppression here if mylar.ZERO_LEVEL == "0": zeroadd = "" else: if mylar.ZERO_LEVEL_N == "none": zeroadd = "" elif mylar.ZERO_LEVEL_N == "0x": zeroadd = "0" elif mylar.ZERO_LEVEL_N == "00x": zeroadd = "00" logger.fdebug("Zero Suppression set to : " + str(mylar.ZERO_LEVEL_N)) if str(len(issueno)) > 1: if int(issueno) < 10: self._log("issue detected less than 10", logger.DEBUG) if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(zeroadd) + str(iss) else: prettycomiss = str(zeroadd) + str(int(issueno)) else: prettycomiss = str(zeroadd) + str(iss) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG) elif int(issueno) >= 10 and int(issueno) < 100: self._log("issue detected greater than 10, but less than 100", logger.DEBUG) if mylar.ZERO_LEVEL_N == "none": zeroadd = "" else: zeroadd = "0" if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(zeroadd) + str(iss) else: prettycomiss = str(zeroadd) + str(int(issueno)) else: prettycomiss = str(zeroadd) + str(iss) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ".Issue will be set as : " + str(prettycomiss), logger.DEBUG) else: self._log("issue detected greater than 100", logger.DEBUG) if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(issueno) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except self._log("Zero level supplement set to " + str(mylar.ZERO_LEVEL_N) + ". Issue will be set as : " + str(prettycomiss), logger.DEBUG) else: prettycomiss = str(issueno) self._log("issue length error - cannot determine length. Defaulting to None: " + str(prettycomiss), logger.DEBUG) logger.fdebug("Pretty Comic Issue is : " + str(prettycomiss)) issueyear = issuenzb['IssueDate'][:4] self._log("Issue Year: " + str(issueyear), logger.DEBUG) logger.fdebug("Issue Year : " + str(issueyear)) comicnzb= myDB.action("SELECT * from comics WHERE comicid=?", [comicid]).fetchone() publisher = comicnzb['ComicPublisher'] self._log("Publisher: " + publisher, logger.DEBUG) logger.fdebug("Publisher: " + str(publisher)) #we need to un-unicode this to make sure we can write the filenames properly for spec.chars series = comicnzb['ComicName'].encode('ascii', 'ignore').strip() self._log("Series: " + series, logger.DEBUG) logger.fdebug("Series: " + str(series)) seriesyear = comicnzb['ComicYear'] self._log("Year: " + seriesyear, logger.DEBUG) logger.fdebug("Year: " + str(seriesyear)) comlocation = comicnzb['ComicLocation'] self._log("Comic Location: " + comlocation, logger.DEBUG) logger.fdebug("Comic Location: " + str(comlocation)) comversion = comicnzb['ComicVersion'] self._log("Comic Version: " + str(comversion), logger.DEBUG) logger.fdebug("Comic Version: " + str(comversion)) if comversion is None: comversion = 'None' #if comversion is None, remove it so it doesn't populate with 'None' if comversion == 'None': chunk_f_f = re.sub('\$VolumeN','',mylar.FILE_FORMAT) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) self._log("No version # found for series - tag will not be available for renaming.", logger.DEBUG) logger.fdebug("No version # found for series, removing from filename") logger.fdebug("new format is now: " + str(chunk_file_format)) else: chunk_file_format = mylar.FILE_FORMAT #Run Pre-script if mylar.ENABLE_PRE_SCRIPTS: nzbn = self.nzb_name #original nzb name nzbf = self.nzb_folder #original nzb folder #name, comicyear, comicid , issueid, issueyear, issue, publisher #create the dic and send it. seriesmeta = [] seriesmetadata = {} seriesmeta.append({ 'name': series, 'comicyear': seriesyear, 'comicid': comicid, 'issueid': issueid, 'issueyear': issueyear, 'issue': issuenum, 'publisher': publisher }) seriesmetadata['seriesmeta'] = seriesmeta self._run_pre_scripts(nzbn, nzbf, seriesmetadata ) #rename file and move to new path #nfilename = series + " " + issueno + " (" + seriesyear + ")" file_values = {'$Series': series, '$Issue': prettycomiss, '$Year': issueyear, '$series': series.lower(), '$Publisher': publisher, '$publisher': publisher.lower(), '$VolumeY': 'V' + str(seriesyear), '$VolumeN': comversion } ofilename = None for root, dirnames, filenames in os.walk(self.nzb_folder): for filename in filenames: if filename.lower().endswith(extensions): ofilename = filename path, ext = os.path.splitext(ofilename) if ofilename is None: logger.error(u"Aborting PostProcessing - the filename doesn't exist in the location given. Make sure that " + str(self.nzb_folder) + " exists and is the correct location.") return self._log("Original Filename: " + ofilename, logger.DEBUG) self._log("Original Extension: " + ext, logger.DEBUG) logger.fdebug("Original Filname: " + str(ofilename)) logger.fdebug("Original Extension: " + str(ext)) if mylar.FILE_FORMAT == '' or not mylar.RENAME_FILES: self._log("Rename Files isn't enabled...keeping original filename.", logger.DEBUG) logger.fdebug("Rename Files isn't enabled - keeping original filename.") #check if extension is in nzb_name - will screw up otherwise if ofilename.lower().endswith(extensions): nfilename = ofilename[:-4] else: nfilename = ofilename else: nfilename = helpers.replace_all(chunk_file_format, file_values) if mylar.REPLACE_SPACES: #mylar.REPLACE_CHAR ...determines what to replace spaces with underscore or dot nfilename = nfilename.replace(' ', mylar.REPLACE_CHAR) nfilename = re.sub('[\,\:\?]', '', nfilename) self._log("New Filename: " + nfilename, logger.DEBUG) logger.fdebug("New Filename: " + str(nfilename)) src = os.path.join(self.nzb_folder, ofilename) filechecker.validateAndCreateDirectory(comlocation, True) if mylar.LOWERCASE_FILENAMES: dst = (comlocation + "/" + nfilename + ext).lower() else: dst = comlocation + "/" + nfilename + ext.lower() self._log("Source:" + src, logger.DEBUG) self._log("Destination:" + dst, logger.DEBUG) logger.fdebug("Source: " + str(src)) logger.fdebug("Destination: " + str(dst)) os.rename(os.path.join(self.nzb_folder, str(ofilename)), os.path.join(self.nzb_folder,str(nfilename + ext))) src = os.path.join(self.nzb_folder, str(nfilename + ext)) try: shutil.move(src, dst) except (OSError, IOError): self._log("Failed to move directory - check directories and manually re-run.", logger.DEBUG) self._log("Post-Processing ABORTED.", logger.DEBUG) return #tidyup old path try: shutil.rmtree(self.nzb_folder) except (OSError, IOError): self._log("Failed to remove temporary directory - check directory and manually re-run.", logger.DEBUG) self._log("Post-Processing ABORTED.", logger.DEBUG) return self._log("Removed temporary directory : " + str(self.nzb_folder), logger.DEBUG) #delete entry from nzblog table myDB.action('DELETE from nzblog WHERE issueid=?', [issueid]) #update snatched table to change status to Downloaded updater.foundsearch(comicid, issueid, down='True') #force rescan of files updater.forceRescan(comicid) logger.info(u"Post-Processing completed for: " + series + " issue: " + str(issuenumOG) ) self._log(u"Post Processing SUCCESSFULL! ", logger.DEBUG) if mylar.PROWL_ENABLED: pushmessage = series + '(' + issueyear + ') - issue #' + issuenumOG logger.info(u"Prowl request") prowl = notifiers.PROWL() prowl.notify(pushmessage,"Download and Postprocessing completed") if mylar.NMA_ENABLED: nma = notifiers.NMA() nma.notify(series, str(issueyear), str(issuenumOG)) if mylar.PUSHOVER_ENABLED: pushmessage = series + ' (' + str(issueyear) + ') - issue #' + str(issuenumOG) logger.info(u"Pushover request") pushover = notifiers.PUSHOVER() pushover.notify(pushmessage, "Download and Post-Processing completed") # retrieve/create the corresponding comic objects if mylar.ENABLE_EXTRA_SCRIPTS: folderp = str(dst) #folder location after move/rename nzbn = self.nzb_name #original nzb name filen = str(nfilename + ext) #new filename #name, comicyear, comicid , issueid, issueyear, issue, publisher #create the dic and send it. seriesmeta = [] seriesmetadata = {} seriesmeta.append({ 'name': series, 'comicyear': seriesyear, 'comicid': comicid, 'issueid': issueid, 'issueyear': issueyear, 'issue': issuenum, 'publisher': publisher }) seriesmetadata['seriesmeta'] = seriesmeta self._run_extra_scripts(nzbname, self.nzb_folder, filen, folderp, seriesmetadata ) return self.log
def run(self): logger.info( '[WEEKLY] Checking Weekly Pull-list for new releases/updates') mylar.weeklypull.pullit() return
def nzbdbsearch(seriesname, issue, comicid=None, nzbprov=None, searchYear=None, ComicVersion=None): myDB = db.DBConnection() seriesname_alt = None if comicid is None or comicid == 'None': pass else: snm = myDB.selectone("SELECT * FROM comics WHERE comicid=?", [comicid]).fetchone() if snm is None: logger.info('Invalid ComicID of ' + str(comicid) + '. Aborting search.') return else: seriesname = snm['ComicName'] seriesname_alt = snm['AlternateSearch'] nsearch_seriesname = re.sub('[\'\!\@\#\$\%\:\;\/\\=\?\.\-\s]', '%', seriesname) formatrem_seriesname = re.sub('[\'\!\@\#\$\%\:\;\/\\=\?\.]', '', seriesname) nsearch = '%' + nsearch_seriesname + "%" nresults = myDB.select("SELECT * FROM rssdb WHERE Title like ? AND Site=?", [nsearch, nzbprov]) if nresults is None: logger.fdebug('nzb search returned no results for ' + seriesname) if seriesname_alt is None: logger.fdebug('no nzb Alternate name given. Aborting search.') return "no results" else: chkthealt = seriesname_alt.split('##') if chkthealt == 0: AS_Alternate = AlternateSearch for calt in chkthealt: AS_Alternate = re.sub('##', '', calt) nresults += myDB.select( "SELECT * FROM rssdb WHERE Title like ? AND Site=?", [AS_Alternate, nzbprov]) if nresults is None: logger.fdebug('nzb alternate name search returned no results.') return "no results" nzbtheinfo = [] nzbinfo = {} if nzbprov == 'experimental': except_list = [ 'releases', 'gold line', 'distribution', '0-day', '0 day' ] if ComicVersion: ComVersChk = re.sub("[^0-9]", "", ComicVersion) if ComVersChk == '': ComVersChk = 0 else: ComVersChk = 0 else: ComVersChk = 0 for results in nresults: title = results['Title'] #logger.fdebug("titlesplit: " + str(title.split("\""))) splitTitle = title.split("\"") noYear = 'False' for subs in splitTitle: #logger.fdebug(subs) if len(subs) > 10 and not any(d in subs.lower() for d in except_list): if ComVersChk == 0: noYear = 'False' if ComVersChk != 0 and searchYear not in subs: noYear = 'True' noYearline = subs if searchYear in subs and noYear == 'True': #this would occur on the next check in the line, if year exists and #the noYear check in the first check came back valid append it subs = noYearline + ' (' + searchYear + ')' noYear = 'False' if noYear == 'False': nzbtheinfo.append({ 'title': subs, 'link': re.sub('\/release\/', '/download/', results['Link']), 'pubdate': str(results['PubDate']), 'site': str(results['Site']), 'length': str(results['Size']) }) else: for nzb in nresults: # no need to parse here, just compile and throw it back .... nzbtheinfo.append({ 'title': nzb['Title'], 'link': nzb['Link'], 'pubdate': nzb['Pubdate'], 'site': nzb['Site'], 'length': nzb['Size'] }) #logger.fdebug("entered info for " + nzb['Title']) nzbinfo['entries'] = nzbtheinfo return nzbinfo
def nzbs(provider=None): nzbprovider = [] nzbp = 0 if mylar.NZBSU == 1: nzbprovider.append('nzb.su') nzbp += 1 if mylar.DOGNZB == 1: nzbprovider.append('dognzb') nzbp += 1 # -------- # Xperimental if mylar.EXPERIMENTAL == 1: nzbprovider.append('experimental') nzbp += 1 newznabs = 0 newznab_hosts = [] if mylar.NEWZNAB == 1: for newznab_host in mylar.EXTRA_NEWZNABS: if newznab_host[4] == '1' or newznab_host[4] == 1: newznab_hosts.append(newznab_host) nzbprovider.append('newznab') newznabs += 1 logger.fdebug('newznab name:' + str(newznab_host[0]) + ' - enabled: ' + str(newznab_host[4])) # -------- providercount = int(nzbp + newznabs) logger.fdebug('there are : ' + str(providercount) + ' nzb RSS search providers you have enabled.') nzbpr = providercount - 1 if nzbpr < 0: nzbpr == 0 feeddata = [] feedthis = [] ft = 0 totNum = 0 nonexp = "no" while (nzbpr >= 0): if nzbprovider[nzbpr] == 'experimental': feed = feedparser.parse( "http://nzbindex.nl/rss/alt.binaries.comics.dcp/?sort=agedesc&max=50&more=1" ) totNum = len(feed.entries) site = 'experimental' keyPair = {} regList = [] entries = [] mres = {} countUp = 0 i = 0 for entry in feed['entries']: tmpsz = feed.entries[i].enclosures[0] feeddata.append({ 'Site': site, 'Title': feed.entries[i].title, 'Link': tmpsz['url'], #feed.entries[i].link, 'Pubdate': feed.entries[i].updated, 'Size': tmpsz['length'] }) # print ("Site:" + str(site)) # print ("Title:" + str(feed.entries[i].title)) # print ("Link:" + str(feed.entries[i].link)) # print ("Pubdate:" + str(feed.entries[i].updated)) # print ("Size:" + str(tmpsz['length'])) i += 1 logger.info(str(i) + ' results from Experimental feed indexed.') nzbpr -= 1 else: if nzbprovider[nzbpr] == 'newznab': for newznab_host in newznab_hosts: if newznab_host[3] is None: newznabuid = '1' newznabcat = '7030' else: if '#' not in newznab_host[3]: newznabuid = newznab_host[3] newznabcat = '7030' else: newzst = newznab_host[3].find('#') newznabuid = newznab_host[3][:newzst] newznabcat = newznab_host[3][newzst + 1:] feed = newznab_host[1].rstrip() + '/rss?t=' + str( newznabcat) + '&dl=1&i=' + str( newznabuid) + '&r=' + newznab_host[2].rstrip() feedme = feedparser.parse(feed) site = newznab_host[0].rstrip() feedthis.append({"feed": feedme, "site": site}) totNum += len(feedme.entries) ft += 1 nonexp = "yes" nzbpr -= 1 elif nzbprovider[nzbpr] == 'nzb.su': if mylar.NZBSU_UID is None: mylar.NZBSU_UID = '1' feed = 'http://api.nzb.su/rss?t=7030&dl=1&i=' + mylar.NZBSU_UID + '&r=' + mylar.NZBSU_APIKEY feedme = feedparser.parse(feed) site = nzbprovider[nzbpr] feedthis.append({"feed": feedme, "site": site}) totNum += len(feedme.entries) ft += 1 nonexp = "yes" nzbpr -= 1 elif nzbprovider[nzbpr] == 'dognzb': if mylar.DOGNZB_UID is None: mylar.DOGNZB_UID = '1' feed = 'https://dognzb.cr/rss.cfm?r=' + mylar.DOGNZB_APIKEY + '&t=7030' feedme = feedparser.parse(feed) site = nzbprovider[nzbpr] ft += 1 nonexp = "yes" feedthis.append({"feed": feedme, "site": site}) totNum += len(feedme.entries) nzbpr -= 1 i = 0 if nonexp == "yes": #print str(ft) + " sites checked. There are " + str(totNum) + " entries to be updated." #print feedme for ft in feedthis: sitei = 0 site = ft['site'] logger.fdebug(str(site) + " now being updated...") #logger.fdebug('feedthis:' + str(ft)) for entry in ft['feed'].entries: if site == 'dognzb': #because the rss of dog doesn't carry the enclosure item, we'll use the newznab size value tmpsz = 0 #for attr in entry['newznab:attrib']: # if attr('@name') == 'size': # tmpsz = attr['@value'] # logger.fdebug('size retrieved as ' + str(tmpsz)) # break feeddata.append({ 'Site': site, 'Title': entry.title, #ft['feed'].entries[i].title, 'Link': entry.link, #ft['feed'].entries[i].link, 'Pubdate': entry.updated, #ft['feed'].entries[i].updated, 'Size': tmpsz }) else: #this should work for all newznabs (nzb.su included) #only difference is the size of the file between this and above (which is probably the same) tmpsz = entry.enclosures[ 0] #ft['feed'].entries[i].enclosures[0] feeddata.append({ 'Site': site, 'Title': entry.title, #ft['feed'].entries[i].title, 'Link': entry.link, #ft['feed'].entries[i].link, 'Pubdate': entry.updated, #ft['feed'].entries[i].updated, 'Size': tmpsz['length'] }) #logger.fdebug("Site: " + str(feeddata[i]['Site'])) #logger.fdebug("Title: " + str(feeddata[i]['Title'])) #logger.fdebug("Link: " + str(feeddata[i]['Link'])) #logger.fdebug("pubdate: " + str(feeddata[i]['Pubdate'])) #logger.fdebug("size: " + str(feeddata[i]['Size'])) sitei += 1 logger.info('[' + str(site) + '] ' + str(sitei) + ' entries indexed.') i += sitei if i > 0: logger.info( '[RSS] ' + str(i) + ' entries have been indexed and are now going to be stored for caching.' ) rssdbupdate(feeddata, i, 'usenet') return
def tehMain(forcerss=None): logger.info('RSS Feed Check was last run at : ' + str(mylar.RSS_LASTRUN)) firstrun = "no" #check the last run of rss to make sure it's not hammering. if mylar.RSS_LASTRUN is None or mylar.RSS_LASTRUN == '' or mylar.RSS_LASTRUN == '0' or forcerss == True: logger.info('RSS Feed Check First Ever Run.') firstrun = "yes" mins = 0 else: c_obj_date = datetime.datetime.strptime(mylar.RSS_LASTRUN, "%Y-%m-%d %H:%M:%S") n_date = datetime.datetime.now() absdiff = abs(n_date - c_obj_date) mins = (absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 60.0 #3600 is for hours. if firstrun == "no" and mins < int(mylar.RSS_CHECKINTERVAL): logger.fdebug( 'RSS Check has taken place less than the threshold - not initiating at this time.' ) return mylar.RSS_LASTRUN = helpers.now() logger.fdebug('Updating RSS Run time to : ' + str(mylar.RSS_LASTRUN)) mylar.config_write() #function for looping through nzbs/torrent feed if mylar.ENABLE_TORRENTS: logger.info('[RSS] Initiating Torrent RSS Check.') if mylar.ENABLE_KAT: logger.info('[RSS] Initiating Torrent RSS Feed Check on KAT.') torrents(pickfeed='3') torrents(pickfeed='6') if mylar.ENABLE_CBT: logger.info('[RSS] Initiating Torrent RSS Feed Check on CBT.') torrents(pickfeed='1') torrents(pickfeed='4') logger.info('[RSS] Initiating RSS Feed Check for NZB Providers.') nzbs() logger.info('[RSS] RSS Feed Check/Update Complete') logger.info('[RSS] Watchlist Check for new Releases') mylar.search.searchforissue(rsscheck='yes') logger.info('[RSS] Watchlist Check complete.') if forcerss: logger.info('Successfully ran RSS Force Check.') return
def NZB_SEARCH(ComicName, IssueNumber, ComicYear, SeriesYear, nzbprov, nzbpr): logger.info(u"Shhh be very quiet...I'm looking for " + ComicName + " issue: " + str(IssueNumber) + " using " + str(nzbprov)) if nzbprov == 'nzb.su': apikey = mylar.NZBSU_APIKEY elif nzbprov == 'dognzb': apikey = mylar.DOGNZB_APIKEY elif nzbprov == 'experimental': apikey = 'none' #print ("-------------------------") if mylar.PREFERRED_QUALITY == 0: filetype = "" elif mylar.PREFERRED_QUALITY == 1: filetype = ".cbr" elif mylar.PREFERRED_QUALITY == 2: filetype = ".cbz" # figure out what was missed via rss feeds and do a manual search via api #tsc = int(tot-1) findcomic = [] findcomiciss = [] findcount = 0 ci = "" comsearch = [] isssearch = [] comyear = str(ComicYear) #print ("-------SEARCH FOR MISSING------------------") findcomic.append(str(ComicName)) IssueNumber = str(re.sub("\.00", "", str(IssueNumber))) #print ("issueNumber" + str(IssueNumber)) findcomiciss.append(str(re.sub("\D", "", str(IssueNumber)))) #print ("we need : " + str(findcomic[findcount]) + " issue: #" + str(findcomiciss[findcount])) # replace whitespace in comic name with %20 for api search cm = re.sub(" ", "%20", str(findcomic[findcount])) #print (cmi) if len(str(findcomiciss[findcount])) == 1: cmloopit = 3 elif len(str(findcomiciss[findcount])) == 2: cmloopit = 2 else: cmloopit = 1 isssearch.append(str(findcomiciss[findcount])) comsearch.append(cm) findcount+=1 # ---- #print ("------RESULTS OF SEARCH-------------------") findloop = 0 foundcomic = [] #---issue problem # if issue is '011' instead of '11' in nzb search results, will not have same # results. '011' will return different than '11', as will '009' and '09'. while (findloop < (findcount) ): comsrc = comsearch[findloop] #print (str(comsearch[findloop])) while (cmloopit >= 1 ): # here we account for issue pattern variations if cmloopit == 3: comsearch[findloop] = comsrc + "%2000" + isssearch[findloop] + "%20" + str(filetype) #print (comsearch[findloop]) elif cmloopit == 2: comsearch[findloop] = comsrc + "%200" + isssearch[findloop] + "%20" + str(filetype) #print (comsearch[findloop]) elif cmloopit == 1: comsearch[findloop] = comsrc + "%20" + isssearch[findloop] + "%20" + str(filetype) #print (comsearch[findloop]) #print ("NZB Provider set to: " + nzbprov) if nzbprov != 'experimental': if nzbprov == 'dognzb': #print ("dog-search.") findurl = "http://dognzb.cr/api?t=search&apikey=" + str(apikey) + "&q=" + str(comsearch[findloop]) + "&o=xml&cat=7030" elif nzbprov == 'nzb.su': #print ("nzb.su search") findurl = "http://nzb.su/api?t=search&q=" + str(comsearch[findloop]) + "&apikey=" + str(apikey) + "&o=xml&cat=7030" bb = feedparser.parse(findurl) #print (findurl) elif nzbprov == 'experimental': #print ("experimental raw search") bb = parseit.MysterBinScrape(comsearch[findloop], comyear) done = False foundc = "no" if bb == "no results": #print ("no results found...attempting alternate search") pass elif (len(bb['entries']) == 0): #print ("Nothing found for : " + str(findcomic[findloop]) + " Issue: #" + str(findcomiciss[findloop])) foundc = "no" else: #print ("Found for: " + str(findcomic[findloop])) for entry in bb['entries']: #print str(entry['title']) cleantitle = helpers.cleanName(str(entry['title'])) if done: break #print ("title: " + str(cleantitle)) #print ("link: " + entry['link']) #let's narrow search down - take out year (2010), (2011), etc #let's check for first occurance of '(' as generally indicates #that the 'title' has ended ripperlist=['digital-', 'empire', 'dcp'] #this takes care of the brackets :) # m = re.findall(r"\((\w+)\)", cleantitle) m = re.findall('[^()]+', cleantitle) lenm = len(m) #print ("there are " + str(lenm) + " words.") cnt = 0 while (cnt < lenm): if m[cnt] is None: break #if m[cnt] == ' ': print ("space detected") #print (str(cnt) + ". Bracket Word: " + m[cnt] ) if cnt == 0: comic_andiss = m[cnt] print ("Comic:" + str(comic_andiss)) if m[cnt][:-2] == '19' or m[cnt][:-2] == '20': print ("year detected!") result_comyear = m[cnt] if str(comyear) in result_comyear: print (str(comyear) + " - right - years match baby!") yearmatch = "true" else: print (str(comyear) + " - not right - years don't match ") yearmatch = "false" if 'digital' in m[cnt] and len(m[cnt]) == 7: pass #print ("digital edition") if ' of ' in m[cnt]: #print ("mini-series detected : " + str(m[cnt])) result_of = m[cnt] if 'cover' in m[cnt]: #print ("covers detected") result_comcovers = m[cnt] for ripper in ripperlist: if ripper in m[cnt]: #print ("Scanner detected:" + str(m[cnt])) result_comscanner = m[cnt] cnt+=1 if yearmatch == "false": break splitit = [] watchcomic_split = [] comic_iss = re.sub('[\-\:\,]', '', str(comic_andiss)) splitit = comic_iss.split(None) watchcomic_split = findcomic[findloop].split(None) bmm = re.findall('v\d', comic_iss) #print ("vers - " + str(bmm)) if len(bmm) > 0: splitst = len(splitit) - 2 else: splitst = len(splitit) - 1 if (splitst) != len(watchcomic_split): print ("incorrect comic lengths...not a match") if str(splitit[0]).lower() == "the": print ("THE word detected...attempting to adjust pattern matching") splitit[0] = splitit[4:] else: print ("length match..proceeding") n = 0 scount = 0 #print ("length:" + str(len(splitit))) while ( n <= len(splitit)-1 ): if n < len(splitit)-1: #print ( str(n) + ". Comparing: " + watchcomic_split[n] + " .to. " + splitit[n] ) if str(watchcomic_split[n].lower()) in str(splitit[n].lower()): #print ("word matched on : " + splitit[n]) scount+=1 #elif ':' in splitit[n] or '-' in splitit[n]: # splitrep = splitit[n].replace('-', '') # print ("non-character keyword...skipped on " + splitit[n]) elif len(splitit[n]) < 3 or (splitit[n][1:]) == "v": #print ("possible verisoning..checking") #we hit a versioning # - account for it if splitit[n][2:].isdigit(): comicversion = str(splitit[n]) #print ("version found:" + str(comicversion)) else: if splitit[n].isdigit(): print ("issue detected") comiss = splitit[n] comicNAMER = n - 1 comNAME = splitit[0] cmnam = 1 while (cmnam < comicNAMER): comNAME = str(comNAME) + " " + str(splitit[cmnam]) cmnam+=1 #print ("comic: " + str(comNAME)) else: #print ("non-match for: " + splitit[n]) pass n+=1 spercent = ( scount/int(len(splitit)) ) * 100 #print (str(spercent) + "% match") #if spercent >= 75: print ("it's a go captain...") #if spercent < 75: print ("failure - we only got " + str(spercent) + "% right!") print ("this should be a match!") #issue comparison now as well if int(findcomiciss[findloop]) == int(comiss): print ("issues match!") ## -- inherit issue. Comic year is non-standard. nzb year is the year ## -- comic was printed, not the start year of the comic series and ## -- thus the deciding component if matches are correct or not linkstart = os.path.splitext(entry['link'])[0] #following is JUST for nzb.su if nzbprov == 'nzb.su': linkit = os.path.splitext(entry['link'])[1] #print ("linkit: " + str(linkit)) linkit = linkit.replace("&", "%26") linkapi = str(linkstart) + str(linkit) else: # this should work for every other provider linkstart = linkstart.replace("&", "%26") linkapi = str(linkstart) #here we distinguish between rename and not. #blackhole functinality--- #let's download the file to a temporary cache. if mylar.BLACKHOLE: if os.path.exists(mylar.BLACKHOLE_DIR): filenamenzb = str(ComicName) + " " + str(IssueNumber) + " (" + str(comyear) + ").nzb" urllib.urlretrieve(linkapi, str(mylar.BLACKHOLE_DIR) + str(filenamenzb)) logger.info(u"Successfully sent .nzb to your Blackhole directory : " + str(mylar.BLACKHOLE_DIR) + str(filenamenzb) ) #end blackhole else: tmppath = mylar.CACHE_DIR print ("cache directory set to: " + str(tmppath)) if os.path.exists(tmppath): filenamenzb = os.path.split(linkapi)[1] #filenzb = os.path.join(tmppath,filenamenzb) if nzbprov == 'nzb.su': filenzb = linkstart[21:] if nzbprov == 'experimental': filenzb = filenamenzb[6:] if nzbprov == 'dognzb': filenzb == str(filenamenzb) savefile = str(tmppath) + "/" + str(filenzb) + ".nzb" else: #let's make the dir. try: os.makedirs(str(mylar.CACHE_DIR)) logger.info(u"Cache Directory successfully created at: " + str(mylar.CACHE_DIR)) savefile = str(mylar.CACHE_DIR) + "/" + str(filenzb) + ".nzb" except OSError.e: if e.errno != errno.EEXIST: raise print ("savefile set to: " + str(savefile)) urllib.urlretrieve(linkapi, str(savefile)) #print (str(mylar.RENAME_FILES)) print ("sucessfully retrieve nzb to : " + str(savefile)) #check sab for current pause status print ("sab host set to :" + str(mylar.SAB_HOST)) sabqstatusapi = str(mylar.SAB_HOST) + "/api?mode=qstatus&output=xml&apikey=" + str(mylar.SAB_APIKEY) from xml.dom.minidom import parseString import urllib2 file = urllib2.urlopen(sabqstatusapi); data = file.read() file.close() dom = parseString(data) for node in dom.getElementsByTagName('paused'): pausestatus = node.firstChild.wholeText #print pausestatus if pausestatus != 'True': #pause sab first because it downloads too quick (cbr's are small!) pauseapi = str(mylar.SAB_HOST) + "/api?mode=pause&apikey=" + str(mylar.SAB_APIKEY) urllib.urlopen(pauseapi); print "Queue paused" else: print "Queue already paused" if mylar.RENAME_FILES == 1: #print ("Saved file to: " + str(savefile)) tmpapi = str(mylar.SAB_HOST) + "/api?mode=addlocalfile&name=" + str(savefile) + "&pp=3&cat=" + str(mylar.SAB_CATEGORY) + "&script=ComicRN.py&apikey=" + str(mylar.SAB_APIKEY) else: tmpapi = str(mylar.SAB_HOST) + "/api?mode=addurl&name=" + str(linkapi) + "&pp=3&cat=" + str(mylar.SAB_CATEGORY) + "&script=ComicRN.py&apikey=" + str(mylar.SAB_APIKEY) print ("sab api string:" + str(tmpapi)) time.sleep(5) urllib.urlopen(tmpapi); if mylar.RENAME_FILES == 1: #let's give it 5 extra seconds to retrieve the nzb data... time.sleep(5) outqueue = str(mylar.SAB_HOST) + "/api?mode=queue&start=START&limit=LIMIT&output=xml&apikey=" + str(mylar.SAB_APIKEY) print ("outqueue line generated") urllib.urlopen(outqueue); time.sleep(5) print ("passed api request to SAB") #<slots><slot><filename>.nzb filename #chang nzbfilename to include series(SAB will auto rename based on this) #api?mode=queue&name=rename&value=<filename_nzi22ks>&value2=NEWNAME from xml.dom.minidom import parseString import urllib2 file = urllib2.urlopen(outqueue); data = file.read() file.close() dom = parseString(data) queue_slots = dom.getElementsByTagName('filename') queue_cnt = len(queue_slots) print ("there are " + str(queue_cnt) + " things in SABnzbd's queue") que = 0 slotmatch = "no" for queue in queue_slots: #retrieve the first xml tag (<tag>data</tag>) #that the parser finds with name tagName: queue_file = dom.getElementsByTagName('filename')[que].firstChild.wholeText while ('Trying to fetch NZB' in queue_file): #let's keep waiting until nzbname is resolved by SABnzbd time.sleep(5) file = urllib2.urlopen(outqueue); data = file.read() file.close() dom = parseString(data) queue_file = dom.getElementsByTagName('filename')[que].firstChild.wholeText print (str(queue_file)) print (str(filenzb)) queue_file = queue_file.replace("_", " ") if str(queue_file) in str(filenzb): print ("matched") slotmatch = "yes" slot_nzoid = dom.getElementsByTagName('nzo_id')[que].firstChild.wholeText print ("slot_nzoid: " + str(slot_nzoid)) break que+=1 if slotmatch == "yes": if mylar.REPLACE_SPACES: repchar = mylar.REPLACE_CHAR else: repchar = ' ' #let's make sure there's no crap in the ComicName since it's O.G. ComicNM = re.sub('[\:\,]', '', str(ComicName)) renameit = str(ComicNM) + " " + str(IssueNumber) + " (" + str(SeriesYear) + ")" + " " + "(" + str(comyear) + ")" renameit = renameit.replace(' ', repchar) nzo_ren = str(mylar.SAB_HOST) + "/api?mode=queue&name=rename&apikey=" + str(mylar.SAB_APIKEY) + "&value=" + str(slot_nzoid) + "&value2=" + str(renameit) print ("attempting to rename queue to " + str(nzo_ren)) urllib2.urlopen(nzo_ren); print ("renamed!") #delete the .nzb now. #delnzb = str(mylar.PROG_DIR) + "/" + str(filenzb) + ".nzb" #if mylar.PROG_DIR is not "/": #os.remove(delnzb) #we need to track nzo_id to make sure finished downloaded with SABnzbd. #controlValueDict = {"nzo_id": str(slot_nzoid)} #newValueDict = {"ComicName": str(ComicName), # "ComicYEAR": str(comyear), # "ComicIssue": str(IssueNumber), # "name": str(filenamenzb)} #print ("updating SABLOG") #myDB = db.DBConnection() #myDB.upsert("sablog", newValueDict, controlValueDict) else: logger.info(u"Couldn't locate file in SAB - are you sure it's being downloaded?") #resume sab if it was running before we started if pausestatus != 'True': #let's unpause queue now that we did our jobs. resumeapi = str(mylar.SAB_HOST) + "/api?mode=resume&apikey=" + str(mylar.SAB_APIKEY) urllib.urlopen(resumeapi); #print "Queue resumed" #else: #print "Queue already paused" #raise an exception to break out of loop foundc = "yes" done = True break else: #print ("issues don't match..") foundc = "no" if done == True: break cmloopit-=1 findloop+=1 if foundc == "yes": foundcomic.append("yes") logger.info(u"Found :" + str(ComicName) + " (" + str(comyear) + ") issue: " + str(IssueNumber) + " using " + str(nzbprov)) break elif foundc == "no" and nzbpr <> 0: logger.info(u"More than one search provider given - trying next one.") elif foundc == "no" and nzbpr == 0: foundcomic.append("no") logger.info(u"Couldn't find Issue " + str(IssueNumber) + " of " + str(ComicName) + "(" + str(comyear) + "). Status kept as wanted." ) break return foundc
def rename_file(self, ofilename, issue=None, annualize=None, arc=False, file_format=None ): #comicname, issue, comicyear=None, issueid=None) comicid = self.comicid # it's coming in unicoded... issueid = self.issueid if file_format is None: file_format = mylar.CONFIG.FILE_FORMAT logger.fdebug(type(comicid)) logger.fdebug(type(issueid)) logger.fdebug('comicid: %s' % comicid) logger.fdebug('issue# as per cv: %s' % issue) logger.fdebug('issueid:' + str(issueid)) if issueid is None: logger.fdebug('annualize is ' + str(annualize)) if arc: #this has to be adjusted to be able to include story arc issues that span multiple arcs chkissue = self.myDB.selectone( "SELECT * from storyarcs WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone() else: chkissue = self.myDB.selectone( "SELECT * from issues WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone() if all([ chkissue is None, annualize is None, not mylar.CONFIG.ANNUALS_ON ]): chkissue = self.myDB.selectone( "SELECT * from annuals WHERE ComicID=? AND Issue_Number=?", [comicid, issue]).fetchone() if chkissue is None: #rechk chkissue against int value of issue # if arc: chkissue = self.myDB.selectone( "SELECT * from storyarcs WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone() else: chkissue = self.myDB.selectone( "SELECT * from issues WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone() if all([ chkissue is None, annualize == 'yes', mylar.CONFIG.ANNUALS_ON ]): chkissue = self.myDB.selectone( "SELECT * from annuals WHERE ComicID=? AND Int_IssueNumber=?", [comicid, issuedigits(issue)]).fetchone() if chkissue is None: logger.error('Invalid Issue_Number - please validate.') return else: logger.info( 'Int Issue_number compare found. continuing...') issueid = chkissue['IssueID'] else: issueid = chkissue['IssueID'] #use issueid to get publisher, series, year, issue number logger.fdebug('issueid is now : ' + str(issueid)) if arc: issueinfo = self.myDB.selectone( "SELECT * from storyarcs WHERE ComicID=? AND IssueID=? AND StoryArc=?", [comicid, issueid, arc]).fetchone() else: issueinfo = self.myDB.selectone( "SELECT * from issues WHERE ComicID=? AND IssueID=?", [comicid, issueid]).fetchone() if issueinfo is None: logger.fdebug('not an issue, checking against annuals') issueinfo = self.myDB.selectone( "SELECT * from annuals WHERE ComicID=? AND IssueID=?", [comicid, issueid]).fetchone() if issueinfo is None: logger.fdebug( 'Unable to rename - cannot locate issue id within db') return else: annualize = True if issueinfo is None: logger.fdebug( 'Unable to rename - cannot locate issue id within db') return #remap the variables to a common factor. if arc: issuenum = issueinfo['IssueNumber'] issuedate = issueinfo['IssueDate'] publisher = issueinfo['IssuePublisher'] series = issueinfo['ComicName'] seriesfilename = series #Alternate FileNaming is not available with story arcs. seriesyear = issueinfo['SeriesYear'] arcdir = helpers.filesafe(issueinfo['StoryArc']) if mylar.CONFIG.REPLACE_SPACES: arcdir = arcdir.replace(' ', mylar.CONFIG.REPLACE_CHAR) if mylar.CONFIG.STORYARCDIR: storyarcd = os.path.join(mylar.CONFIG.DESTINATION_DIR, "StoryArcs", arcdir) logger.fdebug('Story Arc Directory set to : ' + storyarcd) else: logger.fdebug('Story Arc Directory set to : ' + mylar.CONFIG.GRABBAG_DIR) storyarcd = os.path.join(mylar.CONFIG.DESTINATION_DIR, mylar.CONFIG.GRABBAG_DIR) comlocation = storyarcd comversion = None #need to populate this. else: issuenum = issueinfo['Issue_Number'] issuedate = issueinfo['IssueDate'] publisher = self.comic['ComicPublisher'] series = self.comic['ComicName'] if self.comic['AlternateFileName'] is None or self.comic[ 'AlternateFileName'] == 'None': seriesfilename = series else: seriesfilename = self.comic['AlternateFileName'] logger.fdebug( 'Alternate File Naming has been enabled for this series. Will rename series title to : ' + seriesfilename) seriesyear = self.comic['ComicYear'] comlocation = self.comic['ComicLocation'] comversion = self.comic['ComicVersion'] unicodeissue = issuenum if type(issuenum) == str: vals = { '\xbd': '.5', '\xbc': '.25', '\xbe': '.75', '\u221e': '9999999999', '\xe2': '9999999999' } else: vals = { '\xbd': '.5', '\xbc': '.25', '\xbe': '.75', '\\u221e': '9999999999', '\xe2': '9999999999' } x = [vals[key] for key in vals if key in issuenum] if x: issuenum = x[0] logger.fdebug('issue number formatted: %s' % issuenum) #comicid = issueinfo['ComicID'] #issueno = str(issuenum).split('.')[0] issue_except = 'None' issue_exceptions = [ 'AU', 'INH', 'NOW', 'AI', 'MU', 'A', 'B', 'C', 'X', 'O' ] valid_spaces = ('.', '-') for issexcept in issue_exceptions: if issexcept.lower() in issuenum.lower(): logger.fdebug('ALPHANUMERIC EXCEPTION : [' + issexcept + ']') v_chk = [v for v in valid_spaces if v in issuenum] if v_chk: iss_space = v_chk[0] logger.fdebug('character space denoted as : ' + iss_space) else: logger.fdebug('character space not denoted.') iss_space = '' # if issexcept == 'INH': # issue_except = '.INH' if issexcept == 'NOW': if '!' in issuenum: issuenum = re.sub('\!', '', issuenum) # issue_except = '.NOW' issue_except = iss_space + issexcept logger.fdebug('issue_except denoted as : ' + issue_except) issuenum = re.sub("[^0-9]", "", issuenum) break # if 'au' in issuenum.lower() and issuenum[:1].isdigit(): # issue_except = ' AU' # elif 'ai' in issuenum.lower() and issuenum[:1].isdigit(): # issuenum = re.sub("[^0-9]", "", issuenum) # issue_except = ' AI' # elif 'inh' in issuenum.lower() and issuenum[:1].isdigit(): # issuenum = re.sub("[^0-9]", "", issuenum) # issue_except = '.INH' # elif 'now' in issuenum.lower() and issuenum[:1].isdigit(): # if '!' in issuenum: issuenum = re.sub('\!', '', issuenum) # issuenum = re.sub("[^0-9]", "", issuenum) # issue_except = '.NOW' if '.' in issuenum: iss_find = issuenum.find('.') iss_b4dec = issuenum[:iss_find] if iss_find == 0: iss_b4dec = '0' iss_decval = issuenum[iss_find + 1:] if iss_decval.endswith('.'): iss_decval = iss_decval[:-1] if int(iss_decval) == 0: iss = iss_b4dec issdec = int(iss_decval) issueno = iss else: if len(iss_decval) == 1: iss = iss_b4dec + "." + iss_decval issdec = int(iss_decval) * 10 else: iss = iss_b4dec + "." + iss_decval.rstrip('0') issdec = int(iss_decval.rstrip('0')) * 10 issueno = iss_b4dec else: iss = issuenum issueno = iss # issue zero-suppression here if mylar.CONFIG.ZERO_LEVEL == "0": zeroadd = "" else: if mylar.CONFIG.ZERO_LEVEL_N == "none": zeroadd = "" elif mylar.CONFIG.ZERO_LEVEL_N == "0x": zeroadd = "0" elif mylar.CONFIG.ZERO_LEVEL_N == "00x": zeroadd = "00" logger.fdebug('Zero Suppression set to : ' + str(mylar.CONFIG.ZERO_LEVEL_N)) prettycomiss = None if issueno.isalpha(): logger.fdebug('issue detected as an alpha.') prettycomiss = str(issueno) else: try: x = float(issuenum) #validity check if x < 0: logger.info( 'I\'ve encountered a negative issue #: %s. Trying to accomodate.' % issueno) prettycomiss = '-' + str(zeroadd) + str(issueno[1:]) elif x == 9999999999: logger.fdebug('Infinity issue found.') issuenum = 'infinity' elif x >= 0: pass else: raise ValueError except ValueError as e: logger.warn( 'Unable to properly determine issue number [ %s] - you should probably log this on github for help.' % issueno) return if prettycomiss is None and len(str(issueno)) > 0: #if int(issueno) < 0: # self._log("issue detected is a negative") # prettycomiss = '-' + str(zeroadd) + str(abs(issueno)) if int(issueno) < 10: logger.fdebug('issue detected less than 10') if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(zeroadd) + str(iss) else: prettycomiss = str(zeroadd) + str(int(issueno)) else: prettycomiss = str(zeroadd) + str(iss) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '. Issue will be set as : ' + str(prettycomiss)) elif int(issueno) >= 10 and int(issueno) < 100: logger.fdebug( 'issue detected greater than 10, but less than 100') if mylar.CONFIG.ZERO_LEVEL_N == "none": zeroadd = "" else: zeroadd = "0" if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(zeroadd) + str(iss) else: prettycomiss = str(zeroadd) + str(int(issueno)) else: prettycomiss = str(zeroadd) + str(iss) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '.Issue will be set as : ' + str(prettycomiss)) else: logger.fdebug('issue detected greater than 100') if issuenum == 'infinity': prettycomiss = 'infinity' else: if '.' in iss: if int(iss_decval) > 0: issueno = str(iss) prettycomiss = str(issueno) if issue_except != 'None': prettycomiss = str(prettycomiss) + issue_except logger.fdebug('Zero level supplement set to ' + str(mylar.CONFIG.ZERO_LEVEL_N) + '. Issue will be set as : ' + str(prettycomiss)) elif len(str(issueno)) == 0: prettycomiss = str(issueno) logger.fdebug( 'issue length error - cannot determine length. Defaulting to None: ' + str(prettycomiss)) logger.fdebug('Pretty Comic Issue is : ' + str(prettycomiss)) if mylar.CONFIG.UNICODE_ISSUENUMBER: logger.fdebug('Setting this to Unicode format as requested: %s' % prettycomiss) prettycomiss = unicodeissue issueyear = issuedate[:4] month = issuedate[5:7].replace('-', '').strip() month_name = helpers.fullmonth(month) if month_name is None: month_name = 'None' logger.fdebug('Issue Year : ' + str(issueyear)) logger.fdebug('Publisher: ' + publisher) logger.fdebug('Series: ' + series) logger.fdebug('Year: ' + str(seriesyear)) logger.fdebug('Comic Location: ' + comlocation) if self.comic['Corrected_Type'] is not None: if self.comic['Type'] != self.comic['Corrected_Type']: booktype = self.comic['Corrected_Type'] else: booktype = self.comic['Type'] else: booktype = self.comic['Type'] if booktype == 'Print' or all( [booktype != 'Print', mylar.CONFIG.FORMAT_BOOKTYPE is False]): chunk_fb = re.sub('\$Type', '', file_format) chunk_b = re.compile(r'\s+') chunk_file_format = chunk_b.sub(' ', chunk_fb) else: chunk_file_format = file_format if any([comversion is None, booktype != 'Print']): comversion = 'None' #if comversion is None, remove it so it doesn't populate with 'None' if comversion == 'None': chunk_f_f = re.sub('\$VolumeN', '', chunk_file_format) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) logger.fdebug( 'No version # found for series, removing from filename') logger.fdebug("new format: " + str(chunk_file_format)) if annualize is None: chunk_f_f = re.sub('\$Annual', '', chunk_file_format) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) logger.fdebug('not an annual - removing from filename paramaters') logger.fdebug('new format: ' + str(chunk_file_format)) else: logger.fdebug('chunk_file_format is: ' + str(chunk_file_format)) if mylar.CONFIG.ANNUALS_ON: if 'annual' in series.lower(): if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower(): #if it's an annual, but $annual isn't specified in file_format, we need to #force it in there, by default in the format of $Annual $Issue #prettycomiss = "Annual " + str(prettycomiss) logger.fdebug( '[%s][ANNUALS-ON][ANNUAL IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: #because it exists within title, strip it then use formatting tag for placement of wording. chunk_f_f = re.sub('\$Annual', '', chunk_file_format) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) logger.fdebug( '[%s][ANNUALS-ON][ANNUAL IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower(): #if it's an annual, but $annual isn't specified in file_format, we need to #force it in there, by default in the format of $Annual $Issue prettycomiss = "Annual %s" % prettycomiss logger.fdebug( '[%s][ANNUALS-ON][ANNUAL NOT IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: logger.fdebug( '[%s][ANNUALS-ON][ANNUAL NOT IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: #if annuals aren't enabled, then annuals are being tracked as independent series. #annualize will be true since it's an annual in the seriesname. if 'annual' in series.lower(): if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower(): #if it's an annual, but $annual isn't specified in file_format, we need to #force it in there, by default in the format of $Annual $Issue #prettycomiss = "Annual " + str(prettycomiss) logger.fdebug( '[%s][ANNUALS-OFF][ANNUAL IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: #because it exists within title, strip it then use formatting tag for placement of wording. chunk_f_f = re.sub('\$Annual', '', chunk_file_format) chunk_f = re.compile(r'\s+') chunk_file_format = chunk_f.sub(' ', chunk_f_f) logger.fdebug( '[%s][ANNUALS-OFF][ANNUAL IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: if '$Annual' not in chunk_file_format: # and 'annual' not in ofilename.lower(): #if it's an annual, but $annual isn't specified in file_format, we need to #force it in there, by default in the format of $Annual $Issue prettycomiss = "Annual %s" % prettycomiss logger.fdebug( '[%s][ANNUALS-OFF][ANNUAL NOT IN SERIES][NO ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) else: logger.fdebug( '[%s][ANNUALS-OFF][ANNUAL NOT IN SERIES][ANNUAL FORMAT] prettycomiss: %s' % (series, prettycomiss)) logger.fdebug('Annual detected within series title of ' + series + '. Not auto-correcting issue #') seriesfilename = seriesfilename #.encode('ascii', 'ignore').strip() filebad = [ ':', ',', '/', '?', '!', '\'', '\"', '\*' ] #in u_comicname or '/' in u_comicname or ',' in u_comicname or '?' in u_comicname: for dbd in filebad: if dbd in seriesfilename: if any([dbd == '/', dbd == '*']): repthechar = '-' else: repthechar = '' seriesfilename = seriesfilename.replace(dbd, repthechar) logger.fdebug( 'Altering series name due to filenaming restrictions: ' + seriesfilename) publisher = re.sub('!', '', publisher) file_values = { '$Series': seriesfilename, '$Issue': prettycomiss, '$Year': issueyear, '$series': series.lower(), '$Publisher': publisher, '$publisher': publisher.lower(), '$VolumeY': 'V' + str(seriesyear), '$VolumeN': comversion, '$monthname': month_name, '$month': month, '$Annual': 'Annual', '$Type': booktype } extensions = ('.cbr', '.cbz', '.cb7') if ofilename.lower().endswith(extensions): path, ext = os.path.splitext(ofilename) if file_format == '': logger.fdebug( 'Rename Files is not enabled - keeping original filename.') #check if extension is in nzb_name - will screw up otherwise if ofilename.lower().endswith(extensions): nfilename = ofilename[:-4] else: nfilename = ofilename else: chunk_file_format = re.sub('[()|[]]', '', chunk_file_format).strip() nfilename = helpers.replace_all(chunk_file_format, file_values) if mylar.CONFIG.REPLACE_SPACES: #mylar.CONFIG.REPLACE_CHAR ...determines what to replace spaces with underscore or dot nfilename = nfilename.replace(' ', mylar.CONFIG.REPLACE_CHAR) nfilename = re.sub('[\,\:]', '', nfilename) + ext.lower() logger.fdebug('New Filename: ' + nfilename) if mylar.CONFIG.LOWERCASE_FILENAMES: nfilename = nfilename.lower() dst = os.path.join(comlocation, nfilename) else: dst = os.path.join(comlocation, nfilename) logger.fdebug('Source: ' + ofilename) logger.fdebug('Destination: ' + dst) rename_this = { "destination_dir": dst, "nfilename": nfilename, "issueid": issueid, "comicid": comicid } return rename_this
def Process(self): module = '[FAILED-DOWNLOAD]' myDB = db.DBConnection() if self.nzb_name and self.nzb_folder: self._log('Failed download has been detected: ' + self.nzb_name + ' in ' + self.nzb_folder) #since this has already been passed through the search module, which holds the IssueID in the nzblog, #let's find the matching nzbname and pass it the IssueID in order to mark it as Failed and then return #to the search module and continue trucking along. nzbname = self.nzb_name #remove extensions from nzb_name if they somehow got through (Experimental most likely) extensions = ('.cbr', '.cbz') if nzbname.lower().endswith(extensions): fd, ext = os.path.splitext(nzbname) self._log("Removed extension from nzb: " + ext) nzbname = re.sub(str(ext), '', str(nzbname)) #replace spaces nzbname = re.sub(' ', '.', str(nzbname)) nzbname = re.sub('[\,\:\?\'\(\)]', '', str(nzbname)) nzbname = re.sub('[\&]', 'and', str(nzbname)) nzbname = re.sub('_', '.', str(nzbname)) logger.fdebug(module + ' After conversions, nzbname is : ' + str(nzbname)) self._log("nzbname: " + str(nzbname)) nzbiss = myDB.selectone("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone() if nzbiss is None: self._log("Failure - could not initially locate nzbfile in my database to rename.") logger.fdebug(module + ' Failure - could not locate nzbfile initially') # if failed on spaces, change it all to decimals and try again. nzbname = re.sub('_', '.', str(nzbname)) self._log("trying again with this nzbname: " + str(nzbname)) logger.fdebug(module + ' Trying to locate nzbfile again with nzbname of : ' + str(nzbname)) nzbiss = myDB.selectone("SELECT * from nzblog WHERE nzbname=?", [nzbname]).fetchone() if nzbiss is None: logger.error(module + ' Unable to locate downloaded file to rename. PostProcessing aborted.') self._log('Unable to locate downloaded file to rename. PostProcessing aborted.') self.valreturn.append({"self.log": self.log, "mode": 'stop'}) return self.queue.put(self.valreturn) else: self._log("I corrected and found the nzb as : " + str(nzbname)) logger.fdebug(module + ' Auto-corrected and found the nzb as : ' + str(nzbname)) issueid = nzbiss['IssueID'] else: issueid = nzbiss['IssueID'] logger.fdebug(module + ' Issueid: ' + str(issueid)) sarc = nzbiss['SARC'] #use issueid to get publisher, series, year, issue number else: issueid = self.issueid nzbiss = myDB.selectone("SELECT * from nzblog WHERE IssueID=?", [issueid]).fetchone() if nzbiss is None: logger.info(module + ' Cannot locate corresponding record in download history. This will be implemented soon.') self.valreturn.append({"self.log": self.log, "mode": 'stop'}) return self.queue.put(self.valreturn) nzbname = nzbiss['NZBName'] # find the provider. self.prov = nzbiss['PROVIDER'] logger.info(module + ' Provider: ' + self.prov) # grab the id. self.id = nzbiss['ID'] logger.info(module + ' ID: ' + self.id) annchk = "no" if 'annual' in nzbname.lower(): logger.info(module + ' Annual detected.') annchk = "yes" issuenzb = myDB.selectone("SELECT * from annuals WHERE IssueID=? AND ComicName NOT NULL", [issueid]).fetchone() else: issuenzb = myDB.selectone("SELECT * from issues WHERE IssueID=? AND ComicName NOT NULL", [issueid]).fetchone() if issuenzb is not None: logger.info(module + ' issuenzb found.') if helpers.is_number(issueid): sandwich = int(issuenzb['IssueID']) else: logger.info(module + ' issuenzb not found.') #if it's non-numeric, it contains a 'G' at the beginning indicating it's a multi-volume #using GCD data. Set sandwich to 1 so it will bypass and continue post-processing. if 'S' in issueid: sandwich = issueid elif 'G' in issueid or '-' in issueid: sandwich = 1 try: if helpers.is_number(sandwich): if sandwich < 900000: # if sandwich is less than 900000 it's a normal watchlist download. Bypass. pass else: logger.info('Failed download handling for story-arcs and one-off\'s are not supported yet. Be patient!') self._log(' Unable to locate downloaded file to rename. PostProcessing aborted.') self.valreturn.append({"self.log": self.log, "mode": 'stop'}) return self.queue.put(self.valreturn) except NameError: logger.info('sandwich was not defined. Post-processing aborted...') self.valreturn.append({"self.log": self.log, "mode": 'stop'}) return self.queue.put(self.valreturn) comicid = issuenzb['ComicID'] issuenumOG = issuenzb['Issue_Number'] logger.info(module + ' Successfully detected as : ' + issuenzb['ComicName'] + ' issue: ' + str(issuenzb['Issue_Number']) + ' that was downloaded using ' + self.prov) self._log('Successfully detected as : ' + issuenzb['ComicName'] + ' issue: ' + str(issuenzb['Issue_Number']) + ' downloaded using ' + self.prov) logger.info(module + ' Marking as a Failed Download.') self._log('Marking as a Failed Download.') ctrlVal = {"IssueID": issueid} Vals = {"Status": 'Failed'} myDB.upsert("issues", Vals, ctrlVal) ctrlVal = {"ID": self.id, "Provider": self.prov, "NZBName": nzbname} Vals = {"Status": 'Failed', "ComicName": issuenzb['ComicName'], "Issue_Number": issuenzb['Issue_Number'], "IssueID": issueid, "ComicID": comicid, "DateFailed": helpers.now()} myDB.upsert("failed", Vals, ctrlVal) logger.info(module + ' Successfully marked as Failed.') self._log('Successfully marked as Failed.') if mylar.CONFIG.FAILED_AUTO: logger.info(module + ' Sending back to search to see if we can find something that will not fail.') self._log('Sending back to search to see if we can find something better that will not fail.') self.valreturn.append({"self.log": self.log, "mode": 'retry', "issueid": issueid, "comicid": comicid, "comicname": issuenzb['ComicName'], "issuenumber": issuenzb['Issue_Number'], "annchk": annchk}) return self.queue.put(self.valreturn) else: logger.info(module + ' Stopping search here as automatic handling of failed downloads is not enabled *hint*') self._log('Stopping search here as automatic handling of failed downloads is not enabled *hint*') self.valreturn.append({"self.log": self.log, "mode": 'stop'}) return self.queue.put(self.valreturn)
def run(dirName, nzbName=None, issueid=None, comversion=None, manual=None, filename=None, module=None, manualmeta=False): if module is None: module = '' module += '[META-TAGGER]' logger.fdebug(module + ' dirName:' + dirName) # 2015-11-23: Recent CV API changes restrict the rate-limit to 1 api request / second. # ComicTagger has to be included now with the install as a timer had to be added to allow for the 1/second rule. comictagger_cmd = os.path.join(mylar.CMTAGGER_PATH, 'comictagger.py') logger.fdebug( 'ComicTagger Path location for internal comictagger.py set to : ' + comictagger_cmd) # Force mylar to use cmtagger_path = mylar.PROG_DIR to force the use of the included lib. logger.fdebug(module + ' Filename is : ' + filename) filepath = filename og_filepath = filepath try: filename = os.path.split(filename)[1] # just the filename itself except: logger.warn( 'Unable to detect filename within directory - I am aborting the tagging. You best check things out.' ) return "fail" #make use of temporary file location in order to post-process this to ensure that things don't get hammered when converting new_filepath = None new_folder = None try: import tempfile logger.fdebug('Filepath: %s' % filepath) logger.fdebug('Filename: %s' % filename) new_folder = tempfile.mkdtemp( prefix='mylar_', dir=mylar.CONFIG.CACHE_DIR) #prefix, suffix, dir logger.fdebug('New_Folder: %s' % new_folder) new_filepath = os.path.join(new_folder, filename) logger.fdebug('New_Filepath: %s' % new_filepath) if mylar.CONFIG.FILE_OPTS == 'copy' and manualmeta == False: shutil.copy(filepath, new_filepath) else: shutil.copy(filepath, new_filepath) filepath = new_filepath except Exception as e: logger.warn('%s Unexpected Error: %s [%s]' % (module, sys.exc_info()[0], e)) logger.warn( module + ' Unable to create temporary directory to perform meta-tagging. Processing without metatagging.' ) tidyup(og_filepath, new_filepath, new_folder, manualmeta) return "fail" ## Sets up other directories ## scriptname = os.path.basename(sys.argv[0]) downloadpath = os.path.abspath(dirName) sabnzbdscriptpath = os.path.dirname(sys.argv[0]) comicpath = new_folder logger.fdebug(module + ' Paths / Locations:') logger.fdebug(module + ' scriptname : ' + scriptname) logger.fdebug(module + ' downloadpath : ' + downloadpath) logger.fdebug(module + ' sabnzbdscriptpath : ' + sabnzbdscriptpath) logger.fdebug(module + ' comicpath : ' + comicpath) logger.fdebug(module + ' Running the ComicTagger Add-on for Mylar') ##set up default comictagger options here. #used for cbr - to - cbz conversion #depending on copy/move - eitehr we retain the rar or we don't. if mylar.CONFIG.FILE_OPTS == 'move': cbr2cbzoptions = ["-e", "--delete-rar"] else: cbr2cbzoptions = ["-e"] tagoptions = ["-s"] if mylar.CONFIG.CMTAG_VOLUME: if mylar.CONFIG.CMTAG_START_YEAR_AS_VOLUME: comversion = str(comversion) else: if any( [comversion is None, comversion == '', comversion == 'None']): comversion = '1' comversion = re.sub('[^0-9]', '', comversion).strip() cvers = 'volume=' + str(comversion) else: cvers = "volume=" tagoptions.extend(["-m", cvers]) try: #from comictaggerlib import ctversion ct_check = subprocess.check_output( [sys.executable, comictagger_cmd, "--version"], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: #logger.warn(module + "[WARNING] "command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) logger.warn( module + '[WARNING] Make sure that you are using the comictagger included with Mylar.' ) tidyup(filepath, new_filepath, new_folder, manualmeta) return "fail" logger.info('ct_check: %s' % ct_check) ctend = str(ct_check).find(':') ct_version = re.sub("[^0-9]", "", str(ct_check)[:ctend]) from pkg_resources import parse_version if parse_version(ct_version) >= parse_version('1.3.1'): if any([ mylar.CONFIG.COMICVINE_API == 'None', mylar.CONFIG.COMICVINE_API is None ]): logger.fdebug( '%s ComicTagger v.%s being used - no personal ComicVine API Key supplied. Take your chances.' % (module, ct_version)) use_cvapi = "False" else: logger.fdebug( '%s ComicTagger v.%s being used - using personal ComicVine API key supplied via mylar.' % (module, ct_version)) use_cvapi = "True" tagoptions.extend(["--cv-api-key", mylar.CONFIG.COMICVINE_API]) else: logger.fdebug( '%s ComicTagger v.ct_version being used - personal ComicVine API key not supported in this version. Good luck.' % (module, ct_version)) use_cvapi = "False" i = 1 tagcnt = 0 if mylar.CONFIG.CT_TAG_CR: tagcnt = 1 logger.fdebug(module + ' CR Tagging enabled.') if mylar.CONFIG.CT_TAG_CBL: if not mylar.CONFIG.CT_TAG_CR: i = 2 #set the tag to start at cbl and end without doing another tagging. tagcnt = 2 logger.fdebug(module + ' CBL Tagging enabled.') if tagcnt == 0: logger.warn( module + ' You have metatagging enabled, but you have not selected the type(s) of metadata to write. Please fix and re-run manually' ) tidyup(filepath, new_filepath, new_folder, manualmeta) return "fail" #if it's a cbz file - check if no-overwrite existing tags is enabled / disabled in config. if filename.endswith('.cbz'): if mylar.CONFIG.CT_CBZ_OVERWRITE: logger.fdebug( module + ' Will modify existing tag blocks even if it exists.') else: logger.fdebug( module + ' Will NOT modify existing tag blocks even if they exist already.' ) tagoptions.extend(["--nooverwrite"]) if issueid is None: tagoptions.extend(["-f", "-o"]) else: tagoptions.extend(["-o", "--id", issueid]) original_tagoptions = tagoptions og_tagtype = None initial_ctrun = True error_remove = False while (i <= tagcnt): if initial_ctrun: f_tagoptions = cbr2cbzoptions f_tagoptions.extend([filepath]) else: if i == 1: tagtype = 'cr' # CR meta-tagging cycle. tagdisp = 'ComicRack tagging' elif i == 2: tagtype = 'cbl' # Cbl meta-tagging cycle tagdisp = 'Comicbooklover tagging' f_tagoptions = original_tagoptions if og_tagtype is not None: for index, item in enumerate(f_tagoptions): if item == og_tagtype: f_tagoptions[index] = tagtype else: f_tagoptions.extend(["--type", tagtype, filepath]) og_tagtype = tagtype logger.info(module + ' ' + tagdisp + ' meta-tagging processing started.') currentScriptName = [sys.executable, comictagger_cmd] script_cmd = currentScriptName + f_tagoptions if initial_ctrun: logger.fdebug('%s Enabling ComicTagger script with options: %s' % (module, f_tagoptions)) script_cmdlog = script_cmd else: logger.fdebug('%s Enabling ComicTagger script with options: %s' % (module, re.sub( f_tagoptions[f_tagoptions.index( mylar.CONFIG.COMICVINE_API)], 'REDACTED', str(f_tagoptions)))) # generate a safe command line string to execute the script and provide all the parameters script_cmdlog = re.sub( f_tagoptions[f_tagoptions.index(mylar.CONFIG.COMICVINE_API)], 'REDACTED', str(script_cmd)) logger.fdebug(module + ' Executing command: ' + str(script_cmdlog)) logger.fdebug(module + ' Absolute path to script: ' + script_cmd[0]) try: # use subprocess to run the command and capture output p = subprocess.Popen(script_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, err = p.communicate() logger.info(out) logger.info(err) if out is not None: out = out.decode('utf-8') if err is not None: err = err.decode('utf-8') if initial_ctrun and 'exported successfully' in out: logger.fdebug(module + '[COMIC-TAGGER] : ' + str(out)) #Archive exported successfully to: X-Men v4 008 (2014) (Digital) (Nahga-Empire).cbz (Original deleted) if 'Error deleting' in filepath: tf1 = out.find('exported successfully to: ') tmpfilename = out[tf1 + len('exported successfully to: ' ):].strip() error_remove = True else: tmpfilename = re.sub('Archive exported successfully to: ', '', out.rstrip()) if mylar.CONFIG.FILE_OPTS == 'move': tmpfilename = re.sub('\(Original deleted\)', '', tmpfilename).strip() tmpf = tmpfilename #.decode('utf-8') filepath = os.path.join(comicpath, tmpf) if filename.lower() != tmpf.lower() and tmpf.endswith( '(1).cbz'): logger.fdebug( 'New filename [%s] is named incorrectly due to duplication during metatagging - Making sure it\'s named correctly [%s].' % (tmpf, filename)) tmpfilename = filename filepath_new = os.path.join(comicpath, tmpfilename) try: os.rename(filepath, filepath_new) filepath = filepath_new except: logger.warn( '%s unable to rename file to accomodate metatagging cbz to the same filename' % module) if not os.path.isfile(filepath): logger.fdebug(module + 'Trying utf-8 conversion.') tmpf = tmpfilename.encode('utf-8') filepath = os.path.join(comicpath, tmpf) if not os.path.isfile(filepath): logger.fdebug(module + 'Trying latin-1 conversion.') tmpf = tmpfilename.encode('Latin-1') filepath = os.path.join(comicpath, tmpf) logger.fdebug(module + '[COMIC-TAGGER][CBR-TO-CBZ] New filename: ' + filepath) initial_ctrun = False elif initial_ctrun and 'Archive is not a RAR' in out: logger.fdebug('%s Output: %s' % (module, out)) logger.warn(module + '[COMIC-TAGGER] file is not in a RAR format: ' + filename) initial_ctrun = False elif initial_ctrun: initial_ctrun = False if 'file is not expected size' in out: logger.fdebug('%s Output: %s' % (module, out)) tidyup(og_filepath, new_filepath, new_folder, manualmeta) return 'corrupt' else: logger.warn( module + '[COMIC-TAGGER][CBR-TO-CBZ] Failed to convert cbr to cbz - check permissions on folder : ' + mylar.CONFIG.CACHE_DIR + ' and/or the location where Mylar is trying to tag the files from.' ) tidyup(og_filepath, new_filepath, new_folder, manualmeta) return 'fail' elif 'Cannot find' in out: logger.fdebug('%s Output: %s' % (module, out)) logger.warn(module + '[COMIC-TAGGER] Unable to locate file: ' + filename) file_error = 'file not found||' + filename return file_error elif 'not a comic archive!' in out: logger.fdebug('%s Output: %s' % (module, out)) logger.warn(module + '[COMIC-TAGGER] Unable to locate file: ' + filename) file_error = 'file not found||' + filename return file_error else: logger.info(module + '[COMIC-TAGGER] Successfully wrote ' + tagdisp + ' [' + filepath + ']') i += 1 except OSError as e: logger.warn( module + '[COMIC-TAGGER] Unable to run comictagger with the options provided: ' + re.sub( f_tagoptions[f_tagoptions.index( mylar.CONFIG.COMICVINE_API)], 'REDACTED', str(script_cmd))) tidyup(filepath, new_filepath, new_folder, manualmeta) return "fail" if mylar.CONFIG.CBR2CBZ_ONLY and initial_ctrun == False: break return filepath
def failed_check(self): #issueid = self.issueid #comicid = self.comicid # ID = ID passed by search upon a match upon preparing to send it to client to download. # ID is provider dependent, so the same file should be different for every provider. module = '[FAILED_DOWNLOAD_CHECKER]' myDB = db.DBConnection() # Querying on NZBName alone will result in all downloads regardless of provider. # This will make sure that the files being downloaded are different regardless of provider. # Perhaps later improvement might be to break it down by provider so that Mylar will attempt to # download same issues on different providers (albeit it shouldn't matter, if it's broke it's broke). logger.info('prov : ' + str(self.prov) + '[' + str(self.id) + ']') # if this is from nzbhydra, we need to rejig the id line so that the searchid is removed since it's always unique to the search. if 'indexerguid' in self.id: st = self.id.find('searchid:') end = self.id.find(',',st) self.id = '%' + self.id[:st] + '%' + self.id[end+1:len(self.id)-1] + '%' chk_fail = myDB.selectone('SELECT * FROM failed WHERE ID LIKE ?', [self.id]).fetchone() else: chk_fail = myDB.selectone('SELECT * FROM failed WHERE ID=?', [self.id]).fetchone() if chk_fail is None: logger.info(module + ' Successfully marked this download as Good for downloadable content') return 'Good' else: if chk_fail['status'] == 'Good': logger.info(module + ' result has a status of GOOD - which means it does not currently exist in the failed download list.') return chk_fail['status'] elif chk_fail['status'] == 'Failed': logger.info(module + ' result has a status of FAIL which indicates it is not a good choice to download.') logger.info(module + ' continuing search for another download.') return chk_fail['status'] elif chk_fail['status'] == 'Retry': logger.info(module + ' result has a status of RETRY which indicates it was a failed download that retried .') return chk_fail['status'] elif chk_fail['status'] == 'Retrysame': logger.info(module + ' result has a status of RETRYSAME which indicates it was a failed download that retried the initial download.') return chk_fail['status'] else: logger.info(module + ' result has a status of ' + chk_fail['status'] + '. I am not sure what to do now.') return "nope"
def on_login(self, username): """Called on successful login""" logger.info('%s successfully logged on.' % username)
def initialize(options): # HTTPS stuff stolen from sickbeard enable_https = options['enable_https'] https_cert = options['https_cert'] https_key = options['https_key'] https_chain = options['https_chain'] if enable_https: # If either the HTTPS certificate or key do not exist, try to make # self-signed ones. if not (https_cert and os.path.exists(https_cert)) or not ( https_key and os.path.exists(https_key)): if not create_https_certificates(https_cert, https_key): logger.warn("Unable to create certificate and key. Disabling " \ "HTTPS") enable_https = False if not (os.path.exists(https_cert) and os.path.exists(https_key)): logger.warn("Disabled HTTPS because of missing certificate and " \ "key.") enable_https = False options_dict = { 'server.socket_port': options['http_port'], 'server.socket_host': options['http_host'], 'server.thread_pool': 10, 'tools.encode.on': True, 'tools.encode.encoding': 'utf-8', 'tools.encode.text_only': False, 'tools.decode.on': True, 'log.screen': True, 'engine.autoreload.on': False, } if enable_https: options_dict['server.ssl_certificate'] = https_cert options_dict['server.ssl_private_key'] = https_key if https_chain: options_dict['server.ssl_certificate_chain'] = https_chain protocol = "https" else: protocol = "http" logger.info("Starting Mylar on %s://%s:%d%s" % (protocol, options['http_host'], options['http_port'], options['http_root'])) cherrypy.config.update(options_dict) conf = { '/': { 'tools.staticdir.root': os.path.join(mylar.PROG_DIR, 'data'), 'tools.proxy.on': True # pay attention to X-Forwarded-Proto header }, '/interfaces': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "interfaces" }, '/images': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "images" }, '/css': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "css" }, '/js': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "js" }, '/favicon.ico': { 'tools.staticfile.on': True, 'tools.staticfile.filename': os.path.join(os.path.abspath(os.curdir), 'images' + os.sep + 'favicon.ico') }, '/cache': { 'tools.staticdir.on': True, 'tools.staticdir.dir': mylar.CONFIG.CACHE_DIR } } if options['http_password'] is not None: #userpassdict = dict(zip((options['http_username'].encode('utf-8'),), (options['http_password'].encode('utf-8'),))) #get_ha1= cherrypy.lib.auth_digest.get_ha1_dict_plain(userpassdict) if options['authentication'] == 2: # Set up a sessions based login page instead of using basic auth, # using the credentials set for basic auth. Attempting to browse to # a restricted page without a session token will result in a # redirect to the login page. A sucessful login should then redirect # to the originally requested page. # # Login sessions timeout after 43800 minutes (1 month) unless # changed in the config. # Note - the following command doesn't actually work, see update statement 2 lines down # cherrypy.tools.sessions.timeout = options['login_timeout'] conf['/'].update({ 'tools.sessions.on': True, 'tools.auth.on': True, 'tools.sessions.timeout': options['login_timeout'], 'auth.forms_username': options['http_username'], 'auth.forms_password': options['http_password'], # Set all pages to require authentication. # You can also set auth requirements on a per-method basis by # using the @require() decorator on the methods in webserve.py 'auth.require': [] }) # exempt api, login page and static elements from authentication requirements for i in ('/api', '/auth/login', '/css', '/images', '/js', 'favicon.ico'): if i in conf: conf[i].update({'tools.auth.on': False}) else: conf[i] = {'tools.auth.on': False} elif options['authentication'] == 1: conf['/'].update({ 'tools.auth_basic.on': True, 'tools.auth_basic.realm': 'Mylar', 'tools.auth_basic.checkpassword': cherrypy.lib.auth_basic.checkpassword_dict( {options['http_username']: options['http_password']}) }) conf['/api'] = {'tools.auth_basic.on': False} rest_api = { '/': { # the api uses restful method dispatching 'request.dispatch': cherrypy.dispatch.MethodDispatcher(), # all api calls require that the client passes HTTP basic authentication 'tools.auth_basic.on': False, } } if options['opds_authentication']: user_list = {} if len(options['opds_username']) > 0: user_list[options['opds_username']] = options['opds_password'] if options['http_password'] is not None and options[ 'http_username'] != options['opds_username']: user_list[options['http_username']] = options['http_password'] conf['/opds'] = { 'tools.auth.on': False, 'tools.auth_basic.on': True, 'tools.auth_basic.realm': 'Mylar OPDS', 'tools.auth_basic.checkpassword': cherrypy.lib.auth_basic.checkpassword_dict(user_list) } else: conf['/opds'] = {'tools.auth_basic.on': False, 'tools.auth.on': False} # Prevent time-outs #cherrypy.engine.timeout_monitor.unsubscribe() cherrypy.tree.mount(WebInterface(), str(options['http_root']), config=conf) restroot = REST() restroot.comics = restroot.Comics() restroot.comic = restroot.Comic() restroot.watchlist = restroot.Watchlist() #restroot.issues = restroot.comic.Issues() #restroot.issue = restroot.comic.Issue() cherrypy.tree.mount(restroot, '/rest', config=rest_api) try: portend.Checker().assert_free(options['http_host'], options['http_port']) cherrypy.server.start() except Exception as e: logger.error('[ERROR] %s' % e) print('Failed to start on port: %i. Is something else running?' % (options['http_port'])) sys.exit(0) cherrypy.server.wait()
def solicit(month, year): #convert to numerics just to ensure this... month = int(month) year = int(year) #print ( "month: " + str(month) ) #print ( "year: " + str(year) ) # in order to gather ALL upcoming - let's start to loop through months going ahead one at a time # until we get a null then break. (Usually not more than 3 months in advance is available) mnloop = 0 upcoming = [] publishers = { 'DC Comics': 'DC Comics', 'DC\'s': 'DC Comics', 'Marvel': 'Marvel Comics', 'Image': 'Image Comics', 'IDW': 'IDW Publishing', 'Dark Horse': 'Dark Horse' } # -- this is no longer needed (testing) # while (mnloop < 5): # if year == 2014: # if len(str(month)) == 1: # month_string = '0' + str(month) # else: # month_string = str(month) # datestring = str(year) + str(month_string) # else: # datestring = str(month) + str(year) # pagelinks = "http://www.comicbookresources.com/tag/solicits" + str(datestring) #using the solicits+datestring leaves out some entries occasionally #should use http://www.comicbookresources.com/tag/solicitations #then just use the logic below but instead of datestring, find the month term and #go ahead up to +5 months. if month > 0: month_start = month month_end = month + 5 #if month_end > 12: # ms = 8, me=13 [(12-8)+(13-12)] = [4 + 1] = 5 # [(12 - ms) + (me - 12)] = number of months (5) monthlist = [] mongr = month_start #we need to build the months we can grab, but the non-numeric way. while (mongr <= month_end): mon = mongr if mon == 13: mon = 1 year += 1 if len(str(mon)) == 1: mon = '0' + str(mon) monthlist.append({ "month": helpers.fullmonth(str(mon)).lower(), "num_month": mon, "year": str(year) }) mongr += 1 logger.info('months: ' + str(monthlist)) pagelinks = "http://www.comicbookresources.com/tag/solicitations" #logger.info('datestring:' + datestring) #logger.info('checking:' + pagelinks) pageresponse = urllib2.urlopen(pagelinks) soup = BeautifulSoup(pageresponse) cntlinks = soup.findAll('h3') lenlinks = len(cntlinks) #logger.info( str(lenlinks) + ' results' ) publish = [] resultURL = [] resultmonth = [] resultyear = [] x = 0 cnt = 0 while (x < lenlinks): headt = cntlinks[ x] #iterate through the hrefs pulling out only results. if "/?page=article&id=" in str(headt): #print ("titlet: " + str(headt)) headName = headt.findNext(text=True) #print ('headName: ' + headName) if 'Image' in headName: print 'IMAGE FOUND' if not all([ 'Marvel' in headName, 'DC' in headName, 'Image' in headName ]) and ('Solicitations' in headName or 'Solicits' in headName): # test for month here (int(month) + 5) if not any( d.get('month', None) == str(headName).lower() for d in monthlist): for mt in monthlist: if mt['month'] in headName.lower(): logger.info('matched on month: ' + str(mt['month'])) logger.info('matched on year: ' + str(mt['year'])) resultmonth.append(mt['num_month']) resultyear.append(mt['year']) pubstart = headName.find('Solicitations') publishchk = False for pub in publishers: if pub in headName[:pubstart]: #print 'publisher:' + str(publishers[pub]) publish.append(publishers[pub]) publishchk = True break if publishchk == False: break #publish.append( headName[:pubstart].strip() ) abc = headt.findAll('a', href=True)[0] ID_som = abc[ 'href'] #first instance will have the right link... resultURL.append(ID_som) #print '(' + str(cnt) + ') [ ' + publish[cnt] + '] Link URL: ' + resultURL[cnt] cnt += 1 else: logger.info('incorrect month - not using.') x += 1 if cnt == 0: return #break # no results means, end it loopthis = (cnt - 1) #this loops through each 'found' solicit page #shipdate = str(month_string) + '-' + str(year) - not needed. while (loopthis >= 0): #print 'loopthis is : ' + str(loopthis) #print 'resultURL is : ' + str(resultURL[loopthis]) shipdate = str(resultmonth[loopthis]) + '-' + str( resultyear[loopthis]) upcoming += populate(resultURL[loopthis], publish[loopthis], shipdate) loopthis -= 1 logger.info(str(len(upcoming)) + ' upcoming issues discovered.') newfl = mylar.CACHE_DIR + "/future-releases.txt" newtxtfile = open(newfl, 'wb') cntr = 1 for row in upcoming: if row['Extra'] is None or row['Extra'] == '': extrarow = 'N/A' else: extrarow = row['Extra'] newtxtfile.write( str(row['Shipdate']) + '\t' + str(row['Publisher']) + '\t' + str(row['Issue']) + '\t' + str(row['Comic']) + '\t' + str(extrarow) + '\tSkipped' + '\t' + str(cntr) + '\n') cntr += 1 newtxtfile.close() logger.fdebug('attempting to populate future upcoming...') mylardb = os.path.join(mylar.DATA_DIR, "mylar.db") connection = sqlite3.connect(str(mylardb)) cursor = connection.cursor() # we should extract the issues that are being watched, but no data is available yet ('Watch For' status) # once we get the data, store it, wipe the existing table, retrieve the new data, populate the data into # the table, recheck the series against the current watchlist and then restore the Watch For data. cursor.executescript('drop table if exists future;') cursor.execute( "CREATE TABLE IF NOT EXISTS future (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, FutureID text, ComicID text);" ) connection.commit() csvfile = open(newfl, "rb") creader = csv.reader(csvfile, delimiter='\t') t = 1 for row in creader: try: #print ("Row: %s" % row) cursor.execute("INSERT INTO future VALUES (?,?,?,?,?,?,?,null);", row) except Exception, e: logger.fdebug("Error - invald arguments...-skipping") pass t += 1
def load_torrent(self, filepath): logger.info('filepath to torrent file set to : ' + filepath) torrent_id = False if self.client.connected is True: logger.info('Checking if Torrent Exists!') if not filepath.startswith('magnet'): torrentcontent = open(filepath, 'rb').read() hash = str.lower(self.get_the_hash(filepath)) # Deluge expects a lower case hash logger.debug('Torrent Hash (load_torrent): "' + hash + '"') logger.debug('FileName (load_torrent): ' + str(os.path.basename(filepath))) #Check if torrent already added if self.find_torrent(str.lower(hash)): logger.info('load_torrent: Torrent already exists!') #should set something here to denote that it's already loaded, and then the failed download checker not run so it doesn't download #multiple copies of the same issues that's already downloaded else: logger.info('Torrent not added yet, trying to add it now!') try: torrent_id = self.client.call('core.add_torrent_file', str(os.path.basename(filepath)), base64.encodestring(torrentcontent), '') except Exception as e: logger.debug('Torrent not added') return False else: try: torrent_id = self.client.call('core.add_torrent_magnet', str(filepath), {}) except Exception as e: logger.debug('Torrent not added') return False # If label enabled put label on torrent in Deluge if torrent_id and mylar.DELUGE_LABEL: logger.info ('Setting label to ' + mylar.DELUGE_LABEL) try: self.client.call('label.set_torrent', torrent_id, mylar.DELUGE_LABEL) except: #if label isn't set, let's try and create one. try: self.client.call('label.add', mylar.DELUGE_LABEL) self.client.call('label.set_torrent', torrent_id, mylar.DELUGE_LABEL) except: logger.warn('Unable to set label - Either try to create it manually within Deluge, and/or ensure there are no spaces, capitalization or special characters in label') else: logger.info('Succesfully set label to ' + mylar.DELUGE_LABEL) try: torrent_info = self.get_torrent(torrent_id) logger.info('Double checking that the torrent was added.') except Exception as e: logger.warn('Torrent was not added! Please check logs') return False else: logger.info('Torrent successfully added!') return {'hash': torrent_info['hash'], 'label': mylar.DELUGE_LABEL, 'folder': torrent_info['save_path'], 'total_filesize': torrent_info['total_size'], 'name': torrent_info['name'], 'files': torrent_info['files'], 'time_started': torrent_info['active_time'], 'completed': torrent_info['is_finished']}
def validate(self): logger.info('attempting to validate...') req = cherrypy.request.headers logger.info('thekey: %s' % req) logger.info('url: %s' % cherrypy.url()) logger.info('mylar.apikey: %s [%s]' % (mylar.CONFIG.API_KEY, type(mylar.CONFIG.API_KEY))) logger.info('submitted.apikey: %s [%s]' % (req['Api-Key'], type(req['Api-Key']))) if 'Api-Key' not in req or req['Api-Key'] != str(mylar.CONFIG.API_KEY): #str(mylar.API_KEY) or mylar.API_KEY not in cherrypy.url(): logger.info('wrong APIKEY') return 'api-key provided was either not present in auth header, or was incorrect.' else: return True
def solicit(month, year): #convert to numerics just to ensure this... month = int(month) year = int(year) #print ( "month: " + str(month) ) #print ( "year: " + str(year) ) # in order to gather ALL upcoming - let's start to loop through months going ahead one at a time # until we get a null then break. (Usually not more than 3 months in advance is available) mnloop = 0 upcoming = [] publishers = { 'DC Comics': 'DC Comics', 'Marvel': 'Marvel Comics', 'Image': 'Image Comics', 'IDW': 'IDW Publishing', 'Dark Horse': 'Dark Horse Comics' } while (mnloop < 5): if year == 2014: if len(str(month)) == 1: month_string = '0' + str(month) else: month_string = str(month) datestring = str(year) + str(month_string) else: datestring = str(month) + str(year) pagelinks = "http://www.comicbookresources.com/tag/solicits" + str( datestring) pageresponse = urllib2.urlopen(pagelinks) soup = BeautifulSoup(pageresponse) cntlinks = soup.findAll('h3') lenlinks = len(cntlinks) logger.info(str(lenlinks) + ' results') publish = [] resultURL = [] x = 0 cnt = 0 while (x < lenlinks): headt = cntlinks[ x] #iterate through the hrefs pulling out only results. if "/?page=article&id=" in str(headt): #print ("titlet: " + str(headt)) headName = headt.findNext(text=True) if ('Marvel' and 'DC' and 'Image' not in headName) and ( 'Solicitations' in headName or 'Solicits' in headName): pubstart = headName.find('Solicitations') for pub in publishers: if pub in headName[:pubstart]: publish.append(publishers[pub]) #publish.append( headName[:pubstart].strip() ) abc = headt.findAll('a', href=True)[0] ID_som = abc[ 'href'] #first instance will have the right link... resultURL.append(ID_som) #print '[ ' + publish[cnt] + '] Link URL: ' + resultURL[cnt] cnt += 1 x += 1 #print 'cnt:' + str(cnt) if cnt == 0: break # no results means, end it loopthis = (cnt - 1) #this loops through each 'found' solicit page shipdate = str(month) + '-' + str(year) while (loopthis >= 0): upcoming += populate(resultURL[loopthis], publish[loopthis], shipdate) loopthis -= 1 month += 1 #increment month by 1 mnloop += 1 #increment loop by 1 if month > 12: #failsafe failover for months month = 1 year += 1 #print upcoming logger.info(str(len(upcoming)) + ' upcoming issues discovered.') newfl = mylar.CACHE_DIR + "/future-releases.txt" newtxtfile = open(newfl, 'wb') cntr = 1 for row in upcoming: if row['Extra'] is None or row['Extra'] == '': extrarow = 'N/A' else: extrarow = row['Extra'] newtxtfile.write( str(row['Shipdate']) + '\t' + str(row['Publisher']) + '\t' + str(row['Issue']) + '\t' + str(row['Comic']) + '\t' + str(extrarow) + '\tSkipped' + '\t' + str(cntr) + '\n') cntr += 1 newtxtfile.close() logger.fdebug('attempting to populate future upcoming...') mylardb = os.path.join(mylar.DATA_DIR, "mylar.db") connection = sqlite3.connect(str(mylardb)) cursor = connection.cursor() cursor.executescript('drop table if exists future;') cursor.execute( "CREATE TABLE IF NOT EXISTS future (SHIPDATE, PUBLISHER text, ISSUE text, COMIC VARCHAR(150), EXTRA text, STATUS text, FutureID text, ComicID text);" ) connection.commit() csvfile = open(newfl, "rb") creader = csv.reader(csvfile, delimiter='\t') t = 1 for row in creader: try: #print ("Row: %s" % row) cursor.execute("INSERT INTO future VALUES (?,?,?,?,?,?,?,null);", row) except Exception, e: logger.fdebug("Error - invald arguments...-skipping") pass t += 1
def getVersion(): if mylar.CONFIG.GIT_BRANCH is not None and mylar.CONFIG.GIT_BRANCH.startswith( 'win32build'): mylar.INSTALL_TYPE = 'win' # Don't have a way to update exe yet, but don't want to set VERSION to None return 'Windows Install', 'None' elif os.path.isdir(os.path.join(mylar.PROG_DIR, '.git')): mylar.INSTALL_TYPE = 'git' output, err = runGit('rev-parse HEAD') if not output: logger.error('Couldn\'t find latest installed version.') cur_commit_hash = None #branch_history, err = runGit("log --oneline --pretty=format:'%h - %ar - %s' -n 5") #bh = [] #print ("branch_history: " + branch_history) #bh.append(branch_history.split('\n')) #print ("bh1: " + bh[0]) cur_commit_hash = str(output).strip() if not re.match('^[a-z0-9]+$', cur_commit_hash): logger.error('Output does not look like a hash, not using it') cur_commit_hash = None if mylar.CONFIG.GIT_BRANCH: branch = mylar.CONFIG.GIT_BRANCH else: branch = None branch_name, err = runGit('branch --contains %s' % cur_commit_hash) if not branch_name: logger.warn( 'Could not retrieve branch name [%s] from git. Defaulting to Master.' % branch) branch = 'master' else: for line in branch_name.split('\n'): if '*' in line: branch = re.sub('[\*\n]', '', line).strip() break if not branch and mylar.CONFIG.GIT_BRANCH: logger.warn( 'Unable to retrieve branch name [%s] from git. Setting branch to configuration value of : %s' % (branch, mylar.CONFIG.GIT_BRANCH)) branch = mylar.CONFIG.GIT_BRANCH if not branch: logger.warn( 'Could not retrieve branch name [%s] from git. Defaulting to Master.' % branch) branch = 'master' else: logger.info('Branch detected & set to : %s' % branch) return cur_commit_hash, branch else: mylar.INSTALL_TYPE = 'source' version_file = os.path.join(mylar.PROG_DIR, 'version.txt') if not os.path.isfile(version_file): current_version = None else: with open(version_file, 'r') as f: current_version = f.read().strip(' \n\r') if current_version: if mylar.CONFIG.GIT_BRANCH: logger.info('Branch detected & set to : ' + mylar.CONFIG.GIT_BRANCH) return current_version, mylar.CONFIG.GIT_BRANCH else: logger.warn( 'No branch specified within config - will attempt to poll version from mylar' ) try: branch = version.MYLAR_VERSION logger.info('Branch detected & set to : ' + branch) except: branch = 'master' logger.info( 'Unable to detect branch properly - set branch in config.ini, currently defaulting to : ' + branch) return current_version, branch else: if mylar.CONFIG.GIT_BRANCH: logger.info('Branch detected & set to : ' + mylar.CONFIG.GIT_BRANCH) return current_version, mylar.CONFIG.GIT_BRANCH else: logger.warn( 'No branch specified within config - will attempt to poll version from mylar' ) try: branch = version.MYLAR_VERSION logger.info('Branch detected & set to : ' + branch) except: branch = 'master' logger.info( 'Unable to detect branch properly - set branch in config.ini, currently defaulting to : ' + branch) return current_version, branch logger.warn( 'Unable to determine which commit is currently being run. Defaulting to Master branch.' )
def load_torrent(self, filepath): start = bool(mylar.RTORRENT_STARTONLOAD) logger.info('filepath to torrent file set to : ' + filepath) torrent = self.conn.load_torrent(filepath, verify_load=True) if not torrent: return False if mylar.RTORRENT_LABEL: torrent.set_custom(1, mylar.RTORRENT_LABEL) logger.info('Setting label for torrent to : ' + mylar.RTORRENT_LABEL) if mylar.RTORRENT_DIRECTORY: torrent.set_directory(mylar.RTORRENT_DIRECTORY) logger.info('Setting directory for torrent to : ' + mylar.RTORRENT_DIRECTORY) logger.info('Successfully loaded torrent.') #note that if set_directory is enabled, the torrent has to be started AFTER it's loaded or else it will give chunk errors and not seed if start: logger.info('[' + str(start) + '] Now starting torrent.') torrent.start() else: logger.info('[' + str(start) + '] Not starting torrent due to configuration setting.') return True
def sender(self, filename, test=False): if mylar.CONFIG.NZBGET_PRIORITY: if any([ mylar.CONFIG.NZBGET_PRIORITY == 'Default', mylar.CONFIG.NZBGET_PRIORITY == 'Normal' ]): nzbgetpriority = 0 elif mylar.CONFIG.NZBGET_PRIORITY == 'Low': nzbgetpriority = -50 elif mylar.CONFIG.NZBGET_PRIORITY == 'High': nzbgetpriority = 50 elif mylar.CONFIG.NZBGET_PRIORITY == 'Very High': nzbgetpriority = 100 elif mylar.CONFIG.NZBGET_PRIORITY == 'Force': nzbgetpriority = 900 #there's no priority for "paused", so set "Very Low" and deal with that later... elif mylar.CONFIG.NZBGET_PRIORITY == 'Paused': nzbgetpriority = -100 else: #if nzbget priority isn't selected, default to Normal (0) nzbgetpriority = 0 with open(filename, 'rb') as in_file: nzbcontent = in_file.read() nzbcontent64 = standard_b64encode(nzbcontent).decode('utf-8') try: logger.fdebug('sending now to %s' % self.display_url) if mylar.CONFIG.NZBGET_CATEGORY is None: nzb_category = '' else: nzb_category = mylar.CONFIG.NZBGET_CATEGORY sendresponse = self.server.append(filename, nzbcontent64, nzb_category, nzbgetpriority, False, False, '', 0, 'SCORE') except http.client.socket.error as e: nzb_url = re.sub(mylar.CONFIG.NZBGET_PASSWORD, 'REDACTED', str(e)) logger.error( 'Please check your NZBget host and port (if it is running). Tested against: %s' % nzb_url) return {'status': False} except xmlrpc.client.ProtocolError as e: logger.info(e, ) if e.errmsg == "Unauthorized": err = re.sub(mylar.CONFIG.NZBGET_PASSWORD, 'REDACTED', e.errmsg) logger.error('Unauthorized username / password provided: %s' % err) return {'status': False} else: err = "Protocol Error: %s" % re.sub( mylar.CONFIG.NZBGET_PASSWORD, 'REDACTED', e.errmsg) logger.error('Protocol error returned: %s' % err) return {'status': False} except Exception as e: logger.warn( 'uh-oh: %s' % re.sub(mylar.CONFIG.NZBGET_PASSWORD, 'REDACTED', str(e))) return {'status': False} else: if sendresponse <= 0: logger.warn( 'Invalid response received after sending to NZBGet: %s' % sendresponse) return {'status': False} else: #sendresponse is the NZBID that we use to track the progress.... return {'status': True, 'NZBID': sendresponse}
def extract_image(location, single=False, imquality=None): #location = full path to the cbr/cbz (filename included in path) #single = should be set to True so that a single file can have the coverfile # extracted and have the cover location returned to the calling function #imquality = the calling function ('notif' for notifications will initiate a resize image before saving the cover) if PIL_Found is False: return cover = "notfound" pic_extensions = ('.jpg', '.png', '.webp') issue_ends = ('1', '0') modtime = os.path.getmtime(location) low_infile = 9999999999999 low_num = 1000 local_filename = os.path.join(mylar.CONFIG.CACHE_DIR, 'temp_notif') cb_filename = None cb_filenames = [] metadata = None if single is True: if location.endswith(".cbz"): location_in = zipfile.ZipFile(location) dir_opt = 'is_dir' actual_ext = '.cbz' else: try: location_in = rarfile.RarFile(location) dir_opt = 'isdir' actual_ext = '.cbr' except rarfile.BadRarFile as e: logger.warn('[WARNING] %s: %s' % (location, e)) try: logger.info( 'Trying to see if this is a zip renamed as a rar: %s' % (location)) location_in = zipfile.ZipFile(location) dir_opt = 'is_dir' actual_ext = '.cbz' except Exception as e: logger.warn('[EXCEPTION] %s' % e) return except: logger.warn('[EXCEPTION]: %s' % sys.exec_info()[0]) return try: for infile in location_in.infolist(): basename = os.path.basename(infile.filename) if infile.filename == 'ComicInfo.xml': logger.fdebug('Extracting ComicInfo.xml to display.') metadata = location_in.read(infile.filename) if cover == 'found': break filename, extension = os.path.splitext(basename) tmp_infile = re.sub("[^0-9]", "", filename).strip() if any([ tmp_infile == '', not getattr(infile, dir_opt), 'zzz' in filename ]): continue #logger.fdebug('[%s]issue_ends: %s' % (tmp_infile, tmp_infile.endswith(issue_ends))) #logger.fdebug('ext_ends: %s' % infile.filename.lower().endswith(pic_extensions)) #logger.fdebug('(%s) < (%s) == %s' % (int(tmp_infile), int(low_infile), int(tmp_infile)<int(low_infile))) #logger.fdebug('is_dir == %s' % (not getattr(infile, dir_opt))) if all([ infile.filename.lower().endswith(pic_extensions), int(tmp_infile) < int(low_infile) ]): low_infile = tmp_infile low_infile_name = infile.filename elif any([ '00a' in infile.filename, '00b' in infile.filename, '00c' in infile.filename, '00d' in infile.filename, '00e' in infile.filename, '00fc' in infile.filename.lower() ]) and infile.filename.endswith( pic_extensions) and cover == "notfound": altlist = ('00a', '00b', '00c', '00d', '00e', '00fc') for alt in altlist: if alt in infile.filename.lower(): cb_filename = infile.filename cover = "found" #logger.fdebug('[%s] cover found:%s' % (alt, infile.filename)) break elif all([ tmp_infile.endswith(issue_ends), infile.filename.lower().endswith(pic_extensions), int(tmp_infile) < int(low_infile), cover == 'notfound' ]): cb_filenames.append(infile.filename) #logger.fdebug('filename set to: %s' % infile.filename) #low_infile_name = infile.filename #low_infile = tmp_infile if cover != "found" and any( [len(cb_filenames) > 0, low_infile != 9999999999999]): logger.fdebug( 'Invalid naming sequence for jpgs discovered. Attempting to find the lowest sequence and will use as cover (it might not work). Currently : %s' % (low_infile_name)) cb_filename = low_infile_name cover = "found" except Exception as e: logger.error( '[ERROR] Unable to properly retrieve the cover. It\'s probably best to re-tag this file : %s' % e) return logger.fdebug('cb_filename set to : %s' % cb_filename) if extension is not None: ComicImage = local_filename + extension try: insidefile = location_in.getinfo(cb_filename) img = Image.open(BytesIO(location_in.read(insidefile))) wpercent = (600 / float(img.size[0])) hsize = int((float(img.size[1]) * float(wpercent))) img = img.resize((600, hsize), Image.ANTIALIAS) output = BytesIO() img.save(output, format="JPEG") try: ComicImage = str(base64.b64encode(output.getvalue()), 'utf-8') except Exception as e: ComicImage = str( base64.b64encode(output.getvalue() + "==="), 'utf-8') output.close() except Exception as e: logger.warn('[WARNING] Unable to resize existing image: %s' % e) else: ComicImage = local_filename return {'ComicImage': ComicImage, 'metadata': metadata}
def notify(self, snline=None, prline=None, prline2=None, snatched=None, sent_to=None, prov=None, module=None, method=None): if module is None: module = '' module += '[NOTIFIER]' # http_handler = HTTPSConnection("api.pushbullet.com") # if method == 'GET': # uri = '/v2/devices' # else: # method = 'POST' # uri = '/v2/pushes' # authString = base64.b64encode(self.apikey + ":") if method == 'GET': pass # http_handler.request(method, uri, None, headers={'Authorization': 'Basic %s:' % authString}) else: if snatched: if snatched[-1] == '.': snatched = snatched[:-1] event = snline message = "Mylar has snatched: " + snatched + " from " + prov + " and has sent it to " + sent_to else: event = prline + ' complete!' message = prline2 data = { 'type': "note", #'device_iden': self.deviceid, 'title': event.encode('utf-8'), #"mylar", 'body': message.encode('utf-8') } r = self._session.post(self.PUSH_URL, data=json.dumps(data)) dt = r.json() if r.status_code == 200: if method == 'GET': return dt else: logger.info(module + ' PushBullet notifications sent.') return { 'status': True, 'message': 'APIKEY verified OK / notification sent' } elif r.status_code >= 400 and r.status_code < 500: logger.error(module + ' PushBullet request failed: %s' % r.content) return { 'status': False, 'message': '[' + str(r.status_code) + '] ' + dt['error']['message'] } else: logger.error(module + ' PushBullet notification failed serverside: %s' % r.content) return { 'status': False, 'message': '[' + str(r.status_code) + '] ' + dt['error']['message'] }
def main(): # Fixed paths to mylar if hasattr(sys, 'frozen'): mylar.FULL_PATH = os.path.abspath(sys.executable) else: mylar.FULL_PATH = os.path.abspath(__file__) mylar.PROG_DIR = os.path.dirname(mylar.FULL_PATH) mylar.ARGS = sys.argv[1:] # From sickbeard mylar.SYS_ENCODING = None try: locale.setlocale(locale.LC_ALL, "") mylar.SYS_ENCODING = locale.getpreferredencoding() except (locale.Error, IOError): pass # for OSes that are poorly configured I'll just force UTF-8 if not mylar.SYS_ENCODING or mylar.SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): mylar.SYS_ENCODING = 'UTF-8' if not logger.LOG_LANG.startswith('en'): print( 'language detected as non-English (%s). Forcing specific logging module - errors WILL NOT be captured in the logs' % logger.LOG_LANG) else: print('log language set to %s' % logger.LOG_LANG) # Set up and gather command line arguments parser = argparse.ArgumentParser( description='Automated Comic Book Downloader') subparsers = parser.add_subparsers(title='Subcommands', dest='maintenance') parser_maintenance = subparsers.add_parser( 'maintenance', help= 'Enter maintenance mode (no GUI). Additional commands are available (maintenance --help)' ) #main parser parser.add_argument('-v', '--verbose', action='store_true', help='Increase console logging verbosity') parser.add_argument('-q', '--quiet', action='store_true', help='Turn off console logging') parser.add_argument('-d', '--daemon', action='store_true', help='Run as a daemon') parser.add_argument('-p', '--port', type=int, help='Force mylar to run on a specified port') parser.add_argument( '-b', '--backup', action='store_true', help= 'Will automatically backup & keep the last 2 copies of the .db & ini files prior to startup' ) parser.add_argument( '-w', '--noweekly', action='store_true', help= 'Turn off weekly pull list check on startup (quicker boot sequence)') parser.add_argument( '--datadir', help='Specify a directory where to store your data files') parser.add_argument('--config', help='Specify a config file to use') parser.add_argument('--nolaunch', action='store_true', help='Prevent browser from launching on startup') parser.add_argument( '--pidfile', help='Create a pid file (only relevant when running as a daemon)') parser.add_argument( '--safe', action='store_true', help= 'redirect the startup page to point to the Manage Comics screen on startup' ) parser_maintenance.add_argument( '-xj', '--exportjson', action='store', help='Export existing mylar.db to json file') parser_maintenance.add_argument('-id', '--importdatabase', action='store', help='Import a mylar.db into current db') parser_maintenance.add_argument( '-ij', '--importjson', action='store', help= 'Import a specified json file containing just {"ComicID": "XXXXX"} into current db' ) parser_maintenance.add_argument('-st', '--importstatus', action='store_true', help='Provide current maintenance status') parser_maintenance.add_argument( '-u', '--update', action='store_true', help='force mylar to perform an update as if in GUI') parser_maintenance.add_argument( '-fs', '--fixslashes', action='store_true', help='remove double-slashes from within paths in db') #parser_maintenance.add_argument('-it', '--importtext', action='store', help='Import a specified text file into current db') args = parser.parse_args() if args.maintenance: if all([ args.exportjson is None, args.importdatabase is None, args.importjson is None, args.importstatus is False, args.update is False, args.fixslashes is False ]): print( 'Expecting subcommand with the maintenance positional argumeent' ) sys.exit() mylar.MAINTENANCE = True else: mylar.MAINTENANCE = False if args.verbose: print('Verbose/Debugging mode enabled...') mylar.LOG_LEVEL = 2 elif args.quiet: mylar.QUIET = True print('Quiet logging mode enabled...') mylar.LOG_LEVEL = 0 else: mylar.LOG_LEVEL = 1 if args.daemon: if sys.platform == 'win32': print("Daemonize not supported under Windows, starting normally") else: mylar.DAEMON = True if args.pidfile: mylar.PIDFILE = str(args.pidfile) # If the pidfile already exists, mylar may still be running, so exit if os.path.exists(mylar.PIDFILE): sys.exit("PID file '" + mylar.PIDFILE + "' already exists. Exiting.") # The pidfile is only useful in daemon mode, make sure we can write the file properly if mylar.DAEMON: mylar.CREATEPID = True try: open(mylar.PIDFILE, 'w').write("pid\n") except IOError as e: raise SystemExit("Unable to write PID file: %s [%d]" % (e.strerror, e.errno)) else: print("Not running in daemon mode. PID file creation disabled.") if args.datadir: mylar.DATA_DIR = args.datadir else: mylar.DATA_DIR = mylar.PROG_DIR if args.config: mylar.CONFIG_FILE = args.config else: mylar.CONFIG_FILE = os.path.join(mylar.DATA_DIR, 'config.ini') if args.safe: mylar.SAFESTART = True else: mylar.SAFESTART = False if args.noweekly: mylar.NOWEEKLY = True else: mylar.NOWEEKLY = False # Put the database in the DATA_DIR mylar.DB_FILE = os.path.join(mylar.DATA_DIR, 'mylar.db') # Read config and start logging if mylar.MAINTENANCE is False: print('Initializing startup sequence....') #try: mylar.initialize(mylar.CONFIG_FILE) #except Exception as e: # print e # raise SystemExit('FATAL ERROR') if mylar.MAINTENANCE is False: filechecker.validateAndCreateDirectory(mylar.DATA_DIR, True) # Make sure the DATA_DIR is writeable if not os.access(mylar.DATA_DIR, os.W_OK): raise SystemExit('Cannot write to the data directory: ' + mylar.DATA_DIR + '. Exiting...') # backup the db and configs before they load. if args.backup: print('[AUTO-BACKUP] Backing up .db and config.ini files for safety.') backupdir = os.path.join(mylar.DATA_DIR, 'backup') try: os.makedirs(backupdir) print( '[AUTO-BACKUP] Directory does not exist for backup - creating : ' + backupdir) except OSError as exception: if exception.errno != errno.EEXIST: print('[AUTO-BACKUP] Directory already exists.') raise i = 0 while (i < 2): if i == 0: ogfile = mylar.DB_FILE back = os.path.join(backupdir, 'mylar.db') back_1 = os.path.join(backupdir, 'mylar.db.1') else: ogfile = mylar.CONFIG_FILE back = os.path.join(backupdir, 'config.ini') back_1 = os.path.join(backupdir, 'config.ini.1') try: print('[AUTO-BACKUP] Now Backing up mylar.db file') if os.path.isfile(back_1): print('[AUTO-BACKUP] ' + back_1 + ' exists. Deleting and keeping new.') os.remove(back_1) if os.path.isfile(back): print('[AUTO-BACKUP] Now renaming ' + back + ' to ' + back_1) shutil.move(back, back_1) print('[AUTO-BACKUP] Now copying db file to ' + back) shutil.copy(ogfile, back) except OSError as exception: if exception.errno != errno.EXIST: raise i += 1 # Rename the main thread threading.currentThread().name = "MAIN" if mylar.DAEMON: mylar.daemonize() if mylar.MAINTENANCE is True and any([ args.exportjson, args.importjson, args.update is True, args.importstatus is True, args.fixslashes is True ]): loggermode = '[MAINTENANCE-MODE]' if args.importstatus: #mylar.MAINTENANCE is True: cs = maintenance.Maintenance('status') cstat = cs.check_status() else: logger.info('%s Initializing maintenance mode' % loggermode) if args.update is True: logger.info( '%s Attempting to update Mylar so things can work again...' % loggermode) try: mylar.shutdown(restart=True, update=True, maintenance=True) except Exception as e: sys.exit('%s Mylar failed to update: %s' % (loggermode, e)) elif args.importdatabase: #for attempted db import. maintenance_path = args.importdatabase logger.info('%s db path accepted as %s' % (loggermode, maintenance_path)) di = maintenance.Maintenance('database-import', file=maintenance_path) d = di.database_import() elif args.importjson: #for attempted file re-import (json format) maintenance_path = args.importjson logger.info( '%s file indicated as being in json format - path accepted as %s' % (loggermode, maintenance_path)) ij = maintenance.Maintenance('json-import', file=maintenance_path) j = ij.json_import() #elif args.importtext: # #for attempted file re-import (list format) # maintenance_path = args.importtext # logger.info('%s file indicated as being in list format - path accepted as %s' % (loggermode, maintenance_path)) # it = maintenance.Maintenance('list-import', file=maintenance_path) # t = it.list_import() elif args.exportjson: #for export of db comicid's in json format maintenance_path = args.exportjson logger.info( '%s file indicated as being written to json format - destination accepted as %s' % (loggermode, maintenance_path)) ej = maintenance.Maintenance('json-export', output=maintenance_path) j = ej.json_export() elif args.fixslashes: #for running the fix slashes on the db manually logger.info('%s method indicated as fix slashes' % loggermode) fs = maintenance.Maintenance('fixslashes') j = fs.fix_slashes() else: logger.info('%s Not a valid command: %s' % (loggermode, maintenance_info)) sys.exit() logger.info('%s Exiting Maintenance mode' % (loggermode)) #possible option to restart automatically after maintenance has completed... sys.exit() # Force the http port if neccessary if args.port: http_port = args.port logger.info('Starting Mylar on forced port: %i' % http_port) else: http_port = int(mylar.CONFIG.HTTP_PORT) # Check if pyOpenSSL is installed. It is required for certificate generation # and for cherrypy. if mylar.CONFIG.ENABLE_HTTPS: try: import OpenSSL except ImportError: logger.warn("The pyOpenSSL module is missing. Install this " \ "module to enable HTTPS. HTTPS will be disabled.") mylar.CONFIG.ENABLE_HTTPS = False # Try to start the server. Will exit here is address is already in use. web_config = { 'http_port': http_port, 'http_host': mylar.CONFIG.HTTP_HOST, 'http_root': mylar.CONFIG.HTTP_ROOT, 'enable_https': mylar.CONFIG.ENABLE_HTTPS, 'https_cert': mylar.CONFIG.HTTPS_CERT, 'https_key': mylar.CONFIG.HTTPS_KEY, 'https_chain': mylar.CONFIG.HTTPS_CHAIN, 'http_username': mylar.CONFIG.HTTP_USERNAME, 'http_password': mylar.CONFIG.HTTP_PASSWORD, 'authentication': mylar.CONFIG.AUTHENTICATION, 'login_timeout': mylar.CONFIG.LOGIN_TIMEOUT, 'opds_enable': mylar.CONFIG.OPDS_ENABLE, 'opds_authentication': mylar.CONFIG.OPDS_AUTHENTICATION, 'opds_username': mylar.CONFIG.OPDS_USERNAME, 'opds_password': mylar.CONFIG.OPDS_PASSWORD, 'opds_pagesize': mylar.CONFIG.OPDS_PAGESIZE, } # Try to start the server. webstart.initialize(web_config) #check for version here after web server initialized so it doesn't try to repeatidly hit github #for version info if it's already running versioncheck.versionload() if mylar.CONFIG.LAUNCH_BROWSER and not args.nolaunch: mylar.launch_browser(mylar.CONFIG.HTTP_HOST, http_port, mylar.CONFIG.HTTP_ROOT) # Start the background threads mylar.start() signal.signal(signal.SIGTERM, handler_sigterm) while True: if not mylar.SIGNAL: try: time.sleep(1) except KeyboardInterrupt: mylar.SIGNAL = 'shutdown' else: logger.info('Received signal: ' + mylar.SIGNAL) if mylar.SIGNAL == 'shutdown': mylar.shutdown() elif mylar.SIGNAL == 'restart': mylar.shutdown(restart=True) else: mylar.shutdown(restart=True, update=True) mylar.SIGNAL = None return