def run(self):
logger.info('[VersionCheck] Checking for new release on Github.')
helpers.job_management(write=True, job='Check Version', current_run=helpers.utctimestamp(), status='Running')
mylar.VERSION_STATUS = 'Running'
versioncheck.checkGithub()
helpers.job_management(write=True, job='Check Version', last_run_completed=helpers.utctimestamp(), status='Waiting')
mylar.VERSION_STATUS = 'Waiting'
def run(self):
logger.info('[SEARCH] Running Search for Wanted.')
helpers.job_management(write=True, job='Auto-Search', current_run=helpers.utctimestamp(), status='Running')
mylar.SEARCH_STATUS = 'Running'
mylar.search.searchforissue()
helpers.job_management(write=True, job='Auto-Search', last_run_completed=helpers.utctimestamp(), status='Waiting')
mylar.SEARCH_STATUS = 'Waiting'
def run(self):
logger.info('[WEEKLY] Checking Weekly Pull-list for new releases/updates')
helpers.job_management(write=True, job='Weekly Pullist', current_run=helpers.utctimestamp(), status='Running')
mylar.WEEKLY_STATUS = 'Running'
weeklypull.pullit()
weeklypull.future_check()
helpers.job_management(write=True, job='Weekly Pullist', last_run_completed=helpers.utctimestamp(), status='Waiting')
mylar.WEEKLY_STATUS = 'Waiting'
def run(self, sched):
logger.info('[DBUpdate] Updating Database.')
helpers.job_management(write=True,
job='DB Updater',
current_run=helpers.utctimestamp(),
status='Running')
mylar.updater.dbUpdate(sched=sched)
helpers.job_management(write=True,
job='DB Updater',
last_run_completed=helpers.utctimestamp(),
status='Waiting')
def run(self):
logger.info('[SEARCH] Running Search for Wanted.')
helpers.job_management(write=True,
job='Auto-Search',
current_run=helpers.utctimestamp(),
status='Running')
mylar.SEARCH_STATUS = 'Running'
mylar.search.searchforissue()
helpers.job_management(write=True,
job='Auto-Search',
last_run_completed=helpers.utctimestamp(),
status='Waiting')
def run(self):
logger.info('[VersionCheck] Checking for new release on Github.')
helpers.job_management(write=True,
job='Check Version',
current_run=helpers.utctimestamp(),
status='Running')
mylar.VERSION_STATUS = 'Running'
versioncheck.checkGithub()
helpers.job_management(write=True,
job='Check Version',
last_run_completed=helpers.utctimestamp(),
status='Waiting')
mylar.VERSION_STATUS = 'Waiting'
def run(self):
logger.info(
'[WEEKLY] Checking Weekly Pull-list for new releases/updates')
helpers.job_management(write=True,
job='Weekly Pullist',
current_run=helpers.utctimestamp(),
status='Running')
mylar.WEEKLY_STATUS = 'Running'
weeklypull.pullit()
weeklypull.future_check()
helpers.job_management(write=True,
job='Weekly Pullist',
last_run_completed=helpers.utctimestamp(),
status='Waiting')
def update_db(self):
# mylar.MAINTENANCE_UPDATE will indicate what's being updated in the db
if mylar.MAINTENANCE_UPDATE:
self.db_version_check(display=False)
# backup mylar.db here
self.backup_files(dbs=True)
for dmode in mylar.MAINTENANCE_UPDATE:
if dmode['mode'] == 'rss update':
logger.info(
'[MAINTENANCE-MODE][DB-CONVERSION] Updating dB due to RSS table conversion'
)
if dmode['resume'] > 0:
logger.info(
'[MAINTENANCE-MODE][DB-CONVERSION][DB-RECOVERY] Attempting to resume conversion from previous run (starting at record: %s)'
% dmode['resume'])
#force set logging to warning level only so the progress indicator can be displayed in console
prev_log_level = mylar.LOG_LEVEL
self.toggle_logging(level=0)
if dmode['mode'] == 'rss update':
self.sql_attachmylar()
row_cnt = self.dbmylar.execute(
"SELECT COUNT(rowid) as count FROM rssdb")
rowcnt = row_cnt.fetchone()[0]
mylar.MAINTENANCE_DB_TOTAL = rowcnt
if dmode['resume'] > 0:
xt = self.dbmylar.execute(
"SELECT rowid, Title FROM rssdb WHERE rowid >= ? ORDER BY rowid ASC",
[dmode['resume']])
else:
xt = self.dbmylar.execute(
"SELECT rowid, Title FROM rssdb ORDER BY rowid ASC"
)
xlist = xt.fetchall()
mylar.MAINTENANCE_DB_COUNT = 0
if xlist is None:
print('Nothing in the rssdb to update. Ignoring.')
return True
try:
if dmode['resume'] > 0 and xlist is not None:
logger.info('resume set at : %s' %
(xlist[dmode['resume']], ))
#xlist[dmode['resume']:]
mylar.MAINTENANCE_DB_COUNT = dmode['resume']
except Exception as e:
print(
'[ERROR:%s] - table resume location is not accureate. Starting from start, but this should go quick..'
% e)
xt = self.dbmylar.execute(
"SELECT rowid, Title FROM rssdb ORDER BY rowid ASC"
)
xlist = xt.fetchall()
dmode['resume'] = 0
if xlist:
resultlist = []
delete_rows = []
for x in self.progressBar(xlist,
prefix='Progress',
suffix='Complete',
length=50,
resume=dmode['resume']):
#signal capture here since we can't do it as per normal
if any([
mylar.SIGNAL == 'shutdown',
mylar.SIGNAL == 'restart'
]):
try:
self.dbmylar.executemany(
"UPDATE rssdb SET Issue_Number=?, ComicName=? WHERE rowid=?",
(resultlist))
self.sql_closemylar()
except Exception as e:
print('error: %s' % e)
else:
send_it = {
'mode': dmode['mode'],
'version': self.db_version,
'status': 'incomplete',
'total': mylar.MAINTENANCE_DB_TOTAL,
'current': mylar.MAINTENANCE_DB_COUNT,
'last_run': helpers.utctimestamp()
}
self.db_update_status(send_it)
#toggle back the logging level to what it was originally.
self.toggle_logging(level=prev_log_level)
if mylar.SIGNAL == 'shutdown':
logger.info(
'[MAINTENANCE-MODE][DB-CONVERSION][SHUTDOWN]Shutting Down...'
)
return False
else:
logger.info(
'[MAINTENANCE-MODE][DB-CONVERSION][RESTART]Restarting...'
)
return True
mylar.MAINTENANCE_DB_COUNT += 1
if not x[1]:
logger.fdebug(
'[MAINTENANCE-MODE][DB-CONVERSION][JUNK-NAME] %s'
% x[1])
delete_rows.append((x[0], ))
continue
try:
if any(
ext in x[1] for ext in
['yenc', '.pdf', '.rar', '.mp4', '.avi']):
logger.fdebug(
'[MAINTENANCE-MODE][DB-CONVERSION][JUNK-NAME] %s'
% x[1])
delete_rows.append((x[0], ))
continue
else:
flc = filechecker.FileChecker(file=x[1])
filelist = flc.listFiles()
except Exception as e:
logger.fdebug(
'[MAINTENANCE-MODE][DB-CONVERSION][JUNK-NAME] %s'
% x[1])
delete_rows.append((x[0], ))
continue
else:
if all([
filelist['series_name'] != '',
filelist['series_name'] is not None
]) and filelist['issue_number'] != '-':
issuenumber = filelist['issue_number']
seriesname = re.sub(
r'[\u2014|\u2013|\u2e3a|\u2e3b]', '-',
filelist['series_name']).strip()
if seriesname.endswith(
'-') and '#' in seriesname[-6:]:
ck1 = seriesname.rfind('#')
ck2 = seriesname.rfind('-')
if seriesname[ck1 + 1:ck2 -
1].strip().isdigit():
issuenumber = '%s %s' % (
seriesname[ck1:].strip(),
issuenumber)
seriesname = seriesname[:ck1 -
1].strip()
issuenumber.strip()
resultlist.append(
(issuenumber, seriesname.strip(),
x[0]))
if len(resultlist) > 500:
# write it out every 5000 records.
try:
logger.fdebug('resultlist: %s' %
(resultlist, ))
self.dbmylar.executemany(
"UPDATE rssdb SET Issue_Number=?, ComicName=? WHERE rowid=?",
(resultlist))
self.sql_closemylar()
# update the update_db so if it has to resume it doesn't from the beginning or wrong point ( last 5000th write ).
send_it = {
'mode': dmode['mode'],
'version': self.db_version,
'status': 'incomplete',
'total':
mylar.MAINTENANCE_DB_TOTAL,
'current':
mylar.MAINTENANCE_DB_COUNT,
'last_run': helpers.utctimestamp()
}
self.db_update_status(send_it)
except Exception as e:
print('error: %s' % e)
return False
else:
logger.fdebug('reattaching')
self.sql_attachmylar()
resultlist = []
try:
if len(resultlist) > 0:
self.dbmylar.executemany(
"UPDATE rssdb SET Issue_Number=?, ComicName=? WHERE rowid=?",
(resultlist))
self.sql_closemylar()
except Exception as e:
print('error: %s' % e)
return False
else:
try:
send_it = {
'mode': dmode['mode'],
'version': 1,
'status': 'complete',
'total': mylar.MAINTENANCE_DB_TOTAL,
'current': mylar.MAINTENANCE_DB_COUNT,
'last_run': helpers.utctimestamp()
}
except Exception as e:
print('error_sendit: %s' % e)
else:
self.db_update_status(send_it)
if delete_rows:
# only do this on completion, or else the rowids will be different and it will mess up a rerun
try:
self.sql_attachmylar()
print(
'[MAINTENANCE-MODE][DB-CONVERSION][CLEANUP] Removing %s invalid RSS entries from table...'
% len(delete_rows))
self.dbmylar.executemany(
"DELETE FROM rssdb WHERE rowid=?",
(delete_rows))
self.sql_closemylar()
except Exception as e:
print('error: %s' % e)
else:
self.sql_attachmylar()
print(
'[MAINTENANCE-MODE][DB-CONVERSION][CLEANUP] Cleaning up...'
)
self.dbmylar.execute("VACUUM")
else:
print(
'[MAINTENANCE-MODE][DB-CONVERSION][CLEANUP] Cleaning up...'
)
self.sql_attachmylar()
self.dbmylar.execute("VACUUM")
self.sql_closemylar()
#toggle back the logging level to what it was originally.
self.toggle_logging(level=prev_log_level)
logger.info(
'[MAINTENANCE-MODE][DB-CONVERSION] Updating dB complete! (%s / %s)'
% (mylar.MAINTENANCE_DB_COUNT,
mylar.MAINTENANCE_DB_TOTAL))
mylar.MAINTENANCE_UPDATE[:] = [
x for x in mylar.MAINTENANCE_UPDATE
if not ('rss update' == x.get('mode'))
]
else:
mylar.MAINTENANCE_DB_COUNT = 0
logger.info(
'[MAINTENANCE-MODE] Update DB set to start - but nothing was provided as to what. Returning to non-maintenance mode'
)
return True
def run(self, forcerss=None):
with rss_lock:
#logger.info('[RSS-FEEDS] RSS Feed Check was last run at : ' + str(mylar.SCHED_RSS_LAST))
firstrun = "no"
#check the last run of rss to make sure it's not hammering.
if mylar.SCHED_RSS_LAST is None or mylar.SCHED_RSS_LAST == '' or mylar.SCHED_RSS_LAST == '0' or forcerss == True:
logger.info('[RSS-FEEDS] RSS Feed Check Initalizing....')
firstrun = "yes"
duration_diff = 0
else:
tstamp = float(mylar.SCHED_RSS_LAST)
duration_diff = abs(helpers.utctimestamp() - tstamp) / 60
#logger.fdebug('[RSS-FEEDS] Duration diff: %s' % duration_diff)
if firstrun == "no" and duration_diff < int(
mylar.CONFIG.RSS_CHECKINTERVAL):
logger.fdebug(
'[RSS-FEEDS] RSS Check has taken place less than the threshold - not initiating at this time.'
)
return
helpers.job_management(write=True,
job='RSS Feeds',
current_run=helpers.utctimestamp(),
status='Running')
mylar.RSS_STATUS = 'Running'
#logger.fdebug('[RSS-FEEDS] Updated RSS Run time to : ' + str(mylar.SCHED_RSS_LAST))
#function for looping through nzbs/torrent feeds
if mylar.CONFIG.ENABLE_TORRENT_SEARCH:
logger.info('[RSS-FEEDS] Initiating Torrent RSS Check.')
if mylar.CONFIG.ENABLE_PUBLIC:
logger.info(
'[RSS-FEEDS] Initiating Torrent RSS Feed Check on Demonoid / WorldWideTorrents.'
)
#rsscheck.torrents(pickfeed='3') #TP.SE RSS Check (has to be page-parsed)
rsscheck.torrents(pickfeed='Public'
) #TPSE = DEM RSS Check + WWT RSS Check
if mylar.CONFIG.ENABLE_32P:
logger.info(
'[RSS-FEEDS] Initiating Torrent RSS Feed Check on 32P.'
)
if mylar.CONFIG.MODE_32P == 0:
logger.fdebug(
'[RSS-FEEDS] 32P mode set to Legacy mode. Monitoring New Releases feed only.'
)
if any([
mylar.CONFIG.PASSKEY_32P is None,
mylar.CONFIG.PASSKEY_32P == '',
mylar.CONFIG.RSSFEED_32P is None,
mylar.CONFIG.RSSFEED_32P == ''
]):
logger.error(
'[RSS-FEEDS] Unable to validate information from provided RSS Feed. Verify that the feed provided is a current one.'
)
else:
rsscheck.torrents(pickfeed='1',
feedinfo=mylar.KEYS_32P)
else:
logger.fdebug(
'[RSS-FEEDS] 32P mode set to Auth mode. Monitoring all personal notification feeds & New Releases feed'
)
if any([
mylar.CONFIG.USERNAME_32P is None,
mylar.CONFIG.USERNAME_32P == '',
mylar.CONFIG.PASSWORD_32P is None
]):
logger.error(
'[RSS-FEEDS] Unable to sign-on to 32P to validate settings. Please enter/check your username password in the configuration.'
)
else:
if mylar.KEYS_32P is None:
feed32p = auth32p.info32p()
feedinfo = feed32p.authenticate()
if feedinfo != "disable":
pass
else:
mylar.CONFIG.ENABLE_32P = 0
#mylar.config_write()
else:
feedinfo = mylar.FEEDINFO_32P
if feedinfo is None or len(
feedinfo) == 0 or feedinfo == "disable":
logger.error(
'[RSS-FEEDS] Unable to retrieve any information from 32P for RSS Feeds. Skipping for now.'
)
else:
rsscheck.torrents(pickfeed='1',
feedinfo=feedinfo[0])
x = 0
#assign personal feeds for 32p > +8
for fi in feedinfo:
x += 1
pfeed_32p = str(7 + x)
rsscheck.torrents(pickfeed=pfeed_32p,
feedinfo=fi)
logger.info(
'[RSS-FEEDS] Initiating RSS Feed Check for NZB Providers.')
rsscheck.nzbs(forcerss=forcerss)
logger.info('[RSS-FEEDS] RSS Feed Check/Update Complete')
logger.info('[RSS-FEEDS] Watchlist Check for new Releases')
mylar.search.searchforissue(rsscheck='yes')
logger.info('[RSS-FEEDS] Watchlist Check complete.')
if forcerss:
logger.info('[RSS-FEEDS] Successfully ran a forced RSS Check.')
helpers.job_management(write=True,
job='RSS Feeds',
last_run_completed=helpers.utctimestamp(),
status='Waiting')
mylar.RSS_STATUS = 'Waiting'
return True
def run(self, sched):
logger.info('[DBUpdate] Updating Database.')
helpers.job_management(write=True, job='DB Updater', current_run=helpers.utctimestamp(), status='Running')
mylar.updater.dbUpdate(sched=sched)
helpers.job_management(write=True, job='DB Updater', last_run_completed=helpers.utctimestamp(), status='Waiting')
def run(self, forcerss=None):
with rss_lock:
#logger.info('[RSS-FEEDS] RSS Feed Check was last run at : ' + str(mylar.SCHED_RSS_LAST))
firstrun = "no"
#check the last run of rss to make sure it's not hammering.
if mylar.SCHED_RSS_LAST is None or mylar.SCHED_RSS_LAST == '' or mylar.SCHED_RSS_LAST == '0' or forcerss == True:
logger.info('[RSS-FEEDS] RSS Feed Check Initalizing....')
firstrun = "yes"
duration_diff = 0
else:
tstamp = float(mylar.SCHED_RSS_LAST)
duration_diff = abs(helpers.utctimestamp() - tstamp)/60
#logger.fdebug('[RSS-FEEDS] Duration diff: %s' % duration_diff)
if firstrun == "no" and duration_diff < int(mylar.CONFIG.RSS_CHECKINTERVAL):
logger.fdebug('[RSS-FEEDS] RSS Check has taken place less than the threshold - not initiating at this time.')
return
helpers.job_management(write=True, job='RSS Feeds', current_run=helpers.utctimestamp(), status='Running')
mylar.RSS_STATUS = 'Running'
#logger.fdebug('[RSS-FEEDS] Updated RSS Run time to : ' + str(mylar.SCHED_RSS_LAST))
#function for looping through nzbs/torrent feeds
if mylar.CONFIG.ENABLE_TORRENT_SEARCH:
logger.info('[RSS-FEEDS] Initiating Torrent RSS Check.')
if mylar.CONFIG.ENABLE_PUBLIC:
logger.info('[RSS-FEEDS] Initiating Torrent RSS Feed Check on Demonoid / WorldWideTorrents.')
rsscheck.torrents(pickfeed='Public') #TPSE = DEM RSS Check + WWT RSS Check
if mylar.CONFIG.ENABLE_32P is True:
logger.info('[RSS-FEEDS] Initiating Torrent RSS Feed Check on 32P.')
if mylar.CONFIG.MODE_32P == 0:
logger.fdebug('[RSS-FEEDS] 32P mode set to Legacy mode. Monitoring New Releases feed only.')
if any([mylar.CONFIG.PASSKEY_32P is None, mylar.CONFIG.PASSKEY_32P == '', mylar.CONFIG.RSSFEED_32P is None, mylar.CONFIG.RSSFEED_32P == '']):
logger.error('[RSS-FEEDS] Unable to validate information from provided RSS Feed. Verify that the feed provided is a current one.')
else:
rsscheck.torrents(pickfeed='1', feedinfo=mylar.KEYS_32P)
else:
logger.fdebug('[RSS-FEEDS] 32P mode set to Auth mode. Monitoring all personal notification feeds & New Releases feed')
if any([mylar.CONFIG.USERNAME_32P is None, mylar.CONFIG.USERNAME_32P == '', mylar.CONFIG.PASSWORD_32P is None]):
logger.error('[RSS-FEEDS] Unable to sign-on to 32P to validate settings. Please enter/check your username password in the configuration.')
else:
if mylar.KEYS_32P is None:
feed32p = auth32p.info32p()
feedinfo = feed32p.authenticate()
if feedinfo != "disable":
pass
else:
helpers.disable_provider('32P')
else:
feedinfo = mylar.FEEDINFO_32P
if feedinfo is None or len(feedinfo) == 0 or feedinfo == "disable":
logger.error('[RSS-FEEDS] Unable to retrieve any information from 32P for RSS Feeds. Skipping for now.')
else:
rsscheck.torrents(pickfeed='1', feedinfo=feedinfo[0])
x = 0
#assign personal feeds for 32p > +8
for fi in feedinfo:
x+=1
pfeed_32p = str(7 + x)
rsscheck.torrents(pickfeed=pfeed_32p, feedinfo=fi)
logger.info('[RSS-FEEDS] Initiating RSS Feed Check for NZB Providers.')
rsscheck.nzbs(forcerss=forcerss)
logger.info('[RSS-FEEDS] RSS Feed Check/Update Complete')
logger.info('[RSS-FEEDS] Watchlist Check for new Releases')
mylar.search.searchforissue(rsscheck='yes')
logger.info('[RSS-FEEDS] Watchlist Check complete.')
if forcerss:
logger.info('[RSS-FEEDS] Successfully ran a forced RSS Check.')
helpers.job_management(write=True, job='RSS Feeds', last_run_completed=helpers.utctimestamp(), status='Waiting')
mylar.RSS_STATUS = 'Waiting'
return True
def walk_the_walk(self):
folder_location = mylar.CONFIG.FOLDER_CACHE_LOCATION
if folder_location is None:
return {'status': False}
logger.info('checking locally...')
filelist = None
logger.info('check_folder_cache: %s' % (mylar.CHECK_FOLDER_CACHE))
if mylar.CHECK_FOLDER_CACHE is not None:
rd = mylar.CHECK_FOLDER_CACHE #datetime.datetime.utcfromtimestamp(mylar.CHECK_FOLDER_CACHE)
rd_mins = rd + datetime.timedelta(seconds = 600) #10 minute cache retention
rd_now = datetime.datetime.utcfromtimestamp(time.time())
if calendar.timegm(rd_mins.utctimetuple()) > calendar.timegm(rd_now.utctimetuple()):
# if < 10 minutes since last check, use cached listing
logger.info('using cached folder listing since < 10 minutes since last file check.')
filelist = mylar.FOLDER_CACHE
if filelist is None:
logger.info('generating new directory listing for folder_cache')
flc = filechecker.FileChecker(folder_location, justparse=True, pp_mode=True)
mylar.FOLDER_CACHE = flc.listFiles()
mylar.CHECK_FOLDER_CACHE = datetime.datetime.utcfromtimestamp(helpers.utctimestamp())
local_status = False
filepath = None
filename = None
for fl in mylar.FOLDER_CACHE['comiclist']:
logger.info('fl: %s' % (fl,))
if self.arc is not None:
comicname = self.arc['ComicName']
corrected_type = None
alternatesearch = None
booktype = self.arc['Type']
publisher = self.arc['Publisher']
issuenumber = self.arc['IssueNumber']
issuedate = self.arc['IssueDate']
issuename = self.arc['IssueName']
issuestatus = self.arc['Status']
elif self.comic is not None:
comicname = self.comic['ComicName']
booktype = self.comic['Type']
corrected_type = self.comic['Corrected_Type']
alternatesearch = self.comic['AlternateSearch']
publisher = self.comic['ComicPublisher']
issuenumber = self.issue['Issue_Number']
issuedate = self.issue['IssueDate']
issuename = self.issue['IssueName']
issuestatus = self.issue['Status']
else:
# weekly - (one/off)
comicname = self.weekly['COMIC']
booktype = self.weekly['format']
corrected_type = None
alternatesearch = None
publisher = self.weekly['PUBLISHER']
issuenumber = self.weekly['ISSUE']
issuedate = self.weekly['SHIPDATE']
issuename = None
issuestatus = self.weekly['STATUS']
if booktype is not None:
if (all([booktype != 'Print', booktype != 'Digital', booktype != 'None', booktype is not None]) and corrected_type != 'Print') or any([corrected_type == 'TPB', corrected_type == 'GN', corrected_type == 'HC']):
if booktype == 'One-Shot' and corrected_type is None:
booktype = 'One-Shot'
else:
if booktype == 'GN' and corrected_type is None:
booktype = 'GN'
elif booktype == 'HC' and corrected_type is None:
booktype = 'HC'
else:
booktype = 'TPB'
wm = filechecker.FileChecker(watchcomic=comicname, Publisher=publisher, AlternateSearch=alternatesearch)
watchmatch = wm.matchIT(fl)
logger.info('watchmatch: %s' % (watchmatch,))
# this is all for a really general type of match - if passed, the post-processing checks will do the real brunt work
if watchmatch['process_status'] == 'fail':
continue
if watchmatch['justthedigits'] is not None:
temploc= watchmatch['justthedigits'].replace('_', ' ')
if "Director's Cut" not in temploc:
temploc = re.sub('[\#\']', '', temploc)
else:
if any([booktype == 'TPB', booktype =='GN', booktype == 'HC', booktype == 'One-Shot']):
temploc = '1'
else:
temploc = None
continue
int_iss = helpers.issuedigits(issuenumber)
issyear = issuedate[:4]
old_status = issuestatus
issname = issuename
if temploc is not None:
fcdigit = helpers.issuedigits(temploc)
elif any([booktype == 'TPB', booktype == 'GN', booktype == 'GC', booktype == 'One-Shot']) and temploc is None:
fcdigit = helpers.issuedigits('1')
if int(fcdigit) == int_iss:
logger.fdebug('[%s] Issue match - #%s' % (self.issueid, self.issue['Issue_Number']))
local_status = True
if watchmatch['sub'] is None:
filepath = watchmatch['comiclocation']
filename = watchmatch['comicfilename']
else:
filepath = os.path.join(watchmatch['comiclocation'], watchmatch['sub'])
filename = watchmatch['comicfilename']
break
#if local_status is True:
#try:
# copied_folder = os.path.join(mylar.CONFIG.CACHE_DIR, 'tmp_filer')
# if os.path.exists(copied_folder):
# shutil.rmtree(copied_folder)
# os.mkdir(copied_folder)
# logger.info('created temp directory: %s' % copied_folder)
# shutil.copy(os.path.join(filepath, filename), copied_folder)
#except Exception as e:
# logger.error('[%s] error: %s' % (e, filepath))
# filepath = None
# local_status = False
#else:
#filepath = os.path.join(copied_folder, filename)
#logger.info('Successfully copied file : %s' % filepath)
return {'status': local_status,
'filename': filename,
'filepath': filepath}
def start():
global _INITIALIZED, started
with INIT_LOCK:
if _INITIALIZED:
#load up the previous runs from the job sql table so we know stuff...
monitors = helpers.job_management()
SCHED_WEEKLY_LAST = monitors['weekly']
SCHED_SEARCH_LAST = monitors['search']
SCHED_UPDATER_LAST = monitors['dbupdater']
SCHED_MONITOR_LAST = monitors['monitor']
SCHED_VERSION_LAST = monitors['version']
SCHED_RSS_LAST = monitors['rss']
# Start our scheduled background tasks
SCHED.add_job(func=updater.dbUpdate, id='dbupdater', name='DB Updater', args=[None,None,True], trigger=IntervalTrigger(hours=0, minutes=5, timezone='UTC'))
#let's do a run at the Wanted issues here (on startup) if enabled.
ss = searchit.CurrentSearcher()
if CONFIG.NZB_STARTUP_SEARCH:
SCHED.add_job(func=ss.run, id='search', next_run_time=datetime.datetime.utcnow(), name='Auto-Search', trigger=IntervalTrigger(hours=0, minutes=CONFIG.SEARCH_INTERVAL, timezone='UTC'))
else:
if SCHED_SEARCH_LAST is not None:
search_timestamp = float(SCHED_SEARCH_LAST)
logger.fdebug('[AUTO-SEARCH] Search last run @ %s' % datetime.datetime.utcfromtimestamp(search_timestamp))
else:
search_timestamp = helpers.utctimestamp() + (int(CONFIG.SEARCH_INTERVAL) *60)
duration_diff = (helpers.utctimestamp() - search_timestamp)/60
if duration_diff >= int(CONFIG.SEARCH_INTERVAL):
logger.fdebug('[AUTO-SEARCH]Auto-Search set to a delay of one minute before initialization as it has been %s minutes since the last run' % duration_diff)
SCHED.add_job(func=ss.run, id='search', name='Auto-Search', trigger=IntervalTrigger(hours=0, minutes=CONFIG.SEARCH_INTERVAL, timezone='UTC'))
else:
search_diff = datetime.datetime.utcfromtimestamp(helpers.utctimestamp() + ((int(CONFIG.SEARCH_INTERVAL) * 60) - (duration_diff*60)))
logger.fdebug('[AUTO-SEARCH] Scheduling next run @ %s every %s minutes' % (search_diff, CONFIG.SEARCH_INTERVAL))
SCHED.add_job(func=ss.run, id='search', name='Auto-Search', next_run_time=search_diff, trigger=IntervalTrigger(hours=0, minutes=CONFIG.SEARCH_INTERVAL, timezone='UTC'))
if all([CONFIG.ENABLE_TORRENTS, CONFIG.AUTO_SNATCH, OS_DETECT != 'Windows']) and any([CONFIG.TORRENT_DOWNLOADER == 2, CONFIG.TORRENT_DOWNLOADER == 4]):
logger.info('[AUTO-SNATCHER] Auto-Snatch of completed torrents enabled & attempting to background load....')
SNPOOL = threading.Thread(target=helpers.worker_main, args=(SNATCHED_QUEUE,), name="AUTO-SNATCHER")
SNPOOL.start()
logger.info('[AUTO-SNATCHER] Succesfully started Auto-Snatch add-on - will now monitor for completed torrents on client....')
if CONFIG.POST_PROCESSING is True and ( all([CONFIG.NZB_DOWNLOADER == 0, CONFIG.SAB_CLIENT_POST_PROCESSING is True]) or all([CONFIG.NZB_DOWNLOADER == 1, CONFIG.NZBGET_CLIENT_POST_PROCESSING is True]) ):
if CONFIG.NZB_DOWNLOADER == 0:
logger.info('[SAB-MONITOR] Completed post-processing handling enabled for SABnzbd. Attempting to background load....')
elif CONFIG.NZB_DOWNLOADER == 1:
logger.info('[NZBGET-MONITOR] Completed post-processing handling enabled for NZBGet. Attempting to background load....')
NZBPOOL = threading.Thread(target=helpers.nzb_monitor, args=(NZB_QUEUE,), name="AUTO-COMPLETE-NZB")
NZBPOOL.start()
if CONFIG.NZB_DOWNLOADER == 0:
logger.info('[AUTO-COMPLETE-NZB] Succesfully started Completed post-processing handling for SABnzbd - will now monitor for completed nzbs within sabnzbd and post-process automatically....')
elif CONFIG.NZB_DOWNLOADER == 1:
logger.info('[AUTO-COMPLETE-NZB] Succesfully started Completed post-processing handling for NZBGet - will now monitor for completed nzbs within nzbget and post-process automatically....')
helpers.latestdate_fix()
if CONFIG.ALT_PULL == 2:
weektimer = 4
else:
weektimer = 24
#weekly pull list gets messed up if it's not populated first, so let's populate it then set the scheduler.
logger.info('[WEEKLY] Checking for existance of Weekly Comic listing...')
#now the scheduler (check every 24 hours)
weekly_interval = weektimer * 60 * 60
try:
if SCHED_WEEKLY_LAST:
pass
except:
SCHED_WEEKLY_LAST = None
weektimestamp = helpers.utctimestamp()
if SCHED_WEEKLY_LAST is not None:
weekly_timestamp = float(SCHED_WEEKLY_LAST)
else:
weekly_timestamp = weektimestamp + weekly_interval
ws = weeklypullit.Weekly()
duration_diff = (weektimestamp - weekly_timestamp)/60
if abs(duration_diff) >= weekly_interval/60:
logger.info('[WEEKLY] Weekly Pull-Update initializing immediately as it has been %s hours since the last run' % abs(duration_diff/60))
SCHED.add_job(func=ws.run, id='weekly', name='Weekly Pullist', next_run_time=datetime.datetime.utcnow(), trigger=IntervalTrigger(hours=weektimer, minutes=0, timezone='UTC'))
else:
weekly_diff = datetime.datetime.utcfromtimestamp(weektimestamp + (weekly_interval - (duration_diff * 60)))
logger.fdebug('[WEEKLY] Scheduling next run for @ %s every %s hours' % (weekly_diff, weektimer))
SCHED.add_job(func=ws.run, id='weekly', name='Weekly Pullist', next_run_time=weekly_diff, trigger=IntervalTrigger(hours=weektimer, minutes=0, timezone='UTC'))
#initiate startup rss feeds for torrents/nzbs here...
if CONFIG.ENABLE_RSS:
logger.info('[RSS-FEEDS] Initiating startup-RSS feed checks.')
if SCHED_RSS_LAST is not None:
rss_timestamp = float(SCHED_RSS_LAST)
logger.info('[RSS-FEEDS] RSS last run @ %s' % datetime.datetime.utcfromtimestamp(rss_timestamp))
else:
rss_timestamp = helpers.utctimestamp() + (int(CONFIG.RSS_CHECKINTERVAL) *60)
rs = rsscheckit.tehMain()
duration_diff = (helpers.utctimestamp() - rss_timestamp)/60
if duration_diff >= int(CONFIG.RSS_CHECKINTERVAL):
SCHED.add_job(func=rs.run, id='rss', name='RSS Feeds', args=[True], next_run_time=datetime.datetime.utcnow(), trigger=IntervalTrigger(hours=0, minutes=int(CONFIG.RSS_CHECKINTERVAL), timezone='UTC'))
else:
rss_diff = datetime.datetime.utcfromtimestamp(helpers.utctimestamp() + (int(CONFIG.RSS_CHECKINTERVAL) * 60) - (duration_diff * 60))
logger.fdebug('[RSS-FEEDS] Scheduling next run for @ %s every %s minutes' % (rss_diff, CONFIG.RSS_CHECKINTERVAL))
SCHED.add_job(func=rs.run, id='rss', name='RSS Feeds', args=[True], next_run_time=rss_diff, trigger=IntervalTrigger(hours=0, minutes=int(CONFIG.RSS_CHECKINTERVAL), timezone='UTC'))
if CONFIG.CHECK_GITHUB:
vs = versioncheckit.CheckVersion()
SCHED.add_job(func=vs.run, id='version', name='Check Version', trigger=IntervalTrigger(hours=0, minutes=CONFIG.CHECK_GITHUB_INTERVAL, timezone='UTC'))
##run checkFolder every X minutes (basically Manual Run Post-Processing)
if CONFIG.ENABLE_CHECK_FOLDER:
if CONFIG.DOWNLOAD_SCAN_INTERVAL >0:
logger.info('[FOLDER MONITOR] Enabling folder monitor for : ' + str(CONFIG.CHECK_FOLDER) + ' every ' + str(CONFIG.DOWNLOAD_SCAN_INTERVAL) + ' minutes.')
fm = PostProcessor.FolderCheck()
SCHED.add_job(func=fm.run, id='monitor', name='Folder Monitor', trigger=IntervalTrigger(hours=0, minutes=int(CONFIG.DOWNLOAD_SCAN_INTERVAL), timezone='UTC'))
else:
logger.error('[FOLDER MONITOR] You need to specify a monitoring time for the check folder option to work')
logger.info('Firing up the Background Schedulers now....')
try:
SCHED.start()
#update the job db here
logger.info('Background Schedulers successfully started...')
helpers.job_management(write=True)
except Exception as e:
logger.info(e)
SCHED.print_jobs()
started = True
def start():
global _INITIALIZED, started
with INIT_LOCK:
if _INITIALIZED:
#load up the previous runs from the job sql table so we know stuff...
monitors = helpers.job_management()
SCHED_WEEKLY_LAST = monitors['weekly']
SCHED_SEARCH_LAST = monitors['search']
SCHED_UPDATER_LAST = monitors['dbupdater']
SCHED_MONITOR_LAST = monitors['monitor']
SCHED_VERSION_LAST = monitors['version']
SCHED_RSS_LAST = monitors['rss']
# Start our scheduled background tasks
SCHED.add_job(func=updater.dbUpdate, id='dbupdater', name='DB Updater', args=[None,None,True], trigger=IntervalTrigger(hours=5, minutes=5, timezone='UTC'))
#let's do a run at the Wanted issues here (on startup) if enabled.
ss = searchit.CurrentSearcher()
if CONFIG.NZB_STARTUP_SEARCH:
SCHED.add_job(func=ss.run, id='search', next_run_time=datetime.datetime.utcnow(), name='Auto-Search', trigger=IntervalTrigger(hours=0, minutes=CONFIG.SEARCH_INTERVAL, timezone='UTC'))
else:
if SCHED_SEARCH_LAST is not None:
search_timestamp = float(SCHED_SEARCH_LAST)
logger.fdebug('[AUTO-SEARCH] Search last run @ %s' % datetime.datetime.utcfromtimestamp(search_timestamp))
else:
search_timestamp = helpers.utctimestamp() + (int(CONFIG.SEARCH_INTERVAL) *60)
duration_diff = (helpers.utctimestamp() - search_timestamp)/60
if duration_diff >= int(CONFIG.SEARCH_INTERVAL):
logger.fdebug('[AUTO-SEARCH]Auto-Search set to a delay of one minute before initialization as it has been %s minutes since the last run' % duration_diff)
SCHED.add_job(func=ss.run, id='search', name='Auto-Search', trigger=IntervalTrigger(hours=0, minutes=CONFIG.SEARCH_INTERVAL, timezone='UTC'))
else:
search_diff = datetime.datetime.utcfromtimestamp(helpers.utctimestamp() + ((int(CONFIG.SEARCH_INTERVAL) * 60) - (duration_diff*60)))
logger.fdebug('[AUTO-SEARCH] Scheduling next run @ %s every %s minutes' % (search_diff, CONFIG.SEARCH_INTERVAL))
SCHED.add_job(func=ss.run, id='search', name='Auto-Search', next_run_time=search_diff, trigger=IntervalTrigger(hours=0, minutes=CONFIG.SEARCH_INTERVAL, timezone='UTC'))
if all([CONFIG.ENABLE_TORRENTS, CONFIG.AUTO_SNATCH, OS_DETECT != 'Windows']) and any([CONFIG.TORRENT_DOWNLOADER == 2, CONFIG.TORRENT_DOWNLOADER == 4]):
logger.info('[AUTO-SNATCHER] Auto-Snatch of completed torrents enabled & attempting to background load....')
SNPOOL = threading.Thread(target=helpers.worker_main, args=(SNATCHED_QUEUE,), name="AUTO-SNATCHER")
SNPOOL.start()
logger.info('[AUTO-SNATCHER] Succesfully started Auto-Snatch add-on - will now monitor for completed torrents on client....')
if CONFIG.POST_PROCESSING is True and ( all([CONFIG.NZB_DOWNLOADER == 0, CONFIG.SAB_CLIENT_POST_PROCESSING is True]) or all([CONFIG.NZB_DOWNLOADER == 1, CONFIG.NZBGET_CLIENT_POST_PROCESSING is True]) ):
if CONFIG.NZB_DOWNLOADER == 0:
logger.info('[SAB-MONITOR] Completed post-processing handling enabled for SABnzbd. Attempting to background load....')
elif CONFIG.NZB_DOWNLOADER == 1:
logger.info('[NZBGET-MONITOR] Completed post-processing handling enabled for NZBGet. Attempting to background load....')
NZBPOOL = threading.Thread(target=helpers.nzb_monitor, args=(NZB_QUEUE,), name="AUTO-COMPLETE-NZB")
NZBPOOL.start()
if CONFIG.NZB_DOWNLOADER == 0:
logger.info('[AUTO-COMPLETE-NZB] Succesfully started Completed post-processing handling for SABnzbd - will now monitor for completed nzbs within sabnzbd and post-process automatically....')
elif CONFIG.NZB_DOWNLOADER == 1:
logger.info('[AUTO-COMPLETE-NZB] Succesfully started Completed post-processing handling for NZBGet - will now monitor for completed nzbs within nzbget and post-process automatically....')
logger.info('[SEARCH-QUEUE] Attempting to background load the search queue....')
SEARCHPOOL = threading.Thread(target=helpers.search_queue, args=(SEARCH_QUEUE,), name="SEARCH-QUEUE")
SEARCHPOOL.start()
if all([CONFIG.POST_PROCESSING is True, CONFIG.API_ENABLED is True]):
logger.info('[POST-PROCESS-QUEUE] Post Process queue enabled & monitoring for api requests....')
PPPOOL = threading.Thread(target=helpers.postprocess_main, args=(PP_QUEUE,), name="POST-PROCESS-QUEUE")
PPPOOL.start()
logger.info('[POST-PROCESS-QUEUE] Succesfully started Post-Processing Queuer....')
helpers.latestdate_fix()
if CONFIG.ALT_PULL == 2:
weektimer = 4
else:
weektimer = 24
#weekly pull list gets messed up if it's not populated first, so let's populate it then set the scheduler.
logger.info('[WEEKLY] Checking for existance of Weekly Comic listing...')
#now the scheduler (check every 24 hours)
weekly_interval = weektimer * 60 * 60
try:
if SCHED_WEEKLY_LAST:
pass
except:
SCHED_WEEKLY_LAST = None
weektimestamp = helpers.utctimestamp()
if SCHED_WEEKLY_LAST is not None:
weekly_timestamp = float(SCHED_WEEKLY_LAST)
else:
weekly_timestamp = weektimestamp + weekly_interval
ws = weeklypullit.Weekly()
duration_diff = (weektimestamp - weekly_timestamp)/60
if abs(duration_diff) >= weekly_interval/60:
logger.info('[WEEKLY] Weekly Pull-Update initializing immediately as it has been %s hours since the last run' % abs(duration_diff/60))
SCHED.add_job(func=ws.run, id='weekly', name='Weekly Pullist', next_run_time=datetime.datetime.utcnow(), trigger=IntervalTrigger(hours=weektimer, minutes=0, timezone='UTC'))
else:
weekly_diff = datetime.datetime.utcfromtimestamp(weektimestamp + (weekly_interval - (duration_diff * 60)))
logger.fdebug('[WEEKLY] Scheduling next run for @ %s every %s hours' % (weekly_diff, weektimer))
SCHED.add_job(func=ws.run, id='weekly', name='Weekly Pullist', next_run_time=weekly_diff, trigger=IntervalTrigger(hours=weektimer, minutes=0, timezone='UTC'))
#initiate startup rss feeds for torrents/nzbs here...
rs = rsscheckit.tehMain()
if CONFIG.ENABLE_RSS:
logger.info('[RSS-FEEDS] Initiating startup-RSS feed checks.')
if SCHED_RSS_LAST is not None:
rss_timestamp = float(SCHED_RSS_LAST)
logger.info('[RSS-FEEDS] RSS last run @ %s' % datetime.datetime.utcfromtimestamp(rss_timestamp))
else:
rss_timestamp = helpers.utctimestamp() + (int(CONFIG.RSS_CHECKINTERVAL) *60)
duration_diff = (helpers.utctimestamp() - rss_timestamp)/60
if duration_diff >= int(CONFIG.RSS_CHECKINTERVAL):
SCHED.add_job(func=rs.run, id='rss', name='RSS Feeds', args=[True], next_run_time=datetime.datetime.utcnow(), trigger=IntervalTrigger(hours=0, minutes=int(CONFIG.RSS_CHECKINTERVAL), timezone='UTC'))
else:
rss_diff = datetime.datetime.utcfromtimestamp(helpers.utctimestamp() + (int(CONFIG.RSS_CHECKINTERVAL) * 60) - (duration_diff * 60))
logger.fdebug('[RSS-FEEDS] Scheduling next run for @ %s every %s minutes' % (rss_diff, CONFIG.RSS_CHECKINTERVAL))
SCHED.add_job(func=rs.run, id='rss', name='RSS Feeds', args=[True], next_run_time=rss_diff, trigger=IntervalTrigger(hours=0, minutes=int(CONFIG.RSS_CHECKINTERVAL), timezone='UTC'))
#else:
# SCHED.add_job(func=rs.run, id='rss', name='RSS Feeds', args=[True], trigger=IntervalTrigger(hours=0, minutes=int(CONFIG.RSS_CHECKINTERVAL), timezone='UTC'))
# SCHED.pause_job('rss')
if CONFIG.CHECK_GITHUB:
vs = versioncheckit.CheckVersion()
SCHED.add_job(func=vs.run, id='version', name='Check Version', trigger=IntervalTrigger(hours=0, minutes=CONFIG.CHECK_GITHUB_INTERVAL, timezone='UTC'))
##run checkFolder every X minutes (basically Manual Run Post-Processing)
if CONFIG.ENABLE_CHECK_FOLDER:
if CONFIG.DOWNLOAD_SCAN_INTERVAL >0:
logger.info('[FOLDER MONITOR] Enabling folder monitor for : ' + str(CONFIG.CHECK_FOLDER) + ' every ' + str(CONFIG.DOWNLOAD_SCAN_INTERVAL) + ' minutes.')
fm = PostProcessor.FolderCheck()
SCHED.add_job(func=fm.run, id='monitor', name='Folder Monitor', trigger=IntervalTrigger(hours=0, minutes=int(CONFIG.DOWNLOAD_SCAN_INTERVAL), timezone='UTC'))
else:
logger.error('[FOLDER MONITOR] You need to specify a monitoring time for the check folder option to work')
logger.info('Firing up the Background Schedulers now....')
try:
SCHED.start()
#update the job db here
logger.info('Background Schedulers successfully started...')
helpers.job_management(write=True)
except Exception as e:
logger.info(e)
SCHED.print_jobs()
started = True