def _remove_old_zoneinfo():
"""
Removes zoneinfo tar.gz file from repository, as we do not need it
"""
if zoneinfo.ZONEINFOFILE is not None:
cur_zoneinfo = ek(basename, zoneinfo.ZONEINFOFILE)
else:
return
cur_file = helpers.real_path(
ek(join, ek(os.path.dirname, zoneinfo.__file__), cur_zoneinfo))
for (path, dirs, files) in ek(
os.walk, helpers.real_path(ek(os.path.dirname,
zoneinfo.__file__))):
for filename in files:
if filename.endswith('.tar.gz'):
file_w_path = ek(join, path, filename)
if file_w_path != cur_file and ek(isfile, file_w_path):
try:
ek(os.remove, file_w_path)
logger.log(u'Delete unneeded old zoneinfo File: %s' %
file_w_path)
except:
logger.log(u'Unable to delete: %s' % file_w_path,
logger.WARNING)
def _remove_old_zoneinfo():
if lib.dateutil.zoneinfo.ZONEINFOFILE is not None:
cur_zoneinfo = ek.ek(basename, lib.dateutil.zoneinfo.ZONEINFOFILE)
else:
return
cur_file = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__),
cur_zoneinfo))
for (path, dirs, files) in ek.ek(
os.walk,
helpers.real_path(
ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__))):
for filename in files:
if filename.endswith('.tar.gz'):
file_w_path = ek.ek(join, path, filename)
if file_w_path != cur_file and ek.ek(isfile, file_w_path):
try:
ek.ek(os.remove, file_w_path)
logger.log(u'Delete unneeded old zoneinfo File: %s' %
file_w_path)
except:
logger.log(u'Unable to delete: %s' % file_w_path,
logger.ERROR)
def test_real_path(self):
"""
Test real_path
"""
self.assertEqual(
helpers.real_path('/usr/SickChill/../root/real/path/'),
helpers.real_path('/usr/root/real/path/'))
def _delete_folder(self, folder, check_empty=True):
# check if it's a folder
if not ek.ek(os.path.isdir, folder):
return False
# make sure it isn't TV_DOWNLOAD_DIR
if sickbeard.TV_DOWNLOAD_DIR and helpers.real_path(
sickbeard.TV_DOWNLOAD_DIR) == helpers.real_path(folder):
return False
# check if it's empty folder when wanted checked
if check_empty and ek.ek(os.listdir, folder):
return False
# try deleting folder
try:
shutil.rmtree(folder)
except (OSError, IOError) as e:
logger.log(
u'Warning: unable to delete folder: %s: %s' % (folder, ex(e)),
logger.WARNING)
return False
if ek.ek(os.path.isdir, folder):
logger.log(u'Warning: unable to delete folder: %s' % folder,
logger.WARNING)
return False
self._log_helper(u'Deleted folder ' + folder, logger.MESSAGE)
return True
def delete_folder(folder, check_empty=True):
"""
Removes a folder from the filesystem
:param folder: Path to folder to remove
:param check_empty: Boolean, check if the folder is empty before removing it, defaults to True
:return: True on success, False on failure
"""
# check if it's a folder
if not ek(os.path.isdir, folder):
return False
# check if it isn't TV_DOWNLOAD_DIR
if sickbeard.TV_DOWNLOAD_DIR:
if helpers.real_path(folder) == helpers.real_path(sickbeard.TV_DOWNLOAD_DIR):
return False
# check if it's empty folder when wanted checked
if check_empty:
check_files = ek(os.listdir, folder)
if check_files:
logger.log(u"Not deleting folder " + folder + " found the following files: " + str(check_files), logger.INFO)
return False
try:
logger.log(u"Deleting folder (if it's empty): " + folder)
os.rmdir(folder)
except (OSError, IOError), e:
logger.log(u"Warning: unable to delete folder: " + folder + ": " + ex(e), logger.WARNING)
return False
def delete_folder(folder, check_empty=True):
"""
Removes a folder from the filesystem
:param folder: Path to folder to remove
:param check_empty: Boolean, check if the folder is empty before removing it, defaults to True
:return: True on success, False on failure
"""
# check if it's a folder
if not ek(os.path.isdir, folder):
return False
# check if it isn't TV_DOWNLOAD_DIR
if sickbeard.TV_DOWNLOAD_DIR:
if helpers.real_path(folder) == helpers.real_path(sickbeard.TV_DOWNLOAD_DIR):
return False
# check if it's empty folder when wanted checked
if check_empty:
check_files = ek(os.listdir, folder)
if check_files:
logger.log(u"Not deleting folder " + folder + " found the following files: " + str(check_files), logger.INFO)
return False
try:
logger.log(u"Deleting folder (if it's empty): " + folder)
os.rmdir(folder)
except (OSError, IOError), e:
logger.log(u"Warning: unable to delete folder: " + folder + ": " + ex(e), logger.WARNING)
return False
def delete_folder(folder, check_empty=True):
# check if it's a folder
if not ek.ek(os.path.isdir, folder):
return False
# check if it isn't TV_DOWNLOAD_DIR
if sickbeard.TV_DOWNLOAD_DIR:
if helpers.real_path(folder) == helpers.real_path(
sickbeard.TV_DOWNLOAD_DIR):
return False
# check if it's empty folder when wanted checked
if check_empty:
check_files = ek.ek(os.listdir, folder)
if check_files:
return False
# try deleting folder
try:
logger.log(u"Deleting folder: " + folder)
shutil.rmtree(folder)
except (OSError, IOError), e:
logger.log(
u"Warning: unable to delete folder: " + folder + ": " + ex(e),
logger.WARNING)
return False
def _delete_folder(self, folder, check_empty=True):
# check if it's a folder
if not ek.ek(os.path.isdir, folder):
return False
# make sure it isn't TV_DOWNLOAD_DIR
if sickbeard.TV_DOWNLOAD_DIR and helpers.real_path(sickbeard.TV_DOWNLOAD_DIR) == helpers.real_path(folder):
return False
# check if it's empty folder when wanted checked
if check_empty and ek.ek(os.listdir, folder):
return False
# try deleting folder
try:
shutil.rmtree(folder)
except (OSError, IOError) as e:
logger.log(u'Warning: unable to delete folder: %s: %s' % (folder, ex(e)), logger.WARNING)
return False
if ek.ek(os.path.isdir, folder):
logger.log(u'Warning: unable to delete folder: %s' % folder, logger.WARNING)
return False
self._log_helper(u'Deleted folder ' + folder, logger.MESSAGE)
return True
def _remove_old_zoneinfo():
if zoneinfo.ZONEINFOFILE is not None:
cur_zoneinfo = ek.ek(basename, zoneinfo.ZONEINFOFILE)
else:
return
cur_file = helpers.real_path(ek.ek(join, ek.ek(os.path.dirname, zoneinfo.__file__), cur_zoneinfo))
for (path, dirs, files) in ek.ek(os.walk, helpers.real_path(ek.ek(os.path.dirname, zoneinfo.__file__))):
for filename in files:
if filename.endswith(".tar.gz"):
file_w_path = ek.ek(join, path, filename)
if file_w_path != cur_file and ek.ek(isfile, file_w_path):
try:
ek.ek(os.remove, file_w_path)
logger.log(u"Delete unneeded old zoneinfo File: %s" % file_w_path)
except:
logger.log(u"Unable to delete: %s" % file_w_path, logger.WARNING)
def _remove_old_zoneinfo():
zonefilename = zoneinfo._ZONEFILENAME
if None is zonefilename:
return
cur_zoneinfo = ek.ek(basename, zonefilename)
cur_file = helpers.real_path(ek.ek(join, ek.ek(os.path.dirname, zoneinfo.__file__), cur_zoneinfo))
for (path, dirs, files) in ek.ek(os.walk,
helpers.real_path(ek.ek(os.path.dirname, zoneinfo.__file__))):
for filename in files:
if filename.endswith('.tar.gz'):
file_w_path = ek.ek(join, path, filename)
if file_w_path != cur_file and ek.ek(isfile, file_w_path):
try:
ek.ek(os.remove, file_w_path)
logger.log(u'Delete unneeded old zoneinfo File: %s' % file_w_path)
except:
logger.log(u'Unable to delete: %s' % file_w_path, logger.ERROR)
def _remove_old_zoneinfo():
zonefilename = zoneinfo.ZONEFILENAME
if None is zonefilename:
return
cur_zoneinfo = ek.ek(basename, zonefilename)
cur_file = helpers.real_path(ek.ek(join, sickbeard.ZONEINFO_DIR, cur_zoneinfo))
for (path, dirs, files) in chain.from_iterable(ek.ek(os.walk,
helpers.real_path(di)) for di in (sickbeard.ZONEINFO_DIR, ek.ek(os.path.dirname, zoneinfo.__file__))):
for filename in files:
if filename.endswith('.tar.gz'):
file_w_path = ek.ek(join, path, filename)
if file_w_path != cur_file and ek.ek(isfile, file_w_path):
try:
ek.ek(os.remove, file_w_path)
logger.log(u'Delete unneeded old zoneinfo File: %s' % file_w_path)
except:
logger.log(u'Unable to delete: %s' % file_w_path, logger.ERROR)
def remove_zoneinfo(cls):
# delete all existing zoneinfo files
for (path, dirs, files) in ek.ek(os.walk, helpers.real_path(sickbeard.ZONEINFO_DIR)):
for filename in files:
if filename.endswith('.tar.gz'):
file_w_path = ek.ek(os.path.join, path, filename)
try:
ek.ek(os.remove, file_w_path)
except (StandardError, Exception):
pass
def _remove_old_zoneinfo():
"""
Removes zoneinfo tar.gz file from repository, as we do not need it
"""
if zoneinfo.ZONEINFOFILE is not None:
cur_zoneinfo = ek(basename, zoneinfo.ZONEINFOFILE)
else:
return
cur_file = helpers.real_path(ek(join, ek(os.path.dirname, zoneinfo.__file__), cur_zoneinfo))
for (path, dirs, files) in ek(os.walk, helpers.real_path(ek(os.path.dirname, zoneinfo.__file__))):
for filename in files:
if filename.endswith('.tar.gz'):
file_w_path = ek(join, path, filename)
if file_w_path != cur_file and ek(isfile, file_w_path):
try:
ek(os.remove, file_w_path)
logger.log(u'Delete unneeded old zoneinfo File: %s' % file_w_path)
except:
logger.log(u'Unable to delete: %s' % file_w_path, logger.WARNING)
def update_network_dict():
_remove_old_zoneinfo()
_update_zoneinfo()
d = {}
# network timezones are stored in a git submodule
loc = helpers.real_path(ek.ek(join, ek.ek(os.path.dirname, __file__), u'../lib/network_timezones/network_timezones.txt'))
with open(loc, 'r') as file:
data = file.read()
if data is None:
logger.log(u'Updating network timezones failed', logger.ERROR)
load_network_dict()
return
try:
for line in data.splitlines():
(key, val) = line.decode('utf-8').strip().rsplit(u':', 1)
if key is None or val is None:
continue
d[key] = val
except (IOError, OSError):
pass
my_db = db.DBConnection('cache.db')
# load current network timezones
old_d = dict(my_db.select('SELECT * FROM network_timezones'))
# list of sql commands to update the network_timezones table
cl = []
for cur_d, cur_t in d.iteritems():
h_k = old_d.has_key(cur_d)
if h_k and cur_t != old_d[cur_d]:
# update old record
cl.append(
['UPDATE network_timezones SET network_name=?, timezone=? WHERE network_name=?', [cur_d, cur_t, cur_d]])
elif not h_k:
# add new record
cl.append(['INSERT INTO network_timezones (network_name, timezone) VALUES (?,?)', [cur_d, cur_t]])
if h_k:
del old_d[cur_d]
# remove deleted records
if len(old_d) > 0:
old_items = list(va for va in old_d)
cl.append(['DELETE FROM network_timezones WHERE network_name IN (%s)' % ','.join(['?'] * len(old_items)), old_items])
# change all network timezone infos at once (much faster)
if len(cl) > 0:
my_db.mass_action(cl)
load_network_dict()
def remove_zoneinfo(cls):
# delete all existing zoneinfo files
for (path,
dirs, files) in ek.ek(os.walk,
helpers.real_path(sickbeard.ZONEINFO_DIR)):
for filename in files:
if filename.endswith('.tar.gz'):
file_w_path = ek.ek(os.path.join, path, filename)
try:
ek.ek(os.remove, file_w_path)
except (StandardError, Exception):
pass
def delete_folder(folder, check_empty=True):
"""
Removes a folder from the filesystem
:param folder: Path to folder to remove
:param check_empty: Boolean, check if the folder is empty before removing it, defaults to True
:return: True on success, False on failure
"""
# check if it's a folder
if not ek(os.path.isdir, folder):
return False
# check if it isn't TV_DOWNLOAD_DIR
if sickbeard.TV_DOWNLOAD_DIR:
if helpers.real_path(folder) == helpers.real_path(
sickbeard.TV_DOWNLOAD_DIR):
return False
# check if it's empty folder when wanted checked
try:
if check_empty:
check_files = ek(os.listdir, folder)
if check_files:
logging.info(
"Not deleting folder {} found the following files: {}".
format(folder, check_files))
return False
logging.info("Deleting folder (if it's empty): " + folder)
ek(os.rmdir, folder)
else:
logging.info("Deleting folder: " + folder)
ek(removetree, folder)
except (OSError, IOError) as e:
logging.warning("Warning: unable to delete folder: {}: {}".format(
folder, ex(e)))
return False
return True
def delete_folder(folder, check_empty=True):
"""
Removes a folder from the filesystem
:param folder: Path to folder to remove
:param check_empty: Boolean, check if the folder is empty before removing it, defaults to True
:return: True on success, False on failure
"""
# check if it's a folder
if not ek(os.path.isdir, folder):
return False
# check if it isn't TV_DOWNLOAD_DIR
if sickbeard.TV_DOWNLOAD_DIR:
if helpers.real_path(folder) == helpers.real_path(sickbeard.TV_DOWNLOAD_DIR):
return False
# check if it's empty folder when wanted checked
try:
if check_empty:
check_files = ek(os.listdir, folder)
if check_files:
logging.info("Not deleting folder {} found the following files: {}".format(folder, check_files))
return False
logging.info("Deleting folder (if it's empty): " + folder)
ek(os.rmdir, folder)
else:
logging.info("Deleting folder: " + folder)
ek(removetree, folder)
except (OSError, IOError) as e:
logging.warning("Warning: unable to delete folder: {}: {}".format(folder, ex(e)))
return False
return True
def delete_folder(folder, check_empty=True):
# check if it's a folder
if not ek.ek(os.path.isdir, folder):
return False
# check if it isn't TV_DOWNLOAD_DIR
if sickbeard.TV_DOWNLOAD_DIR:
if helpers.real_path(folder) == helpers.real_path(sickbeard.TV_DOWNLOAD_DIR):
return False
# check if it's empty folder when wanted checked
if check_empty:
check_files = ek.ek(os.listdir, folder)
if check_files:
logger.log(u"Not deleting folder " + folder + " found the following files: " + str(check_files), logger.INFO)
return False
try:
logger.log(u"Deleting folder (if it's empty): " + folder)
os.rmdir(folder)
except (OSError, IOError), e:
logger.log(u"Warning: unable to delete folder: " + folder + ": " + ex(e), logger.WARNING)
return False
def find_parent(path):
"""
Test path is inside a parent folder
:param path: Path to check
:type path: String
:return: Parent root dir that matches path, or None
:rtype: String, None
"""
build_path = (lambda old_path: '%s%s' % (helpers.real_path(old_path).rstrip(os.path.sep), os.path.sep))
process_path = build_path(path)
for parent in map(lambda p: build_path(p), sickbeard.ROOT_DIRS.split('|')[1:]):
if process_path.startswith(parent):
return parent.rstrip(os.path.sep)
def update_network_dict():
_remove_old_zoneinfo()
_update_zoneinfo()
loc = helpers.real_path(ek.ek(join, ek.ek(os.path.dirname, __file__), u'../lib/network_timezones/network_timezones.txt'))
with open(loc, 'r') as file:
data = file.read()
if data is None:
logger.log(u'Updating network timezones failed', logger.ERROR)
load_network_dict()
return
d = {}
try:
for line in data.splitlines():
(key, val) = line.decode('utf-8').strip().rsplit(u':', 1)
if key is None or val is None:
continue
d[key] = val
except (IOError, OSError):
pass
my_db = db.DBConnection('cache.db')
network_list = dict(my_db.select('SELECT * FROM network_timezones;'))
queries = []
for network, timezone in d.iteritems():
existing = network_list.has_key(network)
if not existing:
queries.append(['INSERT OR IGNORE INTO network_timezones VALUES (?,?);', [network, timezone]])
elif network_list[network] is not timezone:
queries.append(['UPDATE OR IGNORE network_timezones SET timezone = ? WHERE network_name = ?;', [timezone, network]])
if existing:
del network_list[network]
if network_list:
purged = list(x for x in network_list)
queries.append(['DELETE FROM network_timezones WHERE network_name IN (%s);' % ','.join(['?'] * len(purged)), purged])
if queries:
my_db.mass_action(queries)
load_network_dict()
def retrieve_exceptions():
"""
Looks up the exceptions on github, parses them into a dict, and inserts them into the
scene_exceptions table in cache.db. Also clears the scene name cache.
"""
global exception_dict, anidb_exception_dict, xem_exception_dict
# exceptions are stored in submodules in this repo, sourced from the github repos
# TODO: `git submodule update`
for indexer in sickbeard.indexerApi().indexers:
if shouldRefresh(sickbeard.indexerApi(indexer).name):
logger.log(u"Checking for scene exception updates for " +
sickbeard.indexerApi(indexer).name + "")
loc = sickbeard.indexerApi(indexer).config['scene_loc']
if loc.startswith("http"):
data = helpers.getURL(loc)
else:
loc = helpers.real_path(
ek.ek(os.path.join, ek.ek(os.path.dirname, __file__), loc))
with open(loc, 'r') as file:
data = file.read()
if data is None:
# When data is None, trouble connecting to github, or reading file failed
logger.log(
u"Check scene exceptions update failed. Unable to update from: "
+ loc, logger.WARNING)
continue
setLastRefresh(sickbeard.indexerApi(indexer).name)
# each exception is on one line with the format indexer_id: 'show name 1', 'show name 2', etc
for cur_line in data.splitlines():
indexer_id, sep, aliases = cur_line.partition(
':') # @UnusedVariable
if not aliases:
continue
indexer_id = int(indexer_id)
# regex out the list of shows, taking \' into account
# alias_list = [re.sub(r'\\(.)', r'\1', x) for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
alias_list = [{
re.sub(r'\\(.)', r'\1', x): -1
} for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
exception_dict[indexer_id] = alias_list
del alias_list
# cleanup
del data
# XEM scene exceptions
_xem_exceptions_fetcher()
for xem_ex in xem_exception_dict:
if xem_ex in exception_dict:
exception_dict[
xem_ex] = exception_dict[xem_ex] + xem_exception_dict[xem_ex]
else:
exception_dict[xem_ex] = xem_exception_dict[xem_ex]
# AniDB scene exceptions
_anidb_exceptions_fetcher()
for anidb_ex in anidb_exception_dict:
if anidb_ex in exception_dict:
exception_dict[anidb_ex] = exception_dict[
anidb_ex] + anidb_exception_dict[anidb_ex]
else:
exception_dict[anidb_ex] = anidb_exception_dict[anidb_ex]
changed_exceptions = False
# write all the exceptions we got off the net into the database
myDB = db.DBConnection('cache.db')
for cur_indexer_id in exception_dict:
# get a list of the existing exceptions for this ID
existing_exceptions = [
x["show_name"] for x in myDB.select(
"SELECT * FROM scene_exceptions WHERE indexer_id = ?",
[cur_indexer_id])
]
if not cur_indexer_id in exception_dict:
continue
for cur_exception_dict in exception_dict[cur_indexer_id]:
cur_exception, curSeason = cur_exception_dict.items()[0]
# if this exception isn't already in the DB then add it
if cur_exception not in existing_exceptions:
myDB.action(
"INSERT INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?)",
[cur_indexer_id, cur_exception, curSeason])
changed_exceptions = True
# since this could invalidate the results of the cache we clear it out after updating
if changed_exceptions:
logger.log(u"Updated scene exceptions", logger.DEBUG)
else:
logger.log(u"No scene exceptions update needed", logger.DEBUG)
# cleanup
exception_dict.clear()
anidb_exception_dict.clear()
xem_exception_dict.clear()
def test_root_folder(self):
for root_dirs, path, expected in root_folder_tests:
sickbeard.ROOT_DIRS = root_dirs
self.assertEqual(expected and real_path(expected) or expected, ProcessTVShow.find_parent(path))
def _update_zoneinfo():
global sb_timezone
sb_timezone = tz.tzlocal()
# now check if the zoneinfo needs update
url_zv = 'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/zoneinfo.txt'
url_data = helpers.getURL(url_zv)
if url_data is None:
# When urlData is None, trouble connecting to github
logger.log(u'Loading zoneinfo.txt failed, this can happen from time to time. Unable to get URL: %s' % url_zv,
logger.WARNING)
return
zonefilename = zoneinfo._ZONEFILENAME
cur_zoneinfo = zonefilename
if None is not cur_zoneinfo:
cur_zoneinfo = ek.ek(basename, zonefilename)
zonefile = helpers.real_path(ek.ek(join, ek.ek(os.path.dirname, zoneinfo.__file__), cur_zoneinfo))
zonemetadata = zoneinfo.gettz_db_metadata() if ek.ek(os.path.isfile, zonefile) else None
(new_zoneinfo, zoneinfo_md5) = url_data.decode('utf-8').strip().rsplit(u' ')
newtz_regex = re.search(r'(\d{4}[^.]+)', new_zoneinfo)
if not newtz_regex or len(newtz_regex.groups()) != 1:
return
newtzversion = newtz_regex.group(1)
if cur_zoneinfo is not None and zonemetadata is not None and 'tzversion' in zonemetadata and zonemetadata['tzversion'] == newtzversion:
return
# now load the new zoneinfo
url_tar = u'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/%s' % new_zoneinfo
zonefile_tmp = re.sub(r'\.tar\.gz$', '.tmp', zonefile)
if ek.ek(os.path.exists, zonefile_tmp):
try:
ek.ek(os.remove, zonefile_tmp)
except:
logger.log(u'Unable to delete: %s' % zonefile_tmp, logger.ERROR)
return
if not helpers.download_file(url_tar, zonefile_tmp):
return
if not ek.ek(os.path.exists, zonefile_tmp):
logger.log(u'Download of %s failed.' % zonefile_tmp, logger.ERROR)
return
new_hash = str(helpers.md5_for_file(zonefile_tmp))
if zoneinfo_md5.upper() == new_hash.upper():
logger.log(u'Updating timezone info with new one: %s' % new_zoneinfo, logger.MESSAGE)
try:
# remove the old zoneinfo file
if cur_zoneinfo is not None:
old_file = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, zoneinfo.__file__), cur_zoneinfo))
if ek.ek(os.path.exists, old_file):
ek.ek(os.remove, old_file)
# rename downloaded file
ek.ek(os.rename, zonefile_tmp, zonefile)
from dateutil.zoneinfo import gettz
if '_CLASS_ZONE_INSTANCE' in gettz.func_globals:
gettz.func_globals.__setitem__('_CLASS_ZONE_INSTANCE', list())
sb_timezone = tz.tzlocal()
except:
_remove_zoneinfo_failed(zonefile_tmp)
return
else:
_remove_zoneinfo_failed(zonefile_tmp)
logger.log(u'MD5 hash does not match: %s File: %s' % (zoneinfo_md5.upper(), new_hash.upper()), logger.ERROR)
return
def retrieve_exceptions():
"""
Looks up the exceptions on github, parses them into a dict, and inserts them into the
scene_exceptions table in cache.db. Also clears the scene name cache.
"""
global exception_dict, anidb_exception_dict, xem_exception_dict
# exceptions are stored in submodules in this repo, sourced from the github repos
# TODO: `git submodule update`
for indexer in sickbeard.indexerApi().indexers:
if shouldRefresh(sickbeard.indexerApi(indexer).name):
logger.log(u"Checking for scene exception updates for " + sickbeard.indexerApi(indexer).name + "")
loc = sickbeard.indexerApi(indexer).config['scene_loc']
if loc.startswith("http"):
data = helpers.getURL(loc, session=sickbeard.indexerApi(indexer).session)
else:
loc = helpers.real_path(ek.ek(os.path.join, ek.ek(os.path.dirname, __file__), loc))
with open(loc, 'r') as file:
data = file.read()
if data is None:
# When data is None, trouble connecting to github, or reading file failed
logger.log(u"Check scene exceptions update failed. Unable to update from: " + loc, logger.WARNING)
continue
setLastRefresh(sickbeard.indexerApi(indexer).name)
# each exception is on one line with the format indexer_id: 'show name 1', 'show name 2', etc
for cur_line in data.splitlines():
indexer_id, sep, aliases = cur_line.partition(':') # @UnusedVariable
if not aliases:
continue
indexer_id = int(indexer_id)
# regex out the list of shows, taking \' into account
# alias_list = [re.sub(r'\\(.)', r'\1', x) for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
alias_list = [{re.sub(r'\\(.)', r'\1', x): -1} for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
exception_dict[indexer_id] = alias_list
del alias_list
# cleanup
del data
# XEM scene exceptions
_xem_exceptions_fetcher()
for xem_ex in xem_exception_dict:
if xem_ex in exception_dict:
exception_dict[xem_ex] = exception_dict[xem_ex] + xem_exception_dict[xem_ex]
else:
exception_dict[xem_ex] = xem_exception_dict[xem_ex]
# AniDB scene exceptions
_anidb_exceptions_fetcher()
for anidb_ex in anidb_exception_dict:
if anidb_ex in exception_dict:
exception_dict[anidb_ex] = exception_dict[anidb_ex] + anidb_exception_dict[anidb_ex]
else:
exception_dict[anidb_ex] = anidb_exception_dict[anidb_ex]
changed_exceptions = False
# write all the exceptions we got off the net into the database
myDB = db.DBConnection('cache.db')
for cur_indexer_id in exception_dict:
# get a list of the existing exceptions for this ID
existing_exceptions = [x["show_name"] for x in
myDB.select("SELECT * FROM scene_exceptions WHERE indexer_id = ?", [cur_indexer_id])]
if not cur_indexer_id in exception_dict:
continue
for cur_exception_dict in exception_dict[cur_indexer_id]:
cur_exception, curSeason = cur_exception_dict.items()[0]
# if this exception isn't already in the DB then add it
if cur_exception not in existing_exceptions:
myDB.action("INSERT INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?)",
[cur_indexer_id, cur_exception, curSeason])
changed_exceptions = True
# since this could invalidate the results of the cache we clear it out after updating
if changed_exceptions:
logger.log(u"Updated scene exceptions", logger.DEBUG)
else:
logger.log(u"No scene exceptions update needed", logger.DEBUG)
# cleanup
exception_dict.clear()
anidb_exception_dict.clear()
xem_exception_dict.clear()
def _update_zoneinfo():
global sb_timezone
sb_timezone = tz.tzlocal()
# now check if the zoneinfo needs update
url_zv = 'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/zoneinfo.txt'
url_data = helpers.getURL(url_zv)
if url_data is None:
# When urlData is None, trouble connecting to github
logger.log(u'Loading zoneinfo.txt failed, this can happen from time to time. Unable to get URL: %s' % url_zv,
logger.WARNING)
return
if lib.dateutil.zoneinfo.ZONEINFOFILE is not None:
cur_zoneinfo = ek.ek(basename, lib.dateutil.zoneinfo.ZONEINFOFILE)
else:
cur_zoneinfo = None
(new_zoneinfo, zoneinfo_md5) = url_data.decode('utf-8').strip().rsplit(u' ')
if (cur_zoneinfo is not None) and (new_zoneinfo == cur_zoneinfo):
return
# now load the new zoneinfo
url_tar = u'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/%s' % new_zoneinfo
zonefile = helpers.real_path(ek.ek(join, ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__), new_zoneinfo))
zonefile_tmp = re.sub(r'\.tar\.gz$', '.tmp', zonefile)
if ek.ek(os.path.exists, zonefile_tmp):
try:
ek.ek(os.remove, zonefile_tmp)
except:
logger.log(u'Unable to delete: %s' % zonefile_tmp, logger.ERROR)
return
if not helpers.download_file(url_tar, zonefile_tmp):
return
if not ek.ek(os.path.exists, zonefile_tmp):
logger.log(u'Download of %s failed.' % zonefile_tmp, logger.ERROR)
return
new_hash = str(helpers.md5_for_file(zonefile_tmp))
if zoneinfo_md5.upper() == new_hash.upper():
logger.log(u'Updating timezone info with new one: %s' % new_zoneinfo, logger.MESSAGE)
try:
# remove the old zoneinfo file
if cur_zoneinfo is not None:
old_file = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__), cur_zoneinfo))
if ek.ek(os.path.exists, old_file):
ek.ek(os.remove, old_file)
# rename downloaded file
ek.ek(os.rename, zonefile_tmp, zonefile)
# load the new zoneinfo
reload(lib.dateutil.zoneinfo)
sb_timezone = tz.tzlocal()
except:
_remove_zoneinfo_failed(zonefile_tmp)
return
else:
_remove_zoneinfo_failed(zonefile_tmp)
logger.log(u'MD5 hash does not match: %s File: %s' % (zoneinfo_md5.upper(), new_hash.upper()), logger.ERROR)
return
def _update_zoneinfo():
global sb_timezone
sb_timezone = tz.tzlocal()
# now check if the zoneinfo needs update
url_zv = 'https://github.com/Prinz23/sb_network_timezones/raw/master/zoneinfo.txt'
url_data = helpers.getURL(url_zv)
if url_data is None:
# When urlData is None, trouble connecting to github
logger.log(u"Loading zoneinfo.txt failed. Unable to get URL: " + url_zv, logger.ERROR)
return
if (lib.dateutil.zoneinfo.ZONEINFOFILE is not None):
cur_zoneinfo = ek.ek(basename, lib.dateutil.zoneinfo.ZONEINFOFILE)
else:
cur_zoneinfo = None
(new_zoneinfo, zoneinfo_md5) = url_data.decode('utf-8').strip().rsplit(u' ')
if ((cur_zoneinfo is not None) and (new_zoneinfo == cur_zoneinfo)):
return
# now load the new zoneinfo
url_tar = u'https://github.com/Prinz23/sb_network_timezones/raw/master/' + new_zoneinfo
zonefile = helpers.real_path(ek.ek(join,ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__), new_zoneinfo))
zonefile_tmp = re.sub(r"\.tar\.gz$",'.tmp', zonefile)
if (ek.ek(os.path.exists,zonefile_tmp)):
try:
ek.ek(os.remove,zonefile_tmp)
except:
logger.log(u"Unable to delete: " + zonefile_tmp,logger.ERROR)
return
if not helpers.download_file(url_tar, zonefile_tmp):
return
if not ek.ek(os.path.exists,zonefile_tmp):
logger.log(u"Download of " + zonefile_tmp + " failed.",logger.ERROR)
return
new_hash = str(helpers.md5_for_file(zonefile_tmp))
if (zoneinfo_md5.upper() == new_hash.upper()):
logger.log(u"Updating timezone info with new one: " + new_zoneinfo,logger.MESSAGE)
try:
# remove the old zoneinfo file
if (cur_zoneinfo is not None):
old_file = helpers.real_path(ek.ek(join,ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__), cur_zoneinfo))
if (ek.ek(os.path.exists,old_file)):
ek.ek(os.remove,old_file)
# rename downloaded file
ek.ek(os.rename,zonefile_tmp,zonefile)
# load the new zoneinfo
reload(lib.dateutil.zoneinfo)
sb_timezone = tz.tzlocal()
except:
_remove_zoneinfo_failed(zonefile_tmp)
return
else:
_remove_zoneinfo_failed(zonefile_tmp)
logger.log(u"MD5 HASH doesn't match: " + zoneinfo_md5.upper() + ' File: ' + new_hash.upper(),logger.ERROR)
return
def _update_zoneinfo():
global sb_timezone
sb_timezone = tz.tzlocal()
# now check if the zoneinfo needs update
url_zv = 'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/zoneinfo.txt'
try:
url_data = helpers.getURL(url_zv)
if not url_data:
raise
if lib.dateutil.zoneinfo.ZONEINFOFILE is not None:
cur_zoneinfo = ek.ek(basename, lib.dateutil.zoneinfo.ZONEINFOFILE)
else:
cur_zoneinfo = None
(new_zoneinfo,
zoneinfo_md5) = url_data.decode('utf-8').strip().rsplit(u' ')
except:
# When urlData is None, trouble connecting to github
logger.log(
u'Loading zoneinfo.txt failed, this can happen from time to time. Unable to get URL: %s'
% url_zv, logger.WARNING)
return
if (cur_zoneinfo is not None) and (new_zoneinfo == cur_zoneinfo):
return
# now load the new zoneinfo
url_tar = u'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/%s' % new_zoneinfo
zonefile = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__),
new_zoneinfo))
zonefile_tmp = re.sub(r'\.tar\.gz$', '.tmp', zonefile)
if ek.ek(os.path.exists, zonefile_tmp):
try:
ek.ek(os.remove, zonefile_tmp)
except:
logger.log(u'Unable to delete: %s' % zonefile_tmp, logger.ERROR)
return
if not helpers.download_file(url_tar, zonefile_tmp):
return
if not ek.ek(os.path.exists, zonefile_tmp):
logger.log(u'Download of %s failed.' % zonefile_tmp, logger.ERROR)
return
new_hash = str(helpers.md5_for_file(zonefile_tmp))
if zoneinfo_md5.upper() == new_hash.upper():
logger.log(u'Updating timezone info with new one: %s' % new_zoneinfo,
logger.INFO)
try:
# remove the old zoneinfo file
if cur_zoneinfo is not None:
old_file = helpers.real_path(
ek.ek(
join,
ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__),
cur_zoneinfo))
if ek.ek(os.path.exists, old_file):
ek.ek(os.remove, old_file)
# rename downloaded file
ek.ek(os.rename, zonefile_tmp, zonefile)
# load the new zoneinfo
reload(lib.dateutil.zoneinfo)
sb_timezone = tz.tzlocal()
except:
_remove_zoneinfo_failed(zonefile_tmp)
return
else:
_remove_zoneinfo_failed(zonefile_tmp)
logger.log(
u'MD5 hash does not match: %s File: %s' %
(zoneinfo_md5.upper(), new_hash.upper()), logger.ERROR)
return
def test_real_path(self):
"""
Test real_path
"""
self.assertEqual(helpers.real_path('/usr/SickRage/../root/real/path/'), helpers.real_path('/usr/root/real/path/'))
def _update_zoneinfo():
global sb_timezone
sb_timezone = tz.tzlocal()
# TODO `git subtree pull` commands on updates
loc_zv = helpers.real_path(ek.ek(join, ek.ek(os.path.dirname, __file__), u'../lib/network_timezones/zoneinfo.txt'))
# Read version file
try:
with open(loc_zv, 'r') as file:
data = file.read()
if not data:
raise
# Filename of existing zoneinfo
if lib.dateutil.zoneinfo.ZONEINFOFILE is not None:
cur_zoneinfo = ek.ek(basename, lib.dateutil.zoneinfo.ZONEINFOFILE)
else:
cur_zoneinfo = None
# Filename and hash of new zoneinfo
(new_zoneinfo, zoneinfo_md5) = data.decode('utf-8').strip().rsplit(u' ')
except Exception as e:
logger.log(u'Crazy problem with zoneinfo: %s' % ex(e), logger.ERROR)
return
if (cur_zoneinfo is not None) and (new_zoneinfo == cur_zoneinfo):
return
# now load the new zoneinfo
loc_tar = helpers.real_path(ek.ek(join, ek.ek(os.path.dirname, __file__), u'../lib/network_timezones/%s' % new_zoneinfo))
zonefile = helpers.real_path(ek.ek(join, ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__), new_zoneinfo))
zonefile_tmp = re.sub(r'\.tar\.gz$', '.tmp', zonefile)
if ek.ek(os.path.exists, zonefile_tmp):
try:
ek.ek(os.remove, zonefile_tmp)
except:
logger.log(u'Unable to delete: %s' % zonefile_tmp, logger.ERROR)
return
if not helpers.copyFile(loc_tar, zonefile_tmp):
return
if not ek.ek(os.path.exists, zonefile_tmp):
logger.log(u'Download of %s failed.' % zonefile_tmp, logger.ERROR)
return
new_hash = str(helpers.md5_for_file(zonefile_tmp))
if zoneinfo_md5.upper() == new_hash.upper():
logger.log(u'Updating timezone info with new one: %s' % new_zoneinfo, logger.INFO)
try:
# remove the old zoneinfo file
if cur_zoneinfo is not None:
old_file = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__), cur_zoneinfo))
if ek.ek(os.path.exists, old_file):
ek.ek(os.remove, old_file)
# rename downloaded file
ek.ek(os.rename, zonefile_tmp, zonefile)
# load the new zoneinfo
reload(lib.dateutil.zoneinfo)
sb_timezone = tz.tzlocal()
except:
_remove_zoneinfo_failed(zonefile_tmp)
return
else:
_remove_zoneinfo_failed(zonefile_tmp)
logger.log(u'MD5 hash does not match: %s File: %s' % (zoneinfo_md5.upper(), new_hash.upper()), logger.ERROR)
return
def _update_zoneinfo():
"""
Request new zoneinfo directly from repository
"""
global sb_timezone
sb_timezone = tz.tzlocal()
url_zv = 'http://sickragetv.github.io/network_timezones/zoneinfo.txt'
try:
url_data = helpers.getURL(url_zv, session=requests.Session())
if not url_data:
raise
# Filename of existing zoneinfo
if zoneinfo.ZONEINFOFILE is not None:
cur_zoneinfo = ek(basename, zoneinfo.ZONEINFOFILE)
else:
cur_zoneinfo = None
# Filename and hash of new zoneinfo
(new_zoneinfo, zoneinfo_md5) = url_data.strip().rsplit(u' ')
except Exception as e:
logger.log(u'Loading zoneinfo.txt failed, this can happen from time to time. Unable to get URL: %s' %
url_zv, logger.WARNING)
return
if (cur_zoneinfo is not None) and (new_zoneinfo == cur_zoneinfo):
return
# now load the new zoneinfo
url_tar = u'http://sickragetv.github.io/network_timezones/%s' % new_zoneinfo
zonefile = helpers.real_path(ek(join, ek(os.path.dirname, zoneinfo.__file__), new_zoneinfo))
zonefile_tmp = re.sub(r'\.tar\.gz$', '.tmp', zonefile)
if ek(os.path.exists, zonefile_tmp):
try:
ekk(os.remove, zonefile_tmp)
except:
logger.log(u'Unable to delete: %s' % zonefile_tmp, logger.WARNING)
return
if not helpers.download_file(url_tar, zonefile_tmp, session=requests.Session()):
return
if not ek(os.path.exists, zonefile_tmp):
logger.log(u'Download of %s failed.' % zonefile_tmp, logger.WARNING)
return
new_hash = str(helpers.md5_for_file(zonefile_tmp))
if zoneinfo_md5.upper() == new_hash.upper():
logger.log(u'Updating timezone info with new one: %s' % new_zoneinfo, logger.INFO)
try:
# remove the old zoneinfo file
if cur_zoneinfo is not None:
old_file = helpers.real_path(
ek(join, ek(os.path.dirname, zoneinfo.__file__), cur_zoneinfo))
if ek(os.path.exists, old_file):
ek(os.remove, old_file)
# rename downloaded file
ek(os.rename, zonefile_tmp, zonefile)
# load the new zoneinfo
reload(zoneinfo)
sb_timezone = tz.tzlocal()
except:
_remove_zoneinfo_failed(zonefile_tmp)
return
else:
_remove_zoneinfo_failed(zonefile_tmp)
logger.log(u'MD5 hash does not match: %s File: %s' % (zoneinfo_md5.upper(), new_hash.upper()), logger.WARNING)
return
def validate_dir(process_path, release_name, failed, result): # pylint: disable=too-many-locals,too-many-branches,too-many-return-statements
"""
Check if directory is valid for processing
:param process_path: Directory to check
:param release_name: Original NZB/Torrent name
:param failed: Previously failed objects
:param result: Previous results
:return: True if dir is valid for processing, False if not
"""
result.output += log_helper("Processing folder " + process_path, logger.DEBUG)
upper_name = ek(os.path.basename, process_path).upper()
if upper_name.startswith('_FAILED_') or upper_name.endswith('_FAILED_'):
result.output += log_helper("The directory name indicates it failed to extract.", logger.DEBUG)
failed = True
elif upper_name.startswith('_UNDERSIZED_') or upper_name.endswith('_UNDERSIZED_'):
result.output += log_helper("The directory name indicates that it was previously rejected for being undersized.", logger.DEBUG)
failed = True
elif upper_name.startswith('_UNPACK') or upper_name.endswith('_UNPACK'):
result.output += log_helper("The directory name indicates that this release is in the process of being unpacked.", logger.DEBUG)
result.missed_files.append("{0} : Being unpacked".format(process_path))
return False
if failed:
process_failed(process_path, release_name, result)
result.missed_files.append("{0} : Failed download".format(process_path))
return False
if sickbeard.TV_DOWNLOAD_DIR and helpers.real_path(process_path) != helpers.real_path(sickbeard.TV_DOWNLOAD_DIR) and helpers.is_hidden_folder(process_path):
result.output += log_helper("Ignoring hidden folder: {0}".format(process_path), logger.DEBUG)
result.missed_files.append("{0} : Hidden folder".format(process_path))
return False
# make sure the dir isn't inside a show dir
main_db_con = db.DBConnection()
sql_results = main_db_con.select("SELECT location FROM tv_shows")
for sqlShow in sql_results:
if process_path.lower().startswith(ek(os.path.realpath, sqlShow[b"location"]).lower() + os.sep) or \
process_path.lower() == ek(os.path.realpath, sqlShow[b"location"]).lower():
result.output += log_helper(
"Cannot process an episode that's already been moved to its show dir, skipping " + process_path,
logger.WARNING)
return False
for current_directory, directory_names, file_names in ek(os.walk, process_path, topdown=False, followlinks=sickbeard.PROCESSOR_FOLLOW_SYMLINKS):
sync_files = filter(is_sync_file, file_names)
if sync_files and sickbeard.POSTPONE_IF_SYNC_FILES:
result.output += log_helper("Found temporary sync files: {0} in path: {1}".format(sync_files, ek(os.path.join, process_path, sync_files[0])))
result.output += log_helper("Skipping post processing for folder: {0}".format(process_path))
result.missed_files.append("{0} : Sync files found".format(ek(os.path.join, process_path, sync_files[0])))
continue
found_files = filter(helpers.is_media_file, file_names)
if sickbeard.UNPACK == 1:
found_files += filter(helpers.is_rar_file, file_names)
if current_directory != sickbeard.TV_DOWNLOAD_DIR and found_files:
found_files.append(ek(os.path.basename, current_directory))
for found_file in found_files:
try:
NameParser().parse(found_file, cache_result=False)
except (InvalidNameException, InvalidShowException) as e:
pass
else:
return True
result.output += log_helper("{0} : No processable items found in folder".format(process_path), logger.DEBUG)
return False
def processDir(dirName, nzbName=None, process_method=None, force=False, is_priority=None, failed=False, type="automatic"):
"""
Scans through the files in dirName and processes whatever media files it finds
dirName: The folder name to look in
nzbName: The NZB name which resulted in this folder being downloaded
force: True to postprocess already postprocessed files
failed: Boolean for whether or not the download failed
type: Type of postprocessing automatic or manual
"""
global process_result, returnStr
returnStr = u""
returnStr += logHelper(u"Processing folder " + dirName, logger.DEBUG)
returnStr += logHelper(u"TV_DOWNLOAD_DIR: " + sickbeard.TV_DOWNLOAD_DIR, logger.DEBUG)
# if they passed us a real dir then assume it's the one we want
if ek.ek(os.path.isdir, dirName):
dirName = ek.ek(os.path.realpath, dirName)
# if the client and Sickbeard are not on the same machine translate the Dir in a network dir
elif sickbeard.TV_DOWNLOAD_DIR and ek.ek(os.path.isdir, sickbeard.TV_DOWNLOAD_DIR) \
and helpers.real_path(dirName) != helpers.real_path(sickbeard.TV_DOWNLOAD_DIR):
dirName = ek.ek(os.path.join, sickbeard.TV_DOWNLOAD_DIR, ek.ek(os.path.abspath, dirName).split(os.path.sep)[-1])
returnStr += logHelper(u"Trying to use folder " + dirName, logger.DEBUG)
# if we didn't find a real dir then quit
if not ek.ek(os.path.isdir, dirName):
returnStr += logHelper(u"Unable to figure out what folder to process. If your downloader and Sick Beard aren't on the same PC make sure you fill out your TV download dir in the config.", logger.DEBUG)
return returnStr
path, dirs, files = get_path_dir_files(dirName, nzbName, type)
returnStr += logHelper(u"PostProcessing Path: " + path, logger.DEBUG)
returnStr += logHelper(u"PostProcessing Dirs: " + str(dirs), logger.DEBUG)
rarFiles = filter(helpers.isRarFile, files)
rarContent = unRAR(path, rarFiles, force)
files += rarContent
videoFiles = filter(helpers.isMediaFile, files)
videoInRar = filter(helpers.isMediaFile, rarContent)
returnStr += logHelper(u"PostProcessing Files: " + str(files), logger.DEBUG)
returnStr += logHelper(u"PostProcessing VideoFiles: " + str(videoFiles), logger.DEBUG)
returnStr += logHelper(u"PostProcessing RarContent: " + str(rarContent), logger.DEBUG)
returnStr += logHelper(u"PostProcessing VideoInRar: " + str(videoInRar), logger.DEBUG)
# If nzbName is set and there's more than one videofile in the folder, files will be lost (overwritten).
nzbNameOriginal = nzbName
if len(videoFiles) >= 2:
nzbName = None
if not process_method:
process_method = sickbeard.PROCESS_METHOD
process_result = True
#Don't Link media when the media is extracted from a rar in the same path
if process_method in ('hardlink', 'symlink') and videoInRar:
process_media(path, videoInRar, nzbName, 'move', force, is_priority)
delete_files(path, rarContent)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force, is_priority)
else:
for video in videoFiles:
process_media(path, [video], nzbName, process_method, force, is_priority)
#Process Video File in all TV Subdir
for dir in [x for x in dirs if validateDir(path, x, nzbNameOriginal, failed)]:
process_result = True
for processPath, processDir, fileList in ek.ek(os.walk, ek.ek(os.path.join, path, dir), topdown=False):
rarFiles = filter(helpers.isRarFile, fileList)
rarContent = unRAR(processPath, rarFiles, force)
fileList = set(fileList + rarContent)
videoFiles = filter(helpers.isMediaFile, fileList)
videoInRar = filter(helpers.isMediaFile, rarContent)
notwantedFiles = [x for x in fileList if x not in videoFiles]
#Don't Link media when the media is extracted from a rar in the same path
if process_method in ('hardlink', 'symlink') and videoInRar:
process_media(processPath, videoInRar, nzbName, 'move', force, is_priority)
process_media(processPath, set(videoFiles) - set(videoInRar), nzbName, process_method, force, is_priority)
delete_files(processPath, rarContent)
else:
process_media(processPath, videoFiles, nzbName, process_method, force, is_priority)
#Avoid to delete files
if process_method != "move" or not process_result \
or type == "manual":
continue
delete_files(processPath, notwantedFiles)
if process_method == "move" \
and helpers.real_path(processPath) != helpers.real_path(sickbeard.TV_DOWNLOAD_DIR):
delete_dir(processPath)
return returnStr
def validate_dir(process_path, release_name, failed, result): # pylint: disable=too-many-locals,too-many-branches,too-many-return-statements
"""
Check if directory is valid for processing
:param process_path: Directory to check
:param release_name: Original NZB/Torrent name
:param failed: Previously failed objects
:param result: Previous results
:return: True if dir is valid for processing, False if not
"""
result.output += log_helper("Processing folder " + process_path, logger.DEBUG)
upper_name = ek(os.path.basename, process_path).upper()
if upper_name.startswith('_FAILED_') or upper_name.endswith('_FAILED_'):
result.output += log_helper("The directory name indicates it failed to extract.", logger.DEBUG)
failed = True
elif upper_name.startswith('_UNDERSIZED_') or upper_name.endswith('_UNDERSIZED_'):
result.output += log_helper("The directory name indicates that it was previously rejected for being undersized.", logger.DEBUG)
failed = True
elif upper_name.startswith('_UNPACK') or upper_name.endswith('_UNPACK'):
result.output += log_helper("The directory name indicates that this release is in the process of being unpacked.", logger.DEBUG)
result.missed_files.append("{0} : Being unpacked".format(process_path))
return False
if failed:
process_failed(process_path, release_name, result)
result.missed_files.append("{0} : Failed download".format(process_path))
return False
if sickbeard.TV_DOWNLOAD_DIR and helpers.real_path(process_path) != helpers.real_path(sickbeard.TV_DOWNLOAD_DIR) and helpers.is_hidden_folder(process_path):
result.output += log_helper("Ignoring hidden folder: {0}".format(process_path), logger.DEBUG)
result.missed_files.append("{0} : Hidden folder".format(process_path))
return False
# make sure the dir isn't inside a show dir
main_db_con = db.DBConnection()
sql_results = main_db_con.select("SELECT location FROM tv_shows")
for sqlShow in sql_results:
if process_path.lower().startswith(ek(os.path.realpath, sqlShow[b"location"]).lower() + os.sep) or \
process_path.lower() == ek(os.path.realpath, sqlShow[b"location"]).lower():
result.output += log_helper(
"Cannot process an episode that's already been moved to its show dir, skipping " + process_path,
logger.WARNING)
return False
for current_directory, directory_names, file_names in ek(os.walk, process_path, topdown=False, followlinks=sickbeard.PROCESSOR_FOLLOW_SYMLINKS):
sync_files = filter(is_sync_file, file_names)
if sync_files and sickbeard.POSTPONE_IF_SYNC_FILES:
result.output += log_helper("Found temporary sync files: {0} in path: {1}".format(sync_files, ek(os.path.join, process_path, sync_files[0])))
result.output += log_helper("Skipping post processing for folder: {0}".format(process_path))
result.missed_files.append("{0} : Sync files found".format(ek(os.path.join, process_path, sync_files[0])))
continue
found_files = filter(helpers.is_media_file, file_names)
if sickbeard.UNPACK == 1:
found_files += filter(helpers.is_rar_file, file_names)
if current_directory != sickbeard.TV_DOWNLOAD_DIR and found_files:
found_files.append(ek(os.path.basename, current_directory))
for found_file in found_files:
try:
NameParser().parse(found_file, cache_result=False)
except (InvalidNameException, InvalidShowException) as e:
pass
else:
return True
result.output += log_helper("{0} : No processable items found in folder".format(process_path), logger.DEBUG)
return False
def _update_zoneinfo():
global sb_timezone
sb_timezone = tz.tzlocal()
# now check if the zoneinfo needs update
url_zv = 'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/zoneinfo.txt'
url_data = helpers.getURL(url_zv)
if url_data is None:
# When urlData is None, trouble connecting to github
logger.log(u"Loading zoneinfo.txt failed. Unable to get URL: " + url_zv, logger.ERROR)
return
if (lib.dateutil.zoneinfo.ZONEINFOFILE is not None):
cur_zoneinfo = ek.ek(basename, lib.dateutil.zoneinfo.ZONEINFOFILE)
else:
cur_zoneinfo = None
(new_zoneinfo, zoneinfo_md5) = url_data.decode('utf-8').strip().rsplit(u' ')
if ((cur_zoneinfo is not None) and (new_zoneinfo == cur_zoneinfo)):
return
# now load the new zoneinfo
url_tar = u'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/' + new_zoneinfo
zonefile = helpers.real_path(ek.ek(join, ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__), new_zoneinfo))
zonefile_tmp = re.sub(r"\.tar\.gz$", '.tmp', zonefile)
if (ek.ek(os.path.exists, zonefile_tmp)):
try:
ek.ek(os.remove, zonefile_tmp)
except:
logger.log(u"Unable to delete: " + zonefile_tmp, logger.ERROR)
return
if not helpers.download_file(url_tar, zonefile_tmp):
return
if not ek.ek(os.path.exists, zonefile_tmp):
logger.log(u"Download of " + zonefile_tmp + " failed.", logger.ERROR)
return
new_hash = str(helpers.md5_for_file(zonefile_tmp))
if (zoneinfo_md5.upper() == new_hash.upper()):
logger.log(u"Updating timezone info with new one: " + new_zoneinfo, logger.MESSAGE)
try:
# remove the old zoneinfo file
if (cur_zoneinfo is not None):
old_file = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__), cur_zoneinfo))
if (ek.ek(os.path.exists, old_file)):
ek.ek(os.remove, old_file)
# rename downloaded file
ek.ek(os.rename, zonefile_tmp, zonefile)
# load the new zoneinfo
reload(lib.dateutil.zoneinfo)
sb_timezone = tz.tzlocal()
except:
_remove_zoneinfo_failed(zonefile_tmp)
return
else:
_remove_zoneinfo_failed(zonefile_tmp)
logger.log(u"MD5 HASH doesn't match: " + zoneinfo_md5.upper() + ' File: ' + new_hash.upper(), logger.ERROR)
return
def _update_zoneinfo():
global sb_timezone
sb_timezone = tz.tzlocal()
# now check if the zoneinfo needs update
url_zv = 'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/zoneinfo.txt'
url_data = helpers.getURL(url_zv)
if url_data is None:
# When urlData is None, trouble connecting to github
logger.log(
u'Loading zoneinfo.txt failed, this can happen from time to time. Unable to get URL: %s'
% url_zv, logger.WARNING)
return
zonefilename = zoneinfo._ZONEFILENAME
cur_zoneinfo = zonefilename
if None is not cur_zoneinfo:
cur_zoneinfo = ek.ek(basename, zonefilename)
zonefile = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, zoneinfo.__file__), cur_zoneinfo))
zonemetadata = zoneinfo.gettz_db_metadata() if ek.ek(
os.path.isfile, zonefile) else None
(new_zoneinfo,
zoneinfo_md5) = url_data.decode('utf-8').strip().rsplit(u' ')
newtz_regex = re.search(r'(\d{4}[^.]+)', new_zoneinfo)
if not newtz_regex or len(newtz_regex.groups()) != 1:
return
newtzversion = newtz_regex.group(1)
if cur_zoneinfo is not None and zonemetadata is not None and 'tzversion' in zonemetadata and zonemetadata[
'tzversion'] == newtzversion:
return
# now load the new zoneinfo
url_tar = u'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/%s' % new_zoneinfo
zonefile_tmp = re.sub(r'\.tar\.gz$', '.tmp', zonefile)
if ek.ek(os.path.exists, zonefile_tmp):
try:
ek.ek(os.remove, zonefile_tmp)
except:
logger.log(u'Unable to delete: %s' % zonefile_tmp, logger.ERROR)
return
if not helpers.download_file(url_tar, zonefile_tmp):
return
if not ek.ek(os.path.exists, zonefile_tmp):
logger.log(u'Download of %s failed.' % zonefile_tmp, logger.ERROR)
return
new_hash = str(helpers.md5_for_file(zonefile_tmp))
if zoneinfo_md5.upper() == new_hash.upper():
logger.log(u'Updating timezone info with new one: %s' % new_zoneinfo,
logger.MESSAGE)
try:
# remove the old zoneinfo file
if cur_zoneinfo is not None:
old_file = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, zoneinfo.__file__),
cur_zoneinfo))
if ek.ek(os.path.exists, old_file):
ek.ek(os.remove, old_file)
# rename downloaded file
ek.ek(os.rename, zonefile_tmp, zonefile)
from dateutil.zoneinfo import gettz
if '_CLASS_ZONE_INSTANCE' in gettz.func_globals:
gettz.func_globals.__setitem__('_CLASS_ZONE_INSTANCE', list())
sb_timezone = tz.tzlocal()
except:
_remove_zoneinfo_failed(zonefile_tmp)
return
else:
_remove_zoneinfo_failed(zonefile_tmp)
logger.log(
u'MD5 hash does not match: %s File: %s' %
(zoneinfo_md5.upper(), new_hash.upper()), logger.ERROR)
return
def process_dir(process_path,
release_name=None,
process_method=None,
force=False,
is_priority=None,
delete_on=False,
failed=False,
mode="auto"):
"""
Scans through the files in process_path and processes whatever media files it finds
:param process_path: The folder name to look in
:param release_name: The NZB/Torrent name which resulted in this folder being downloaded
:param force: True to process previously processed files
:param failed: Boolean for whether or not the download failed
:param mode: Type of postprocessing auto or manual
"""
result = ProcessResult()
# if they passed us a real dir then assume it's the one we want
if ek(os.path.isdir, process_path):
process_path = ek(os.path.realpath, process_path)
result.output += log_helper(
u"Processing folder {0}".format(process_path), logger.DEBUG)
# if the client and SickRage are not on the same machine translate the directory into a network directory
elif all([
sickbeard.TV_DOWNLOAD_DIR,
ek(os.path.isdir, sickbeard.TV_DOWNLOAD_DIR),
ek(os.path.normpath, process_path) == ek(os.path.normpath,
sickbeard.TV_DOWNLOAD_DIR)
]):
process_path = ek(
os.path.join, sickbeard.TV_DOWNLOAD_DIR,
ek(os.path.abspath, process_path).split(os.path.sep)[-1])
result.output += log_helper(
u"Trying to use folder: {0} ".format(process_path), logger.DEBUG)
# if we didn't find a real dir then quit
if not ek(os.path.isdir, process_path):
result.output += log_helper(
u"Unable to figure out what folder to process. "
u"If your downloader and SickRage aren't on the same PC "
u"make sure you fill out your TV download dir in the config.",
logger.DEBUG)
return result.output
process_method = process_method or sickbeard.PROCESS_METHOD
directories_from_rars = []
for current_directory, directory_names, file_names in ek(
os.walk, process_path):
if not validate_dir(current_directory, release_name, failed, result):
continue
result.result = True
file_names = [f for f in file_names if not is_torrent_or_nzb_file(f)]
rar_files = [
x for x in file_names
if helpers.is_rar_file(ek(os.path.join, current_directory, x))
]
if rar_files:
extracted_directories = unrar(current_directory, rar_files, force,
result)
# Add the directories to the walk directories
result.output += log_helper(
u"Adding extracted directories to the list of directories to process: {0}"
.format(extracted_directories), logger.DEBUG)
directories_from_rars += extracted_directories
video_files = filter(helpers.is_media_file, file_names)
unwanted_files = [
x for x in file_names if x not in video_files and x != '.stfolder'
]
if unwanted_files:
result.output += log_helper(
u"Found unwanted files: {0}".format(unwanted_files),
logger.DEBUG)
process_media(current_directory, video_files, release_name,
process_method, force, is_priority, result)
# Delete all file not needed and avoid deleting files if Manual PostProcessing
if not (process_method == u"move"
and result.result) or (mode == u"manual" and not delete_on):
continue
delete_folder(ek(os.path.join, current_directory, u'@eaDir'), False)
delete_files(current_directory, unwanted_files, result)
if all([
not sickbeard.NO_DELETE or mode == u"manual",
process_method == u"move", sickbeard.TV_DOWNLOAD_DIR
and helpers.real_path(current_directory) != helpers.real_path(
sickbeard.TV_DOWNLOAD_DIR)
]):
if delete_folder(current_directory, check_empty=True):
result.output += log_helper(
u"Deleted folder: {0}".format(current_directory),
logger.DEBUG)
for directory_from_rar in directories_from_rars:
process_dir(
process_path=directory_from_rar,
release_name=ek(os.path.basename, directory_from_rar),
process_method=('move',
process_method)[process_method in ('move',
'copy')],
force=force,
is_priority=is_priority,
delete_on=sickbeard.DELRARCONTENTS,
failed=failed,
mode=mode)
if sickbeard.DELRARCONTENTS:
delete_folder(directory_from_rar, False)
result.output += log_helper(
(u"Processing Failed", u"Successfully processed")[result.aggresult],
(logger.WARNING, logger.INFO)[result.aggresult])
if result.missed_files:
result.output += log_helper(u"Some items were not processed.")
for missed_file in result.missed_files:
result.output += log_helper(missed_file)
return result.output
def _update_zoneinfo():
global sb_timezone
sb_timezone = tz.tzlocal()
# TODO `git subtree pull` commands on updates
loc_zv = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, __file__),
u'../lib/network_timezones/zoneinfo.txt'))
# Read version file
try:
with open(loc_zv, 'r') as file:
data = file.read()
if not data:
raise
# Filename of existing zoneinfo
if lib.dateutil.zoneinfo.ZONEINFOFILE is not None:
cur_zoneinfo = ek.ek(basename, lib.dateutil.zoneinfo.ZONEINFOFILE)
else:
cur_zoneinfo = None
# Filename and hash of new zoneinfo
(new_zoneinfo,
zoneinfo_md5) = data.decode('utf-8').strip().rsplit(u' ')
except Exception as e:
logger.log(u'Crazy problem with zoneinfo: %s' % ex(e), logger.ERROR)
return
if (cur_zoneinfo is not None) and (new_zoneinfo == cur_zoneinfo):
return
# now load the new zoneinfo
loc_tar = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, __file__),
u'../lib/network_timezones/%s' % new_zoneinfo))
zonefile = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__),
new_zoneinfo))
zonefile_tmp = re.sub(r'\.tar\.gz$', '.tmp', zonefile)
if ek.ek(os.path.exists, zonefile_tmp):
try:
ek.ek(os.remove, zonefile_tmp)
except:
logger.log(u'Unable to delete: %s' % zonefile_tmp, logger.ERROR)
return
if not helpers.copyFile(loc_tar, zonefile_tmp):
return
if not ek.ek(os.path.exists, zonefile_tmp):
logger.log(u'Download of %s failed.' % zonefile_tmp, logger.ERROR)
return
new_hash = str(helpers.md5_for_file(zonefile_tmp))
if zoneinfo_md5.upper() == new_hash.upper():
logger.log(u'Updating timezone info with new one: %s' % new_zoneinfo,
logger.INFO)
try:
# remove the old zoneinfo file
if cur_zoneinfo is not None:
old_file = helpers.real_path(
ek.ek(
join,
ek.ek(os.path.dirname, lib.dateutil.zoneinfo.__file__),
cur_zoneinfo))
if ek.ek(os.path.exists, old_file):
ek.ek(os.remove, old_file)
# rename downloaded file
ek.ek(os.rename, zonefile_tmp, zonefile)
# load the new zoneinfo
reload(lib.dateutil.zoneinfo)
sb_timezone = tz.tzlocal()
except:
_remove_zoneinfo_failed(zonefile_tmp)
return
else:
_remove_zoneinfo_failed(zonefile_tmp)
logger.log(
u'MD5 hash does not match: %s File: %s' %
(zoneinfo_md5.upper(), new_hash.upper()), logger.ERROR)
return
def update_network_dict():
_remove_old_zoneinfo()
_update_zoneinfo()
d = {}
# network timezones are stored in a git submodule
loc = helpers.real_path(
ek.ek(join, ek.ek(os.path.dirname, __file__),
u'../lib/network_timezones/network_timezones.txt'))
with open(loc, 'r') as file:
data = file.read()
if data is None:
logger.log(u'Updating network timezones failed', logger.ERROR)
load_network_dict()
return
try:
for line in data.splitlines():
(key, val) = line.decode('utf-8').strip().rsplit(u':', 1)
if key is None or val is None:
continue
d[key] = val
except (IOError, OSError):
pass
my_db = db.DBConnection('cache.db')
# load current network timezones
old_d = dict(my_db.select('SELECT * FROM network_timezones'))
# list of sql commands to update the network_timezones table
cl = []
for cur_d, cur_t in d.iteritems():
h_k = old_d.has_key(cur_d)
if h_k and cur_t != old_d[cur_d]:
# update old record
cl.append([
'UPDATE network_timezones SET network_name=?, timezone=? WHERE network_name=?',
[cur_d, cur_t, cur_d]
])
elif not h_k:
# add new record
cl.append([
'INSERT INTO network_timezones (network_name, timezone) VALUES (?,?)',
[cur_d, cur_t]
])
if h_k:
del old_d[cur_d]
# remove deleted records
if len(old_d) > 0:
old_items = list(va for va in old_d)
cl.append([
'DELETE FROM network_timezones WHERE network_name IN (%s)' %
','.join(['?'] * len(old_items)), old_items
])
# change all network timezone infos at once (much faster)
if len(cl) > 0:
my_db.mass_action(cl)
load_network_dict()
def processDir(dirName,
nzbName=None,
process_method=None,
force=False,
is_priority=None,
failed=False,
type="automatic"):
"""
Scans through the files in dirName and processes whatever media files it finds
dirName: The folder name to look in
nzbName: The NZB name which resulted in this folder being downloaded
force: True to postprocess already postprocessed files
failed: Boolean for whether or not the download failed
type: Type of postprocessing automatic or manual
"""
global process_result, returnStr
returnStr = u""
returnStr += logHelper(u"Processing folder " + dirName, logger.DEBUG)
returnStr += logHelper(u"TV_DOWNLOAD_DIR: " + sickbeard.TV_DOWNLOAD_DIR,
logger.DEBUG)
# if they passed us a real dir then assume it's the one we want
if ek.ek(os.path.isdir, dirName):
dirName = ek.ek(os.path.realpath, dirName)
# if the client and Sickbeard are not on the same machine translate the Dir in a network dir
elif sickbeard.TV_DOWNLOAD_DIR and ek.ek(os.path.isdir, sickbeard.TV_DOWNLOAD_DIR) \
and helpers.real_path(dirName) != helpers.real_path(sickbeard.TV_DOWNLOAD_DIR):
dirName = ek.ek(os.path.join, sickbeard.TV_DOWNLOAD_DIR,
ek.ek(os.path.abspath, dirName).split(os.path.sep)[-1])
returnStr += logHelper(u"Trying to use folder " + dirName,
logger.DEBUG)
# if we didn't find a real dir then quit
if not ek.ek(os.path.isdir, dirName):
returnStr += logHelper(
u"Unable to figure out what folder to process. If your downloader and Sick Beard aren't on the same PC make sure you fill out your TV download dir in the config.",
logger.DEBUG)
return returnStr
path, dirs, files = get_path_dir_files(dirName, nzbName, type)
returnStr += logHelper(u"PostProcessing Path: " + path, logger.DEBUG)
returnStr += logHelper(u"PostProcessing Dirs: " + str(dirs), logger.DEBUG)
rarFiles = filter(helpers.isRarFile, files)
rarContent = unRAR(path, rarFiles, force)
files += rarContent
videoFiles = filter(helpers.isMediaFile, files)
videoInRar = filter(helpers.isMediaFile, rarContent)
returnStr += logHelper(u"PostProcessing Files: " + str(files),
logger.DEBUG)
returnStr += logHelper(u"PostProcessing VideoFiles: " + str(videoFiles),
logger.DEBUG)
returnStr += logHelper(u"PostProcessing RarContent: " + str(rarContent),
logger.DEBUG)
returnStr += logHelper(u"PostProcessing VideoInRar: " + str(videoInRar),
logger.DEBUG)
# If nzbName is set and there's more than one videofile in the folder, files will be lost (overwritten).
nzbNameOriginal = nzbName
if len(videoFiles) >= 2:
nzbName = None
if not process_method:
process_method = sickbeard.PROCESS_METHOD
process_result = True
#Don't Link media when the media is extracted from a rar in the same path
if process_method in ('hardlink', 'symlink') and videoInRar:
process_media(path, videoInRar, nzbName, 'move', force, is_priority)
delete_files(path, rarContent)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force,
is_priority)
else:
for video in videoFiles:
process_media(path, [video], nzbName, process_method, force,
is_priority)
#Process Video File in all TV Subdir
for dir in [
x for x in dirs if validateDir(path, x, nzbNameOriginal, failed)
]:
process_result = True
for processPath, processDir, fileList in ek.ek(os.walk,
ek.ek(
os.path.join, path,
dir),
topdown=False):
rarFiles = filter(helpers.isRarFile, fileList)
rarContent = unRAR(processPath, rarFiles, force)
fileList = set(fileList + rarContent)
videoFiles = filter(helpers.isMediaFile, fileList)
videoInRar = filter(helpers.isMediaFile, rarContent)
notwantedFiles = [x for x in fileList if x not in videoFiles]
#Don't Link media when the media is extracted from a rar in the same path
if process_method in ('hardlink', 'symlink') and videoInRar:
process_media(processPath, videoInRar, nzbName, 'move', force,
is_priority)
process_media(processPath,
set(videoFiles) - set(videoInRar), nzbName,
process_method, force, is_priority)
delete_files(processPath, rarContent)
else:
process_media(processPath, videoFiles, nzbName, process_method,
force, is_priority)
#Avoid to delete files
if process_method != "move" or not process_result \
or type == "manual":
continue
delete_files(processPath, notwantedFiles)
if process_method == "move" \
and helpers.real_path(processPath) != helpers.real_path(sickbeard.TV_DOWNLOAD_DIR):
delete_dir(processPath)
return returnStr