def cleanDatabase(self):
log.trace('ScanSeriesLibrary: Start')
for user in self.settings.Hostnames:
_table_entries = []
try:
cursor = self.db.cursor()
cursor.execute('SELECT FileName FROM Downloads WHERE Name = "{}"'.format(user))
for row in cursor:
_table_entries.append(unicodedata.normalize('NFKD', row[0]).encode('ascii', 'ignore'))
except:
self.db.close()
log.error("Processing Stopped: SQLITE3 Error")
return []
cursor = self.db.cursor()
for _file_name in _table_entries:
_target = os.path.join('/mnt/Links', user, self.settings.IncrementalsDir, re.split(self.settings.SeriesDir, _file_name)[1].lstrip(os.sep))
if os.path.lexists(_target):
continue
else:
try:
# SQL #
cursor.execute('DELETE FROM Downloads WHERE Name = "{}" AND Filename = "{}"'.format(user, _file_name))
except sqlite3.Error, e:
raise UnexpectedErrorOccured("File Information Deletion: {} {}".format(e, _file_name))
self.db.commit()
def move_files(old, new):
files_to_move = listdir(dirname(old))
file_list = []
for entry in files_to_move:
entry = join(dirname(old), entry)
if any(fnmatch.fnmatch(entry.lower(), pattern) for pattern in DadVision.settings.AdditionalGlob):
file_list.append(entry)
else:
if ignored(entry):
if isfile(entry): del_file(entry)
if isdir(entry): del_dir(entry, Tree=True)
if file_list:
try:
cmd = ['rsync', '-rptvhogLR', '--progress', '--remove-source-files',
'--partial-dir=.rsync-partial']
cmd.extend(file_list)
cmd.append(dirname(old))
log.verbose(' '.join(cmd))
check_call(cmd, shell=False, stdin=None, stdout=None, stderr=None, cwd=dirname(old))
except CalledProcessError, exc:
log.error("Incremental rsync Command returned with RC=%d, Ending" % (exc.returncode))
raise UnexpectedErrorOccured(exc)
def _rename_file(self, movie_details):
log.trace("=================================================")
log.trace(
"_rename_file method: movie_details: {!s}".format(movie_details))
try:
_fq_new_file_name = self._get_new_filename(movie_details)
if not self._duplicate_file(_fq_new_file_name,
movie_details['FileName']):
log.info('Renaming Movie: %s to %s' % (os.path.basename(
movie_details['FileName']), _fq_new_file_name))
if not os.path.exists(os.path.dirname(_fq_new_file_name)):
_new_dir = os.path.dirname(_fq_new_file_name)
os.makedirs(_new_dir)
os.chmod(_new_dir, 0775)
os.rename(movie_details['FileName'], _fq_new_file_name)
os.chmod(_fq_new_file_name, 0664)
log.info("Successfully Renamed: %s" % _fq_new_file_name)
if len(os.listdir(os.path.dirname(
movie_details['FileName']))) > 0:
return
if self.args.move:
_del_dir(os.path.dirname(movie_details['FileName']))
except OSError, exc:
log.error("Skipping, Unable to Rename File: %s" %
movie_details['FileName'])
log.error("Unexpected error: %s" % exc)
raise UnexpectedErrorOccured("Unexpected error: %s" % exc)
def make_dir(pathname):
try:
if not exists(pathname):
makedirs(pathname)
chmod(pathname, 0775)
log.info("Successfully Created: %s" % pathname)
except OSError, e:
log.error("Unexpected error: %s" % e)
raise UnexpectedErrorOccured("Unexpected error: %s" % e)
def ScanFileEntries(self):
log.trace('ScanSeriesLibrary: Start')
self.cursor.execute('SELECT COUNT(*)e FROM Files')
Result = self.cursor.fetchall()
File_Count = Result[0][0]
Files_Processed = 0
Files_Deleted = 0
Files_Updated = 0
log.info('Number of File to be Checked: %s' % File_Count)
self.cursor.execute('SELECT f.FileName FROM Files f')
Result = self.cursor.fetchall()
for row in Result:
Files_Processed += 1
if os.path.exists(row[0]):
pass
else:
_target = re.split(self.directory_old, row[0])
if len(_target) > 1:
try:
Files_Updated += 1
episode = os.path.join(self.settings.SeriesDir,
_target[1].lstrip(os.sep))
self.cursor.execute(
'UPDATE Files SET FileName="{}" WHERE FileName="{}"'
.format(episode.encode('ascii', 'ignore'),
row[0].encode('ascii', 'ignore')))
except sqlite3.Error, e:
if e.message == 'column FileName is not unique':
self.cursor.execute(
'DELETE FROM Files WHERE FileName="{}"'.format(
row[0]))
Files_Deleted += 1
else:
sys.exc_clear()
# print 'ERROR: {}'.format(e.message.encode('ascii', 'ignore'))
else:
log.info('Entry Removed: {}'.format(row[0].encode(
'ascii', 'ignore')))
try:
self.cursor.execute(
'DELETE FROM Files WHERE FileName="{}"'.format(
row[0].encode('ascii', 'ignore')))
Files_Deleted += 1
except sqlite3.Error, e:
raise UnexpectedErrorOccured(
"File Information Insert: {} {}".format(
e.message, row[0].encode('ascii', 'ignore')))
def rename_file(old, new):
if isdir(old): raise InvalidPath('Directory was Requested for File Rename: {}'.format(old))
try:
if not exists(dirname(new)): make_dir(dirname(new))
rename(old, new)
chmod(new, 0664)
log.info("Successfully Renamed: %s" % new)
if len(listdir(dirname(old))) > 1:
move_files(old, new)
if len(listdir(dirname(old))) == 0:
del_dir(dirname(old))
except OSError, e:
log.error("Unable to Rename File: {} - {}".format(old, e))
raise UnexpectedErrorOccured("Unable to Rename File: {} - {}".format(old, e))
def _rename_directory(self, directory):
log.trace("=================================================")
log.trace("_rename_directory method: pathname:{!s}".format(directory))
_directory_details = self.fileparser.getFileDetails(directory + '.avi')
_directory_details['FileName'] = directory
_directory_details = self.tmdbinfo.tmdb(_directory_details)
if 'Year' in _directory_details:
_new_dir = '%s (%s)' % (_directory_details['MovieName'],
_directory_details['Year'])
else:
_new_dir = '%s' % (_directory_details['MovieName'])
_target_dir = os.path.join(self.settings.MoviesDir, _new_dir)
if os.path.exists(_target_dir):
if _target_dir == directory:
log.trace(
'Skipping: Directory already properly named and in: {}'.
format(directory))
return
else:
_target_dir = os.path.join(
os.path.split(self.settings.MoviesDir)[0], _new_dir)
if os.path.exists(_target_dir):
log.warn(
"Unable to process, Directory: {}, already at destination!"
.format(_target_dir))
return
log.info('Renaming Movie Directory: %s to %s' %
(os.path.basename(directory), _target_dir))
try:
os.rename(directory, _target_dir)
log.info("Successfully Renamed: %s" % _target_dir)
except OSError, exc:
log.error("Skipping, Unable to Rename Directory: %s" % directory)
log.error("Unexpected error: %s" % exc)
raise UnexpectedErrorOccured("Unexpected error: %s" % exc)
class DownloadDatabase(Library):
def __init__(self):
log.trace('__init__ method: Started')
super(DownloadDatabase, self).__init__()
self.db = sqlite3.connect(self.settings.DBFile)
self.cursor = self.db.cursor()
self.fileparser = FileParser()
def ScanSeriesLibrary(self):
log.trace('ScanSeriesLibrary: Start')
FilesToBeAdded = []
Files_Loaded = 0
Files_Processed = 0
Files_Already_Processed = 0
Files_Non_Video = 0
Files_with_Errors = 0
File_Count = countFiles(self.settings.SeriesDir,
exclude_list=self.settings.ExcludeList)
log.info('Number of File to be Checked: %s' % File_Count)
for _root, _dirs, _files in os.walk(self.settings.SeriesDir):
if _dirs is not None:
_dirs.sort()
for _dir in _dirs[:]:
# Process Enbedded Directories
if self._ignored(_dir):
_dirs.remove(_dir)
for _file_name in _files:
quotient, remainder = divmod(Files_Processed, 250)
if remainder == 0:
self.db.commit()
log.info('Checked: %2.2f%% - %5s of %5s Errors: %s Present: %s Non-Video: %s ' \
% ((Files_Processed/ File_Count)*100,
Files_Processed,
File_Count,
Files_with_Errors,
Files_Already_Processed,
Files_Non_Video
)
)
Files_Processed += 1
_fq_name = os.path.join(_root, _file_name)
log.trace('Processing File: %s' % _fq_name)
try:
_ext = os.path.splitext(_file_name)[1][1:]
if _ext not in self.settings.MediaExt:
Files_Non_Video += 1
continue
_file_details = self.fileparser.getFileDetails(_fq_name)
self.load_entry(_file_details)
Files_Loaded += 1
except InvalidFilename:
log.info('Skipping Series Not Found: {}'.format(_fq_name))
Files_with_Errors += 1
continue
except DuplicateRecord:
Files_Already_Processed += 1
continue
self.db.commit()
log.info('Complete: Files Checked: %5s Number of Errors: %s' %
(Files_Processed, Files_with_Errors))
def load_entry(self, file_details):
# t = os.path.getmtime(file_details['FileName'])
# timestamp = datetime.datetime.fromtimestamp(t)
try:
# SQL #
self.cursor.execute(
'INSERT INTO Files(SeriesName, SeasonNum, EpisodeNum, Filename) \
VALUES ("{}", {}, {}, "{}")'.format(file_details['SeriesName'],
file_details['SeasonNum'],
file_details['EpisodeNums'][0],
file_details['FileName']))
file_id = int(self.cursor.lastrowid)
except sqlite3.IntegrityError, e:
raise DuplicateRecord
except sqlite3.Error, e:
raise UnexpectedErrorOccured(
"File Information Insert: {} {}".format(e, file_details))
def renameFile(self, pathname):
log.info('-' * 70)
_series = None
try:
_series = self.getShowInfo(pathname)
_series = self.getFileName(_series)
except (SeasonNotFound, EpisodeNotFound):
if _series and _series.title:
log.error('SERIES: {}'.format(_series.title))
raise
log.info('SERIES: {}'.format(_series.fileDetails.seriesTitle))
_season_folder = os.path.dirname(_series.fileDetails.newName)
_series_folder = os.path.dirname(_season_folder)
log.info('Renamed: SEASON: {}'.format(_series.fileDetails.seasonNum))
if not os.path.exists(_season_folder):
os.makedirs(_season_folder)
os.chmod(_season_folder, 0775)
os.chmod(_series_folder, 0775)
else:
_series = self._replace_existing(_series)
if _series.fileDetails.fileName == _series.fileDetails.newName:
log.info('Processed: SEASON: {} FILE: {}'.format(
_series.fileDetails.seasonNum,
os.path.basename(_series.fileDetails.newName)))
self._update_date(_series)
return
else:
os.rename(_series.fileDetails.fileName,
_series.fileDetails.newName)
os.chmod(_series.fileDetails.newName, 0664)
log.info('Renamed: FILE: {}'.format(
os.path.basename(_series.fileDetails.newName)))
log.info('Renamed: New: {}'.format(
os.path.basename(_series.fileDetails.fileName)))
self._update_date(_series)
self._del_dir(os.path.dirname(_series.fileDetails.fileName))
self.xbmc_update_required = True
if self.hostname == 'grumpy':
try:
self.db = sqlite3.connect(DadVision.settings.DBFile)
self.cursor = self.db.cursor()
self.cursor.execute(
'INSERT INTO Files(SeriesName, SeasonNum, EpisodeNum, Filename) \
VALUES ("{}", {}, {}, "{}")'.format(_series.fileDetails.seriesTitle,
_series.fileDetails.seasonNum,
_series.fileDetails.episodeNums[0],
_series.fileDetails.fileName))
self.db.commit()
self.db.close()
except sqlite3.IntegrityError, e:
self.db.close()
except sqlite3.Error, e:
self.db.close()
raise UnexpectedErrorOccured(
"File Information Insert: {} {}".format(
e, _series.__dict__))
def ScanSeriesLibrary(self, type_scan, user_list):
log.trace('ScanSeriesLibrary: Start')
user_profiles = config.GetSubscribers()
for user in config.Users:
if user_list and user not in user_list:
continue
user_profile = user_profiles[user]
Files_Loaded = 0
Files_Processed = 0
source_directory = os.path.join(config.SubscriptionDir, user,
type_scan)
File_Count = countFiles(source_directory,
exclude_list=config.ExcludeList)
log.info('{:5s} - Number of File to be Checked: {}'.format(
user, File_Count))
for _symlink_dir in os.listdir(source_directory):
target_dir = os.path.join(config.SeriesDir, _symlink_dir)
if not os.path.isdir(target_dir):
raise UnexpectedErrorOccured(
'Series Library referenced in setting NOT FOUND: {}'.
format(target_dir))
sys.exit(1)
for _root, _dirs, _files in os.walk(target_dir):
if _dirs != None:
_dirs.sort()
for _exclude_dir in config.ExcludeList:
try:
_index = _dirs.index(_exclude_dir)
_dirs.pop(_index)
logger.TRACE('Removing Dir: %s' % _exclude_dir)
except:
continue
for _file_name in _files:
Files_Processed += 1
_fq_name = os.path.join(_root, _file_name)
log.trace('Processing File: %s' % _fq_name)
try:
_file_ext = os.path.splitext(
_fq_name)[1][1:].lower()
if _file_ext in config.MediaExt:
load_entry(user, _fq_name)
Files_Loaded += 1
else:
continue
log.info('Skipping Non-VIdeo File: {}'.format(
_fq_name))
except DuplicateRecord:
Files_Loaded += 1
quotient, remainder = divmod(Files_Processed, 250)
if remainder == 0:
db.commit()
log.info(
'%-5s - Files Checked: %2.2f%% %5s of %5s Number of Errors: %s'
% (user, Files_Processed / File_Count,
Files_Processed, File_Count,
Files_Processed - Files_Loaded))
db.commit()
log.info(
'{:5s} - Complete: Files Checked: {:5n} Number of Errors: {}'
.format(user, Files_Processed, Files_Processed - Files_Loaded))
def load_entry(user, file_name):
t = os.path.getmtime(file_name)
timestamp = datetime.datetime.fromtimestamp(t)
try:
file_details = fileparser.getFileDetails(file_name)
# SQL #
cursor.execute(
'INSERT INTO Downloads(Name, SeriesName, Filename, DownloadTimeStamp) VALUES ("{}", "{}", "{}", "{}")'
.format(user, file_details['SeriesName'], file_name, timestamp))
except sqlite3.IntegrityError, e:
raise DuplicateRecord
except sqlite3.Error, e:
raise UnexpectedErrorOccured("File Information Insert: {} {}".format(
e, file_name))
except InvalidFilename:
pass
class DownloadDatabase(object):
def __init__(self):
pass
def ScanSeriesLibrary(self, type_scan, user_list):
log.trace('ScanSeriesLibrary: Start')
user_profiles = config.GetSubscribers()
for user in config.Users:
if user_list and user not in user_list:
continue
request = Request(url, headers=headers)
try:
response_body = urlopen(request).read()
except HTTPError, e:
return e
data = json.loads(response_body.decode('UTF-8', 'ignore'))
if rtn == str:
return data
elif rtn is dict:
_list = {}
elif rtn is list:
_list = []
else:
raise UnexpectedErrorOccured('Invalid rtn object type requested')
for entry in data:
if 'type' in entry:
if entry['type'] in [u'show', u'episode']:
_object = Series(**entry)
elif entry['type'] == u'movie':
_object = Movie(**entry)
else:
sys.exit(99)
elif 'show' in entry:
_object = Series(**entry)
elif 'movie' in entry:
_object = Movie(**entry['movie'])
else:
_object = Series(**data)
def __init__(self):
log.trace('__init__')
super(Distribute, self).__init__()
dist1 = DadVision.cmdoptions.parser.add_argument_group(
"Media Type Options", description=None)
dist1.add_argument("-s",
"--series",
dest="content",
default=None,
action="store_const",
const="Series",
help="Process as Series")
dist1.add_argument("-m",
"--movies",
dest="content",
action="store_const",
const="Movies",
help="Process as Movies")
dist1.add_argument("-n",
"--non-video",
dest="content",
action="store_const",
const="NonVideo",
help="Process as Non-Video")
dist2 = DadVision.cmdoptions.parser.add_argument_group(
"Distribute Options", description=None)
dist2.add_argument("--no-cleanup",
"--nc",
dest="clean_up_name",
action="store_false",
default=True,
help="Do not clean-up names from unpack")
dist2.add_argument("--no-ignore",
"--ni",
dest="ignore",
action="store_false",
default=True,
help="Process all files, Ignore nothing")
dist2.add_argument("--no-movies",
dest="suppress_movies",
action="store_true",
default=False,
help="Do Not Process Movie files")
dist2.add_argument("--no-rename",
"--nr",
dest="rename",
action="store_false",
default=True,
help="Do Not Rename the File from NEW")
self.rename_series = RenameSeries()
self.movie = Movie()
self.contentType = None
self.RegEx = []
try:
self.RegEx.append(
re.compile(
'(.*)[\._ \-][s|season]([0-9]+)[e|x]?[0-9]?([^\\/]*)',
re.IGNORECASE))
self.RegEx.append(
re.compile('(.*)[\._ \-][s]?([0-9]+)[e|x][0-9]?([^\\/]*)',
re.IGNORECASE))
self.RegEx.append(
re.compile('(.*)[\._ \-]([0-9]+\.[0-9]+\.[0-9]+)([^\\/]*)',
re.IGNORECASE))
self.RegEx.append(
re.compile('(.*)[\._ \-]season[\._ \-]([0-9]+)([^\\/]*)',
re.IGNORECASE))
except re.error, errormsg:
log.error("Distribute.__init__: Invalid Series RegEx pattern: %s" %
(errormsg))
raise UnexpectedErrorOccured(
"Distribute.__init__: Invalid Series RegEx: %s" % (errormsg))
def distributeFile(self, pathname):
log.trace('distributeFile: %s %s' % (self.contentType, pathname))
""" Move or copy a single file.
"""
# ignored file?
if DadVision.args.ignore and ignored(os.path.basename(pathname)):
log.verbose("Ignoring %r!" % (pathname))
return
if os.path.splitext(
pathname)[1][1:].lower() not in DadVision.settings.MediaExt:
log.verbose("Ignoring %r!" % (pathname))
return
self._setContentType(pathname)
if self.contentType == 'Movies' and DadVision.args.suppress_movies:
log.verbose('Skipping Movie: %s' % pathname)
return
if self.contentType == "Series":
_destinationDir = DadVision.settings.NewSeriesDir
elif self.contentType == "Movies":
_destinationDir = DadVision.settings.NewMoviesDir
else:
_destinationDir = DadVision.settings.UnpackDir
# Adds the last directory to target name
_destinationDir = os.path.join(
_destinationDir, os.path.basename(os.path.dirname(pathname)))
_distributedFile = os.path.join(_destinationDir,
os.path.basename(pathname))
if os.path.exists(_distributedFile) and os.path.getsize(
_distributedFile) == os.path.getsize(pathname):
log.verbose("Skipped Copy: Already at Destination:")
log.verbose(" {}".format(os.path.basename(pathname)))
log.verbose(" {}".format(
os.path.basename(_distributedFile)))
else:
# make sure target folder exists
if not os.path.exists(_destinationDir):
log.debug(
"Creating %r" % _destinationDir.rstrip(os.sep) + os.sep, )
os.makedirs(_destinationDir)
os.chmod(_destinationDir, 0775)
# copy file, possibly across devices.
try:
cmd = [
'rsync', '-rptvhogLR', '--progress',
'--partial-dir=.rsync-partial',
os.path.basename(pathname), _destinationDir
]
log.verbose(' '.join(cmd))
check_call(cmd,
shell=False,
stdin=None,
stdout=None,
stderr=None,
cwd=os.path.dirname(pathname))
except CalledProcessError, exc:
log.error(
"Incremental rsync Command returned with RC=%d, Ending" %
(exc.returncode))
raise UnexpectedErrorOccured(exc)