def processComplete(self, release_download, delete_files):
log.debug('Requesting rTorrent to remove the torrent %s%s.',
(release_download['name'], ' and cleanup the downloaded files' if delete_files else ''))
if not self.connect():
return False
torrent = self.rt.find_torrent(release_download['id'])
if torrent is None:
return False
if delete_files:
for file_item in torrent.get_files(): # will only delete files, not dir/sub-dir
os.unlink(os.path.join(torrent.directory, file_item.path))
if torrent.is_multi_file() and torrent.directory.endswith(torrent.name):
# Remove empty directories bottom up
try:
for path, _, _ in scandir.walk(torrent.directory, topdown = False):
os.rmdir(path)
except OSError:
log.info('Directory "%s" contains extra files, unable to remove', torrent.directory)
torrent.erase() # just removes the torrent, doesn't delete data
return True
def replaceWith(self, path):
app_dir = ss(Env.get('app_dir'))
data_dir = ss(Env.get('data_dir'))
# Get list of files we want to overwrite
self.deletePyc()
existing_files = []
for root, subfiles, filenames in scandir.walk(app_dir):
for filename in filenames:
existing_files.append(os.path.join(root, filename))
for root, subfiles, filenames in scandir.walk(path):
for filename in filenames:
fromfile = os.path.join(root, filename)
tofile = os.path.join(app_dir, fromfile.replace(path + os.path.sep, ''))
if not Env.get('dev'):
try:
if os.path.isfile(tofile):
os.remove(tofile)
dirname = os.path.dirname(tofile)
if not os.path.isdir(dirname):
self.makeDir(dirname)
shutil.move(fromfile, tofile)
try:
existing_files.remove(tofile)
except ValueError:
pass
except:
log.error('Failed overwriting file "%s": %s', (tofile, traceback.format_exc()))
return False
for still_exists in existing_files:
if data_dir in still_exists:
continue
try:
os.remove(still_exists)
except:
log.error('Failed removing non-used file: %s', traceback.format_exc())
return True
def deleteEmptyFolder(self, folder, show_error = True):
folder = sp(folder)
for root, dirs, files in scandir.walk(folder):
for dir_name in dirs:
full_path = os.path.join(root, dir_name)
if len(os.listdir(full_path)) == 0:
try:
os.rmdir(full_path)
except:
if show_error:
log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
try:
os.rmdir(folder)
except:
if show_error:
log.error('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc()))
def deletePyc(self, only_excess = True, show_logs = True):
for root, dirs, files in scandir.walk(ss(Env.get('app_dir'))):
pyc_files = filter(lambda filename: filename.endswith('.pyc'), files)
py_files = set(filter(lambda filename: filename.endswith('.py'), files))
excess_pyc_files = filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) if only_excess else pyc_files
for excess_pyc_file in excess_pyc_files:
full_path = os.path.join(root, excess_pyc_file)
if show_logs: log.debug('Removing old PYC file: %s', full_path)
try:
os.remove(full_path)
except:
log.error('Couldn\'t remove %s: %s', (full_path, traceback.format_exc()))
for dir_name in dirs:
full_path = os.path.join(root, dir_name)
if len(os.listdir(full_path)) == 0:
try:
os.rmdir(full_path)
except:
log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
def scan(
self,
folder=None,
files=None,
release_download=None,
simple=False,
newer_than=0,
return_ignored=True,
on_found=None,
):
folder = sp(folder)
if not folder or not os.path.isdir(folder):
log.error("Folder doesn't exists: %s", folder)
return {}
# Get movie "master" files
movie_files = {}
leftovers = []
# Scan all files of the folder if no files are set
if not files:
check_file_date = True
try:
files = []
for root, dirs, walk_files in scandir.walk(folder, followlinks=True):
files.extend([sp(os.path.join(root, filename)) for filename in walk_files])
# Break if CP wants to shut down
if self.shuttingDown():
break
except:
log.error("Failed getting files from %s: %s", (folder, traceback.format_exc()))
log.debug("Found %s files to scan and group in %s", (len(files), folder))
else:
check_file_date = False
files = [sp(x) for x in files]
for file_path in files:
if not os.path.exists(file_path):
continue
# Remove ignored files
if self.isSampleFile(file_path):
leftovers.append(file_path)
continue
elif not self.keepFile(file_path):
continue
is_dvd_file = self.isDVDFile(file_path)
if (
self.filesizeBetween(file_path, self.file_sizes["movie"]) or is_dvd_file
): # Minimal 300MB files or is DVD file
# Normal identifier
identifier = self.createStringIdentifier(file_path, folder, exclude_filename=is_dvd_file)
identifiers = [identifier]
# Identifier with quality
quality = (
fireEvent("quality.guess", [file_path], single=True) if not is_dvd_file else {"identifier": "dvdr"}
)
if quality:
identifier_with_quality = "%s %s" % (identifier, quality.get("identifier", ""))
identifiers = [identifier_with_quality, identifier]
if not movie_files.get(identifier):
movie_files[identifier] = {"unsorted_files": [], "identifiers": identifiers, "is_dvd": is_dvd_file}
movie_files[identifier]["unsorted_files"].append(file_path)
else:
leftovers.append(file_path)
# Break if CP wants to shut down
if self.shuttingDown():
break
# Cleanup
del files
# Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2"
# files will be grouped first.
leftovers = set(sorted(leftovers, reverse=True))
# Group files minus extension
ignored_identifiers = []
for identifier, group in movie_files.items():
if identifier not in group["identifiers"] and len(identifier) > 0:
group["identifiers"].append(identifier)
log.debug("Grouping files: %s", identifier)
has_ignored = 0
for file_path in list(group["unsorted_files"]):
ext = getExt(file_path)
wo_ext = file_path[: -(len(ext) + 1)]
found_files = set([i for i in leftovers if wo_ext in i])
group["unsorted_files"].extend(found_files)
leftovers = leftovers - found_files
has_ignored += 1 if ext == "ignore" else 0
if has_ignored == 0:
for file_path in list(group["unsorted_files"]):
ext = getExt(file_path)
has_ignored += 1 if ext == "ignore" else 0
if has_ignored > 0:
ignored_identifiers.append(identifier)
# Break if CP wants to shut down
if self.shuttingDown():
break
# Create identifiers for all leftover files
path_identifiers = {}
for file_path in leftovers:
identifier = self.createStringIdentifier(file_path, folder)
if not path_identifiers.get(identifier):
path_identifiers[identifier] = []
path_identifiers[identifier].append(file_path)
# Group the files based on the identifier
delete_identifiers = []
for identifier, found_files in path_identifiers.items():
log.debug("Grouping files on identifier: %s", identifier)
group = movie_files.get(identifier)
if group:
group["unsorted_files"].extend(found_files)
delete_identifiers.append(identifier)
# Remove the found files from the leftover stack
leftovers = leftovers - set(found_files)
# Break if CP wants to shut down
if self.shuttingDown():
break
# Cleaning up used
for identifier in delete_identifiers:
if path_identifiers.get(identifier):
del path_identifiers[identifier]
del delete_identifiers
# Group based on folder
delete_identifiers = []
for identifier, found_files in path_identifiers.items():
log.debug("Grouping files on foldername: %s", identifier)
for ff in found_files:
new_identifier = self.createStringIdentifier(os.path.dirname(ff), folder)
group = movie_files.get(new_identifier)
if group:
group["unsorted_files"].extend([ff])
delete_identifiers.append(identifier)
# Remove the found files from the leftover stack
leftovers -= leftovers - set([ff])
# Break if CP wants to shut down
if self.shuttingDown():
break
# leftovers should be empty
if leftovers:
log.debug("Some files are still left over: %s", leftovers)
# Cleaning up used
for identifier in delete_identifiers:
if path_identifiers.get(identifier):
del path_identifiers[identifier]
del delete_identifiers
# Make sure we remove older / still extracting files
valid_files = {}
while True and not self.shuttingDown():
try:
identifier, group = movie_files.popitem()
except:
break
# Check if movie is fresh and maybe still unpacking, ignore files newer than 1 minute
if check_file_date:
files_too_new, time_string = self.checkFilesChanged(group["unsorted_files"])
if files_too_new:
log.info(
"Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s",
(time_string, identifier),
)
# Delete the unsorted list
del group["unsorted_files"]
continue
# Only process movies newer than x
if newer_than and newer_than > 0:
has_new_files = False
for cur_file in group["unsorted_files"]:
file_time = self.getFileTimes(cur_file)
if file_time[0] > newer_than or file_time[1] > newer_than:
has_new_files = True
break
if not has_new_files:
log.debug(
"None of the files have changed since %s for %s, skipping.",
(time.ctime(newer_than), identifier),
)
# Delete the unsorted list
del group["unsorted_files"]
continue
valid_files[identifier] = group
del movie_files
total_found = len(valid_files)
# Make sure only one movie was found if a download ID is provided
if release_download and total_found == 0:
log.info(
"Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).",
release_download.get("imdb_id"),
)
elif release_download and total_found > 1:
log.info(
"Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...",
(release_download.get("imdb_id"), len(valid_files)),
)
release_download = None
# Determine file types
processed_movies = {}
while True and not self.shuttingDown():
try:
identifier, group = valid_files.popitem()
except:
break
if return_ignored is False and identifier in ignored_identifiers:
log.debug("Ignore file found, ignoring release: %s", identifier)
continue
# Group extra (and easy) files first
group["files"] = {
"movie_extra": self.getMovieExtras(group["unsorted_files"]),
"subtitle": self.getSubtitles(group["unsorted_files"]),
"subtitle_extra": self.getSubtitlesExtras(group["unsorted_files"]),
"nfo": self.getNfo(group["unsorted_files"]),
"trailer": self.getTrailers(group["unsorted_files"]),
"leftover": set(group["unsorted_files"]),
}
# Media files
if group["is_dvd"]:
group["files"]["movie"] = self.getDVDFiles(group["unsorted_files"])
else:
group["files"]["movie"] = self.getMediaFiles(group["unsorted_files"])
if len(group["files"]["movie"]) == 0:
log.error("Couldn't find any movie files for %s", identifier)
continue
log.debug("Getting metadata for %s", identifier)
group["meta_data"] = self.getMetaData(group, folder=folder, release_download=release_download)
# Subtitle meta
group["subtitle_language"] = self.getSubtitleLanguage(group) if not simple else {}
# Get parent dir from movie files
for movie_file in group["files"]["movie"]:
group["parentdir"] = os.path.dirname(movie_file)
group["dirname"] = None
folder_names = group["parentdir"].replace(folder, "").split(os.path.sep)
folder_names.reverse()
# Try and get a proper dirname, so no "A", "Movie", "Download" etc
for folder_name in folder_names:
if folder_name.lower() not in self.ignore_names and len(folder_name) > 2:
group["dirname"] = folder_name
break
break
# Leftover "sorted" files
for file_type in group["files"]:
if not file_type is "leftover":
group["files"]["leftover"] -= set(group["files"][file_type])
group["files"][file_type] = list(group["files"][file_type])
group["files"]["leftover"] = list(group["files"]["leftover"])
# Delete the unsorted list
del group["unsorted_files"]
# Determine movie
group["media"] = self.determineMedia(group, release_download=release_download)
if not group["media"]:
log.error("Unable to determine media: %s", group["identifiers"])
else:
group["identifier"] = getIdentifier(group["media"]) or group["media"]["info"].get("imdb")
processed_movies[identifier] = group
# Notify parent & progress on something found
if on_found:
on_found(group, total_found, total_found - len(processed_movies))
# Wait for all the async events calm down a bit
while threading.activeCount() > 100 and not self.shuttingDown():
log.debug("Too many threads active, waiting a few seconds")
time.sleep(10)
if len(processed_movies) > 0:
log.info("Found %s movies in the folder %s", (len(processed_movies), folder))
else:
log.debug("Found no movies in the folder %s", folder)
return processed_movies
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
Env.set('encoding', encoding)
# Do db stuff
db_path = toUnicode(os.path.join(data_dir, 'database'))
# Check if database exists
db = ThreadSafeDatabase(db_path)
db_exists = db.exists()
if db_exists:
# Backup before start and cleanup old backups
backup_path = toUnicode(os.path.join(data_dir, 'db_backup'))
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in scandir.walk(backup_path):
for backup_file in files:
ints = re.findall('\d+', backup_file)
# Delete non zip files
if len(ints) != 1:
os.remove(os.path.join(backup_path, backup_file))
else:
existing_backups.append((int(ints[0]), backup_file))
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = toUnicode(os.path.join(backup_path, '%s.tar.gz' % int(time.time())))
zipf = tarfile.open(new_backup, 'w:gz')
for root, dirs, files in scandir.walk(db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename))
zipf.close()
# Open last
db.open()
else:
db.create()
# Register environment settings
Env.set('app_dir', toUnicode(base_path))
Env.set('data_dir', toUnicode(data_dir))
Env.set('log_path', toUnicode(os.path.join(log_dir, 'CouchPotato.log')))
Env.set('db', db)
Env.set('cache_dir', toUnicode(os.path.join(data_dir, 'cache')))
Env.set('cache', FileSystemCache(toUnicode(os.path.join(Env.get('cache_dir'), 'python'))))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('daemonized', options.daemon)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default = False, type = 'bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default = False, type = 'bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Use reloader
reloader = debug is True and development and not Env.get('desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding'))
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
# noinspection PyUnresolvedReferences
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
def customwarn(message, category, filename, lineno, file = None, line = None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Create app
from couchpotato import WebHandler
web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/'
Env.set('web_base', web_base)
api_key = Env.setting('api_key')
if not api_key:
api_key = uuid4().hex
Env.setting('api_key', value = api_key)
api_base = r'%sapi/%s/' % (web_base, api_key)
Env.set('api_base', api_base)
# Basic config
host = Env.setting('host', default = '0.0.0.0')
# app.debug = development
config = {
'use_reloader': reloader,
'port': tryInt(Env.setting('port', default = 5050)),
'host': host if host and len(host) > 0 else '0.0.0.0',
'ssl_cert': Env.setting('ssl_cert', default = None),
'ssl_key': Env.setting('ssl_key', default = None),
}
# Load the app
application = Application(
[],
log_function = lambda x: None,
debug = config['use_reloader'],
gzip = True,
cookie_secret = api_key,
login_url = '%slogin/' % web_base,
)
Env.set('app', application)
# Request handlers
application.add_handlers(".*$", [
(r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler),
# API handlers
(r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler
(r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key
(r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs
# Login handlers
(r'%slogin(/?)' % web_base, LoginHandler),
(r'%slogout(/?)' % web_base, LogoutHandler),
# Catch all webhandlers
(r'%s(.*)(/?)' % web_base, WebHandler),
(r'(.*)', WebHandler),
])
# Static paths
static_path = '%sstatic/' % web_base
for dir_name in ['fonts', 'images', 'scripts', 'style']:
application.add_handlers(".*$", [
('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': toUnicode(os.path.join(base_path, 'couchpotato', 'static', dir_name))})
])
Env.set('static_path', static_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root = toUnicode(base_path))
loader.run()
# Fill database with needed stuff
fireEvent('database.setup')
if not db_exists:
fireEvent('app.initialize', in_order = True)
fireEvent('app.migrate')
# Go go go!
from tornado.ioloop import IOLoop
loop = IOLoop.current()
# Some logging and fire load event
try: log.info('Starting server on port %(port)s', config)
except: pass
fireEventAsync('app.load')
if config['ssl_cert'] and config['ssl_key']:
server = HTTPServer(application, no_keep_alive = True, ssl_options = {
'certfile': config['ssl_cert'],
'keyfile': config['ssl_key'],
})
else:
server = HTTPServer(application, no_keep_alive = True)
try_restart = True
restart_tries = 5
while try_restart:
try:
server.listen(config['port'], config['host'])
loop.start()
except Exception as e:
log.error('Failed starting: %s', traceback.format_exc())
try:
nr, msg = e
if nr == 48:
log.info('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), restart_tries))
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except:
pass
raise
try_restart = False