def processComplete(self, release_download, delete_files): log.debug('Requesting rTorrent to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) if not self.connect(): return False torrent = self.rt.find_torrent(release_download['id']) if torrent is None: return False if delete_files: for file_item in torrent.get_files(): # will only delete files, not dir/sub-dir os.unlink(os.path.join(torrent.directory, file_item.path)) if torrent.is_multi_file() and torrent.directory.endswith(torrent.name): # Remove empty directories bottom up try: for path, _, _ in scandir.walk(torrent.directory, topdown = False): os.rmdir(path) except OSError: log.info('Directory "%s" contains extra files, unable to remove', torrent.directory) torrent.erase() # just removes the torrent, doesn't delete data return True
def replaceWith(self, path): app_dir = ss(Env.get('app_dir')) data_dir = ss(Env.get('data_dir')) # Get list of files we want to overwrite self.deletePyc() existing_files = [] for root, subfiles, filenames in scandir.walk(app_dir): for filename in filenames: existing_files.append(os.path.join(root, filename)) for root, subfiles, filenames in scandir.walk(path): for filename in filenames: fromfile = os.path.join(root, filename) tofile = os.path.join(app_dir, fromfile.replace(path + os.path.sep, '')) if not Env.get('dev'): try: if os.path.isfile(tofile): os.remove(tofile) dirname = os.path.dirname(tofile) if not os.path.isdir(dirname): self.makeDir(dirname) shutil.move(fromfile, tofile) try: existing_files.remove(tofile) except ValueError: pass except: log.error('Failed overwriting file "%s": %s', (tofile, traceback.format_exc())) return False for still_exists in existing_files: if data_dir in still_exists: continue try: os.remove(still_exists) except: log.error('Failed removing non-used file: %s', traceback.format_exc()) return True
def deleteEmptyFolder(self, folder, show_error = True): folder = sp(folder) for root, dirs, files in scandir.walk(folder): for dir_name in dirs: full_path = os.path.join(root, dir_name) if len(os.listdir(full_path)) == 0: try: os.rmdir(full_path) except: if show_error: log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc())) try: os.rmdir(folder) except: if show_error: log.error('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc()))
def deletePyc(self, only_excess = True, show_logs = True): for root, dirs, files in scandir.walk(ss(Env.get('app_dir'))): pyc_files = filter(lambda filename: filename.endswith('.pyc'), files) py_files = set(filter(lambda filename: filename.endswith('.py'), files)) excess_pyc_files = filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) if only_excess else pyc_files for excess_pyc_file in excess_pyc_files: full_path = os.path.join(root, excess_pyc_file) if show_logs: log.debug('Removing old PYC file: %s', full_path) try: os.remove(full_path) except: log.error('Couldn\'t remove %s: %s', (full_path, traceback.format_exc())) for dir_name in dirs: full_path = os.path.join(root, dir_name) if len(os.listdir(full_path)) == 0: try: os.rmdir(full_path) except: log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
def scan( self, folder=None, files=None, release_download=None, simple=False, newer_than=0, return_ignored=True, on_found=None, ): folder = sp(folder) if not folder or not os.path.isdir(folder): log.error("Folder doesn't exists: %s", folder) return {} # Get movie "master" files movie_files = {} leftovers = [] # Scan all files of the folder if no files are set if not files: check_file_date = True try: files = [] for root, dirs, walk_files in scandir.walk(folder, followlinks=True): files.extend([sp(os.path.join(root, filename)) for filename in walk_files]) # Break if CP wants to shut down if self.shuttingDown(): break except: log.error("Failed getting files from %s: %s", (folder, traceback.format_exc())) log.debug("Found %s files to scan and group in %s", (len(files), folder)) else: check_file_date = False files = [sp(x) for x in files] for file_path in files: if not os.path.exists(file_path): continue # Remove ignored files if self.isSampleFile(file_path): leftovers.append(file_path) continue elif not self.keepFile(file_path): continue is_dvd_file = self.isDVDFile(file_path) if ( self.filesizeBetween(file_path, self.file_sizes["movie"]) or is_dvd_file ): # Minimal 300MB files or is DVD file # Normal identifier identifier = self.createStringIdentifier(file_path, folder, exclude_filename=is_dvd_file) identifiers = [identifier] # Identifier with quality quality = ( fireEvent("quality.guess", [file_path], single=True) if not is_dvd_file else {"identifier": "dvdr"} ) if quality: identifier_with_quality = "%s %s" % (identifier, quality.get("identifier", "")) identifiers = [identifier_with_quality, identifier] if not movie_files.get(identifier): movie_files[identifier] = {"unsorted_files": [], "identifiers": identifiers, "is_dvd": is_dvd_file} movie_files[identifier]["unsorted_files"].append(file_path) else: leftovers.append(file_path) # Break if CP wants to shut down if self.shuttingDown(): break # Cleanup del files # Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2" # files will be grouped first. leftovers = set(sorted(leftovers, reverse=True)) # Group files minus extension ignored_identifiers = [] for identifier, group in movie_files.items(): if identifier not in group["identifiers"] and len(identifier) > 0: group["identifiers"].append(identifier) log.debug("Grouping files: %s", identifier) has_ignored = 0 for file_path in list(group["unsorted_files"]): ext = getExt(file_path) wo_ext = file_path[: -(len(ext) + 1)] found_files = set([i for i in leftovers if wo_ext in i]) group["unsorted_files"].extend(found_files) leftovers = leftovers - found_files has_ignored += 1 if ext == "ignore" else 0 if has_ignored == 0: for file_path in list(group["unsorted_files"]): ext = getExt(file_path) has_ignored += 1 if ext == "ignore" else 0 if has_ignored > 0: ignored_identifiers.append(identifier) # Break if CP wants to shut down if self.shuttingDown(): break # Create identifiers for all leftover files path_identifiers = {} for file_path in leftovers: identifier = self.createStringIdentifier(file_path, folder) if not path_identifiers.get(identifier): path_identifiers[identifier] = [] path_identifiers[identifier].append(file_path) # Group the files based on the identifier delete_identifiers = [] for identifier, found_files in path_identifiers.items(): log.debug("Grouping files on identifier: %s", identifier) group = movie_files.get(identifier) if group: group["unsorted_files"].extend(found_files) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers = leftovers - set(found_files) # Break if CP wants to shut down if self.shuttingDown(): break # Cleaning up used for identifier in delete_identifiers: if path_identifiers.get(identifier): del path_identifiers[identifier] del delete_identifiers # Group based on folder delete_identifiers = [] for identifier, found_files in path_identifiers.items(): log.debug("Grouping files on foldername: %s", identifier) for ff in found_files: new_identifier = self.createStringIdentifier(os.path.dirname(ff), folder) group = movie_files.get(new_identifier) if group: group["unsorted_files"].extend([ff]) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers -= leftovers - set([ff]) # Break if CP wants to shut down if self.shuttingDown(): break # leftovers should be empty if leftovers: log.debug("Some files are still left over: %s", leftovers) # Cleaning up used for identifier in delete_identifiers: if path_identifiers.get(identifier): del path_identifiers[identifier] del delete_identifiers # Make sure we remove older / still extracting files valid_files = {} while True and not self.shuttingDown(): try: identifier, group = movie_files.popitem() except: break # Check if movie is fresh and maybe still unpacking, ignore files newer than 1 minute if check_file_date: files_too_new, time_string = self.checkFilesChanged(group["unsorted_files"]) if files_too_new: log.info( "Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s", (time_string, identifier), ) # Delete the unsorted list del group["unsorted_files"] continue # Only process movies newer than x if newer_than and newer_than > 0: has_new_files = False for cur_file in group["unsorted_files"]: file_time = self.getFileTimes(cur_file) if file_time[0] > newer_than or file_time[1] > newer_than: has_new_files = True break if not has_new_files: log.debug( "None of the files have changed since %s for %s, skipping.", (time.ctime(newer_than), identifier), ) # Delete the unsorted list del group["unsorted_files"] continue valid_files[identifier] = group del movie_files total_found = len(valid_files) # Make sure only one movie was found if a download ID is provided if release_download and total_found == 0: log.info( "Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).", release_download.get("imdb_id"), ) elif release_download and total_found > 1: log.info( "Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...", (release_download.get("imdb_id"), len(valid_files)), ) release_download = None # Determine file types processed_movies = {} while True and not self.shuttingDown(): try: identifier, group = valid_files.popitem() except: break if return_ignored is False and identifier in ignored_identifiers: log.debug("Ignore file found, ignoring release: %s", identifier) continue # Group extra (and easy) files first group["files"] = { "movie_extra": self.getMovieExtras(group["unsorted_files"]), "subtitle": self.getSubtitles(group["unsorted_files"]), "subtitle_extra": self.getSubtitlesExtras(group["unsorted_files"]), "nfo": self.getNfo(group["unsorted_files"]), "trailer": self.getTrailers(group["unsorted_files"]), "leftover": set(group["unsorted_files"]), } # Media files if group["is_dvd"]: group["files"]["movie"] = self.getDVDFiles(group["unsorted_files"]) else: group["files"]["movie"] = self.getMediaFiles(group["unsorted_files"]) if len(group["files"]["movie"]) == 0: log.error("Couldn't find any movie files for %s", identifier) continue log.debug("Getting metadata for %s", identifier) group["meta_data"] = self.getMetaData(group, folder=folder, release_download=release_download) # Subtitle meta group["subtitle_language"] = self.getSubtitleLanguage(group) if not simple else {} # Get parent dir from movie files for movie_file in group["files"]["movie"]: group["parentdir"] = os.path.dirname(movie_file) group["dirname"] = None folder_names = group["parentdir"].replace(folder, "").split(os.path.sep) folder_names.reverse() # Try and get a proper dirname, so no "A", "Movie", "Download" etc for folder_name in folder_names: if folder_name.lower() not in self.ignore_names and len(folder_name) > 2: group["dirname"] = folder_name break break # Leftover "sorted" files for file_type in group["files"]: if not file_type is "leftover": group["files"]["leftover"] -= set(group["files"][file_type]) group["files"][file_type] = list(group["files"][file_type]) group["files"]["leftover"] = list(group["files"]["leftover"]) # Delete the unsorted list del group["unsorted_files"] # Determine movie group["media"] = self.determineMedia(group, release_download=release_download) if not group["media"]: log.error("Unable to determine media: %s", group["identifiers"]) else: group["identifier"] = getIdentifier(group["media"]) or group["media"]["info"].get("imdb") processed_movies[identifier] = group # Notify parent & progress on something found if on_found: on_found(group, total_found, total_found - len(processed_movies)) # Wait for all the async events calm down a bit while threading.activeCount() > 100 and not self.shuttingDown(): log.debug("Too many threads active, waiting a few seconds") time.sleep(10) if len(processed_movies) > 0: log.info("Found %s movies in the folder %s", (len(processed_movies), folder)) else: log.debug("Found no movies in the folder %s", folder) return processed_movies
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None): try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): encoding = None # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): encoding = 'UTF-8' Env.set('encoding', encoding) # Do db stuff db_path = toUnicode(os.path.join(data_dir, 'database')) # Check if database exists db = ThreadSafeDatabase(db_path) db_exists = db.exists() if db_exists: # Backup before start and cleanup old backups backup_path = toUnicode(os.path.join(data_dir, 'db_backup')) backup_count = 5 existing_backups = [] if not os.path.isdir(backup_path): os.makedirs(backup_path) for root, dirs, files in scandir.walk(backup_path): for backup_file in files: ints = re.findall('\d+', backup_file) # Delete non zip files if len(ints) != 1: os.remove(os.path.join(backup_path, backup_file)) else: existing_backups.append((int(ints[0]), backup_file)) # Remove all but the last 5 for eb in existing_backups[:-backup_count]: os.remove(os.path.join(backup_path, eb[1])) # Create new backup new_backup = toUnicode(os.path.join(backup_path, '%s.tar.gz' % int(time.time()))) zipf = tarfile.open(new_backup, 'w:gz') for root, dirs, files in scandir.walk(db_path): for zfilename in files: zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename)) zipf.close() # Open last db.open() else: db.create() # Register environment settings Env.set('app_dir', toUnicode(base_path)) Env.set('data_dir', toUnicode(data_dir)) Env.set('log_path', toUnicode(os.path.join(log_dir, 'CouchPotato.log'))) Env.set('db', db) Env.set('cache_dir', toUnicode(os.path.join(data_dir, 'cache'))) Env.set('cache', FileSystemCache(toUnicode(os.path.join(Env.get('cache_dir'), 'python')))) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) Env.set('daemonized', options.daemon) Env.set('args', args) Env.set('options', options) # Determine debug debug = options.debug or Env.setting('debug', default = False, type = 'bool') Env.set('debug', debug) # Development development = Env.setting('development', default = False, type = 'bool') Env.set('dev', development) # Disable logging for some modules for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ['gntp']: logging.getLogger(logger_name).setLevel(logging.WARNING) # Use reloader reloader = debug is True and development and not Env.get('desktop') and not options.daemon # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) logging.addLevelName(19, 'INFO') # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding')) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors # noinspection PyUnresolvedReferences import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s', options) def customwarn(message, category, filename, lineno, file = None, line = None): log.warning('%s %s %s line:%s', (category, message, filename, lineno)) warnings.showwarning = customwarn # Create app from couchpotato import WebHandler web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/' Env.set('web_base', web_base) api_key = Env.setting('api_key') if not api_key: api_key = uuid4().hex Env.setting('api_key', value = api_key) api_base = r'%sapi/%s/' % (web_base, api_key) Env.set('api_base', api_base) # Basic config host = Env.setting('host', default = '0.0.0.0') # app.debug = development config = { 'use_reloader': reloader, 'port': tryInt(Env.setting('port', default = 5050)), 'host': host if host and len(host) > 0 else '0.0.0.0', 'ssl_cert': Env.setting('ssl_cert', default = None), 'ssl_key': Env.setting('ssl_key', default = None), } # Load the app application = Application( [], log_function = lambda x: None, debug = config['use_reloader'], gzip = True, cookie_secret = api_key, login_url = '%slogin/' % web_base, ) Env.set('app', application) # Request handlers application.add_handlers(".*$", [ (r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler), # API handlers (r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler (r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key (r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs # Login handlers (r'%slogin(/?)' % web_base, LoginHandler), (r'%slogout(/?)' % web_base, LogoutHandler), # Catch all webhandlers (r'%s(.*)(/?)' % web_base, WebHandler), (r'(.*)', WebHandler), ]) # Static paths static_path = '%sstatic/' % web_base for dir_name in ['fonts', 'images', 'scripts', 'style']: application.add_handlers(".*$", [ ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': toUnicode(os.path.join(base_path, 'couchpotato', 'static', dir_name))}) ]) Env.set('static_path', static_path) # Load configs & plugins loader = Env.get('loader') loader.preload(root = toUnicode(base_path)) loader.run() # Fill database with needed stuff fireEvent('database.setup') if not db_exists: fireEvent('app.initialize', in_order = True) fireEvent('app.migrate') # Go go go! from tornado.ioloop import IOLoop loop = IOLoop.current() # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass fireEventAsync('app.load') if config['ssl_cert'] and config['ssl_key']: server = HTTPServer(application, no_keep_alive = True, ssl_options = { 'certfile': config['ssl_cert'], 'keyfile': config['ssl_key'], }) else: server = HTTPServer(application, no_keep_alive = True) try_restart = True restart_tries = 5 while try_restart: try: server.listen(config['port'], config['host']) loop.start() except Exception as e: log.error('Failed starting: %s', traceback.format_exc()) try: nr, msg = e if nr == 48: log.info('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), restart_tries)) time.sleep(1) restart_tries -= 1 if restart_tries > 0: continue else: return except: pass raise try_restart = False