Exemplo n.º 1
0
    def processComplete(self, release_download, delete_files):
        log.debug('Requesting rTorrent to remove the torrent %s%s.',
                  (release_download['name'], ' and cleanup the downloaded files' if delete_files else ''))

        if not self.connect():
            return False

        torrent = self.rt.find_torrent(release_download['id'])

        if torrent is None:
            return False

        if delete_files:
            for file_item in torrent.get_files(): # will only delete files, not dir/sub-dir
                os.unlink(os.path.join(torrent.directory, file_item.path))

            if torrent.is_multi_file() and torrent.directory.endswith(torrent.name):
                # Remove empty directories bottom up
                try:
                    for path, _, _ in scandir.walk(torrent.directory, topdown = False):
                        os.rmdir(path)
                except OSError:
                    log.info('Directory "%s" contains extra files, unable to remove', torrent.directory)

        torrent.erase() # just removes the torrent, doesn't delete data

        return True
Exemplo n.º 2
0
    def replaceWith(self, path):
        app_dir = ss(Env.get('app_dir'))
        data_dir = ss(Env.get('data_dir'))

        # Get list of files we want to overwrite
        self.deletePyc()
        existing_files = []
        for root, subfiles, filenames in scandir.walk(app_dir):
            for filename in filenames:
                existing_files.append(os.path.join(root, filename))

        for root, subfiles, filenames in scandir.walk(path):
            for filename in filenames:
                fromfile = os.path.join(root, filename)
                tofile = os.path.join(app_dir, fromfile.replace(path + os.path.sep, ''))

                if not Env.get('dev'):
                    try:
                        if os.path.isfile(tofile):
                            os.remove(tofile)

                        dirname = os.path.dirname(tofile)
                        if not os.path.isdir(dirname):
                            self.makeDir(dirname)

                        shutil.move(fromfile, tofile)
                        try:
                            existing_files.remove(tofile)
                        except ValueError:
                            pass
                    except:
                        log.error('Failed overwriting file "%s": %s', (tofile, traceback.format_exc()))
                        return False

        for still_exists in existing_files:

            if data_dir in still_exists:
                continue

            try:
                os.remove(still_exists)
            except:
                log.error('Failed removing non-used file: %s', traceback.format_exc())

        return True
Exemplo n.º 3
0
    def deleteEmptyFolder(self, folder, show_error = True):
        folder = sp(folder)

        for root, dirs, files in scandir.walk(folder):

            for dir_name in dirs:
                full_path = os.path.join(root, dir_name)
                if len(os.listdir(full_path)) == 0:
                    try:
                        os.rmdir(full_path)
                    except:
                        if show_error:
                            log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))

        try:
            os.rmdir(folder)
        except:
            if show_error:
                log.error('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc()))
Exemplo n.º 4
0
    def deletePyc(self, only_excess = True, show_logs = True):

        for root, dirs, files in scandir.walk(ss(Env.get('app_dir'))):

            pyc_files = filter(lambda filename: filename.endswith('.pyc'), files)
            py_files = set(filter(lambda filename: filename.endswith('.py'), files))
            excess_pyc_files = filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) if only_excess else pyc_files

            for excess_pyc_file in excess_pyc_files:
                full_path = os.path.join(root, excess_pyc_file)
                if show_logs: log.debug('Removing old PYC file: %s', full_path)
                try:
                    os.remove(full_path)
                except:
                    log.error('Couldn\'t remove %s: %s', (full_path, traceback.format_exc()))

            for dir_name in dirs:
                full_path = os.path.join(root, dir_name)
                if len(os.listdir(full_path)) == 0:
                    try:
                        os.rmdir(full_path)
                    except:
                        log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
Exemplo n.º 5
0
    def scan(
        self,
        folder=None,
        files=None,
        release_download=None,
        simple=False,
        newer_than=0,
        return_ignored=True,
        on_found=None,
    ):

        folder = sp(folder)

        if not folder or not os.path.isdir(folder):
            log.error("Folder doesn't exists: %s", folder)
            return {}

        # Get movie "master" files
        movie_files = {}
        leftovers = []

        # Scan all files of the folder if no files are set
        if not files:
            check_file_date = True
            try:
                files = []
                for root, dirs, walk_files in scandir.walk(folder, followlinks=True):
                    files.extend([sp(os.path.join(root, filename)) for filename in walk_files])

                    # Break if CP wants to shut down
                    if self.shuttingDown():
                        break

            except:
                log.error("Failed getting files from %s: %s", (folder, traceback.format_exc()))

            log.debug("Found %s files to scan and group in %s", (len(files), folder))
        else:
            check_file_date = False
            files = [sp(x) for x in files]

        for file_path in files:

            if not os.path.exists(file_path):
                continue

            # Remove ignored files
            if self.isSampleFile(file_path):
                leftovers.append(file_path)
                continue
            elif not self.keepFile(file_path):
                continue

            is_dvd_file = self.isDVDFile(file_path)
            if (
                self.filesizeBetween(file_path, self.file_sizes["movie"]) or is_dvd_file
            ):  # Minimal 300MB files or is DVD file

                # Normal identifier
                identifier = self.createStringIdentifier(file_path, folder, exclude_filename=is_dvd_file)
                identifiers = [identifier]

                # Identifier with quality
                quality = (
                    fireEvent("quality.guess", [file_path], single=True) if not is_dvd_file else {"identifier": "dvdr"}
                )
                if quality:
                    identifier_with_quality = "%s %s" % (identifier, quality.get("identifier", ""))
                    identifiers = [identifier_with_quality, identifier]

                if not movie_files.get(identifier):
                    movie_files[identifier] = {"unsorted_files": [], "identifiers": identifiers, "is_dvd": is_dvd_file}

                movie_files[identifier]["unsorted_files"].append(file_path)
            else:
                leftovers.append(file_path)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Cleanup
        del files

        # Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2"
        # files will be grouped first.
        leftovers = set(sorted(leftovers, reverse=True))

        # Group files minus extension
        ignored_identifiers = []
        for identifier, group in movie_files.items():
            if identifier not in group["identifiers"] and len(identifier) > 0:
                group["identifiers"].append(identifier)

            log.debug("Grouping files: %s", identifier)

            has_ignored = 0
            for file_path in list(group["unsorted_files"]):
                ext = getExt(file_path)
                wo_ext = file_path[: -(len(ext) + 1)]
                found_files = set([i for i in leftovers if wo_ext in i])
                group["unsorted_files"].extend(found_files)
                leftovers = leftovers - found_files

                has_ignored += 1 if ext == "ignore" else 0

            if has_ignored == 0:
                for file_path in list(group["unsorted_files"]):
                    ext = getExt(file_path)
                    has_ignored += 1 if ext == "ignore" else 0

            if has_ignored > 0:
                ignored_identifiers.append(identifier)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Create identifiers for all leftover files
        path_identifiers = {}
        for file_path in leftovers:
            identifier = self.createStringIdentifier(file_path, folder)

            if not path_identifiers.get(identifier):
                path_identifiers[identifier] = []

            path_identifiers[identifier].append(file_path)

        # Group the files based on the identifier
        delete_identifiers = []
        for identifier, found_files in path_identifiers.items():
            log.debug("Grouping files on identifier: %s", identifier)

            group = movie_files.get(identifier)
            if group:
                group["unsorted_files"].extend(found_files)
                delete_identifiers.append(identifier)

                # Remove the found files from the leftover stack
                leftovers = leftovers - set(found_files)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Cleaning up used
        for identifier in delete_identifiers:
            if path_identifiers.get(identifier):
                del path_identifiers[identifier]
        del delete_identifiers

        # Group based on folder
        delete_identifiers = []
        for identifier, found_files in path_identifiers.items():
            log.debug("Grouping files on foldername: %s", identifier)

            for ff in found_files:
                new_identifier = self.createStringIdentifier(os.path.dirname(ff), folder)

                group = movie_files.get(new_identifier)
                if group:
                    group["unsorted_files"].extend([ff])
                    delete_identifiers.append(identifier)

                    # Remove the found files from the leftover stack
                    leftovers -= leftovers - set([ff])

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # leftovers should be empty
        if leftovers:
            log.debug("Some files are still left over: %s", leftovers)

        # Cleaning up used
        for identifier in delete_identifiers:
            if path_identifiers.get(identifier):
                del path_identifiers[identifier]
        del delete_identifiers

        # Make sure we remove older / still extracting files
        valid_files = {}
        while True and not self.shuttingDown():
            try:
                identifier, group = movie_files.popitem()
            except:
                break

            # Check if movie is fresh and maybe still unpacking, ignore files newer than 1 minute
            if check_file_date:
                files_too_new, time_string = self.checkFilesChanged(group["unsorted_files"])
                if files_too_new:
                    log.info(
                        "Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s",
                        (time_string, identifier),
                    )

                    # Delete the unsorted list
                    del group["unsorted_files"]

                    continue

            # Only process movies newer than x
            if newer_than and newer_than > 0:
                has_new_files = False
                for cur_file in group["unsorted_files"]:
                    file_time = self.getFileTimes(cur_file)
                    if file_time[0] > newer_than or file_time[1] > newer_than:
                        has_new_files = True
                        break

                if not has_new_files:
                    log.debug(
                        "None of the files have changed since %s for %s, skipping.",
                        (time.ctime(newer_than), identifier),
                    )

                    # Delete the unsorted list
                    del group["unsorted_files"]

                    continue

            valid_files[identifier] = group

        del movie_files

        total_found = len(valid_files)

        # Make sure only one movie was found if a download ID is provided
        if release_download and total_found == 0:
            log.info(
                "Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).",
                release_download.get("imdb_id"),
            )
        elif release_download and total_found > 1:
            log.info(
                "Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...",
                (release_download.get("imdb_id"), len(valid_files)),
            )
            release_download = None

        # Determine file types
        processed_movies = {}
        while True and not self.shuttingDown():
            try:
                identifier, group = valid_files.popitem()
            except:
                break

            if return_ignored is False and identifier in ignored_identifiers:
                log.debug("Ignore file found, ignoring release: %s", identifier)
                continue

            # Group extra (and easy) files first
            group["files"] = {
                "movie_extra": self.getMovieExtras(group["unsorted_files"]),
                "subtitle": self.getSubtitles(group["unsorted_files"]),
                "subtitle_extra": self.getSubtitlesExtras(group["unsorted_files"]),
                "nfo": self.getNfo(group["unsorted_files"]),
                "trailer": self.getTrailers(group["unsorted_files"]),
                "leftover": set(group["unsorted_files"]),
            }

            # Media files
            if group["is_dvd"]:
                group["files"]["movie"] = self.getDVDFiles(group["unsorted_files"])
            else:
                group["files"]["movie"] = self.getMediaFiles(group["unsorted_files"])

            if len(group["files"]["movie"]) == 0:
                log.error("Couldn't find any movie files for %s", identifier)
                continue

            log.debug("Getting metadata for %s", identifier)
            group["meta_data"] = self.getMetaData(group, folder=folder, release_download=release_download)

            # Subtitle meta
            group["subtitle_language"] = self.getSubtitleLanguage(group) if not simple else {}

            # Get parent dir from movie files
            for movie_file in group["files"]["movie"]:
                group["parentdir"] = os.path.dirname(movie_file)
                group["dirname"] = None

                folder_names = group["parentdir"].replace(folder, "").split(os.path.sep)
                folder_names.reverse()

                # Try and get a proper dirname, so no "A", "Movie", "Download" etc
                for folder_name in folder_names:
                    if folder_name.lower() not in self.ignore_names and len(folder_name) > 2:
                        group["dirname"] = folder_name
                        break

                break

            # Leftover "sorted" files
            for file_type in group["files"]:
                if not file_type is "leftover":
                    group["files"]["leftover"] -= set(group["files"][file_type])
                    group["files"][file_type] = list(group["files"][file_type])
            group["files"]["leftover"] = list(group["files"]["leftover"])

            # Delete the unsorted list
            del group["unsorted_files"]

            # Determine movie
            group["media"] = self.determineMedia(group, release_download=release_download)
            if not group["media"]:
                log.error("Unable to determine media: %s", group["identifiers"])
            else:
                group["identifier"] = getIdentifier(group["media"]) or group["media"]["info"].get("imdb")

            processed_movies[identifier] = group

            # Notify parent & progress on something found
            if on_found:
                on_found(group, total_found, total_found - len(processed_movies))

            # Wait for all the async events calm down a bit
            while threading.activeCount() > 100 and not self.shuttingDown():
                log.debug("Too many threads active, waiting a few seconds")
                time.sleep(10)

        if len(processed_movies) > 0:
            log.info("Found %s movies in the folder %s", (len(processed_movies), folder))
        else:
            log.debug("Found no movies in the folder %s", folder)

        return processed_movies
Exemplo n.º 6
0
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None):

    try:
        locale.setlocale(locale.LC_ALL, "")
        encoding = locale.getpreferredencoding()
    except (locale.Error, IOError):
        encoding = None

    # for OSes that are poorly configured I'll just force UTF-8
    if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
        encoding = 'UTF-8'

    Env.set('encoding', encoding)

    # Do db stuff
    db_path = toUnicode(os.path.join(data_dir, 'database'))

    # Check if database exists
    db = ThreadSafeDatabase(db_path)
    db_exists = db.exists()
    if db_exists:

        # Backup before start and cleanup old backups
        backup_path = toUnicode(os.path.join(data_dir, 'db_backup'))
        backup_count = 5
        existing_backups = []
        if not os.path.isdir(backup_path): os.makedirs(backup_path)

        for root, dirs, files in scandir.walk(backup_path):
            for backup_file in files:
                ints = re.findall('\d+', backup_file)

                # Delete non zip files
                if len(ints) != 1:
                    os.remove(os.path.join(backup_path, backup_file))
                else:
                    existing_backups.append((int(ints[0]), backup_file))

        # Remove all but the last 5
        for eb in existing_backups[:-backup_count]:
            os.remove(os.path.join(backup_path, eb[1]))

        # Create new backup
        new_backup = toUnicode(os.path.join(backup_path, '%s.tar.gz' % int(time.time())))
        zipf = tarfile.open(new_backup, 'w:gz')
        for root, dirs, files in scandir.walk(db_path):
            for zfilename in files:
                zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename))
        zipf.close()

        # Open last
        db.open()

    else:
        db.create()

    # Register environment settings
    Env.set('app_dir', toUnicode(base_path))
    Env.set('data_dir', toUnicode(data_dir))
    Env.set('log_path', toUnicode(os.path.join(log_dir, 'CouchPotato.log')))
    Env.set('db', db)
    Env.set('cache_dir', toUnicode(os.path.join(data_dir, 'cache')))
    Env.set('cache', FileSystemCache(toUnicode(os.path.join(Env.get('cache_dir'), 'python'))))
    Env.set('console_log', options.console_log)
    Env.set('quiet', options.quiet)
    Env.set('desktop', desktop)
    Env.set('daemonized', options.daemon)
    Env.set('args', args)
    Env.set('options', options)

    # Determine debug
    debug = options.debug or Env.setting('debug', default = False, type = 'bool')
    Env.set('debug', debug)

    # Development
    development = Env.setting('development', default = False, type = 'bool')
    Env.set('dev', development)

    # Disable logging for some modules
    for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']:
        logging.getLogger(logger_name).setLevel(logging.ERROR)

    for logger_name in ['gntp']:
        logging.getLogger(logger_name).setLevel(logging.WARNING)

    # Use reloader
    reloader = debug is True and development and not Env.get('desktop') and not options.daemon

    # Logger
    logger = logging.getLogger()
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S')
    level = logging.DEBUG if debug else logging.INFO
    logger.setLevel(level)
    logging.addLevelName(19, 'INFO')

    # To screen
    if (debug or options.console_log) and not options.quiet and not options.daemon:
        hdlr = logging.StreamHandler(sys.stderr)
        hdlr.setFormatter(formatter)
        logger.addHandler(hdlr)

    # To file
    hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding'))
    hdlr2.setFormatter(formatter)
    logger.addHandler(hdlr2)

    # Start logging & enable colors
    # noinspection PyUnresolvedReferences
    import color_logs
    from couchpotato.core.logger import CPLog
    log = CPLog(__name__)
    log.debug('Started with options %s', options)

    def customwarn(message, category, filename, lineno, file = None, line = None):
        log.warning('%s %s %s line:%s', (category, message, filename, lineno))
    warnings.showwarning = customwarn

    # Create app
    from couchpotato import WebHandler
    web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/'
    Env.set('web_base', web_base)

    api_key = Env.setting('api_key')
    if not api_key:
        api_key = uuid4().hex
        Env.setting('api_key', value = api_key)

    api_base = r'%sapi/%s/' % (web_base, api_key)
    Env.set('api_base', api_base)

    # Basic config
    host = Env.setting('host', default = '0.0.0.0')
    # app.debug = development
    config = {
        'use_reloader': reloader,
        'port': tryInt(Env.setting('port', default = 5050)),
        'host': host if host and len(host) > 0 else '0.0.0.0',
        'ssl_cert': Env.setting('ssl_cert', default = None),
        'ssl_key': Env.setting('ssl_key', default = None),
    }

    # Load the app
    application = Application(
        [],
        log_function = lambda x: None,
        debug = config['use_reloader'],
        gzip = True,
        cookie_secret = api_key,
        login_url = '%slogin/' % web_base,
    )
    Env.set('app', application)

    # Request handlers
    application.add_handlers(".*$", [
        (r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler),

        # API handlers
        (r'%s(.*)(/?)' % api_base, ApiHandler),  # Main API handler
        (r'%sgetkey(/?)' % web_base, KeyHandler),  # Get API key
        (r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}),  # API docs

        # Login handlers
        (r'%slogin(/?)' % web_base, LoginHandler),
        (r'%slogout(/?)' % web_base, LogoutHandler),

        # Catch all webhandlers
        (r'%s(.*)(/?)' % web_base, WebHandler),
        (r'(.*)', WebHandler),
    ])

    # Static paths
    static_path = '%sstatic/' % web_base
    for dir_name in ['fonts', 'images', 'scripts', 'style']:
        application.add_handlers(".*$", [
            ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': toUnicode(os.path.join(base_path, 'couchpotato', 'static', dir_name))})
        ])
    Env.set('static_path', static_path)

    # Load configs & plugins
    loader = Env.get('loader')
    loader.preload(root = toUnicode(base_path))
    loader.run()

    # Fill database with needed stuff
    fireEvent('database.setup')
    if not db_exists:
        fireEvent('app.initialize', in_order = True)
    fireEvent('app.migrate')

    # Go go go!
    from tornado.ioloop import IOLoop
    loop = IOLoop.current()

    # Some logging and fire load event
    try: log.info('Starting server on port %(port)s', config)
    except: pass
    fireEventAsync('app.load')

    if config['ssl_cert'] and config['ssl_key']:
        server = HTTPServer(application, no_keep_alive = True, ssl_options = {
            'certfile': config['ssl_cert'],
            'keyfile': config['ssl_key'],
        })
    else:
        server = HTTPServer(application, no_keep_alive = True)

    try_restart = True
    restart_tries = 5

    while try_restart:
        try:
            server.listen(config['port'], config['host'])
            loop.start()
        except Exception as e:
            log.error('Failed starting: %s', traceback.format_exc())
            try:
                nr, msg = e
                if nr == 48:
                    log.info('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), restart_tries))
                    time.sleep(1)
                    restart_tries -= 1

                    if restart_tries > 0:
                        continue
                    else:
                        return
            except:
                pass

            raise

        try_restart = False