Example #1
0
    def guess(self, files, extra = None, size = None):
        if not extra: extra = {}

        # Create hash for cache
        cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files])
        cached = self.getCache(cache_key)
        if cached and len(extra) == 0:
            return cached

        qualities = self.all()

        # Start with 0
        score = {}
        for quality in qualities:
            score[quality.get('identifier')] = {
                'score': 0,
                '3d': {}
            }

        for cur_file in files:
            words = re.split('\W+', cur_file.lower())

            for quality in qualities:
                contains_score = self.containsTagScore(quality, words, cur_file)
                threedscore = self.contains3D(quality, words, cur_file) if quality.get('allow_3d') else (0, None)

                self.calcScore(score, quality, contains_score, threedscore)

        # Evaluate score based on size
        for quality in qualities:
            size_score = self.guessSizeScore(quality, size = size)
            self.calcScore(score, quality, size_score, penalty = False)

        # Try again with loose testing
        for quality in qualities:
            loose_score = self.guessLooseScore(quality, extra = extra)
            self.calcScore(score, quality, loose_score, penalty = False)

        # Return nothing if all scores are <= 0
        has_non_zero = 0
        for s in score:
            if score[s]['score'] > 0:
                has_non_zero += 1

        if not has_non_zero:
            return None

        heighest_quality = max(score, key = lambda p: score[p]['score'])
        if heighest_quality:
            for quality in qualities:
                if quality.get('identifier') == heighest_quality:
                    quality['is_3d'] = False
                    if score[heighest_quality].get('3d'):
                        quality['is_3d'] = True
                    return self.setCache(cache_key, quality)

        return None
    def guess(self, files, extra = None):
        if not extra: extra = {}

        # Create hash for cache
#        cache_key = md5(str([f.replace('.' + getExt(f), '') for f in files]))
#        cached = self.getCache(cache_key)
#        if cached and len(extra) == 0: return cached
        # Create hash for cache
        cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files])
        cached = self.getCache(cache_key)
        if cached and len(extra) == 0:
            return cached
        qualities = self.all()

        # Start with 0
        score = {}
        for quality in qualities:
            score[quality.get('identifier')] = 0

        for cur_file in files:
            words = re.split('[\W\-]+', cur_file.lower(), flags=re.U)

            for quality in qualities:
                contains_score = self.containsTagScore(quality, words, cur_file)
                self.calcScore(score, quality, contains_score)

        # Try again with loose testing
        for quality in qualities:
            loose_score = self.guessLooseScore(quality, files = files, extra = extra)
            self.calcScore(score, quality, loose_score)


        # Return nothing if all scores are 0
        has_non_zero = 0
        for s in score:
            if score[s] > 0:
                has_non_zero += 1

        if not has_non_zero:
            return None

        heighest_quality = max(score, key = score.get)
        if heighest_quality:
            for quality in qualities:
                if quality.get('identifier') == heighest_quality:
                    return self.setCache(cache_key, quality)

        return None
Example #3
0
    def searchSingle(self, message = None, group = None):
        if not group: group = {}
        if self.isDisabled() or len(group['files']['trailer']) > 0: return

        trailers = fireEvent('trailer.search', group = group, merge = True)
        if not trailers or trailers == []:
            log.info('No trailers found for: %s', getTitle(group))
            return False

        for trailer in trailers.get(self.conf('quality'), []):

            ext = getExt(trailer)
            filename = self.conf('name').replace('<filename>', group['filename']) + ('.%s' % ('mp4' if len(ext) > 5 else ext))
            destination = os.path.join(group['destination_dir'], filename)
            if not os.path.isfile(destination):
                trailer_file = fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True)
                if os.path.getsize(trailer_file) < (1024 * 1024):  # Don't trust small trailers (1MB), try next one
                    os.unlink(trailer_file)
                    continue
            else:
                log.debug('Trailer already exists: %s', destination)

            group['renamed_files'].append(destination)

            # Download first and break
            break

        return True
Example #4
0
    def registerStatic(self, plugin_file, add_to_head = True):

        # Register plugin path
        self.plugin_path = os.path.dirname(plugin_file)
        static_folder = toUnicode(os.path.join(self.plugin_path, 'static'))

        if not os.path.isdir(static_folder):
            return

        # Get plugin_name from PluginName
        s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__)
        class_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

        # View path
        path = 'static/plugin/%s/' % class_name

        # Add handler to Tornado
        Env.get('app').add_handlers(".*$", [(Env.get('web_base') + path + '(.*)', StaticFileHandler, {'path': static_folder})])

        # Register for HTML <HEAD>
        if add_to_head:
            for f in glob.glob(os.path.join(self.plugin_path, 'static', '*')):
                ext = getExt(f)
                if ext in ['js', 'css']:
                    fireEvent('register_%s' % ('script' if ext in 'js' else 'style'), path + os.path.basename(f), f)
Example #5
0
    def registerStatic(self, plugin_file, add_to_head=True):

        # Register plugin path
        self.plugin_path = os.path.dirname(plugin_file)
        static_folder = toUnicode(os.path.join(self.plugin_path, "static"))

        if not os.path.isdir(static_folder):
            return

        # Get plugin_name from PluginName
        s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", self.__class__.__name__)
        class_name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()

        # View path
        path = "static/plugin/%s/" % class_name

        # Add handler to Tornado
        Env.get("app").add_handlers(
            ".*$", [(Env.get("web_base") + path + "(.*)", StaticFileHandler, {"path": static_folder})]
        )

        # Register for HTML <HEAD>
        if add_to_head:
            for f in glob.glob(os.path.join(self.plugin_path, "static", "*")):
                ext = getExt(f)
                if ext in ["js", "css"]:
                    fireEvent("register_%s" % ("script" if ext in "js" else "style"), path + os.path.basename(f), f)
Example #6
0
    def guess(self, files, extra = {}, loose = False):

        # Create hash for cache
        hash = md5(str([f.replace('.' + getExt(f), '') for f in files]))
        cached = self.getCache(hash)
        if cached and extra is {}: return cached

        for cur_file in files:
            size = (os.path.getsize(cur_file) / 1024 / 1024) if os.path.isfile(cur_file) else 0
            words = re.split('\W+', cur_file.lower())

            for quality in self.all():

                # Check tags
                if quality['identifier'] in words:
                    log.debug('Found via identifier "%s" in %s' % (quality['identifier'], cur_file))
                    return self.setCache(hash, quality)

                if list(set(quality.get('alternative', [])) & set(words)):
                    log.debug('Found %s via alt %s in %s' % (quality['identifier'], quality.get('alternative'), cur_file))
                    return self.setCache(hash, quality)

                for tag in quality.get('tags', []):
                    if isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words):
                        log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), cur_file))
                        return self.setCache(hash, quality)

                if list(set(quality.get('tags', [])) & set(words)):
                    log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), cur_file))
                    return self.setCache(hash, quality)

                # Check on unreliable stuff
                if loose:

                    # Last check on resolution only
                    if quality.get('width', 480) == extra.get('resolution_width', 0):
                        log.debug('Found %s via resolution_width: %s == %s' % (quality['identifier'], quality.get('width', 480), extra.get('resolution_width', 0)))
                        return self.setCache(hash, quality)

                    # Check extension + filesize
                    if list(set(quality.get('ext', [])) & set(words)) and size >= quality['size_min'] and size <= quality['size_max']:
                        log.debug('Found %s via ext and filesize %s in %s' % (quality['identifier'], quality.get('ext'), words))
                        return self.setCache(hash, quality)


        # Try again with loose testing
        if not loose:
            quality = self.guess(files, extra = extra, loose = True)
            if quality:
                return self.setCache(hash, quality)

        log.debug('Could not identify quality for: %s' % files)
        return None
Example #7
0
    def download(self, url = '', dest = None, overwrite = False):

        try:
            filedata = self.urlopen(url)
        except:
            return False

        if not dest: # to Cache
            dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url)))

        if overwrite or not os.path.isfile(dest):
            self.createFile(dest, filedata, binary = True)

        return dest
Example #8
0
    def download(self, url="", dest=None, overwrite=False):

        if not dest:  # to Cache
            dest = os.path.join(Env.get("cache_dir"), "%s.%s" % (md5(url), getExt(url)))

        if not overwrite and os.path.isfile(dest):
            return dest

        try:
            filedata = self.urlopen(url)
        except:
            return False

        self.createFile(dest, filedata, binary=True)
        return dest
Example #9
0
    def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = {}):

        if not dest: # to Cache
            dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url)))

        if not overwrite and os.path.isfile(dest):
            return dest

        try:
            filedata = self.urlopen(url, **urlopen_kwargs)
        except:
            return False

        self.createFile(dest, filedata, binary = True)
        return dest
Example #10
0
    def registerStatic(self, plugin_file, add_to_head = True):

        # Register plugin path
        self.plugin_path = os.path.dirname(plugin_file)

        # Get plugin_name from PluginName
        s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__)
        class_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

        path = 'static/' + class_name + '/'
        addView(path + '<path:file>', self.showStatic, static = True)

        if add_to_head:
            for f in glob.glob(os.path.join(self.plugin_path, 'static', '*')):
                fireEvent('register_%s' % ('script' if getExt(f) in 'js' else 'style'), path + os.path.basename(f))
Example #11
0
    def searchSingle(self, group):

        if self.isDisabled() or len(group['files']['trailer']) > 0: return

        trailers = fireEvent('trailer.search', group = group, merge = True)

        for trailer in trailers.get(self.conf('quality'), []):
            destination = '%s-trailer.%s' % (self.getRootName(group), getExt(trailer))
            if not os.path.isfile(destination):
                fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True)
            else:
                log.debug('Trailer already exists: %s' % destination)

            # Download first and break
            break
Example #12
0
    def getRenameExtras(self, extra_type = '', replacements = {}, folder_name = '', file_name = '', destination = '', group = {}, current_file = ''):

        rename_files = {}

        def test(s):
            return current_file[:-len(replacements['ext'])] in s

        for extra in set(filter(test, group['files'][extra_type])):
            replacements['ext'] = getExt(extra)

            final_folder_name = self.doReplace(folder_name, replacements)
            final_file_name = self.doReplace(file_name, replacements)
            rename_files[extra] = os.path.join(destination, final_folder_name, final_file_name)

        return rename_files
Example #13
0
    def getPoster(self, media, image_urls):
        if 'files' not in media:
            media['files'] = {}

        existing_files = media['files']

        image_type = 'poster'
        file_type = 'image_%s' % image_type

        # Make existing unique
        unique_files = list(set(existing_files.get(file_type, [])))

        # Remove files that can't be found
        for ef in unique_files:
            if not os.path.isfile(ef):
                unique_files.remove(ef)

        # Replace new files list
        existing_files[file_type] = unique_files
        if len(existing_files) == 0:
            del existing_files[file_type]

        images = image_urls.get(image_type, [])
        for y in ['SX300', 'tmdb']:
            initially_try = [x for x in images if y in x]
            images[:-1] = initially_try

        # Loop over type
        for image in images:
            if not isinstance(image, (str, unicode)):
                continue

            # Check if it has top image
            filename = '%s.%s' % (md5(image), getExt(image))
            existing = existing_files.get(file_type, [])
            has_latest = False
            for x in existing:
                if filename in x:
                    has_latest = True

            if not has_latest or file_type not in existing_files or len(existing_files.get(file_type, [])) == 0:
                file_path = fireEvent('file.download', url = image, single = True)
                if file_path:
                    existing_files[file_type] = [toUnicode(file_path)]
                    break
            else:
                break
Example #14
0
    def guess(self, files, extra={}):

        # Create hash for cache
        hash = md5(str([f.replace('.' + getExt(f), '') for f in files]))
        cached = self.getCache(hash)
        if cached and extra is {}: return cached

        for cur_file in files:
            size = (os.path.getsize(cur_file) / 1024 /
                    1024) if os.path.isfile(cur_file) else 0
            words = re.split('\W+', cur_file.lower())

            for quality in self.all():

                # Check tags
                if quality['identifier'] in words:
                    log.debug('Found via identifier "%s" in %s',
                              (quality['identifier'], cur_file))
                    return self.setCache(hash, quality)

                if list(set(quality.get('alternative', [])) & set(words)):
                    log.debug('Found %s via alt %s in %s',
                              (quality['identifier'],
                               quality.get('alternative'), cur_file))
                    return self.setCache(hash, quality)

                for tag in quality.get('tags', []):
                    if isinstance(tag,
                                  tuple) and '.'.join(tag) in '.'.join(words):
                        log.debug('Found %s via tag %s in %s',
                                  (quality['identifier'], quality.get('tags'),
                                   cur_file))
                        return self.setCache(hash, quality)

                if list(set(quality.get('tags', [])) & set(words)):
                    log.debug(
                        'Found %s via tag %s in %s',
                        (quality['identifier'], quality.get('tags'), cur_file))
                    return self.setCache(hash, quality)

        # Try again with loose testing
        quality = self.guessLoose(hash, extra=extra)
        if quality:
            return self.setCache(hash, quality)

        log.debug('Could not identify quality for: %s', files)
        return None
Example #15
0
    def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = None):
        if not urlopen_kwargs: urlopen_kwargs = {}

        if not dest:  # to Cache
            dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url)))

        if not overwrite and os.path.isfile(dest):
            return dest

        try:
            filedata = self.urlopen(url, **urlopen_kwargs)
        except:
            log.error('Failed downloading file %s: %s', (url, traceback.format_exc()))
            return False

        self.createFile(dest, filedata, binary = True)
        return dest
Example #16
0
    def registerStatic(self, plugin_file, add_to_head=True):

        # Register plugin path
        self.plugin_path = os.path.dirname(plugin_file)

        # Get plugin_name from PluginName
        s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", self.__class__.__name__)
        class_name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()

        path = "api/%s/static/%s/" % (Env.setting("api_key"), class_name)
        addView(path + "<path:filename>", self.showStatic, static=True)

        if add_to_head:
            for f in glob.glob(os.path.join(self.plugin_path, "static", "*")):
                ext = getExt(f)
                if ext in ["js", "css"]:
                    fireEvent("register_%s" % ("script" if ext in "js" else "style"), path + os.path.basename(f))
Example #17
0
    def registerStatic(self, plugin_file, add_to_head = True):

        # Register plugin path
        self.plugin_path = os.path.dirname(plugin_file)

        # Get plugin_name from PluginName
        s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__)
        class_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

        path = 'api/%s/static/%s/' % (Env.setting('api_key'), class_name)
        addView(path + '<path:filename>', self.showStatic, static = True)

        if add_to_head:
            for f in glob.glob(os.path.join(self.plugin_path, 'static', '*')):
                ext = getExt(f)
                if ext in ['js', 'css']:
                    fireEvent('register_%s' % ('script' if ext in 'js' else 'style'), path + os.path.basename(f))
Example #18
0
    def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = None):
        if not urlopen_kwargs: urlopen_kwargs = {}

        if not dest:  # to Cache
            dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url)))

        if not overwrite and os.path.isfile(dest):
            return dest

        try:
            filedata = self.urlopen(url, **urlopen_kwargs)
        except:
            log.error('Failed downloading file %s: %s', (url, traceback.format_exc()))
            return False

        self.createFile(dest, filedata, binary = True)
        return dest
Example #19
0
    def guess(self, files, extra=None):
        if not extra: extra = {}

        # Create hash for cache
        cache_key = md5(str([f.replace('.' + getExt(f), '') for f in files]))
        cached = self.getCache(cache_key)
        if cached and len(extra) == 0: return cached

        qualities = self.all()

        # Start with 0
        score = {}
        for quality in qualities:
            score[quality.get('identifier')] = 0

        for cur_file in files:
            words = re.split('\W+', cur_file.lower())

            for quality in qualities:
                contains_score = self.containsTagScore(quality, words,
                                                       cur_file)
                self.calcScore(score, quality, contains_score)

        # Try again with loose testing
        for quality in qualities:
            loose_score = self.guessLooseScore(quality,
                                               files=files,
                                               extra=extra)
            self.calcScore(score, quality, loose_score)

        # Return nothing if all scores are 0
        has_non_zero = 0
        for s in score:
            if score[s] > 0:
                has_non_zero += 1

        if not has_non_zero:
            return None

        heighest_quality = max(score, key=score.get)
        if heighest_quality:
            for quality in qualities:
                if quality.get('identifier') == heighest_quality:
                    return self.setCache(cache_key, quality)

        return None
Example #20
0
    def getRenameExtras(self, extra_type = '', replacements = None, folder_name = '', file_name = '', destination = '', group = None, current_file = '', remove_multiple = False):
        if not group: group = {}
        if not replacements: replacements = {}

        replacements = replacements.copy()
        rename_files = {}

        def test(s):
            return current_file[:-len(replacements['ext'])] in s

        for extra in set(filter(test, group['files'][extra_type])):
            replacements['ext'] = getExt(extra)

            final_folder_name = self.doReplace(folder_name, replacements, remove_multiple = remove_multiple, folder = True)
            final_file_name = self.doReplace(file_name, replacements, remove_multiple = remove_multiple)
            rename_files[extra] = os.path.join(destination, final_folder_name, final_file_name)

        return rename_files
Example #21
0
    def searchSingle(self, message = None, group = None):
        if not group: group = {}
        if self.isDisabled() or len(group['files']['trailer']) > 0: return

        if self.conf('usevf'):
            try:
                filename = self.conf('name').replace('<filename>', group['filename'].decode('latin-1'))
            except:
                filename = self.conf('name').replace('<filename>', group['filename'])
            try:
                destination = os.path.join(group['destination_dir'].decode('latin-1'), filename)
            except:
                destination = os.path.join(group['destination_dir'], filename)
            trailers = fireEvent('vftrailer.search', group = group, filename=filename, destination=destination, merge = True)
        else :
            trailers = fireEvent('trailer.search', group = group, merge = True)
        if not trailers or trailers == []:
            log.info('No trailers found for: %s', getTitle(group))
            return False

        if self.conf('usevf'):
            log.info('Trailers found in VF for: %s', getTitle(group))
            return True
        else:

            for trailer in trailers.get(self.conf('quality'), []):

                ext = getExt(trailer)
                filename = self.conf('name').replace('<filename>', group['filename']) + ('.%s' % ('mp4' if len(ext) > 5 else ext))
                destination = os.path.join(group['destination_dir'], filename)
                if not os.path.isfile(destination):
                    trailer_file = fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True)
                    if os.path.getsize(trailer_file) < (1024 * 1024):  # Don't trust small trailers (1MB), try next one
                        os.unlink(trailer_file)
                        continue
                else:
                    log.debug('Trailer already exists: %s', destination)

                group['renamed_files'].append(destination)

                # Download first and break
                break

            return True
Example #22
0
    def getPoster(self, media, image_urls):
        if 'files' not in media:
            media['files'] = {}

        existing_files = media['files']

        image_type = 'poster'
        file_type = 'image_%s' % image_type

        # Make existing unique
        unique_files = list(set(existing_files.get(file_type, [])))

        # Remove files that can't be found
        for ef in unique_files:
            if not os.path.isfile(ef):
                unique_files.remove(ef)

        # Replace new files list
        existing_files[file_type] = unique_files
        if len(existing_files) == 0:
            del existing_files[file_type]

        # Loop over type
        for image in image_urls.get(image_type, []):
            if not isinstance(image, (str, unicode)):
                continue

            # Check if it has top image
            filename = '%s.%s' % (md5(image), getExt(image))
            existing = existing_files.get(file_type, [])
            has_latest = False
            for x in existing:
                if filename in x:
                    has_latest = True

            if not has_latest or file_type not in existing_files or len(
                    existing_files.get(file_type, [])) == 0:
                file_path = fireEvent('file.download', url=image, single=True)
                if file_path:
                    existing_files[file_type] = [toUnicode(file_path)]
                    break
            else:
                break
Example #23
0
    def download(self, url="", dest=None, overwrite=False):

        file = self.urlopen(url)
        if not file:
            log.error("File is empty, don't download")
            return False

        if not dest:  # to Cache
            dest = os.path.join(Env.get("cache_dir"), "%s.%s" % (md5(url), getExt(url)))

        if overwrite or not os.path.exists(dest):
            log.debug("Writing file to: %s" % dest)
            output = open(dest, "wb")
            output.write(file)
            output.close()
        else:
            log.debug("File already exists: %s" % dest)

        return dest
Example #24
0
    def searchSingle(self, message = None, group = {}):

        if self.isDisabled() or len(group['files']['trailer']) > 0: return

        trailers = fireEvent('trailer.search', group = group, merge = True)
        if not trailers or trailers == []:
            log.info('No trailers found for: %s', getTitle(group['library']))
            return

        for trailer in trailers.get(self.conf('quality'), []):
            destination = '%s-trailer.%s' % (self.getRootName(group), getExt(trailer))
            if not os.path.isfile(destination):
                fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True)
            else:
                log.debug('Trailer already exists: %s', destination)

            group['renamed_files'].append(destination)

            # Download first and break
            break
Example #25
0
    def download(self, url='', dest=None, overwrite=False):

        file = self.urlopen(url)
        if not file:
            log.error('File is empty, don\'t download')
            return False

        if not dest:  # to Cache
            dest = os.path.join(Env.get('cache_dir'),
                                '%s.%s' % (md5(url), getExt(url)))

        if overwrite or not os.path.exists(dest):
            log.debug('Writing file to: %s' % dest)
            output = open(dest, 'wb')
            output.write(file)
            output.close()
        else:
            log.debug('File already exists: %s' % dest)

        return dest
Example #26
0
    def download(self, url = '', dest = None, overwrite = False):

        try:
            file = urllib2.urlopen(url)

            if not dest: # to Cache
                dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url)))

            if overwrite or not os.path.exists(dest):
                log.debug('Writing file to: %s' % dest)
                output = open(dest, 'wb')
                output.write(file.read())
                output.close()
            else:
                log.debug('File already exists: %s' % dest)

            return dest

        except Exception, e:
            log.error('Unable to download file "%s": %s' % (url, e))
Example #27
0
    def guess(self, files, extra={}):

        # Create hash for cache
        hash = md5(str([f.replace("." + getExt(f), "") for f in files]))
        cached = self.getCache(hash)
        if cached and extra is {}:
            return cached

        for cur_file in files:
            size = (os.path.getsize(cur_file) / 1024 / 1024) if os.path.isfile(cur_file) else 0
            words = re.split("\W+", cur_file.lower())

            for quality in self.all():

                # Check tags
                if quality["identifier"] in words:
                    log.debug('Found via identifier "%s" in %s' % (quality["identifier"], cur_file))
                    return self.setCache(hash, quality)

                if list(set(quality.get("alternative", [])) & set(words)):
                    log.debug(
                        "Found %s via alt %s in %s" % (quality["identifier"], quality.get("alternative"), cur_file)
                    )
                    return self.setCache(hash, quality)

                for tag in quality.get("tags", []):
                    if isinstance(tag, tuple) and ".".join(tag) in ".".join(words):
                        log.debug("Found %s via tag %s in %s" % (quality["identifier"], quality.get("tags"), cur_file))
                        return self.setCache(hash, quality)

                if list(set(quality.get("tags", [])) & set(words)):
                    log.debug("Found %s via tag %s in %s" % (quality["identifier"], quality.get("tags"), cur_file))
                    return self.setCache(hash, quality)

        # Try again with loose testing
        quality = self.guessLoose(hash, extra=extra)
        if quality:
            return self.setCache(hash, quality)

        log.debug("Could not identify quality for: %s" % files)
        return None
Example #28
0
    def download(self, url='', dest=None, overwrite=False):

        try:
            file = urllib2.urlopen(url)

            if not dest:  # to Cache
                dest = os.path.join(Env.get('cache_dir'),
                                    '%s.%s' % (md5(url), getExt(url)))

            if overwrite or not os.path.exists(dest):
                log.debug('Writing file to: %s' % dest)
                output = open(dest, 'wb')
                output.write(file.read())
                output.close()
            else:
                log.debug('File already exists: %s' % dest)

            return dest

        except Exception, e:
            log.error('Unable to download file "%s": %s' % (url, e))
Example #29
0
    def guess(self, files, extra = None):
        if not extra: extra = {}

        # Create hash for cache
        cache_key = md5(str([f.replace('.' + getExt(f), '') for f in files]))
        cached = self.getCache(cache_key)
        if cached and len(extra) == 0: return cached

        qualities = self.all()
        for cur_file in files:
            words = re.split('\W+', cur_file.lower())

            found = {}
            for quality in qualities:
                contains = self.containsTag(quality, words, cur_file)
                if contains:
                    found[quality['identifier']] = True

            for quality in qualities:

                # Check identifier
                if quality['identifier'] in words:
                    if len(found) == 0 or len(found) == 1 and found.get(quality['identifier']):
                        log.debug('Found via identifier "%s" in %s', (quality['identifier'], cur_file))
                        return self.setCache(cache_key, quality)

                # Check alt and tags
                contains = self.containsTag(quality, words, cur_file)
                if contains:
                    return self.setCache(cache_key, quality)

        # Try again with loose testing
        quality = self.guessLoose(cache_key, files = files, extra = extra)
        if quality:
            return self.setCache(cache_key, quality)

        log.debug('Could not identify quality for: %s', files)
        return None
Example #30
0
    def download(self, url="", dest=None, overwrite=False, urlopen_kwargs=None):
        if not urlopen_kwargs:
            urlopen_kwargs = {}

        # Return response object to stream download
        urlopen_kwargs["stream"] = True

        if not dest:  # to Cache
            dest = os.path.join(Env.get("cache_dir"), ss("%s.%s" % (md5(url), getExt(url))))

        dest = sp(dest)

        if not overwrite and os.path.isfile(dest):
            return dest

        try:
            filedata = self.urlopen(url, **urlopen_kwargs)
        except:
            log.error("Failed downloading file %s: %s", (url, traceback.format_exc()))
            return False

        self.createFile(dest, filedata, binary=True)
        return dest
Example #31
0
    def getRenameExtras(self,
                        extra_type='',
                        replacements={},
                        folder_name='',
                        file_name='',
                        destination='',
                        group={},
                        current_file=''):

        rename_files = {}

        def test(s):
            return current_file[:-len(replacements['ext'])] in s

        for extra in set(filter(test, group['files'][extra_type])):
            replacements['ext'] = getExt(extra)

            final_folder_name = self.doReplace(folder_name, replacements)
            final_file_name = self.doReplace(file_name, replacements)
            rename_files[extra] = os.path.join(destination, final_folder_name,
                                               final_file_name)

        return rename_files
Example #32
0
    def searchSingle(self, message=None, group={}):

        if self.isDisabled() or len(group["files"]["trailer"]) > 0:
            return

        trailers = fireEvent("trailer.search", group=group, merge=True)
        if not trailers or trailers == []:
            log.info("No trailers found for: %s", getTitle(group["library"]))
            return False

        for trailer in trailers.get(self.conf("quality"), []):

            ext = getExt(trailer)
            filename = self.conf("name").replace("<filename>", group["filename"]) + (
                ".%s" % ("mp4" if len(ext) > 5 else ext)
            )
            destination = os.path.join(group["destination_dir"], filename)
            if not os.path.isfile(destination):
                trailer_file = fireEvent(
                    "file.download",
                    url=trailer,
                    dest=destination,
                    urlopen_kwargs={"headers": {"User-Agent": "Quicktime"}},
                    single=True,
                )
                if os.path.getsize(trailer_file) < (1024 * 1024):  # Don't trust small trailers (1MB), try next one
                    os.unlink(trailer_file)
                    continue
            else:
                log.debug("Trailer already exists: %s", destination)

            group["renamed_files"].append(destination)

            # Download first and break
            break

        return True
Example #33
0
    def guess(self, files, extra = {}):

        # Create hash for cache
        hash = md5(str([f.replace('.' + getExt(f), '') for f in files]))
        cached = self.getCache(hash)
        if cached and extra is {}: return cached

        for cur_file in files:
            words = re.split('\W+', cur_file.lower())

            for quality in self.all():

                # Check tags
                if quality['identifier'] in words:
                    log.debug('Found via identifier "%s" in %s', (quality['identifier'], cur_file))
                    return self.setCache(hash, quality)

                if list(set(quality.get('alternative', [])) & set(words)):
                    log.debug('Found %s via alt %s in %s', (quality['identifier'], quality.get('alternative'), cur_file))
                    return self.setCache(hash, quality)

                for tag in quality.get('tags', []):
                    if isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words):
                        log.debug('Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file))
                        return self.setCache(hash, quality)

                if list(set(quality.get('tags', [])) & set(words)):
                    log.debug('Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file))
                    return self.setCache(hash, quality)

        # Try again with loose testing
        quality = self.guessLoose(hash, files = files, extra = extra)
        if quality:
            return self.setCache(hash, quality)

        log.debug('Could not identify quality for: %s', files)
        return None
Example #34
0
    def scan(self):

        if self.isDisabled():
            return

        if self.renaming_started is True:
            log.info(
                'Renamer is disabled to avoid infinite looping of the same error.'
            )
            return

        # Check to see if the "to" folder is inside the "from" folder.
        if not os.path.isdir(self.conf('from')) or not os.path.isdir(
                self.conf('to')):
            log.debug('"To" and "From" have to exist.')
            return
        elif self.conf('from') in self.conf('to'):
            log.error(
                'The "to" can\'t be inside of the "from" folder. You\'ll get an infinite loop.'
            )
            return

        groups = fireEvent('scanner.scan',
                           folder=self.conf('from'),
                           single=True)

        self.renaming_started = True

        destination = self.conf('to')
        folder_name = self.conf('folder_name')
        file_name = self.conf('file_name')
        trailer_name = self.conf('trailer_name')
        nfo_name = self.conf('nfo_name')
        separator = self.conf('separator')

        # Statusses
        done_status = fireEvent('status.get', 'done', single=True)
        active_status = fireEvent('status.get', 'active', single=True)
        downloaded_status = fireEvent('status.get', 'downloaded', single=True)
        snatched_status = fireEvent('status.get', 'snatched', single=True)

        db = get_session()

        for group_identifier in groups:

            group = groups[group_identifier]
            rename_files = {}
            remove_files = []
            remove_releases = []

            movie_title = getTitle(group['library'])

            # Add _UNKNOWN_ if no library item is connected
            if not group['library'] or not movie_title:
                self.tagDir(group, 'unknown')
                continue
            # Rename the files using the library data
            else:
                group['library'] = fireEvent(
                    'library.update',
                    identifier=group['library']['identifier'],
                    single=True)
                if not group['library']:
                    log.error(
                        'Could not rename, no library item to work with: %s',
                        group_identifier)
                    continue

                library = group['library']
                movie_title = getTitle(library)

                # Find subtitle for renaming
                fireEvent('renamer.before', group)

                # Remove weird chars from moviename
                movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', movie_title)

                # Put 'The' at the end
                name_the = movie_name
                if movie_name[:4].lower() == 'the ':
                    name_the = movie_name[4:] + ', The'

                replacements = {
                    'ext':
                    'mkv',
                    'namethe':
                    name_the.strip(),
                    'thename':
                    movie_name.strip(),
                    'year':
                    library['year'],
                    'first':
                    name_the[0].upper(),
                    'quality':
                    group['meta_data']['quality']['label'],
                    'quality_type':
                    group['meta_data']['quality_type'],
                    'video':
                    group['meta_data'].get('video'),
                    'audio':
                    group['meta_data'].get('audio'),
                    'group':
                    group['meta_data']['group'],
                    'source':
                    group['meta_data']['source'],
                    'resolution_width':
                    group['meta_data'].get('resolution_width'),
                    'resolution_height':
                    group['meta_data'].get('resolution_height'),
                    'imdb_id':
                    library['identifier'],
                }

                for file_type in group['files']:

                    # Move nfo depending on settings
                    if file_type is 'nfo' and not self.conf('rename_nfo'):
                        log.debug('Skipping, renaming of %s disabled',
                                  file_type)
                        if self.conf('cleanup'):
                            for current_file in group['files'][file_type]:
                                remove_files.append(current_file)
                        continue

                    # Subtitle extra
                    if file_type is 'subtitle_extra':
                        continue

                    # Move other files
                    multiple = len(
                        group['files']['movie']) > 1 and not group['is_dvd']
                    cd = 1 if multiple else 0

                    for current_file in sorted(list(
                            group['files'][file_type])):

                        # Original filename
                        replacements['original'] = os.path.splitext(
                            os.path.basename(current_file))[0]
                        replacements['original_folder'] = fireEvent(
                            'scanner.remove_cptag',
                            group['dirname'],
                            single=True)

                        # Extension
                        replacements['ext'] = getExt(current_file)

                        # cd #
                        replacements['cd'] = ' cd%d' % cd if cd else ''
                        replacements['cd_nr'] = cd

                        # Naming
                        final_folder_name = self.doReplace(
                            folder_name, replacements)
                        final_file_name = self.doReplace(
                            file_name, replacements)
                        replacements['filename'] = final_file_name[:-(
                            len(getExt(final_file_name)) + 1)]

                        # Group filename without cd extension
                        replacements['cd'] = ''
                        replacements['cd_nr'] = ''

                        # Meta naming
                        if file_type is 'trailer':
                            final_file_name = self.doReplace(
                                trailer_name, replacements)
                        elif file_type is 'nfo':
                            final_file_name = self.doReplace(
                                nfo_name, replacements)

                        # Seperator replace
                        if separator:
                            final_file_name = final_file_name.replace(
                                ' ', separator)

                        # Move DVD files (no structure renaming)
                        if group['is_dvd'] and file_type is 'movie':
                            found = False
                            for top_dir in [
                                    'video_ts', 'audio_ts', 'bdmv',
                                    'certificate'
                            ]:
                                has_string = current_file.lower().find(
                                    os.path.sep + top_dir + os.path.sep)
                                if has_string >= 0:
                                    structure_dir = current_file[
                                        has_string:].lstrip(os.path.sep)
                                    rename_files[current_file] = os.path.join(
                                        destination, final_folder_name,
                                        structure_dir)
                                    found = True
                                    break

                            if not found:
                                log.error(
                                    'Could not determine dvd structure for: %s',
                                    current_file)

                        # Do rename others
                        else:
                            if file_type is 'leftover':
                                if self.conf('move_leftover'):
                                    rename_files[current_file] = os.path.join(
                                        destination, final_folder_name,
                                        os.path.basename(current_file))
                            elif file_type not in ['subtitle']:
                                rename_files[current_file] = os.path.join(
                                    destination, final_folder_name,
                                    final_file_name)

                        # Check for extra subtitle files
                        if file_type is 'subtitle':

                            # rename subtitles with or without language
                            rename_files[current_file] = os.path.join(
                                destination, final_folder_name,
                                final_file_name)
                            sub_langs = group['subtitle_language'].get(
                                current_file, [])

                            rename_extras = self.getRenameExtras(
                                extra_type='subtitle_extra',
                                replacements=replacements,
                                folder_name=folder_name,
                                file_name=file_name,
                                destination=destination,
                                group=group,
                                current_file=current_file)

                            # Don't add language if multiple languages in 1 file
                            if len(sub_langs) > 1:
                                rename_files[current_file] = os.path.join(
                                    destination, final_folder_name,
                                    final_file_name)
                            elif len(sub_langs) == 1:
                                sub_name = final_file_name.replace(
                                    replacements['ext'], '%s.%s' %
                                    (sub_langs[0], replacements['ext']))
                                rename_files[current_file] = os.path.join(
                                    destination, final_folder_name, sub_name)

                            rename_files = mergeDicts(rename_files,
                                                      rename_extras)

                        # Filename without cd etc
                        if file_type is 'movie':
                            rename_extras = self.getRenameExtras(
                                extra_type='movie_extra',
                                replacements=replacements,
                                folder_name=folder_name,
                                file_name=file_name,
                                destination=destination,
                                group=group,
                                current_file=current_file)
                            rename_files = mergeDicts(rename_files,
                                                      rename_extras)

                            group['filename'] = self.doReplace(
                                file_name,
                                replacements)[:-(len(getExt(final_file_name)) +
                                                 1)]
                            group['destination_dir'] = os.path.join(
                                destination, final_folder_name)

                        if multiple:
                            cd += 1

                # Before renaming, remove the lower quality files
                library = db.query(Library).filter_by(
                    identifier=group['library']['identifier']).first()
                remove_leftovers = True

                # Add it to the wanted list before we continue
                if len(library.movies) == 0:
                    profile = db.query(Profile).filter_by(
                        core=True,
                        label=group['meta_data']['quality']['label']).first()
                    fireEvent('movie.add',
                              params={
                                  'identifier': group['library']['identifier'],
                                  'profile_id': profile.id
                              },
                              search_after=False)
                    db.expire_all()
                    library = db.query(Library).filter_by(
                        identifier=group['library']['identifier']).first()

                for movie in library.movies:

                    # Mark movie "done" onces it found the quality with the finish check
                    try:
                        if movie.status_id == active_status.get(
                                'id') and movie.profile:
                            for profile_type in movie.profile.types:
                                if profile_type.quality_id == group[
                                        'meta_data']['quality'][
                                            'id'] and profile_type.finish:
                                    movie.status_id = done_status.get('id')
                                    db.commit()
                    except Exception, e:
                        log.error('Failed marking movie finished: %s %s',
                                  (e, traceback.format_exc()))

                    # Go over current movie releases
                    for release in movie.releases:

                        # When a release already exists
                        if release.status_id is done_status.get('id'):

                            # This is where CP removes older, lesser quality releases
                            if release.quality.order > group['meta_data'][
                                    'quality']['order']:
                                log.info('Removing lesser quality %s for %s.',
                                         (movie.library.titles[0].title,
                                          release.quality.label))
                                for current_file in release.files:
                                    remove_files.append(current_file)
                                remove_releases.append(release)
                            # Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc
                            elif release.quality.order is group['meta_data'][
                                    'quality']['order']:
                                log.info(
                                    'Same quality release already exists for %s, with quality %s. Assuming repack.',
                                    (movie.library.titles[0].title,
                                     release.quality.label))
                                for current_file in release.files:
                                    remove_files.append(current_file)
                                remove_releases.append(release)

                            # Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan
                            else:
                                log.info(
                                    'Better quality release already exists for %s, with quality %s',
                                    (movie.library.titles[0].title,
                                     release.quality.label))

                                # Add _EXISTS_ to the parent dir
                                self.tagDir(group, 'exists')

                                # Notify on rename fail
                                download_message = 'Renaming of %s (%s) canceled, exists in %s already.' % (
                                    movie.library.titles[0].title,
                                    group['meta_data']['quality']['label'],
                                    release.quality.label)
                                fireEvent('movie.renaming.canceled',
                                          message=download_message,
                                          data=group)
                                remove_leftovers = False

                                break
                        elif release.status_id is snatched_status.get('id'):
                            if release.quality.id is group['meta_data'][
                                    'quality']['id']:
                                log.debug('Marking release as downloaded')
                                release.status_id = downloaded_status.get('id')
                                db.commit()

                # Remove leftover files
                if self.conf('cleanup') and not self.conf(
                        'move_leftover') and remove_leftovers:
                    log.debug('Removing leftover files')
                    for current_file in group['files']['leftover']:
                        remove_files.append(current_file)
                elif not remove_leftovers:  # Don't remove anything
                    break

            # Remove files
            delete_folders = []
            for src in remove_files:

                if isinstance(src, File):
                    src = src.path

                if rename_files.get(src):
                    log.debug('Not removing file that will be renamed: %s',
                              src)
                    continue

                log.info('Removing "%s"', src)
                try:
                    if os.path.isfile(src):
                        os.remove(src)

                        parent_dir = os.path.normpath(os.path.dirname(src))
                        if delete_folders.count(
                                parent_dir) == 0 and os.path.isdir(
                                    parent_dir) and destination != parent_dir:
                            delete_folders.append(parent_dir)

                except:
                    log.error('Failed removing %s: %s',
                              (src, traceback.format_exc()))
                    self.tagDir(group, 'failed_remove')

            # Delete leftover folder from older releases
            for delete_folder in delete_folders:
                self.deleteEmptyFolder(delete_folder, show_error=False)

            # Rename all files marked
            group['renamed_files'] = []
            for src in rename_files:
                if rename_files[src]:
                    dst = rename_files[src]
                    log.info('Renaming "%s" to "%s"', (src, dst))

                    # Create dir
                    self.makeDir(os.path.dirname(dst))

                    try:
                        self.moveFile(src, dst)
                        group['renamed_files'].append(dst)
                    except:
                        log.error(
                            'Failed moving the file "%s" : %s',
                            (os.path.basename(src), traceback.format_exc()))
                        self.tagDir(group, 'failed_rename')

            # Remove matching releases
            for release in remove_releases:
                log.debug('Removing release %s', release.identifier)
                try:
                    db.delete(release)
                except:
                    log.error('Failed removing %s: %s',
                              (release.identifier, traceback.format_exc()))

            if group['dirname'] and group['parentdir']:
                try:
                    log.info('Deleting folder: %s', group['parentdir'])
                    self.deleteEmptyFolder(group['parentdir'])
                except:
                    log.error('Failed removing %s: %s',
                              (group['parentdir'], traceback.format_exc()))

            # Search for trailers etc
            fireEventAsync('renamer.after', group)

            # Notify on download
            download_message = 'Downloaded %s (%s)' % (movie_title,
                                                       replacements['quality'])
            fireEventAsync('movie.downloaded',
                           message=download_message,
                           data=group)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break
Example #35
0
    def scan(self):

        groups = fireEvent('scanner.scan',
                           folder=self.conf('from'),
                           single=True)
        if groups is None: return

        destination = self.conf('to')
        folder_name = self.conf('folder_name')
        file_name = self.conf('file_name')
        trailer_name = self.conf('trailer_name')
        backdrop_name = self.conf('fanart_name')
        nfo_name = self.conf('nfo_name')
        separator = self.conf('separator')

        for group_identifier in groups:

            group = groups[group_identifier]
            rename_files = {}

            # Add _UNKNOWN_ if no library item is connected
            if not group['library']:
                if group['dirname']:
                    rename_files[
                        group['parentdir']] = group['parentdir'].replace(
                            group['dirname'], '_UNKNOWN_%s' % group['dirname'])
                else:  # Add it to filename
                    for file_type in group['files']:
                        for rename_me in group['files'][file_type]:
                            filename = os.path.basename(rename_me)
                            rename_files[rename_me] = rename_me.replace(
                                filename, '_UNKNOWN_%s' % filename)

            # Rename the files using the library data
            else:
                group['library'] = fireEvent(
                    'library.update',
                    identifier=group['library']['identifier'],
                    single=True)
                if not group['library']:
                    log.error(
                        'Could not rename, no library item to work with: %s' %
                        group_identifier)
                    continue

                library = group['library']

                # Find subtitle for renaming
                fireEvent('renamer.before', group)

                # Remove weird chars from moviename
                movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '',
                                    group['library']['titles'][0]['title'])

                # Put 'The' at the end
                name_the = movie_name
                if movie_name[:4].lower() == 'the ':
                    name_the = movie_name[4:] + ', The'

                replacements = {
                    'ext':
                    'mkv',
                    'namethe':
                    name_the.strip(),
                    'thename':
                    movie_name.strip(),
                    'year':
                    library['year'],
                    'first':
                    name_the[0].upper(),
                    'dirname':
                    group['dirname'],
                    'quality':
                    group['meta_data']['quality']['label'],
                    'quality_type':
                    group['meta_data']['quality_type'],
                    'video':
                    group['meta_data'].get('video'),
                    'audio':
                    group['meta_data'].get('audio'),
                    'group':
                    group['meta_data']['group'],
                    'source':
                    group['meta_data']['source'],
                    'resolution_width':
                    group['meta_data'].get('resolution_width'),
                    'resolution_height':
                    group['meta_data'].get('resolution_height'),
                }

                for file_type in group['files']:

                    # Move nfo depending on settings
                    if file_type is 'nfo' and not self.conf('rename_nfo'):
                        log.debug('Skipping, renaming of %s disabled' %
                                  file_type)
                        continue

                    # Subtitle extra
                    if file_type is 'subtitle_extra':
                        continue

                    # Move other files
                    multiple = len(
                        group['files']['movie']) > 1 and not group['is_dvd']
                    cd = 1 if multiple else 0

                    for file in sorted(list(group['files'][file_type])):

                        # Original filename
                        replacements['original'] = os.path.basename(file)
                        replacements['original_folder'] = os.path.basename(
                            os.path.dirname(file))

                        # Extension
                        replacements['ext'] = getExt(file)

                        # cd #
                        replacements['cd'] = ' cd%d' % cd if cd else ''
                        replacements['cd_nr'] = cd

                        # Naming
                        final_folder_name = self.doReplace(
                            folder_name, replacements)
                        final_file_name = self.doReplace(
                            file_name, replacements)
                        replacements['filename'] = final_file_name[:-(
                            len(getExt(final_file_name)) + 1)]

                        # Meta naming
                        if file_type is 'trailer':
                            final_file_name = self.doReplace(
                                trailer_name, replacements)
                        elif file_type is 'nfo':
                            final_file_name = self.doReplace(
                                nfo_name, replacements)
                        elif file_type is 'backdrop':
                            final_file_name = self.doReplace(
                                backdrop_name, replacements)

                        # Seperator replace
                        if separator:
                            final_file_name = final_file_name.replace(
                                ' ', separator)

                        # Move DVD files (no structure renaming)
                        if group['is_dvd'] and file_type is 'movie':
                            found = False
                            for top_dir in [
                                    'video_ts', 'audio_ts', 'bdmv',
                                    'certificate'
                            ]:
                                has_string = file.lower().find(os.path.sep +
                                                               top_dir +
                                                               os.path.sep)
                                if has_string >= 0:
                                    structure_dir = file[has_string:].lstrip(
                                        os.path.sep)
                                    rename_files[file] = os.path.join(
                                        destination, final_folder_name,
                                        structure_dir)
                                    found = True
                                    break

                            if not found:
                                log.error(
                                    'Could not determin dvd structure for: %s'
                                    % file)

                        # Do rename others
                        else:
                            rename_files[file] = os.path.join(
                                destination, final_folder_name,
                                final_file_name)

                        # Check for extra subtitle files
                        if file_type is 'subtitle':

                            def test(s):
                                return file[:-len(replacements['ext'])] in s

                            for subtitle_extra in set(
                                    filter(test,
                                           group['files']['subtitle_extra'])):
                                replacements['ext'] = getExt(subtitle_extra)

                                final_folder_name = self.doReplace(
                                    folder_name, replacements)
                                final_file_name = self.doReplace(
                                    file_name, replacements)
                                rename_files[subtitle_extra] = os.path.join(
                                    destination, final_folder_name,
                                    final_file_name)

                        # Filename without cd etc
                        if file_type is 'movie':
                            group['destination_dir'] = os.path.join(
                                destination, final_folder_name)

                        if multiple:
                            cd += 1

                # Before renaming, remove the lower quality files
                db = get_session()

                library = db.query(Library).filter_by(
                    identifier=group['library']['identifier']).first()
                done_status = fireEvent('status.get', 'done', single=True)
                active_status = fireEvent('status.get', 'active', single=True)

                for movie in library.movies:

                    # Mark movie "done" onces it found the quality with the finish check
                    try:
                        if movie.status_id == active_status.get('id'):
                            for type in movie.profile.types:
                                if type.quality_id == group['meta_data'][
                                        'quality']['id'] and type.finish:
                                    movie.status_id = done_status.get('id')
                                    db.commit()
                    except Exception, e:
                        log.error('Failed marking movie finished: %s %s' %
                                  (e, traceback.format_exc()))

                    # Go over current movie releases
                    for release in movie.releases:

                        # This is where CP removes older, lesser quality releases
                        if release.quality.order > group['meta_data'][
                                'quality']['order']:
                            log.info(
                                'Removing older release for %s, with quality %s'
                                % (movie.library.titles[0].title,
                                   release.quality.label))

                            for file in release.files:
                                log.info('Removing (not really) "%s"' %
                                         file.path)

                        # When a release already exists
                        elif release.status_id is done_status.get('id'):

                            # Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc
                            if release.quality.order is group['meta_data'][
                                    'quality']['order']:
                                log.info(
                                    'Same quality release already exists for %s, with quality %s. Assuming repack.'
                                    % (movie.library.titles[0].title,
                                       release.quality.label))

                            # Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan
                            else:
                                log.info(
                                    'Better quality release already exists for %s, with quality %s'
                                    % (movie.library.titles[0].title,
                                       release.quality.label))

                                # Add _EXISTS_ to the parent dir
                                if group['dirname']:
                                    for rename_me in rename_files:  # Don't rename anything in this group
                                        rename_files[rename_me] = None
                                    rename_files[group['parentdir']] = group[
                                        'parentdir'].replace(
                                            group['dirname'],
                                            '_EXISTS_%s' % group['dirname'])
                                else:  # Add it to filename
                                    for rename_me in rename_files:
                                        filename = os.path.basename(rename_me)
                                        rename_files[
                                            rename_me] = rename_me.replace(
                                                filename,
                                                '_EXISTS_%s' % filename)

                                # Notify on rename fail
                                download_message = 'Renaming of %s (%s) canceled, exists in %s already.' % (
                                    movie.library.titles[0].title,
                                    group['meta_data']['quality']['label'],
                                    release.quality.label)
                                fireEvent('movie.renaming.canceled',
                                          message=download_message,
                                          data=group)

                                break

            # Rename all files marked
            for src in rename_files:
                if rename_files[src]:

                    dst = rename_files[src]

                    log.info('Renaming "%s" to "%s"' % (src, dst))

                    path = os.path.dirname(dst)

                    # Create dir
                    self.makeDir(path)

                    try:
                        pass
                        #shutil.move(src, dst)
                    except:
                        log.error(
                            'Failed moving the file "%s" : %s' %
                            (os.path.basename(src), traceback.format_exc()))

                #print rename_me, rename_files[rename_me]

            # Search for trailers etc
            fireEventAsync('renamer.after', group)

            # Notify on download
            download_message = 'Download of %s (%s) successful.' % (
                group['library']['titles'][0]['title'],
                replacements['quality'])
            fireEventAsync('movie.downloaded',
                           message=download_message,
                           data=group)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break
Example #36
0
    def guess(self, files, extra=None, size=None, use_cache=True):
        if not extra: extra = {}

        # Create hash for cache
        cache_key = str([
            f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f
            for f in files
        ])
        if use_cache:
            cached = self.getCache(cache_key)
            if cached and len(extra) == 0:
                return cached

        qualities = self.all()

        # Start with 0
        score = {}
        for quality in qualities:
            score[quality.get('identifier')] = {'score': 0, '3d': {}}

        # Use metadata titles as extra check
        if extra and extra.get('titles'):
            files.extend(extra.get('titles'))

        for cur_file in files:
            words = re.split('\W+', cur_file.lower())
            name_year = fireEvent('scanner.name_year',
                                  cur_file,
                                  file_name=cur_file,
                                  single=True)
            threed_words = words
            if name_year and name_year.get('name'):
                split_name = splitString(name_year.get('name'), ' ')
                threed_words = [x for x in words if x not in split_name]

            for quality in qualities:
                contains_score = self.containsTagScore(quality, words,
                                                       cur_file)
                threedscore = self.contains3D(
                    quality, threed_words,
                    cur_file) if quality.get('allow_3d') else (0, None)

                self.calcScore(score,
                               quality,
                               contains_score,
                               threedscore,
                               penalty=contains_score)

        size_scores = []
        for quality in qualities:

            # Evaluate score based on size
            size_score = self.guessSizeScore(quality, size=size)
            loose_score = self.guessLooseScore(quality, extra=extra)

            if size_score > 0:
                size_scores.append(quality)

            self.calcScore(score, quality, size_score + loose_score)

        # Add additional size score if only 1 size validated
        if len(size_scores) == 1:
            self.calcScore(score, size_scores[0], 7)
        del size_scores

        # Return nothing if all scores are <= 0
        has_non_zero = 0
        for s in score:
            if score[s]['score'] > 0:
                has_non_zero += 1

        if not has_non_zero:
            return None

        heighest_quality = max(score, key=lambda p: score[p]['score'])
        if heighest_quality:
            for quality in qualities:
                if quality.get('identifier') == heighest_quality:
                    quality['is_3d'] = False
                    if score[heighest_quality].get('3d'):
                        quality['is_3d'] = True
                    return self.setCache(cache_key, quality)

        return None
Example #37
0
 def getNfo(self, files):
     return set(filter(lambda s: getExt(s.lower()) in self.extensions['nfo'], files))
Example #38
0
 def getMovieExtras(self, files):
     return set(filter(lambda s: getExt(s.lower()) in self.extensions['movie_extra'], files))
Example #39
0
    def scan(self, folder = None, files = [], simple = False, newer_than = 0):

        folder = ss(os.path.normpath(folder))

        if not folder or not os.path.isdir(folder):
            log.error('Folder doesn\'t exists: %s', folder)
            return {}

        # Get movie "master" files
        movie_files = {}
        leftovers = []

        # Scan all files of the folder if no files are set
        if len(files) == 0:
            try:
                files = []
                for root, dirs, walk_files in os.walk(folder):
                    for filename in walk_files:
                        files.append(os.path.join(root, filename))
            except:
                log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))
        else:
            files = [ss(x) for x in files]

        db = get_session()

        for file_path in files:

            if not os.path.exists(file_path):
                continue

            # Remove ignored files
            if self.isSampleFile(file_path):
                leftovers.append(file_path)
                continue
            elif not self.keepFile(file_path):
                continue

            is_dvd_file = self.isDVDFile(file_path)
            if os.path.getsize(file_path) > self.minimal_filesize['media'] or is_dvd_file: # Minimal 300MB files or is DVD file

                # Normal identifier
                identifier = self.createStringIdentifier(file_path, folder, exclude_filename = is_dvd_file)
                identifiers = [identifier]

                # Identifier with quality
                quality = fireEvent('quality.guess', [file_path], single = True) if not is_dvd_file else {'identifier':'dvdr'}
                if quality:
                    identifier_with_quality = '%s %s' % (identifier, quality.get('identifier', ''))
                    identifiers = [identifier_with_quality, identifier]

                if not movie_files.get(identifier):
                    movie_files[identifier] = {
                        'unsorted_files': [],
                        'identifiers': identifiers,
                        'is_dvd': is_dvd_file,
                    }

                movie_files[identifier]['unsorted_files'].append(file_path)
            else:
                leftovers.append(file_path)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Cleanup
        del files

        # Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2"
        # files will be grouped first.
        leftovers = set(sorted(leftovers, reverse = True))


        # Group files minus extension
        for identifier, group in movie_files.iteritems():
            if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier)

            log.debug('Grouping files: %s', identifier)

            for file_path in group['unsorted_files']:
                wo_ext = file_path[:-(len(getExt(file_path)) + 1)]
                found_files = set([i for i in leftovers if wo_ext in i])
                group['unsorted_files'].extend(found_files)
                leftovers = leftovers - found_files

            # Break if CP wants to shut down
            if self.shuttingDown():
                break


        # Create identifiers for all leftover files
        for file_path in leftovers:
            identifier = self.createStringIdentifier(file_path, folder)

            if not self.path_identifiers.get(identifier):
                self.path_identifiers[identifier] = []

            self.path_identifiers[identifier].append(file_path)


        # Group the files based on the identifier
        delete_identifiers = []
        for identifier, found_files in self.path_identifiers.iteritems():
            log.debug('Grouping files on identifier: %s', identifier)

            group = movie_files.get(identifier)
            if group:
                group['unsorted_files'].extend(found_files)
                delete_identifiers.append(identifier)

                # Remove the found files from the leftover stack
                leftovers = leftovers - set(found_files)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Cleaning up used
        for identifier in delete_identifiers:
            if self.path_identifiers.get(identifier):
                del self.path_identifiers[identifier]
        del delete_identifiers

        # Group based on folder
        delete_identifiers = []
        for identifier, found_files in self.path_identifiers.iteritems():
            log.debug('Grouping files on foldername: %s', identifier)

            for ff in found_files:
                new_identifier = self.createStringIdentifier(os.path.dirname(ff), folder)

                group = movie_files.get(new_identifier)
                if group:
                    group['unsorted_files'].extend([ff])
                    delete_identifiers.append(identifier)

                    # Remove the found files from the leftover stack
                    leftovers = leftovers - set([ff])

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Cleaning up used
        for identifier in delete_identifiers:
            if self.path_identifiers.get(identifier):
                del self.path_identifiers[identifier]
        del delete_identifiers

        # Determine file types
        processed_movies = {}
        while True and not self.shuttingDown():
            try:
                identifier, group = movie_files.popitem()
            except:
                break

            # Check if movie is fresh and maybe still unpacking, ignore files new then 1 minute
            file_too_new = False
            for cur_file in group['unsorted_files']:
                if not os.path.isfile(cur_file):
                    file_too_new = time.time()
                    break
                file_time = [os.path.getmtime(cur_file), os.path.getctime(cur_file)]
                for t in file_time:
                    if t > time.time() - 60:
                        file_too_new = tryInt(time.time() - t)
                        break

                if file_too_new:
                    break

            if file_too_new:
                log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s', (time.ctime(file_time[0]), identifier))

                # Delete the unsorted list
                del group['unsorted_files']

                continue

            # Only process movies newer than x
            if newer_than and newer_than > 0:
                for cur_file in group['unsorted_files']:
                    file_time = [os.path.getmtime(cur_file), os.path.getctime(cur_file)]
                    if file_time[0] > time.time() or file_time[1] > time.time():
                        break

                log.debug('None of the files have changed since %s for %s, skipping.', (time.ctime(newer_than), identifier))

                # Delete the unsorted list
                del group['unsorted_files']

                continue

            # Group extra (and easy) files first
            # images = self.getImages(group['unsorted_files'])
            group['files'] = {
                'movie_extra': self.getMovieExtras(group['unsorted_files']),
                'subtitle': self.getSubtitles(group['unsorted_files']),
                'subtitle_extra': self.getSubtitlesExtras(group['unsorted_files']),
                'nfo': self.getNfo(group['unsorted_files']),
                'trailer': self.getTrailers(group['unsorted_files']),
                #'backdrop': images['backdrop'],
                'leftover': set(group['unsorted_files']),
            }

            # Media files
            if group['is_dvd']:
                group['files']['movie'] = self.getDVDFiles(group['unsorted_files'])
            else:
                group['files']['movie'] = self.getMediaFiles(group['unsorted_files'])

            if len(group['files']['movie']) == 0:
                log.error('Couldn\t find any movie files for %s', identifier)
                continue

            log.debug('Getting metadata for %s', identifier)
            group['meta_data'] = self.getMetaData(group)

            # Subtitle meta
            group['subtitle_language'] = self.getSubtitleLanguage(group) if not simple else {}

            # Get parent dir from movie files
            for movie_file in group['files']['movie']:
                group['parentdir'] = os.path.dirname(movie_file)
                group['dirname'] = None

                folder_names = group['parentdir'].replace(folder, '').split(os.path.sep)
                folder_names.reverse()

                # Try and get a proper dirname, so no "A", "Movie", "Download" etc
                for folder_name in folder_names:
                    if folder_name.lower() not in self.ignore_names and len(folder_name) > 2:
                        group['dirname'] = folder_name
                        break

                break

            # Leftover "sorted" files
            for file_type in group['files']:
                if not file_type is 'leftover':
                    group['files']['leftover'] -= set(group['files'][file_type])

            # Delete the unsorted list
            del group['unsorted_files']

            # Determine movie
            group['library'] = self.determineMovie(group)
            if not group['library']:
                log.error('Unable to determine movie: %s', group['identifiers'])
            else:
                movie = db.query(Movie).filter_by(library_id = group['library']['id']).first()
                group['movie_id'] = None if not movie else movie.id


            processed_movies[identifier] = group


        # Clean up
        self.path_identifiers = {}

        if len(processed_movies) > 0:
            log.info('Found %s movies in the folder %s', (len(processed_movies), folder))
        else:
            log.debug('Found no movies in the folder %s', (folder))
        return processed_movies
Example #40
0
 def test(s):
     return self.filesizeBetween(s, self.file_sizes['movie']) and getExt(s.lower()) in self.extensions['movie'] and not self.isSampleFile(s)
Example #41
0
    def scan(self, folder = None, files = None, release_download = None, simple = False, newer_than = 0, return_ignored = True, check_file_date = True, on_found = None):

        folder = sp(folder)

        if not folder or not os.path.isdir(folder):
            log.error('Folder doesn\'t exists: %s', folder)
            return {}

        # Get movie "master" files
        movie_files = {}
        leftovers = []

        # Scan all files of the folder if no files are set
        if not files:
            try:
                files = []
                for root, dirs, walk_files in os.walk(folder, followlinks=True):
                    files.extend([sp(os.path.join(sp(root), ss(filename))) for filename in walk_files])

                    # Break if CP wants to shut down
                    if self.shuttingDown():
                        break

            except:
                log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))

            log.debug('Found %s files to scan and group in %s', (len(files), folder))
        else:
            check_file_date = False
            files = [sp(x) for x in files]

        for file_path in files:

            if not os.path.exists(file_path):
                continue

            # Remove ignored files
            if self.isSampleFile(file_path):
                leftovers.append(file_path)
                continue
            elif not self.keepFile(file_path):
                continue

            is_dvd_file = self.isDVDFile(file_path)
            if self.filesizeBetween(file_path, self.file_sizes['movie']) or is_dvd_file: # Minimal 300MB files or is DVD file

                # Normal identifier
                identifier = self.createStringIdentifier(file_path, folder, exclude_filename = is_dvd_file)
                identifiers = [identifier]

                # Identifier with quality
                quality = fireEvent('quality.guess', files = [file_path], size = self.getFileSize(file_path), single = True) if not is_dvd_file else {'identifier':'dvdr'}
                if quality:
                    identifier_with_quality = '%s %s' % (identifier, quality.get('identifier', ''))
                    identifiers = [identifier_with_quality, identifier]

                if not movie_files.get(identifier):
                    movie_files[identifier] = {
                        'unsorted_files': [],
                        'identifiers': identifiers,
                        'is_dvd': is_dvd_file,
                    }

                movie_files[identifier]['unsorted_files'].append(file_path)
            else:
                leftovers.append(file_path)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Cleanup
        del files

        # Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2"
        # files will be grouped first.
        leftovers = set(sorted(leftovers, reverse = True))

        # Group files minus extension
        ignored_identifiers = []
        for identifier, group in movie_files.items():
            if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier)

            log.debug('Grouping files: %s', identifier)

            has_ignored = 0
            for file_path in list(group['unsorted_files']):
                ext = getExt(file_path)
                wo_ext = file_path[:-(len(ext) + 1)]
                found_files = set([i for i in leftovers if wo_ext in i])
                group['unsorted_files'].extend(found_files)
                leftovers = leftovers - found_files

                has_ignored += 1 if ext == 'ignore' else 0

            if has_ignored == 0:
                for file_path in list(group['unsorted_files']):
                    ext = getExt(file_path)
                    has_ignored += 1 if ext == 'ignore' else 0

            if has_ignored > 0:
                ignored_identifiers.append(identifier)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break


        # Create identifiers for all leftover files
        path_identifiers = {}
        for file_path in leftovers:
            identifier = self.createStringIdentifier(file_path, folder)

            if not path_identifiers.get(identifier):
                path_identifiers[identifier] = []

            path_identifiers[identifier].append(file_path)


        # Group the files based on the identifier
        delete_identifiers = []
        for identifier, found_files in path_identifiers.items():
            log.debug('Grouping files on identifier: %s', identifier)

            group = movie_files.get(identifier)
            if group:
                group['unsorted_files'].extend(found_files)
                delete_identifiers.append(identifier)

                # Remove the found files from the leftover stack
                leftovers = leftovers - set(found_files)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Cleaning up used
        for identifier in delete_identifiers:
            if path_identifiers.get(identifier):
                del path_identifiers[identifier]
        del delete_identifiers

        # Group based on folder
        delete_identifiers = []
        for identifier, found_files in path_identifiers.items():
            log.debug('Grouping files on foldername: %s', identifier)

            for ff in found_files:
                new_identifier = self.createStringIdentifier(os.path.dirname(ff), folder)

                group = movie_files.get(new_identifier)
                if group:
                    group['unsorted_files'].extend([ff])
                    delete_identifiers.append(identifier)

                    # Remove the found files from the leftover stack
                    leftovers -= leftovers - set([ff])

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # leftovers should be empty
        if leftovers:
            log.debug('Some files are still left over: %s', leftovers)

        # Cleaning up used
        for identifier in delete_identifiers:
            if path_identifiers.get(identifier):
                del path_identifiers[identifier]
        del delete_identifiers

        # Make sure we remove older / still extracting files
        valid_files = {}
        while True and not self.shuttingDown():
            try:
                identifier, group = movie_files.popitem()
            except:
                break

            # Check if movie is fresh and maybe still unpacking, ignore files newer than 1 minute
            if check_file_date:
                files_too_new, time_string = self.checkFilesChanged(group['unsorted_files'])
                if files_too_new:
                    log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s', (time_string, identifier))

                    # Delete the unsorted list
                    del group['unsorted_files']

                    continue

            # Only process movies newer than x
            if newer_than and newer_than > 0:
                has_new_files = False
                for cur_file in group['unsorted_files']:
                    file_time = self.getFileTimes(cur_file)
                    if file_time[0] > newer_than or file_time[1] > newer_than:
                        has_new_files = True
                        break

                if not has_new_files:
                    log.debug('None of the files have changed since %s for %s, skipping.', (time.ctime(newer_than), identifier))

                    # Delete the unsorted list
                    del group['unsorted_files']

                    continue

            valid_files[identifier] = group

        del movie_files

        total_found = len(valid_files)

        # Make sure only one movie was found if a download ID is provided
        if release_download and total_found == 0:
            log.info('Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).', release_download.get('imdb_id'))
        elif release_download and total_found > 1:
            log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (release_download.get('imdb_id'), len(valid_files)))
            release_download = None

        # Determine file types
        processed_movies = {}
        while True and not self.shuttingDown():
            try:
                identifier, group = valid_files.popitem()
            except:
                break

            if return_ignored is False and identifier in ignored_identifiers:
                log.debug('Ignore file found, ignoring release: %s', identifier)
                total_found -= 1
                continue

            # Group extra (and easy) files first
            group['files'] = {
                'movie_extra': self.getMovieExtras(group['unsorted_files']),
                'subtitle': self.getSubtitles(group['unsorted_files']),
                'subtitle_extra': self.getSubtitlesExtras(group['unsorted_files']),
                'nfo': self.getNfo(group['unsorted_files']),
                'trailer': self.getTrailers(group['unsorted_files']),
                'leftover': set(group['unsorted_files']),
            }

            # Media files
            if group['is_dvd']:
                group['files']['movie'] = self.getDVDFiles(group['unsorted_files'])
            else:
                group['files']['movie'] = self.getMediaFiles(group['unsorted_files'])

            if len(group['files']['movie']) == 0:
                log.error('Couldn\'t find any movie files for %s', identifier)
                total_found -= 1
                continue

            log.debug('Getting metadata for %s', identifier)
            group['meta_data'] = self.getMetaData(group, folder = folder, release_download = release_download)

            # Subtitle meta
            group['subtitle_language'] = self.getSubtitleLanguage(group) if not simple else {}

            # Get parent dir from movie files
            for movie_file in group['files']['movie']:
                group['parentdir'] = os.path.dirname(movie_file)
                group['dirname'] = None

                folder_names = group['parentdir'].replace(folder, '').split(os.path.sep)
                folder_names.reverse()

                # Try and get a proper dirname, so no "A", "Movie", "Download" etc
                for folder_name in folder_names:
                    if folder_name.lower() not in self.ignore_names and len(folder_name) > 2:
                        group['dirname'] = folder_name
                        break

                break

            # Leftover "sorted" files
            for file_type in group['files']:
                if not file_type is 'leftover':
                    group['files']['leftover'] -= set(group['files'][file_type])
                    group['files'][file_type] = list(group['files'][file_type])
            group['files']['leftover'] = list(group['files']['leftover'])

            # Delete the unsorted list
            del group['unsorted_files']

            # Determine movie
            group['media'] = self.determineMedia(group, release_download = release_download)
            if not group['media']:
                log.error('Unable to determine media: %s', group['identifiers'])
            else:
                group['identifier'] = getIdentifier(group['media']) or group['media']['info'].get('imdb')

            processed_movies[identifier] = group

            # Notify parent & progress on something found
            if on_found:
                on_found(group, total_found, len(valid_files))

            # Wait for all the async events calm down a bit
            while threading.activeCount() > 100 and not self.shuttingDown():
                log.debug('Too many threads active, waiting a few seconds')
                time.sleep(10)

        if len(processed_movies) > 0:
            log.info('Found %s movies in the folder %s', (len(processed_movies), folder))
        else:
            log.debug('Found no movies in the folder %s', folder)

        return processed_movies
Example #42
0
 def test(s):
     return self.filesizeBetween(s, 300, 100000) and getExt(
         s.lower()) in self.extensions['movie']
Example #43
0
    def scan(self, movie_folder = None, download_info = None):

        if self.isDisabled():
            return

        if self.renaming_started is True:
            log.info('Renamer is already running, if you see this often, check the logs above for errors.')
            return

        # Check to see if the "to" folder is inside the "from" folder.
        if movie_folder and not os.path.isdir(movie_folder) or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')):
            l = log.debug if movie_folder else log.error
            l('Both the "To" and "From" have to exist.')
            return
        elif self.conf('from') in self.conf('to'):
            log.error('The "to" can\'t be inside of the "from" folder. You\'ll get an infinite loop.')
            return
        elif (movie_folder and movie_folder in [self.conf('to'), self.conf('from')]):
            log.error('The "to" and "from" folders can\'t be inside of or the same as the provided movie folder.')
            return

        self.renaming_started = True

        # make sure the movie folder name is included in the search
        folder = None
        files = []
        if movie_folder:
            log.info('Scanning movie folder %s...', movie_folder)
            movie_folder = movie_folder.rstrip(os.path.sep)
            folder = os.path.dirname(movie_folder)

            # Get all files from the specified folder
            try:
                for root, folders, names in os.walk(movie_folder):
                    files.extend([os.path.join(root, name) for name in names])
            except:
                log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc()))

        db = get_session()

        # Extend the download info with info stored in the downloaded release
        download_info = self.extendDownloadInfo(download_info)

        groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'),
                           files = files, download_info = download_info, return_ignored = False, single = True)

        destination = self.conf('to')
        folder_name = self.conf('folder_name')
        file_name = self.conf('file_name')
        trailer_name = self.conf('trailer_name')
        nfo_name = self.conf('nfo_name')
        separator = self.conf('separator')

        # Statusses
        done_status, active_status, downloaded_status, snatched_status = \
            fireEvent('status.get', ['done', 'active', 'downloaded', 'snatched'], single = True)

        for group_identifier in groups:

            group = groups[group_identifier]
            rename_files = {}
            remove_files = []
            remove_releases = []

            movie_title = getTitle(group['library'])

            # Add _UNKNOWN_ if no library item is connected
            if not group['library'] or not movie_title:
                self.tagDir(group, 'unknown')
                continue
            # Rename the files using the library data
            else:
                group['library'] = fireEvent('library.update', identifier = group['library']['identifier'], single = True)
                if not group['library']:
                    log.error('Could not rename, no library item to work with: %s', group_identifier)
                    continue

                library = group['library']
                movie_title = getTitle(library)

                # Find subtitle for renaming
                fireEvent('renamer.before', group)

                # Remove weird chars from moviename
                movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', movie_title)

                # Put 'The' at the end
                name_the = movie_name
                if movie_name[:4].lower() == 'the ':
                    name_the = movie_name[4:] + ', The'

                replacements = {
                     'ext': 'mkv',
                     'namethe': name_the.strip(),
                     'thename': movie_name.strip(),
                     'year': library['year'],
                     'first': name_the[0].upper(),
                     'quality': group['meta_data']['quality']['label'],
                     'quality_type': group['meta_data']['quality_type'],
                     'video': group['meta_data'].get('video'),
                     'audio': group['meta_data'].get('audio'),
                     'group': group['meta_data']['group'],
                     'source': group['meta_data']['source'],
                     'resolution_width': group['meta_data'].get('resolution_width'),
                     'resolution_height': group['meta_data'].get('resolution_height'),
                     'imdb_id': library['identifier'],
                     'cd': '',
                     'cd_nr': '',
                }

                for file_type in group['files']:

                    # Move nfo depending on settings
                    if file_type is 'nfo' and not self.conf('rename_nfo'):
                        log.debug('Skipping, renaming of %s disabled', file_type)
                        if self.conf('cleanup'):
                            for current_file in group['files'][file_type]:
                                remove_files.append(current_file)
                        continue

                    # Subtitle extra
                    if file_type is 'subtitle_extra':
                        continue

                    # Move other files
                    multiple = len(group['files'][file_type]) > 1 and not group['is_dvd']
                    cd = 1 if multiple else 0

                    for current_file in sorted(list(group['files'][file_type])):
                        current_file = toUnicode(current_file)

                        # Original filename
                        replacements['original'] = os.path.splitext(os.path.basename(current_file))[0]
                        replacements['original_folder'] = fireEvent('scanner.remove_cptag', group['dirname'], single = True)

                        # Extension
                        replacements['ext'] = getExt(current_file)

                        # cd #
                        replacements['cd'] = ' cd%d' % cd if multiple else ''
                        replacements['cd_nr'] = cd if multiple else ''

                        # Naming
                        final_folder_name = self.doReplace(folder_name, replacements).lstrip('. ')
                        final_file_name = self.doReplace(file_name, replacements).lstrip('. ')
                        replacements['filename'] = final_file_name[:-(len(getExt(final_file_name)) + 1)]

                        # Meta naming
                        if file_type is 'trailer':
                            final_file_name = self.doReplace(trailer_name, replacements, remove_multiple = True).lstrip('. ')
                        elif file_type is 'nfo':
                            final_file_name = self.doReplace(nfo_name, replacements, remove_multiple = True).lstrip('. ')

                        # Seperator replace
                        if separator:
                            final_file_name = final_file_name.replace(' ', separator)

                        # Move DVD files (no structure renaming)
                        if group['is_dvd'] and file_type is 'movie':
                            found = False
                            for top_dir in ['video_ts', 'audio_ts', 'bdmv', 'certificate']:
                                has_string = current_file.lower().find(os.path.sep + top_dir + os.path.sep)
                                if has_string >= 0:
                                    structure_dir = current_file[has_string:].lstrip(os.path.sep)
                                    rename_files[current_file] = os.path.join(destination, final_folder_name, structure_dir)
                                    found = True
                                    break

                            if not found:
                                log.error('Could not determine dvd structure for: %s', current_file)

                        # Do rename others
                        else:
                            if file_type is 'leftover':
                                if self.conf('move_leftover'):
                                    rename_files[current_file] = os.path.join(destination, final_folder_name, os.path.basename(current_file))
                            elif file_type not in ['subtitle']:
                                rename_files[current_file] = os.path.join(destination, final_folder_name, final_file_name)

                        # Check for extra subtitle files
                        if file_type is 'subtitle':

                            remove_multiple = False
                            if len(group['files']['movie']) == 1:
                                remove_multiple = True

                            sub_langs = group['subtitle_language'].get(current_file, [])

                            # rename subtitles with or without language
                            sub_name = self.doReplace(file_name, replacements, remove_multiple = remove_multiple)
                            rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)

                            rename_extras = self.getRenameExtras(
                                extra_type = 'subtitle_extra',
                                replacements = replacements,
                                folder_name = folder_name,
                                file_name = file_name,
                                destination = destination,
                                group = group,
                                current_file = current_file,
                                remove_multiple = remove_multiple,
                            )

                            # Don't add language if multiple languages in 1 subtitle file
                            if len(sub_langs) == 1:
                                sub_name = final_file_name.replace(replacements['ext'], '%s.%s' % (sub_langs[0], replacements['ext']))
                                rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)

                            rename_files = mergeDicts(rename_files, rename_extras)

                        # Filename without cd etc
                        elif file_type is 'movie':
                            rename_extras = self.getRenameExtras(
                                extra_type = 'movie_extra',
                                replacements = replacements,
                                folder_name = folder_name,
                                file_name = file_name,
                                destination = destination,
                                group = group,
                                current_file = current_file
                            )
                            rename_files = mergeDicts(rename_files, rename_extras)

                            group['filename'] = self.doReplace(file_name, replacements, remove_multiple = True)[:-(len(getExt(final_file_name)) + 1)]
                            group['destination_dir'] = os.path.join(destination, final_folder_name)

                        if multiple:
                            cd += 1

                # Before renaming, remove the lower quality files
                library = db.query(Library).filter_by(identifier = group['library']['identifier']).first()
                remove_leftovers = True

                # Add it to the wanted list before we continue
                if len(library.movies) == 0:
                    profile = db.query(Profile).filter_by(core = True, label = group['meta_data']['quality']['label']).first()
                    fireEvent('movie.add', params = {'identifier': group['library']['identifier'], 'profile_id': profile.id}, search_after = False)
                    db.expire_all()
                    library = db.query(Library).filter_by(identifier = group['library']['identifier']).first()

                for movie in library.movies:

                    # Mark movie "done" onces it found the quality with the finish check
                    try:
                        if movie.status_id == active_status.get('id') and movie.profile:
                            for profile_type in movie.profile.types:
                                if profile_type.quality_id == group['meta_data']['quality']['id'] and profile_type.finish:
                                    movie.status_id = done_status.get('id')
                                    movie.last_edit = int(time.time())
                                    db.commit()
                    except Exception, e:
                        log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc()))

                    # Go over current movie releases
                    for release in movie.releases:

                        # When a release already exists
                        if release.status_id is done_status.get('id'):

                            # This is where CP removes older, lesser quality releases
                            if release.quality.order > group['meta_data']['quality']['order']:
                                log.info('Removing lesser quality %s for %s.', (movie.library.titles[0].title, release.quality.label))
                                for current_file in release.files:
                                    remove_files.append(current_file)
                                remove_releases.append(release)
                            # Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc
                            elif release.quality.order is group['meta_data']['quality']['order']:
                                log.info('Same quality release already exists for %s, with quality %s. Assuming repack.', (movie.library.titles[0].title, release.quality.label))
                                for current_file in release.files:
                                    remove_files.append(current_file)
                                remove_releases.append(release)

                            # Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan
                            else:
                                log.info('Better quality release already exists for %s, with quality %s', (movie.library.titles[0].title, release.quality.label))

                                # Add exists tag to the .ignore file
                                self.tagDir(group, 'exists')

                                # Notify on rename fail
                                download_message = 'Renaming of %s (%s) canceled, exists in %s already.' % (movie.library.titles[0].title, group['meta_data']['quality']['label'], release.quality.label)
                                fireEvent('movie.renaming.canceled', message = download_message, data = group)
                                remove_leftovers = False

                                break
                        elif release.status_id is snatched_status.get('id'):
                            if release.quality.id is group['meta_data']['quality']['id']:
                                log.debug('Marking release as downloaded')
                                try:
                                    release.status_id = downloaded_status.get('id')
                                    release.last_edit = int(time.time())
                                except Exception, e:
                                    log.error('Failed marking release as finished: %s %s', (e, traceback.format_exc()))

                                db.commit()

                # Remove leftover files
                if self.conf('cleanup') and not self.conf('move_leftover') and remove_leftovers and \
                        not (self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info)):
                    log.debug('Removing leftover files')
                    for current_file in group['files']['leftover']:
                        remove_files.append(current_file)
                elif not remove_leftovers: # Don't remove anything
                    break
Example #44
0
 def getMovieExtras(self, files):
     return set(filter(lambda s: getExt(s.lower()) in self.extensions['movie_extra'], files))
Example #45
0
    def guess(self, files, extra=None, size=None):
        if not extra: extra = {}

        # Create hash for cache
        cache_key = str([
            f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f
            for f in files
        ])
        cached = self.getCache(cache_key)
        if cached and len(extra) == 0:
            return cached

        qualities = self.all()

        # Start with 0
        score = {}
        for quality in qualities:
            score[quality.get('identifier')] = {'score': 0, '3d': {}}

        for cur_file in files:
            words = re.split('\W+', cur_file.lower())

            for quality in qualities:
                contains_score = self.containsTagScore(quality, words,
                                                       cur_file)
                threedscore = self.contains3D(
                    quality, words,
                    cur_file) if quality.get('allow_3d') else (0, None)

                self.calcScore(score, quality, contains_score, threedscore)

        size_scores = []
        for quality in qualities:

            # Evaluate score based on size
            size_score = self.guessSizeScore(quality, size=size)
            loose_score = self.guessLooseScore(quality, extra=extra)

            if size_score > 0:
                size_scores.append(quality)

            self.calcScore(score,
                           quality,
                           size_score + loose_score,
                           penalty=False)

        # Add additional size score if only 1 size validated
        if len(size_scores) == 1:
            self.calcScore(score, size_scores[0], 10, penalty=False)
        del size_scores

        # Return nothing if all scores are <= 0
        has_non_zero = 0
        for s in score:
            if score[s]['score'] > 0:
                has_non_zero += 1

        if not has_non_zero:
            return None

        heighest_quality = max(score, key=lambda p: score[p]['score'])
        if heighest_quality:
            for quality in qualities:
                if quality.get('identifier') == heighest_quality:
                    quality['is_3d'] = False
                    if score[heighest_quality].get('3d'):
                        quality['is_3d'] = True
                    return self.setCache(cache_key, quality)

        return None
Example #46
0
 def getSubtitlesExtras(self, files):
     return set(filter(lambda s: getExt(s.lower()) in self.extensions['subtitle_extra'], files))
Example #47
0
 def test(s):
     return self.filesizeBetween(s, 300, 100000) and getExt(s.lower()) in self.extensions['movie'] and not self.isSampleFile(s)
Example #48
0
 def getNfo(self, files):
     return set(filter(lambda s: getExt(s.lower()) in self.extensions['nfo'], files))
Example #49
0
 def getSubtitlesExtras(self, files):
     return set(filter(lambda s: getExt(s.lower()) in self.extensions['subtitle_extra'], files))
Example #50
0
 def test(s):
     return getExt(s.lower()) in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tbn']
Example #51
0
 def test(s):
     return getExt(s.lower()) in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tbn']
Example #52
0
    def scan(self,
             folder=None,
             files=None,
             simple=False,
             newer_than=0,
             on_found=None):

        folder = ss(os.path.normpath(folder))

        if not folder or not os.path.isdir(folder):
            log.error('Folder doesn\'t exists: %s', folder)
            return {}

        # Get movie "master" files
        movie_files = {}
        leftovers = []

        # Scan all files of the folder if no files are set
        if not files:
            check_file_date = True
            try:
                files = []
                for root, dirs, walk_files in os.walk(folder):
                    for filename in walk_files:
                        files.append(os.path.join(root, filename))
            except:
                log.error('Failed getting files from %s: %s',
                          (folder, traceback.format_exc()))
        else:
            check_file_date = False
            files = [ss(x) for x in files]

        db = get_session()

        for file_path in files:

            if not os.path.exists(file_path):
                continue

            # Remove ignored files
            if self.isSampleFile(file_path):
                leftovers.append(file_path)
                continue
            elif not self.keepFile(file_path):
                continue

            is_dvd_file = self.isDVDFile(file_path)
            if os.path.getsize(file_path) > self.minimal_filesize[
                    'media'] or is_dvd_file:  # Minimal 300MB files or is DVD file

                # Normal identifier
                identifier = self.createStringIdentifier(
                    file_path, folder, exclude_filename=is_dvd_file)
                identifiers = [identifier]

                # Identifier with quality
                quality = fireEvent('quality.guess', [file_path],
                                    single=True) if not is_dvd_file else {
                                        'identifier': 'dvdr'
                                    }
                if quality:
                    identifier_with_quality = '%s %s' % (
                        identifier, quality.get('identifier', ''))
                    identifiers = [identifier_with_quality, identifier]

                if not movie_files.get(identifier):
                    movie_files[identifier] = {
                        'unsorted_files': [],
                        'identifiers': identifiers,
                        'is_dvd': is_dvd_file,
                    }

                movie_files[identifier]['unsorted_files'].append(file_path)
            else:
                leftovers.append(file_path)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Cleanup
        del files

        # Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2"
        # files will be grouped first.
        leftovers = set(sorted(leftovers, reverse=True))

        # Group files minus extension
        for identifier, group in movie_files.iteritems():
            if identifier not in group['identifiers'] and len(identifier) > 0:
                group['identifiers'].append(identifier)

            log.debug('Grouping files: %s', identifier)

            for file_path in group['unsorted_files']:
                wo_ext = file_path[:-(len(getExt(file_path)) + 1)]
                found_files = set([i for i in leftovers if wo_ext in i])
                group['unsorted_files'].extend(found_files)
                leftovers = leftovers - found_files

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Create identifiers for all leftover files
        path_identifiers = {}
        for file_path in leftovers:
            identifier = self.createStringIdentifier(file_path, folder)

            if not path_identifiers.get(identifier):
                path_identifiers[identifier] = []

            path_identifiers[identifier].append(file_path)

        # Group the files based on the identifier
        delete_identifiers = []
        for identifier, found_files in path_identifiers.iteritems():
            log.debug('Grouping files on identifier: %s', identifier)

            group = movie_files.get(identifier)
            if group:
                group['unsorted_files'].extend(found_files)
                delete_identifiers.append(identifier)

                # Remove the found files from the leftover stack
                leftovers = leftovers - set(found_files)

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Cleaning up used
        for identifier in delete_identifiers:
            if path_identifiers.get(identifier):
                del path_identifiers[identifier]
        del delete_identifiers

        # Group based on folder
        delete_identifiers = []
        for identifier, found_files in path_identifiers.iteritems():
            log.debug('Grouping files on foldername: %s', identifier)

            for ff in found_files:
                new_identifier = self.createStringIdentifier(
                    os.path.dirname(ff), folder)

                group = movie_files.get(new_identifier)
                if group:
                    group['unsorted_files'].extend([ff])
                    delete_identifiers.append(identifier)

                    # Remove the found files from the leftover stack
                    leftovers = leftovers - set([ff])

            # Break if CP wants to shut down
            if self.shuttingDown():
                break

        # Cleaning up used
        for identifier in delete_identifiers:
            if path_identifiers.get(identifier):
                del path_identifiers[identifier]
        del delete_identifiers

        # Make sure we remove older / still extracting files
        valid_files = {}
        while True and not self.shuttingDown():
            try:
                identifier, group = movie_files.popitem()
            except:
                break

            # Check if movie is fresh and maybe still unpacking, ignore files new then 1 minute
            file_too_new = False
            for cur_file in group['unsorted_files']:
                if not os.path.isfile(cur_file):
                    file_too_new = time.time()
                    break
                file_time = [
                    os.path.getmtime(cur_file),
                    os.path.getctime(cur_file)
                ]
                for t in file_time:
                    if t > time.time() - 60:
                        file_too_new = tryInt(time.time() - t)
                        break

                if file_too_new:
                    break

            if check_file_date and file_too_new:
                try:
                    time_string = time.ctime(file_time[0])
                except:
                    try:
                        time_string = time.ctime(file_time[1])
                    except:
                        time_string = 'unknown'

                log.info(
                    'Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s',
                    (time_string, identifier))

                # Delete the unsorted list
                del group['unsorted_files']

                continue

            # Only process movies newer than x
            if newer_than and newer_than > 0:
                has_new_files = False
                for cur_file in group['unsorted_files']:
                    file_time = [
                        os.path.getmtime(cur_file),
                        os.path.getctime(cur_file)
                    ]
                    if file_time[0] > newer_than or file_time[1] > newer_than:
                        has_new_files = True
                        break

                if not has_new_files:
                    log.debug(
                        'None of the files have changed since %s for %s, skipping.',
                        (time.ctime(newer_than), identifier))

                    # Delete the unsorted list
                    del group['unsorted_files']

                    continue

            valid_files[identifier] = group

        del movie_files

        # Determine file types
        processed_movies = {}
        total_found = len(valid_files)
        while True and not self.shuttingDown():
            try:
                identifier, group = valid_files.popitem()
            except:
                break

            # Group extra (and easy) files first
            # images = self.getImages(group['unsorted_files'])
            group['files'] = {
                'movie_extra': self.getMovieExtras(group['unsorted_files']),
                'subtitle': self.getSubtitles(group['unsorted_files']),
                'subtitle_extra':
                self.getSubtitlesExtras(group['unsorted_files']),
                'nfo': self.getNfo(group['unsorted_files']),
                'trailer': self.getTrailers(group['unsorted_files']),
                #'backdrop': images['backdrop'],
                'leftover': set(group['unsorted_files']),
            }

            # Media files
            if group['is_dvd']:
                group['files']['movie'] = self.getDVDFiles(
                    group['unsorted_files'])
            else:
                group['files']['movie'] = self.getMediaFiles(
                    group['unsorted_files'])

            if len(group['files']['movie']) == 0:
                log.error('Couldn\'t find any movie files for %s', identifier)
                continue

            log.debug('Getting metadata for %s', identifier)
            group['meta_data'] = self.getMetaData(group, folder=folder)

            # Subtitle meta
            group['subtitle_language'] = self.getSubtitleLanguage(
                group) if not simple else {}

            # Get parent dir from movie files
            for movie_file in group['files']['movie']:
                group['parentdir'] = os.path.dirname(movie_file)
                group['dirname'] = None

                folder_names = group['parentdir'].replace(folder, '').split(
                    os.path.sep)
                folder_names.reverse()

                # Try and get a proper dirname, so no "A", "Movie", "Download" etc
                for folder_name in folder_names:
                    if folder_name.lower(
                    ) not in self.ignore_names and len(folder_name) > 2:
                        group['dirname'] = folder_name
                        break

                break

            # Leftover "sorted" files
            for file_type in group['files']:
                if not file_type is 'leftover':
                    group['files']['leftover'] -= set(
                        group['files'][file_type])

            # Delete the unsorted list
            del group['unsorted_files']

            # Determine movie
            group['library'] = self.determineMovie(group)
            if not group['library']:
                log.error('Unable to determine movie: %s',
                          group['identifiers'])
            else:
                movie = db.query(Movie).filter_by(
                    library_id=group['library']['id']).first()
                group['movie_id'] = None if not movie else movie.id

            processed_movies[identifier] = group

            # Notify parent & progress on something found
            if on_found:
                on_found(group, total_found,
                         total_found - len(processed_movies))

        if len(processed_movies) > 0:
            log.info('Found %s movies in the folder %s',
                     (len(processed_movies), folder))
        else:
            log.debug('Found no movies in the folder %s', (folder))

        return processed_movies