def guess(self, files, extra = None, size = None): if not extra: extra = {} # Create hash for cache cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files]) cached = self.getCache(cache_key) if cached and len(extra) == 0: return cached qualities = self.all() # Start with 0 score = {} for quality in qualities: score[quality.get('identifier')] = { 'score': 0, '3d': {} } for cur_file in files: words = re.split('\W+', cur_file.lower()) for quality in qualities: contains_score = self.containsTagScore(quality, words, cur_file) threedscore = self.contains3D(quality, words, cur_file) if quality.get('allow_3d') else (0, None) self.calcScore(score, quality, contains_score, threedscore) # Evaluate score based on size for quality in qualities: size_score = self.guessSizeScore(quality, size = size) self.calcScore(score, quality, size_score, penalty = False) # Try again with loose testing for quality in qualities: loose_score = self.guessLooseScore(quality, extra = extra) self.calcScore(score, quality, loose_score, penalty = False) # Return nothing if all scores are <= 0 has_non_zero = 0 for s in score: if score[s]['score'] > 0: has_non_zero += 1 if not has_non_zero: return None heighest_quality = max(score, key = lambda p: score[p]['score']) if heighest_quality: for quality in qualities: if quality.get('identifier') == heighest_quality: quality['is_3d'] = False if score[heighest_quality].get('3d'): quality['is_3d'] = True return self.setCache(cache_key, quality) return None
def guess(self, files, extra = None): if not extra: extra = {} # Create hash for cache # cache_key = md5(str([f.replace('.' + getExt(f), '') for f in files])) # cached = self.getCache(cache_key) # if cached and len(extra) == 0: return cached # Create hash for cache cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files]) cached = self.getCache(cache_key) if cached and len(extra) == 0: return cached qualities = self.all() # Start with 0 score = {} for quality in qualities: score[quality.get('identifier')] = 0 for cur_file in files: words = re.split('[\W\-]+', cur_file.lower(), flags=re.U) for quality in qualities: contains_score = self.containsTagScore(quality, words, cur_file) self.calcScore(score, quality, contains_score) # Try again with loose testing for quality in qualities: loose_score = self.guessLooseScore(quality, files = files, extra = extra) self.calcScore(score, quality, loose_score) # Return nothing if all scores are 0 has_non_zero = 0 for s in score: if score[s] > 0: has_non_zero += 1 if not has_non_zero: return None heighest_quality = max(score, key = score.get) if heighest_quality: for quality in qualities: if quality.get('identifier') == heighest_quality: return self.setCache(cache_key, quality) return None
def searchSingle(self, message = None, group = None): if not group: group = {} if self.isDisabled() or len(group['files']['trailer']) > 0: return trailers = fireEvent('trailer.search', group = group, merge = True) if not trailers or trailers == []: log.info('No trailers found for: %s', getTitle(group)) return False for trailer in trailers.get(self.conf('quality'), []): ext = getExt(trailer) filename = self.conf('name').replace('<filename>', group['filename']) + ('.%s' % ('mp4' if len(ext) > 5 else ext)) destination = os.path.join(group['destination_dir'], filename) if not os.path.isfile(destination): trailer_file = fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True) if os.path.getsize(trailer_file) < (1024 * 1024): # Don't trust small trailers (1MB), try next one os.unlink(trailer_file) continue else: log.debug('Trailer already exists: %s', destination) group['renamed_files'].append(destination) # Download first and break break return True
def registerStatic(self, plugin_file, add_to_head = True): # Register plugin path self.plugin_path = os.path.dirname(plugin_file) static_folder = toUnicode(os.path.join(self.plugin_path, 'static')) if not os.path.isdir(static_folder): return # Get plugin_name from PluginName s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__) class_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() # View path path = 'static/plugin/%s/' % class_name # Add handler to Tornado Env.get('app').add_handlers(".*$", [(Env.get('web_base') + path + '(.*)', StaticFileHandler, {'path': static_folder})]) # Register for HTML <HEAD> if add_to_head: for f in glob.glob(os.path.join(self.plugin_path, 'static', '*')): ext = getExt(f) if ext in ['js', 'css']: fireEvent('register_%s' % ('script' if ext in 'js' else 'style'), path + os.path.basename(f), f)
def registerStatic(self, plugin_file, add_to_head=True): # Register plugin path self.plugin_path = os.path.dirname(plugin_file) static_folder = toUnicode(os.path.join(self.plugin_path, "static")) if not os.path.isdir(static_folder): return # Get plugin_name from PluginName s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", self.__class__.__name__) class_name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower() # View path path = "static/plugin/%s/" % class_name # Add handler to Tornado Env.get("app").add_handlers( ".*$", [(Env.get("web_base") + path + "(.*)", StaticFileHandler, {"path": static_folder})] ) # Register for HTML <HEAD> if add_to_head: for f in glob.glob(os.path.join(self.plugin_path, "static", "*")): ext = getExt(f) if ext in ["js", "css"]: fireEvent("register_%s" % ("script" if ext in "js" else "style"), path + os.path.basename(f), f)
def guess(self, files, extra = {}, loose = False): # Create hash for cache hash = md5(str([f.replace('.' + getExt(f), '') for f in files])) cached = self.getCache(hash) if cached and extra is {}: return cached for cur_file in files: size = (os.path.getsize(cur_file) / 1024 / 1024) if os.path.isfile(cur_file) else 0 words = re.split('\W+', cur_file.lower()) for quality in self.all(): # Check tags if quality['identifier'] in words: log.debug('Found via identifier "%s" in %s' % (quality['identifier'], cur_file)) return self.setCache(hash, quality) if list(set(quality.get('alternative', [])) & set(words)): log.debug('Found %s via alt %s in %s' % (quality['identifier'], quality.get('alternative'), cur_file)) return self.setCache(hash, quality) for tag in quality.get('tags', []): if isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words): log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), cur_file)) return self.setCache(hash, quality) if list(set(quality.get('tags', [])) & set(words)): log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), cur_file)) return self.setCache(hash, quality) # Check on unreliable stuff if loose: # Last check on resolution only if quality.get('width', 480) == extra.get('resolution_width', 0): log.debug('Found %s via resolution_width: %s == %s' % (quality['identifier'], quality.get('width', 480), extra.get('resolution_width', 0))) return self.setCache(hash, quality) # Check extension + filesize if list(set(quality.get('ext', [])) & set(words)) and size >= quality['size_min'] and size <= quality['size_max']: log.debug('Found %s via ext and filesize %s in %s' % (quality['identifier'], quality.get('ext'), words)) return self.setCache(hash, quality) # Try again with loose testing if not loose: quality = self.guess(files, extra = extra, loose = True) if quality: return self.setCache(hash, quality) log.debug('Could not identify quality for: %s' % files) return None
def download(self, url = '', dest = None, overwrite = False): try: filedata = self.urlopen(url) except: return False if not dest: # to Cache dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) if overwrite or not os.path.isfile(dest): self.createFile(dest, filedata, binary = True) return dest
def download(self, url="", dest=None, overwrite=False): if not dest: # to Cache dest = os.path.join(Env.get("cache_dir"), "%s.%s" % (md5(url), getExt(url))) if not overwrite and os.path.isfile(dest): return dest try: filedata = self.urlopen(url) except: return False self.createFile(dest, filedata, binary=True) return dest
def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = {}): if not dest: # to Cache dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) if not overwrite and os.path.isfile(dest): return dest try: filedata = self.urlopen(url, **urlopen_kwargs) except: return False self.createFile(dest, filedata, binary = True) return dest
def registerStatic(self, plugin_file, add_to_head = True): # Register plugin path self.plugin_path = os.path.dirname(plugin_file) # Get plugin_name from PluginName s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__) class_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() path = 'static/' + class_name + '/' addView(path + '<path:file>', self.showStatic, static = True) if add_to_head: for f in glob.glob(os.path.join(self.plugin_path, 'static', '*')): fireEvent('register_%s' % ('script' if getExt(f) in 'js' else 'style'), path + os.path.basename(f))
def searchSingle(self, group): if self.isDisabled() or len(group['files']['trailer']) > 0: return trailers = fireEvent('trailer.search', group = group, merge = True) for trailer in trailers.get(self.conf('quality'), []): destination = '%s-trailer.%s' % (self.getRootName(group), getExt(trailer)) if not os.path.isfile(destination): fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True) else: log.debug('Trailer already exists: %s' % destination) # Download first and break break
def getRenameExtras(self, extra_type = '', replacements = {}, folder_name = '', file_name = '', destination = '', group = {}, current_file = ''): rename_files = {} def test(s): return current_file[:-len(replacements['ext'])] in s for extra in set(filter(test, group['files'][extra_type])): replacements['ext'] = getExt(extra) final_folder_name = self.doReplace(folder_name, replacements) final_file_name = self.doReplace(file_name, replacements) rename_files[extra] = os.path.join(destination, final_folder_name, final_file_name) return rename_files
def getPoster(self, media, image_urls): if 'files' not in media: media['files'] = {} existing_files = media['files'] image_type = 'poster' file_type = 'image_%s' % image_type # Make existing unique unique_files = list(set(existing_files.get(file_type, []))) # Remove files that can't be found for ef in unique_files: if not os.path.isfile(ef): unique_files.remove(ef) # Replace new files list existing_files[file_type] = unique_files if len(existing_files) == 0: del existing_files[file_type] images = image_urls.get(image_type, []) for y in ['SX300', 'tmdb']: initially_try = [x for x in images if y in x] images[:-1] = initially_try # Loop over type for image in images: if not isinstance(image, (str, unicode)): continue # Check if it has top image filename = '%s.%s' % (md5(image), getExt(image)) existing = existing_files.get(file_type, []) has_latest = False for x in existing: if filename in x: has_latest = True if not has_latest or file_type not in existing_files or len(existing_files.get(file_type, [])) == 0: file_path = fireEvent('file.download', url = image, single = True) if file_path: existing_files[file_type] = [toUnicode(file_path)] break else: break
def guess(self, files, extra={}): # Create hash for cache hash = md5(str([f.replace('.' + getExt(f), '') for f in files])) cached = self.getCache(hash) if cached and extra is {}: return cached for cur_file in files: size = (os.path.getsize(cur_file) / 1024 / 1024) if os.path.isfile(cur_file) else 0 words = re.split('\W+', cur_file.lower()) for quality in self.all(): # Check tags if quality['identifier'] in words: log.debug('Found via identifier "%s" in %s', (quality['identifier'], cur_file)) return self.setCache(hash, quality) if list(set(quality.get('alternative', [])) & set(words)): log.debug('Found %s via alt %s in %s', (quality['identifier'], quality.get('alternative'), cur_file)) return self.setCache(hash, quality) for tag in quality.get('tags', []): if isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words): log.debug('Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file)) return self.setCache(hash, quality) if list(set(quality.get('tags', [])) & set(words)): log.debug( 'Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file)) return self.setCache(hash, quality) # Try again with loose testing quality = self.guessLoose(hash, extra=extra) if quality: return self.setCache(hash, quality) log.debug('Could not identify quality for: %s', files) return None
def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = None): if not urlopen_kwargs: urlopen_kwargs = {} if not dest: # to Cache dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) if not overwrite and os.path.isfile(dest): return dest try: filedata = self.urlopen(url, **urlopen_kwargs) except: log.error('Failed downloading file %s: %s', (url, traceback.format_exc())) return False self.createFile(dest, filedata, binary = True) return dest
def registerStatic(self, plugin_file, add_to_head=True): # Register plugin path self.plugin_path = os.path.dirname(plugin_file) # Get plugin_name from PluginName s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", self.__class__.__name__) class_name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower() path = "api/%s/static/%s/" % (Env.setting("api_key"), class_name) addView(path + "<path:filename>", self.showStatic, static=True) if add_to_head: for f in glob.glob(os.path.join(self.plugin_path, "static", "*")): ext = getExt(f) if ext in ["js", "css"]: fireEvent("register_%s" % ("script" if ext in "js" else "style"), path + os.path.basename(f))
def registerStatic(self, plugin_file, add_to_head = True): # Register plugin path self.plugin_path = os.path.dirname(plugin_file) # Get plugin_name from PluginName s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__) class_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() path = 'api/%s/static/%s/' % (Env.setting('api_key'), class_name) addView(path + '<path:filename>', self.showStatic, static = True) if add_to_head: for f in glob.glob(os.path.join(self.plugin_path, 'static', '*')): ext = getExt(f) if ext in ['js', 'css']: fireEvent('register_%s' % ('script' if ext in 'js' else 'style'), path + os.path.basename(f))
def guess(self, files, extra=None): if not extra: extra = {} # Create hash for cache cache_key = md5(str([f.replace('.' + getExt(f), '') for f in files])) cached = self.getCache(cache_key) if cached and len(extra) == 0: return cached qualities = self.all() # Start with 0 score = {} for quality in qualities: score[quality.get('identifier')] = 0 for cur_file in files: words = re.split('\W+', cur_file.lower()) for quality in qualities: contains_score = self.containsTagScore(quality, words, cur_file) self.calcScore(score, quality, contains_score) # Try again with loose testing for quality in qualities: loose_score = self.guessLooseScore(quality, files=files, extra=extra) self.calcScore(score, quality, loose_score) # Return nothing if all scores are 0 has_non_zero = 0 for s in score: if score[s] > 0: has_non_zero += 1 if not has_non_zero: return None heighest_quality = max(score, key=score.get) if heighest_quality: for quality in qualities: if quality.get('identifier') == heighest_quality: return self.setCache(cache_key, quality) return None
def getRenameExtras(self, extra_type = '', replacements = None, folder_name = '', file_name = '', destination = '', group = None, current_file = '', remove_multiple = False): if not group: group = {} if not replacements: replacements = {} replacements = replacements.copy() rename_files = {} def test(s): return current_file[:-len(replacements['ext'])] in s for extra in set(filter(test, group['files'][extra_type])): replacements['ext'] = getExt(extra) final_folder_name = self.doReplace(folder_name, replacements, remove_multiple = remove_multiple, folder = True) final_file_name = self.doReplace(file_name, replacements, remove_multiple = remove_multiple) rename_files[extra] = os.path.join(destination, final_folder_name, final_file_name) return rename_files
def searchSingle(self, message = None, group = None): if not group: group = {} if self.isDisabled() or len(group['files']['trailer']) > 0: return if self.conf('usevf'): try: filename = self.conf('name').replace('<filename>', group['filename'].decode('latin-1')) except: filename = self.conf('name').replace('<filename>', group['filename']) try: destination = os.path.join(group['destination_dir'].decode('latin-1'), filename) except: destination = os.path.join(group['destination_dir'], filename) trailers = fireEvent('vftrailer.search', group = group, filename=filename, destination=destination, merge = True) else : trailers = fireEvent('trailer.search', group = group, merge = True) if not trailers or trailers == []: log.info('No trailers found for: %s', getTitle(group)) return False if self.conf('usevf'): log.info('Trailers found in VF for: %s', getTitle(group)) return True else: for trailer in trailers.get(self.conf('quality'), []): ext = getExt(trailer) filename = self.conf('name').replace('<filename>', group['filename']) + ('.%s' % ('mp4' if len(ext) > 5 else ext)) destination = os.path.join(group['destination_dir'], filename) if not os.path.isfile(destination): trailer_file = fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True) if os.path.getsize(trailer_file) < (1024 * 1024): # Don't trust small trailers (1MB), try next one os.unlink(trailer_file) continue else: log.debug('Trailer already exists: %s', destination) group['renamed_files'].append(destination) # Download first and break break return True
def getPoster(self, media, image_urls): if 'files' not in media: media['files'] = {} existing_files = media['files'] image_type = 'poster' file_type = 'image_%s' % image_type # Make existing unique unique_files = list(set(existing_files.get(file_type, []))) # Remove files that can't be found for ef in unique_files: if not os.path.isfile(ef): unique_files.remove(ef) # Replace new files list existing_files[file_type] = unique_files if len(existing_files) == 0: del existing_files[file_type] # Loop over type for image in image_urls.get(image_type, []): if not isinstance(image, (str, unicode)): continue # Check if it has top image filename = '%s.%s' % (md5(image), getExt(image)) existing = existing_files.get(file_type, []) has_latest = False for x in existing: if filename in x: has_latest = True if not has_latest or file_type not in existing_files or len( existing_files.get(file_type, [])) == 0: file_path = fireEvent('file.download', url=image, single=True) if file_path: existing_files[file_type] = [toUnicode(file_path)] break else: break
def download(self, url="", dest=None, overwrite=False): file = self.urlopen(url) if not file: log.error("File is empty, don't download") return False if not dest: # to Cache dest = os.path.join(Env.get("cache_dir"), "%s.%s" % (md5(url), getExt(url))) if overwrite or not os.path.exists(dest): log.debug("Writing file to: %s" % dest) output = open(dest, "wb") output.write(file) output.close() else: log.debug("File already exists: %s" % dest) return dest
def searchSingle(self, message = None, group = {}): if self.isDisabled() or len(group['files']['trailer']) > 0: return trailers = fireEvent('trailer.search', group = group, merge = True) if not trailers or trailers == []: log.info('No trailers found for: %s', getTitle(group['library'])) return for trailer in trailers.get(self.conf('quality'), []): destination = '%s-trailer.%s' % (self.getRootName(group), getExt(trailer)) if not os.path.isfile(destination): fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True) else: log.debug('Trailer already exists: %s', destination) group['renamed_files'].append(destination) # Download first and break break
def download(self, url='', dest=None, overwrite=False): file = self.urlopen(url) if not file: log.error('File is empty, don\'t download') return False if not dest: # to Cache dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) if overwrite or not os.path.exists(dest): log.debug('Writing file to: %s' % dest) output = open(dest, 'wb') output.write(file) output.close() else: log.debug('File already exists: %s' % dest) return dest
def download(self, url = '', dest = None, overwrite = False): try: file = urllib2.urlopen(url) if not dest: # to Cache dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) if overwrite or not os.path.exists(dest): log.debug('Writing file to: %s' % dest) output = open(dest, 'wb') output.write(file.read()) output.close() else: log.debug('File already exists: %s' % dest) return dest except Exception, e: log.error('Unable to download file "%s": %s' % (url, e))
def guess(self, files, extra={}): # Create hash for cache hash = md5(str([f.replace("." + getExt(f), "") for f in files])) cached = self.getCache(hash) if cached and extra is {}: return cached for cur_file in files: size = (os.path.getsize(cur_file) / 1024 / 1024) if os.path.isfile(cur_file) else 0 words = re.split("\W+", cur_file.lower()) for quality in self.all(): # Check tags if quality["identifier"] in words: log.debug('Found via identifier "%s" in %s' % (quality["identifier"], cur_file)) return self.setCache(hash, quality) if list(set(quality.get("alternative", [])) & set(words)): log.debug( "Found %s via alt %s in %s" % (quality["identifier"], quality.get("alternative"), cur_file) ) return self.setCache(hash, quality) for tag in quality.get("tags", []): if isinstance(tag, tuple) and ".".join(tag) in ".".join(words): log.debug("Found %s via tag %s in %s" % (quality["identifier"], quality.get("tags"), cur_file)) return self.setCache(hash, quality) if list(set(quality.get("tags", [])) & set(words)): log.debug("Found %s via tag %s in %s" % (quality["identifier"], quality.get("tags"), cur_file)) return self.setCache(hash, quality) # Try again with loose testing quality = self.guessLoose(hash, extra=extra) if quality: return self.setCache(hash, quality) log.debug("Could not identify quality for: %s" % files) return None
def download(self, url='', dest=None, overwrite=False): try: file = urllib2.urlopen(url) if not dest: # to Cache dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) if overwrite or not os.path.exists(dest): log.debug('Writing file to: %s' % dest) output = open(dest, 'wb') output.write(file.read()) output.close() else: log.debug('File already exists: %s' % dest) return dest except Exception, e: log.error('Unable to download file "%s": %s' % (url, e))
def guess(self, files, extra = None): if not extra: extra = {} # Create hash for cache cache_key = md5(str([f.replace('.' + getExt(f), '') for f in files])) cached = self.getCache(cache_key) if cached and len(extra) == 0: return cached qualities = self.all() for cur_file in files: words = re.split('\W+', cur_file.lower()) found = {} for quality in qualities: contains = self.containsTag(quality, words, cur_file) if contains: found[quality['identifier']] = True for quality in qualities: # Check identifier if quality['identifier'] in words: if len(found) == 0 or len(found) == 1 and found.get(quality['identifier']): log.debug('Found via identifier "%s" in %s', (quality['identifier'], cur_file)) return self.setCache(cache_key, quality) # Check alt and tags contains = self.containsTag(quality, words, cur_file) if contains: return self.setCache(cache_key, quality) # Try again with loose testing quality = self.guessLoose(cache_key, files = files, extra = extra) if quality: return self.setCache(cache_key, quality) log.debug('Could not identify quality for: %s', files) return None
def download(self, url="", dest=None, overwrite=False, urlopen_kwargs=None): if not urlopen_kwargs: urlopen_kwargs = {} # Return response object to stream download urlopen_kwargs["stream"] = True if not dest: # to Cache dest = os.path.join(Env.get("cache_dir"), ss("%s.%s" % (md5(url), getExt(url)))) dest = sp(dest) if not overwrite and os.path.isfile(dest): return dest try: filedata = self.urlopen(url, **urlopen_kwargs) except: log.error("Failed downloading file %s: %s", (url, traceback.format_exc())) return False self.createFile(dest, filedata, binary=True) return dest
def getRenameExtras(self, extra_type='', replacements={}, folder_name='', file_name='', destination='', group={}, current_file=''): rename_files = {} def test(s): return current_file[:-len(replacements['ext'])] in s for extra in set(filter(test, group['files'][extra_type])): replacements['ext'] = getExt(extra) final_folder_name = self.doReplace(folder_name, replacements) final_file_name = self.doReplace(file_name, replacements) rename_files[extra] = os.path.join(destination, final_folder_name, final_file_name) return rename_files
def searchSingle(self, message=None, group={}): if self.isDisabled() or len(group["files"]["trailer"]) > 0: return trailers = fireEvent("trailer.search", group=group, merge=True) if not trailers or trailers == []: log.info("No trailers found for: %s", getTitle(group["library"])) return False for trailer in trailers.get(self.conf("quality"), []): ext = getExt(trailer) filename = self.conf("name").replace("<filename>", group["filename"]) + ( ".%s" % ("mp4" if len(ext) > 5 else ext) ) destination = os.path.join(group["destination_dir"], filename) if not os.path.isfile(destination): trailer_file = fireEvent( "file.download", url=trailer, dest=destination, urlopen_kwargs={"headers": {"User-Agent": "Quicktime"}}, single=True, ) if os.path.getsize(trailer_file) < (1024 * 1024): # Don't trust small trailers (1MB), try next one os.unlink(trailer_file) continue else: log.debug("Trailer already exists: %s", destination) group["renamed_files"].append(destination) # Download first and break break return True
def guess(self, files, extra = {}): # Create hash for cache hash = md5(str([f.replace('.' + getExt(f), '') for f in files])) cached = self.getCache(hash) if cached and extra is {}: return cached for cur_file in files: words = re.split('\W+', cur_file.lower()) for quality in self.all(): # Check tags if quality['identifier'] in words: log.debug('Found via identifier "%s" in %s', (quality['identifier'], cur_file)) return self.setCache(hash, quality) if list(set(quality.get('alternative', [])) & set(words)): log.debug('Found %s via alt %s in %s', (quality['identifier'], quality.get('alternative'), cur_file)) return self.setCache(hash, quality) for tag in quality.get('tags', []): if isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words): log.debug('Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file)) return self.setCache(hash, quality) if list(set(quality.get('tags', [])) & set(words)): log.debug('Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file)) return self.setCache(hash, quality) # Try again with loose testing quality = self.guessLoose(hash, files = files, extra = extra) if quality: return self.setCache(hash, quality) log.debug('Could not identify quality for: %s', files) return None
def scan(self): if self.isDisabled(): return if self.renaming_started is True: log.info( 'Renamer is disabled to avoid infinite looping of the same error.' ) return # Check to see if the "to" folder is inside the "from" folder. if not os.path.isdir(self.conf('from')) or not os.path.isdir( self.conf('to')): log.debug('"To" and "From" have to exist.') return elif self.conf('from') in self.conf('to'): log.error( 'The "to" can\'t be inside of the "from" folder. You\'ll get an infinite loop.' ) return groups = fireEvent('scanner.scan', folder=self.conf('from'), single=True) self.renaming_started = True destination = self.conf('to') folder_name = self.conf('folder_name') file_name = self.conf('file_name') trailer_name = self.conf('trailer_name') nfo_name = self.conf('nfo_name') separator = self.conf('separator') # Statusses done_status = fireEvent('status.get', 'done', single=True) active_status = fireEvent('status.get', 'active', single=True) downloaded_status = fireEvent('status.get', 'downloaded', single=True) snatched_status = fireEvent('status.get', 'snatched', single=True) db = get_session() for group_identifier in groups: group = groups[group_identifier] rename_files = {} remove_files = [] remove_releases = [] movie_title = getTitle(group['library']) # Add _UNKNOWN_ if no library item is connected if not group['library'] or not movie_title: self.tagDir(group, 'unknown') continue # Rename the files using the library data else: group['library'] = fireEvent( 'library.update', identifier=group['library']['identifier'], single=True) if not group['library']: log.error( 'Could not rename, no library item to work with: %s', group_identifier) continue library = group['library'] movie_title = getTitle(library) # Find subtitle for renaming fireEvent('renamer.before', group) # Remove weird chars from moviename movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', movie_title) # Put 'The' at the end name_the = movie_name if movie_name[:4].lower() == 'the ': name_the = movie_name[4:] + ', The' replacements = { 'ext': 'mkv', 'namethe': name_the.strip(), 'thename': movie_name.strip(), 'year': library['year'], 'first': name_the[0].upper(), 'quality': group['meta_data']['quality']['label'], 'quality_type': group['meta_data']['quality_type'], 'video': group['meta_data'].get('video'), 'audio': group['meta_data'].get('audio'), 'group': group['meta_data']['group'], 'source': group['meta_data']['source'], 'resolution_width': group['meta_data'].get('resolution_width'), 'resolution_height': group['meta_data'].get('resolution_height'), 'imdb_id': library['identifier'], } for file_type in group['files']: # Move nfo depending on settings if file_type is 'nfo' and not self.conf('rename_nfo'): log.debug('Skipping, renaming of %s disabled', file_type) if self.conf('cleanup'): for current_file in group['files'][file_type]: remove_files.append(current_file) continue # Subtitle extra if file_type is 'subtitle_extra': continue # Move other files multiple = len( group['files']['movie']) > 1 and not group['is_dvd'] cd = 1 if multiple else 0 for current_file in sorted(list( group['files'][file_type])): # Original filename replacements['original'] = os.path.splitext( os.path.basename(current_file))[0] replacements['original_folder'] = fireEvent( 'scanner.remove_cptag', group['dirname'], single=True) # Extension replacements['ext'] = getExt(current_file) # cd # replacements['cd'] = ' cd%d' % cd if cd else '' replacements['cd_nr'] = cd # Naming final_folder_name = self.doReplace( folder_name, replacements) final_file_name = self.doReplace( file_name, replacements) replacements['filename'] = final_file_name[:-( len(getExt(final_file_name)) + 1)] # Group filename without cd extension replacements['cd'] = '' replacements['cd_nr'] = '' # Meta naming if file_type is 'trailer': final_file_name = self.doReplace( trailer_name, replacements) elif file_type is 'nfo': final_file_name = self.doReplace( nfo_name, replacements) # Seperator replace if separator: final_file_name = final_file_name.replace( ' ', separator) # Move DVD files (no structure renaming) if group['is_dvd'] and file_type is 'movie': found = False for top_dir in [ 'video_ts', 'audio_ts', 'bdmv', 'certificate' ]: has_string = current_file.lower().find( os.path.sep + top_dir + os.path.sep) if has_string >= 0: structure_dir = current_file[ has_string:].lstrip(os.path.sep) rename_files[current_file] = os.path.join( destination, final_folder_name, structure_dir) found = True break if not found: log.error( 'Could not determine dvd structure for: %s', current_file) # Do rename others else: if file_type is 'leftover': if self.conf('move_leftover'): rename_files[current_file] = os.path.join( destination, final_folder_name, os.path.basename(current_file)) elif file_type not in ['subtitle']: rename_files[current_file] = os.path.join( destination, final_folder_name, final_file_name) # Check for extra subtitle files if file_type is 'subtitle': # rename subtitles with or without language rename_files[current_file] = os.path.join( destination, final_folder_name, final_file_name) sub_langs = group['subtitle_language'].get( current_file, []) rename_extras = self.getRenameExtras( extra_type='subtitle_extra', replacements=replacements, folder_name=folder_name, file_name=file_name, destination=destination, group=group, current_file=current_file) # Don't add language if multiple languages in 1 file if len(sub_langs) > 1: rename_files[current_file] = os.path.join( destination, final_folder_name, final_file_name) elif len(sub_langs) == 1: sub_name = final_file_name.replace( replacements['ext'], '%s.%s' % (sub_langs[0], replacements['ext'])) rename_files[current_file] = os.path.join( destination, final_folder_name, sub_name) rename_files = mergeDicts(rename_files, rename_extras) # Filename without cd etc if file_type is 'movie': rename_extras = self.getRenameExtras( extra_type='movie_extra', replacements=replacements, folder_name=folder_name, file_name=file_name, destination=destination, group=group, current_file=current_file) rename_files = mergeDicts(rename_files, rename_extras) group['filename'] = self.doReplace( file_name, replacements)[:-(len(getExt(final_file_name)) + 1)] group['destination_dir'] = os.path.join( destination, final_folder_name) if multiple: cd += 1 # Before renaming, remove the lower quality files library = db.query(Library).filter_by( identifier=group['library']['identifier']).first() remove_leftovers = True # Add it to the wanted list before we continue if len(library.movies) == 0: profile = db.query(Profile).filter_by( core=True, label=group['meta_data']['quality']['label']).first() fireEvent('movie.add', params={ 'identifier': group['library']['identifier'], 'profile_id': profile.id }, search_after=False) db.expire_all() library = db.query(Library).filter_by( identifier=group['library']['identifier']).first() for movie in library.movies: # Mark movie "done" onces it found the quality with the finish check try: if movie.status_id == active_status.get( 'id') and movie.profile: for profile_type in movie.profile.types: if profile_type.quality_id == group[ 'meta_data']['quality'][ 'id'] and profile_type.finish: movie.status_id = done_status.get('id') db.commit() except Exception, e: log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc())) # Go over current movie releases for release in movie.releases: # When a release already exists if release.status_id is done_status.get('id'): # This is where CP removes older, lesser quality releases if release.quality.order > group['meta_data'][ 'quality']['order']: log.info('Removing lesser quality %s for %s.', (movie.library.titles[0].title, release.quality.label)) for current_file in release.files: remove_files.append(current_file) remove_releases.append(release) # Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc elif release.quality.order is group['meta_data'][ 'quality']['order']: log.info( 'Same quality release already exists for %s, with quality %s. Assuming repack.', (movie.library.titles[0].title, release.quality.label)) for current_file in release.files: remove_files.append(current_file) remove_releases.append(release) # Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan else: log.info( 'Better quality release already exists for %s, with quality %s', (movie.library.titles[0].title, release.quality.label)) # Add _EXISTS_ to the parent dir self.tagDir(group, 'exists') # Notify on rename fail download_message = 'Renaming of %s (%s) canceled, exists in %s already.' % ( movie.library.titles[0].title, group['meta_data']['quality']['label'], release.quality.label) fireEvent('movie.renaming.canceled', message=download_message, data=group) remove_leftovers = False break elif release.status_id is snatched_status.get('id'): if release.quality.id is group['meta_data'][ 'quality']['id']: log.debug('Marking release as downloaded') release.status_id = downloaded_status.get('id') db.commit() # Remove leftover files if self.conf('cleanup') and not self.conf( 'move_leftover') and remove_leftovers: log.debug('Removing leftover files') for current_file in group['files']['leftover']: remove_files.append(current_file) elif not remove_leftovers: # Don't remove anything break # Remove files delete_folders = [] for src in remove_files: if isinstance(src, File): src = src.path if rename_files.get(src): log.debug('Not removing file that will be renamed: %s', src) continue log.info('Removing "%s"', src) try: if os.path.isfile(src): os.remove(src) parent_dir = os.path.normpath(os.path.dirname(src)) if delete_folders.count( parent_dir) == 0 and os.path.isdir( parent_dir) and destination != parent_dir: delete_folders.append(parent_dir) except: log.error('Failed removing %s: %s', (src, traceback.format_exc())) self.tagDir(group, 'failed_remove') # Delete leftover folder from older releases for delete_folder in delete_folders: self.deleteEmptyFolder(delete_folder, show_error=False) # Rename all files marked group['renamed_files'] = [] for src in rename_files: if rename_files[src]: dst = rename_files[src] log.info('Renaming "%s" to "%s"', (src, dst)) # Create dir self.makeDir(os.path.dirname(dst)) try: self.moveFile(src, dst) group['renamed_files'].append(dst) except: log.error( 'Failed moving the file "%s" : %s', (os.path.basename(src), traceback.format_exc())) self.tagDir(group, 'failed_rename') # Remove matching releases for release in remove_releases: log.debug('Removing release %s', release.identifier) try: db.delete(release) except: log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc())) if group['dirname'] and group['parentdir']: try: log.info('Deleting folder: %s', group['parentdir']) self.deleteEmptyFolder(group['parentdir']) except: log.error('Failed removing %s: %s', (group['parentdir'], traceback.format_exc())) # Search for trailers etc fireEventAsync('renamer.after', group) # Notify on download download_message = 'Downloaded %s (%s)' % (movie_title, replacements['quality']) fireEventAsync('movie.downloaded', message=download_message, data=group) # Break if CP wants to shut down if self.shuttingDown(): break
def scan(self): groups = fireEvent('scanner.scan', folder=self.conf('from'), single=True) if groups is None: return destination = self.conf('to') folder_name = self.conf('folder_name') file_name = self.conf('file_name') trailer_name = self.conf('trailer_name') backdrop_name = self.conf('fanart_name') nfo_name = self.conf('nfo_name') separator = self.conf('separator') for group_identifier in groups: group = groups[group_identifier] rename_files = {} # Add _UNKNOWN_ if no library item is connected if not group['library']: if group['dirname']: rename_files[ group['parentdir']] = group['parentdir'].replace( group['dirname'], '_UNKNOWN_%s' % group['dirname']) else: # Add it to filename for file_type in group['files']: for rename_me in group['files'][file_type]: filename = os.path.basename(rename_me) rename_files[rename_me] = rename_me.replace( filename, '_UNKNOWN_%s' % filename) # Rename the files using the library data else: group['library'] = fireEvent( 'library.update', identifier=group['library']['identifier'], single=True) if not group['library']: log.error( 'Could not rename, no library item to work with: %s' % group_identifier) continue library = group['library'] # Find subtitle for renaming fireEvent('renamer.before', group) # Remove weird chars from moviename movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', group['library']['titles'][0]['title']) # Put 'The' at the end name_the = movie_name if movie_name[:4].lower() == 'the ': name_the = movie_name[4:] + ', The' replacements = { 'ext': 'mkv', 'namethe': name_the.strip(), 'thename': movie_name.strip(), 'year': library['year'], 'first': name_the[0].upper(), 'dirname': group['dirname'], 'quality': group['meta_data']['quality']['label'], 'quality_type': group['meta_data']['quality_type'], 'video': group['meta_data'].get('video'), 'audio': group['meta_data'].get('audio'), 'group': group['meta_data']['group'], 'source': group['meta_data']['source'], 'resolution_width': group['meta_data'].get('resolution_width'), 'resolution_height': group['meta_data'].get('resolution_height'), } for file_type in group['files']: # Move nfo depending on settings if file_type is 'nfo' and not self.conf('rename_nfo'): log.debug('Skipping, renaming of %s disabled' % file_type) continue # Subtitle extra if file_type is 'subtitle_extra': continue # Move other files multiple = len( group['files']['movie']) > 1 and not group['is_dvd'] cd = 1 if multiple else 0 for file in sorted(list(group['files'][file_type])): # Original filename replacements['original'] = os.path.basename(file) replacements['original_folder'] = os.path.basename( os.path.dirname(file)) # Extension replacements['ext'] = getExt(file) # cd # replacements['cd'] = ' cd%d' % cd if cd else '' replacements['cd_nr'] = cd # Naming final_folder_name = self.doReplace( folder_name, replacements) final_file_name = self.doReplace( file_name, replacements) replacements['filename'] = final_file_name[:-( len(getExt(final_file_name)) + 1)] # Meta naming if file_type is 'trailer': final_file_name = self.doReplace( trailer_name, replacements) elif file_type is 'nfo': final_file_name = self.doReplace( nfo_name, replacements) elif file_type is 'backdrop': final_file_name = self.doReplace( backdrop_name, replacements) # Seperator replace if separator: final_file_name = final_file_name.replace( ' ', separator) # Move DVD files (no structure renaming) if group['is_dvd'] and file_type is 'movie': found = False for top_dir in [ 'video_ts', 'audio_ts', 'bdmv', 'certificate' ]: has_string = file.lower().find(os.path.sep + top_dir + os.path.sep) if has_string >= 0: structure_dir = file[has_string:].lstrip( os.path.sep) rename_files[file] = os.path.join( destination, final_folder_name, structure_dir) found = True break if not found: log.error( 'Could not determin dvd structure for: %s' % file) # Do rename others else: rename_files[file] = os.path.join( destination, final_folder_name, final_file_name) # Check for extra subtitle files if file_type is 'subtitle': def test(s): return file[:-len(replacements['ext'])] in s for subtitle_extra in set( filter(test, group['files']['subtitle_extra'])): replacements['ext'] = getExt(subtitle_extra) final_folder_name = self.doReplace( folder_name, replacements) final_file_name = self.doReplace( file_name, replacements) rename_files[subtitle_extra] = os.path.join( destination, final_folder_name, final_file_name) # Filename without cd etc if file_type is 'movie': group['destination_dir'] = os.path.join( destination, final_folder_name) if multiple: cd += 1 # Before renaming, remove the lower quality files db = get_session() library = db.query(Library).filter_by( identifier=group['library']['identifier']).first() done_status = fireEvent('status.get', 'done', single=True) active_status = fireEvent('status.get', 'active', single=True) for movie in library.movies: # Mark movie "done" onces it found the quality with the finish check try: if movie.status_id == active_status.get('id'): for type in movie.profile.types: if type.quality_id == group['meta_data'][ 'quality']['id'] and type.finish: movie.status_id = done_status.get('id') db.commit() except Exception, e: log.error('Failed marking movie finished: %s %s' % (e, traceback.format_exc())) # Go over current movie releases for release in movie.releases: # This is where CP removes older, lesser quality releases if release.quality.order > group['meta_data'][ 'quality']['order']: log.info( 'Removing older release for %s, with quality %s' % (movie.library.titles[0].title, release.quality.label)) for file in release.files: log.info('Removing (not really) "%s"' % file.path) # When a release already exists elif release.status_id is done_status.get('id'): # Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc if release.quality.order is group['meta_data'][ 'quality']['order']: log.info( 'Same quality release already exists for %s, with quality %s. Assuming repack.' % (movie.library.titles[0].title, release.quality.label)) # Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan else: log.info( 'Better quality release already exists for %s, with quality %s' % (movie.library.titles[0].title, release.quality.label)) # Add _EXISTS_ to the parent dir if group['dirname']: for rename_me in rename_files: # Don't rename anything in this group rename_files[rename_me] = None rename_files[group['parentdir']] = group[ 'parentdir'].replace( group['dirname'], '_EXISTS_%s' % group['dirname']) else: # Add it to filename for rename_me in rename_files: filename = os.path.basename(rename_me) rename_files[ rename_me] = rename_me.replace( filename, '_EXISTS_%s' % filename) # Notify on rename fail download_message = 'Renaming of %s (%s) canceled, exists in %s already.' % ( movie.library.titles[0].title, group['meta_data']['quality']['label'], release.quality.label) fireEvent('movie.renaming.canceled', message=download_message, data=group) break # Rename all files marked for src in rename_files: if rename_files[src]: dst = rename_files[src] log.info('Renaming "%s" to "%s"' % (src, dst)) path = os.path.dirname(dst) # Create dir self.makeDir(path) try: pass #shutil.move(src, dst) except: log.error( 'Failed moving the file "%s" : %s' % (os.path.basename(src), traceback.format_exc())) #print rename_me, rename_files[rename_me] # Search for trailers etc fireEventAsync('renamer.after', group) # Notify on download download_message = 'Download of %s (%s) successful.' % ( group['library']['titles'][0]['title'], replacements['quality']) fireEventAsync('movie.downloaded', message=download_message, data=group) # Break if CP wants to shut down if self.shuttingDown(): break
def guess(self, files, extra=None, size=None, use_cache=True): if not extra: extra = {} # Create hash for cache cache_key = str([ f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files ]) if use_cache: cached = self.getCache(cache_key) if cached and len(extra) == 0: return cached qualities = self.all() # Start with 0 score = {} for quality in qualities: score[quality.get('identifier')] = {'score': 0, '3d': {}} # Use metadata titles as extra check if extra and extra.get('titles'): files.extend(extra.get('titles')) for cur_file in files: words = re.split('\W+', cur_file.lower()) name_year = fireEvent('scanner.name_year', cur_file, file_name=cur_file, single=True) threed_words = words if name_year and name_year.get('name'): split_name = splitString(name_year.get('name'), ' ') threed_words = [x for x in words if x not in split_name] for quality in qualities: contains_score = self.containsTagScore(quality, words, cur_file) threedscore = self.contains3D( quality, threed_words, cur_file) if quality.get('allow_3d') else (0, None) self.calcScore(score, quality, contains_score, threedscore, penalty=contains_score) size_scores = [] for quality in qualities: # Evaluate score based on size size_score = self.guessSizeScore(quality, size=size) loose_score = self.guessLooseScore(quality, extra=extra) if size_score > 0: size_scores.append(quality) self.calcScore(score, quality, size_score + loose_score) # Add additional size score if only 1 size validated if len(size_scores) == 1: self.calcScore(score, size_scores[0], 7) del size_scores # Return nothing if all scores are <= 0 has_non_zero = 0 for s in score: if score[s]['score'] > 0: has_non_zero += 1 if not has_non_zero: return None heighest_quality = max(score, key=lambda p: score[p]['score']) if heighest_quality: for quality in qualities: if quality.get('identifier') == heighest_quality: quality['is_3d'] = False if score[heighest_quality].get('3d'): quality['is_3d'] = True return self.setCache(cache_key, quality) return None
def getNfo(self, files): return set(filter(lambda s: getExt(s.lower()) in self.extensions['nfo'], files))
def getMovieExtras(self, files): return set(filter(lambda s: getExt(s.lower()) in self.extensions['movie_extra'], files))
def scan(self, folder = None, files = [], simple = False, newer_than = 0): folder = ss(os.path.normpath(folder)) if not folder or not os.path.isdir(folder): log.error('Folder doesn\'t exists: %s', folder) return {} # Get movie "master" files movie_files = {} leftovers = [] # Scan all files of the folder if no files are set if len(files) == 0: try: files = [] for root, dirs, walk_files in os.walk(folder): for filename in walk_files: files.append(os.path.join(root, filename)) except: log.error('Failed getting files from %s: %s', (folder, traceback.format_exc())) else: files = [ss(x) for x in files] db = get_session() for file_path in files: if not os.path.exists(file_path): continue # Remove ignored files if self.isSampleFile(file_path): leftovers.append(file_path) continue elif not self.keepFile(file_path): continue is_dvd_file = self.isDVDFile(file_path) if os.path.getsize(file_path) > self.minimal_filesize['media'] or is_dvd_file: # Minimal 300MB files or is DVD file # Normal identifier identifier = self.createStringIdentifier(file_path, folder, exclude_filename = is_dvd_file) identifiers = [identifier] # Identifier with quality quality = fireEvent('quality.guess', [file_path], single = True) if not is_dvd_file else {'identifier':'dvdr'} if quality: identifier_with_quality = '%s %s' % (identifier, quality.get('identifier', '')) identifiers = [identifier_with_quality, identifier] if not movie_files.get(identifier): movie_files[identifier] = { 'unsorted_files': [], 'identifiers': identifiers, 'is_dvd': is_dvd_file, } movie_files[identifier]['unsorted_files'].append(file_path) else: leftovers.append(file_path) # Break if CP wants to shut down if self.shuttingDown(): break # Cleanup del files # Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2" # files will be grouped first. leftovers = set(sorted(leftovers, reverse = True)) # Group files minus extension for identifier, group in movie_files.iteritems(): if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier) log.debug('Grouping files: %s', identifier) for file_path in group['unsorted_files']: wo_ext = file_path[:-(len(getExt(file_path)) + 1)] found_files = set([i for i in leftovers if wo_ext in i]) group['unsorted_files'].extend(found_files) leftovers = leftovers - found_files # Break if CP wants to shut down if self.shuttingDown(): break # Create identifiers for all leftover files for file_path in leftovers: identifier = self.createStringIdentifier(file_path, folder) if not self.path_identifiers.get(identifier): self.path_identifiers[identifier] = [] self.path_identifiers[identifier].append(file_path) # Group the files based on the identifier delete_identifiers = [] for identifier, found_files in self.path_identifiers.iteritems(): log.debug('Grouping files on identifier: %s', identifier) group = movie_files.get(identifier) if group: group['unsorted_files'].extend(found_files) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers = leftovers - set(found_files) # Break if CP wants to shut down if self.shuttingDown(): break # Cleaning up used for identifier in delete_identifiers: if self.path_identifiers.get(identifier): del self.path_identifiers[identifier] del delete_identifiers # Group based on folder delete_identifiers = [] for identifier, found_files in self.path_identifiers.iteritems(): log.debug('Grouping files on foldername: %s', identifier) for ff in found_files: new_identifier = self.createStringIdentifier(os.path.dirname(ff), folder) group = movie_files.get(new_identifier) if group: group['unsorted_files'].extend([ff]) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers = leftovers - set([ff]) # Break if CP wants to shut down if self.shuttingDown(): break # Cleaning up used for identifier in delete_identifiers: if self.path_identifiers.get(identifier): del self.path_identifiers[identifier] del delete_identifiers # Determine file types processed_movies = {} while True and not self.shuttingDown(): try: identifier, group = movie_files.popitem() except: break # Check if movie is fresh and maybe still unpacking, ignore files new then 1 minute file_too_new = False for cur_file in group['unsorted_files']: if not os.path.isfile(cur_file): file_too_new = time.time() break file_time = [os.path.getmtime(cur_file), os.path.getctime(cur_file)] for t in file_time: if t > time.time() - 60: file_too_new = tryInt(time.time() - t) break if file_too_new: break if file_too_new: log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s', (time.ctime(file_time[0]), identifier)) # Delete the unsorted list del group['unsorted_files'] continue # Only process movies newer than x if newer_than and newer_than > 0: for cur_file in group['unsorted_files']: file_time = [os.path.getmtime(cur_file), os.path.getctime(cur_file)] if file_time[0] > time.time() or file_time[1] > time.time(): break log.debug('None of the files have changed since %s for %s, skipping.', (time.ctime(newer_than), identifier)) # Delete the unsorted list del group['unsorted_files'] continue # Group extra (and easy) files first # images = self.getImages(group['unsorted_files']) group['files'] = { 'movie_extra': self.getMovieExtras(group['unsorted_files']), 'subtitle': self.getSubtitles(group['unsorted_files']), 'subtitle_extra': self.getSubtitlesExtras(group['unsorted_files']), 'nfo': self.getNfo(group['unsorted_files']), 'trailer': self.getTrailers(group['unsorted_files']), #'backdrop': images['backdrop'], 'leftover': set(group['unsorted_files']), } # Media files if group['is_dvd']: group['files']['movie'] = self.getDVDFiles(group['unsorted_files']) else: group['files']['movie'] = self.getMediaFiles(group['unsorted_files']) if len(group['files']['movie']) == 0: log.error('Couldn\t find any movie files for %s', identifier) continue log.debug('Getting metadata for %s', identifier) group['meta_data'] = self.getMetaData(group) # Subtitle meta group['subtitle_language'] = self.getSubtitleLanguage(group) if not simple else {} # Get parent dir from movie files for movie_file in group['files']['movie']: group['parentdir'] = os.path.dirname(movie_file) group['dirname'] = None folder_names = group['parentdir'].replace(folder, '').split(os.path.sep) folder_names.reverse() # Try and get a proper dirname, so no "A", "Movie", "Download" etc for folder_name in folder_names: if folder_name.lower() not in self.ignore_names and len(folder_name) > 2: group['dirname'] = folder_name break break # Leftover "sorted" files for file_type in group['files']: if not file_type is 'leftover': group['files']['leftover'] -= set(group['files'][file_type]) # Delete the unsorted list del group['unsorted_files'] # Determine movie group['library'] = self.determineMovie(group) if not group['library']: log.error('Unable to determine movie: %s', group['identifiers']) else: movie = db.query(Movie).filter_by(library_id = group['library']['id']).first() group['movie_id'] = None if not movie else movie.id processed_movies[identifier] = group # Clean up self.path_identifiers = {} if len(processed_movies) > 0: log.info('Found %s movies in the folder %s', (len(processed_movies), folder)) else: log.debug('Found no movies in the folder %s', (folder)) return processed_movies
def test(s): return self.filesizeBetween(s, self.file_sizes['movie']) and getExt(s.lower()) in self.extensions['movie'] and not self.isSampleFile(s)
def scan(self, folder = None, files = None, release_download = None, simple = False, newer_than = 0, return_ignored = True, check_file_date = True, on_found = None): folder = sp(folder) if not folder or not os.path.isdir(folder): log.error('Folder doesn\'t exists: %s', folder) return {} # Get movie "master" files movie_files = {} leftovers = [] # Scan all files of the folder if no files are set if not files: try: files = [] for root, dirs, walk_files in os.walk(folder, followlinks=True): files.extend([sp(os.path.join(sp(root), ss(filename))) for filename in walk_files]) # Break if CP wants to shut down if self.shuttingDown(): break except: log.error('Failed getting files from %s: %s', (folder, traceback.format_exc())) log.debug('Found %s files to scan and group in %s', (len(files), folder)) else: check_file_date = False files = [sp(x) for x in files] for file_path in files: if not os.path.exists(file_path): continue # Remove ignored files if self.isSampleFile(file_path): leftovers.append(file_path) continue elif not self.keepFile(file_path): continue is_dvd_file = self.isDVDFile(file_path) if self.filesizeBetween(file_path, self.file_sizes['movie']) or is_dvd_file: # Minimal 300MB files or is DVD file # Normal identifier identifier = self.createStringIdentifier(file_path, folder, exclude_filename = is_dvd_file) identifiers = [identifier] # Identifier with quality quality = fireEvent('quality.guess', files = [file_path], size = self.getFileSize(file_path), single = True) if not is_dvd_file else {'identifier':'dvdr'} if quality: identifier_with_quality = '%s %s' % (identifier, quality.get('identifier', '')) identifiers = [identifier_with_quality, identifier] if not movie_files.get(identifier): movie_files[identifier] = { 'unsorted_files': [], 'identifiers': identifiers, 'is_dvd': is_dvd_file, } movie_files[identifier]['unsorted_files'].append(file_path) else: leftovers.append(file_path) # Break if CP wants to shut down if self.shuttingDown(): break # Cleanup del files # Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2" # files will be grouped first. leftovers = set(sorted(leftovers, reverse = True)) # Group files minus extension ignored_identifiers = [] for identifier, group in movie_files.items(): if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier) log.debug('Grouping files: %s', identifier) has_ignored = 0 for file_path in list(group['unsorted_files']): ext = getExt(file_path) wo_ext = file_path[:-(len(ext) + 1)] found_files = set([i for i in leftovers if wo_ext in i]) group['unsorted_files'].extend(found_files) leftovers = leftovers - found_files has_ignored += 1 if ext == 'ignore' else 0 if has_ignored == 0: for file_path in list(group['unsorted_files']): ext = getExt(file_path) has_ignored += 1 if ext == 'ignore' else 0 if has_ignored > 0: ignored_identifiers.append(identifier) # Break if CP wants to shut down if self.shuttingDown(): break # Create identifiers for all leftover files path_identifiers = {} for file_path in leftovers: identifier = self.createStringIdentifier(file_path, folder) if not path_identifiers.get(identifier): path_identifiers[identifier] = [] path_identifiers[identifier].append(file_path) # Group the files based on the identifier delete_identifiers = [] for identifier, found_files in path_identifiers.items(): log.debug('Grouping files on identifier: %s', identifier) group = movie_files.get(identifier) if group: group['unsorted_files'].extend(found_files) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers = leftovers - set(found_files) # Break if CP wants to shut down if self.shuttingDown(): break # Cleaning up used for identifier in delete_identifiers: if path_identifiers.get(identifier): del path_identifiers[identifier] del delete_identifiers # Group based on folder delete_identifiers = [] for identifier, found_files in path_identifiers.items(): log.debug('Grouping files on foldername: %s', identifier) for ff in found_files: new_identifier = self.createStringIdentifier(os.path.dirname(ff), folder) group = movie_files.get(new_identifier) if group: group['unsorted_files'].extend([ff]) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers -= leftovers - set([ff]) # Break if CP wants to shut down if self.shuttingDown(): break # leftovers should be empty if leftovers: log.debug('Some files are still left over: %s', leftovers) # Cleaning up used for identifier in delete_identifiers: if path_identifiers.get(identifier): del path_identifiers[identifier] del delete_identifiers # Make sure we remove older / still extracting files valid_files = {} while True and not self.shuttingDown(): try: identifier, group = movie_files.popitem() except: break # Check if movie is fresh and maybe still unpacking, ignore files newer than 1 minute if check_file_date: files_too_new, time_string = self.checkFilesChanged(group['unsorted_files']) if files_too_new: log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s', (time_string, identifier)) # Delete the unsorted list del group['unsorted_files'] continue # Only process movies newer than x if newer_than and newer_than > 0: has_new_files = False for cur_file in group['unsorted_files']: file_time = self.getFileTimes(cur_file) if file_time[0] > newer_than or file_time[1] > newer_than: has_new_files = True break if not has_new_files: log.debug('None of the files have changed since %s for %s, skipping.', (time.ctime(newer_than), identifier)) # Delete the unsorted list del group['unsorted_files'] continue valid_files[identifier] = group del movie_files total_found = len(valid_files) # Make sure only one movie was found if a download ID is provided if release_download and total_found == 0: log.info('Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).', release_download.get('imdb_id')) elif release_download and total_found > 1: log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (release_download.get('imdb_id'), len(valid_files))) release_download = None # Determine file types processed_movies = {} while True and not self.shuttingDown(): try: identifier, group = valid_files.popitem() except: break if return_ignored is False and identifier in ignored_identifiers: log.debug('Ignore file found, ignoring release: %s', identifier) total_found -= 1 continue # Group extra (and easy) files first group['files'] = { 'movie_extra': self.getMovieExtras(group['unsorted_files']), 'subtitle': self.getSubtitles(group['unsorted_files']), 'subtitle_extra': self.getSubtitlesExtras(group['unsorted_files']), 'nfo': self.getNfo(group['unsorted_files']), 'trailer': self.getTrailers(group['unsorted_files']), 'leftover': set(group['unsorted_files']), } # Media files if group['is_dvd']: group['files']['movie'] = self.getDVDFiles(group['unsorted_files']) else: group['files']['movie'] = self.getMediaFiles(group['unsorted_files']) if len(group['files']['movie']) == 0: log.error('Couldn\'t find any movie files for %s', identifier) total_found -= 1 continue log.debug('Getting metadata for %s', identifier) group['meta_data'] = self.getMetaData(group, folder = folder, release_download = release_download) # Subtitle meta group['subtitle_language'] = self.getSubtitleLanguage(group) if not simple else {} # Get parent dir from movie files for movie_file in group['files']['movie']: group['parentdir'] = os.path.dirname(movie_file) group['dirname'] = None folder_names = group['parentdir'].replace(folder, '').split(os.path.sep) folder_names.reverse() # Try and get a proper dirname, so no "A", "Movie", "Download" etc for folder_name in folder_names: if folder_name.lower() not in self.ignore_names and len(folder_name) > 2: group['dirname'] = folder_name break break # Leftover "sorted" files for file_type in group['files']: if not file_type is 'leftover': group['files']['leftover'] -= set(group['files'][file_type]) group['files'][file_type] = list(group['files'][file_type]) group['files']['leftover'] = list(group['files']['leftover']) # Delete the unsorted list del group['unsorted_files'] # Determine movie group['media'] = self.determineMedia(group, release_download = release_download) if not group['media']: log.error('Unable to determine media: %s', group['identifiers']) else: group['identifier'] = getIdentifier(group['media']) or group['media']['info'].get('imdb') processed_movies[identifier] = group # Notify parent & progress on something found if on_found: on_found(group, total_found, len(valid_files)) # Wait for all the async events calm down a bit while threading.activeCount() > 100 and not self.shuttingDown(): log.debug('Too many threads active, waiting a few seconds') time.sleep(10) if len(processed_movies) > 0: log.info('Found %s movies in the folder %s', (len(processed_movies), folder)) else: log.debug('Found no movies in the folder %s', folder) return processed_movies
def test(s): return self.filesizeBetween(s, 300, 100000) and getExt( s.lower()) in self.extensions['movie']
def scan(self, movie_folder = None, download_info = None): if self.isDisabled(): return if self.renaming_started is True: log.info('Renamer is already running, if you see this often, check the logs above for errors.') return # Check to see if the "to" folder is inside the "from" folder. if movie_folder and not os.path.isdir(movie_folder) or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')): l = log.debug if movie_folder else log.error l('Both the "To" and "From" have to exist.') return elif self.conf('from') in self.conf('to'): log.error('The "to" can\'t be inside of the "from" folder. You\'ll get an infinite loop.') return elif (movie_folder and movie_folder in [self.conf('to'), self.conf('from')]): log.error('The "to" and "from" folders can\'t be inside of or the same as the provided movie folder.') return self.renaming_started = True # make sure the movie folder name is included in the search folder = None files = [] if movie_folder: log.info('Scanning movie folder %s...', movie_folder) movie_folder = movie_folder.rstrip(os.path.sep) folder = os.path.dirname(movie_folder) # Get all files from the specified folder try: for root, folders, names in os.walk(movie_folder): files.extend([os.path.join(root, name) for name in names]) except: log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc())) db = get_session() # Extend the download info with info stored in the downloaded release download_info = self.extendDownloadInfo(download_info) groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = files, download_info = download_info, return_ignored = False, single = True) destination = self.conf('to') folder_name = self.conf('folder_name') file_name = self.conf('file_name') trailer_name = self.conf('trailer_name') nfo_name = self.conf('nfo_name') separator = self.conf('separator') # Statusses done_status, active_status, downloaded_status, snatched_status = \ fireEvent('status.get', ['done', 'active', 'downloaded', 'snatched'], single = True) for group_identifier in groups: group = groups[group_identifier] rename_files = {} remove_files = [] remove_releases = [] movie_title = getTitle(group['library']) # Add _UNKNOWN_ if no library item is connected if not group['library'] or not movie_title: self.tagDir(group, 'unknown') continue # Rename the files using the library data else: group['library'] = fireEvent('library.update', identifier = group['library']['identifier'], single = True) if not group['library']: log.error('Could not rename, no library item to work with: %s', group_identifier) continue library = group['library'] movie_title = getTitle(library) # Find subtitle for renaming fireEvent('renamer.before', group) # Remove weird chars from moviename movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', movie_title) # Put 'The' at the end name_the = movie_name if movie_name[:4].lower() == 'the ': name_the = movie_name[4:] + ', The' replacements = { 'ext': 'mkv', 'namethe': name_the.strip(), 'thename': movie_name.strip(), 'year': library['year'], 'first': name_the[0].upper(), 'quality': group['meta_data']['quality']['label'], 'quality_type': group['meta_data']['quality_type'], 'video': group['meta_data'].get('video'), 'audio': group['meta_data'].get('audio'), 'group': group['meta_data']['group'], 'source': group['meta_data']['source'], 'resolution_width': group['meta_data'].get('resolution_width'), 'resolution_height': group['meta_data'].get('resolution_height'), 'imdb_id': library['identifier'], 'cd': '', 'cd_nr': '', } for file_type in group['files']: # Move nfo depending on settings if file_type is 'nfo' and not self.conf('rename_nfo'): log.debug('Skipping, renaming of %s disabled', file_type) if self.conf('cleanup'): for current_file in group['files'][file_type]: remove_files.append(current_file) continue # Subtitle extra if file_type is 'subtitle_extra': continue # Move other files multiple = len(group['files'][file_type]) > 1 and not group['is_dvd'] cd = 1 if multiple else 0 for current_file in sorted(list(group['files'][file_type])): current_file = toUnicode(current_file) # Original filename replacements['original'] = os.path.splitext(os.path.basename(current_file))[0] replacements['original_folder'] = fireEvent('scanner.remove_cptag', group['dirname'], single = True) # Extension replacements['ext'] = getExt(current_file) # cd # replacements['cd'] = ' cd%d' % cd if multiple else '' replacements['cd_nr'] = cd if multiple else '' # Naming final_folder_name = self.doReplace(folder_name, replacements).lstrip('. ') final_file_name = self.doReplace(file_name, replacements).lstrip('. ') replacements['filename'] = final_file_name[:-(len(getExt(final_file_name)) + 1)] # Meta naming if file_type is 'trailer': final_file_name = self.doReplace(trailer_name, replacements, remove_multiple = True).lstrip('. ') elif file_type is 'nfo': final_file_name = self.doReplace(nfo_name, replacements, remove_multiple = True).lstrip('. ') # Seperator replace if separator: final_file_name = final_file_name.replace(' ', separator) # Move DVD files (no structure renaming) if group['is_dvd'] and file_type is 'movie': found = False for top_dir in ['video_ts', 'audio_ts', 'bdmv', 'certificate']: has_string = current_file.lower().find(os.path.sep + top_dir + os.path.sep) if has_string >= 0: structure_dir = current_file[has_string:].lstrip(os.path.sep) rename_files[current_file] = os.path.join(destination, final_folder_name, structure_dir) found = True break if not found: log.error('Could not determine dvd structure for: %s', current_file) # Do rename others else: if file_type is 'leftover': if self.conf('move_leftover'): rename_files[current_file] = os.path.join(destination, final_folder_name, os.path.basename(current_file)) elif file_type not in ['subtitle']: rename_files[current_file] = os.path.join(destination, final_folder_name, final_file_name) # Check for extra subtitle files if file_type is 'subtitle': remove_multiple = False if len(group['files']['movie']) == 1: remove_multiple = True sub_langs = group['subtitle_language'].get(current_file, []) # rename subtitles with or without language sub_name = self.doReplace(file_name, replacements, remove_multiple = remove_multiple) rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name) rename_extras = self.getRenameExtras( extra_type = 'subtitle_extra', replacements = replacements, folder_name = folder_name, file_name = file_name, destination = destination, group = group, current_file = current_file, remove_multiple = remove_multiple, ) # Don't add language if multiple languages in 1 subtitle file if len(sub_langs) == 1: sub_name = final_file_name.replace(replacements['ext'], '%s.%s' % (sub_langs[0], replacements['ext'])) rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name) rename_files = mergeDicts(rename_files, rename_extras) # Filename without cd etc elif file_type is 'movie': rename_extras = self.getRenameExtras( extra_type = 'movie_extra', replacements = replacements, folder_name = folder_name, file_name = file_name, destination = destination, group = group, current_file = current_file ) rename_files = mergeDicts(rename_files, rename_extras) group['filename'] = self.doReplace(file_name, replacements, remove_multiple = True)[:-(len(getExt(final_file_name)) + 1)] group['destination_dir'] = os.path.join(destination, final_folder_name) if multiple: cd += 1 # Before renaming, remove the lower quality files library = db.query(Library).filter_by(identifier = group['library']['identifier']).first() remove_leftovers = True # Add it to the wanted list before we continue if len(library.movies) == 0: profile = db.query(Profile).filter_by(core = True, label = group['meta_data']['quality']['label']).first() fireEvent('movie.add', params = {'identifier': group['library']['identifier'], 'profile_id': profile.id}, search_after = False) db.expire_all() library = db.query(Library).filter_by(identifier = group['library']['identifier']).first() for movie in library.movies: # Mark movie "done" onces it found the quality with the finish check try: if movie.status_id == active_status.get('id') and movie.profile: for profile_type in movie.profile.types: if profile_type.quality_id == group['meta_data']['quality']['id'] and profile_type.finish: movie.status_id = done_status.get('id') movie.last_edit = int(time.time()) db.commit() except Exception, e: log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc())) # Go over current movie releases for release in movie.releases: # When a release already exists if release.status_id is done_status.get('id'): # This is where CP removes older, lesser quality releases if release.quality.order > group['meta_data']['quality']['order']: log.info('Removing lesser quality %s for %s.', (movie.library.titles[0].title, release.quality.label)) for current_file in release.files: remove_files.append(current_file) remove_releases.append(release) # Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc elif release.quality.order is group['meta_data']['quality']['order']: log.info('Same quality release already exists for %s, with quality %s. Assuming repack.', (movie.library.titles[0].title, release.quality.label)) for current_file in release.files: remove_files.append(current_file) remove_releases.append(release) # Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan else: log.info('Better quality release already exists for %s, with quality %s', (movie.library.titles[0].title, release.quality.label)) # Add exists tag to the .ignore file self.tagDir(group, 'exists') # Notify on rename fail download_message = 'Renaming of %s (%s) canceled, exists in %s already.' % (movie.library.titles[0].title, group['meta_data']['quality']['label'], release.quality.label) fireEvent('movie.renaming.canceled', message = download_message, data = group) remove_leftovers = False break elif release.status_id is snatched_status.get('id'): if release.quality.id is group['meta_data']['quality']['id']: log.debug('Marking release as downloaded') try: release.status_id = downloaded_status.get('id') release.last_edit = int(time.time()) except Exception, e: log.error('Failed marking release as finished: %s %s', (e, traceback.format_exc())) db.commit() # Remove leftover files if self.conf('cleanup') and not self.conf('move_leftover') and remove_leftovers and \ not (self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info)): log.debug('Removing leftover files') for current_file in group['files']['leftover']: remove_files.append(current_file) elif not remove_leftovers: # Don't remove anything break
def guess(self, files, extra=None, size=None): if not extra: extra = {} # Create hash for cache cache_key = str([ f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files ]) cached = self.getCache(cache_key) if cached and len(extra) == 0: return cached qualities = self.all() # Start with 0 score = {} for quality in qualities: score[quality.get('identifier')] = {'score': 0, '3d': {}} for cur_file in files: words = re.split('\W+', cur_file.lower()) for quality in qualities: contains_score = self.containsTagScore(quality, words, cur_file) threedscore = self.contains3D( quality, words, cur_file) if quality.get('allow_3d') else (0, None) self.calcScore(score, quality, contains_score, threedscore) size_scores = [] for quality in qualities: # Evaluate score based on size size_score = self.guessSizeScore(quality, size=size) loose_score = self.guessLooseScore(quality, extra=extra) if size_score > 0: size_scores.append(quality) self.calcScore(score, quality, size_score + loose_score, penalty=False) # Add additional size score if only 1 size validated if len(size_scores) == 1: self.calcScore(score, size_scores[0], 10, penalty=False) del size_scores # Return nothing if all scores are <= 0 has_non_zero = 0 for s in score: if score[s]['score'] > 0: has_non_zero += 1 if not has_non_zero: return None heighest_quality = max(score, key=lambda p: score[p]['score']) if heighest_quality: for quality in qualities: if quality.get('identifier') == heighest_quality: quality['is_3d'] = False if score[heighest_quality].get('3d'): quality['is_3d'] = True return self.setCache(cache_key, quality) return None
def getSubtitlesExtras(self, files): return set(filter(lambda s: getExt(s.lower()) in self.extensions['subtitle_extra'], files))
def test(s): return self.filesizeBetween(s, 300, 100000) and getExt(s.lower()) in self.extensions['movie'] and not self.isSampleFile(s)
def test(s): return getExt(s.lower()) in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tbn']
def scan(self, folder=None, files=None, simple=False, newer_than=0, on_found=None): folder = ss(os.path.normpath(folder)) if not folder or not os.path.isdir(folder): log.error('Folder doesn\'t exists: %s', folder) return {} # Get movie "master" files movie_files = {} leftovers = [] # Scan all files of the folder if no files are set if not files: check_file_date = True try: files = [] for root, dirs, walk_files in os.walk(folder): for filename in walk_files: files.append(os.path.join(root, filename)) except: log.error('Failed getting files from %s: %s', (folder, traceback.format_exc())) else: check_file_date = False files = [ss(x) for x in files] db = get_session() for file_path in files: if not os.path.exists(file_path): continue # Remove ignored files if self.isSampleFile(file_path): leftovers.append(file_path) continue elif not self.keepFile(file_path): continue is_dvd_file = self.isDVDFile(file_path) if os.path.getsize(file_path) > self.minimal_filesize[ 'media'] or is_dvd_file: # Minimal 300MB files or is DVD file # Normal identifier identifier = self.createStringIdentifier( file_path, folder, exclude_filename=is_dvd_file) identifiers = [identifier] # Identifier with quality quality = fireEvent('quality.guess', [file_path], single=True) if not is_dvd_file else { 'identifier': 'dvdr' } if quality: identifier_with_quality = '%s %s' % ( identifier, quality.get('identifier', '')) identifiers = [identifier_with_quality, identifier] if not movie_files.get(identifier): movie_files[identifier] = { 'unsorted_files': [], 'identifiers': identifiers, 'is_dvd': is_dvd_file, } movie_files[identifier]['unsorted_files'].append(file_path) else: leftovers.append(file_path) # Break if CP wants to shut down if self.shuttingDown(): break # Cleanup del files # Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2" # files will be grouped first. leftovers = set(sorted(leftovers, reverse=True)) # Group files minus extension for identifier, group in movie_files.iteritems(): if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier) log.debug('Grouping files: %s', identifier) for file_path in group['unsorted_files']: wo_ext = file_path[:-(len(getExt(file_path)) + 1)] found_files = set([i for i in leftovers if wo_ext in i]) group['unsorted_files'].extend(found_files) leftovers = leftovers - found_files # Break if CP wants to shut down if self.shuttingDown(): break # Create identifiers for all leftover files path_identifiers = {} for file_path in leftovers: identifier = self.createStringIdentifier(file_path, folder) if not path_identifiers.get(identifier): path_identifiers[identifier] = [] path_identifiers[identifier].append(file_path) # Group the files based on the identifier delete_identifiers = [] for identifier, found_files in path_identifiers.iteritems(): log.debug('Grouping files on identifier: %s', identifier) group = movie_files.get(identifier) if group: group['unsorted_files'].extend(found_files) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers = leftovers - set(found_files) # Break if CP wants to shut down if self.shuttingDown(): break # Cleaning up used for identifier in delete_identifiers: if path_identifiers.get(identifier): del path_identifiers[identifier] del delete_identifiers # Group based on folder delete_identifiers = [] for identifier, found_files in path_identifiers.iteritems(): log.debug('Grouping files on foldername: %s', identifier) for ff in found_files: new_identifier = self.createStringIdentifier( os.path.dirname(ff), folder) group = movie_files.get(new_identifier) if group: group['unsorted_files'].extend([ff]) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers = leftovers - set([ff]) # Break if CP wants to shut down if self.shuttingDown(): break # Cleaning up used for identifier in delete_identifiers: if path_identifiers.get(identifier): del path_identifiers[identifier] del delete_identifiers # Make sure we remove older / still extracting files valid_files = {} while True and not self.shuttingDown(): try: identifier, group = movie_files.popitem() except: break # Check if movie is fresh and maybe still unpacking, ignore files new then 1 minute file_too_new = False for cur_file in group['unsorted_files']: if not os.path.isfile(cur_file): file_too_new = time.time() break file_time = [ os.path.getmtime(cur_file), os.path.getctime(cur_file) ] for t in file_time: if t > time.time() - 60: file_too_new = tryInt(time.time() - t) break if file_too_new: break if check_file_date and file_too_new: try: time_string = time.ctime(file_time[0]) except: try: time_string = time.ctime(file_time[1]) except: time_string = 'unknown' log.info( 'Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s', (time_string, identifier)) # Delete the unsorted list del group['unsorted_files'] continue # Only process movies newer than x if newer_than and newer_than > 0: has_new_files = False for cur_file in group['unsorted_files']: file_time = [ os.path.getmtime(cur_file), os.path.getctime(cur_file) ] if file_time[0] > newer_than or file_time[1] > newer_than: has_new_files = True break if not has_new_files: log.debug( 'None of the files have changed since %s for %s, skipping.', (time.ctime(newer_than), identifier)) # Delete the unsorted list del group['unsorted_files'] continue valid_files[identifier] = group del movie_files # Determine file types processed_movies = {} total_found = len(valid_files) while True and not self.shuttingDown(): try: identifier, group = valid_files.popitem() except: break # Group extra (and easy) files first # images = self.getImages(group['unsorted_files']) group['files'] = { 'movie_extra': self.getMovieExtras(group['unsorted_files']), 'subtitle': self.getSubtitles(group['unsorted_files']), 'subtitle_extra': self.getSubtitlesExtras(group['unsorted_files']), 'nfo': self.getNfo(group['unsorted_files']), 'trailer': self.getTrailers(group['unsorted_files']), #'backdrop': images['backdrop'], 'leftover': set(group['unsorted_files']), } # Media files if group['is_dvd']: group['files']['movie'] = self.getDVDFiles( group['unsorted_files']) else: group['files']['movie'] = self.getMediaFiles( group['unsorted_files']) if len(group['files']['movie']) == 0: log.error('Couldn\'t find any movie files for %s', identifier) continue log.debug('Getting metadata for %s', identifier) group['meta_data'] = self.getMetaData(group, folder=folder) # Subtitle meta group['subtitle_language'] = self.getSubtitleLanguage( group) if not simple else {} # Get parent dir from movie files for movie_file in group['files']['movie']: group['parentdir'] = os.path.dirname(movie_file) group['dirname'] = None folder_names = group['parentdir'].replace(folder, '').split( os.path.sep) folder_names.reverse() # Try and get a proper dirname, so no "A", "Movie", "Download" etc for folder_name in folder_names: if folder_name.lower( ) not in self.ignore_names and len(folder_name) > 2: group['dirname'] = folder_name break break # Leftover "sorted" files for file_type in group['files']: if not file_type is 'leftover': group['files']['leftover'] -= set( group['files'][file_type]) # Delete the unsorted list del group['unsorted_files'] # Determine movie group['library'] = self.determineMovie(group) if not group['library']: log.error('Unable to determine movie: %s', group['identifiers']) else: movie = db.query(Movie).filter_by( library_id=group['library']['id']).first() group['movie_id'] = None if not movie else movie.id processed_movies[identifier] = group # Notify parent & progress on something found if on_found: on_found(group, total_found, total_found - len(processed_movies)) if len(processed_movies) > 0: log.info('Found %s movies in the folder %s', (len(processed_movies), folder)) else: log.debug('Found no movies in the folder %s', (folder)) return processed_movies