def rename(self, files, current_path): logging.debug("Renaming Date file") renamed = None #find the master file to rename for file in files: if is_full_path(file): filepath = file.replace('_UNPACK_', '') else: filepath = os.path.join(current_path, file) if os.path.exists(filepath): size = os.stat(filepath).st_size if size > 130000000: if 'sample' not in file: tmp, ext = os.path.splitext(file) self.fname = tmp newname = "%s%s" % (self.filename_set, ext) newname = newname.replace('%fn', tmp) newpath = os.path.join(current_path, newname) if not os.path.exists(newpath): try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(Ta('Failed to rename: %s to %s'), current_path, newpath) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, file, self.filename_set) break
def rename_similar(folder, skip_ext, name, skipped_files): """ Rename all other files in the 'folder' hierarchy after 'name' and move them to the root of 'folder'. Files having extension 'skip_ext' will be moved, but not renamed. Don't touch files in list `skipped_files` """ logging.debug('Give files in set "%s" matching names.', name) folder = os.path.normpath(folder) skip_ext = skip_ext.lower() for root, dirs, files in os.walk(folder): for f in files: path = os.path.join(root, f) if path in skipped_files: continue org, ext = os.path.splitext(f) if ext.lower() == skip_ext: # Move file, but do not rename newpath = os.path.join(folder, f) else: # Move file and rename newname = "%s%s" % (name, ext) newname = newname.replace('%fn', org) newpath = os.path.join(folder, newname) if path != newpath: newpath = get_unique_filename(newpath) try: logging.debug("Rename: %s to %s", path, newpath) renamer(path, newpath) except: logging.error(T('Failed to rename similar file: %s to %s'), clip_path(path), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) cleanup_empty_directories(folder)
def rename_and_collapse_folder(oldpath, newpath, files): """ Rename folder, collapsing when there's just a single subfolder oldpath --> newpath OR oldpath/subfolder --> newpath Modify list of filenames accordingly """ orgpath = oldpath items = globber(oldpath) if len(items) == 1: folder = items[0] folder_path = os.path.join(oldpath, folder) if os.path.isdir(folder_path) and folder not in ('VIDEO_TS', 'AUDIO_TS'): logging.info('Collapsing %s', os.path.join(newpath, folder)) oldpath = folder_path oldpath = os.path.normpath(oldpath) newpath = os.path.normpath(newpath) files = [os.path.normpath(f).replace(oldpath, newpath) for f in files] renamer(oldpath, newpath) try: remove_dir(orgpath) except: pass return files
def move(self, workdir_complete): ok = True if self.type == 'movie': move_to_parent = True # check if we should leave the files inside an extra folder if cfg.movie_extra_folders(): # if there is a folder in the download, leave it in an extra folder move_to_parent = not check_for_folder(workdir_complete) if move_to_parent: workdir_complete, ok = move_to_parent_folder(workdir_complete) else: workdir_complete, ok = move_to_parent_folder(workdir_complete) if not ok: return workdir_complete, False path, part = os.path.split(workdir_complete) if '%fn' in part and self.sorter.fname: old = workdir_complete workdir_complete = os.path.join(path, part.replace('%fn', self.sorter.fname)) workdir_complete = get_unique_path(workdir_complete, create_dir=False) try: renamer(old, workdir_complete) except: logging.error(T('Cannot create directory %s'), clip_path(workdir_complete)) workdir_complete = old ok = False return workdir_complete, ok
def move(self, workdir_complete): ok = True if self.type == 'movie': move_to_parent = True # check if we should leave the files inside an extra folder if cfg.movie_extra_folders(): # if there is a folder in the download, leave it in an extra folder move_to_parent = not check_for_folder(workdir_complete) if move_to_parent: workdir_complete, ok = move_to_parent_folder(workdir_complete) else: workdir_complete, ok = move_to_parent_folder(workdir_complete) if not ok: return workdir_complete, False path, part = os.path.split(workdir_complete) if '%fn' in part and self.sorter.fname: old = workdir_complete workdir_complete = os.path.join( path, part.replace('%fn', self.sorter.fname)) workdir_complete = get_unique_path(workdir_complete, create_dir=False) try: renamer(old, workdir_complete) except: logging.error(T('Cannot create directory %s'), clip_path(workdir_complete)) workdir_complete = old ok = False return workdir_complete, ok
def rename(self, files, current_path): """ Renaming Date file """ logging.debug("Renaming Date file") # find the master file to rename for file in files: if is_full_path(file): filepath = os.path.normpath(file) else: filepath = os.path.normpath(os.path.join(current_path, file)) if os.path.exists(filepath): size = os.stat(filepath).st_size if size > cfg.movie_rename_limit.get_int(): if 'sample' not in file: self.fname, ext = os.path.splitext( os.path.split(file)[1]) newname = "%s%s" % (self.filename_set, ext) newname = newname.replace('%fn', self.fname) newpath = os.path.join(current_path, newname) if not os.path.exists(newpath): try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(T('Failed to rename: %s to %s'), clip_path(current_path), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, ext, self.filename_set, ()) break
def rename(self, files, current_path): """ Renaming Date file """ logging.debug("Renaming Date file") # find the master file to rename for file in files: if is_full_path(file): filepath = os.path.normpath(file) else: filepath = os.path.normpath(os.path.join(current_path, file)) if os.path.exists(filepath): size = os.stat(filepath).st_size if size > cfg.movie_rename_limit.get_int(): if 'sample' not in file: self.fname, ext = os.path.splitext(os.path.split(file)[1]) newname = "%s%s" % (self.filename_set, ext) newname = newname.replace('%fn', self.fname) newpath = os.path.join(current_path, newname) if not os.path.exists(newpath): try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(T('Failed to rename: %s to %s'), clip_path(current_path), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, ext, self.filename_set, ()) break
def _assemble(nzf, path, dupe): if os.path.exists(path): unique_path = get_unique_filename(path) if dupe: path = unique_path else: renamer(path, unique_path) fout = open(path, 'ab') if cfg.quick_check(): md5 = new_md5() else: md5 = None _type = nzf.type decodetable = nzf.decodetable for articlenum in decodetable: sleep(0.001) article = decodetable[articlenum] data = ArticleCache.do.load_article(article) if not data: logging.info(T('%s missing'), article) else: # yenc data already decoded, flush it out if _type == 'yenc': fout.write(data) if md5: md5.update(data) # need to decode uu data now elif _type == 'uu': data = data.split('\r\n') chunks = [] for line in data: if not line: continue if line == '-- ' or line.startswith('Posted via '): continue try: tmpdata = binascii.a2b_uu(line) chunks.append(tmpdata) except binascii.Error, msg: # Workaround for broken uuencoders by # /Fredrik Lundh nbytes = (((ord(line[0]) - 32) & 63) * 4 + 5) / 3 try: tmpdata = binascii.a2b_uu(line[:nbytes]) chunks.append(tmpdata) except binascii.Error, msg: logging.info('Decode failed in part %s: %s', article.article, msg) data = ''.join(chunks) fout.write(data) if md5: md5.update(data)
def rename(self, files, current_path): logging.debug("Renaming Date file") renamed = None #find the master file to rename for file in files: if is_full_path(file): filepath = file.replace('_UNPACK_', '') else: filepath = os.path.join(current_path, file) if os.path.exists(filepath): size = os.stat(filepath).st_size if size > 130000000: if 'sample' not in file: tmp, ext = os.path.splitext(file) newname = "%s%s" % (self.filename_set,ext) newname = newname.replace('%fn',tmp) newpath = os.path.join(current_path, newname) if not os.path.exists(newpath): try: logging.debug("Rename: %s to %s", filepath,newpath) renamer(filepath,newpath) except: logging.error(Ta('error-tvRename@2'), current_path, newpath) logging.debug("Traceback: ", exc_info = True) rename_similar(current_path, file, self.filename_set) break
def prepare_extraction_path(nzo): """ Based on the information that we have, generate the extraction path and create the directory. Separated so it can be called from DirectUnpacker """ one_folder = False marker_file = None # Determine class directory catdir = config.get_categories(nzo.cat).dir() if catdir.endswith('*'): catdir = catdir.strip('*') one_folder = True complete_dir = real_path(cfg.complete_dir.get_path(), catdir) complete_dir = long_path(complete_dir) # TV/Movie/Date Renaming code part 1 - detect and construct paths if cfg.enable_meta(): file_sorter = Sorter(nzo, nzo.cat) else: file_sorter = Sorter(None, nzo.cat) complete_dir = file_sorter.detect(nzo.final_name, complete_dir) if file_sorter.sort_file: one_folder = False complete_dir = sanitize_and_trim_path(complete_dir) if one_folder: workdir_complete = create_dirs(complete_dir) else: workdir_complete = get_unique_path(os.path.join( complete_dir, nzo.final_name), create_dir=True) marker_file = set_marker(workdir_complete) if not workdir_complete or not os.path.exists(workdir_complete): logging.error( T('Cannot create final folder %s') % unicoder(os.path.join(complete_dir, nzo.final_name))) raise IOError if cfg.folder_rename() and not one_folder: prefixed_path = prefix(workdir_complete, '_UNPACK_') tmp_workdir_complete = get_unique_path(prefix(workdir_complete, '_UNPACK_'), create_dir=False) try: renamer(workdir_complete, tmp_workdir_complete) except: pass # On failure, just use the original name # Is the unique path different? Then we also need to modify the final path if prefixed_path != tmp_workdir_complete: workdir_complete = workdir_complete + os.path.splitext( tmp_workdir_complete)[1] else: tmp_workdir_complete = workdir_complete return tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file
def rename(self, files, current_path): """ Rename for Series """ logging.debug("Renaming Series") largest = (None, None, 0) def to_filepath(f, current_path): if is_full_path(f): filepath = f.replace('_UNPACK_', '') else: filepath = os.path.join(current_path, f) return filepath # Create a generator of filepaths, ignore sample files and excluded files (vobs ect) filepaths = ((file, to_filepath(file, current_path)) for file in files if not RE_SAMPLE.search(file) \ and get_ext(file) not in EXCLUDED_FILE_EXTS) # Find the largest existing file for file, fp in filepaths: # If for some reason the file no longer exists, skip if not os.path.exists(fp): continue size = os.stat(fp).st_size f_file, f_fp, f_size = largest if size > f_size: largest = (file, fp, size) file, filepath, size = largest # >20MB if filepath and size > 20971520: tmp, self.ext = os.path.splitext(file) self.fname = tmp newname = "%s%s" % (self.filename_set, self.ext) # Replace %fn with the original filename newname = newname.replace('%fn', tmp) newpath = os.path.join(current_path, newname) # Replace %ext with extension newpath = newpath.replace('%ext', self.ext) if not os.path.exists(newpath): try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error("Failed to rename: %s to %s", current_path, newpath) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, self.ext, self.filename_set) else: logging.debug( 'Current path already exists, skipping rename, %s', newpath) else: logging.debug('Nothing to rename, %s', files)
def rename_with_ext(self, workdir_complete): """ Special renamer for %ext """ if self.sorter.rename_or_not and '%ext' in workdir_complete and self.ext: # Replace %ext with extension newpath = workdir_complete.replace('%ext', self.ext) try: renamer(workdir_complete, newpath) except: return newpath, False return newpath, True else: return workdir_complete, True
def prepare_extraction_path(nzo): """ Based on the information that we have, generate the extraction path and create the directory. Separated so it can be called from DirectUnpacker """ one_folder = False marker_file = None # Determine class directory catdir = config.get_categories(nzo.cat).dir() if catdir.endswith('*'): catdir = catdir.strip('*') one_folder = True complete_dir = real_path(cfg.complete_dir.get_path(), catdir) complete_dir = long_path(complete_dir) # TV/Movie/Date Renaming code part 1 - detect and construct paths if cfg.enable_meta(): file_sorter = Sorter(nzo, nzo.cat) else: file_sorter = Sorter(None, nzo.cat) complete_dir = file_sorter.detect(nzo.final_name, complete_dir) if file_sorter.sort_file: one_folder = False complete_dir = sanitize_and_trim_path(complete_dir) if one_folder: workdir_complete = create_dirs(complete_dir) else: workdir_complete = get_unique_path(os.path.join(complete_dir, nzo.final_name), create_dir=True) marker_file = set_marker(workdir_complete) if not workdir_complete or not os.path.exists(workdir_complete): logging.error(T('Cannot create final folder %s') % unicoder(os.path.join(complete_dir, nzo.final_name))) raise IOError if cfg.folder_rename() and not one_folder: prefixed_path = prefix(workdir_complete, '_UNPACK_') tmp_workdir_complete = get_unique_path(prefix(workdir_complete, '_UNPACK_'), create_dir=False) try: renamer(workdir_complete, tmp_workdir_complete) except: pass # On failure, just use the original name # Is the unique path different? Then we also need to modify the final path if prefixed_path != tmp_workdir_complete: workdir_complete = workdir_complete + os.path.splitext(tmp_workdir_complete)[1] else: tmp_workdir_complete = workdir_complete return tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file
def rename(self, files, current_path): """ Rename for Series """ logging.debug("Renaming Series") largest = (None, None, 0) def to_filepath(f, current_path): if is_full_path(f): filepath = f.replace('_UNPACK_', '') else: filepath = os.path.join(current_path, f) return filepath # Create a generator of filepaths, ignore sample files and excluded files (vobs ect) filepaths = ((file, to_filepath(file, current_path)) for file in files if not RE_SAMPLE.search(file) \ and get_ext(file) not in EXCLUDED_FILE_EXTS) # Find the largest existing file for file, fp in filepaths: # If for some reason the file no longer exists, skip if not os.path.exists(fp): continue size = os.stat(fp).st_size f_file, f_fp, f_size = largest if size > f_size: largest = (file, fp, size) file, filepath, size = largest # >20MB if filepath and size > 20971520: tmp, self.ext = os.path.splitext(file) self.fname = tmp newname = "%s%s" % (self.filename_set, self.ext) # Replace %fn with the original filename newname = newname.replace('%fn', tmp) newpath = os.path.join(current_path, newname) # Replace %ext with extension newpath = newpath.replace('%ext', self.ext) if not os.path.exists(newpath): try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error("Failed to rename: %s to %s", current_path, newpath) logging.info("Traceback: ", exc_info = True) rename_similar(current_path, self.ext, self.filename_set) else: logging.debug('Current path already exists, skipping rename, %s', newpath) else: logging.debug('Nothing to rename, %s', files)
def rename_similar(path, file, name): logging.debug('Renaming files similar to: %s to %s', file, name) file_prefix, ext = os.path.splitext(file) for root, dirs, files in os.walk(path): for _file in files: fpath = os.path.join(root, _file) tmp, ext = os.path.splitext(_file) if tmp == file_prefix: newname = "%s%s" % (name,ext) newname = newname.replace('%fn',tmp) newpath = os.path.join(path, newname) if not os.path.exists(newpath): try: logging.debug("Rename: %s to %s", fpath,newpath) renamer(fpath,newpath) except: logging.error(Ta('error-tvSimRename@2'), path, newpath) logging.debug("Traceback: ", exc_info = True)
def collapse_folder(oldpath, newpath): """ Rename folder, collapsing when there's just a single subfolder oldpath --> newpath OR oldpath/subfolder --> newpath """ orgpath = oldpath items = globber(oldpath) if len(items) == 1: folder_path = items[0] folder = os.path.split(folder_path)[1] if os.path.isdir(folder_path) and folder not in ('VIDEO_TS', 'AUDIO_TS'): logging.info('Collapsing %s', os.path.join(newpath, folder)) oldpath = folder_path renamer(oldpath, newpath) try: remove_dir(orgpath) except: pass
def _assemble(nzf, path, dupe): if os.path.exists(path): unique_path = get_unique_filename(path) if dupe: path = unique_path else: renamer(path, unique_path) fout = open(path, "ab") if cfg.quick_check(): md5 = new_md5() else: md5 = None decodetable = nzf.decodetable for articlenum in decodetable: # Break if deleted during writing if nzf.nzo.status is Status.DELETED: break # Sleep to allow decoder/assembler switching sleep(0.001) article = decodetable[articlenum] data = ArticleCache.do.load_article(article) if not data: logging.info(T("%s missing"), article) else: # yenc data already decoded, flush it out fout.write(data) if md5: md5.update(data) fout.flush() fout.close() set_permissions(path) if md5: nzf.md5sum = md5.digest() del md5 return path
def _assemble(nzf, path, dupe): if os.path.exists(path): unique_path = get_unique_filename(path) if dupe: path = unique_path else: renamer(path, unique_path) fout = open(path, 'ab') if cfg.quick_check(): md5 = hashlib.md5() else: md5 = None decodetable = nzf.decodetable for articlenum in decodetable: # Break if deleted during writing if nzf.nzo.status is Status.DELETED: break # Sleep to allow decoder/assembler switching sleep(0.0001) article = decodetable[articlenum] data = ArticleCache.do.load_article(article) if not data: logging.info(T('%s missing'), article) else: # yenc data already decoded, flush it out fout.write(data) if md5: md5.update(data) fout.flush() fout.close() set_permissions(path) if md5: nzf.md5sum = md5.digest() del md5 return path
def rename_similar(path, file, name): logging.debug('Renaming files similar to: %s to %s', file, name) file_prefix, ext = os.path.splitext(file) for root, dirs, files in os.walk(path): for _file in files: fpath = os.path.join(root, _file) tmp, ext = os.path.splitext(_file) if tmp == file_prefix: newname = "%s%s" % (name, ext) newname = newname.replace('%fn', tmp) newpath = os.path.join(path, newname) if not os.path.exists(newpath): try: logging.debug("Rename: %s to %s", fpath, newpath) renamer(fpath, newpath) except: logging.error( Ta('Failed to rename similar file: %s to %s'), path, newpath) logging.info("Traceback: ", exc_info=True)
def rename_and_collapse_folder(oldpath, newpath, files): """ Rename folder, collapsing when there's just a single subfolder oldpath --> newpath OR oldpath/subfolder --> newpath Modify list of filenames accordingly """ orgpath = oldpath items = globber(oldpath) if len(items) == 1: folder_path = items[0] folder = os.path.split(folder_path)[1] if os.path.isdir(folder_path) and folder not in ('VIDEO_TS', 'AUDIO_TS'): logging.info('Collapsing %s', os.path.join(newpath, folder)) oldpath = folder_path oldpath = os.path.normpath(oldpath) newpath = os.path.normpath(newpath) files = [os.path.normpath(f).replace(oldpath, newpath) for f in files] renamer(oldpath, newpath) try: remove_dir(orgpath) except: pass return files
def process_job(nzo): """ Process one job """ assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject) start = time.time() # keep track of whether we can continue all_ok = True # keep track of par problems par_error = False # keep track of any unpacking errors unpack_error = False # Signal empty download, for when 'empty_postproc' is enabled empty = False nzb_list = [] # These need to be initialized in case of a crash workdir_complete = "" postproc_time = 0 # @UnusedVariable -- pep8 bug? script_log = "" script_line = "" crash_msg = "" # Get the job flags nzo.save_attribs() flag_repair, flag_unpack, flag_delete = nzo.repair_opts # Normalize PP if flag_delete: flag_unpack = True if flag_unpack: flag_repair = True # Get the NZB name filename = nzo.final_name if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete): # After streaming, force +D nzo.set_pp(3) nzo.status = Status.FAILED nzo.save_attribs() all_ok = False if nzo.fail_msg: # Special case: aborted due to too many missing data nzo.status = Status.FAILED nzo.save_attribs() all_ok = False par_error = True unpack_error = 1 try: # Get the folder containing the download result workdir = nzo.downpath tmp_workdir_complete = None # if no files are present (except __admin__), fail the job if all_ok and len(globber(workdir)) < 2: if nzo.precheck: _enough, ratio = nzo.check_quality() req_ratio = float(cfg.req_completion_rate()) / 100.0 # Make sure that rounded ratio doesn't equal required ratio # when it is actually below required if (ratio < req_ratio) and (req_ratio - ratio) < 0.001: ratio = req_ratio - 0.001 emsg = "%.1f%%" % (ratio * 100.0) emsg2 = "%.1f%%" % float(cfg.req_completion_rate()) emsg = T("Download might fail, only %s of required %s available") % (emsg, emsg2) else: emsg = T("Download failed - Not on your server(s)") empty = True nzo.fail_msg = emsg nzo.set_unpack_info("Fail", emsg) nzo.status = Status.FAILED # do not run unpacking or parity verification flag_repair = flag_unpack = False all_ok = cfg.empty_postproc() and empty if not all_ok: par_error = True unpack_error = 1 script = nzo.script cat = nzo.cat logging.info( "Starting PostProcessing on %s" + " => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s", filename, flag_repair, flag_unpack, flag_delete, script, cat, ) # Set complete dir to workdir in case we need to abort workdir_complete = workdir dirname = nzo.final_name marker_file = None # Par processing, if enabled if all_ok and flag_repair: if not check_win_maxpath(workdir): crash_msg = T('Path exceeds 260, repair by "par2" is not possible') raise WindowsError par_error, re_add = parring(nzo, workdir) if re_add: # Try to get more par files return False # Check if user allows unsafe post-processing if flag_repair and cfg.safe_postproc(): all_ok = all_ok and not par_error if all_ok: fix_unix_encoding(workdir) one_folder = False # Determine class directory if cfg.create_group_folders(): complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix) complete_dir = create_dirs(complete_dir) else: catdir = config.get_categories(cat).dir() if catdir.endswith("*"): catdir = catdir.strip("*") one_folder = True complete_dir = real_path(cfg.complete_dir.get_path(), catdir) complete_dir = long_path(complete_dir) # TV/Movie/Date Renaming code part 1 - detect and construct paths if cfg.enable_meta(): file_sorter = Sorter(nzo, cat) else: file_sorter = Sorter(None, cat) complete_dir = file_sorter.detect(dirname, complete_dir) if file_sorter.sort_file: one_folder = False complete_dir = sanitize_and_trim_path(complete_dir) if one_folder: workdir_complete = create_dirs(complete_dir) else: workdir_complete = get_unique_path(os.path.join(complete_dir, dirname), create_dir=True) marker_file = set_marker(workdir_complete) if not workdir_complete or not os.path.exists(workdir_complete): crash_msg = T("Cannot create final folder %s") % unicoder(os.path.join(complete_dir, dirname)) raise IOError if cfg.folder_rename() and not one_folder: tmp_workdir_complete = prefix(workdir_complete, "_UNPACK_") try: renamer(workdir_complete, tmp_workdir_complete) except: pass # On failure, just use the original name else: tmp_workdir_complete = workdir_complete newfiles = [] # Run Stage 2: Unpack if flag_unpack: if all_ok: # set the current nzo status to "Extracting...". Used in History nzo.status = Status.EXTRACTING logging.info("Running unpack_magic on %s", filename) short_complete = short_path(tmp_workdir_complete) unpack_error, newfiles = unpack_magic( nzo, short_path(workdir), short_complete, flag_delete, one_folder, (), (), (), (), () ) if short_complete != tmp_workdir_complete: newfiles = [f.replace(short_complete, tmp_workdir_complete) for f in newfiles] logging.info("unpack_magic finished on %s", filename) else: nzo.set_unpack_info("Unpack", T("No post-processing because of failed verification")) if cfg.safe_postproc(): all_ok = all_ok and not unpack_error if all_ok: # Move any (left-over) files to destination nzo.status = Status.MOVING nzo.set_action_line(T("Moving"), "...") for root, _dirs, files in os.walk(workdir): if not root.endswith(JOB_ADMIN): for file_ in files: path = os.path.join(root, file_) new_path = path.replace(workdir, tmp_workdir_complete) ok, new_path = move_to_path(path, new_path) newfiles.append(new_path) if not ok: nzo.set_unpack_info( "Unpack", T("Failed moving %s to %s") % (unicoder(path), unicoder(new_path)) ) all_ok = False break # Set permissions right set_permissions(tmp_workdir_complete) if all_ok and marker_file: del_marker(os.path.join(tmp_workdir_complete, marker_file)) remove_from_list(marker_file, newfiles) if all_ok: # Remove files matching the cleanup list cleanup_list(tmp_workdir_complete, True) # Check if this is an NZB-only download, if so redirect to queue # except when PP was Download-only if flag_repair: nzb_list = nzb_redirect( tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority ) else: nzb_list = None if nzb_list: nzo.set_unpack_info("Download", T("Sent %s to queue") % unicoder(nzb_list)) cleanup_empty_directories(tmp_workdir_complete) else: cleanup_list(tmp_workdir_complete, False) script_output = "" script_ret = 0 if not nzb_list: # Give destination its final name if cfg.folder_rename() and tmp_workdir_complete and not one_folder: if all_ok: try: newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles) except: logging.error( T('Error renaming "%s" to "%s"'), clip_path(tmp_workdir_complete), clip_path(workdir_complete), ) logging.info("Traceback: ", exc_info=True) # Better disable sorting because filenames are all off now file_sorter.sort_file = None else: workdir_complete = tmp_workdir_complete.replace("_UNPACK_", "_FAILED_") workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False) workdir_complete = workdir_complete if empty: job_result = -1 else: job_result = int(par_error) + int(bool(unpack_error)) * 2 if cfg.ignore_samples(): remove_samples(workdir_complete) # TV/Movie/Date Renaming code part 2 - rename and move files to parent folder if all_ok and file_sorter.sort_file: if newfiles: file_sorter.rename(newfiles, workdir_complete) workdir_complete, ok = file_sorter.move(workdir_complete) else: workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete) if not ok: nzo.set_unpack_info("Unpack", T("Failed to move files")) all_ok = False # Run the user script script_path = make_script_path(script) if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path: # set the current nzo status to "Ext Script...". Used in History nzo.status = Status.RUNNING nzo.set_action_line(T("Running script"), unicoder(script)) nzo.set_unpack_info("Script", T("Running user script %s") % unicoder(script), unique=True) script_log, script_ret = external_processing( short_path(script_path, False), short_path(workdir_complete, False), nzo.filename, dirname, cat, nzo.group, job_result, nzo.nzo_info.get("failure", ""), ) script_line = get_last_line(script_log) if script_log: script_output = nzo.nzo_id if script_line: nzo.set_unpack_info("Script", unicoder(script_line), unique=True) else: nzo.set_unpack_info("Script", T("Ran %s") % unicoder(script), unique=True) else: script = "" script_line = "" script_ret = 0 # Maybe bad script result should fail job if script_ret and cfg.script_can_fail(): script_error = True all_ok = False nzo.fail_msg = T("Script exit code is %s") % script_ret else: script_error = False # Email the results if (not nzb_list) and cfg.email_endjob(): if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error or script_error)): emailer.endjob( dirname, cat, all_ok, workdir_complete, nzo.bytes_downloaded, nzo.fail_msg, nzo.unpack_info, script, TRANS(script_log), script_ret, ) if script_output: # Can do this only now, otherwise it would show up in the email if script_ret: script_ret = "Exit(%s) " % script_ret else: script_ret = "" if script_line: nzo.set_unpack_info( "Script", u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, unicoder(script_line), urllib.quote(script_output), T("More")), unique=True, ) else: nzo.set_unpack_info( "Script", u'%s<a href="./scriptlog?name=%s">%s</a>' % (script_ret, urllib.quote(script_output), T("View script output")), unique=True, ) # Cleanup again, including NZB files if all_ok: cleanup_list(workdir_complete, False) # Force error for empty result all_ok = all_ok and not empty # Update indexer with results if cfg.rating_enable(): if nzo.encrypted > 0: Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED) if empty: hosts = map(lambda s: s.host, sabnzbd.downloader.Downloader.do.nzo_servers(nzo)) if not hosts: hosts = [None] for host in hosts: Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED, host) # Show final status in history if all_ok: growler.send_notification(T("Download Completed"), filename, "complete") nzo.status = Status.COMPLETED else: growler.send_notification(T("Download Failed"), filename, "failed") nzo.status = Status.FAILED except: logging.error(T("Post Processing Failed for %s (%s)"), filename, crash_msg) if not crash_msg: logging.info("Traceback: ", exc_info=True) crash_msg = T("see logfile") nzo.fail_msg = T("PostProcessing was aborted (%s)") % unicoder(crash_msg) growler.send_notification(T("Download Failed"), filename, "failed") nzo.status = Status.FAILED par_error = True all_ok = False if cfg.email_endjob(): emailer.endjob( dirname, cat, all_ok, clip_path(workdir_complete), nzo.bytes_downloaded, nzo.fail_msg, nzo.unpack_info, "", "", 0, ) if all_ok: # If the folder only contains one file OR folder, have that as the path # Be aware that series/generic/date sorting may move a single file into a folder containing other files workdir_complete = one_file_or_folder(workdir_complete) workdir_complete = os.path.normpath(workdir_complete) # Log the overall time taken for postprocessing postproc_time = int(time.time() - start) # Create the history DB instance history_db = database.get_history_handle() # Add the nzo to the database. Only the path, script and time taken is passed # Other information is obtained from the nzo history_db.add_history_db(nzo, clip_path(workdir_complete), nzo.downpath, postproc_time, script_log, script_line) # The connection is only used once, so close it here history_db.close() # Clean up the NZO try: logging.info("Cleaning up %s (keep_basic=%s)", filename, str(not all_ok)) sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok) except: logging.error(T("Cleanup of %s failed."), nzo.final_name) logging.info("Traceback: ", exc_info=True) # Remove download folder if all_ok: try: if os.path.exists(workdir): logging.debug("Removing workdir %s", workdir) remove_all(workdir, recursive=True) except: logging.error(T("Error removing workdir (%s)"), clip_path(workdir)) logging.info("Traceback: ", exc_info=True) # Use automatic retry link on par2 errors and encrypted/bad RARs if par_error or unpack_error in (2, 3): try_alt_nzb(nzo) return True
def parring(nzo, workdir): """ Perform par processing. Returns: (par_error, re_add) """ filename = nzo.final_name notifier.send_notification(T('Post-processing'), filename, 'pp') logging.info('Starting verification and repair of %s', filename) # Get verification status of sets verified = sabnzbd.load_data(VERIFIED_FILE, nzo.workpath, remove=False) or {} repair_sets = nzo.partable.keys() re_add = False par_error = False single = len(repair_sets) == 1 if repair_sets: for setname in repair_sets: if cfg.ignore_samples() and RE_SAMPLE.search(setname.lower()): continue if not verified.get(setname, False): logging.info("Running verification and repair on set %s", setname) parfile_nzf = nzo.partable[setname] if os.path.exists(os.path.join(nzo.downpath, parfile_nzf.filename)) or parfile_nzf.extrapars: need_re_add, res = par2_repair(parfile_nzf, nzo, workdir, setname, single=single) # Was it aborted? if not nzo.pp_active: re_add = False par_error = True break re_add = re_add or need_re_add verified[setname] = res else: continue par_error = par_error or not res else: # Obfuscated par2 check logging.info('No par2 sets found, running obfuscated check on %s', filename) # Get the NZF's and sort them based on size nzfs_sorted = sorted(nzo.finished_files, key=lambda x: x.bytes) # We will have to make 'fake' par files that are recognized par2_vol = 0 par2_filename = None for nzf_try in nzfs_sorted: # run through list of files, looking for par2 signature.. logging.debug("Checking par2 signature of %s", nzf_try.filename) try: nzf_path = os.path.join(workdir, nzf_try.filename) if(is_parfile(nzf_path)): # We need 1 base-name so they are recognized as 1 set if not par2_filename: par2_filename = nzf_path # Rename so handle_par2() picks it up newpath = '%s.vol%d+%d.par2' % (par2_filename, par2_vol, par2_vol + 1) renamer(nzf_path, newpath) nzf_try.filename = os.path.split(newpath)[1] # Let the magic happen nzo.handle_par2(nzf_try, file_done=True) par2_vol += 1 except: pass if par2_vol > 0: # Pars found, we do it again par_error, re_add = parring(nzo, workdir) else: # We must not have found any par2.. logging.info("No par2 sets for %s", filename) nzo.set_unpack_info('Repair', T('[%s] No par2 sets') % unicoder(filename)) if cfg.sfv_check() and not verified.get('', False): par_error = not try_sfv_check(nzo, workdir, '') verified[''] = not par_error # If still no success, do RAR-check if not par_error and cfg.enable_unrar(): par_error = not try_rar_check(nzo, workdir, '') verified[''] = not par_error if re_add: logging.info('Re-added %s to queue', filename) if nzo.priority != TOP_PRIORITY: nzo.priority = REPAIR_PRIORITY sabnzbd.nzbqueue.add_nzo(nzo) sabnzbd.downloader.Downloader.do.resume_from_postproc() sabnzbd.save_data(verified, VERIFIED_FILE, nzo.workpath) logging.info('Verification and repair finished for %s', filename) return par_error, re_add
def process_job(nzo): """ Process one job """ assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject) start = time.time() # keep track of whether we can continue all_ok = True # keep track of par problems par_error = False # keep track of any unpacking errors unpack_error = False # Signal empty download, for when 'empty_postproc' is enabled empty = False nzb_list = [] # These need to be initialised incase of a crash workdir_complete = '' postproc_time = 0 script_log = '' script_line = '' crash_msg = '' ## Get the job flags nzo.save_attribs() flag_repair, flag_unpack, flag_delete = nzo.repair_opts # Normalize PP if flag_delete: flag_unpack = True if flag_unpack: flag_repair = True # Get the NZB name filename = nzo.final_name msgid = nzo.msgid if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete): # After streaming, force +D nzo.set_pp(3) nzo.status = Status.FAILED nzo.save_attribs() all_ok = False if nzo.fail_msg: # Special case: aborted due to too many missing data nzo.status = Status.FAILED nzo.save_attribs() all_ok = False par_error = unpack_error = True try: # Get the folder containing the download result workdir = nzo.downpath tmp_workdir_complete = None # if no files are present (except __admin__), fail the job if all_ok and len(globber(workdir)) < 2: if nzo.precheck: enough, ratio = nzo.check_quality() req_ratio = float(cfg.req_completion_rate()) / 100.0 # Make sure that rounded ratio doesn't equal required ratio # when it is actually below required if (ratio < req_ratio) and (req_ratio - ratio) < 0.001: ratio = req_ratio - 0.001 emsg = '%.1f%%' % (ratio * 100.0) emsg2 = '%.1f%%' % float(cfg.req_completion_rate()) emsg = T('Download might fail, only %s of required %s available') % (emsg, emsg2) else: emsg = T('Download failed - Out of your server\'s retention?') empty = True nzo.fail_msg = emsg nzo.set_unpack_info('Fail', emsg) nzo.status = Status.FAILED # do not run unpacking or parity verification flag_repair = flag_unpack = False all_ok = cfg.empty_postproc() and empty if not all_ok: par_error = unpack_error = True script = nzo.script cat = nzo.cat logging.info('Starting PostProcessing on %s' + \ ' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s', filename, flag_repair, flag_unpack, flag_delete, script, cat) ## Par processing, if enabled if all_ok and flag_repair: par_error, re_add = parring(nzo, workdir) if re_add: # Try to get more par files return False ## Check if user allows unsafe post-processing if flag_repair and cfg.safe_postproc(): all_ok = all_ok and not par_error # Set complete dir to workdir in case we need to abort workdir_complete = workdir dirname = nzo.final_name marker_file = None if all_ok: one_folder = False ## Determine class directory if cfg.create_group_folders(): complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix) complete_dir = create_dirs(complete_dir) else: catdir = config.get_categories(cat).dir() if catdir.endswith('*'): catdir = catdir.strip('*') one_folder = True complete_dir = real_path(cfg.complete_dir.get_path(), catdir) ## TV/Movie/Date Renaming code part 1 - detect and construct paths file_sorter = Sorter(cat) complete_dir = file_sorter.detect(dirname, complete_dir) if file_sorter.sort_file: one_folder = False if one_folder: workdir_complete = create_dirs(complete_dir) else: workdir_complete = get_unique_path(os.path.join(complete_dir, dirname), create_dir=True) marker_file = set_marker(workdir_complete) if not workdir_complete or not os.path.exists(workdir_complete): crash_msg = T('Cannot create final folder %s') % unicoder(os.path.join(complete_dir, dirname)) raise IOError if cfg.folder_rename() and not one_folder: tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_') try: renamer(workdir_complete, tmp_workdir_complete) except: pass # On failure, just use the original name else: tmp_workdir_complete = workdir_complete newfiles = [] ## Run Stage 2: Unpack if flag_unpack: if all_ok: #set the current nzo status to "Extracting...". Used in History nzo.status = Status.EXTRACTING logging.info("Running unpack_magic on %s", filename) unpack_error, newfiles = unpack_magic(nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), ()) logging.info("unpack_magic finished on %s", filename) else: nzo.set_unpack_info('Unpack', T('No post-processing because of failed verification')) if cfg.safe_postproc(): all_ok = all_ok and not unpack_error if all_ok: ## Move any (left-over) files to destination nzo.status = Status.MOVING nzo.set_action_line(T('Moving'), '...') for root, dirs, files in os.walk(workdir): if not root.endswith(JOB_ADMIN): for file_ in files: path = os.path.join(root, file_) new_path = path.replace(workdir, tmp_workdir_complete) ok, new_path = move_to_path(path, new_path) newfiles.append(new_path) if not ok: nzo.set_unpack_info('Unpack', T('Failed moving %s to %s') % (unicoder(path), unicoder(new_path))) all_ok = False break ## Set permissions right set_permissions(tmp_workdir_complete) if all_ok and marker_file: del_marker(os.path.join(tmp_workdir_complete, marker_file)) remove_from_list(marker_file, newfiles) if all_ok: ## Remove files matching the cleanup list cleanup_list(tmp_workdir_complete, True) ## Check if this is an NZB-only download, if so redirect to queue ## except when PP was Download-only if flag_repair: nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority) else: nzb_list = None if nzb_list: nzo.set_unpack_info('Download', T('Sent %s to queue') % unicoder(nzb_list)) cleanup_empty_directories(tmp_workdir_complete) else: cleanup_list(tmp_workdir_complete, False) script_output = '' script_ret = 0 if not nzb_list: ## Give destination its final name if cfg.folder_rename() and tmp_workdir_complete and not one_folder: if all_ok: try: newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles) except: logging.error(Ta('Error renaming "%s" to "%s"'), tmp_workdir_complete, workdir_complete) logging.info('Traceback: ', exc_info = True) # Better disable sorting because filenames are all off now file_sorter.sort_file = None else: workdir_complete = tmp_workdir_complete.replace('_UNPACK_', '_FAILED_') workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False) if empty: job_result = -1 else: job_result = int(par_error) + int(unpack_error)*2 if cfg.ignore_samples() > 0: remove_samples(workdir_complete) ## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder if all_ok and file_sorter.sort_file: if newfiles: file_sorter.rename(newfiles, workdir_complete) workdir_complete, ok = file_sorter.move(workdir_complete) else: workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete) if not ok: nzo.set_unpack_info('Unpack', T('Failed to move files')) all_ok = False ## Run the user script script_path = make_script_path(script) if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path: #set the current nzo status to "Ext Script...". Used in History nzo.status = Status.RUNNING nzo.set_action_line(T('Running script'), unicoder(script)) nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True) script_log, script_ret = external_processing(script_path, workdir_complete, nzo.filename, msgid, dirname, cat, nzo.group, job_result) script_line = get_last_line(script_log) if script_log: script_output = nzo.nzo_id if script_line: nzo.set_unpack_info('Script', unicoder(script_line), unique=True) else: nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True) else: script = "" script_line = "" script_ret = 0 ## Email the results if (not nzb_list) and cfg.email_endjob(): if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error)): emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded, nzo.fail_msg, nzo.unpack_info, script, TRANS(script_log), script_ret) if script_output: # Can do this only now, otherwise it would show up in the email if script_ret: script_ret = 'Exit(%s) ' % script_ret else: script_ret = '' if script_line: nzo.set_unpack_info('Script', u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, unicoder(script_line), urllib.quote(script_output), T('More')), unique=True) else: nzo.set_unpack_info('Script', u'%s<a href="./scriptlog?name=%s">%s</a>' % (script_ret, urllib.quote(script_output), T('View script output')), unique=True) ## Cleanup again, including NZB files if all_ok: cleanup_list(workdir_complete, False) ## Remove newzbin bookmark, if any if msgid and all_ok: Bookmarks.do.del_bookmark(msgid) elif all_ok and isinstance(nzo.url, str): sabnzbd.proxy_rm_bookmark(nzo.url) ## Force error for empty result all_ok = all_ok and not empty ## Show final status in history if all_ok: growler.send_notification(T('Download Completed'), filename, 'complete') nzo.status = Status.COMPLETED else: growler.send_notification(T('Download Failed'), filename, 'complete') nzo.status = Status.FAILED except: logging.error(Ta('Post Processing Failed for %s (%s)'), filename, crash_msg) if not crash_msg: logging.info("Traceback: ", exc_info = True) crash_msg = T('see logfile') nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(crash_msg) growler.send_notification(T('Download Failed'), filename, 'complete') nzo.status = Status.FAILED par_error = True all_ok = False if cfg.email_endjob(): emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded, nzo.fail_msg, nzo.unpack_info, '', '', 0) if all_ok: # If the folder only contains one file OR folder, have that as the path # Be aware that series/generic/date sorting may move a single file into a folder containing other files workdir_complete = one_file_or_folder(workdir_complete) workdir_complete = os.path.normpath(workdir_complete) # Log the overall time taken for postprocessing postproc_time = int(time.time() - start) # Create the history DB instance history_db = database.get_history_handle() # Add the nzo to the database. Only the path, script and time taken is passed # Other information is obtained from the nzo history_db.add_history_db(nzo, workdir_complete, nzo.downpath, postproc_time, script_log, script_line) # The connection is only used once, so close it here history_db.close() ## Clean up the NZO try: logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok)) sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok) except: logging.error(Ta('Cleanup of %s failed.'), nzo.final_name) logging.info("Traceback: ", exc_info = True) ## Remove download folder if all_ok: try: if os.path.exists(workdir): logging.debug('Removing workdir %s', workdir) remove_all(workdir, recursive=True) except: logging.error(Ta('Error removing workdir (%s)'), workdir) logging.info("Traceback: ", exc_info = True) return True
def rename(self, _files, current_path): logging.debug("Renaming Generic file") def filter_files(_file, current_path): if is_full_path(_file): filepath = _file.replace('_UNPACK_', '') else: filepath = os.path.join(current_path, _file) if os.path.exists(filepath): size = os.stat(filepath).st_size if size > 314572800 and not RE_SAMPLE.search(_file) \ and get_ext(_file) not in EXCLUDED_FILE_EXTS: return True return False renamed = False # remove any files below 300MB from this list files = [_file for _file in _files if filter_files(_file, current_path)] length = len(files) ## Single File Handling if length == 1: file = files[0] if is_full_path(file): filepath = file.replace('_UNPACK_', '') else: filepath = os.path.join(current_path, file) if os.path.exists(filepath): tmp, ext = os.path.splitext(file) newname = "%s%s" % (self.filename_set,ext) newname = newname.replace('%fn',tmp) newpath = os.path.join(current_path, newname) try: logging.debug("Rename: %s to %s", filepath,newpath) renamer(filepath,newpath) except: logging.error(Ta('error-tvRename@2'), filepath, newpath) logging.debug("Traceback: ", exc_info = True) rename_similar(current_path, file, self.filename_set) ## Sequence File Handling # if there is more than one extracted file check for CD1/1/A in the title elif self.extra: matched_files = check_for_multiple(files) # rename files marked as in a set if matched_files: logging.debug("Renaming a series of generic files (%s)", matched_files) for index, file in matched_files.iteritems(): filepath = os.path.join(current_path, file) tmp, ext = os.path.splitext(file) name = '%s%s' % (self.filename_set, self.extra) name = name.replace('%1', str(index)).replace('%fn',tmp) name = name + ext newpath = os.path.join(current_path, name) try: logging.debug("Rename: %s to %s", filepath,newpath) renamer(filepath,newpath) except: logging.error(Ta('error-tvRename@2'), filepath, newpath) logging.debug("Traceback: ", exc_info = True) rename_similar(current_path, file, self.filename_set) else: logging.debug("Movie files not in sequence %s", _files)
def process_job(nzo): """ Process one job """ assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject) start = time.time() # keep track of whether we can continue all_ok = True # keep track of par problems par_error = False # keep track of any unpacking errors unpack_error = False nzb_list = [] # These need to be initialised incase of a crash workdir_complete = '' postproc_time = 0 script_log = '' script_line = '' crash_msg = '' ## Get the job flags nzo.save_attribs() flag_repair, flag_unpack, flag_delete = nzo.repair_opts # Normalize PP if flag_delete: flag_unpack = True if flag_unpack: flag_repair = True # Get the NZB name filename = nzo.final_name msgid = nzo.msgid if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete): # After streaming, force +D nzo.set_pp(3) nzo.status = 'Failed' nzo.save_attribs() all_ok = False try: # Get the folder containing the download result workdir = nzo.downpath tmp_workdir_complete = None # if no files are present (except __admin__), fail the job if len(globber(workdir)) < 2: emsg = T('Download failed - Out of your server\'s retention?') nzo.fail_msg = emsg nzo.status = 'Failed' # do not run unpacking or parity verification flag_repair = flag_unpack = False par_error = unpack_error = True all_ok = False script = nzo.script cat = nzo.cat logging.info('Starting PostProcessing on %s' + \ ' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s', filename, flag_repair, flag_unpack, flag_delete, script, cat) ## Par processing, if enabled if flag_repair: par_error, re_add = parring(nzo, workdir) if re_add: # Try to get more par files return False ## Check if user allows unsafe post-processing if flag_repair and cfg.safe_postproc(): all_ok = all_ok and not par_error # Set complete dir to workdir in case we need to abort workdir_complete = workdir dirname = nzo.final_name if all_ok: one_folder = False ## Determine class directory if cfg.create_group_folders(): complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix) complete_dir = create_dirs(complete_dir) else: catdir = config.get_categories(cat).dir() if catdir.endswith('*'): catdir = catdir.strip('*') one_folder = True complete_dir = real_path(cfg.complete_dir.get_path(), catdir) ## TV/Movie/Date Renaming code part 1 - detect and construct paths file_sorter = Sorter(cat) complete_dir = file_sorter.detect(dirname, complete_dir) if file_sorter.is_sortfile(): one_folder = False if one_folder: workdir_complete = create_dirs(complete_dir) else: workdir_complete = get_unique_path(os.path.join( complete_dir, dirname), create_dir=True) if not workdir_complete or not os.path.exists(workdir_complete): crash_msg = T('Cannot create final folder %s') % unicoder( os.path.join(complete_dir, dirname)) raise IOError if cfg.folder_rename() and not one_folder: tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_') try: renamer(workdir_complete, tmp_workdir_complete) except: pass # On failure, just use the original name else: tmp_workdir_complete = workdir_complete newfiles = [] ## Run Stage 2: Unpack if flag_unpack: if all_ok: #set the current nzo status to "Extracting...". Used in History nzo.status = 'Extracting' logging.info("Running unpack_magic on %s", filename) unpack_error, newfiles = unpack_magic( nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), ()) logging.info("unpack_magic finished on %s", filename) else: nzo.set_unpack_info( 'Unpack', T('No post-processing because of failed verification')) if cfg.safe_postproc(): all_ok = all_ok and not unpack_error if all_ok: ## Move any (left-over) files to destination nzo.status = 'Moving' nzo.set_action_line(T('Moving'), '...') for root, dirs, files in os.walk(workdir): if not root.endswith(JOB_ADMIN): for file_ in files: path = os.path.join(root, file_) new_path = path.replace(workdir, tmp_workdir_complete) new_path = get_unique_filename(new_path) move_to_path(path, new_path, unique=False) ## Set permissions right if not sabnzbd.WIN32: perm_script(tmp_workdir_complete, cfg.umask()) if all_ok: ## Remove files matching the cleanup list cleanup_list(tmp_workdir_complete, True) ## Check if this is an NZB-only download, if so redirect to queue ## except when PP was Download-only if flag_repair: nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority) else: nzb_list = None if nzb_list: nzo.set_unpack_info( 'Download', T('Sent %s to queue') % unicoder(nzb_list)) try: remove_dir(tmp_workdir_complete) except: pass else: cleanup_list(tmp_workdir_complete, False) script_output = '' script_ret = 0 if not nzb_list: ## Give destination its final name if cfg.folder_rename() and tmp_workdir_complete and not one_folder: if not all_ok: workdir_complete = tmp_workdir_complete.replace( '_UNPACK_', '_FAILED_') workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False) try: collapse_folder(tmp_workdir_complete, workdir_complete) except: logging.error(Ta('Error renaming "%s" to "%s"'), tmp_workdir_complete, workdir_complete) logging.info("Traceback: ", exc_info=True) job_result = int(par_error) + int(unpack_error) * 2 if cfg.ignore_samples() > 0: remove_samples(workdir_complete) ## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder if all_ok: if newfiles and file_sorter.is_sortfile(): file_sorter.rename(newfiles, workdir_complete) workdir_complete = file_sorter.move(workdir_complete) ## Run the user script script_path = make_script_path(script) if all_ok and (not nzb_list) and script_path: #set the current nzo status to "Ext Script...". Used in History nzo.status = 'Running' nzo.set_action_line(T('Running script'), unicoder(script)) nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True) script_log, script_ret = external_processing( script_path, workdir_complete, nzo.filename, msgid, dirname, cat, nzo.group, job_result) script_line = get_last_line(script_log) if script_log: script_output = nzo.nzo_id if script_line: nzo.set_unpack_info('Script', unicoder(script_line), unique=True) else: nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True) else: script = "" script_line = "" script_ret = 0 ## Email the results if (not nzb_list) and cfg.email_endjob(): if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error)): emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded, nzo.unpack_info, script, TRANS(script_log), script_ret) if script_output: # Can do this only now, otherwise it would show up in the email if script_ret: script_ret = 'Exit(%s) ' % script_ret else: script_ret = '' if script_line: nzo.set_unpack_info( 'Script', u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, unicoder(script_line), urllib.quote(script_output), T('More')), unique=True) else: nzo.set_unpack_info('Script', u'%s<a href="./scriptlog?name=%s">%s</a>' % (script_ret, urllib.quote(script_output), T('View script output')), unique=True) ## Cleanup again, including NZB files cleanup_list(workdir_complete, False) ## Remove newzbin bookmark, if any if msgid and all_ok: Bookmarks.do.del_bookmark(msgid) elif all_ok: sabnzbd.proxy_rm_bookmark(nzo.url) ## Show final status in history if all_ok: osx.sendGrowlMsg(T('Download Completed'), filename, osx.NOTIFICATION['complete']) nzo.status = 'Completed' else: osx.sendGrowlMsg(T('Download Failed'), filename, osx.NOTIFICATION['complete']) nzo.status = 'Failed' except: logging.error(Ta('Post Processing Failed for %s (%s)'), filename, crash_msg) if not crash_msg: logging.info("Traceback: ", exc_info=True) crash_msg = T('see logfile') nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder( crash_msg) osx.sendGrowlMsg(T('Download Failed'), filename, osx.NOTIFICATION['complete']) nzo.status = 'Failed' par_error = True all_ok = False if all_ok: # If the folder only contains one file OR folder, have that as the path # Be aware that series/generic/date sorting may move a single file into a folder containing other files workdir_complete = one_file_or_folder(workdir_complete) workdir_complete = os.path.normpath(workdir_complete) # Log the overall time taken for postprocessing postproc_time = int(time.time() - start) # Create the history DB instance history_db = database.get_history_handle() # Add the nzo to the database. Only the path, script and time taken is passed # Other information is obtained from the nzo history_db.add_history_db(nzo, workdir_complete, nzo.downpath, postproc_time, script_log, script_line) # The connection is only used once, so close it here history_db.close() ## Clean up the NZO try: logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok)) sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok) except: logging.error(Ta('Cleanup of %s failed.'), nzo.final_name) logging.info("Traceback: ", exc_info=True) ## Remove download folder if all_ok: try: if os.path.exists(workdir): logging.debug('Removing workdir %s', workdir) remove_all(workdir, recursive=True) except: logging.error(Ta('Error removing workdir (%s)'), workdir) logging.info("Traceback: ", exc_info=True) return True
def parring(nzo, workdir): """ Perform par processing. Returns: (par_error, re_add) """ if 0: assert isinstance( nzo, sabnzbd.nzbstuff.NzbObject) # Assert only for debug purposes filename = nzo.final_name notifier.send_notification(T('Post-processing'), filename, 'pp') logging.info('Starting verification and repair of %s', filename) # Get verification status of sets verified = sabnzbd.load_data(VERIFIED_FILE, nzo.workpath, remove=False) or {} # Collect the par files if nzo.partable: par_table = nzo.partable.copy() else: par_table = {} repair_sets = par_table.keys() re_add = False par_error = False single = len(repair_sets) == 1 if repair_sets: for setname in repair_sets: if cfg.ignore_samples() and 'sample' in setname.lower(): continue if not verified.get(setname, False): logging.info("Running verification and repair on set %s", setname) parfile_nzf = par_table[setname] if os.path.exists( os.path.join( nzo.downpath, parfile_nzf.filename)) or parfile_nzf.extrapars: need_re_add, res = par2_repair(parfile_nzf, nzo, workdir, setname, single=single) # Was it aborted? if not nzo.pp_active: re_add = False par_error = True break re_add = re_add or need_re_add if not res and not need_re_add and cfg.sfv_check(): res = try_sfv_check(nzo, workdir, setname) if not res and not need_re_add and cfg.enable_unrar(): res = try_rar_check(nzo, workdir, setname) verified[setname] = res else: continue par_error = par_error or not res else: # Obfuscated par2 check logging.info('No par2 sets found, running obfuscated check on %s', filename) # Get the NZF's and sort them based on size nzfs_sorted = sorted(nzo.finished_files, key=lambda x: x.bytes) # We will have to make 'fake' par files that are recognized par2_vol = 0 par2_filename = None for nzf_try in nzfs_sorted: # run through list of files, looking for par2 signature.. logging.debug("Checking par2 signature of %s", nzf_try.filename) try: nzf_path = os.path.join(workdir, nzf_try.filename) if (is_parfile(nzf_path)): # We need 1 base-name so they are recognized as 1 set if not par2_filename: par2_filename = nzf_path # Rename so handle_par2() picks it up newpath = '%s.vol%d+%d.par2' % (par2_filename, par2_vol, par2_vol + 1) renamer(nzf_path, newpath) nzf_try.filename = os.path.split(newpath)[1] # Let the magic happen nzo.handle_par2(nzf_try, file_done=True) par2_vol += 1 except: pass if par2_vol > 0: # Pars found, we do it again par_error, re_add = parring(nzo, workdir) else: # We must not have found any par2.. logging.info("No par2 sets for %s", filename) nzo.set_unpack_info('Repair', T('[%s] No par2 sets') % unicoder(filename)) if cfg.sfv_check() and not verified.get('', False): par_error = not try_sfv_check(nzo, workdir, '') verified[''] = not par_error # If still no success, do RAR-check if not par_error and cfg.enable_unrar(): par_error = not try_rar_check(nzo, workdir, '') verified[''] = not par_error if re_add: logging.info('Re-added %s to queue', filename) if nzo.priority != TOP_PRIORITY: nzo.priority = REPAIR_PRIORITY sabnzbd.nzbqueue.add_nzo(nzo) sabnzbd.downloader.Downloader.do.resume_from_postproc() sabnzbd.save_data(verified, VERIFIED_FILE, nzo.workpath) logging.info('Verification and repair finished for %s', filename) return par_error, re_add
def rename(self, _files, current_path): logging.debug("Renaming Generic file") def filter_files(_file, current_path): if is_full_path(_file): filepath = _file.replace('_UNPACK_', '') else: filepath = os.path.join(current_path, _file) if os.path.exists(filepath): size = os.stat(filepath).st_size if size > 314572800 and not RE_SAMPLE.search(_file) \ and get_ext(_file) not in EXCLUDED_FILE_EXTS: return True return False renamed = False # remove any files below 300MB from this list files = [ _file for _file in _files if filter_files(_file, current_path) ] length = len(files) ## Single File Handling if length == 1: file = files[0] if is_full_path(file): filepath = file.replace('_UNPACK_', '') else: filepath = os.path.join(current_path, file) if os.path.exists(filepath): tmp, ext = os.path.splitext(file) self.fname = tmp newname = "%s%s" % (self.filename_set, ext) newname = newname.replace('%fn', tmp) newpath = os.path.join(current_path, newname) try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(Ta('Failed to rename: %s to %s'), filepath, newpath) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, file, self.filename_set) ## Sequence File Handling # if there is more than one extracted file check for CD1/1/A in the title elif self.extra: matched_files = check_for_multiple(files) # rename files marked as in a set if matched_files: logging.debug("Renaming a series of generic files (%s)", matched_files) for index, file in matched_files.iteritems(): filepath = os.path.join(current_path, file) tmp, ext = os.path.splitext(file) self.fname = tmp name = '%s%s' % (self.filename_set, self.extra) name = name.replace('%1', str(index)).replace('%fn', tmp) name = name + ext newpath = os.path.join(current_path, name) try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(Ta('Failed to rename: %s to %s'), filepath, newpath) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, file, self.filename_set) else: logging.debug("Movie files not in sequence %s", _files)
def rename(self, _files, current_path): """ Rename for Generic files """ logging.debug("Renaming Generic file") def filter_files(_file, current_path): if is_full_path(_file): filepath = os.path.normpath(_file) else: filepath = os.path.normpath(os.path.join(current_path, _file)) if os.path.exists(filepath): size = os.stat(filepath).st_size if size >= cfg.movie_rename_limit.get_int() and not RE_SAMPLE.search(_file) \ and get_ext(_file) not in EXCLUDED_FILE_EXTS: return True return False # remove any files below the limit from this list files = [_file for _file in _files if filter_files(_file, current_path)] length = len(files) # Single File Handling if length == 1: file = files[0] if is_full_path(file): filepath = os.path.normpath(file) else: filepath = os.path.normpath(os.path.join(current_path, file)) if os.path.exists(filepath): self.fname, ext = os.path.splitext(os.path.split(file)[1]) newname = "%s%s" % (self.filename_set, ext) newname = newname.replace('%fn', self.fname) newpath = os.path.join(current_path, newname) try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(T('Failed to rename: %s to %s'), clip_path(filepath), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, ext, self.filename_set, ()) # Sequence File Handling # if there is more than one extracted file check for CD1/1/A in the title elif self.extra: matched_files = check_for_multiple(files) # rename files marked as in a set if matched_files: logging.debug("Renaming a series of generic files (%s)", matched_files) renamed = matched_files.values() for index, file in matched_files.iteritems(): filepath = os.path.join(current_path, file) renamed.append(filepath) self.fname, ext = os.path.splitext(os.path.split(file)[1]) name = '%s%s' % (self.filename_set, self.extra) name = name.replace('%1', str(index)).replace('%fn', self.fname) name = name + ext newpath = os.path.join(current_path, name) try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(T('Failed to rename: %s to %s'), clip_path(filepath), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, ext, self.filename_set, renamed) else: logging.debug("Movie files not in sequence %s", _files)
def process_job(nzo): """ Process one job """ assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject) start = time.time() # keep track of whether we can continue all_ok = True # keep track of par problems par_error = False # keep track of any unpacking errors unpack_error = False # Signal empty download, for when 'empty_postproc' is enabled empty = False nzb_list = [] # These need to be initialised incase of a crash workdir_complete = '' postproc_time = 0 script_log = '' script_line = '' crash_msg = '' ## Get the job flags nzo.save_attribs() flag_repair, flag_unpack, flag_delete = nzo.repair_opts # Normalize PP if flag_delete: flag_unpack = True if flag_unpack: flag_repair = True # Get the NZB name filename = nzo.final_name msgid = nzo.msgid if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete): # After streaming, force +D nzo.set_pp(3) nzo.status = Status.FAILED nzo.save_attribs() all_ok = False if nzo.fail_msg: # Special case: aborted due to too many missing data nzo.status = Status.FAILED nzo.save_attribs() all_ok = False par_error = unpack_error = True try: # Get the folder containing the download result workdir = nzo.downpath tmp_workdir_complete = None # if no files are present (except __admin__), fail the job if all_ok and len(globber(workdir)) < 2: if nzo.precheck: enough, ratio = nzo.check_quality() req_ratio = float(cfg.req_completion_rate()) / 100.0 # Make sure that rounded ratio doesn't equal required ratio # when it is actually below required if (ratio < req_ratio) and (req_ratio - ratio) < 0.001: ratio = req_ratio - 0.001 emsg = '%.1f%%' % (ratio * 100.0) emsg2 = '%.1f%%' % float(cfg.req_completion_rate()) emsg = T( 'Download might fail, only %s of required %s available' ) % (emsg, emsg2) else: emsg = T('Download failed - Out of your server\'s retention?') empty = True nzo.fail_msg = emsg nzo.set_unpack_info('Fail', emsg) nzo.status = Status.FAILED # do not run unpacking or parity verification flag_repair = flag_unpack = False all_ok = cfg.empty_postproc() and empty if not all_ok: par_error = unpack_error = True script = nzo.script cat = nzo.cat logging.info('Starting PostProcessing on %s' + \ ' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s', filename, flag_repair, flag_unpack, flag_delete, script, cat) ## Par processing, if enabled if all_ok and flag_repair: par_error, re_add = parring(nzo, workdir) if re_add: # Try to get more par files return False ## Check if user allows unsafe post-processing if flag_repair and cfg.safe_postproc(): all_ok = all_ok and not par_error # Set complete dir to workdir in case we need to abort workdir_complete = workdir dirname = nzo.final_name marker_file = None if all_ok: one_folder = False ## Determine class directory if cfg.create_group_folders(): complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix) complete_dir = create_dirs(complete_dir) else: catdir = config.get_categories(cat).dir() if catdir.endswith('*'): catdir = catdir.strip('*') one_folder = True complete_dir = real_path(cfg.complete_dir.get_path(), catdir) ## TV/Movie/Date Renaming code part 1 - detect and construct paths if cfg.enable_meta(): file_sorter = Sorter(nzo, cat) else: file_sorter = Sorter(None, cat) complete_dir = file_sorter.detect(dirname, complete_dir) if file_sorter.sort_file: one_folder = False if one_folder: workdir_complete = create_dirs(complete_dir) else: workdir_complete = get_unique_path(os.path.join( complete_dir, dirname), create_dir=True) marker_file = set_marker(workdir_complete) if not workdir_complete or not os.path.exists(workdir_complete): crash_msg = T('Cannot create final folder %s') % unicoder( os.path.join(complete_dir, dirname)) raise IOError if cfg.folder_rename() and not one_folder: tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_') try: renamer(workdir_complete, tmp_workdir_complete) except: pass # On failure, just use the original name else: tmp_workdir_complete = workdir_complete newfiles = [] ## Run Stage 2: Unpack if flag_unpack: if all_ok: #set the current nzo status to "Extracting...". Used in History nzo.status = Status.EXTRACTING logging.info("Running unpack_magic on %s", filename) unpack_error, newfiles = unpack_magic( nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), ()) logging.info("unpack_magic finished on %s", filename) else: nzo.set_unpack_info( 'Unpack', T('No post-processing because of failed verification')) if cfg.safe_postproc(): all_ok = all_ok and not unpack_error if all_ok: ## Move any (left-over) files to destination nzo.status = Status.MOVING nzo.set_action_line(T('Moving'), '...') for root, dirs, files in os.walk(workdir): if not root.endswith(JOB_ADMIN): for file_ in files: path = os.path.join(root, file_) new_path = path.replace(workdir, tmp_workdir_complete) ok, new_path = move_to_path(path, new_path) newfiles.append(new_path) if not ok: nzo.set_unpack_info( 'Unpack', T('Failed moving %s to %s') % (unicoder(path), unicoder(new_path))) all_ok = False break ## Set permissions right set_permissions(tmp_workdir_complete) if all_ok and marker_file: del_marker(os.path.join(tmp_workdir_complete, marker_file)) remove_from_list(marker_file, newfiles) if all_ok: ## Remove files matching the cleanup list cleanup_list(tmp_workdir_complete, True) ## Check if this is an NZB-only download, if so redirect to queue ## except when PP was Download-only if flag_repair: nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority) else: nzb_list = None if nzb_list: nzo.set_unpack_info( 'Download', T('Sent %s to queue') % unicoder(nzb_list)) cleanup_empty_directories(tmp_workdir_complete) else: cleanup_list(tmp_workdir_complete, False) script_output = '' script_ret = 0 if not nzb_list: ## Give destination its final name if cfg.folder_rename() and tmp_workdir_complete and not one_folder: if all_ok: try: newfiles = rename_and_collapse_folder( tmp_workdir_complete, workdir_complete, newfiles) except: logging.error(Ta('Error renaming "%s" to "%s"'), tmp_workdir_complete, workdir_complete) logging.info('Traceback: ', exc_info=True) # Better disable sorting because filenames are all off now file_sorter.sort_file = None else: workdir_complete = tmp_workdir_complete.replace( '_UNPACK_', '_FAILED_') workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False) if empty: job_result = -1 else: job_result = int(par_error) + int(unpack_error) * 2 if cfg.ignore_samples() > 0: remove_samples(workdir_complete) ## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder if all_ok and file_sorter.sort_file: if newfiles: file_sorter.rename(newfiles, workdir_complete) workdir_complete, ok = file_sorter.move(workdir_complete) else: workdir_complete, ok = file_sorter.rename_with_ext( workdir_complete) if not ok: nzo.set_unpack_info('Unpack', T('Failed to move files')) all_ok = False ## Run the user script script_path = make_script_path(script) if (all_ok or not cfg.safe_postproc()) and ( not nzb_list) and script_path: #set the current nzo status to "Ext Script...". Used in History nzo.status = Status.RUNNING nzo.set_action_line(T('Running script'), unicoder(script)) nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True) script_log, script_ret = external_processing( script_path, workdir_complete, nzo.filename, msgid, dirname, cat, nzo.group, job_result, nzo.nzo_info.get('failure', '')) script_line = get_last_line(script_log) if script_log: script_output = nzo.nzo_id if script_line: nzo.set_unpack_info('Script', unicoder(script_line), unique=True) else: nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True) else: script = "" script_line = "" script_ret = 0 ## Email the results if (not nzb_list) and cfg.email_endjob(): if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error)): emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded, nzo.fail_msg, nzo.unpack_info, script, TRANS(script_log), script_ret) if script_output: # Can do this only now, otherwise it would show up in the email if script_ret: script_ret = 'Exit(%s) ' % script_ret else: script_ret = '' if script_line: nzo.set_unpack_info( 'Script', u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, unicoder(script_line), urllib.quote(script_output), T('More')), unique=True) else: nzo.set_unpack_info('Script', u'%s<a href="./scriptlog?name=%s">%s</a>' % (script_ret, urllib.quote(script_output), T('View script output')), unique=True) ## Cleanup again, including NZB files if all_ok: cleanup_list(workdir_complete, False) ## Remove newzbin bookmark, if any if msgid and all_ok: Bookmarks.do.del_bookmark(msgid) elif all_ok and isinstance(nzo.url, str): sabnzbd.proxy_rm_bookmark(nzo.url) ## Force error for empty result all_ok = all_ok and not empty ## Update indexer with results if nzo.encrypted > 0: Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED) if empty: hosts = map(lambda s: s.host, sabnzbd.downloader.Downloader.do.nzo_servers(nzo)) if not hosts: hosts = [None] for host in hosts: Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED, host) ## Show final status in history if all_ok: growler.send_notification(T('Download Completed'), filename, 'complete') nzo.status = Status.COMPLETED else: growler.send_notification(T('Download Failed'), filename, 'complete') nzo.status = Status.FAILED except: logging.error(Ta('Post Processing Failed for %s (%s)'), filename, crash_msg) if not crash_msg: logging.info("Traceback: ", exc_info=True) crash_msg = T('see logfile') nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder( crash_msg) growler.send_notification(T('Download Failed'), filename, 'complete') nzo.status = Status.FAILED par_error = True all_ok = False if cfg.email_endjob(): emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded, nzo.fail_msg, nzo.unpack_info, '', '', 0) if all_ok: # If the folder only contains one file OR folder, have that as the path # Be aware that series/generic/date sorting may move a single file into a folder containing other files workdir_complete = one_file_or_folder(workdir_complete) workdir_complete = os.path.normpath(workdir_complete) # Log the overall time taken for postprocessing postproc_time = int(time.time() - start) # Create the history DB instance history_db = database.get_history_handle() # Add the nzo to the database. Only the path, script and time taken is passed # Other information is obtained from the nzo history_db.add_history_db(nzo, workdir_complete, nzo.downpath, postproc_time, script_log, script_line) # The connection is only used once, so close it here history_db.close() ## Clean up the NZO try: logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok)) sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok) except: logging.error(Ta('Cleanup of %s failed.'), nzo.final_name) logging.info("Traceback: ", exc_info=True) ## Remove download folder if all_ok: try: if os.path.exists(workdir): logging.debug('Removing workdir %s', workdir) remove_all(workdir, recursive=True) except: logging.error(Ta('Error removing workdir (%s)'), workdir) logging.info("Traceback: ", exc_info=True) return True
def rename(self, _files, current_path): """ Rename for Generic files """ logging.debug("Renaming Generic file") def filter_files(_file, current_path): if is_full_path(_file): filepath = os.path.normpath(_file) else: filepath = os.path.normpath(os.path.join(current_path, _file)) if os.path.exists(filepath): size = os.stat(filepath).st_size if size >= cfg.movie_rename_limit.get_int() and not RE_SAMPLE.search(_file) \ and get_ext(_file) not in EXCLUDED_FILE_EXTS: return True return False # remove any files below the limit from this list files = [ _file for _file in _files if filter_files(_file, current_path) ] length = len(files) # Single File Handling if length == 1: file = files[0] if is_full_path(file): filepath = os.path.normpath(file) else: filepath = os.path.normpath(os.path.join(current_path, file)) if os.path.exists(filepath): self.fname, ext = os.path.splitext(os.path.split(file)[1]) newname = "%s%s" % (self.filename_set, ext) newname = newname.replace('%fn', self.fname) newpath = os.path.join(current_path, newname) try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(T('Failed to rename: %s to %s'), clip_path(filepath), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, ext, self.filename_set, ()) # Sequence File Handling # if there is more than one extracted file check for CD1/1/A in the title elif self.extra: matched_files = check_for_multiple(files) # rename files marked as in a set if matched_files: logging.debug("Renaming a series of generic files (%s)", matched_files) renamed = matched_files.values() for index, file in matched_files.iteritems(): filepath = os.path.join(current_path, file) renamed.append(filepath) self.fname, ext = os.path.splitext(os.path.split(file)[1]) name = '%s%s' % (self.filename_set, self.extra) name = name.replace('%1', str(index)).replace('%fn', self.fname) name = name + ext newpath = os.path.join(current_path, name) try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(T('Failed to rename: %s to %s'), clip_path(filepath), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, ext, self.filename_set, renamed) else: logging.debug("Movie files not in sequence %s", _files)