def _permissions_runner(self, test_base, perms_base="0700", apply_umask=True): # Create base directory and set the base permissions perms_base_int = int(perms_base, 8) self.fs.create_dir(test_base, perms_base_int) assert os.path.exists(test_base) is True self.assert_dir_perms(test_base, perms_base_int) # Create directories with permissions new_dir = os.path.join(test_base, "se 1", "ep1") filesystem.create_all_dirs(new_dir, apply_umask=apply_umask) # If permissions needed to be set, verify the new folder has the # right permissions and verify the base didn't change if apply_umask and cfg.umask(): perms_test_int = int(cfg.umask(), 8) | int("0700", 8) else: # Get the current umask, since os.mkdir masks that out cur_umask = os.umask(0) os.umask(cur_umask) perms_test_int = int("0777", 8) & ~cur_umask self.assert_dir_perms(new_dir, perms_test_int) self.assert_dir_perms(test_base, perms_base_int)
def create_all_dirs(path, umask=False): """ Create all required path elements and set umask on all Return True if last element could be made or exists """ result = True if sabnzbd.WIN32: try: os.makedirs(path) except: result = False else: lst = [] lst.extend(path.split('/')) path = '' for d in lst: if d: path += '/' + d if not os.path.exists(path): try: os.mkdir(path) result = True except: result = False if umask: mask = cfg.umask() if mask: try: os.chmod(path, int(mask, 8) | 0700) except: pass return result
def set_permissions(path, recursive=True): """ Give folder tree and its files their proper permissions """ if not sabnzbd.WIN32: umask = cfg.umask() try: # Make sure that user R+W+X is on umask = int(umask, 8) | int('0700', 8) report = True except ValueError: # No or no valid permissions # Use the effective permissions of the session # Don't report errors (because the system might not support it) umask = int('0777', 8) & (sabnzbd.ORG_UMASK ^ int('0777', 8)) report = False # Remove X bits for files umask_file = umask & int('7666', 8) if os.path.isdir(path): if recursive: # Parse the dir/file tree and set permissions for root, _dirs, files in os.walk(path): set_chmod(root, umask, report) for name in files: set_chmod(os.path.join(root, name), umask_file, report) else: set_chmod(path, umask, report) else: set_chmod(path, umask_file, report)
def create_all_dirs(path, umask=False): """ Create all required path elements and set umask on all Return True if last elelent could be made or exists """ result = True if sabnzbd.WIN32: try: os.makedirs(path) except: result = False else: list = [] list.extend(path.split('/')) path = '' for d in list: if d: path += '/' + d if not os.path.exists(path): try: os.mkdir(path) result = True except: result = False if umask: mask = cfg.umask() if mask: try: os.chmod(path, int(mask, 8) | 0700) except: pass return result
def set_permissions(path, recursive=True): """ Give folder tree and its files their proper permissions """ if not sabnzbd.WIN32: umask = cfg.umask() try: # Make sure that user R is on umask = int(umask, 8) | int('0400', 8) report = True except ValueError: # No or no valid permissions # Use the effective permissions of the session # Don't report errors (because the system might not support it) umask = int('0777', 8) & (sabnzbd.ORG_UMASK ^ int('0777', 8)) report = False # Remove X bits for files umask_file = umask & int('7666', 8) if os.path.isdir(path): if recursive: # Parse the dir/file tree and set permissions for root, dirs, files in os.walk(path): set_chmod(root, umask, report) for name in files: set_chmod(os.path.join(root, name), umask_file, report) else: set_chmod(path, umask, report) else: set_chmod(path, umask_file, report)
def process_job(nzo): """ Process one job """ assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject) start = time.time() # keep track of whether we can continue all_ok = True # keep track of par problems par_error = False # keep track of any unpacking errors unpack_error = False nzb_list = [] # These need to be initialised incase of a crash workdir_complete = '' postproc_time = 0 script_log = '' script_line = '' crash_msg = '' ## Get the job flags nzo.save_attribs() flag_repair, flag_unpack, flag_delete = nzo.repair_opts # Normalize PP if flag_delete: flag_unpack = True if flag_unpack: flag_repair = True # Get the NZB name filename = nzo.final_name msgid = nzo.msgid if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete): # After streaming, force +D nzo.set_pp(3) nzo.status = 'Failed' nzo.save_attribs() all_ok = False try: # Get the folder containing the download result workdir = nzo.downpath tmp_workdir_complete = None # if no files are present (except __admin__), fail the job if len(globber(workdir)) < 2: emsg = T('Download failed - Out of your server\'s retention?') nzo.fail_msg = emsg nzo.status = 'Failed' # do not run unpacking or parity verification flag_repair = flag_unpack = False par_error = unpack_error = True all_ok = False script = nzo.script cat = nzo.cat logging.info('Starting PostProcessing on %s' + \ ' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s', filename, flag_repair, flag_unpack, flag_delete, script, cat) ## Par processing, if enabled if flag_repair: par_error, re_add = parring(nzo, workdir) if re_add: # Try to get more par files return False ## Check if user allows unsafe post-processing if flag_repair and cfg.safe_postproc(): all_ok = all_ok and not par_error # Set complete dir to workdir in case we need to abort workdir_complete = workdir dirname = nzo.final_name if all_ok: one_folder = False ## Determine class directory if cfg.create_group_folders(): complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix) complete_dir = create_dirs(complete_dir) else: catdir = config.get_categories(cat).dir() if catdir.endswith('*'): catdir = catdir.strip('*') one_folder = True complete_dir = real_path(cfg.complete_dir.get_path(), catdir) ## TV/Movie/Date Renaming code part 1 - detect and construct paths file_sorter = Sorter(cat) complete_dir = file_sorter.detect(dirname, complete_dir) if file_sorter.is_sortfile(): one_folder = False if one_folder: workdir_complete = create_dirs(complete_dir) else: workdir_complete = get_unique_path(os.path.join(complete_dir, dirname), create_dir=True) if not workdir_complete or not os.path.exists(workdir_complete): crash_msg = T('Cannot create final folder %s') % unicoder(os.path.join(complete_dir, dirname)) raise IOError if cfg.folder_rename() and not one_folder: tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_') try: renamer(workdir_complete, tmp_workdir_complete) except: pass # On failure, just use the original name else: tmp_workdir_complete = workdir_complete newfiles = [] ## Run Stage 2: Unpack if flag_unpack: if all_ok: #set the current nzo status to "Extracting...". Used in History nzo.status = 'Extracting' logging.info("Running unpack_magic on %s", filename) unpack_error, newfiles = unpack_magic(nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), ()) logging.info("unpack_magic finished on %s", filename) else: nzo.set_unpack_info('Unpack', T('No post-processing because of failed verification')) if cfg.safe_postproc(): all_ok = all_ok and not unpack_error if all_ok: ## Move any (left-over) files to destination nzo.status = 'Moving' nzo.set_action_line(T('Moving'), '...') for root, dirs, files in os.walk(workdir): if not root.endswith(JOB_ADMIN): for file_ in files: path = os.path.join(root, file_) new_path = path.replace(workdir, tmp_workdir_complete) new_path = get_unique_filename(new_path) move_to_path(path, new_path, unique=False) ## Set permissions right if not sabnzbd.WIN32: perm_script(tmp_workdir_complete, cfg.umask()) if all_ok: ## Remove files matching the cleanup list cleanup_list(tmp_workdir_complete, True) ## Check if this is an NZB-only download, if so redirect to queue ## except when PP was Download-only if flag_repair: nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority) else: nzb_list = None if nzb_list: nzo.set_unpack_info('Download', T('Sent %s to queue') % unicoder(nzb_list)) try: remove_dir(tmp_workdir_complete) except: pass else: cleanup_list(tmp_workdir_complete, False) script_output = '' script_ret = 0 if not nzb_list: ## Give destination its final name if cfg.folder_rename() and tmp_workdir_complete and not one_folder: if not all_ok: workdir_complete = tmp_workdir_complete.replace('_UNPACK_', '_FAILED_') workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False) try: collapse_folder(tmp_workdir_complete, workdir_complete) except: logging.error(Ta('Error renaming "%s" to "%s"'), tmp_workdir_complete, workdir_complete) logging.info("Traceback: ", exc_info = True) job_result = int(par_error) + int(unpack_error)*2 if cfg.ignore_samples() > 0: remove_samples(workdir_complete) ## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder if all_ok: if newfiles and file_sorter.is_sortfile(): file_sorter.rename(newfiles, workdir_complete) workdir_complete = file_sorter.move(workdir_complete) ## Run the user script script_path = make_script_path(script) if all_ok and (not nzb_list) and script_path: #set the current nzo status to "Ext Script...". Used in History nzo.status = 'Running' nzo.set_action_line(T('Running script'), unicoder(script)) nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True) script_log, script_ret = external_processing(script_path, workdir_complete, nzo.filename, msgid, dirname, cat, nzo.group, job_result) script_line = get_last_line(script_log) if script_log: script_output = nzo.nzo_id if script_line: nzo.set_unpack_info('Script', unicoder(script_line), unique=True) else: nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True) else: script = "" script_line = "" script_ret = 0 ## Email the results if (not nzb_list) and cfg.email_endjob(): if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error)): emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded, nzo.unpack_info, script, TRANS(script_log), script_ret) if script_output: # Can do this only now, otherwise it would show up in the email if script_ret: script_ret = 'Exit(%s) ' % script_ret else: script_ret = '' if script_line: nzo.set_unpack_info('Script', u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, unicoder(script_line), urllib.quote(script_output), T('More')), unique=True) else: nzo.set_unpack_info('Script', u'%s<a href="./scriptlog?name=%s">%s</a>' % (script_ret, urllib.quote(script_output), T('View script output')), unique=True) ## Cleanup again, including NZB files cleanup_list(workdir_complete, False) ## Remove newzbin bookmark, if any if msgid and all_ok: Bookmarks.do.del_bookmark(msgid) elif all_ok: sabnzbd.proxy_rm_bookmark(nzo.url) ## Show final status in history if all_ok: osx.sendGrowlMsg(T('Download Completed'), filename, osx.NOTIFICATION['complete']) nzo.status = 'Completed' else: osx.sendGrowlMsg(T('Download Failed'), filename, osx.NOTIFICATION['complete']) nzo.status = 'Failed' except: logging.error(Ta('Post Processing Failed for %s (%s)'), filename, crash_msg) if not crash_msg: logging.info("Traceback: ", exc_info = True) crash_msg = T('see logfile') nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(crash_msg) osx.sendGrowlMsg(T('Download Failed'), filename, osx.NOTIFICATION['complete']) nzo.status = 'Failed' par_error = True all_ok = False if all_ok: # If the folder only contains one file OR folder, have that as the path # Be aware that series/generic/date sorting may move a single file into a folder containing other files workdir_complete = one_file_or_folder(workdir_complete) workdir_complete = os.path.normpath(workdir_complete) # Log the overall time taken for postprocessing postproc_time = int(time.time() - start) # Create the history DB instance history_db = database.get_history_handle() # Add the nzo to the database. Only the path, script and time taken is passed # Other information is obtained from the nzo history_db.add_history_db(nzo, workdir_complete, nzo.downpath, postproc_time, script_log, script_line) # The connection is only used once, so close it here history_db.close() ## Clean up the NZO try: logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok)) sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok) except: logging.error(Ta('Cleanup of %s failed.'), nzo.final_name) logging.info("Traceback: ", exc_info = True) ## Remove download folder if all_ok: try: if os.path.exists(workdir): logging.debug('Removing workdir %s', workdir) remove_all(os.path.join(workdir, JOB_ADMIN)) remove_dir(workdir) except: logging.error(Ta('Error removing workdir (%s)'), workdir) logging.info("Traceback: ", exc_info = True) return True
def process_job(nzo): """ Process one job """ assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject) start = time.time() # keep track of whether we can continue all_ok = True # keep track of par problems par_error = False # keep track of any unpacking errors unpack_error = False nzb_list = [] # These need to be initialised incase of a crash workdir_complete = '' postproc_time = 0 script_log = '' script_line = '' crash_msg = '' ## Get the job flags nzo.save_attribs() flag_repair, flag_unpack, flag_delete = nzo.repair_opts # Normalize PP if flag_delete: flag_unpack = True if flag_unpack: flag_repair = True # Get the NZB name filename = nzo.final_name msgid = nzo.msgid if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete): # After streaming, force +D nzo.set_pp(3) nzo.status = 'Failed' nzo.save_attribs() all_ok = False try: # Get the folder containing the download result workdir = nzo.downpath tmp_workdir_complete = None # if no files are present (except __admin__), fail the job if len(globber(workdir)) < 2: emsg = T('Download failed - Out of your server\'s retention?') nzo.fail_msg = emsg nzo.status = 'Failed' # do not run unpacking or parity verification flag_repair = flag_unpack = False par_error = unpack_error = True all_ok = False script = nzo.script cat = nzo.cat logging.info('Starting PostProcessing on %s' + \ ' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s', filename, flag_repair, flag_unpack, flag_delete, script, cat) ## Par processing, if enabled if flag_repair: par_error, re_add = parring(nzo, workdir) if re_add: # Try to get more par files return False ## Check if user allows unsafe post-processing if flag_repair and cfg.safe_postproc(): all_ok = all_ok and not par_error # Set complete dir to workdir in case we need to abort workdir_complete = workdir dirname = nzo.final_name if all_ok: one_folder = False ## Determine class directory if cfg.create_group_folders(): complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix) complete_dir = create_dirs(complete_dir) else: catdir = config.get_categories(cat).dir() if catdir.endswith('*'): catdir = catdir.strip('*') one_folder = True complete_dir = real_path(cfg.complete_dir.get_path(), catdir) ## TV/Movie/Date Renaming code part 1 - detect and construct paths file_sorter = Sorter(cat) complete_dir = file_sorter.detect(dirname, complete_dir) if file_sorter.is_sortfile(): one_folder = False if one_folder: workdir_complete = create_dirs(complete_dir) else: workdir_complete = get_unique_path(os.path.join( complete_dir, dirname), create_dir=True) if not workdir_complete or not os.path.exists(workdir_complete): crash_msg = T('Cannot create final folder %s') % unicoder( os.path.join(complete_dir, dirname)) raise IOError if cfg.folder_rename() and not one_folder: tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_') try: renamer(workdir_complete, tmp_workdir_complete) except: pass # On failure, just use the original name else: tmp_workdir_complete = workdir_complete newfiles = [] ## Run Stage 2: Unpack if flag_unpack: if all_ok: #set the current nzo status to "Extracting...". Used in History nzo.status = 'Extracting' logging.info("Running unpack_magic on %s", filename) unpack_error, newfiles = unpack_magic( nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), ()) logging.info("unpack_magic finished on %s", filename) else: nzo.set_unpack_info( 'Unpack', T('No post-processing because of failed verification')) if cfg.safe_postproc(): all_ok = all_ok and not unpack_error if all_ok: ## Move any (left-over) files to destination nzo.status = 'Moving' nzo.set_action_line(T('Moving'), '...') for root, dirs, files in os.walk(workdir): if not root.endswith(JOB_ADMIN): for file_ in files: path = os.path.join(root, file_) new_path = path.replace(workdir, tmp_workdir_complete) new_path = get_unique_filename(new_path) move_to_path(path, new_path, unique=False) ## Set permissions right if not sabnzbd.WIN32: perm_script(tmp_workdir_complete, cfg.umask()) if all_ok: ## Remove files matching the cleanup list cleanup_list(tmp_workdir_complete, True) ## Check if this is an NZB-only download, if so redirect to queue ## except when PP was Download-only if flag_repair: nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority) else: nzb_list = None if nzb_list: nzo.set_unpack_info( 'Download', T('Sent %s to queue') % unicoder(nzb_list)) try: remove_dir(tmp_workdir_complete) except: pass else: cleanup_list(tmp_workdir_complete, False) script_output = '' script_ret = 0 if not nzb_list: ## Give destination its final name if cfg.folder_rename() and tmp_workdir_complete and not one_folder: if not all_ok: workdir_complete = tmp_workdir_complete.replace( '_UNPACK_', '_FAILED_') workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False) try: collapse_folder(tmp_workdir_complete, workdir_complete) except: logging.error(Ta('Error renaming "%s" to "%s"'), tmp_workdir_complete, workdir_complete) logging.info("Traceback: ", exc_info=True) job_result = int(par_error) + int(unpack_error) * 2 if cfg.ignore_samples() > 0: remove_samples(workdir_complete) ## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder if all_ok: if newfiles and file_sorter.is_sortfile(): file_sorter.rename(newfiles, workdir_complete) workdir_complete = file_sorter.move(workdir_complete) ## Run the user script script_path = make_script_path(script) if all_ok and (not nzb_list) and script_path: #set the current nzo status to "Ext Script...". Used in History nzo.status = 'Running' nzo.set_action_line(T('Running script'), unicoder(script)) nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True) script_log, script_ret = external_processing( script_path, workdir_complete, nzo.filename, msgid, dirname, cat, nzo.group, job_result) script_line = get_last_line(script_log) if script_log: script_output = nzo.nzo_id if script_line: nzo.set_unpack_info('Script', unicoder(script_line), unique=True) else: nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True) else: script = "" script_line = "" script_ret = 0 ## Email the results if (not nzb_list) and cfg.email_endjob(): if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error)): emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded, nzo.unpack_info, script, TRANS(script_log), script_ret) if script_output: # Can do this only now, otherwise it would show up in the email if script_ret: script_ret = 'Exit(%s) ' % script_ret else: script_ret = '' if script_line: nzo.set_unpack_info( 'Script', u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, unicoder(script_line), urllib.quote(script_output), T('More')), unique=True) else: nzo.set_unpack_info('Script', u'%s<a href="./scriptlog?name=%s">%s</a>' % (script_ret, urllib.quote(script_output), T('View script output')), unique=True) ## Cleanup again, including NZB files cleanup_list(workdir_complete, False) ## Remove newzbin bookmark, if any if msgid and all_ok: Bookmarks.do.del_bookmark(msgid) elif all_ok: sabnzbd.proxy_rm_bookmark(nzo.url) ## Show final status in history if all_ok: osx.sendGrowlMsg(T('Download Completed'), filename, osx.NOTIFICATION['complete']) nzo.status = 'Completed' else: osx.sendGrowlMsg(T('Download Failed'), filename, osx.NOTIFICATION['complete']) nzo.status = 'Failed' except: logging.error(Ta('Post Processing Failed for %s (%s)'), filename, crash_msg) if not crash_msg: logging.info("Traceback: ", exc_info=True) crash_msg = T('see logfile') nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder( crash_msg) osx.sendGrowlMsg(T('Download Failed'), filename, osx.NOTIFICATION['complete']) nzo.status = 'Failed' par_error = True all_ok = False if all_ok: # If the folder only contains one file OR folder, have that as the path # Be aware that series/generic/date sorting may move a single file into a folder containing other files workdir_complete = one_file_or_folder(workdir_complete) workdir_complete = os.path.normpath(workdir_complete) # Log the overall time taken for postprocessing postproc_time = int(time.time() - start) # Create the history DB instance history_db = database.get_history_handle() # Add the nzo to the database. Only the path, script and time taken is passed # Other information is obtained from the nzo history_db.add_history_db(nzo, workdir_complete, nzo.downpath, postproc_time, script_log, script_line) # The connection is only used once, so close it here history_db.close() ## Clean up the NZO try: logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok)) sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok) except: logging.error(Ta('Cleanup of %s failed.'), nzo.final_name) logging.info("Traceback: ", exc_info=True) ## Remove download folder if all_ok: try: if os.path.exists(workdir): logging.debug('Removing workdir %s', workdir) remove_all(workdir, recursive=True) except: logging.error(Ta('Error removing workdir (%s)'), workdir) logging.info("Traceback: ", exc_info=True) return True