Exemplo n.º 1
0
    def repair_job(self, folder, new_nzb=None, password=None):
        """ Reconstruct admin for a single job folder, optionally with new NZB """
        def all_verified(path):
            """ Return True when all sets have been successfully verified """
            verified = sabnzbd.load_data(VERIFIED_FILE, path, remove=False) or {'x': False}
            return all(verified[x] for x in verified)

        name = os.path.basename(folder)
        path = os.path.join(folder, JOB_ADMIN)
        if hasattr(new_nzb, 'filename'):
            filename = new_nzb.filename
        else:
            filename = ''
        if not filename:
            if not all_verified(path):
                filename = globber_full(path, '*.gz')
            if len(filename) > 0:
                logging.debug('Repair job %s by reparsing stored NZB', name)
                nzo_id = sabnzbd.add_nzbfile(filename[0], pp=None, script=None, cat=None, priority=None, nzbname=name,
                                             reuse=True, password=password)[1]
            else:
                logging.debug('Repair job %s without stored NZB', name)
                nzo = NzbObject(name, pp=None, script=None, nzb='', cat=None, priority=None, nzbname=name, reuse=True)
                nzo.password = password
                self.add(nzo)
                nzo_id = nzo.nzo_id
        else:
            remove_all(path, '*.gz')
            logging.debug('Repair job %s with new NZB (%s)', name, filename)
            nzo_id = sabnzbd.add_nzbfile(new_nzb, pp=None, script=None, cat=None, priority=None, nzbname=name,
                                         reuse=True, password=password)[1]

        return nzo_id
Exemplo n.º 2
0
    def repair_job(self, folder, new_nzb=None, password=None):
        """ Reconstruct admin for a single job folder, optionally with new NZB """
        def all_verified(path):
            """ Return True when all sets have been successfully verified """
            verified = sabnzbd.load_data(VERIFIED_FILE, path, remove=False) or {'x': False}
            return all(verified[x] for x in verified)

        name = os.path.basename(folder)
        path = os.path.join(folder, JOB_ADMIN)
        if hasattr(new_nzb, 'filename'):
            filename = new_nzb.filename
        else:
            filename = ''
        if not filename:
            if not all_verified(path):
                filename = globber_full(path, '*.gz')
            if len(filename) > 0:
                logging.debug('Repair job %s by reparsing stored NZB', name)
                nzo_id = sabnzbd.add_nzbfile(filename[0], pp=None, script=None, cat=None, priority=None, nzbname=name,
                                             reuse=True, password=password)[1]
            else:
                logging.debug('Repair job %s without stored NZB', name)
                nzo = NzbObject(name, pp=None, script=None, nzb='', cat=None, priority=None, nzbname=name, reuse=True)
                nzo.password = password
                self.add(nzo)
                nzo_id = nzo.nzo_id
        else:
            remove_all(path, '*.gz')
            logging.debug('Repair job %s with new NZB (%s)', name, filename)
            nzo_id = sabnzbd.add_nzbfile(new_nzb, pp=None, script=None, cat=None, priority=None, nzbname=name,
                                         reuse=True, password=password)[1]

        return nzo_id
Exemplo n.º 3
0
    def abort(self):
        """ Abort running instance and delete generated files """
        if not self.killed:
            logging.info('Aborting DirectUnpack for %s', self.cur_setname)
            self.killed = True

            # Save reference to the first rarfile
            rarfile_nzf = self.rarfile_nzf

            # Abort Unrar
            if self.active_instance:
                self.active_instance.kill()
                # We need to wait for it to kill the process
                self.active_instance.wait()

            # Wake up the thread
            with self.next_file_lock:
                self.next_file_lock.notify()

            # No new sets
            self.next_sets = []
            self.success_sets = {}

            # Remove files
            if self.unpack_dir_info:
                extraction_path, _, _, one_folder, _ = self.unpack_dir_info
                # In case of flat-unpack we need to remove the files manually
                if one_folder:
                    # RarFile can fail for mysterious reasons
                    try:
                        rar_contents = RarFile(os.path.join(
                            self.nzo.downpath, rarfile_nzf.filename),
                                               all_names=True).filelist()
                        for rm_file in rar_contents:
                            # Flat-unpack, so remove foldername from RarFile output
                            f = os.path.join(extraction_path,
                                             os.path.basename(rm_file))
                            logging.debug('Removing file %s', f)
                            os.remove(f)
                    except:
                        # The user will have to remove it themselves
                        logging.info(
                            'Failed to clean Direct Unpack after aborting %s',
                            rarfile_nzf.filename,
                            exc_info=True)
                        pass
                else:
                    # We can just remove the whole path
                    remove_all(extraction_path, recursive=True)
                # Remove dir-info
                self.unpack_dir_info = None

            # Reset settings
            self.reset_active()
Exemplo n.º 4
0
    def abort(self):
        """ Abort running instance and delete generated files """
        if not self.killed and self.cur_setname:
            logging.info('Aborting DirectUnpack for %s', self.cur_setname)
            self.killed = True

            # Save reference to the first rarfile
            rarfile_nzf = self.rarfile_nzf

            # Abort Unrar
            if self.active_instance:
                self.active_instance.kill()
                # We need to wait for it to kill the process
                self.active_instance.wait()

            # Wake up the thread
            with self.next_file_lock:
                self.next_file_lock.notify()

            # No new sets
            self.next_sets = []
            self.success_sets = {}

            # Remove files
            if self.unpack_dir_info:
                extraction_path, _, _, one_folder, _ = self.unpack_dir_info
                # In case of flat-unpack we need to remove the files manually
                if one_folder:
                    # RarFile can fail for mysterious reasons
                    try:
                        rar_contents = RarFile(os.path.join(self.nzo.downpath, rarfile_nzf.filename), all_names=True).filelist()
                        for rm_file in rar_contents:
                            # Flat-unpack, so remove foldername from RarFile output
                            f = os.path.join(extraction_path, os.path.basename(rm_file))
                            remove_file(f)
                    except:
                        # The user will have to remove it themselves
                        logging.info('Failed to clean Direct Unpack after aborting %s', rarfile_nzf.filename, exc_info=True)
                        pass
                else:
                    # We can just remove the whole path
                    remove_all(extraction_path, recursive=True)
                # Remove dir-info
                self.unpack_dir_info = None

            # Reset settings
            self.reset_active()
Exemplo n.º 5
0
 def repair_job(self, folder, new_nzb=None):
     """ Reconstruct admin for a single job folder, optionally with new NZB """
     name = os.path.basename(folder)
     path = os.path.join(folder, JOB_ADMIN)
     if new_nzb is None or not new_nzb.filename:
         filename = globber(path, '*.gz')
         if len(filename) > 0:
             logging.debug('Repair job %s by reparsing stored NZB', latin1(name))
             sabnzbd.add_nzbfile(filename[0], pp=None, script=None, cat=None, priority=None, nzbname=name, reuse=True)
         else:
             logging.debug('Repair job %s without stored NZB', latin1(name))
             nzo = NzbObject(name, 0, pp=None, script=None, nzb='', cat=None, priority=None, nzbname=name, reuse=True)
             self.add(nzo)
     else:
         remove_all(path, '*.gz')
         logging.debug('Repair job %s without new NZB (%s)', latin1(name), latin1(new_nzb.filename))
         sabnzbd.add_nzbfile(new_nzb, pp=None, script=None, cat=None, priority=None, nzbname=name, reuse=True)
Exemplo n.º 6
0
def initialize(pause_downloader=False,
               clean_up=False,
               evalSched=False,
               repair=0):
    global __INITIALIZED__, __SHUTTING_DOWN__,\
           LOGFILE, WEBLOGFILE, LOGHANDLER, GUIHANDLER, AMBI_LOCALHOST, WAITEXIT, \
           DAEMON, MY_NAME, MY_FULLNAME, NEW_VERSION, \
           DIR_HOME, DIR_APPDATA, DIR_LCLDATA, DIR_PROG , DIR_INTERFACES, \
           DARWIN, RESTART_REQ, OLD_QUEUE

    if __INITIALIZED__:
        return False

    __SHUTTING_DOWN__ = False

    ### Set global database connection for Web-UI threads
    cherrypy.engine.subscribe('start_thread', connect_db)

    ### Paused?
    pause_downloader = pause_downloader or cfg.start_paused()

    ### Clean-up, if requested
    if clean_up:
        # Old cache folder
        misc.remove_all(cfg.cache_dir.get_path(), '*.sab')
        misc.remove_all(cfg.cache_dir.get_path(), 'SABnzbd_*')
        # New admin folder
        misc.remove_all(cfg.admin_dir.get_path(), '*.sab')

    ### Optionally wait for "incomplete" to become online
    if cfg.wait_for_dfolder():
        wait_for_download_folder()
    else:
        cfg.download_dir.set(cfg.download_dir(), create=True)
    cfg.download_dir.set_create(True)

    ### Set access rights for "incomplete" base folder
    misc.set_permissions(cfg.download_dir.get_path(), recursive=False)

    ### If dirscan_dir cannot be created, set a proper value anyway.
    ### Maybe it's a network path that's temporarily missing.
    path = cfg.dirscan_dir.get_path()
    if not os.path.exists(path):
        sabnzbd.misc.create_real_path(cfg.dirscan_dir.ident(), '', path, False)

    ### Set call backs for Config items
    cfg.cache_limit.callback(new_limit)
    cfg.cherryhost.callback(guard_restart)
    cfg.cherryport.callback(guard_restart)
    cfg.web_dir.callback(guard_restart)
    cfg.web_dir2.callback(guard_restart)
    cfg.web_color.callback(guard_restart)
    cfg.web_color2.callback(guard_restart)
    cfg.log_dir.callback(guard_restart)
    cfg.cache_dir.callback(guard_restart)
    cfg.https_port.callback(guard_restart)
    cfg.https_cert.callback(guard_restart)
    cfg.https_key.callback(guard_restart)
    cfg.enable_https.callback(guard_restart)
    cfg.bandwidth_limit.callback(guard_speedlimit)
    cfg.top_only.callback(guard_top_only)
    cfg.pause_on_post_processing.callback(guard_pause_on_pp)
    cfg.growl_server.callback(sabnzbd.growler.change_value)
    cfg.growl_password.callback(sabnzbd.growler.change_value)
    cfg.quota_size.callback(guard_quota_size)
    cfg.quota_day.callback(guard_quota_dp)
    cfg.quota_period.callback(guard_quota_dp)
    cfg.fsys_type.callback(guard_fsys_type)
    cfg.language.callback(sabnzbd.growler.reset_growl)

    ### Set Posix filesystem encoding
    sabnzbd.encoding.change_fsys(cfg.fsys_type())

    ### Set cache limit
    if (sabnzbd.WIN32 or sabnzbd.DARWIN) and not cfg.cache_limit():
        cfg.cache_limit.set('200M')
    ArticleCache.do.new_limit(cfg.cache_limit.get_int())

    check_incomplete_vs_complete()

    ### Handle language upgrade from 0.5.x to 0.6.x
    cfg.language.set(LANG_MAP.get(cfg.language(), cfg.language()))

    ### Set language files
    lang.set_locale_info('SABnzbd', DIR_LANGUAGE)
    lang.set_language(cfg.language())
    sabnzbd.api.clear_trans_cache()

    ### Check for old queue (when a new queue is not present)
    if not os.path.exists(
            os.path.join(cfg.cache_dir.get_path(), QUEUE_FILE_NAME)):
        OLD_QUEUE = bool(
            misc.globber(cfg.cache_dir.get_path(), QUEUE_FILE_TMPL % '?'))

    sabnzbd.change_queue_complete_action(cfg.queue_complete(), new=False)

    if check_repair_request():
        repair = 2
        pause_downloader = True
    else:
        # Check crash detection file
        #if load_admin(TERM_FLAG_FILE, remove=True):
        # Repair mode 2 is a bit over an over-reaction!
        pass  # repair = 2

    # Set crash detection file
    #save_admin(1, TERM_FLAG_FILE)

    ###
    ### Initialize threads
    ###

    Bookmarks()
    rss.init()

    paused = BPSMeter.do.read()

    PostProcessor()

    NzbQueue()

    Assembler()

    NzbQueue.do.read_queue(repair)

    Downloader(pause_downloader or paused)

    DirScanner()

    MSGIDGrabber()

    Rating()

    URLGrabber()

    scheduler.init()

    if evalSched:
        scheduler.analyse(pause_downloader)

    logging.info('All processes started')
    RESTART_REQ = False
    __INITIALIZED__ = True
    return True
Exemplo n.º 7
0
def process_job(nzo):
    """ Process one job """
    start = time.time()

    # keep track of whether we can continue
    all_ok = True
    # keep track of par problems
    par_error = False
    # keep track of any unpacking errors
    unpack_error = False
    # Signal empty download, for when 'empty_postproc' is enabled
    empty = False
    nzb_list = []
    # These need to be initialized in case of a crash
    workdir_complete = ''
    script_log = ''
    script_line = ''

    # Get the job flags
    nzo.save_attribs()
    flag_repair, flag_unpack, flag_delete = nzo.repair_opts
    # Normalize PP
    if flag_delete:
        flag_unpack = True
    if flag_unpack:
        flag_repair = True

    # Get the NZB name
    filename = nzo.final_name

    if nzo.fail_msg:  # Special case: aborted due to too many missing data
        nzo.status = Status.FAILED
        nzo.save_attribs()
        all_ok = False
        par_error = True
        unpack_error = 1

    try:
        # Get the folder containing the download result
        workdir = nzo.downpath
        tmp_workdir_complete = None

        # if no files are present (except __admin__), fail the job
        if all_ok and len(globber(workdir)) < 2:
            if nzo.precheck:
                _enough, ratio = nzo.check_quality()
                req_ratio = float(cfg.req_completion_rate()) / 100.0
                # Make sure that rounded ratio doesn't equal required ratio
                # when it is actually below required
                if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
                    ratio = req_ratio - 0.001
                emsg = '%.1f%%' % (ratio * 100.0)
                emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
                emsg = T(
                    'Download might fail, only %s of required %s available'
                ) % (emsg, emsg2)
            else:
                emsg = T('Download failed - Not on your server(s)')
                empty = True
            emsg += ' - https://sabnzbd.org/not-complete'
            nzo.fail_msg = emsg
            nzo.set_unpack_info('Fail', emsg)
            nzo.status = Status.FAILED
            # do not run unpacking or parity verification
            flag_repair = flag_unpack = False
            all_ok = cfg.empty_postproc() and empty
            if not all_ok:
                par_error = True
                unpack_error = 1

        script = nzo.script
        logging.info(
            'Starting Post-Processing on %s' +
            ' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s', filename,
            flag_repair, flag_unpack, flag_delete, script, nzo.cat)

        # Set complete dir to workdir in case we need to abort
        workdir_complete = workdir

        # Par processing, if enabled
        if all_ok and flag_repair:
            par_error, re_add = parring(nzo, workdir)
            if re_add:
                # Try to get more par files
                return False

        # If we don't need extra par2, we can disconnect
        if sabnzbd.nzbqueue.NzbQueue.do.actives(
                grabs=False) == 0 and cfg.autodisconnect():
            # This was the last job, close server connections
            sabnzbd.downloader.Downloader.do.disconnect()

        # Sanitize the resulting files
        if sabnzbd.WIN32:
            sanitize_files_in_folder(workdir)

        # Check if user allows unsafe post-processing
        if flag_repair and cfg.safe_postproc():
            all_ok = all_ok and not par_error

        if all_ok:
            # Fix encodings
            fix_unix_encoding(workdir)

            # Use dirs generated by direct-unpacker
            if nzo.direct_unpacker and nzo.direct_unpacker.unpack_dir_info:
                tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file = nzo.direct_unpacker.unpack_dir_info
            else:
                # Generate extraction path
                tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file = prepare_extraction_path(
                    nzo)

            newfiles = []
            # Run Stage 2: Unpack
            if flag_unpack:
                # set the current nzo status to "Extracting...". Used in History
                nzo.status = Status.EXTRACTING
                logging.info("Running unpack_magic on %s", filename)
                unpack_error, newfiles = unpack_magic(nzo, workdir,
                                                      tmp_workdir_complete,
                                                      flag_delete, one_folder,
                                                      (), (), (), (), ())
                logging.info("Unpacked files %s", newfiles)

                if sabnzbd.WIN32:
                    # Sanitize the resulting files
                    newfiles = sanitize_files_in_folder(tmp_workdir_complete)
                logging.info("Finished unpack_magic on %s", filename)

            if cfg.safe_postproc():
                all_ok = all_ok and not unpack_error

            if all_ok:
                # Move any (left-over) files to destination
                nzo.status = Status.MOVING
                nzo.set_action_line(T('Moving'), '...')
                for root, _dirs, files in os.walk(workdir):
                    if not root.endswith(JOB_ADMIN):
                        for file_ in files:
                            path = os.path.join(root, file_)
                            new_path = path.replace(workdir,
                                                    tmp_workdir_complete)
                            ok, new_path = move_to_path(path, new_path)
                            if new_path:
                                newfiles.append(new_path)
                            if not ok:
                                nzo.set_unpack_info(
                                    'Unpack',
                                    T('Failed moving %s to %s') %
                                    (unicoder(path), unicoder(new_path)))
                                all_ok = False
                                break

            # Set permissions right
            set_permissions(tmp_workdir_complete)

            if all_ok and marker_file:
                del_marker(os.path.join(tmp_workdir_complete, marker_file))
                remove_from_list(marker_file, newfiles)

            if all_ok:
                # Remove files matching the cleanup list
                cleanup_list(tmp_workdir_complete, True)

                # Check if this is an NZB-only download, if so redirect to queue
                # except when PP was Download-only
                if flag_repair:
                    nzb_list = nzb_redirect(tmp_workdir_complete,
                                            nzo.final_name,
                                            nzo.pp,
                                            script,
                                            nzo.cat,
                                            priority=nzo.priority)
                else:
                    nzb_list = None
                if nzb_list:
                    nzo.set_unpack_info(
                        'Download',
                        T('Sent %s to queue') % unicoder(nzb_list))
                    cleanup_empty_directories(tmp_workdir_complete)
                else:
                    cleanup_list(tmp_workdir_complete, False)

        script_output = ''
        script_ret = 0
        if not nzb_list:
            # Give destination its final name
            if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
                if all_ok:
                    try:
                        newfiles = rename_and_collapse_folder(
                            tmp_workdir_complete, workdir_complete, newfiles)
                    except:
                        logging.error(T('Error renaming "%s" to "%s"'),
                                      clip_path(tmp_workdir_complete),
                                      clip_path(workdir_complete))
                        logging.info('Traceback: ', exc_info=True)
                        # Better disable sorting because filenames are all off now
                        file_sorter.sort_file = None
                else:
                    workdir_complete = tmp_workdir_complete.replace(
                        '_UNPACK_', '_FAILED_')
                    workdir_complete = get_unique_path(workdir_complete,
                                                       n=0,
                                                       create_dir=False)

            if empty:
                job_result = -1
            else:
                job_result = int(par_error) + int(bool(unpack_error)) * 2

            if cfg.ignore_samples():
                remove_samples(workdir_complete)

            # TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
            if all_ok and file_sorter.sort_file:
                if newfiles:
                    file_sorter.rename(newfiles, workdir_complete)
                    workdir_complete, ok = file_sorter.move(workdir_complete)
                else:
                    workdir_complete, ok = file_sorter.rename_with_ext(
                        workdir_complete)
                if not ok:
                    nzo.set_unpack_info('Unpack', T('Failed to move files'))
                    all_ok = False

            # Run the user script
            script_path = make_script_path(script)
            if (all_ok or not cfg.safe_postproc()) and (
                    not nzb_list) and script_path:
                # Set the current nzo status to "Ext Script...". Used in History
                nzo.status = Status.RUNNING
                nzo.set_action_line(T('Running script'), unicoder(script))
                nzo.set_unpack_info('Script',
                                    T('Running user script %s') %
                                    unicoder(script),
                                    unique=True)
                script_log, script_ret = external_processing(
                    script_path, nzo, clip_path(workdir_complete),
                    nzo.final_name, job_result)
                script_line = get_last_line(script_log)
                if script_log:
                    script_output = nzo.nzo_id
                if script_line:
                    nzo.set_unpack_info('Script',
                                        unicoder(script_line),
                                        unique=True)
                else:
                    nzo.set_unpack_info('Script',
                                        T('Ran %s') % unicoder(script),
                                        unique=True)
            else:
                script = ""
                script_line = ""
                script_ret = 0

        # Maybe bad script result should fail job
        if script_ret and cfg.script_can_fail():
            script_error = True
            all_ok = False
            nzo.fail_msg = T('Script exit code is %s') % script_ret
        else:
            script_error = False

        # Email the results
        if (not nzb_list) and cfg.email_endjob():
            if (cfg.email_endjob()
                    == 1) or (cfg.email_endjob() == 2 and
                              (unpack_error or par_error or script_error)):
                emailer.endjob(nzo.final_name, nzo.cat, all_ok,
                               workdir_complete, nzo.bytes_downloaded,
                               nzo.fail_msg, nzo.unpack_info, script,
                               TRANS(script_log), script_ret)

        if script_output:
            # Can do this only now, otherwise it would show up in the email
            if script_ret:
                script_ret = 'Exit(%s) ' % script_ret
            else:
                script_ret = ''
            if len(script_log.rstrip().split('\n')) > 1:
                nzo.set_unpack_info(
                    'Script',
                    u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' %
                    (script_ret, script_line,
                     xml.sax.saxutils.escape(script_output), T('More')),
                    unique=True)
            else:
                # No '(more)' button needed
                nzo.set_unpack_info('Script',
                                    u'%s%s ' % (script_ret, script_line),
                                    unique=True)

        # Cleanup again, including NZB files
        if all_ok:
            cleanup_list(workdir_complete, False)

        # Force error for empty result
        all_ok = all_ok and not empty

        # Update indexer with results
        if cfg.rating_enable():
            if nzo.encrypted > 0:
                Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
            if empty:
                hosts = map(lambda s: s.host,
                            sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
                if not hosts:
                    hosts = [None]
                for host in hosts:
                    Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED,
                                               host)

    except:
        logging.error(T('Post Processing Failed for %s (%s)'), filename,
                      T('see logfile'))
        logging.info("Traceback: ", exc_info=True)

        nzo.fail_msg = T('PostProcessing was aborted (%s)') % T('see logfile')
        notifier.send_notification(T('Download Failed'), filename, 'failed',
                                   nzo.cat)
        nzo.status = Status.FAILED
        par_error = True
        all_ok = False

        if cfg.email_endjob():
            emailer.endjob(nzo.final_name, nzo.cat, all_ok,
                           clip_path(workdir_complete), nzo.bytes_downloaded,
                           nzo.fail_msg, nzo.unpack_info, '', '', 0)

    if all_ok:
        # If the folder only contains one file OR folder, have that as the path
        # Be aware that series/generic/date sorting may move a single file into a folder containing other files
        workdir_complete = one_file_or_folder(workdir_complete)
        workdir_complete = os.path.normpath(workdir_complete)

    # Clean up the NZO
    try:
        logging.info('Cleaning up %s (keep_basic=%s)', filename,
                     str(not all_ok))
        sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
    except:
        logging.error(T('Cleanup of %s failed.'), nzo.final_name)
        logging.info("Traceback: ", exc_info=True)

    # Remove download folder
    if all_ok:
        try:
            if os.path.exists(workdir):
                logging.debug('Removing workdir %s', workdir)
                remove_all(workdir, recursive=True)
        except:
            logging.error(T('Error removing workdir (%s)'), clip_path(workdir))
            logging.info("Traceback: ", exc_info=True)

    # Use automatic retry link on par2 errors and encrypted/bad RARs
    if par_error or unpack_error in (2, 3):
        try_alt_nzb(nzo)

    # Show final status in history
    if all_ok:
        notifier.send_notification(T('Download Completed'), filename,
                                   'complete', nzo.cat)
        nzo.status = Status.COMPLETED
    else:
        notifier.send_notification(T('Download Failed'), filename, 'failed',
                                   nzo.cat)
        nzo.status = Status.FAILED

    # Log the overall time taken for postprocessing
    postproc_time = int(time.time() - start)

    # Create the history DB instance
    history_db = database.HistoryDB()
    # Add the nzo to the database. Only the path, script and time taken is passed
    # Other information is obtained from the nzo
    history_db.add_history_db(nzo, clip_path(workdir_complete), nzo.downpath,
                              postproc_time, script_log, script_line)
    # Purge items
    history_db.auto_history_purge()
    # The connection is only used once, so close it here
    history_db.close()
    sabnzbd.history_updated()
    return True
Exemplo n.º 8
0
    def abort(self):
        """ Abort running instance and delete generated files """
        if not self.killed and self.cur_setname:
            logging.info('Aborting DirectUnpack for %s', self.cur_setname)
            self.killed = True

            # Save reference to the first rarfile
            rarfile_nzf = self.rarfile_nzf

            # Abort Unrar
            if self.active_instance:
                # First we try to abort gracefully
                try:
                    self.active_instance.stdin.write('Q\n')
                    time.sleep(0.2)
                except IOError:
                    pass

                # Now force kill and give it a bit of time
                try:
                    self.active_instance.kill()
                    time.sleep(0.2)
                except AttributeError:
                    # Already killed by the Quit command
                    pass

            # Wake up the thread
            with self.next_file_lock:
                self.next_file_lock.notify()

            # No new sets
            self.next_sets = []
            self.success_sets = {}

            # Remove files
            if self.unpack_dir_info:
                extraction_path, _, _, one_folder, _ = self.unpack_dir_info
                # In case of flat-unpack we need to remove the files manually
                if one_folder:
                    # RarFile can fail for mysterious reasons
                    try:
                        rar_contents = RarFile(os.path.join(
                            self.nzo.downpath, rarfile_nzf.filename),
                                               all_names=True).filelist()
                        for rm_file in rar_contents:
                            # Flat-unpack, so remove foldername from RarFile output
                            f = os.path.join(extraction_path,
                                             os.path.basename(rm_file))
                            remove_file(f)
                    except:
                        # The user will have to remove it themselves
                        logging.info(
                            'Failed to clean Direct Unpack after aborting %s',
                            rarfile_nzf.filename,
                            exc_info=True)
                else:
                    # We can just remove the whole path
                    remove_all(extraction_path, recursive=True)
                # Remove dir-info
                self.unpack_dir_info = None

            # Reset settings
            self.reset_active()
Exemplo n.º 9
0
def process_job(nzo):
    """ Process one job """
    start = time.time()

    # keep track of whether we can continue
    all_ok = True
    # keep track of par problems
    par_error = False
    # keep track of any unpacking errors
    unpack_error = False
    # Signal empty download, for when 'empty_postproc' is enabled
    empty = False
    nzb_list = []
    # These need to be initialized in case of a crash
    workdir_complete = ''
    script_log = ''
    script_line = ''

    # Get the job flags
    nzo.save_attribs()
    flag_repair, flag_unpack, flag_delete = nzo.repair_opts
    # Normalize PP
    if flag_delete:
        flag_unpack = True
    if flag_unpack:
        flag_repair = True

    # Get the NZB name
    filename = nzo.final_name

    if nzo.fail_msg:  # Special case: aborted due to too many missing data
        nzo.status = Status.FAILED
        nzo.save_attribs()
        all_ok = False
        par_error = True
        unpack_error = 1

    try:
        # Get the folder containing the download result
        workdir = nzo.downpath
        tmp_workdir_complete = None

        # if no files are present (except __admin__), fail the job
        if all_ok and len(globber(workdir)) < 2:
            if nzo.precheck:
                _enough, ratio = nzo.check_quality()
                req_ratio = float(cfg.req_completion_rate()) / 100.0
                # Make sure that rounded ratio doesn't equal required ratio
                # when it is actually below required
                if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
                    ratio = req_ratio - 0.001
                emsg = '%.1f%%' % (ratio * 100.0)
                emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
                emsg = T('Download might fail, only %s of required %s available') % (emsg, emsg2)
            else:
                emsg = T('Download failed - Not on your server(s)')
                empty = True
            emsg += ' - https://sabnzbd.org/not-complete'
            nzo.fail_msg = emsg
            nzo.set_unpack_info('Fail', emsg)
            nzo.status = Status.FAILED
            # do not run unpacking or parity verification
            flag_repair = flag_unpack = False
            all_ok = cfg.empty_postproc() and empty
            if not all_ok:
                par_error = True
                unpack_error = 1

        script = nzo.script
        logging.info('Starting Post-Processing on %s' +
                     ' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
                     filename, flag_repair, flag_unpack, flag_delete, script, nzo.cat)

        # Set complete dir to workdir in case we need to abort
        workdir_complete = workdir

        # Par processing, if enabled
        if all_ok and flag_repair:
            par_error, re_add = parring(nzo, workdir)
            if re_add:
                # Try to get more par files
                return False

        # If we don't need extra par2, we can disconnect
        if sabnzbd.nzbqueue.NzbQueue.do.actives(grabs=False) == 0 and cfg.autodisconnect():
            # This was the last job, close server connections
            sabnzbd.downloader.Downloader.do.disconnect()

        # Sanitize the resulting files
        if sabnzbd.WIN32:
            sanitize_files_in_folder(workdir)

        # Check if user allows unsafe post-processing
        if flag_repair and cfg.safe_postproc():
            all_ok = all_ok and not par_error

        if all_ok:
            # Fix encodings
            fix_unix_encoding(workdir)

            # Use dirs generated by direct-unpacker
            if nzo.direct_unpacker and nzo.direct_unpacker.unpack_dir_info:
                tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file = nzo.direct_unpacker.unpack_dir_info
            else:
                # Generate extraction path
                tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file = prepare_extraction_path(nzo)

            newfiles = []
            # Run Stage 2: Unpack
            if flag_unpack:
                if all_ok:
                    # set the current nzo status to "Extracting...". Used in History
                    nzo.status = Status.EXTRACTING
                    logging.info("Running unpack_magic on %s", filename)
                    unpack_error, newfiles = unpack_magic(nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), (), ())
                    logging.info("Unpacked files %s", newfiles)

                    if sabnzbd.WIN32:
                        # Sanitize the resulting files
                        newfiles = sanitize_files_in_folder(tmp_workdir_complete)
                    logging.info("Finished unpack_magic on %s", filename)
                else:
                    nzo.set_unpack_info('Unpack', T('No post-processing because of failed verification'))

            if cfg.safe_postproc():
                all_ok = all_ok and not unpack_error

            if all_ok:
                # Move any (left-over) files to destination
                nzo.status = Status.MOVING
                nzo.set_action_line(T('Moving'), '...')
                for root, _dirs, files in os.walk(workdir):
                    if not root.endswith(JOB_ADMIN):
                        for file_ in files:
                            path = os.path.join(root, file_)
                            new_path = path.replace(workdir, tmp_workdir_complete)
                            ok, new_path = move_to_path(path, new_path)
                            if new_path:
                                newfiles.append(new_path)
                            if not ok:
                                nzo.set_unpack_info('Unpack', T('Failed moving %s to %s') % (unicoder(path), unicoder(new_path)))
                                all_ok = False
                                break

            # Set permissions right
            set_permissions(tmp_workdir_complete)

            if all_ok and marker_file:
                del_marker(os.path.join(tmp_workdir_complete, marker_file))
                remove_from_list(marker_file, newfiles)

            if all_ok:
                # Remove files matching the cleanup list
                cleanup_list(tmp_workdir_complete, True)

                # Check if this is an NZB-only download, if so redirect to queue
                # except when PP was Download-only
                if flag_repair:
                    nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, nzo.cat, priority=nzo.priority)
                else:
                    nzb_list = None
                if nzb_list:
                    nzo.set_unpack_info('Download', T('Sent %s to queue') % unicoder(nzb_list))
                    cleanup_empty_directories(tmp_workdir_complete)
                else:
                    cleanup_list(tmp_workdir_complete, False)

        script_output = ''
        script_ret = 0
        if not nzb_list:
            # Give destination its final name
            if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
                if all_ok:
                    try:
                        newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles)
                    except:
                        logging.error(T('Error renaming "%s" to "%s"'), clip_path(tmp_workdir_complete), clip_path(workdir_complete))
                        logging.info('Traceback: ', exc_info=True)
                        # Better disable sorting because filenames are all off now
                        file_sorter.sort_file = None
                else:
                    workdir_complete = tmp_workdir_complete.replace('_UNPACK_', '_FAILED_')
                    workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False)
                    workdir_complete = workdir_complete

            if empty:
                job_result = -1
            else:
                job_result = int(par_error) + int(bool(unpack_error)) * 2

            if cfg.ignore_samples():
                remove_samples(workdir_complete)

            # TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
            if all_ok and file_sorter.sort_file:
                if newfiles:
                    file_sorter.rename(newfiles, workdir_complete)
                    workdir_complete, ok = file_sorter.move(workdir_complete)
                else:
                    workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete)
                if not ok:
                    nzo.set_unpack_info('Unpack', T('Failed to move files'))
                    all_ok = False

            # Run the user script
            script_path = make_script_path(script)
            if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path:
                # Set the current nzo status to "Ext Script...". Used in History
                nzo.status = Status.RUNNING
                nzo.set_action_line(T('Running script'), unicoder(script))
                nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True)
                script_log, script_ret = external_processing(script_path, nzo, clip_path(workdir_complete),
                                                             nzo.final_name, job_result)
                script_line = get_last_line(script_log)
                if script_log:
                    script_output = nzo.nzo_id
                if script_line:
                    nzo.set_unpack_info('Script', unicoder(script_line), unique=True)
                else:
                    nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True)
            else:
                script = ""
                script_line = ""
                script_ret = 0

        # Maybe bad script result should fail job
        if script_ret and cfg.script_can_fail():
            script_error = True
            all_ok = False
            nzo.fail_msg = T('Script exit code is %s') % script_ret
        else:
            script_error = False

        # Email the results
        if (not nzb_list) and cfg.email_endjob():
            if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error or script_error)):
                emailer.endjob(nzo.final_name, nzo.cat, all_ok, workdir_complete, nzo.bytes_downloaded,
                               nzo.fail_msg, nzo.unpack_info, script, TRANS(script_log), script_ret)

        if script_output:
            # Can do this only now, otherwise it would show up in the email
            if script_ret:
                script_ret = 'Exit(%s) ' % script_ret
            else:
                script_ret = ''
            if len(script_log.rstrip().split('\n')) > 1:
                nzo.set_unpack_info('Script',
                                    u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, script_line,
                                    xml.sax.saxutils.escape(script_output), T('More')), unique=True)
            else:
                # No '(more)' button needed
                nzo.set_unpack_info('Script', u'%s%s ' % (script_ret, script_line), unique=True)

        # Cleanup again, including NZB files
        if all_ok:
            cleanup_list(workdir_complete, False)

        # Force error for empty result
        all_ok = all_ok and not empty

        # Update indexer with results
        if cfg.rating_enable():
            if nzo.encrypted > 0:
                Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
            if empty:
                hosts = map(lambda s: s.host, sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
                if not hosts:
                    hosts = [None]
                for host in hosts:
                    Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED, host)

    except:
        logging.error(T('Post Processing Failed for %s (%s)'), filename, T('see logfile'))
        logging.info("Traceback: ", exc_info=True)

        nzo.fail_msg = T('PostProcessing was aborted (%s)') % T('see logfile')
        notifier.send_notification(T('Download Failed'), filename, 'failed', nzo.cat)
        nzo.status = Status.FAILED
        par_error = True
        all_ok = False

        if cfg.email_endjob():
            emailer.endjob(nzo.final_name, nzo.cat, all_ok, clip_path(workdir_complete), nzo.bytes_downloaded,
                           nzo.fail_msg, nzo.unpack_info, '', '', 0)

    if all_ok:
        # If the folder only contains one file OR folder, have that as the path
        # Be aware that series/generic/date sorting may move a single file into a folder containing other files
        workdir_complete = one_file_or_folder(workdir_complete)
        workdir_complete = os.path.normpath(workdir_complete)

    # Clean up the NZO
    try:
        logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok))
        sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
    except:
        logging.error(T('Cleanup of %s failed.'), nzo.final_name)
        logging.info("Traceback: ", exc_info=True)

    # Remove download folder
    if all_ok:
        try:
            if os.path.exists(workdir):
                logging.debug('Removing workdir %s', workdir)
                remove_all(workdir, recursive=True)
        except:
            logging.error(T('Error removing workdir (%s)'), clip_path(workdir))
            logging.info("Traceback: ", exc_info=True)

    # Use automatic retry link on par2 errors and encrypted/bad RARs
    if par_error or unpack_error in (2, 3):
        try_alt_nzb(nzo)

    # Show final status in history
    if all_ok:
        notifier.send_notification(T('Download Completed'), filename, 'complete', nzo.cat)
        nzo.status = Status.COMPLETED
    else:
        notifier.send_notification(T('Download Failed'), filename, 'failed', nzo.cat)
        nzo.status = Status.FAILED

    # Log the overall time taken for postprocessing
    postproc_time = int(time.time() - start)

    # Create the history DB instance
    history_db = database.HistoryDB()
    # Add the nzo to the database. Only the path, script and time taken is passed
    # Other information is obtained from the nzo
    history_db.add_history_db(nzo, clip_path(workdir_complete), nzo.downpath, postproc_time, script_log, script_line)
    # Purge items
    history_db.auto_history_purge()
    # The connection is only used once, so close it here
    history_db.close()
    sabnzbd.history_updated()
    return True
Exemplo n.º 10
0
def initialize(pause_downloader=False,
               clean_up=False,
               evalSched=False,
               repair=0):
    global __INITIALIZED__, __SHUTTING_DOWN__,\
        LOGFILE, WEBLOGFILE, LOGHANDLER, GUIHANDLER, AMBI_LOCALHOST, WAITEXIT, \
        DAEMON, MY_NAME, MY_FULLNAME, NEW_VERSION, \
        DIR_HOME, DIR_APPDATA, DIR_LCLDATA, DIR_PROG, DIR_INTERFACES, \
        DARWIN, RESTART_REQ, OLD_QUEUE

    if __INITIALIZED__:
        return False

    __SHUTTING_DOWN__ = False

    # Set global database connection for Web-UI threads
    cherrypy.engine.subscribe('start_thread', get_db_connection)

    # Paused?
    pause_downloader = pause_downloader or cfg.start_paused()

    # Clean-up, if requested
    if clean_up:
        # New admin folder
        misc.remove_all(cfg.admin_dir.get_path(), '*.sab')

    # Optionally wait for "incomplete" to become online
    if cfg.wait_for_dfolder():
        wait_for_download_folder()
    else:
        cfg.download_dir.set(cfg.download_dir(), create=True)
    cfg.download_dir.set_create(True)

    # Set access rights for "incomplete" base folder
    misc.set_permissions(cfg.download_dir.get_path(), recursive=False)

    # If dirscan_dir cannot be created, set a proper value anyway.
    # Maybe it's a network path that's temporarily missing.
    path = cfg.dirscan_dir.get_path()
    if not os.path.exists(path):
        sabnzbd.misc.create_real_path(cfg.dirscan_dir.ident(), '', path, False)

    # Set call backs for Config items
    cfg.cache_limit.callback(new_limit)
    cfg.cherryhost.callback(guard_restart)
    cfg.cherryport.callback(guard_restart)
    cfg.web_dir.callback(guard_restart)
    cfg.web_color.callback(guard_restart)
    cfg.username.callback(guard_restart)
    cfg.password.callback(guard_restart)
    cfg.log_dir.callback(guard_restart)
    cfg.https_port.callback(guard_restart)
    cfg.https_cert.callback(guard_restart)
    cfg.https_key.callback(guard_restart)
    cfg.enable_https.callback(guard_restart)
    cfg.top_only.callback(guard_top_only)
    cfg.pause_on_post_processing.callback(guard_pause_on_pp)
    cfg.growl_server.callback(sabnzbd.notifier.change_value)
    cfg.growl_password.callback(sabnzbd.notifier.change_value)
    cfg.quota_size.callback(guard_quota_size)
    cfg.quota_day.callback(guard_quota_dp)
    cfg.quota_period.callback(guard_quota_dp)
    cfg.fsys_type.callback(guard_fsys_type)
    cfg.language.callback(guard_language)
    cfg.enable_https_verification.callback(guard_https_ver)
    guard_https_ver()

    # Set Posix filesystem encoding
    sabnzbd.encoding.change_fsys(cfg.fsys_type())

    # Set cache limit
    if not cfg.cache_limit() or (cfg.cache_limit() in ('200M', '450M') and
                                 (sabnzbd.WIN32 or sabnzbd.DARWIN)):
        cfg.cache_limit.set(misc.get_cache_limit())
    ArticleCache.do.new_limit(cfg.cache_limit.get_int())

    check_incomplete_vs_complete()

    # Set language files
    lang.set_locale_info('SABnzbd', DIR_LANGUAGE)
    lang.set_language(cfg.language())
    sabnzbd.api.clear_trans_cache()

    OLD_QUEUE = check_old_queue()

    sabnzbd.change_queue_complete_action(cfg.queue_complete(), new=False)

    # One time conversion "speedlimit" in schedules.
    if not cfg.sched_converted():
        schedules = cfg.schedules()
        newsched = []
        for sched in schedules:
            if 'speedlimit' in sched:
                newsched.append(re.sub(r'(speedlimit \d+)$', r'\1K', sched))
            else:
                newsched.append(sched)
        cfg.schedules.set(newsched)
        cfg.sched_converted.set(1)

    # Second time schedule conversion
    if cfg.sched_converted() != 2:
        cfg.schedules.set(
            ['%s %s' % (1, schedule) for schedule in cfg.schedules()])
        cfg.sched_converted.set(2)
        config.save_config()

    # Add hostname to the whitelist
    if not cfg.host_whitelist():
        cfg.host_whitelist.set(socket.gethostname())

    # Do repair if requested
    if check_repair_request():
        repair = 2
        pause_downloader = True

    # Initialize threads
    rss.init()

    paused = BPSMeter.do.read()

    NzbQueue()

    Downloader(pause_downloader or paused)

    Assembler()

    PostProcessor()

    NzbQueue.do.read_queue(repair)

    DirScanner()

    Rating()

    URLGrabber()

    scheduler.init()

    if evalSched:
        scheduler.analyse(pause_downloader)

    logging.info('All processes started')
    RESTART_REQ = False
    __INITIALIZED__ = True
    return True
Exemplo n.º 11
0
def initialize(pause_downloader=False, clean_up=False, evalSched=False, repair=0):
    global __INITIALIZED__, __SHUTTING_DOWN__,\
        LOGFILE, WEBLOGFILE, LOGHANDLER, GUIHANDLER, AMBI_LOCALHOST, WAITEXIT, \
        DAEMON, MY_NAME, MY_FULLNAME, NEW_VERSION, \
        DIR_HOME, DIR_APPDATA, DIR_LCLDATA, DIR_PROG, DIR_INTERFACES, \
        DARWIN, RESTART_REQ, OLD_QUEUE

    if __INITIALIZED__:
        return False

    __SHUTTING_DOWN__ = False

    # Set global database connection for Web-UI threads
    cherrypy.engine.subscribe('start_thread', connect_db)

    # Paused?
    pause_downloader = pause_downloader or cfg.start_paused()

    # Clean-up, if requested
    if clean_up:
        # New admin folder
        misc.remove_all(cfg.admin_dir.get_path(), '*.sab')

    # Optionally wait for "incomplete" to become online
    if cfg.wait_for_dfolder():
        wait_for_download_folder()
    else:
        cfg.download_dir.set(cfg.download_dir(), create=True)
    cfg.download_dir.set_create(True)

    # Set access rights for "incomplete" base folder
    misc.set_permissions(cfg.download_dir.get_path(), recursive=False)

    # If dirscan_dir cannot be created, set a proper value anyway.
    # Maybe it's a network path that's temporarily missing.
    path = cfg.dirscan_dir.get_path()
    if not os.path.exists(path):
        sabnzbd.misc.create_real_path(cfg.dirscan_dir.ident(), '', path, False)

    # Set call backs for Config items
    cfg.cache_limit.callback(new_limit)
    cfg.cherryhost.callback(guard_restart)
    cfg.cherryport.callback(guard_restart)
    cfg.web_dir.callback(guard_restart)
    cfg.web_color.callback(guard_restart)
    cfg.username.callback(guard_restart)
    cfg.password.callback(guard_restart)
    cfg.log_dir.callback(guard_restart)
    cfg.https_port.callback(guard_restart)
    cfg.https_cert.callback(guard_restart)
    cfg.https_key.callback(guard_restart)
    cfg.enable_https.callback(guard_restart)
    cfg.top_only.callback(guard_top_only)
    cfg.pause_on_post_processing.callback(guard_pause_on_pp)
    cfg.growl_server.callback(sabnzbd.notifier.change_value)
    cfg.growl_password.callback(sabnzbd.notifier.change_value)
    cfg.quota_size.callback(guard_quota_size)
    cfg.quota_day.callback(guard_quota_dp)
    cfg.quota_period.callback(guard_quota_dp)
    cfg.fsys_type.callback(guard_fsys_type)
    cfg.language.callback(sabnzbd.notifier.reset_growl)
    cfg.enable_https_verification.callback(guard_https_ver)
    guard_https_ver()

    # Set Posix filesystem encoding
    sabnzbd.encoding.change_fsys(cfg.fsys_type())

    # Set cache limit
    if not cfg.cache_limit() or (cfg.cache_limit() == '200M' and (sabnzbd.WIN32 or sabnzbd.DARWIN)):
        cfg.cache_limit.set(misc.get_cache_limit())
    ArticleCache.do.new_limit(cfg.cache_limit.get_int())

    check_incomplete_vs_complete()

    # Set language files
    lang.set_locale_info('SABnzbd', DIR_LANGUAGE)
    lang.set_language(cfg.language())
    sabnzbd.api.clear_trans_cache()

    OLD_QUEUE = check_old_queue()

    sabnzbd.change_queue_complete_action(cfg.queue_complete(), new=False)

    # One time conversion "speedlimit" in schedules.
    if not cfg.sched_converted():
        schedules = cfg.schedules()
        newsched = []
        for sched in schedules:
            if 'speedlimit' in sched:
                newsched.append(re.sub(r'(speedlimit \d+)$', r'\1K', sched))
            else:
                newsched.append(sched)
        cfg.schedules.set(newsched)
        cfg.sched_converted.set(1)

    # Second time schedule conversion
    if cfg.sched_converted() != 2:
        cfg.schedules.set(['%s %s' % (1, schedule) for schedule in cfg.schedules()])
        cfg.sched_converted.set(2)

    if check_repair_request():
        repair = 2
        pause_downloader = True

    # Initialize threads
    rss.init()

    paused = BPSMeter.do.read()

    PostProcessor()

    NzbQueue()

    Assembler()

    NzbQueue.do.read_queue(repair)

    Downloader(pause_downloader or paused)

    DirScanner()

    Rating()

    URLGrabber()

    scheduler.init()

    if evalSched:
        scheduler.analyse(pause_downloader)

    logging.info('All processes started')
    RESTART_REQ = False
    __INITIALIZED__ = True
    return True
Exemplo n.º 12
0
def initialize(pause_downloader=False, clean_up=False, evalSched=False, repair=0):
    global __INITIALIZED__, __SHUTTING_DOWN__, LOGFILE, WEBLOGFILE, LOGHANDLER, GUIHANDLER, AMBI_LOCALHOST, WAITEXIT, DAEMON, MY_NAME, MY_FULLNAME, NEW_VERSION, DIR_HOME, DIR_APPDATA, DIR_LCLDATA, DIR_PROG, DIR_INTERFACES, DARWIN, RESTART_REQ, OLD_QUEUE

    if __INITIALIZED__:
        return False

    __SHUTTING_DOWN__ = False

    # Set global database connection for Web-UI threads
    cherrypy.engine.subscribe("start_thread", connect_db)

    # Paused?
    pause_downloader = pause_downloader or cfg.start_paused()

    # Clean-up, if requested
    if clean_up:
        # New admin folder
        misc.remove_all(cfg.admin_dir.get_path(), "*.sab")

    # Optionally wait for "incomplete" to become online
    if cfg.wait_for_dfolder():
        wait_for_download_folder()
    else:
        cfg.download_dir.set(cfg.download_dir(), create=True)
    cfg.download_dir.set_create(True)

    # Set access rights for "incomplete" base folder
    misc.set_permissions(cfg.download_dir.get_path(), recursive=False)

    # If dirscan_dir cannot be created, set a proper value anyway.
    # Maybe it's a network path that's temporarily missing.
    path = cfg.dirscan_dir.get_path()
    if not os.path.exists(path):
        sabnzbd.misc.create_real_path(cfg.dirscan_dir.ident(), "", path, False)

    # Set call backs for Config items
    cfg.cache_limit.callback(new_limit)
    cfg.cherryhost.callback(guard_restart)
    cfg.cherryport.callback(guard_restart)
    cfg.web_dir.callback(guard_restart)
    cfg.web_dir2.callback(guard_restart)
    cfg.web_color.callback(guard_restart)
    cfg.web_color2.callback(guard_restart)
    cfg.log_dir.callback(guard_restart)
    cfg.https_port.callback(guard_restart)
    cfg.https_cert.callback(guard_restart)
    cfg.https_key.callback(guard_restart)
    cfg.enable_https.callback(guard_restart)
    cfg.top_only.callback(guard_top_only)
    cfg.pause_on_post_processing.callback(guard_pause_on_pp)
    cfg.growl_server.callback(sabnzbd.growler.change_value)
    cfg.growl_password.callback(sabnzbd.growler.change_value)
    cfg.quota_size.callback(guard_quota_size)
    cfg.quota_day.callback(guard_quota_dp)
    cfg.quota_period.callback(guard_quota_dp)
    cfg.fsys_type.callback(guard_fsys_type)
    cfg.language.callback(sabnzbd.growler.reset_growl)
    cfg.enable_https_verification.callback(guard_https_ver)
    guard_https_ver()

    # Set Posix filesystem encoding
    sabnzbd.encoding.change_fsys(cfg.fsys_type())

    # Set cache limit
    if (sabnzbd.WIN32 or sabnzbd.DARWIN) and not cfg.cache_limit():
        cfg.cache_limit.set("200M")
    ArticleCache.do.new_limit(cfg.cache_limit.get_int())

    check_incomplete_vs_complete()

    # Handle language upgrade from 0.5.x to 0.6.x
    cfg.language.set(LANG_MAP.get(cfg.language(), cfg.language()))

    # Set language files
    lang.set_locale_info("SABnzbd", DIR_LANGUAGE)
    lang.set_language(cfg.language())
    sabnzbd.api.clear_trans_cache()

    OLD_QUEUE = check_old_queue()

    sabnzbd.change_queue_complete_action(cfg.queue_complete(), new=False)

    sabnzbd.EXTERNAL_IPV6 = test_ipv6()
    logging.debug("External IPv6 test result: %s", sabnzbd.EXTERNAL_IPV6)

    # One time conversion "speedlimit" in schedules.
    if not cfg.sched_converted():
        schedules = cfg.schedules()
        newsched = []
        for sched in schedules:
            if "speedlimit" in sched:
                newsched.append(re.sub(r"(speedlimit \d+)$", r"\1K", sched))
            else:
                newsched.append(sched)
        cfg.schedules.set(newsched)
        cfg.sched_converted.set(True)

    if check_repair_request():
        repair = 2
        pause_downloader = True
    else:
        # Check crash detection file
        # if load_admin(TERM_FLAG_FILE, remove=True):
        # Repair mode 2 is a bit over an over-reaction!
        pass  # repair = 2

    # Set crash detection file
    # save_admin(1, TERM_FLAG_FILE)

    # Initialize threads
    rss.init()

    paused = BPSMeter.do.read()

    PostProcessor()

    NzbQueue()

    Assembler()

    NzbQueue.do.read_queue(repair)

    Downloader(pause_downloader or paused)

    DirScanner()

    Rating()

    URLGrabber()

    scheduler.init()

    if evalSched:
        scheduler.analyse(pause_downloader)

    logging.info("All processes started")
    RESTART_REQ = False
    __INITIALIZED__ = True
    return True
Exemplo n.º 13
0
def initialize(pause_downloader = False, clean_up = False, evalSched=False, repair=0):
    global __INITIALIZED__, __SHUTTING_DOWN__,\
           LOGFILE, WEBLOGFILE, LOGHANDLER, GUIHANDLER, AMBI_LOCALHOST, WAITEXIT, \
           DAEMON, MY_NAME, MY_FULLNAME, NEW_VERSION, \
           DIR_HOME, DIR_APPDATA, DIR_LCLDATA, DIR_PROG , DIR_INTERFACES, \
           DARWIN, RESTART_REQ, OSX_ICON, OLD_QUEUE

    if __INITIALIZED__:
        return False

    __SHUTTING_DOWN__ = False

    ### Set global database connection for Web-UI threads
    cherrypy.engine.subscribe('start_thread', connect_db)

    ### Clean-up, if requested
    if clean_up:
        # Old cache folder
        misc.remove_all(cfg.cache_dir.get_path(), '*.sab')
        misc.remove_all(cfg.cache_dir.get_path(), 'SABnzbd_*')
        # New admin folder
        misc.remove_all(cfg.admin_dir.get_path(), '*.sab')

    ### If dirscan_dir cannot be created, set a proper value anyway.
    ### Maybe it's a network path that's temporarily missing.
    path = cfg.dirscan_dir.get_path()
    if not os.path.exists(path):
        sabnzbd.misc.create_real_path(cfg.dirscan_dir.ident(), '', path, False)

    ### Set call backs for Config items
    cfg.cache_limit.callback(new_limit)
    cfg.cherryhost.callback(guard_restart)
    cfg.cherryport.callback(guard_restart)
    cfg.web_dir.callback(guard_restart)
    cfg.web_dir2.callback(guard_restart)
    cfg.web_color.callback(guard_restart)
    cfg.web_color2.callback(guard_restart)
    cfg.log_dir.callback(guard_restart)
    cfg.cache_dir.callback(guard_restart)
    cfg.https_port.callback(guard_restart)
    cfg.https_cert.callback(guard_restart)
    cfg.https_key.callback(guard_restart)
    cfg.enable_https.callback(guard_restart)
    cfg.bandwidth_limit.callback(guard_speedlimit)
    cfg.top_only.callback(guard_top_only)
    cfg.pause_on_post_processing.callback(guard_pause_on_pp)

    ### Set cache limit
    ArticleCache.do.new_limit(cfg.cache_limit.get_int())

    ### Handle language upgrade from 0.5.x to 0.6.x
    cfg.language.set(LANG_MAP.get(cfg.language(), cfg.language()))

    ### Set language files
    lang.set_locale_info('SABnzbd', DIR_LANGUAGE)
    lang.set_language(cfg.language())
    sabnzbd.api.cache_skin_trans()

    ### Check for old queue (when a new queue is not present)
    if not os.path.exists(os.path.join(cfg.cache_dir.get_path(), QUEUE_FILE_NAME)):
        OLD_QUEUE = bool(misc.globber(cfg.cache_dir.get_path(), QUEUE_FILE_TMPL % '?'))

    sabnzbd.change_queue_complete_action(cfg.queue_complete(), new=False)

    if check_repair_request():
        repair = 2
        pause_downloader = True
    else:
        # Check crash detection file
        #if load_admin(TERM_FLAG_FILE, remove=True):
            # Repair mode 2 is a bit over an over-reaction!
        pass # repair = 2

    # Set crash detection file
    #save_admin(1, TERM_FLAG_FILE)

    ###
    ### Initialize threads
    ###

    Bookmarks()
    rss.init()

    BPSMeter.do.read()

    PostProcessor()

    NzbQueue()
    NzbQueue.do.read_queue(repair)

    Assembler()

    Downloader(pause_downloader)

    DirScanner()

    MSGIDGrabber()

    URLGrabber()

    scheduler.init()

    if evalSched:
        scheduler.analyse(pause_downloader)

    logging.info('All processes started')
    RESTART_REQ = False
    __INITIALIZED__ = True
    return True
Exemplo n.º 14
0
    def abort(self):
        """ Abort running instance and delete generated files """
        if not self.killed and self.cur_setname:
            logging.info('Aborting DirectUnpack for %s', self.cur_setname)
            self.killed = True

            # Save reference to the first rarfile
            rarfile_nzf = self.rarfile_nzf

            # Abort Unrar
            if self.active_instance:
                # First we try to abort gracefully
                try:
                    self.active_instance.stdin.write('Q\n')
                    time.sleep(0.2)
                except IOError:
                    pass

                # Now force kill and give it a bit of time
                self.active_instance.kill()
                time.sleep(0.2)

                # Have to collect the return-code to avoid zombie
                # But it will block forever if the process is in special state.
                # That should never happen, but it can happen on broken unrar's
                if self.active_instance.poll():
                    self.active_instance.communicate()
                else:
                    # It is still running?!? This should never happen
                    # Wait a little bit longer just to be sure..
                    time.sleep(2.0)
                    if not self.active_instance.poll():
                        logging.warning(T('Unable to stop the unrar process.'))

            # Wake up the thread
            with self.next_file_lock:
                self.next_file_lock.notify()

            # No new sets
            self.next_sets = []
            self.success_sets = {}

            # Remove files
            if self.unpack_dir_info:
                extraction_path, _, _, one_folder, _ = self.unpack_dir_info
                # In case of flat-unpack we need to remove the files manually
                if one_folder:
                    # RarFile can fail for mysterious reasons
                    try:
                        rar_contents = RarFile(os.path.join(
                            self.nzo.downpath, rarfile_nzf.filename),
                                               all_names=True).filelist()
                        for rm_file in rar_contents:
                            # Flat-unpack, so remove foldername from RarFile output
                            f = os.path.join(extraction_path,
                                             os.path.basename(rm_file))
                            remove_file(f)
                    except:
                        # The user will have to remove it themselves
                        logging.info(
                            'Failed to clean Direct Unpack after aborting %s',
                            rarfile_nzf.filename,
                            exc_info=True)
                else:
                    # We can just remove the whole path
                    remove_all(extraction_path, recursive=True)
                # Remove dir-info
                self.unpack_dir_info = None

            # Reset settings
            self.reset_active()
Exemplo n.º 15
0
def process_job(nzo):
    """ Process one job """
    assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
    start = time.time()

    # keep track of whether we can continue
    all_ok = True
    # keep track of par problems
    par_error = False
    # keep track of any unpacking errors
    unpack_error = False
    nzb_list = []
    # These need to be initialised incase of a crash
    workdir_complete = ''
    postproc_time = 0
    script_log = ''
    script_line = ''
    crash_msg = ''

    ## Get the job flags
    nzo.save_attribs()
    flag_repair, flag_unpack, flag_delete = nzo.repair_opts
    # Normalize PP
    if flag_delete: flag_unpack = True
    if flag_unpack: flag_repair = True

    # Get the NZB name
    filename = nzo.final_name
    msgid = nzo.msgid

    if cfg.allow_streaming() and not (flag_repair or flag_unpack
                                      or flag_delete):
        # After streaming, force +D
        nzo.set_pp(3)
        nzo.status = 'Failed'
        nzo.save_attribs()
        all_ok = False

    try:

        # Get the folder containing the download result
        workdir = nzo.downpath
        tmp_workdir_complete = None

        # if no files are present (except __admin__), fail the job
        if len(globber(workdir)) < 2:
            emsg = T('Download failed - Out of your server\'s retention?')
            nzo.fail_msg = emsg
            nzo.status = 'Failed'
            # do not run unpacking or parity verification
            flag_repair = flag_unpack = False
            par_error = unpack_error = True
            all_ok = False

        script = nzo.script
        cat = nzo.cat

        logging.info('Starting PostProcessing on %s' + \
                     ' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
                     filename, flag_repair, flag_unpack, flag_delete, script, cat)

        ## Par processing, if enabled
        if flag_repair:
            par_error, re_add = parring(nzo, workdir)
            if re_add:
                # Try to get more par files
                return False

        ## Check if user allows unsafe post-processing
        if flag_repair and cfg.safe_postproc():
            all_ok = all_ok and not par_error

        # Set complete dir to workdir in case we need to abort
        workdir_complete = workdir
        dirname = nzo.final_name

        if all_ok:
            one_folder = False
            ## Determine class directory
            if cfg.create_group_folders():
                complete_dir = addPrefixes(cfg.complete_dir.get_path(),
                                           nzo.dirprefix)
                complete_dir = create_dirs(complete_dir)
            else:
                catdir = config.get_categories(cat).dir()
                if catdir.endswith('*'):
                    catdir = catdir.strip('*')
                    one_folder = True
                complete_dir = real_path(cfg.complete_dir.get_path(), catdir)

            ## TV/Movie/Date Renaming code part 1 - detect and construct paths
            file_sorter = Sorter(cat)
            complete_dir = file_sorter.detect(dirname, complete_dir)
            if file_sorter.is_sortfile():
                one_folder = False

            if one_folder:
                workdir_complete = create_dirs(complete_dir)
            else:
                workdir_complete = get_unique_path(os.path.join(
                    complete_dir, dirname),
                                                   create_dir=True)
            if not workdir_complete or not os.path.exists(workdir_complete):
                crash_msg = T('Cannot create final folder %s') % unicoder(
                    os.path.join(complete_dir, dirname))
                raise IOError

            if cfg.folder_rename() and not one_folder:
                tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_')
                try:
                    renamer(workdir_complete, tmp_workdir_complete)
                except:
                    pass  # On failure, just use the original name
            else:
                tmp_workdir_complete = workdir_complete

            newfiles = []
            ## Run Stage 2: Unpack
            if flag_unpack:
                if all_ok:
                    #set the current nzo status to "Extracting...". Used in History
                    nzo.status = 'Extracting'
                    logging.info("Running unpack_magic on %s", filename)
                    unpack_error, newfiles = unpack_magic(
                        nzo, workdir, tmp_workdir_complete, flag_delete,
                        one_folder, (), (), (), ())
                    logging.info("unpack_magic finished on %s", filename)
                else:
                    nzo.set_unpack_info(
                        'Unpack',
                        T('No post-processing because of failed verification'))

            if cfg.safe_postproc():
                all_ok = all_ok and not unpack_error

            if all_ok:
                ## Move any (left-over) files to destination
                nzo.status = 'Moving'
                nzo.set_action_line(T('Moving'), '...')
                for root, dirs, files in os.walk(workdir):
                    if not root.endswith(JOB_ADMIN):
                        for file_ in files:
                            path = os.path.join(root, file_)
                            new_path = path.replace(workdir,
                                                    tmp_workdir_complete)
                            new_path = get_unique_filename(new_path)
                            move_to_path(path, new_path, unique=False)

            ## Set permissions right
            if not sabnzbd.WIN32:
                perm_script(tmp_workdir_complete, cfg.umask())

            if all_ok:
                ## Remove files matching the cleanup list
                cleanup_list(tmp_workdir_complete, True)

                ## Check if this is an NZB-only download, if so redirect to queue
                ## except when PP was Download-only
                if flag_repair:
                    nzb_list = nzb_redirect(tmp_workdir_complete,
                                            nzo.final_name,
                                            nzo.pp,
                                            script,
                                            cat,
                                            priority=nzo.priority)
                else:
                    nzb_list = None
                if nzb_list:
                    nzo.set_unpack_info(
                        'Download',
                        T('Sent %s to queue') % unicoder(nzb_list))
                    try:
                        remove_dir(tmp_workdir_complete)
                    except:
                        pass
                else:
                    cleanup_list(tmp_workdir_complete, False)

        script_output = ''
        script_ret = 0
        if not nzb_list:
            ## Give destination its final name
            if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
                if not all_ok:
                    workdir_complete = tmp_workdir_complete.replace(
                        '_UNPACK_', '_FAILED_')
                    workdir_complete = get_unique_path(workdir_complete,
                                                       n=0,
                                                       create_dir=False)
                try:
                    collapse_folder(tmp_workdir_complete, workdir_complete)
                except:
                    logging.error(Ta('Error renaming "%s" to "%s"'),
                                  tmp_workdir_complete, workdir_complete)
                    logging.info("Traceback: ", exc_info=True)

            job_result = int(par_error) + int(unpack_error) * 2

            if cfg.ignore_samples() > 0:
                remove_samples(workdir_complete)

            ## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
            if all_ok:
                if newfiles and file_sorter.is_sortfile():
                    file_sorter.rename(newfiles, workdir_complete)
                    workdir_complete = file_sorter.move(workdir_complete)

            ## Run the user script
            script_path = make_script_path(script)
            if all_ok and (not nzb_list) and script_path:
                #set the current nzo status to "Ext Script...". Used in History
                nzo.status = 'Running'
                nzo.set_action_line(T('Running script'), unicoder(script))
                nzo.set_unpack_info('Script',
                                    T('Running user script %s') %
                                    unicoder(script),
                                    unique=True)
                script_log, script_ret = external_processing(
                    script_path, workdir_complete, nzo.filename, msgid,
                    dirname, cat, nzo.group, job_result)
                script_line = get_last_line(script_log)
                if script_log:
                    script_output = nzo.nzo_id
                if script_line:
                    nzo.set_unpack_info('Script',
                                        unicoder(script_line),
                                        unique=True)
                else:
                    nzo.set_unpack_info('Script',
                                        T('Ran %s') % unicoder(script),
                                        unique=True)
            else:
                script = ""
                script_line = ""
                script_ret = 0

        ## Email the results
        if (not nzb_list) and cfg.email_endjob():
            if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and
                                             (unpack_error or par_error)):
                emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete,
                               nzo.bytes_downloaded, nzo.unpack_info, script,
                               TRANS(script_log), script_ret)

        if script_output:
            # Can do this only now, otherwise it would show up in the email
            if script_ret:
                script_ret = 'Exit(%s) ' % script_ret
            else:
                script_ret = ''
            if script_line:
                nzo.set_unpack_info(
                    'Script',
                    u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' %
                    (script_ret, unicoder(script_line),
                     urllib.quote(script_output), T('More')),
                    unique=True)
            else:
                nzo.set_unpack_info('Script',
                                    u'%s<a href="./scriptlog?name=%s">%s</a>' %
                                    (script_ret, urllib.quote(script_output),
                                     T('View script output')),
                                    unique=True)

        ## Cleanup again, including NZB files
        cleanup_list(workdir_complete, False)

        ## Remove newzbin bookmark, if any
        if msgid and all_ok:
            Bookmarks.do.del_bookmark(msgid)
        elif all_ok:
            sabnzbd.proxy_rm_bookmark(nzo.url)

        ## Show final status in history
        if all_ok:
            osx.sendGrowlMsg(T('Download Completed'), filename,
                             osx.NOTIFICATION['complete'])
            nzo.status = 'Completed'
        else:
            osx.sendGrowlMsg(T('Download Failed'), filename,
                             osx.NOTIFICATION['complete'])
            nzo.status = 'Failed'

    except:
        logging.error(Ta('Post Processing Failed for %s (%s)'), filename,
                      crash_msg)
        if not crash_msg:
            logging.info("Traceback: ", exc_info=True)
            crash_msg = T('see logfile')
        nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(
            crash_msg)
        osx.sendGrowlMsg(T('Download Failed'), filename,
                         osx.NOTIFICATION['complete'])
        nzo.status = 'Failed'
        par_error = True
        all_ok = False

    if all_ok:
        # If the folder only contains one file OR folder, have that as the path
        # Be aware that series/generic/date sorting may move a single file into a folder containing other files
        workdir_complete = one_file_or_folder(workdir_complete)
        workdir_complete = os.path.normpath(workdir_complete)

    # Log the overall time taken for postprocessing
    postproc_time = int(time.time() - start)

    # Create the history DB instance
    history_db = database.get_history_handle()
    # Add the nzo to the database. Only the path, script and time taken is passed
    # Other information is obtained from the nzo
    history_db.add_history_db(nzo, workdir_complete, nzo.downpath,
                              postproc_time, script_log, script_line)
    # The connection is only used once, so close it here
    history_db.close()

    ## Clean up the NZO
    try:
        logging.info('Cleaning up %s (keep_basic=%s)', filename,
                     str(not all_ok))
        sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
    except:
        logging.error(Ta('Cleanup of %s failed.'), nzo.final_name)
        logging.info("Traceback: ", exc_info=True)

    ## Remove download folder
    if all_ok:
        try:
            if os.path.exists(workdir):
                logging.debug('Removing workdir %s', workdir)
                remove_all(workdir, recursive=True)
        except:
            logging.error(Ta('Error removing workdir (%s)'), workdir)
            logging.info("Traceback: ", exc_info=True)

    return True
Exemplo n.º 16
0
def process_job(nzo):
    """ Process one job """
    assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
    start = time.time()

    # keep track of whether we can continue
    all_ok = True
    # keep track of par problems
    par_error = False
    # keep track of any unpacking errors
    unpack_error = False
    # Signal empty download, for when 'empty_postproc' is enabled
    empty = False
    nzb_list = []
    # These need to be initialised incase of a crash
    workdir_complete = ''
    postproc_time = 0
    script_log = ''
    script_line = ''
    crash_msg = ''

    ## Get the job flags
    nzo.save_attribs()
    flag_repair, flag_unpack, flag_delete = nzo.repair_opts
    # Normalize PP
    if flag_delete: flag_unpack = True
    if flag_unpack: flag_repair = True

    # Get the NZB name
    filename = nzo.final_name
    msgid = nzo.msgid

    if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete):
        # After streaming, force +D
        nzo.set_pp(3)
        nzo.status = Status.FAILED
        nzo.save_attribs()
        all_ok = False

    if nzo.fail_msg: # Special case: aborted due to too many missing data
        nzo.status = Status.FAILED
        nzo.save_attribs()
        all_ok = False
        par_error = unpack_error = True

    try:

        # Get the folder containing the download result
        workdir = nzo.downpath
        tmp_workdir_complete = None

        # if no files are present (except __admin__), fail the job
        if all_ok and len(globber(workdir)) < 2:
            if nzo.precheck:
                enough, ratio = nzo.check_quality()
                req_ratio = float(cfg.req_completion_rate()) / 100.0
                # Make sure that rounded ratio doesn't equal required ratio
                # when it is actually below required
                if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
                    ratio = req_ratio - 0.001
                emsg = '%.1f%%' % (ratio * 100.0)
                emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
                emsg = T('Download might fail, only %s of required %s available') % (emsg, emsg2)
            else:
                emsg = T('Download failed - Out of your server\'s retention?')
                empty = True
            nzo.fail_msg = emsg
            nzo.set_unpack_info('Fail', emsg)
            nzo.status = Status.FAILED
            # do not run unpacking or parity verification
            flag_repair = flag_unpack = False
            all_ok = cfg.empty_postproc() and empty
            if not all_ok:
                par_error = unpack_error = True

        script = nzo.script
        cat = nzo.cat

        logging.info('Starting PostProcessing on %s' + \
                     ' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
                     filename, flag_repair, flag_unpack, flag_delete, script, cat)

        ## Par processing, if enabled
        if all_ok and flag_repair:
            par_error, re_add = parring(nzo, workdir)
            if re_add:
                # Try to get more par files
                return False

        ## Check if user allows unsafe post-processing
        if flag_repair and cfg.safe_postproc():
            all_ok = all_ok and not par_error

        # Set complete dir to workdir in case we need to abort
        workdir_complete = workdir
        dirname = nzo.final_name
        marker_file = None

        if all_ok:
            one_folder = False
            ## Determine class directory
            if cfg.create_group_folders():
                complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix)
                complete_dir = create_dirs(complete_dir)
            else:
                catdir = config.get_categories(cat).dir()
                if catdir.endswith('*'):
                    catdir = catdir.strip('*')
                    one_folder = True
                complete_dir = real_path(cfg.complete_dir.get_path(), catdir)

            ## TV/Movie/Date Renaming code part 1 - detect and construct paths
            file_sorter = Sorter(cat)
            complete_dir = file_sorter.detect(dirname, complete_dir)
            if file_sorter.sort_file:
                one_folder = False

            if one_folder:
                workdir_complete = create_dirs(complete_dir)
            else:
                workdir_complete = get_unique_path(os.path.join(complete_dir, dirname), create_dir=True)
                marker_file = set_marker(workdir_complete)

            if not workdir_complete or not os.path.exists(workdir_complete):
                crash_msg = T('Cannot create final folder %s') % unicoder(os.path.join(complete_dir, dirname))
                raise IOError

            if cfg.folder_rename() and not one_folder:
                tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_')
                try:
                    renamer(workdir_complete, tmp_workdir_complete)
                except:
                    pass # On failure, just use the original name
            else:
                tmp_workdir_complete = workdir_complete

            newfiles = []
            ## Run Stage 2: Unpack
            if flag_unpack:
                if all_ok:
                    #set the current nzo status to "Extracting...". Used in History
                    nzo.status = Status.EXTRACTING
                    logging.info("Running unpack_magic on %s", filename)
                    unpack_error, newfiles = unpack_magic(nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), ())
                    logging.info("unpack_magic finished on %s", filename)
                else:
                    nzo.set_unpack_info('Unpack', T('No post-processing because of failed verification'))

            if cfg.safe_postproc():
                all_ok = all_ok and not unpack_error

            if all_ok:
                ## Move any (left-over) files to destination
                nzo.status = Status.MOVING
                nzo.set_action_line(T('Moving'), '...')
                for root, dirs, files in os.walk(workdir):
                    if not root.endswith(JOB_ADMIN):
                        for file_ in files:
                            path = os.path.join(root, file_)
                            new_path = path.replace(workdir, tmp_workdir_complete)
                            ok, new_path = move_to_path(path, new_path)
                            newfiles.append(new_path)
                            if not ok:
                                nzo.set_unpack_info('Unpack', T('Failed moving %s to %s') % (unicoder(path), unicoder(new_path)))
                                all_ok = False
                                break

            ## Set permissions right
            set_permissions(tmp_workdir_complete)

            if all_ok and marker_file:
                del_marker(os.path.join(tmp_workdir_complete, marker_file))
                remove_from_list(marker_file, newfiles)

            if all_ok:
                ## Remove files matching the cleanup list
                cleanup_list(tmp_workdir_complete, True)

                ## Check if this is an NZB-only download, if so redirect to queue
                ## except when PP was Download-only
                if flag_repair:
                    nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority)
                else:
                    nzb_list = None
                if nzb_list:
                    nzo.set_unpack_info('Download', T('Sent %s to queue') % unicoder(nzb_list))
                    cleanup_empty_directories(tmp_workdir_complete)
                else:
                    cleanup_list(tmp_workdir_complete, False)

        script_output = ''
        script_ret = 0
        if not nzb_list:
            ## Give destination its final name
            if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
                if all_ok:
                    try:
                        newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles)
                    except:
                        logging.error(Ta('Error renaming "%s" to "%s"'), tmp_workdir_complete, workdir_complete)
                        logging.info('Traceback: ', exc_info = True)
                        # Better disable sorting because filenames are all off now
                        file_sorter.sort_file = None
                else:
                    workdir_complete = tmp_workdir_complete.replace('_UNPACK_', '_FAILED_')
                    workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False)

            if empty:
                job_result = -1
            else:
                job_result = int(par_error) + int(unpack_error)*2

            if cfg.ignore_samples() > 0:
                remove_samples(workdir_complete)

            ## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
            if all_ok and file_sorter.sort_file:
                if newfiles:
                    file_sorter.rename(newfiles, workdir_complete)
                    workdir_complete, ok = file_sorter.move(workdir_complete)
                else:
                    workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete)
                if not ok:
                    nzo.set_unpack_info('Unpack', T('Failed to move files'))
                    all_ok = False

            ## Run the user script
            script_path = make_script_path(script)
            if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path:
                #set the current nzo status to "Ext Script...". Used in History
                nzo.status = Status.RUNNING
                nzo.set_action_line(T('Running script'), unicoder(script))
                nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True)
                script_log, script_ret = external_processing(script_path, workdir_complete, nzo.filename,
                                                             msgid, dirname, cat, nzo.group, job_result)
                script_line = get_last_line(script_log)
                if script_log:
                    script_output = nzo.nzo_id
                if script_line:
                    nzo.set_unpack_info('Script', unicoder(script_line), unique=True)
                else:
                    nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True)
            else:
                script = ""
                script_line = ""
                script_ret = 0

        ## Email the results
        if (not nzb_list) and cfg.email_endjob():
            if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error)):
                emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded,
                               nzo.fail_msg, nzo.unpack_info, script, TRANS(script_log), script_ret)

        if script_output:
            # Can do this only now, otherwise it would show up in the email
            if script_ret:
                script_ret = 'Exit(%s) ' % script_ret
            else:
                script_ret = ''
            if script_line:
                nzo.set_unpack_info('Script',
                                    u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, unicoder(script_line), urllib.quote(script_output),
                                     T('More')), unique=True)
            else:
                nzo.set_unpack_info('Script',
                                    u'%s<a href="./scriptlog?name=%s">%s</a>' % (script_ret, urllib.quote(script_output),
                                    T('View script output')), unique=True)

        ## Cleanup again, including NZB files
        if all_ok:
            cleanup_list(workdir_complete, False)

        ## Remove newzbin bookmark, if any
        if msgid and all_ok:
            Bookmarks.do.del_bookmark(msgid)
        elif all_ok and isinstance(nzo.url, str):
            sabnzbd.proxy_rm_bookmark(nzo.url)

        ## Force error for empty result
        all_ok = all_ok and not empty

        ## Show final status in history
        if all_ok:
            growler.send_notification(T('Download Completed'), filename, 'complete')
            nzo.status = Status.COMPLETED
        else:
            growler.send_notification(T('Download Failed'), filename, 'complete')
            nzo.status = Status.FAILED

    except:
        logging.error(Ta('Post Processing Failed for %s (%s)'), filename, crash_msg)
        if not crash_msg:
            logging.info("Traceback: ", exc_info = True)
            crash_msg = T('see logfile')
        nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(crash_msg)
        growler.send_notification(T('Download Failed'), filename, 'complete')
        nzo.status = Status.FAILED
        par_error = True
        all_ok = False
        if cfg.email_endjob():
            emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded,
                           nzo.fail_msg, nzo.unpack_info, '', '', 0)


    if all_ok:
        # If the folder only contains one file OR folder, have that as the path
        # Be aware that series/generic/date sorting may move a single file into a folder containing other files
        workdir_complete = one_file_or_folder(workdir_complete)
        workdir_complete = os.path.normpath(workdir_complete)

    # Log the overall time taken for postprocessing
    postproc_time = int(time.time() - start)

    # Create the history DB instance
    history_db = database.get_history_handle()
    # Add the nzo to the database. Only the path, script and time taken is passed
    # Other information is obtained from the nzo
    history_db.add_history_db(nzo, workdir_complete, nzo.downpath, postproc_time, script_log, script_line)
    # The connection is only used once, so close it here
    history_db.close()

    ## Clean up the NZO
    try:
        logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok))
        sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
    except:
        logging.error(Ta('Cleanup of %s failed.'), nzo.final_name)
        logging.info("Traceback: ", exc_info = True)

    ## Remove download folder
    if all_ok:
        try:
            if os.path.exists(workdir):
                logging.debug('Removing workdir %s', workdir)
                remove_all(workdir, recursive=True)
        except:
            logging.error(Ta('Error removing workdir (%s)'), workdir)
            logging.info("Traceback: ", exc_info = True)

    return True
Exemplo n.º 17
0
def process_job(nzo):
    """ Process one job """
    assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
    start = time.time()

    # keep track of whether we can continue
    all_ok = True
    # keep track of par problems
    par_error = False
    # keep track of any unpacking errors
    unpack_error = False
    # Signal empty download, for when 'empty_postproc' is enabled
    empty = False
    nzb_list = []
    # These need to be initialised incase of a crash
    workdir_complete = ''
    postproc_time = 0
    script_log = ''
    script_line = ''
    crash_msg = ''

    ## Get the job flags
    nzo.save_attribs()
    flag_repair, flag_unpack, flag_delete = nzo.repair_opts
    # Normalize PP
    if flag_delete: flag_unpack = True
    if flag_unpack: flag_repair = True

    # Get the NZB name
    filename = nzo.final_name
    msgid = nzo.msgid

    if cfg.allow_streaming() and not (flag_repair or flag_unpack
                                      or flag_delete):
        # After streaming, force +D
        nzo.set_pp(3)
        nzo.status = Status.FAILED
        nzo.save_attribs()
        all_ok = False

    if nzo.fail_msg:  # Special case: aborted due to too many missing data
        nzo.status = Status.FAILED
        nzo.save_attribs()
        all_ok = False
        par_error = unpack_error = True

    try:

        # Get the folder containing the download result
        workdir = nzo.downpath
        tmp_workdir_complete = None

        # if no files are present (except __admin__), fail the job
        if all_ok and len(globber(workdir)) < 2:
            if nzo.precheck:
                enough, ratio = nzo.check_quality()
                req_ratio = float(cfg.req_completion_rate()) / 100.0
                # Make sure that rounded ratio doesn't equal required ratio
                # when it is actually below required
                if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
                    ratio = req_ratio - 0.001
                emsg = '%.1f%%' % (ratio * 100.0)
                emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
                emsg = T(
                    'Download might fail, only %s of required %s available'
                ) % (emsg, emsg2)
            else:
                emsg = T('Download failed - Out of your server\'s retention?')
                empty = True
            nzo.fail_msg = emsg
            nzo.set_unpack_info('Fail', emsg)
            nzo.status = Status.FAILED
            # do not run unpacking or parity verification
            flag_repair = flag_unpack = False
            all_ok = cfg.empty_postproc() and empty
            if not all_ok:
                par_error = unpack_error = True

        script = nzo.script
        cat = nzo.cat

        logging.info('Starting PostProcessing on %s' + \
                     ' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
                     filename, flag_repair, flag_unpack, flag_delete, script, cat)

        ## Par processing, if enabled
        if all_ok and flag_repair:
            par_error, re_add = parring(nzo, workdir)
            if re_add:
                # Try to get more par files
                return False

        ## Check if user allows unsafe post-processing
        if flag_repair and cfg.safe_postproc():
            all_ok = all_ok and not par_error

        # Set complete dir to workdir in case we need to abort
        workdir_complete = workdir
        dirname = nzo.final_name
        marker_file = None

        if all_ok:
            one_folder = False
            ## Determine class directory
            if cfg.create_group_folders():
                complete_dir = addPrefixes(cfg.complete_dir.get_path(),
                                           nzo.dirprefix)
                complete_dir = create_dirs(complete_dir)
            else:
                catdir = config.get_categories(cat).dir()
                if catdir.endswith('*'):
                    catdir = catdir.strip('*')
                    one_folder = True
                complete_dir = real_path(cfg.complete_dir.get_path(), catdir)

            ## TV/Movie/Date Renaming code part 1 - detect and construct paths
            if cfg.enable_meta():
                file_sorter = Sorter(nzo, cat)
            else:
                file_sorter = Sorter(None, cat)
            complete_dir = file_sorter.detect(dirname, complete_dir)
            if file_sorter.sort_file:
                one_folder = False

            if one_folder:
                workdir_complete = create_dirs(complete_dir)
            else:
                workdir_complete = get_unique_path(os.path.join(
                    complete_dir, dirname),
                                                   create_dir=True)
                marker_file = set_marker(workdir_complete)

            if not workdir_complete or not os.path.exists(workdir_complete):
                crash_msg = T('Cannot create final folder %s') % unicoder(
                    os.path.join(complete_dir, dirname))
                raise IOError

            if cfg.folder_rename() and not one_folder:
                tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_')
                try:
                    renamer(workdir_complete, tmp_workdir_complete)
                except:
                    pass  # On failure, just use the original name
            else:
                tmp_workdir_complete = workdir_complete

            newfiles = []
            ## Run Stage 2: Unpack
            if flag_unpack:
                if all_ok:
                    #set the current nzo status to "Extracting...". Used in History
                    nzo.status = Status.EXTRACTING
                    logging.info("Running unpack_magic on %s", filename)
                    unpack_error, newfiles = unpack_magic(
                        nzo, workdir, tmp_workdir_complete, flag_delete,
                        one_folder, (), (), (), ())
                    logging.info("unpack_magic finished on %s", filename)
                else:
                    nzo.set_unpack_info(
                        'Unpack',
                        T('No post-processing because of failed verification'))

            if cfg.safe_postproc():
                all_ok = all_ok and not unpack_error

            if all_ok:
                ## Move any (left-over) files to destination
                nzo.status = Status.MOVING
                nzo.set_action_line(T('Moving'), '...')
                for root, dirs, files in os.walk(workdir):
                    if not root.endswith(JOB_ADMIN):
                        for file_ in files:
                            path = os.path.join(root, file_)
                            new_path = path.replace(workdir,
                                                    tmp_workdir_complete)
                            ok, new_path = move_to_path(path, new_path)
                            newfiles.append(new_path)
                            if not ok:
                                nzo.set_unpack_info(
                                    'Unpack',
                                    T('Failed moving %s to %s') %
                                    (unicoder(path), unicoder(new_path)))
                                all_ok = False
                                break

            ## Set permissions right
            set_permissions(tmp_workdir_complete)

            if all_ok and marker_file:
                del_marker(os.path.join(tmp_workdir_complete, marker_file))
                remove_from_list(marker_file, newfiles)

            if all_ok:
                ## Remove files matching the cleanup list
                cleanup_list(tmp_workdir_complete, True)

                ## Check if this is an NZB-only download, if so redirect to queue
                ## except when PP was Download-only
                if flag_repair:
                    nzb_list = nzb_redirect(tmp_workdir_complete,
                                            nzo.final_name,
                                            nzo.pp,
                                            script,
                                            cat,
                                            priority=nzo.priority)
                else:
                    nzb_list = None
                if nzb_list:
                    nzo.set_unpack_info(
                        'Download',
                        T('Sent %s to queue') % unicoder(nzb_list))
                    cleanup_empty_directories(tmp_workdir_complete)
                else:
                    cleanup_list(tmp_workdir_complete, False)

        script_output = ''
        script_ret = 0
        if not nzb_list:
            ## Give destination its final name
            if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
                if all_ok:
                    try:
                        newfiles = rename_and_collapse_folder(
                            tmp_workdir_complete, workdir_complete, newfiles)
                    except:
                        logging.error(Ta('Error renaming "%s" to "%s"'),
                                      tmp_workdir_complete, workdir_complete)
                        logging.info('Traceback: ', exc_info=True)
                        # Better disable sorting because filenames are all off now
                        file_sorter.sort_file = None
                else:
                    workdir_complete = tmp_workdir_complete.replace(
                        '_UNPACK_', '_FAILED_')
                    workdir_complete = get_unique_path(workdir_complete,
                                                       n=0,
                                                       create_dir=False)

            if empty:
                job_result = -1
            else:
                job_result = int(par_error) + int(unpack_error) * 2

            if cfg.ignore_samples() > 0:
                remove_samples(workdir_complete)

            ## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
            if all_ok and file_sorter.sort_file:
                if newfiles:
                    file_sorter.rename(newfiles, workdir_complete)
                    workdir_complete, ok = file_sorter.move(workdir_complete)
                else:
                    workdir_complete, ok = file_sorter.rename_with_ext(
                        workdir_complete)
                if not ok:
                    nzo.set_unpack_info('Unpack', T('Failed to move files'))
                    all_ok = False

            ## Run the user script
            script_path = make_script_path(script)
            if (all_ok or not cfg.safe_postproc()) and (
                    not nzb_list) and script_path:
                #set the current nzo status to "Ext Script...". Used in History
                nzo.status = Status.RUNNING
                nzo.set_action_line(T('Running script'), unicoder(script))
                nzo.set_unpack_info('Script',
                                    T('Running user script %s') %
                                    unicoder(script),
                                    unique=True)
                script_log, script_ret = external_processing(
                    script_path, workdir_complete, nzo.filename, msgid,
                    dirname, cat, nzo.group, job_result,
                    nzo.nzo_info.get('failure', ''))
                script_line = get_last_line(script_log)
                if script_log:
                    script_output = nzo.nzo_id
                if script_line:
                    nzo.set_unpack_info('Script',
                                        unicoder(script_line),
                                        unique=True)
                else:
                    nzo.set_unpack_info('Script',
                                        T('Ran %s') % unicoder(script),
                                        unique=True)
            else:
                script = ""
                script_line = ""
                script_ret = 0

        ## Email the results
        if (not nzb_list) and cfg.email_endjob():
            if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and
                                             (unpack_error or par_error)):
                emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete,
                               nzo.bytes_downloaded,
                               nzo.fail_msg, nzo.unpack_info, script,
                               TRANS(script_log), script_ret)

        if script_output:
            # Can do this only now, otherwise it would show up in the email
            if script_ret:
                script_ret = 'Exit(%s) ' % script_ret
            else:
                script_ret = ''
            if script_line:
                nzo.set_unpack_info(
                    'Script',
                    u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' %
                    (script_ret, unicoder(script_line),
                     urllib.quote(script_output), T('More')),
                    unique=True)
            else:
                nzo.set_unpack_info('Script',
                                    u'%s<a href="./scriptlog?name=%s">%s</a>' %
                                    (script_ret, urllib.quote(script_output),
                                     T('View script output')),
                                    unique=True)

        ## Cleanup again, including NZB files
        if all_ok:
            cleanup_list(workdir_complete, False)

        ## Remove newzbin bookmark, if any
        if msgid and all_ok:
            Bookmarks.do.del_bookmark(msgid)
        elif all_ok and isinstance(nzo.url, str):
            sabnzbd.proxy_rm_bookmark(nzo.url)

        ## Force error for empty result
        all_ok = all_ok and not empty

        ## Update indexer with results
        if nzo.encrypted > 0:
            Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
        if empty:
            hosts = map(lambda s: s.host,
                        sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
            if not hosts: hosts = [None]
            for host in hosts:
                Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED,
                                           host)

        ## Show final status in history
        if all_ok:
            growler.send_notification(T('Download Completed'), filename,
                                      'complete')
            nzo.status = Status.COMPLETED
        else:
            growler.send_notification(T('Download Failed'), filename,
                                      'complete')
            nzo.status = Status.FAILED

    except:
        logging.error(Ta('Post Processing Failed for %s (%s)'), filename,
                      crash_msg)
        if not crash_msg:
            logging.info("Traceback: ", exc_info=True)
            crash_msg = T('see logfile')
        nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(
            crash_msg)
        growler.send_notification(T('Download Failed'), filename, 'complete')
        nzo.status = Status.FAILED
        par_error = True
        all_ok = False
        if cfg.email_endjob():
            emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete,
                           nzo.bytes_downloaded, nzo.fail_msg, nzo.unpack_info,
                           '', '', 0)

    if all_ok:
        # If the folder only contains one file OR folder, have that as the path
        # Be aware that series/generic/date sorting may move a single file into a folder containing other files
        workdir_complete = one_file_or_folder(workdir_complete)
        workdir_complete = os.path.normpath(workdir_complete)

    # Log the overall time taken for postprocessing
    postproc_time = int(time.time() - start)

    # Create the history DB instance
    history_db = database.get_history_handle()
    # Add the nzo to the database. Only the path, script and time taken is passed
    # Other information is obtained from the nzo
    history_db.add_history_db(nzo, workdir_complete, nzo.downpath,
                              postproc_time, script_log, script_line)
    # The connection is only used once, so close it here
    history_db.close()

    ## Clean up the NZO
    try:
        logging.info('Cleaning up %s (keep_basic=%s)', filename,
                     str(not all_ok))
        sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
    except:
        logging.error(Ta('Cleanup of %s failed.'), nzo.final_name)
        logging.info("Traceback: ", exc_info=True)

    ## Remove download folder
    if all_ok:
        try:
            if os.path.exists(workdir):
                logging.debug('Removing workdir %s', workdir)
                remove_all(workdir, recursive=True)
        except:
            logging.error(Ta('Error removing workdir (%s)'), workdir)
            logging.info("Traceback: ", exc_info=True)

    return True
Exemplo n.º 18
0
def process_job(nzo):
    """ Process one job """
    assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
    start = time.time()

    # keep track of whether we can continue
    all_ok = True
    # keep track of par problems
    par_error = False
    # keep track of any unpacking errors
    unpack_error = False
    # Signal empty download, for when 'empty_postproc' is enabled
    empty = False
    nzb_list = []
    # These need to be initialized in case of a crash
    workdir_complete = ""
    postproc_time = 0  # @UnusedVariable -- pep8 bug?
    script_log = ""
    script_line = ""
    crash_msg = ""

    # Get the job flags
    nzo.save_attribs()
    flag_repair, flag_unpack, flag_delete = nzo.repair_opts
    # Normalize PP
    if flag_delete:
        flag_unpack = True
    if flag_unpack:
        flag_repair = True

    # Get the NZB name
    filename = nzo.final_name

    if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete):
        # After streaming, force +D
        nzo.set_pp(3)
        nzo.status = Status.FAILED
        nzo.save_attribs()
        all_ok = False

    if nzo.fail_msg:  # Special case: aborted due to too many missing data
        nzo.status = Status.FAILED
        nzo.save_attribs()
        all_ok = False
        par_error = True
        unpack_error = 1

    try:

        # Get the folder containing the download result
        workdir = nzo.downpath
        tmp_workdir_complete = None

        # if no files are present (except __admin__), fail the job
        if all_ok and len(globber(workdir)) < 2:
            if nzo.precheck:
                _enough, ratio = nzo.check_quality()
                req_ratio = float(cfg.req_completion_rate()) / 100.0
                # Make sure that rounded ratio doesn't equal required ratio
                # when it is actually below required
                if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
                    ratio = req_ratio - 0.001
                emsg = "%.1f%%" % (ratio * 100.0)
                emsg2 = "%.1f%%" % float(cfg.req_completion_rate())
                emsg = T("Download might fail, only %s of required %s available") % (emsg, emsg2)
            else:
                emsg = T("Download failed - Not on your server(s)")
                empty = True
            nzo.fail_msg = emsg
            nzo.set_unpack_info("Fail", emsg)
            nzo.status = Status.FAILED
            # do not run unpacking or parity verification
            flag_repair = flag_unpack = False
            all_ok = cfg.empty_postproc() and empty
            if not all_ok:
                par_error = True
                unpack_error = 1

        script = nzo.script
        cat = nzo.cat

        logging.info(
            "Starting PostProcessing on %s" + " => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s",
            filename,
            flag_repair,
            flag_unpack,
            flag_delete,
            script,
            cat,
        )

        # Set complete dir to workdir in case we need to abort
        workdir_complete = workdir
        dirname = nzo.final_name
        marker_file = None

        # Par processing, if enabled
        if all_ok and flag_repair:
            if not check_win_maxpath(workdir):
                crash_msg = T('Path exceeds 260, repair by "par2" is not possible')
                raise WindowsError
            par_error, re_add = parring(nzo, workdir)
            if re_add:
                # Try to get more par files
                return False

        # Check if user allows unsafe post-processing
        if flag_repair and cfg.safe_postproc():
            all_ok = all_ok and not par_error

        if all_ok:
            fix_unix_encoding(workdir)
            one_folder = False
            # Determine class directory
            if cfg.create_group_folders():
                complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix)
                complete_dir = create_dirs(complete_dir)
            else:
                catdir = config.get_categories(cat).dir()
                if catdir.endswith("*"):
                    catdir = catdir.strip("*")
                    one_folder = True
                complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
            complete_dir = long_path(complete_dir)

            # TV/Movie/Date Renaming code part 1 - detect and construct paths
            if cfg.enable_meta():
                file_sorter = Sorter(nzo, cat)
            else:
                file_sorter = Sorter(None, cat)
            complete_dir = file_sorter.detect(dirname, complete_dir)
            if file_sorter.sort_file:
                one_folder = False

            complete_dir = sanitize_and_trim_path(complete_dir)

            if one_folder:
                workdir_complete = create_dirs(complete_dir)
            else:
                workdir_complete = get_unique_path(os.path.join(complete_dir, dirname), create_dir=True)
                marker_file = set_marker(workdir_complete)

            if not workdir_complete or not os.path.exists(workdir_complete):
                crash_msg = T("Cannot create final folder %s") % unicoder(os.path.join(complete_dir, dirname))
                raise IOError

            if cfg.folder_rename() and not one_folder:
                tmp_workdir_complete = prefix(workdir_complete, "_UNPACK_")
                try:
                    renamer(workdir_complete, tmp_workdir_complete)
                except:
                    pass  # On failure, just use the original name
            else:
                tmp_workdir_complete = workdir_complete

            newfiles = []
            # Run Stage 2: Unpack
            if flag_unpack:
                if all_ok:
                    # set the current nzo status to "Extracting...". Used in History
                    nzo.status = Status.EXTRACTING
                    logging.info("Running unpack_magic on %s", filename)
                    short_complete = short_path(tmp_workdir_complete)
                    unpack_error, newfiles = unpack_magic(
                        nzo, short_path(workdir), short_complete, flag_delete, one_folder, (), (), (), (), ()
                    )
                    if short_complete != tmp_workdir_complete:
                        newfiles = [f.replace(short_complete, tmp_workdir_complete) for f in newfiles]
                    logging.info("unpack_magic finished on %s", filename)
                else:
                    nzo.set_unpack_info("Unpack", T("No post-processing because of failed verification"))

            if cfg.safe_postproc():
                all_ok = all_ok and not unpack_error

            if all_ok:
                # Move any (left-over) files to destination
                nzo.status = Status.MOVING
                nzo.set_action_line(T("Moving"), "...")
                for root, _dirs, files in os.walk(workdir):
                    if not root.endswith(JOB_ADMIN):
                        for file_ in files:
                            path = os.path.join(root, file_)
                            new_path = path.replace(workdir, tmp_workdir_complete)
                            ok, new_path = move_to_path(path, new_path)
                            newfiles.append(new_path)
                            if not ok:
                                nzo.set_unpack_info(
                                    "Unpack", T("Failed moving %s to %s") % (unicoder(path), unicoder(new_path))
                                )
                                all_ok = False
                                break

            # Set permissions right
            set_permissions(tmp_workdir_complete)

            if all_ok and marker_file:
                del_marker(os.path.join(tmp_workdir_complete, marker_file))
                remove_from_list(marker_file, newfiles)

            if all_ok:
                # Remove files matching the cleanup list
                cleanup_list(tmp_workdir_complete, True)

                # Check if this is an NZB-only download, if so redirect to queue
                # except when PP was Download-only
                if flag_repair:
                    nzb_list = nzb_redirect(
                        tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority
                    )
                else:
                    nzb_list = None
                if nzb_list:
                    nzo.set_unpack_info("Download", T("Sent %s to queue") % unicoder(nzb_list))
                    cleanup_empty_directories(tmp_workdir_complete)
                else:
                    cleanup_list(tmp_workdir_complete, False)

        script_output = ""
        script_ret = 0
        if not nzb_list:
            # Give destination its final name
            if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
                if all_ok:
                    try:
                        newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles)
                    except:
                        logging.error(
                            T('Error renaming "%s" to "%s"'),
                            clip_path(tmp_workdir_complete),
                            clip_path(workdir_complete),
                        )
                        logging.info("Traceback: ", exc_info=True)
                        # Better disable sorting because filenames are all off now
                        file_sorter.sort_file = None
                else:
                    workdir_complete = tmp_workdir_complete.replace("_UNPACK_", "_FAILED_")
                    workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False)
                    workdir_complete = workdir_complete

            if empty:
                job_result = -1
            else:
                job_result = int(par_error) + int(bool(unpack_error)) * 2

            if cfg.ignore_samples():
                remove_samples(workdir_complete)

            # TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
            if all_ok and file_sorter.sort_file:
                if newfiles:
                    file_sorter.rename(newfiles, workdir_complete)
                    workdir_complete, ok = file_sorter.move(workdir_complete)
                else:
                    workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete)
                if not ok:
                    nzo.set_unpack_info("Unpack", T("Failed to move files"))
                    all_ok = False

            # Run the user script
            script_path = make_script_path(script)
            if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path:
                # set the current nzo status to "Ext Script...". Used in History
                nzo.status = Status.RUNNING
                nzo.set_action_line(T("Running script"), unicoder(script))
                nzo.set_unpack_info("Script", T("Running user script %s") % unicoder(script), unique=True)
                script_log, script_ret = external_processing(
                    short_path(script_path, False),
                    short_path(workdir_complete, False),
                    nzo.filename,
                    dirname,
                    cat,
                    nzo.group,
                    job_result,
                    nzo.nzo_info.get("failure", ""),
                )
                script_line = get_last_line(script_log)
                if script_log:
                    script_output = nzo.nzo_id
                if script_line:
                    nzo.set_unpack_info("Script", unicoder(script_line), unique=True)
                else:
                    nzo.set_unpack_info("Script", T("Ran %s") % unicoder(script), unique=True)
            else:
                script = ""
                script_line = ""
                script_ret = 0

        # Maybe bad script result should fail job
        if script_ret and cfg.script_can_fail():
            script_error = True
            all_ok = False
            nzo.fail_msg = T("Script exit code is %s") % script_ret
        else:
            script_error = False

        # Email the results
        if (not nzb_list) and cfg.email_endjob():
            if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error or script_error)):
                emailer.endjob(
                    dirname,
                    cat,
                    all_ok,
                    workdir_complete,
                    nzo.bytes_downloaded,
                    nzo.fail_msg,
                    nzo.unpack_info,
                    script,
                    TRANS(script_log),
                    script_ret,
                )

        if script_output:
            # Can do this only now, otherwise it would show up in the email
            if script_ret:
                script_ret = "Exit(%s) " % script_ret
            else:
                script_ret = ""
            if script_line:
                nzo.set_unpack_info(
                    "Script",
                    u'%s%s <a href="./scriptlog?name=%s">(%s)</a>'
                    % (script_ret, unicoder(script_line), urllib.quote(script_output), T("More")),
                    unique=True,
                )
            else:
                nzo.set_unpack_info(
                    "Script",
                    u'%s<a href="./scriptlog?name=%s">%s</a>'
                    % (script_ret, urllib.quote(script_output), T("View script output")),
                    unique=True,
                )

        # Cleanup again, including NZB files
        if all_ok:
            cleanup_list(workdir_complete, False)

        # Force error for empty result
        all_ok = all_ok and not empty

        # Update indexer with results
        if cfg.rating_enable():
            if nzo.encrypted > 0:
                Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
            if empty:
                hosts = map(lambda s: s.host, sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
                if not hosts:
                    hosts = [None]
                for host in hosts:
                    Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED, host)

        # Show final status in history
        if all_ok:
            growler.send_notification(T("Download Completed"), filename, "complete")
            nzo.status = Status.COMPLETED
        else:
            growler.send_notification(T("Download Failed"), filename, "failed")
            nzo.status = Status.FAILED

    except:
        logging.error(T("Post Processing Failed for %s (%s)"), filename, crash_msg)
        if not crash_msg:
            logging.info("Traceback: ", exc_info=True)
            crash_msg = T("see logfile")
        nzo.fail_msg = T("PostProcessing was aborted (%s)") % unicoder(crash_msg)
        growler.send_notification(T("Download Failed"), filename, "failed")
        nzo.status = Status.FAILED
        par_error = True
        all_ok = False
        if cfg.email_endjob():
            emailer.endjob(
                dirname,
                cat,
                all_ok,
                clip_path(workdir_complete),
                nzo.bytes_downloaded,
                nzo.fail_msg,
                nzo.unpack_info,
                "",
                "",
                0,
            )

    if all_ok:
        # If the folder only contains one file OR folder, have that as the path
        # Be aware that series/generic/date sorting may move a single file into a folder containing other files
        workdir_complete = one_file_or_folder(workdir_complete)
        workdir_complete = os.path.normpath(workdir_complete)

    # Log the overall time taken for postprocessing
    postproc_time = int(time.time() - start)

    # Create the history DB instance
    history_db = database.get_history_handle()
    # Add the nzo to the database. Only the path, script and time taken is passed
    # Other information is obtained from the nzo
    history_db.add_history_db(nzo, clip_path(workdir_complete), nzo.downpath, postproc_time, script_log, script_line)
    # The connection is only used once, so close it here
    history_db.close()

    # Clean up the NZO
    try:
        logging.info("Cleaning up %s (keep_basic=%s)", filename, str(not all_ok))
        sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
    except:
        logging.error(T("Cleanup of %s failed."), nzo.final_name)
        logging.info("Traceback: ", exc_info=True)

    # Remove download folder
    if all_ok:
        try:
            if os.path.exists(workdir):
                logging.debug("Removing workdir %s", workdir)
                remove_all(workdir, recursive=True)
        except:
            logging.error(T("Error removing workdir (%s)"), clip_path(workdir))
            logging.info("Traceback: ", exc_info=True)

    # Use automatic retry link on par2 errors and encrypted/bad RARs
    if par_error or unpack_error in (2, 3):
        try_alt_nzb(nzo)

    return True