Beispiel #1
0
def is_archive(path):
    """ Check if file in path is an ZIP, RAR or 7z file
    :param path: path to file
    :return: (zf, status, expected_extension)
            status: -1==Error/Retry, 0==OK, 1==Ignore
    """
    if zipfile.is_zipfile(path):
        try:
            zf = zipfile.ZipFile(path)
            return 0, zf, '.zip'
        except:
            logging.info(T('Cannot read %s'), path, exc_info=True)
            return -1, None, ''
    elif rarfile.is_rarfile(path):
        try:
            # Set path to tool to open it
            rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
            zf = rarfile.RarFile(path)
            return 0, zf, '.rar'
        except:
            logging.info(T('Cannot read %s'), path, exc_info=True)
            return -1, None, ''
    elif is_sevenfile(path):
        try:
            zf = SevenZip(path)
            return 0, zf, '.7z'
        except:
            logging.info(T('Cannot read %s'), path, exc_info=True)
            return -1, None, ''
    else:
        logging.info('Archive %s is not a real archive!',
                     os.path.basename(path))
        return 1, None, ''
Beispiel #2
0
def is_archive(path):
    """ Check if file in path is an ZIP, RAR or 7z file
    :param path: path to file
    :return: (zf, status, expected_extension)
            status: -1==Error/Retry, 0==OK, 1==Ignore
    """
    if zipfile.is_zipfile(path):
        try:
            zf = zipfile.ZipFile(path)
            return 0, zf, '.zip'
        except:
            return -1, None, ''
    elif rarfile.is_rarfile(path):
        try:
            zf = rarfile.RarFile(path)
            # Set path to tool to open it
            rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
            return 0, zf, '.rar'
        except:
            return -1, None, ''
    elif is_sevenfile(path):
        try:
            zf = SevenZip(path)
            return 0, zf, '.7z'
        except:
            return -1, None, ''
    else:
        return 1, None, ''
Beispiel #3
0
def is_archive(path: str) -> Tuple[int, Any, str]:
    """Check if file in path is an ZIP, RAR or 7z file
    :param path: path to file
    :return: (zf, status, expected_extension)
            status: -1==Error/Retry, 0==OK, 1==Ignore
    """
    if zipfile.is_zipfile(path):
        try:
            zf = zipfile.ZipFile(path)
            return 0, zf, ".zip"
        except:
            logging.info(T("Cannot read %s"), path, exc_info=True)
            return -1, None, ""
    elif rarfile.is_rarfile(path):
        try:
            # Set path to tool to open it
            rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
            zf = rarfile.RarFile(path)
            return 0, zf, ".rar"
        except:
            logging.info(T("Cannot read %s"), path, exc_info=True)
            return -1, None, ""
    elif sabnzbd.newsunpack.is_sevenfile(path):
        try:
            zf = sabnzbd.newsunpack.SevenZip(path)
            return 0, zf, ".7z"
        except:
            logging.info(T("Cannot read %s"), path, exc_info=True)
            return -1, None, ""
    else:
        logging.info("Archive %s is not a real archive!",
                     os.path.basename(path))
        return 1, None, ""
Beispiel #4
0
def is_archive(path):
    """ Check if file in path is an ZIP, RAR or 7z file
    :param path: path to file
    :return: (zf, status, expected_extension)
            status: -1==Error/Retry, 0==OK, 1==Ignore
    """
    if zipfile.is_zipfile(path):
        try:
            zf = zipfile.ZipFile(path)
            return 0, zf, '.zip'
        except:
            return -1, None, ''
    elif is_rarfile(path):
        try:
            zf = RarFile(path)
            return 0, zf, '.rar'
        except:
            return -1, None, ''
    elif is_sevenfile(path):
        try:
            zf = SevenZip(path)
            return 0, zf, '.7z'
        except:
            return -1, None, ''
    else:
        return 1, None, ''
Beispiel #5
0
def is_archive(path):
    """ Check if file in path is an ZIP, RAR or 7z file
    :param path: path to file
    :return: (zf, status, expected_extension)
            status: -1==Error/Retry, 0==OK, 1==Ignore
    """
    if zipfile.is_zipfile(path):
        try:
            zf = zipfile.ZipFile(path)
            return 0, zf, '.zip'
        except:
            return -1, None, ''
    elif is_rarfile(path):
        try:
            zf = RarFile(path)
            return 0, zf, '.rar'
        except:
            return -1, None, ''
    elif is_sevenfile(path):
        try:
            zf = SevenZip(path)
            return 0, zf, '.7z'
        except:
            return -1, None, ''
    else:
        return 1, None, ''
Beispiel #6
0
def is_archive(path):
    """ Check if file in path is an ZIP, RAR or 7z file
    :param path: path to file
    :return: (zf, status, expected_extension)
            status: -1==Error/Retry, 0==OK, 1==Ignore
    """
    if zipfile.is_zipfile(path):
        try:
            zf = zipfile.ZipFile(path)
            return 0, zf, '.zip'
        except:
            return -1, None, ''
    elif rarfile.is_rarfile(path):
        try:
            zf = rarfile.RarFile(path)
            # Set path to tool to open it
            rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
            return 0, zf, '.rar'
        except:
            return -1, None, ''
    elif is_sevenfile(path):
        try:
            zf = SevenZip(path)
            return 0, zf, '.7z'
        except:
            return -1, None, ''
    else:
        return 1, None, ''
Beispiel #7
0
def check_encrypted_rar(nzo, filepath):
    """ Check if file is rar and is encrypted """
    encrypted = False
    if not nzo.password and cfg.pause_on_pwrar() and is_rarfile(filepath):
        try:
            zf = RarFile(filepath)
            encrypted = zf.encrypted
            if encrypted and int(nzo.encrypted) < 2:
                nzo.encrypted = 1
            else:
                encrypted = False
            zf.close()
            del zf
        except:
            logging.debug('RAR file %s cannot be inspected', filepath)
    return encrypted
Beispiel #8
0
def check_encrypted_rar(nzo, filepath):
    """ Check if file is rar and is encrypted """
    encrypted = False
    if not nzo.password and cfg.pause_on_pwrar() and is_rarfile(filepath):
        try:
            zf = RarFile(filepath, all_names=True)
            encrypted = zf.encrypted or is_cloaked(filepath, zf.namelist())
            if encrypted and int(nzo.encrypted) < 2:
                nzo.encrypted = 1
            else:
                encrypted = False
            zf.close()
            del zf
        except:
            logging.debug('RAR file %s cannot be inspected', filepath)
    return encrypted
Beispiel #9
0
def check_encrypted_rar(nzo, filepath):
    """ Check if file is rar and is encrypted """
    encrypted = False
    if not nzo.password and not nzo.meta.get(
            'password') and cfg.pause_on_pwrar() and is_rarfile(filepath):
        try:
            zf = RarFile(filepath, all_names=True)
            encrypted = zf.encrypted or is_cloaked(filepath, zf.namelist())
            if encrypted and int(nzo.encrypted) < 2 and not nzo.reuse:
                nzo.encrypted = 1
            else:
                encrypted = False
            zf.close()
            del zf
        except:
            logging.debug('RAR file %s cannot be inspected', filepath)
    return encrypted
Beispiel #10
0
def rar_contains_unwanted_file(filepath):
    # checks for unwanted extensions in the rar file 'filepath'
    # ... unwanted extensions are defined in global variable cfg.unwanted_extensions()
    # returns False if no unwanted extensions are found in the rar file
    # returns name of file if unwanted extension is found in the rar file
    unwanted = None
    if is_rarfile(filepath):
        #logging.debug('rar file to check: %s',filepath)
        #logging.debug('unwanted extensions are: %s', cfg.unwanted_extensions())
        try:
            zf = RarFile(filepath, all_names=True)
            #logging.debug('files in rar file: %s', zf.namelist())
            for somefile in zf.namelist() :
                logging.debug('file in rar file: %s', somefile)
                if os.path.splitext(somefile)[1].replace('.', '').lower() in cfg.unwanted_extensions():
                    logging.debug('Unwanted file %s', somefile)
                    unwanted = somefile
                    zf.close()
        except:
            logging.debug('RAR file %s cannot be inspected.', filepath)
    return unwanted
Beispiel #11
0
def rar_contains_unwanted_file(filepath):
    # checks for unwanted extensions in the rar file 'filepath'
    # ... unwanted extensions are defined in global variable cfg.unwanted_extensions()
    # returns False if no unwanted extensions are found in the rar file
    # returns name of file if unwanted extension is found in the rar file
    unwanted = None
    if cfg.unwanted_extensions() and is_rarfile(filepath):
        # logging.debug('rar file to check: %s',filepath)
        # logging.debug('unwanted extensions are: %s', cfg.unwanted_extensions())
        try:
            zf = RarFile(filepath, all_names=True)
            # logging.debug('files in rar file: %s', zf.namelist())
            for somefile in zf.namelist():
                logging.debug("file in rar file: %s", somefile)
                if os.path.splitext(somefile)[1].replace(".", "").lower() in cfg.unwanted_extensions():
                    logging.debug("Unwanted file %s", somefile)
                    unwanted = somefile
                    zf.close()
        except:
            logging.debug("RAR file %s cannot be inspected.", filepath)
    return unwanted
Beispiel #12
0
def check_encrypted_rar(nzo, filepath):
    """ Check if file is rar and is encrypted """
    encrypted = False
    if (
        nzo.encrypted == 0
        and not nzo.password
        and not nzo.meta.get("password")
        and cfg.pause_on_pwrar()
        and is_rarfile(filepath)
    ):
        try:
            zf = RarFile(filepath, all_names=True)
            encrypted = zf.encrypted or is_cloaked(filepath, zf.namelist())
            if encrypted and not nzo.reuse:
                nzo.encrypted = 1
            else:
                # Don't check other files
                nzo.encrypted = -1
                encrypted = False
            zf.close()
            del zf
        except:
            logging.debug("RAR file %s cannot be inspected", filepath)
    return encrypted
Beispiel #13
0
def get_rar_extension(myrarfile):
    """
    Find out orginal extension of a rar file. Returns "" in case of file problems
    So ... returns:     "part001.rar", ... "part005.rar"
    or old number scheme (can only happen for rar3/rar4 files): "rar", r00, ... r89
    """
    # When things go wrong
    volumenumber = -1
    org_extension = False

    try:
        rar_ver = rarfile.is_rarfile(myrarfile)
        with open(myrarfile, "rb") as fh:
            if rar_ver.endswith("3"):
                # As it's rar3, let's first find the numbering scheme: old (rNN) or new (partNN.rar)
                mybuf = fh.read(100)  # first 100 bytes is enough
                HEAD_FLAGS_LSB = mybuf[10]  # LSB = Least Significant Byte
                newnumbering = HEAD_FLAGS_LSB & 0x10

                # For the volume number, At the end of the file, we need about 20 bytes
                fh.seek(-20, os.SEEK_END)
                mybuf = fh.read()
                volumenumber = 1 + mybuf[-9] + 256 * mybuf[-8]

                if newnumbering:
                    org_extension = "part%02d.rar" % volumenumber
                else:
                    # 1, 2, 3, 4 resp refers to .rar, .r00, .r01, .r02 ...
                    if volumenumber == 1:
                        org_extension = "rar"
                    else:
                        org_extension = "r%02d" % (volumenumber - 2)

            elif rar_ver.endswith("5"):
                mybuf = fh.read(100)  # first 100 bytes is enough

                # Get (and skip) the first 8 + 4 bytes
                rar5sig, newpos = rarfile.load_bytes(mybuf, 8,
                                                     0)  # Rar5 signature
                crc32, newpos = rarfile.load_bytes(mybuf, 4, newpos)  # crc32

                # Then get the VINT values (with variable size, so parse them all):
                headersize, newpos = rarfile.load_vint(mybuf, newpos)
                headertype, newpos = rarfile.load_vint(mybuf, newpos)
                headerflags, newpos = rarfile.load_vint(mybuf, newpos)
                extraareasize, newpos = rarfile.load_vint(mybuf, newpos)
                archiveflags, newpos = rarfile.load_vint(mybuf, newpos)

                # Now we're ready for the volume number:
                if archiveflags & 2:
                    value, newpos = rarfile.load_vint(mybuf, newpos)
                    volumenumber = value + 1
                else:
                    # first volume, aka 1
                    volumenumber = 1

                # Combine into the extension
                org_extension = "part%03d.rar" % volumenumber
    except:
        pass

    return volumenumber, org_extension
Beispiel #14
0
def check_encrypted_and_unwanted_files(nzo, filepath):
    """ Combines check for unwanted and encrypted files to save on CPU and IO """
    encrypted = False
    unwanted = None

    if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or (nzo.encrypted == 0 and cfg.pause_on_pwrar()):
        # These checks should not break the assembler
        try:
            # Rarfile freezes on Windows special names, so don't try those!
            if sabnzbd.WIN32 and has_win_device(filepath):
                return encrypted, unwanted

            # Is it even a rarfile?
            if rarfile.is_rarfile(filepath):
                # Open the rar
                rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
                zf = rarfile.RarFile(filepath, all_names=True)

                # Check for encryption
                if nzo.encrypted == 0 and cfg.pause_on_pwrar() and (zf.needs_password() or is_cloaked(nzo, filepath, zf.namelist())):
                    # Load all passwords
                    passwords = get_all_passwords(nzo)

                    # Cloaked job?
                    if is_cloaked(nzo, filepath, zf.namelist()):
                        encrypted = True
                    elif not sabnzbd.HAVE_CRYPTOGRAPHY and not passwords:
                        # if no cryptography installed, only error when no password was set
                        logging.info(T('%s missing'), 'Python Cryptography')
                        nzo.encrypted = 1
                        encrypted = True

                    elif sabnzbd.HAVE_CRYPTOGRAPHY:
                        # Lets test if any of the password work
                        password_hit = False

                        for password in passwords:
                            if password:
                                logging.info('Trying password "%s" on job "%s"', password, nzo.final_name)
                                try:
                                    zf.setpassword(password)
                                except:
                                    # On weird passwords the setpassword() will fail
                                    # but the actual rartest() will work
                                    pass
                                try:
                                    zf.testrar()
                                    password_hit = password
                                    break
                                except rarfile.RarCRCError:
                                    # On CRC error we can continue!
                                    password_hit = password
                                    break
                                except Exception as e:
                                    # Did we start from the right volume?
                                    if 'need to start extraction from a previous volume' in e[0]:
                                        return encrypted, unwanted
                                    # This one failed
                                    pass

                        # Did any work?
                        if password_hit:
                            # Don't check other files
                            logging.info('Password "%s" matches for job "%s"', password_hit, nzo.final_name)
                            nzo.encrypted = -1
                            encrypted = False
                        else:
                            # Encrypted and none of them worked
                            nzo.encrypted = 1
                            encrypted = True
                    else:
                        # Don't check other files
                        nzo.encrypted = -1
                        encrypted = False

                # Check for unwanted extensions
                if cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions():
                    for somefile in zf.namelist():
                        logging.debug('File contains: %s', somefile)
                        if os.path.splitext(somefile)[1].replace('.', '').lower() in cfg.unwanted_extensions():
                            logging.debug('Unwanted file %s', somefile)
                            unwanted = somefile
                zf.close()
                del zf
        except:
            logging.info('Error during inspection of RAR-file %s', filepath, exc_info=True)

    return encrypted, unwanted
Beispiel #15
0
def ProcessArchiveFile(filename, path, pp=None, script=None, cat=None, catdir=None, keep=False,
                       priority=None, url='', nzbname=None, password=None, nzo_id=None):
    """ Analyse ZIP file and create job(s).
        Accepts ZIP files with ONLY nzb/nfo/folder files in it.
        returns (status, nzo_ids)
            status: -1==Error/Retry, 0==OK, 1==Ignore
    """
    from sabnzbd.nzbqueue import add_nzo
    nzo_ids = []
    if catdir is None:
        catdir = cat

    filename, cat = name_to_cat(filename, catdir)

    if zipfile.is_zipfile(path):
        try:
            zf = zipfile.ZipFile(path)
        except:
            return -1, []
    elif is_rarfile(path):
        try:
            zf = RarFile(path)
        except:
            return -1, []
    elif is_sevenfile(path):
        try:
            zf = SevenZip(path)
        except:
            return -1, []
    else:
        return 1, []

    status = 1
    names = zf.namelist()
    names.sort()
    nzbcount = 0
    for name in names:
        name = name.lower()
        if not (name.endswith('.nzb') or name.endswith('.nfo') or name.endswith('/')):
            status = 1
            break
        elif name.endswith('.nzb'):
            status = 0
            nzbcount += 1
    if status == 0:
        if nzbcount != 1:
            nzbname = None
        for name in names:
            if name.lower().endswith('.nzb'):
                try:
                    data = zf.read(name)
                except:
                    zf.close()
                    return -1, []
                name = os.path.basename(name)
                if data:
                    try:
                        nzo = nzbstuff.NzbObject(name, pp, script, data, cat=cat, url=url,
                                                 priority=priority, nzbname=nzbname)
                        if not nzo.password:
                            nzo.password = password
                    except:
                        nzo = None
                    if nzo:
                        if nzo_id:
                            # Re-use existing nzo_id, when a "future" job gets it payload
                            sabnzbd.nzbqueue.NzbQueue.do.remove(nzo_id, add_to_history=False)
                            nzo.nzo_id = nzo_id
                        nzo_ids.append(add_nzo(nzo))
                        nzo.update_rating()
        zf.close()
        try:
            if not keep:
                os.remove(path)
        except:
            logging.error(T('Error removing %s'), misc.clip_path(path))
            logging.info("Traceback: ", exc_info=True)
            status = 1
    else:
        zf.close()
        status = 1

    return status, nzo_ids
Beispiel #16
0
def ProcessArchiveFile(filename, path, pp=None, script=None, cat=None, catdir=None, keep=False,
                       priority=None, url='', nzbname=None):
    """ Analyse ZIP file and create job(s).
        Accepts ZIP files with ONLY nzb/nfo/folder files in it.
        returns (status, nzo_ids)
            status: -1==Error/Retry, 0==OK, 1==Ignore
    """
    from sabnzbd.nzbqueue import add_nzo
    nzo_ids = []
    if catdir is None:
        catdir = cat

    filename, cat = name_to_cat(filename, catdir)

    if zipfile.is_zipfile(path):
        try:
            zf = zipfile.ZipFile(path)
        except:
            return -1, []
    elif is_rarfile(path):
        try:
            zf = RarFile(path)
        except:
            return -1, []
    else:
        return 1, []

    status = 1
    names = zf.namelist()
    names.sort()
    nzbcount = 0
    for name in names:
        name = name.lower()
        if not (name.endswith('.nzb') or name.endswith('.nfo') or name.endswith('/')):
            status = 1
            break
        elif name.endswith('.nzb'):
            status = 0
            nzbcount += 1
    if status == 0:
        if nzbcount != 1:
            nzbname = None
        for name in names:
            if name.lower().endswith('.nzb'):
                try:
                    data = zf.read(name)
                except:
                    zf.close()
                    return -1, []
                name = re.sub(r'\[.*nzbmatrix.com\]', '', name)
                name = os.path.basename(name)
                name = misc.sanitize_foldername(name)
                if data:
                    try:
                        nzo = nzbstuff.NzbObject(name, 0, pp, script, data, cat=cat, url=url,
                                                 priority=priority, nzbname=nzbname)
                    except:
                        nzo = None
                    if nzo:
                        nzo_ids.append(add_nzo(nzo))
                        nzo.update_rating()
        zf.close()
        try:
            if not keep: os.remove(path)
        except:
            logging.error(Ta('Error removing %s'), path)
            logging.info("Traceback: ", exc_info = True)
            status = 1
    else:
        zf.close()
        status = 1

    return status, nzo_ids
Beispiel #17
0
def check_encrypted_and_unwanted_files(nzo: NzbObject, filepath: str) -> Tuple[bool, Optional[str]]:
    """ Combines check for unwanted and encrypted files to save on CPU and IO """
    encrypted = False
    unwanted = None

    if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or (
        nzo.encrypted == 0 and cfg.pause_on_pwrar()
    ):
        # These checks should not break the assembler
        try:
            # Rarfile freezes on Windows special names, so don't try those!
            if sabnzbd.WIN32 and has_win_device(filepath):
                return encrypted, unwanted

            # Is it even a rarfile?
            if rarfile.is_rarfile(filepath):
                # Open the rar
                rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
                zf = rarfile.RarFile(filepath, single_file_check=True)

                # Check for encryption
                if (
                    nzo.encrypted == 0
                    and cfg.pause_on_pwrar()
                    and (zf.needs_password() or is_cloaked(nzo, filepath, zf.namelist()))
                ):
                    # Load all passwords
                    passwords = get_all_passwords(nzo)

                    # Cloaked job?
                    if is_cloaked(nzo, filepath, zf.namelist()):
                        encrypted = True
                    elif not passwords:
                        # Only error when no password was set
                        nzo.encrypted = 1
                        encrypted = True
                    else:
                        # Lets test if any of the password work
                        password_hit = False

                        for password in passwords:
                            if password:
                                logging.info('Trying password "%s" on job "%s"', password, nzo.final_name)
                                try:
                                    zf.setpassword(password)
                                except rarfile.Error:
                                    # On weird passwords the setpassword() will fail
                                    # but the actual testrar() will work
                                    pass
                                try:
                                    zf.testrar()
                                    password_hit = password
                                    break
                                except rarfile.RarWrongPassword:
                                    # This one really didn't work
                                    pass
                                except rarfile.RarCRCError as e:
                                    # CRC errors can be thrown for wrong password or
                                    # missing the next volume (with correct password)
                                    if "cannot find volume" in str(e).lower():
                                        # We assume this one worked!
                                        password_hit = password
                                        break
                                    # This one didn't work
                                    pass
                                except:
                                    # All the other errors we skip, they might be fixable in post-proc.
                                    # For example starting from the wrong volume, or damaged files
                                    # This will cause the check to be performed again for the next rar, might
                                    # be disk-intensive! Could be removed later and just accept the password.
                                    return encrypted, unwanted

                        # Did any work?
                        if password_hit:
                            # We always trust the user's input
                            if not nzo.password:
                                nzo.password = password_hit
                            # Don't check other files
                            logging.info('Password "%s" matches for job "%s"', password_hit, nzo.final_name)
                            nzo.encrypted = -1
                            encrypted = False
                        else:
                            # Encrypted and none of them worked
                            nzo.encrypted = 1
                            encrypted = True

                # Check for unwanted extensions
                if cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions():
                    for somefile in zf.namelist():
                        logging.debug("File contains: %s", somefile)
                        if get_ext(somefile).replace(".", "").lower() in cfg.unwanted_extensions():
                            logging.debug("Unwanted file %s", somefile)
                            unwanted = somefile
                zf.close()
                del zf
        except:
            logging.info("Error during inspection of RAR-file %s", filepath)
            logging.debug("Traceback: ", exc_info=True)

    return encrypted, unwanted
Beispiel #18
0
    def run(self):
        while 1:
            # Set NzbObject and NzbFile objects to None so references
            # from this thread do not keep the objects alive (see #1628)
            nzo = nzf = None
            nzo, nzf, file_done = self.queue.get()
            if not nzo:
                logging.info("Shutting down")
                break

            if nzf:
                # Check if enough disk space is free after each file is done
                # If not enough space left, pause downloader and send email
                if file_done and not sabnzbd.Downloader.paused:
                    freespace = diskspace(force=True)
                    full_dir = None
                    required_space = (cfg.download_free.get_float() + nzf.bytes) / GIGI
                    if freespace["download_dir"][1] < required_space:
                        full_dir = "download_dir"

                    # Enough space in download_dir, check complete_dir
                    complete_free = cfg.complete_free.get_float()
                    if complete_free > 0 and not full_dir:
                        required_space = 0
                        if cfg.direct_unpack():
                            required_space = (complete_free + nzo.bytes_downloaded) / GIGI
                        else:
                            # Continue downloading until 95% complete before checking
                            if nzo.bytes_tried > (nzo.bytes - nzo.bytes_par2) * 0.95:
                                required_space = (complete_free + nzo.bytes) / GIGI

                        if required_space and freespace["complete_dir"][1] < required_space:
                            full_dir = "complete_dir"

                    if full_dir:
                        logging.warning(T("Too little diskspace forcing PAUSE"))
                        # Pause downloader, but don't save, since the disk is almost full!
                        sabnzbd.Downloader.pause()
                        if cfg.fulldisk_autoresume():
                            sabnzbd.Scheduler.plan_diskspace_resume(full_dir, required_space)
                        sabnzbd.emailer.diskfull_mail()

                # Prepare filepath
                filepath = nzf.prepare_filepath()

                if filepath:
                    logging.debug("Decoding part of %s", filepath)
                    try:
                        self.assemble(nzf, file_done)
                    except IOError as err:
                        # If job was deleted or in active post-processing, ignore error
                        if not nzo.deleted and not nzo.is_gone() and not nzo.pp_active:
                            # 28 == disk full => pause downloader
                            if err.errno == 28:
                                logging.error(T("Disk full! Forcing Pause"))
                            else:
                                logging.error(T("Disk error on creating file %s"), clip_path(filepath))
                            # Log traceback
                            logging.info("Traceback: ", exc_info=True)
                            # Pause without saving
                            sabnzbd.Downloader.pause()
                        continue
                    except:
                        logging.error(T("Fatal error in Assembler"), exc_info=True)
                        break

                    # Continue after partly written data
                    if not file_done:
                        continue

                    # Clean-up admin data
                    logging.info("Decoding finished %s", filepath)
                    nzf.remove_admin()

                    # Do rar-related processing
                    if rarfile.is_rarfile(filepath):
                        # Encryption and unwanted extension detection
                        rar_encrypted, unwanted_file = check_encrypted_and_unwanted_files(nzo, filepath)
                        if rar_encrypted:
                            if cfg.pause_on_pwrar() == 1:
                                logging.warning(
                                    T(
                                        'Paused job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
                                    ),
                                    nzo.final_name,
                                )
                                nzo.pause()
                            else:
                                logging.warning(
                                    T(
                                        'Aborted job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
                                    ),
                                    nzo.final_name,
                                )
                                nzo.fail_msg = T("Aborted, encryption detected")
                                sabnzbd.NzbQueue.end_job(nzo)

                        if unwanted_file:
                            # Don't repeat the warning after a user override of an unwanted extension pause
                            if nzo.unwanted_ext == 0:
                                logging.warning(
                                    T('In "%s" unwanted extension in RAR file. Unwanted file is %s '),
                                    nzo.final_name,
                                    unwanted_file,
                                )
                            logging.debug(T("Unwanted extension is in rar file %s"), filepath)
                            if cfg.action_on_unwanted_extensions() == 1 and nzo.unwanted_ext == 0:
                                logging.debug("Unwanted extension ... pausing")
                                nzo.unwanted_ext = 1
                                nzo.pause()
                            if cfg.action_on_unwanted_extensions() == 2:
                                logging.debug("Unwanted extension ... aborting")
                                nzo.fail_msg = T("Aborted, unwanted extension detected")
                                sabnzbd.NzbQueue.end_job(nzo)

                        # Add to direct unpack
                        nzo.add_to_direct_unpacker(nzf)

                    elif par2file.is_parfile(filepath):
                        # Parse par2 files, cloaked or not
                        nzo.handle_par2(nzf, filepath)

                    filter_output, reason = nzo_filtered_by_rating(nzo)
                    if filter_output == 1:
                        logging.warning(
                            T('Paused job "%s" because of rating (%s)'),
                            nzo.final_name,
                            reason,
                        )
                        nzo.pause()
                    elif filter_output == 2:
                        logging.warning(
                            T('Aborted job "%s" because of rating (%s)'),
                            nzo.final_name,
                            reason,
                        )
                        nzo.fail_msg = T("Aborted, rating filter matched (%s)") % reason
                        sabnzbd.NzbQueue.end_job(nzo)

            else:
                sabnzbd.NzbQueue.remove(nzo.nzo_id, cleanup=False)
                sabnzbd.PostProcessor.process(nzo)
Beispiel #19
0
    def run(self):
        while 1:
            job = self.queue.get()
            if not job:
                logging.info("Shutting down")
                break

            nzo, nzf, file_done = job

            if nzf:
                # Check if enough disk space is free after each file is done
                # If not enough space left, pause downloader and send email
                if (file_done and diskspace(force=True)["download_dir"][1] <
                    (cfg.download_free.get_float() + nzf.bytes) / GIGI):
                    # Only warn and email once
                    if not sabnzbd.downloader.Downloader.do.paused:
                        logging.warning(
                            T("Too little diskspace forcing PAUSE"))
                        # Pause downloader, but don't save, since the disk is almost full!
                        sabnzbd.downloader.Downloader.do.pause()
                        sabnzbd.emailer.diskfull_mail()
                        # Abort all direct unpackers, just to be sure
                        sabnzbd.directunpacker.abort_all()

                # Prepare filepath
                filepath = nzf.prepare_filepath()

                if filepath:
                    logging.debug("Decoding part of %s", filepath)
                    try:
                        self.assemble(nzf, file_done)
                    except IOError as err:
                        # If job was deleted or in active post-processing, ignore error
                        if not nzo.deleted and not nzo.is_gone(
                        ) and not nzo.pp_active:
                            # 28 == disk full => pause downloader
                            if err.errno == 28:
                                logging.error(T("Disk full! Forcing Pause"))
                            else:
                                logging.error(
                                    T("Disk error on creating file %s"),
                                    clip_path(filepath))
                            # Log traceback
                            logging.info("Traceback: ", exc_info=True)
                            # Pause without saving
                            sabnzbd.downloader.Downloader.do.pause()
                        continue
                    except:
                        logging.error(T("Fatal error in Assembler"),
                                      exc_info=True)
                        break

                    # Continue after partly written data
                    if not file_done:
                        continue

                    # Clean-up admin data
                    logging.info("Decoding finished %s", filepath)
                    nzf.remove_admin()

                    # Do rar-related processing
                    if rarfile.is_rarfile(filepath):
                        # Encryption and unwanted extension detection
                        rar_encrypted, unwanted_file = check_encrypted_and_unwanted_files(
                            nzo, filepath)
                        if rar_encrypted:
                            if cfg.pause_on_pwrar() == 1:
                                logging.warning(
                                    T('Paused job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
                                      ),
                                    nzo.final_name,
                                )
                                nzo.pause()
                            else:
                                logging.warning(
                                    T('Aborted job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
                                      ),
                                    nzo.final_name,
                                )
                                nzo.fail_msg = T(
                                    "Aborted, encryption detected")
                                sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)

                        if unwanted_file:
                            logging.warning(
                                T('In "%s" unwanted extension in RAR file. Unwanted file is %s '
                                  ),
                                nzo.final_name,
                                unwanted_file,
                            )
                            logging.debug(
                                T("Unwanted extension is in rar file %s"),
                                filepath)
                            if cfg.action_on_unwanted_extensions(
                            ) == 1 and nzo.unwanted_ext == 0:
                                logging.debug("Unwanted extension ... pausing")
                                nzo.unwanted_ext = 1
                                nzo.pause()
                            if cfg.action_on_unwanted_extensions() == 2:
                                logging.debug(
                                    "Unwanted extension ... aborting")
                                nzo.fail_msg = T(
                                    "Aborted, unwanted extension detected")
                                sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)

                        # Add to direct unpack
                        nzo.add_to_direct_unpacker(nzf)

                    elif par2file.is_parfile(filepath):
                        # Parse par2 files, cloaked or not
                        nzo.handle_par2(nzf, filepath)

                    filter_output, reason = nzo_filtered_by_rating(nzo)
                    if filter_output == 1:
                        logging.warning(
                            T('Paused job "%s" because of rating (%s)'),
                            nzo.final_name,
                            reason,
                        )
                        nzo.pause()
                    elif filter_output == 2:
                        logging.warning(
                            T('Aborted job "%s" because of rating (%s)'),
                            nzo.final_name,
                            reason,
                        )
                        nzo.fail_msg = T(
                            "Aborted, rating filter matched (%s)") % reason
                        sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)

            else:
                sabnzbd.nzbqueue.NzbQueue.do.remove(nzo.nzo_id,
                                                    add_to_history=False,
                                                    cleanup=False)
                PostProcessor.do.process(nzo)
Beispiel #20
0
def check_encrypted_and_unwanted_files(nzo, filepath):
    """ Combines check for unwanted and encrypted files to save on CPU and IO """
    encrypted = False
    unwanted = None

    if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or (
            nzo.encrypted == 0 and cfg.pause_on_pwrar()):
        # These checks should not break the assembler
        try:
            # Rarfile freezes on Windows special names, so don't try those!
            if sabnzbd.WIN32 and has_win_device(filepath):
                return encrypted, unwanted

            # Is it even a rarfile?
            if rarfile.is_rarfile(filepath):
                # Open the rar
                rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
                zf = rarfile.RarFile(filepath, all_names=True)

                # Check for encryption
                if nzo.encrypted == 0 and cfg.pause_on_pwrar() and (
                        zf.needs_password()
                        or is_cloaked(nzo, filepath, zf.namelist())):
                    # Load all passwords
                    passwords = get_all_passwords(nzo)

                    # Cloaked job?
                    if is_cloaked(nzo, filepath, zf.namelist()):
                        encrypted = True
                    elif not sabnzbd.HAVE_CRYPTOGRAPHY and not passwords:
                        # if no cryptography installed, only error when no password was set
                        logging.info(T('%s missing'), 'Python Cryptography')
                        nzo.encrypted = 1
                        encrypted = True

                    elif sabnzbd.HAVE_CRYPTOGRAPHY:
                        # Lets test if any of the password work
                        password_hit = False

                        for password in passwords:
                            if password:
                                logging.info(
                                    'Trying password "%s" on job "%s"',
                                    password, nzo.final_name)
                                try:
                                    zf.setpassword(password)
                                except:
                                    # On weird passwords the setpassword() will fail
                                    # but the actual rartest() will work
                                    pass
                                try:
                                    zf.testrar()
                                    password_hit = password
                                    break
                                except rarfile.RarCRCError:
                                    # On CRC error we can continue!
                                    password_hit = password
                                    break
                                except Exception as e:
                                    # Did we start from the right volume?
                                    if 'need to start extraction from a previous volume' in e[
                                            0]:
                                        return encrypted, unwanted
                                    # This one failed
                                    pass

                        # Did any work?
                        if password_hit:
                            # We always trust the user's input
                            if not nzo.password:
                                nzo.password = password_hit
                            # Don't check other files
                            logging.info('Password "%s" matches for job "%s"',
                                         password_hit, nzo.final_name)
                            nzo.encrypted = -1
                            encrypted = False
                        else:
                            # Encrypted and none of them worked
                            nzo.encrypted = 1
                            encrypted = True
                    else:
                        # Don't check other files
                        nzo.encrypted = -1
                        encrypted = False

                # Check for unwanted extensions
                if cfg.unwanted_extensions(
                ) and cfg.action_on_unwanted_extensions():
                    for somefile in zf.namelist():
                        logging.debug('File contains: %s', somefile)
                        if get_ext(somefile).replace(
                                '.', '').lower() in cfg.unwanted_extensions():
                            logging.debug('Unwanted file %s', somefile)
                            unwanted = somefile
                zf.close()
                del zf
        except:
            logging.info('Error during inspection of RAR-file %s', filepath)
            logging.debug('Traceback: ', exc_info=True)

    return encrypted, unwanted
Beispiel #21
0
class Assembler(Thread):
    do = None  # Link to the instance of this method

    def __init__(self, queue=None):
        Thread.__init__(self)

        if queue:
            self.queue = queue
        else:
            self.queue = Queue.Queue()
        Assembler.do = self

    def stop(self):
        self.process(None)

    def process(self, job):
        self.queue.put(job)

    def run(self):
        while 1:
            job = self.queue.get()
            if not job:
                logging.info("Shutting down")
                break

            nzo, nzf = job

            if nzf:
                # Check if enough disk space is free, if not pause downloader and send email
                if diskspace(force=True)['download_dir'][1] < (
                        cfg.download_free.get_float() + nzf.bytes) / GIGI:
                    # Only warn and email once
                    if not sabnzbd.downloader.Downloader.do.paused:
                        logging.warning(
                            T('Too little diskspace forcing PAUSE'))
                        # Pause downloader, but don't save, since the disk is almost full!
                        sabnzbd.downloader.Downloader.do.pause()
                        sabnzbd.emailer.diskfull()
                        # Abort all direct unpackers, just to be sure
                        sabnzbd.directunpacker.abort_all()

                    # Place job back in queue and wait 30 seconds to hope it gets resolved
                    self.process(job)
                    sleep(30)
                    continue

                # Prepare filename
                nzo.verify_nzf_filename(nzf)
                nzf.filename = sanitize_filename(nzf.filename)
                filepath = get_filepath(long_path(cfg.download_dir.get_path()),
                                        nzo, nzf.filename)
                nzf.filename = get_filename(filepath)

                if filepath:
                    logging.info('Decoding %s %s', filepath, nzf.type)
                    try:
                        filepath = self.assemble(nzf, filepath)
                    except IOError, (errno, strerror):
                        # If job was deleted or in active post-processing, ignore error
                        if not nzo.deleted and not nzo.is_gone(
                        ) and not nzo.pp_active:
                            # 28 == disk full => pause downloader
                            if errno == 28:
                                logging.error(T('Disk full! Forcing Pause'))
                            else:
                                logging.error(
                                    T('Disk error on creating file %s'),
                                    clip_path(filepath))
                            # Log traceback
                            logging.info('Traceback: ', exc_info=True)
                            # Pause without saving
                            sabnzbd.downloader.Downloader.do.pause()
                        continue
                    except:
                        logging.error(T('Fatal error in Assembler'),
                                      exc_info=True)
                        break

                    # Clean-up admin data
                    nzf.remove_admin()

                    # Do rar-related processing
                    if rarfile.is_rarfile(filepath):
                        # Encryption and unwanted extension detection
                        rar_encrypted, unwanted_file = check_encrypted_and_unwanted_files(
                            nzo, filepath)
                        if rar_encrypted:
                            if cfg.pause_on_pwrar() == 1:
                                logging.warning(
                                    remove_warning_label(
                                        T('WARNING: Paused job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
                                          )), nzo.final_name)
                                nzo.pause()
                            else:
                                logging.warning(
                                    remove_warning_label(
                                        T('WARNING: Aborted job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
                                          )), nzo.final_name)
                                nzo.fail_msg = T(
                                    'Aborted, encryption detected')
                                sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)

                        if unwanted_file:
                            logging.warning(
                                remove_warning_label(
                                    T('WARNING: In "%s" unwanted extension in RAR file. Unwanted file is %s '
                                      )), nzo.final_name, unwanted_file)
                            logging.debug(
                                T('Unwanted extension is in rar file %s'),
                                filepath)
                            if cfg.action_on_unwanted_extensions(
                            ) == 1 and nzo.unwanted_ext == 0:
                                logging.debug('Unwanted extension ... pausing')
                                nzo.unwanted_ext = 1
                                nzo.pause()
                            if cfg.action_on_unwanted_extensions() == 2:
                                logging.debug(
                                    'Unwanted extension ... aborting')
                                nzo.fail_msg = T(
                                    'Aborted, unwanted extension detected')
                                sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)

                        # Add to direct unpack
                        nzo.add_to_direct_unpacker(nzf)

                    elif par2file.is_parfile(filepath):
                        # Parse par2 files, cloaked or not
                        nzo.handle_par2(nzf, filepath)

                    filter, reason = nzo_filtered_by_rating(nzo)
                    if filter == 1:
                        logging.warning(
                            remove_warning_label(
                                T('WARNING: Paused job "%s" because of rating (%s)'
                                  )), nzo.final_name, reason)
                        nzo.pause()
                    elif filter == 2:
                        logging.warning(
                            remove_warning_label(
                                T('WARNING: Aborted job "%s" because of rating (%s)'
                                  )), nzo.final_name, reason)
                        nzo.fail_msg = T(
                            'Aborted, rating filter matched (%s)') % reason
                        sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
Beispiel #22
0
def check_encrypted_and_unwanted_files(nzo, filepath):
    """ Combines check for unwanted and encrypted files to save on CPU and IO """
    encrypted = False
    unwanted = None

    if cfg.unwanted_extensions() or (nzo.encrypted == 0 and cfg.pause_on_pwrar()):
        # Safe-format for Windows
        # RarFile requires de-unicoded filenames for zf.testrar()
        filepath_split = os.path.split(filepath)
        workdir_short = short_path(filepath_split[0])
        filepath = deunicode(os.path.join(workdir_short, filepath_split[1]))

        # Is it even a rarfile?
        if rarfile.is_rarfile(filepath):
            try:
                zf = rarfile.RarFile(filepath, all_names=True)
                # Check for encryption
                if nzo.encrypted == 0 and cfg.pause_on_pwrar() and (zf.needs_password() or is_cloaked(filepath, zf.namelist())):
                    # Load all passwords
                    passwords = get_all_passwords(nzo)

                    # if no cryptography installed, only error when no password was set
                    if not sabnzbd.HAVE_CRYPTOGRAPHY and not passwords:
                        logging.info(T('%s missing'), 'Python Cryptography')
                        nzo.encrypted = 1
                        encrypted = True

                    elif sabnzbd.HAVE_CRYPTOGRAPHY:
                        # Lets test if any of the password work
                        password_hit = False
                        rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND

                        for password in passwords:
                            if password:
                                logging.info('Trying password "%s" on job "%s"', password, nzo.final_name)
                                try:
                                    zf.setpassword(password)
                                    zf.testrar()
                                    password_hit = password
                                    break
                                except rarfile.RarCRCError:
                                    # On CRC error we can continue!
                                    password_hit = password
                                    break
                                except:
                                    pass

                        # Did any work?
                        if password_hit:
                            # Don't check other files
                            logging.info('Password "%s" matches for job "%s"', password_hit, nzo.final_name)
                            nzo.encrypted = -1
                            encrypted = False
                        else:
                            # Encrypted and none of them worked
                            nzo.encrypted = 1
                            encrypted = True
                    else:
                        # Don't check other files
                        nzo.encrypted = -1
                        encrypted = False

                # Check for unwanted extensions
                if cfg.unwanted_extensions():
                    for somefile in zf.namelist():
                        logging.debug('File contains: %s', somefile)
                        if os.path.splitext(somefile)[1].replace('.', '').lower() in cfg.unwanted_extensions():
                            logging.debug('Unwanted file %s', somefile)
                            unwanted = somefile
                            zf.close()
                zf.close()
                del zf
            except:
                logging.debug('RAR file %s cannot be inspected', filepath)

    return encrypted, unwanted