Exemplo n.º 1
0
def relative_path_from_nevra(nevra, org_id, package_type=None, checksum_type=None, checksum=None):
    #4/18/05 wregglej. if 1051 is in the header's keys, then it's a nosrc package.
    if nevra[4] == 'src' or nevra[4] == 'nosrc':
        is_source = 1
    else:
        is_source = 0
    log_debug(4, nevra, is_source)
    return get_package_path(nevra, org_id=org_id, source=is_source, 
        prepend=CFG.PREPENDED_DIR, omit_epoch=1, package_type=package_type,
        checksum_type=checksum_type, checksum=checksum)
Exemplo n.º 2
0
def relative_path_from_nevra(nevra, org_id, package_type=None, checksum_type=None, checksum=None):
    # 4/18/05 wregglej. if 1051 is in the header's keys, then it's a nosrc package.
    if nevra[4] == 'src' or nevra[4] == 'nosrc':
        is_source = 1
    else:
        is_source = 0
    log_debug(4, nevra, is_source)
    return get_package_path(nevra, org_id=org_id, source=is_source,
                            prepend=CFG.PREPENDED_DIR, omit_epoch=None, package_type=package_type,
                            checksum_type=checksum_type, checksum=checksum)
Exemplo n.º 3
0
def process_sha256_packages():
    if debug:
        log = rhnLog('/var/log/rhn/update-packages.log', 5)

    _get_sha256_packages_sql = rhnSQL.prepare(_get_sha256_packages_query)
    _get_sha256_packages_sql.execute()
    packages = _get_sha256_packages_sql.fetchall_dict()

    if not packages:
        print "No SHA256 capable packages to process."
        if debug:
            log.writeMessage("No SHA256 capable packages to process.")

        return

    if verbose:
        print "Processing %s SHA256 capable packages" % len(packages)

    pb = ProgressBar(prompt='standby: ', endTag=' - Complete!', \
                     finalSize=len(packages), finalBarLength=40, stream=sys.stdout)
    pb.printAll(1)

    _update_sha256_package_sql = rhnSQL.prepare(_update_sha256_package)
    _update_package_files_sql = rhnSQL.prepare(_update_package_files)

    for package in packages:
        pb.addTo(1)
        pb.printIncrement()

        old_abs_path = os.path.join(CFG.MOUNT_POINT, package['path'])
        if debug and verbose:
            log.writeMessage("Processing package: %s" % old_abs_path)
        temp_file = open(old_abs_path, 'rb')
        header, _payload_stream, _header_start, _header_end = \
                rhnPackageUpload.load_package(temp_file)
        checksum_type = header.checksum_type()
        checksum = getFileChecksum(checksum_type, file_obj=temp_file)

        old_path = package['path'].split('/')
        nevra = parseRPMFilename(old_path[-1])
        org_id = old_path[1]
        new_path = get_package_path(nevra,
                                    org_id,
                                    prepend=old_path[0],
                                    checksum=checksum)
        new_abs_path = os.path.join(CFG.MOUNT_POINT, new_path)

        # Filer content relocation
        try:
            if old_abs_path != new_abs_path:
                if debug:
                    log.writeMessage("Relocating %s to %s on filer" %
                                     (old_abs_path, new_abs_path))

                new_abs_dir = os.path.dirname(new_abs_path)
                if not os.path.isdir(new_abs_dir):
                    os.makedirs(new_abs_dir)

                # link() the old path to the new path
                if not os.path.exists(new_abs_path):
                    os.link(old_abs_path, new_abs_path)
                elif debug:
                    log.writeMessage("File %s already exists" % new_abs_path)

                # Make the new path readable
                os.chmod(new_abs_path, 0644)
        except OSError, e:
            message = "Error when relocating %s to %s on filer: %s" % \
                      (old_abs_path, new_abs_path, str(e))
            print message
            if debug:
                log.writeMessage(message)
            sys.exit(1)

        # Update package checksum in the database
        _update_sha256_package_sql.execute(ctype=checksum_type,
                                           csum=checksum,
                                           path=new_path,
                                           id=package['id'])

        _select_checksum_type_id_sql = rhnSQL.prepare(_select_checksum_type_id)
        _select_checksum_type_id_sql.execute(ctype=checksum_type)
        checksum_type_id = _select_checksum_type_id_sql.fetchone()[0]

        # Update checksum of every single file in a package
        for i, f in enumerate(header['filenames']):
            csum = header['filemd5s'][i]

            # Do not update checksums for directories & links
            if not csum:
                continue

            _update_package_files_sql.execute(ctype_id=checksum_type_id,
                                              csum=csum,
                                              pid=package['id'],
                                              filename=f)

        rhnSQL.commit()

        try:
            if os.path.exists(old_abs_path):
                os.unlink(old_abs_path)
            if os.path.exists(os.path.dirname(old_abs_path)):
                os.removedirs(os.path.dirname(old_abs_path))
        except OSError, e:
            message = "Error when removing %s: %s" % (old_abs_path, str(e))
            print message
            if debug:
                log.writeMessage(message)

            sys.exit(1)
Exemplo n.º 4
0
def process_package_data():
    if debug:
        log = rhnLog('/var/log/rhn/update-packages.log', 5)

    _get_path_sql = rhnSQL.prepare(_get_path_query)
    _update_package_path = rhnSQL.prepare(_update_pkg_path_query)

    _get_path_sql.execute()
    paths = _get_path_sql.fetchall_dict()

    if not paths:
        # Nothing to change
        return
    if verbose:
        print "Processing %s packages" % len(paths)
    pb = ProgressBar(prompt='standby: ', endTag=' - Complete!', \
                     finalSize=len(paths), finalBarLength=40, stream=sys.stdout)
    pb.printAll(1)
    skip_list = []
    new_ok_list = []
    i = 0
    for path in paths:
        pb.addTo(1)
        pb.printIncrement()
        old_path_nvrea = path['path'].split('/')
        org_id = old_path_nvrea[1]
        # pylint: disable=W0703
        try:
            nevra = parseRPMFilename(old_path_nvrea[-1])
            if nevra[1] in [None, '']:
                nevra[1] = path['epoch']
        except Exception:
            # probably not an rpm skip
            if debug:
                log.writeMessage("Skipping: %s Not a valid rpm" \
                                  % old_path_nvrea[-1])
            continue
        old_abs_path = os.path.join(CFG.MOUNT_POINT, path['path'])

        checksum_type = path['checksum_type']
        checksum = path['checksum']
        new_path = get_package_path(nevra,
                                    org_id,
                                    prepend=old_path_nvrea[0],
                                    checksum=checksum)
        new_abs_path = os.path.join(CFG.MOUNT_POINT, new_path)

        bad_abs_path = os.path.join(CFG.MOUNT_POINT, \
                   get_package_path(nevra, org_id, prepend=old_path_nvrea[0],
                             omit_epoch = True, checksum=checksum))

        if not os.path.exists(old_abs_path):
            if os.path.exists(new_abs_path):
                new_ok_list.append(new_abs_path)
                if debug:
                    log.writeMessage("File %s already on final path %s" %
                                     (path['path'], new_abs_path))
                old_abs_path = new_abs_path
            elif os.path.exists(bad_abs_path):
                log.writeMessage("File %s found on %s" %
                                 (path['path'], bad_abs_path))
                old_abs_path = bad_abs_path
            else:
                skip_list.append(old_abs_path)
                if debug:
                    log.writeMessage("Missing path %s for package %d" %
                                     (old_abs_path, path['id']))
                continue

        # pylint: disable=W0703
        try:
            hdr = rhn_rpm.get_package_header(filename=old_abs_path)
        except Exception, e:
            msg = "Exception occurred when reading package header %s: %s" % \
                (old_abs_path, str(e))
            print msg
            if debug:
                log.writeMessage(msg)
            rhnSQL.commit()
            sys.exit(1)

        if old_abs_path != new_abs_path:
            new_abs_dir = os.path.dirname(new_abs_path)
            # relocate the package on the filer
            if debug:
                log.writeMessage("Relocating %s to %s on filer" \
                           % (old_abs_path, new_abs_path))
            if not os.path.isdir(new_abs_dir):
                os.makedirs(new_abs_dir)
            shutil.move(old_abs_path, new_abs_path)
            # Clean up left overs
            os.removedirs(os.path.dirname(old_abs_path))
            # make the path readable
            os.chmod(new_abs_path, 0644)

        # Update the db paths
        _update_package_path.execute(the_id= path['id'], \
                             new_path = new_path )
        if debug:
            log.writeMessage("query Executed: update rhnPackage %d to %s" \
                               % ( path['id'], new_path ))
        # Process gpg key ids
        server_packages.processPackageKeyAssociations(hdr, checksum_type,
                                                      checksum)
        if debug:
            log.writeMessage("gpg key info updated from %s" % new_abs_path)
        i = i + 1
        # we need to break the transaction to smaller pieces
        if i % 1000 == 0:
            rhnSQL.commit()
Exemplo n.º 5
0
def process_sha256_packages():
    if debug:
        log = rhnLog('/var/log/rhn/update-packages.log', 5)

    _get_sha256_packages_sql = rhnSQL.prepare(_get_sha256_packages_query)
    _get_sha256_packages_sql.execute()
    packages = _get_sha256_packages_sql.fetchall_dict()

    if not packages:
        print("No SHA256 capable packages to process.")
        if debug:
            log.writeMessage("No SHA256 capable packages to process.")

        return

    if verbose:
        print("Processing %s SHA256 capable packages" % len(packages))

    pb = ProgressBar(prompt='standby: ', endTag=' - Complete!',
                     finalSize=len(packages), finalBarLength=40, stream=sys.stdout)
    pb.printAll(1)

    _update_sha256_package_sql = rhnSQL.prepare(_update_sha256_package)
    _update_package_files_sql = rhnSQL.prepare(_update_package_files)

    for package in packages:
        pb.addTo(1)
        pb.printIncrement()

        old_abs_path = os.path.join(CFG.MOUNT_POINT, package['path'])
        if debug and verbose:
            log.writeMessage("Processing package: %s" % old_abs_path)
        temp_file = open(old_abs_path, 'rb')
        header, _payload_stream, _header_start, _header_end = \
            rhnPackageUpload.load_package(temp_file)
        checksum_type = header.checksum_type()
        checksum = getFileChecksum(checksum_type, file_obj=temp_file)

        old_path = package['path'].split('/')
        nevra = parseRPMFilename(old_path[-1])
        org_id = old_path[1]
        new_path = get_package_path(nevra, org_id, prepend=old_path[0], checksum=checksum)
        new_abs_path = os.path.join(CFG.MOUNT_POINT, new_path)

        # Filer content relocation
        try:
            if old_abs_path != new_abs_path:
                if debug:
                    log.writeMessage("Relocating %s to %s on filer" % (old_abs_path, new_abs_path))

                new_abs_dir = os.path.dirname(new_abs_path)
                if not os.path.isdir(new_abs_dir):
                    os.makedirs(new_abs_dir)

                # link() the old path to the new path
                if not os.path.exists(new_abs_path):
                    os.link(old_abs_path, new_abs_path)
                elif debug:
                    log.writeMessage("File %s already exists" % new_abs_path)

                # Make the new path readable
                os.chmod(new_abs_path, int('0644', 8))
        except OSError:
            e = sys.exc_info()[1]
            message = "Error when relocating %s to %s on filer: %s" % \
                      (old_abs_path, new_abs_path, str(e))
            print(message)
            if debug:
                log.writeMessage(message)
            sys.exit(1)

        # Update package checksum in the database
        _update_sha256_package_sql.execute(ctype=checksum_type, csum=checksum,
                                           path=new_path, id=package['id'])

        _select_checksum_type_id_sql = rhnSQL.prepare(_select_checksum_type_id)
        _select_checksum_type_id_sql.execute(ctype=checksum_type)
        checksum_type_id = _select_checksum_type_id_sql.fetchone()[0]

        # Update checksum of every single file in a package
        for i, f in enumerate(header['filenames']):
            csum = header['filemd5s'][i]

            # Do not update checksums for directories & links
            if not csum:
                continue

            _update_package_files_sql.execute(ctype_id=checksum_type_id, csum=csum,
                                              pid=package['id'], filename=f)

        rhnSQL.commit()

        try:
            if os.path.exists(old_abs_path):
                os.unlink(old_abs_path)
            if os.path.exists(os.path.dirname(old_abs_path)):
                os.removedirs(os.path.dirname(old_abs_path))
        except OSError:
            e = sys.exc_info()[1]
            message = "Error when removing %s: %s" % (old_abs_path, str(e))
            print(message)
            if debug:
                log.writeMessage(message)

            sys.exit(1)

    pb.printComplete()
Exemplo n.º 6
0
def process_package_data():
    if debug:
        log = rhnLog('/var/log/rhn/update-packages.log', 5)

    _get_path_sql = rhnSQL.prepare(_get_path_query)
    _update_package_path = rhnSQL.prepare(_update_pkg_path_query)

    _get_path_sql.execute()
    paths = _get_path_sql.fetchall_dict()

    if not paths:
        # Nothing to change
        return
    if verbose:
        print("Processing %s packages" % len(paths))
    pb = ProgressBar(prompt='standby: ', endTag=' - Complete!',
                     finalSize=len(paths), finalBarLength=40, stream=sys.stdout)
    pb.printAll(1)
    skip_list = []
    new_ok_list = []
    i = 0
    for path in paths:
        pb.addTo(1)
        pb.printIncrement()
        old_path_nvrea = path['path'].split('/')
        org_id = old_path_nvrea[1]
        # pylint: disable=W0703
        try:
            nevra = parseRPMFilename(old_path_nvrea[-1])
            if nevra[1] in [None, '']:
                nevra[1] = path['epoch']
        except Exception:
            # probably not an rpm skip
            if debug:
                log.writeMessage("Skipping: %s Not a valid rpm"
                                 % old_path_nvrea[-1])
            continue
        old_abs_path = os.path.join(CFG.MOUNT_POINT, path['path'])

        checksum_type = path['checksum_type']
        checksum = path['checksum']
        new_path = get_package_path(nevra, org_id, prepend=old_path_nvrea[0],
                                    checksum=checksum)
        new_abs_path = os.path.join(CFG.MOUNT_POINT, new_path)

        bad_abs_path = os.path.join(CFG.MOUNT_POINT,
                                    get_package_path(nevra, org_id, prepend=old_path_nvrea[0],
                                                     omit_epoch=True, checksum=checksum))

        if not os.path.exists(old_abs_path):
            if os.path.exists(new_abs_path):
                new_ok_list.append(new_abs_path)
                if debug:
                    log.writeMessage("File %s already on final path %s" % (path['path'], new_abs_path))
                old_abs_path = new_abs_path
            elif os.path.exists(bad_abs_path):
                log.writeMessage("File %s found on %s" % (path['path'], bad_abs_path))
                old_abs_path = bad_abs_path
            else:
                skip_list.append(old_abs_path)
                if debug:
                    log.writeMessage("Missing path %s for package %d" % (old_abs_path, path['id']))
                continue

        # pylint: disable=W0703
        try:
            hdr = rhn_rpm.get_package_header(filename=old_abs_path)
        except Exception:
            e = sys.exc_info()[1]
            msg = "Exception occurred when reading package header %s: %s" % \
                (old_abs_path, str(e))
            print(msg)
            if debug:
                log.writeMessage(msg)
            rhnSQL.commit()
            sys.exit(1)

        if old_abs_path != new_abs_path:
            new_abs_dir = os.path.dirname(new_abs_path)
            # relocate the package on the filer
            if debug:
                log.writeMessage("Relocating %s to %s on filer"
                                 % (old_abs_path, new_abs_path))
            if not os.path.isdir(new_abs_dir):
                os.makedirs(new_abs_dir)
            shutil.move(old_abs_path, new_abs_path)
            # Clean up left overs
            os.removedirs(os.path.dirname(old_abs_path))
            # make the path readable
            os.chmod(new_abs_path, int('0644', 8))

        # Update the db paths
        _update_package_path.execute(the_id=path['id'],
                                     new_path=new_path)
        if debug:
            log.writeMessage("query Executed: update rhnPackage %d to %s"
                             % (path['id'], new_path))
        # Process gpg key ids
        server_packages.processPackageKeyAssociations(hdr, checksum_type, checksum)
        if debug:
            log.writeMessage("gpg key info updated from %s" % new_abs_path)
        i = i + 1
        # we need to break the transaction to smaller pieces
        if i % 1000 == 0:
            rhnSQL.commit()
    pb.printComplete()
    # All done, final commit
    rhnSQL.commit()
    sys.stderr.write("Transaction Committed! \n")
    if verbose:
        print(" Skipping %s packages, paths not found" % len(skip_list))
    if len(new_ok_list) > 0 and verbose:
        print(" There were %s packages found in the correct location" % len(new_ok_list))
    return