def get_package_path_by_filename(self, fileName, channel, checksum):
log_debug(3, fileName, channel, checksum)
fileName = str(fileName)
n, e, v, r, a = rhnLib.parseRPMFilename(fileName)
h = rhnSQL.prepare(self._query_get_package_path_by_nvra_and_checksum)
h.execute(name=n,
version=v,
release=r,
epoch=e,
arch=a,
channel=channel,
checksum=checksum)
try:
return _get_path_from_cursor(h)
except InvalidPackageError:
log_debug(4, "Error", "Non-existent package requested", fileName)
raise_with_tb(
rhnFault(17,
_("Invalid RPM package %s requested") % fileName),
sys.exc_info()[2])
except NullPathPackageError:
e = sys.exc_info()[1]
package_id = e[0]
log_error("Package path null for package id", package_id)
raise_with_tb(
rhnFault(17,
_("Invalid RPM package %s requested") % fileName),
sys.exc_info()[2])
except MissingPackageError:
e = sys.exc_info()[1]
filePath = e[0]
log_error("Package not found", filePath)
raise_with_tb(rhnFault(17, _("Package not found")),
sys.exc_info()[2])
def get_package_path_by_filename(self, fileName, channel):
log_debug(3, fileName, channel)
fileName = str(fileName)
n, e, v, r, a = rhnLib.parseRPMFilename(fileName)
h = rhnSQL.prepare(self._query_get_package_path_by_nvra)
h.execute(name=n,
version=v,
release=r,
epoch=e,
arch=a,
channel=channel)
try:
return _get_path_from_cursor(h)
except InvalidPackageError:
log_debug(4, "Error", "Non-existent package requested", fileName)
raise rhnFault(17,
_("Invalid RPM package %s requested") %
fileName), None, sys.exc_info()[2]
except NullPathPackageError, e:
package_id = e[0]
log_error("Package path null for package id", package_id)
raise rhnFault(17,
_("Invalid RPM package %s requested") %
fileName), None, sys.exc_info()[2]
def get_package_path_by_filename(self, fileName, channel):
log_debug(3, fileName, channel)
fileName = str(fileName)
n, e, v, r, a = rhnLib.parseRPMFilename(fileName)
h = rhnSQL.prepare(self._query_get_package_path_by_nvra)
h.execute(name=n, version=v, release=r, epoch=e, arch=a, channel=channel)
try:
return _get_path_from_cursor(h)
except InvalidPackageError:
log_debug(4, "Error", "Non-existent package requested", fileName)
raise rhnFault(17, _("Invalid RPM package %s requested") % fileName), None, sys.exc_info()[2]
except NullPathPackageError, e:
package_id = e[0]
log_error("Package path null for package id", package_id)
raise rhnFault(17, _("Invalid RPM package %s requested") % fileName), None, sys.exc_info()[2]
def get_path(channel, filename):
with dblock:
nevra = rhnLib.parseRPMFilename(filename)
c = rhnSQL.prepare("""
SELECT rhnPackage.path
FROM rhnPackage, rhnChannelPackage, rhnChannel, rhnPackageName,
rhnPackageEVR, rhnPackageArch
WHERE rhnPackage.name_id = rhnPackageName.id AND
rhnPackageName.name = :name AND
rhnPackage.evr_id = rhnPackageEVR.id AND
rhnPackageEVR.version = :version AND
rhnPackageEVR.release = :release AND
rhnPackage.package_arch_id = rhnPackageArch.id AND
rhnPackageArch.label = :arch AND
rhnPackage.id = rhnChannelPackage.package_id AND
rhnChannelPackage.channel_id = rhnChannel.id AND
rhnChannel.label = :channel
""")
c.execute(channel=channel, name=nevra[0], version=nevra[2],
release=nevra[3], arch=nevra[4])
return c.fetchone_dict()["path"]
def process_sha256_packages():
if debug:
log = rhnLog('/var/log/rhn/update-packages.log', 5)
_get_sha256_packages_sql = rhnSQL.prepare(_get_sha256_packages_query)
_get_sha256_packages_sql.execute()
packages = _get_sha256_packages_sql.fetchall_dict()
if not packages:
print "No SHA256 capable packages to process."
if debug:
log.writeMessage("No SHA256 capable packages to process.")
return
if verbose:
print "Processing %s SHA256 capable packages" % len(packages)
pb = ProgressBar(prompt='standby: ', endTag=' - Complete!', \
finalSize=len(packages), finalBarLength=40, stream=sys.stdout)
pb.printAll(1)
_update_sha256_package_sql = rhnSQL.prepare(_update_sha256_package)
_update_package_files_sql = rhnSQL.prepare(_update_package_files)
for package in packages:
pb.addTo(1)
pb.printIncrement()
old_abs_path = os.path.join(CFG.MOUNT_POINT, package['path'])
if debug and verbose:
log.writeMessage("Processing package: %s" % old_abs_path)
temp_file = open(old_abs_path, 'rb')
header, _payload_stream, _header_start, _header_end = \
rhnPackageUpload.load_package(temp_file)
checksum_type = header.checksum_type()
checksum = getFileChecksum(checksum_type, file_obj=temp_file)
old_path = package['path'].split('/')
nevra = parseRPMFilename(old_path[-1])
org_id = old_path[1]
new_path = get_package_path(nevra,
org_id,
prepend=old_path[0],
checksum=checksum)
new_abs_path = os.path.join(CFG.MOUNT_POINT, new_path)
# Filer content relocation
try:
if old_abs_path != new_abs_path:
if debug:
log.writeMessage("Relocating %s to %s on filer" %
(old_abs_path, new_abs_path))
new_abs_dir = os.path.dirname(new_abs_path)
if not os.path.isdir(new_abs_dir):
os.makedirs(new_abs_dir)
# link() the old path to the new path
if not os.path.exists(new_abs_path):
os.link(old_abs_path, new_abs_path)
elif debug:
log.writeMessage("File %s already exists" % new_abs_path)
# Make the new path readable
os.chmod(new_abs_path, 0644)
except OSError, e:
message = "Error when relocating %s to %s on filer: %s" % \
(old_abs_path, new_abs_path, str(e))
print message
if debug:
log.writeMessage(message)
sys.exit(1)
# Update package checksum in the database
_update_sha256_package_sql.execute(ctype=checksum_type,
csum=checksum,
path=new_path,
id=package['id'])
_select_checksum_type_id_sql = rhnSQL.prepare(_select_checksum_type_id)
_select_checksum_type_id_sql.execute(ctype=checksum_type)
checksum_type_id = _select_checksum_type_id_sql.fetchone()[0]
# Update checksum of every single file in a package
for i, f in enumerate(header['filenames']):
csum = header['filemd5s'][i]
# Do not update checksums for directories & links
if not csum:
continue
_update_package_files_sql.execute(ctype_id=checksum_type_id,
csum=csum,
pid=package['id'],
filename=f)
rhnSQL.commit()
try:
if os.path.exists(old_abs_path):
os.unlink(old_abs_path)
if os.path.exists(os.path.dirname(old_abs_path)):
os.removedirs(os.path.dirname(old_abs_path))
except OSError, e:
message = "Error when removing %s: %s" % (old_abs_path, str(e))
print message
if debug:
log.writeMessage(message)
sys.exit(1)
def process_package_data():
if debug:
log = rhnLog('/var/log/rhn/update-packages.log', 5)
_get_path_sql = rhnSQL.prepare(_get_path_query)
_update_package_path = rhnSQL.prepare(_update_pkg_path_query)
_get_path_sql.execute()
paths = _get_path_sql.fetchall_dict()
if not paths:
# Nothing to change
return
if verbose:
print "Processing %s packages" % len(paths)
pb = ProgressBar(prompt='standby: ', endTag=' - Complete!', \
finalSize=len(paths), finalBarLength=40, stream=sys.stdout)
pb.printAll(1)
skip_list = []
new_ok_list = []
i = 0
for path in paths:
pb.addTo(1)
pb.printIncrement()
old_path_nvrea = path['path'].split('/')
org_id = old_path_nvrea[1]
# pylint: disable=W0703
try:
nevra = parseRPMFilename(old_path_nvrea[-1])
if nevra[1] in [None, '']:
nevra[1] = path['epoch']
except Exception:
# probably not an rpm skip
if debug:
log.writeMessage("Skipping: %s Not a valid rpm" \
% old_path_nvrea[-1])
continue
old_abs_path = os.path.join(CFG.MOUNT_POINT, path['path'])
checksum_type = path['checksum_type']
checksum = path['checksum']
new_path = get_package_path(nevra,
org_id,
prepend=old_path_nvrea[0],
checksum=checksum)
new_abs_path = os.path.join(CFG.MOUNT_POINT, new_path)
bad_abs_path = os.path.join(CFG.MOUNT_POINT, \
get_package_path(nevra, org_id, prepend=old_path_nvrea[0],
omit_epoch = True, checksum=checksum))
if not os.path.exists(old_abs_path):
if os.path.exists(new_abs_path):
new_ok_list.append(new_abs_path)
if debug:
log.writeMessage("File %s already on final path %s" %
(path['path'], new_abs_path))
old_abs_path = new_abs_path
elif os.path.exists(bad_abs_path):
log.writeMessage("File %s found on %s" %
(path['path'], bad_abs_path))
old_abs_path = bad_abs_path
else:
skip_list.append(old_abs_path)
if debug:
log.writeMessage("Missing path %s for package %d" %
(old_abs_path, path['id']))
continue
# pylint: disable=W0703
try:
hdr = rhn_rpm.get_package_header(filename=old_abs_path)
except Exception, e:
msg = "Exception occurred when reading package header %s: %s" % \
(old_abs_path, str(e))
print msg
if debug:
log.writeMessage(msg)
rhnSQL.commit()
sys.exit(1)
if old_abs_path != new_abs_path:
new_abs_dir = os.path.dirname(new_abs_path)
# relocate the package on the filer
if debug:
log.writeMessage("Relocating %s to %s on filer" \
% (old_abs_path, new_abs_path))
if not os.path.isdir(new_abs_dir):
os.makedirs(new_abs_dir)
shutil.move(old_abs_path, new_abs_path)
# Clean up left overs
os.removedirs(os.path.dirname(old_abs_path))
# make the path readable
os.chmod(new_abs_path, 0644)
# Update the db paths
_update_package_path.execute(the_id= path['id'], \
new_path = new_path )
if debug:
log.writeMessage("query Executed: update rhnPackage %d to %s" \
% ( path['id'], new_path ))
# Process gpg key ids
server_packages.processPackageKeyAssociations(hdr, checksum_type,
checksum)
if debug:
log.writeMessage("gpg key info updated from %s" % new_abs_path)
i = i + 1
# we need to break the transaction to smaller pieces
if i % 1000 == 0:
rhnSQL.commit()
def process_sha256_packages():
if debug:
log = rhnLog('/var/log/rhn/update-packages.log', 5)
_get_sha256_packages_sql = rhnSQL.prepare(_get_sha256_packages_query)
_get_sha256_packages_sql.execute()
packages = _get_sha256_packages_sql.fetchall_dict()
if not packages:
print("No SHA256 capable packages to process.")
if debug:
log.writeMessage("No SHA256 capable packages to process.")
return
if verbose:
print("Processing %s SHA256 capable packages" % len(packages))
pb = ProgressBar(prompt='standby: ', endTag=' - Complete!',
finalSize=len(packages), finalBarLength=40, stream=sys.stdout)
pb.printAll(1)
_update_sha256_package_sql = rhnSQL.prepare(_update_sha256_package)
_update_package_files_sql = rhnSQL.prepare(_update_package_files)
for package in packages:
pb.addTo(1)
pb.printIncrement()
old_abs_path = os.path.join(CFG.MOUNT_POINT, package['path'])
if debug and verbose:
log.writeMessage("Processing package: %s" % old_abs_path)
temp_file = open(old_abs_path, 'rb')
header, _payload_stream, _header_start, _header_end = \
rhnPackageUpload.load_package(temp_file)
checksum_type = header.checksum_type()
checksum = getFileChecksum(checksum_type, file_obj=temp_file)
old_path = package['path'].split('/')
nevra = parseRPMFilename(old_path[-1])
org_id = old_path[1]
new_path = get_package_path(nevra, org_id, prepend=old_path[0], checksum=checksum)
new_abs_path = os.path.join(CFG.MOUNT_POINT, new_path)
# Filer content relocation
try:
if old_abs_path != new_abs_path:
if debug:
log.writeMessage("Relocating %s to %s on filer" % (old_abs_path, new_abs_path))
new_abs_dir = os.path.dirname(new_abs_path)
if not os.path.isdir(new_abs_dir):
os.makedirs(new_abs_dir)
# link() the old path to the new path
if not os.path.exists(new_abs_path):
os.link(old_abs_path, new_abs_path)
elif debug:
log.writeMessage("File %s already exists" % new_abs_path)
# Make the new path readable
os.chmod(new_abs_path, int('0644', 8))
except OSError:
e = sys.exc_info()[1]
message = "Error when relocating %s to %s on filer: %s" % \
(old_abs_path, new_abs_path, str(e))
print(message)
if debug:
log.writeMessage(message)
sys.exit(1)
# Update package checksum in the database
_update_sha256_package_sql.execute(ctype=checksum_type, csum=checksum,
path=new_path, id=package['id'])
_select_checksum_type_id_sql = rhnSQL.prepare(_select_checksum_type_id)
_select_checksum_type_id_sql.execute(ctype=checksum_type)
checksum_type_id = _select_checksum_type_id_sql.fetchone()[0]
# Update checksum of every single file in a package
for i, f in enumerate(header['filenames']):
csum = header['filemd5s'][i]
# Do not update checksums for directories & links
if not csum:
continue
_update_package_files_sql.execute(ctype_id=checksum_type_id, csum=csum,
pid=package['id'], filename=f)
rhnSQL.commit()
try:
if os.path.exists(old_abs_path):
os.unlink(old_abs_path)
if os.path.exists(os.path.dirname(old_abs_path)):
os.removedirs(os.path.dirname(old_abs_path))
except OSError:
e = sys.exc_info()[1]
message = "Error when removing %s: %s" % (old_abs_path, str(e))
print(message)
if debug:
log.writeMessage(message)
sys.exit(1)
pb.printComplete()
def process_package_data():
if debug:
log = rhnLog('/var/log/rhn/update-packages.log', 5)
_get_path_sql = rhnSQL.prepare(_get_path_query)
_update_package_path = rhnSQL.prepare(_update_pkg_path_query)
_get_path_sql.execute()
paths = _get_path_sql.fetchall_dict()
if not paths:
# Nothing to change
return
if verbose:
print("Processing %s packages" % len(paths))
pb = ProgressBar(prompt='standby: ', endTag=' - Complete!',
finalSize=len(paths), finalBarLength=40, stream=sys.stdout)
pb.printAll(1)
skip_list = []
new_ok_list = []
i = 0
for path in paths:
pb.addTo(1)
pb.printIncrement()
old_path_nvrea = path['path'].split('/')
org_id = old_path_nvrea[1]
# pylint: disable=W0703
try:
nevra = parseRPMFilename(old_path_nvrea[-1])
if nevra[1] in [None, '']:
nevra[1] = path['epoch']
except Exception:
# probably not an rpm skip
if debug:
log.writeMessage("Skipping: %s Not a valid rpm"
% old_path_nvrea[-1])
continue
old_abs_path = os.path.join(CFG.MOUNT_POINT, path['path'])
checksum_type = path['checksum_type']
checksum = path['checksum']
new_path = get_package_path(nevra, org_id, prepend=old_path_nvrea[0],
checksum=checksum)
new_abs_path = os.path.join(CFG.MOUNT_POINT, new_path)
bad_abs_path = os.path.join(CFG.MOUNT_POINT,
get_package_path(nevra, org_id, prepend=old_path_nvrea[0],
omit_epoch=True, checksum=checksum))
if not os.path.exists(old_abs_path):
if os.path.exists(new_abs_path):
new_ok_list.append(new_abs_path)
if debug:
log.writeMessage("File %s already on final path %s" % (path['path'], new_abs_path))
old_abs_path = new_abs_path
elif os.path.exists(bad_abs_path):
log.writeMessage("File %s found on %s" % (path['path'], bad_abs_path))
old_abs_path = bad_abs_path
else:
skip_list.append(old_abs_path)
if debug:
log.writeMessage("Missing path %s for package %d" % (old_abs_path, path['id']))
continue
# pylint: disable=W0703
try:
hdr = rhn_rpm.get_package_header(filename=old_abs_path)
except Exception:
e = sys.exc_info()[1]
msg = "Exception occurred when reading package header %s: %s" % \
(old_abs_path, str(e))
print(msg)
if debug:
log.writeMessage(msg)
rhnSQL.commit()
sys.exit(1)
if old_abs_path != new_abs_path:
new_abs_dir = os.path.dirname(new_abs_path)
# relocate the package on the filer
if debug:
log.writeMessage("Relocating %s to %s on filer"
% (old_abs_path, new_abs_path))
if not os.path.isdir(new_abs_dir):
os.makedirs(new_abs_dir)
shutil.move(old_abs_path, new_abs_path)
# Clean up left overs
os.removedirs(os.path.dirname(old_abs_path))
# make the path readable
os.chmod(new_abs_path, int('0644', 8))
# Update the db paths
_update_package_path.execute(the_id=path['id'],
new_path=new_path)
if debug:
log.writeMessage("query Executed: update rhnPackage %d to %s"
% (path['id'], new_path))
# Process gpg key ids
server_packages.processPackageKeyAssociations(hdr, checksum_type, checksum)
if debug:
log.writeMessage("gpg key info updated from %s" % new_abs_path)
i = i + 1
# we need to break the transaction to smaller pieces
if i % 1000 == 0:
rhnSQL.commit()
pb.printComplete()
# All done, final commit
rhnSQL.commit()
sys.stderr.write("Transaction Committed! \n")
if verbose:
print(" Skipping %s packages, paths not found" % len(skip_list))
if len(new_ok_list) > 0 and verbose:
print(" There were %s packages found in the correct location" % len(new_ok_list))
return
