def read_dsc(package, version, component, distro_name, archive_root):
source_dir, dsc_path = unpack_dsc(package, version, component,
distro_name, archive_root)
dsc = open(dsc_path).read().strip()
fullpath = os.path.join(source_dir, "debian", "changelog")
changelog = None
if os.path.exists(fullpath):
changelog = open(fullpath).read().strip()
else:
log.warn("No changelog file found for %s in %s" %
(package, source_dir))
changelog = None
copyright = None
globpath = os.path.join(source_dir, "debian", "*copyright")
for fullpath in glob.glob(globpath):
if not os.path.exists(fullpath):
continue
copyright = open(fullpath).read().strip()
if copyright is None:
log.warn(
"No copyright file found for %s in %s" % (package, source_dir))
copyright = ''
return dsc, changelog, copyright
def locate_sourcepackage(self, binarypackagedata, distroseries):
# This function uses a list of versions to deal with the fact
# that we may need to munge the version number as we search for
# bin-only-NMUs. The fast path is dealt with the first cycle of
# the loop; we only cycle more than once if the source package
# is really missing.
versions = [binarypackagedata.source_version]
is_binnmu = self.binnmu_re2.match(binarypackagedata.source_version)
if is_binnmu:
# DEB is jikes-sablevm_1.1.5-1.0.1_all.deb
# bin version is 1.1.5-1.0.1
# DSC is sablevm_1.1.5-1.dsc
# src version is 1.1.5-1
versions.append(is_binnmu.group(1))
is_binnmu = self.binnmu_re.match(binarypackagedata.source_version)
if is_binnmu:
# DEB is jikes-sablevm_1.1.5-1.1_all.deb
# bin version is 1.1.5-1.1
# DSC is sablevm_1.1.5-1.dsc
# src version is 1.1.5-1
versions.append(is_binnmu.group(1))
for version in versions:
sourcepackage = self.sphandler.checkSource(
binarypackagedata.source, version, distroseries)
if sourcepackage:
return sourcepackage
# We couldn't find a sourcepackagerelease in the database.
# Perhaps we can opportunistically pick one out of the archive.
log.warn("No source package %s (%s) listed for %s (%s), "
"scrubbing archive..." %
(binarypackagedata.source,
version, binarypackagedata.package,
binarypackagedata.version))
# XXX kiko 2005-11-03: I question whether
# binarypackagedata.section here is actually correct -- but
# where can we obtain this information from introspecting
# the archive?
sourcepackage = self.sphandler.findUnlistedSourcePackage(
binarypackagedata.source, version,
binarypackagedata.component, binarypackagedata.section,
distroseries)
if sourcepackage:
return sourcepackage
log.warn("Nope, couldn't find it. Could it be a "
"bin-only-NMU? Checking version %s" % version)
# XXX kiko 2005-11-03: Testing a third cycle of this loop
# isn't done.
return None
def _buildArchiveFilesystemInfo(self, archive_root, distroseries,
component, arch=None, source_only=False):
"""Create and store the ArchiveFilesystemInfo objects."""
try:
archive_info = ArchiveFilesystemInfo(
archive_root, distroseries, component, arch, source_only)
except NoBinaryArchive:
log.warn("The archive for %s/%s doesn't contain "
"a directory for %s, skipping" %
(distroseries, component, arch))
return
self._archive_archs.append(archive_info)
def import_binarypackages(distro, packages_map, package_root,
importer_handler):
nosource = []
# Run over all the architectures we have
for archtag in packages_map.bin_map.keys():
npacks = len(packages_map.bin_map[archtag])
log.info('%i Binary Packages to be imported for %s', npacks, archtag)
# Go over binarypackages importing them for this architecture
for package_name in sorted(packages_map.bin_map[archtag].iterkeys()):
binary = packages_map.bin_map[archtag][package_name]
try:
try:
do_one_binarypackage(distro, binary, archtag, package_root,
importer_handler)
except psycopg2.Error:
log.exception(
"Database errors when importing a BinaryPackage "
"for %s. Retrying once..", package_name)
importer_handler.abort()
time.sleep(15)
do_one_binarypackage(distro, binary, archtag, package_root,
importer_handler)
except (InvalidVersionError, MissingRequiredArguments):
log.exception("Unable to create BinaryPackageData for %s",
package_name)
continue
except (PoolFileNotFound, ExecutionError):
# Problems with katie db stuff of opening files
log.exception("Error processing package files for %s",
package_name)
continue
except MultiplePackageReleaseError:
log.exception("Database duplication processing %s",
package_name)
continue
except psycopg2.Error:
log.exception(
"Database errors made me give up: unable to create "
"BinaryPackage for %s", package_name)
importer_handler.abort()
continue
except NoSourcePackageError:
log.exception("Failed to create Binary Package for %s",
package_name)
nosource.append(binary)
continue
if nosource:
# XXX kiko 2005-10-23: untested
log.warn('%i source packages not found', len(nosource))
for pkg in nosource:
log.warn(pkg)
def ensure_complete(self):
if self.format is None:
# XXX kiko 2005-11-05: this is very funny. We care so much about
# it here, but we don't do anything about this in handlers.py!
self.format = "1.0"
log.warn(
"Invalid format in %s, assumed %r", self.package, self.format)
if self.urgency not in ChangesFile.urgency_map:
log.warn(
"Invalid urgency in %s, %r, assumed %r",
self.package, self.urgency, "low")
self.urgency = "low"
def __init__(self, **args):
for k, v in args.items():
if k == 'Binary':
self.binaries = stripseq(v.split(","))
elif k == 'Section':
self.section = parse_section(v)
elif k == 'Urgency':
urgency = v
# This is to handle cases like:
# - debget: 'high (actually works)
# - lxtools: 'low, closes=90239'
if " " in urgency:
urgency = urgency.split()[0]
if "," in urgency:
urgency = urgency.split(",")[0]
self.urgency = urgency
elif k == 'Maintainer':
displayname, emailaddress = parse_person(v)
try:
self.maintainer = (
encoding.guess(displayname),
emailaddress,
)
except UnicodeDecodeError:
raise DisplayNameDecodingError(
"Could not decode name %s" % displayname)
elif k == 'Files':
self.files = []
files = v.split("\n")
for f in files:
self.files.append(stripseq(f.split(" ")))
else:
setattr(self, k.lower().replace("-", "_"), v)
if self.section is None:
self.section = 'misc'
log.warn(
"Source package %s lacks section, assumed %r",
self.package, self.section)
if '/' in self.section:
# this apparently happens with packages in universe.
# 3dchess, for instance, uses "universe/games"
self.section = self.section.split("/", 1)[1]
AbstractPackageData.__init__(self)
def import_binarypackages(distro, packages_map, package_root, importer_handler):
nosource = []
# Run over all the architectures we have
for archtag in packages_map.bin_map.keys():
npacks = len(packages_map.bin_map[archtag])
log.info("%i Binary Packages to be imported for %s", npacks, archtag)
# Go over binarypackages importing them for this architecture
for package_name in sorted(packages_map.bin_map[archtag].iterkeys()):
binary = packages_map.bin_map[archtag][package_name]
try:
try:
do_one_binarypackage(distro, binary, archtag, package_root, importer_handler)
except psycopg2.Error:
log.exception(
"Database errors when importing a BinaryPackage " "for %s. Retrying once..", package_name
)
importer_handler.abort()
time.sleep(15)
do_one_binarypackage(distro, binary, archtag, package_root, importer_handler)
except (InvalidVersionError, MissingRequiredArguments):
log.exception("Unable to create BinaryPackageData for %s", package_name)
continue
except (PoolFileNotFound, ExecutionError):
# Problems with katie db stuff of opening files
log.exception("Error processing package files for %s", package_name)
continue
except MultiplePackageReleaseError:
log.exception("Database duplication processing %s", package_name)
continue
except psycopg2.Error:
log.exception("Database errors made me give up: unable to create " "BinaryPackage for %s", package_name)
importer_handler.abort()
continue
except NoSourcePackageError:
log.exception("Failed to create Binary Package for %s", package_name)
nosource.append(binary)
continue
if nosource:
# XXX kiko 2005-10-23: untested
log.warn("%i source packages not found", len(nosource))
for pkg in nosource:
log.warn(pkg)
def do_package(self, distro_name, archive_root):
"""Get the Changelog and urgency from the package on archive.
If successful processing of the package occurs, this method
sets the changelog and urgency attributes.
"""
dsc, changelog, copyright = read_dsc(
self.package, self.version, self.component, distro_name,
archive_root)
self.dsc = encoding.guess(dsc)
self.copyright = encoding.guess(copyright)
parsed_changelog = None
if changelog:
parsed_changelog = parse_changelog(changelog.split('\n'))
self.urgency = None
self.changelog = None
self.changelog_entry = None
if parsed_changelog and parsed_changelog[0]:
cldata = parsed_changelog[0]
if 'changes' in cldata:
if cldata["package"] != self.package:
log.warn("Changelog package %s differs from %s" %
(cldata["package"], self.package))
if cldata["version"] != self.version:
log.warn("Changelog version %s differs from %s" %
(cldata["version"], self.version))
self.changelog_entry = encoding.guess(cldata["changes"])
self.changelog = changelog
self.urgency = cldata["urgency"]
else:
log.warn("Changelog empty for source %s (%s)" %
(self.package, self.version))
def __init__(self, **args):
for k, v in args.items():
if k == "Maintainer":
self.maintainer = parse_person(v)
elif k == "Essential":
self.essential = (v == "yes")
elif k == 'Section':
self.section = parse_section(v)
elif k == "Description":
self.description = encoding.guess(v)
summary = self.description.split("\n")[0].strip()
if not summary.endswith('.'):
summary = summary + '.'
self.summary = summary
elif k == "Installed-Size":
try:
self.installed_size = int(v)
except ValueError:
raise MissingRequiredArguments("Installed-Size is "
"not a valid integer: %r" % v)
else:
setattr(self, k.lower().replace("-", "_"), v)
if self.source:
# We need to handle cases like "Source: myspell
# (1:3.0+pre3.1-6)". apt-pkg kindly splits this for us
# already, but sometimes fails.
# XXX: dsilvers 2005-09-22: Work out why this happens and
# file an upstream bug against python-apt once we've worked
# it out.
if self.source_version is None:
match = self.source_version_re.match(self.source)
if match:
self.source = match.group(1)
self.source_version = match.group(2)
else:
# XXX kiko 2005-10-18:
# This is probably a best-guess and might fail.
self.source_version = self.version
else:
# Some packages have Source, some don't -- the ones that
# don't have the same package name.
self.source = self.package
self.source_version = self.version
if (self.source_version is None or
self.source_version != self.version and
not valid_debian_version(self.source_version)):
raise InvalidSourceVersionError(
"Binary package %s (%s) refers to source package %s "
"with invalid version: %s" %
(self.package, self.version, self.source,
self.source_version))
if self.section is None:
self.section = 'misc'
log.warn(
"Binary package %s lacks a section, assumed %r",
self.package, self.section)
if self.priority is None:
self.priority = 'extra'
log.warn(
"Binary package %s lacks valid priority, assumed %r",
self.package, self.priority)
AbstractPackageData.__init__(self)
