def test_parse_archive(self):
"""Germinator.parse_archive successfully parses a simple archive."""
self.addSource("warty", "main", "hello", "1.0-1",
["hello", "hello-dependency"],
fields={"Maintainer": "Test Person <*****@*****.**>"})
self.addPackage("warty", "main", "i386", "hello", "1.0-1",
fields={
"Maintainer": "Test Person <*****@*****.**>",
"Depends": "hello-dependency",
})
self.addPackage("warty", "main", "i386", "hello-dependency", "1.0-1",
fields={"Source": "hello"})
self.addSeed("ubuntu.warty", "supported")
self.addSeedPackage("ubuntu.warty", "supported", "hello")
germinator = Germinator("i386")
archive = TagFile(
"warty", "main", "i386", "file://%s" % self.archive_dir)
germinator.parse_archive(archive)
self.assertIn("hello", germinator._sources)
self.assertEqual({
"Maintainer": u"Test Person <*****@*****.**>",
"Version": "1.0-1",
"Build-Depends": [],
"Build-Depends-Indep": [],
"Binaries": ["hello", "hello-dependency"],
}, germinator._sources["hello"])
self.assertIn("hello", germinator._packages)
self.assertEqual({
"Section": "",
"Version": "1.0-1",
"Maintainer": u"Test Person <*****@*****.**>",
"Essential": "",
"Pre-Depends": [],
"Depends": [[("hello-dependency", "", "")]],
"Recommends": [],
"Size": 0,
"Installed-Size": 0,
"Source": "hello",
"Provides": [],
"Kernel-Version": "",
}, germinator._packages["hello"])
self.assertEqual("deb", germinator._packagetype["hello"])
self.assertIn("hello-dependency", germinator._packages)
self.assertEqual({
"Section": "",
"Version": "1.0-1",
"Maintainer": u"",
"Essential": "",
"Pre-Depends": [],
"Depends": [],
"Recommends": [],
"Size": 0,
"Installed-Size": 0,
"Source": "hello",
"Provides": [],
"Kernel-Version": "",
}, germinator._packages["hello-dependency"])
self.assertEqual("deb", germinator._packagetype["hello-dependency"])
self.assertEqual({}, germinator._provides)
def test_build_depends_profiles(self):
"""Test that https://wiki.debian.org/BuildProfileSpec restrictions
are parseable.
"""
self.addSource("precise", "main", "hello", "1.0-1", ["hello"],
fields={"Build-Depends":
"gettext <!stage1> <!cross>, "
"base-files <stage1>, "
"gettext (<< 0.7) | debhelper (>= 9)"})
self.addPackage("precise", "main", "i386", "hello", "1.0-1")
self.addSource("precise", "main", "gettext", "0.8.1.1-5ubuntu3",
["gettext"])
self.addPackage("precise", "main", "i386", "gettext",
"0.8.1.1-5ubuntu3")
self.addSource("precise", "main", "base-files", "6.5ubuntu6",
["base-files"])
self.addPackage("precise", "main", "i386", "base-files", "6.5ubuntu6")
self.addSource("precise", "main", "debhelper", "9.20120115ubuntu3",
["debhelper"])
self.addPackage("precise", "main", "i386", "debhelper",
"9.20120115ubuntu3")
branch = "collection.precise"
self.addSeed(branch, "base")
self.addSeedPackage(branch, "base", "hello")
germinator = Germinator("i386")
archive = TagFile(
"precise", "main", "i386", "file://%s" % self.archive_dir)
germinator.parse_archive(archive)
structure = self.openSeedStructure(branch)
germinator.plant_seeds(structure)
germinator.grow(structure)
self.assertEqual(
set(["gettext", "debhelper"]),
germinator.get_build_depends(structure, "base"))
def test_depends_multiarch(self):
"""Compare Depends behaviour against the multiarch specification.
https://wiki.ubuntu.com/MultiarchSpec
"""
for ma, qual, allowed in (
(None, "", True),
(None, ":any", False),
(None, ":native", False),
("same", "", True),
("same", ":any", False),
("same", ":native", False),
("foreign", "", True),
("foreign", ":any", False),
("foreign", ":native", False),
("allowed", "", True),
("allowed", ":any", True),
("allowed", ":native", False),
):
self.addSource("precise", "main", "hello", "1.0-1", ["hello"])
self.addPackage("precise",
"main",
"i386",
"hello",
"1.0-1",
fields={"Depends": "gettext%s" % qual})
self.addSource("precise", "main", "gettext", "0.18.1.1-5ubuntu3",
["gettext"])
package_fields = {}
if ma is not None:
package_fields["Multi-Arch"] = ma
self.addPackage("precise",
"main",
"i386",
"gettext",
"0.18.1.1-5ubuntu3",
fields=package_fields)
branch = "collection.precise"
self.addSeed(branch, "base")
self.addSeedPackage(branch, "base", "hello")
germinator = Germinator("i386")
archive = TagFile("precise", "main", "i386",
"file://%s" % self.archive_dir)
germinator.parse_archive(archive)
structure = self.openSeedStructure(branch)
germinator.plant_seeds(structure)
germinator.grow(structure)
expected = set()
if allowed:
expected.add("gettext")
self.assertEqual(
expected, germinator.get_depends(structure, "base"),
"Depends: gettext%s on Multi-Arch: %s incorrectly %s" %
(qual, ma if ma else "none",
"disallowed" if allowed else "allowed"))
shutil.rmtree(self.archive_dir)
shutil.rmtree(self.seeds_dir)
def set_seeds(self, options, seeds):
self.seeds = seeds
# Suppress most log information
germinate_logging(logging.CRITICAL)
logging.getLogger('germinate.archive').setLevel(logging.INFO)
global MIRRORS, COMPONENTS
print("Germinating")
g = Germinator(options.arch)
archive = germinate.archive.TagFile(options.dist,
COMPONENTS,
options.arch,
MIRRORS,
cleanup=True)
g.parse_archive(archive)
needed_seeds = []
build_tree = False
try:
structure = SeedStructure(options.release, options.seeds)
for seedname in self.seeds:
if seedname == ('%s+build-depends' % structure.supported):
seedname = structure.supported
build_tree = True
needed_seeds.append(seedname)
g.plant_seeds(structure, seeds=needed_seeds)
except SeedError:
sys.exit(1)
g.grow(structure)
for seedname in structure.names:
for pkg in g.get_seed_entries(structure, seedname):
self.package.setdefault(pkg, Package(pkg))
self.package[pkg].set_seed(seedname + ".seed")
for pkg in g.get_seed_recommends_entries(structure, seedname):
self.package.setdefault(pkg, Package(pkg))
self.package[pkg].set_seed(seedname + ".seed-recommends")
for pkg in g.get_depends(structure, seedname):
self.package.setdefault(pkg, Package(pkg))
self.package[pkg].set_seed(seedname + ".depends")
if build_tree:
build_depends = set(g.get_build_depends(structure, seedname))
for inner in structure.inner_seeds(structure.supported):
build_depends -= set(g.get_seed_entries(structure, inner))
build_depends -= set(
g.get_seed_recommends_entries(structure, inner))
build_depends -= g.get_depends(structure, inner)
for pkg in build_depends:
self.package.setdefault(pkg, Package(pkg))
self.package[pkg].set_seed(structure.supported +
".build-depends")
def set_seeds(self, options, seeds):
self.seeds = seeds
# Suppress most log information
germinate_logging(logging.CRITICAL)
logging.getLogger('germinate.archive').setLevel(logging.INFO)
global MIRRORS, COMPONENTS
print("Germinating")
g = Germinator(options.arch)
archive = germinate.archive.TagFile(
options.dist, COMPONENTS, options.arch, MIRRORS, cleanup=True)
g.parse_archive(archive)
needed_seeds = []
build_tree = False
try:
structure = SeedStructure(options.release, options.seeds)
for seedname in self.seeds:
if seedname == ('%s+build-depends' % structure.supported):
seedname = structure.supported
build_tree = True
needed_seeds.append(seedname)
g.plant_seeds(structure, seeds=needed_seeds)
except SeedError:
sys.exit(1)
g.grow(structure)
for seedname in structure.names:
for pkg in g.get_seed_entries(structure, seedname):
self.package.setdefault(pkg, Package(pkg))
self.package[pkg].set_seed(seedname + ".seed")
for pkg in g.get_seed_recommends_entries(structure, seedname):
self.package.setdefault(pkg, Package(pkg))
self.package[pkg].set_seed(seedname + ".seed-recommends")
for pkg in g.get_depends(structure, seedname):
self.package.setdefault(pkg, Package(pkg))
self.package[pkg].set_seed(seedname + ".depends")
if build_tree:
build_depends = set(g.get_build_depends(structure, seedname))
for inner in structure.inner_seeds(structure.supported):
build_depends -= set(g.get_seed_entries(structure, inner))
build_depends -= set(g.get_seed_recommends_entries(
structure, inner))
build_depends -= g.get_depends(structure, inner)
for pkg in build_depends:
self.package.setdefault(pkg, Package(pkg))
self.package[pkg].set_seed(structure.supported +
".build-depends")
def test_build_depends_multiarch(self):
"""Compare Build-Depends behaviour against the multiarch specification.
https://wiki.ubuntu.com/MultiarchCross#Build_Dependencies
"""
for ma, qual, allowed in (
(None, "", True),
(None, ":any", False),
(None, ":native", True),
("same", "", True),
("same", ":any", False),
("same", ":native", True),
("foreign", "", True),
("foreign", ":any", False),
("foreign", ":native", False),
("allowed", "", True),
("allowed", ":any", True),
("allowed", ":native", True),
):
self.addSource("precise", "main", "hello", "1.0-1", ["hello"],
fields={"Build-Depends": "gettext%s" % qual})
self.addPackage("precise", "main", "i386", "hello", "1.0-1")
self.addSource("precise", "main", "gettext", "0.18.1.1-5ubuntu3",
["gettext"])
package_fields = {}
if ma is not None:
package_fields["Multi-Arch"] = ma
self.addPackage("precise", "main", "i386", "gettext",
"0.18.1.1-5ubuntu3", fields=package_fields)
branch = "collection.precise"
self.addSeed(branch, "base")
self.addSeedPackage(branch, "base", "hello")
germinator = Germinator("i386")
archive = TagFile(
"precise", "main", "i386", "file://%s" % self.archive_dir)
germinator.parse_archive(archive)
structure = self.openSeedStructure(branch)
germinator.plant_seeds(structure)
germinator.grow(structure)
expected = set()
if allowed:
expected.add("gettext")
self.assertEqual(
expected, germinator.get_build_depends(structure, "base"),
"Build-Depends: gettext%s on Multi-Arch: %s incorrectly %s" % (
qual, ma if ma else "none",
"disallowed" if allowed else "allowed"))
shutil.rmtree(self.archive_dir)
shutil.rmtree(self.seeds_dir)
def test_different_providers_between_suites(self):
"""Provides from later versions override those from earlier ones."""
self.addSource("warty", "main", "hello", "1.0-1", ["hello"])
self.addPackage("warty", "main", "i386", "hello", "1.0-1",
fields={"Provides": "goodbye"})
self.addSource("warty-updates", "main", "hello", "1.0-1.1", ["hello"])
self.addPackage("warty-updates", "main", "i386", "hello", "1.0-1.1",
fields={"Provides": "hello-goodbye"})
germinator = Germinator("i386")
archive = TagFile(
["warty", "warty-updates"], "main", "i386",
"file://%s" % self.archive_dir)
germinator.parse_archive(archive)
self.assertNotIn("goodbye", germinator._provides)
self.assertIn("hello-goodbye", germinator._provides)
self.assertEqual(["hello"], germinator._provides["hello-goodbye"])
def test_build_depends_profiles(self):
"""Test that https://wiki.debian.org/BuildProfileSpec restrictions
are parseable.
"""
self.addSource("precise",
"main",
"hello",
"1.0-1", ["hello"],
fields={
"Build-Depends":
"gettext <!stage1> <!cross>, "
"base-files <stage1>, "
"gettext (<< 0.7) | debhelper (>= 9)"
})
self.addPackage("precise", "main", "i386", "hello", "1.0-1")
self.addSource("precise", "main", "gettext", "0.8.1.1-5ubuntu3",
["gettext"])
self.addPackage("precise", "main", "i386", "gettext",
"0.8.1.1-5ubuntu3")
self.addSource("precise", "main", "base-files", "6.5ubuntu6",
["base-files"])
self.addPackage("precise", "main", "i386", "base-files", "6.5ubuntu6")
self.addSource("precise", "main", "debhelper", "9.20120115ubuntu3",
["debhelper"])
self.addPackage("precise", "main", "i386", "debhelper",
"9.20120115ubuntu3")
branch = "collection.precise"
self.addSeed(branch, "base")
self.addSeedPackage(branch, "base", "hello")
germinator = Germinator("i386")
archive = TagFile("precise", "main", "i386",
"file://%s" % self.archive_dir)
germinator.parse_archive(archive)
structure = self.openSeedStructure(branch)
germinator.plant_seeds(structure)
germinator.grow(structure)
self.assertEqual(set(["gettext", "debhelper"]),
germinator.get_build_depends(structure, "base"))
def germinateArch(self,
override_file,
series_name,
components,
arch,
flavours,
structures,
seed_outputs=None):
"""Germinate seeds on all flavours for a single architecture."""
germinator = Germinator(arch)
# Read archive metadata.
archive = TagFile(series_name,
components,
arch,
"file://%s" % self.config.archiveroot,
cleanup=True)
germinator.parse_archive(archive)
for flavour in flavours:
self.logger.info("Germinating for %s/%s/%s", flavour, series_name,
arch)
# Add this to the germinate log as well so that that can be
# debugged more easily. Log a separator line first.
self.logGerminateProgress("")
self.logGerminateProgress("Germinating for %s/%s/%s", flavour,
series_name, arch)
self.germinateArchFlavour(override_file,
germinator,
series_name,
arch,
flavour,
structures[flavour],
flavour == flavours[0],
seed_outputs=seed_outputs)
def germinateArch(self, override_file, series_name, components, arch,
flavours, structures, seed_outputs=None):
"""Germinate seeds on all flavours for a single architecture."""
germinator = Germinator(arch)
# Read archive metadata.
archive = TagFile(
series_name, components, arch,
"file://%s" % self.config.archiveroot, cleanup=True)
germinator.parse_archive(archive)
for flavour in flavours:
self.logger.info(
"Germinating for %s/%s/%s", flavour, series_name, arch)
# Add this to the germinate log as well so that that can be
# debugged more easily. Log a separator line first.
self.logGerminateProgress("")
self.logGerminateProgress(
"Germinating for %s/%s/%s", flavour, series_name, arch)
self.germinateArchFlavour(
override_file, germinator, series_name, arch, flavour,
structures[flavour], flavour == flavours[0],
seed_outputs=seed_outputs)
def test_different_providers_between_suites(self):
"""Provides from later versions override those from earlier ones."""
self.addSource("warty", "main", "hello", "1.0-1", ["hello"])
self.addPackage("warty",
"main",
"i386",
"hello",
"1.0-1",
fields={"Provides": "goodbye"})
self.addSource("warty-updates", "main", "hello", "1.0-1.1", ["hello"])
self.addPackage("warty-updates",
"main",
"i386",
"hello",
"1.0-1.1",
fields={"Provides": "hello-goodbye"})
germinator = Germinator("i386")
archive = TagFile(["warty", "warty-updates"], "main", "i386",
"file://%s" % self.archive_dir)
germinator.parse_archive(archive)
self.assertNotIn("goodbye", germinator._provides)
self.assertIn("hello-goodbye", germinator._provides)
self.assertEqual({"hello": ""}, germinator._provides["hello-goodbye"])
def main(argv):
options, args = parse_options(argv)
if not os.path.exists('debian/control'):
error_exit('must be run from the top level of a source package')
this_source = None
with open('debian/control') as control:
for line in control:
if line.startswith('Source:'):
this_source = line[7:].strip()
break
elif line == '':
break
if this_source is None:
error_exit('cannot find Source: in debian/control')
if not this_source.endswith('-meta'):
error_exit('source package name must be *-meta')
metapackage = this_source[:-5]
print("[info] Initialising %s-* package lists update..." % metapackage)
config = SafeConfigParser()
with open('update.cfg') as config_file:
try:
# >= 3.2
config.read_file(config_file)
except AttributeError:
# < 3.2
config.readfp(config_file)
if len(args) > 0:
dist = args[0]
else:
dist = config.get('DEFAULT', 'dist')
seeds = config.get(dist, 'seeds').split()
try:
output_seeds = config.get(dist, 'output_seeds').split()
except NoOptionError:
output_seeds = list(seeds)
architectures = config.get(dist, 'architectures').split()
try:
archive_base_default = config.get(dist, 'archive_base/default')
archive_base_default = re.split(r'[, ]+', archive_base_default)
except (NoSectionError, NoOptionError):
archive_base_default = None
archive_base = {}
for arch in architectures:
try:
archive_base[arch] = config.get(dist, 'archive_base/%s' % arch)
archive_base[arch] = re.split(r'[, ]+', archive_base[arch])
except (NoSectionError, NoOptionError):
if archive_base_default is not None:
archive_base[arch] = archive_base_default
else:
error_exit('no archive_base configured for %s' % arch)
if options.vcs and config.has_option("%s/vcs" % dist, 'seed_base'):
seed_base = config.get("%s/vcs" % dist, 'seed_base')
elif options.vcs and config.has_option("%s/bzr" % dist, 'seed_base'):
# Backward compatibility.
seed_base = config.get("%s/bzr" % dist, 'seed_base')
else:
seed_base = config.get(dist, 'seed_base')
seed_base = re.split(r'[, ]+', seed_base)
if options.vcs and config.has_option("%s/vcs" % dist, 'seed_dist'):
seed_dist = config.get("%s/vcs" % dist, 'seed_dist')
elif options.vcs and config.has_option("%s/bzr" % dist, 'seed_dist'):
# Backward compatibility.
seed_dist = config.get("%s/bzr" % dist, 'seed_dist')
elif config.has_option(dist, 'seed_dist'):
seed_dist = config.get(dist, 'seed_dist')
else:
seed_dist = dist
if config.has_option(dist, 'dists'):
dists = config.get(dist, 'dists').split()
else:
dists = [dist]
components = config.get(dist, 'components').split()
def seed_packages(germinator_method, structure, seed_name):
if config.has_option(dist, "seed_map/%s" % seed_name):
mapped_seeds = config.get(dist, "seed_map/%s" % seed_name).split()
else:
mapped_seeds = []
task_seeds_re = re.compile('^Task-Seeds:\s*(.*)', re.I)
with structure[seed_name] as seed:
for line in seed:
task_seeds_match = task_seeds_re.match(line)
if task_seeds_match is not None:
mapped_seeds = task_seeds_match.group(1).split()
break
if seed_name not in mapped_seeds:
mapped_seeds.append(seed_name)
packages = []
for mapped_seed in mapped_seeds:
packages.extend(germinator_method(structure, mapped_seed))
return packages
def metapackage_name(structure, seed_name):
if config.has_option(dist, "metapackage_map/%s" % seed_name):
return config.get(dist, "metapackage_map/%s" % seed_name)
else:
task_meta_re = re.compile('^Task-Metapackage:\s*(.*)', re.I)
with structure[seed_name] as seed:
for line in seed:
task_meta_match = task_meta_re.match(line)
if task_meta_match is not None:
return task_meta_match.group(1)
return "%s-%s" % (metapackage, seed_name)
debootstrap_version_file = 'debootstrap-version'
def get_debootstrap_version():
version_cmd = subprocess.Popen(
['dpkg-query', '-W', '--showformat', '${Version}', 'debootstrap'],
stdout=subprocess.PIPE,
universal_newlines=True)
version, _ = version_cmd.communicate()
if not version:
error_exit('debootstrap does not appear to be installed')
return version
def debootstrap_packages(arch):
env = dict(os.environ)
if 'PATH' in env:
env['PATH'] = '/usr/sbin:/sbin:%s' % env['PATH']
else:
env['PATH'] = '/usr/sbin:/sbin:/usr/bin:/bin'
debootstrap = subprocess.Popen([
'debootstrap', '--arch', arch, '--components',
','.join(components), '--print-debs', dist, 'debootstrap-dir',
archive_base[arch][0]
],
stdout=subprocess.PIPE,
env=env,
stderr=subprocess.PIPE,
universal_newlines=True)
(debootstrap_stdout, debootstrap_stderr) = debootstrap.communicate()
if debootstrap.returncode != 0:
error_exit('Unable to retrieve package list from debootstrap; '
'stdout: %s\nstderr: %s' %
(debootstrap_stdout, debootstrap_stderr))
# sometimes debootstrap gives empty packages / multiple separators
packages = [pkg for pkg in debootstrap_stdout.split() if pkg]
return sorted(packages)
def check_debootstrap_version():
if os.path.exists(debootstrap_version_file):
with open(debootstrap_version_file) as debootstrap:
old_debootstrap_version = debootstrap.read().strip()
debootstrap_version = get_debootstrap_version()
failed = subprocess.call([
'dpkg', '--compare-versions', debootstrap_version, 'ge',
old_debootstrap_version
])
if failed:
error_exit('Installed debootstrap is older than in the '
'previous version! (%s < %s)' %
(debootstrap_version, old_debootstrap_version))
def update_debootstrap_version():
with open(debootstrap_version_file, 'w') as debootstrap:
debootstrap.write(get_debootstrap_version() + '\n')
def format_changes(items):
by_arch = defaultdict(set)
for pkg, arch in items:
by_arch[pkg].add(arch)
all_pkgs = sorted(by_arch)
chunks = []
for pkg in all_pkgs:
arches = by_arch[pkg]
if set(architectures) - arches:
# only some architectures
chunks.append('%s [%s]' % (pkg, ' '.join(sorted(arches))))
else:
# all architectures
chunks.append(pkg)
return ', '.join(chunks)
germinate_logging(logging.DEBUG)
check_debootstrap_version()
additions = defaultdict(list)
removals = defaultdict(list)
moves = defaultdict(list)
metapackage_map = {}
for architecture in architectures:
print("[%s] Downloading available package lists..." % architecture)
germinator = Germinator(architecture)
archive = germinate.archive.TagFile(
dists,
components,
architecture,
archive_base[architecture],
source_mirrors=archive_base_default,
cleanup=True)
germinator.parse_archive(archive)
debootstrap_base = set(debootstrap_packages(architecture))
print("[%s] Loading seed lists..." % architecture)
try:
structure = SeedStructure(seed_dist, seed_base, options.vcs)
germinator.plant_seeds(structure, seeds=seeds)
except SeedError:
sys.exit(1)
print("[%s] Merging seeds with available package lists..." %
architecture)
for seed_name in output_seeds:
meta_name = metapackage_name(structure, seed_name)
metapackage_map[seed_name] = meta_name
output_filename = os.path.join(options.outdir,
'%s-%s' % (seed_name, architecture))
old_list = None
if os.path.exists(output_filename):
with open(output_filename) as output:
old_list = set(map(str.strip, output.readlines()))
os.rename(output_filename, output_filename + '.old')
# work on the depends
new_list = []
packages = seed_packages(germinator.get_seed_entries, structure,
seed_name)
for package in packages:
if package == meta_name:
print("%s/%s: Skipping package %s (metapackage)" %
(seed_name, architecture, package))
elif (seed_name == 'minimal'
and package not in debootstrap_base):
print("%s/%s: Skipping package %s (package not in "
"debootstrap)" % (seed_name, architecture, package))
elif germinator.is_essential(package):
print("%s/%s: Skipping package %s (essential)" %
(seed_name, architecture, package))
else:
new_list.append(package)
new_list.sort()
with open(output_filename, 'w') as output:
for package in new_list:
output.write(package)
output.write('\n')
# work on the recommends
old_recommends_list = None
new_recommends_list = []
packages = seed_packages(germinator.get_seed_recommends_entries,
structure, seed_name)
for package in packages:
if package == meta_name:
print("%s/%s: Skipping package %s (metapackage)" %
(seed_name, architecture, package))
continue
if seed_name == 'minimal' and package not in debootstrap_base:
print("%s/%s: Skipping package %s (package not in "
"debootstrap)" % (seed_name, architecture, package))
else:
new_recommends_list.append(package)
new_recommends_list.sort()
seed_name_recommends = '%s-recommends' % seed_name
output_recommends_filename = os.path.join(
options.outdir, '%s-%s' % (seed_name_recommends, architecture))
if os.path.exists(output_recommends_filename):
with open(output_recommends_filename) as output:
old_recommends_list = set(
map(str.strip, output.readlines()))
os.rename(output_recommends_filename,
output_recommends_filename + '.old')
with open(output_recommends_filename, 'w') as output:
for package in new_recommends_list:
output.write(package)
output.write('\n')
# Calculate deltas
merged = defaultdict(int)
recommends_merged = defaultdict(int)
if old_list is not None:
for package in new_list:
merged[package] += 1
for package in old_list:
merged[package] -= 1
if old_recommends_list is not None:
for package in new_recommends_list:
recommends_merged[package] += 1
for package in old_recommends_list:
recommends_merged[package] -= 1
mergeditems = sorted(merged.items())
for package, value in mergeditems:
#print(package, value)
if value == 1:
if recommends_merged.get(package, 0) == -1:
moves[package].append([seed_name, architecture])
recommends_merged[package] += 1
else:
additions[package].append([seed_name, architecture])
elif value == -1:
if recommends_merged.get(package, 0) == 1:
moves[package].append(
[seed_name_recommends, architecture])
recommends_merged[package] -= 1
else:
removals[package].append([seed_name, architecture])
mergedrecitems = sorted(recommends_merged.items())
for package, value in mergedrecitems:
#print(package, value)
if value == 1:
additions[package].append(
[seed_name_recommends, architecture])
elif value == -1:
removals[package].append(
[seed_name_recommends, architecture])
with open('metapackage-map', 'w') as metapackage_map_file:
for seed_name in output_seeds:
print(seed_name,
metapackage_map[seed_name],
file=metapackage_map_file)
if not options.nodch and (additions or removals or moves):
dch_help = subprocess.Popen(['dch', '--help'],
stdout=subprocess.PIPE,
universal_newlines=True)
try:
have_U = '-U' in dch_help.stdout.read()
finally:
if dch_help.stdout:
dch_help.stdout.close()
dch_help.wait()
if have_U:
subprocess.check_call(['dch', '-iU', 'Refreshed dependencies'])
else:
subprocess.check_call(['dch', '-i', 'Refreshed dependencies'])
changes = []
for package in sorted(additions):
changes.append('Added %s to %s' %
(package, format_changes(additions[package])))
for package in sorted(removals):
changes.append('Removed %s from %s' %
(package, format_changes(removals[package])))
for package in sorted(moves):
# TODO: We should really list where it moved from as well, but
# that gets wordy very quickly, and at the moment this is only
# implemented for depends->recommends or vice versa. In future,
# using this for moves between seeds might also be useful.
changes.append('Moved %s to %s' %
(package, format_changes(moves[package])))
for change in changes:
print(change)
subprocess.check_call(['dch', '-a', change])
update_debootstrap_version()
else:
if not options.nodch:
print("No changes found")
return 0
def test_versioned_provides(self):
"""Germinator.parse_archive resolves versioned provides."""
self.addSource("bionic",
"main",
"hello",
"1.0-1", ["hello", "hello-dependency", "hello-bad"],
fields={"Maintainer": "Test Person <*****@*****.**>"})
self.addPackage("bionic",
"main",
"i386",
"hello",
"1.0-1",
fields={
"Maintainer": "Test Person <*****@*****.**>",
"Depends": "hello-virtual (>= 2.0)",
})
self.addPackage("bionic",
"main",
"i386",
"hello-bad",
"2.0-1",
fields={
"Source": "hello (= 1.0-1)",
"Provides": "hello-virtual (= 1.0)"
})
self.addPackage("bionic",
"main",
"i386",
"hello-dependency",
"1.0-1",
fields={
"Source": "hello",
"Provides": "hello-virtual (= 2.0)"
})
branch = "ubuntu.bionic"
self.addSeed(branch, "supported")
self.addSeedPackage(branch, "supported", "hello")
germinator = Germinator("i386")
archive = TagFile("bionic", "main", "i386",
"file://%s" % self.archive_dir)
germinator.parse_archive(archive)
self.assertIn("hello", germinator._sources)
self.assertIn("hello", germinator._packages)
self.assertEqual("deb", germinator._packagetype["hello"])
self.assertIn("hello-dependency", germinator._packages)
self.assertEqual("deb", germinator._packagetype["hello-dependency"])
self.assertIn("hello-bad", germinator._packages)
self.assertEqual("deb", germinator._packagetype["hello-bad"])
self.assertEqual(
{"hello-virtual": {
"hello-bad": "1.0",
"hello-dependency": "2.0"
}}, germinator._provides)
structure = self.openSeedStructure(branch)
germinator.plant_seeds(structure)
germinator.grow(structure)
expected = set(["hello-dependency"])
self.assertEqual(expected,
germinator.get_depends(structure, "supported"))
def main(argv):
options = parse_options(argv)
if options.verbose:
germinate_logging(logging.DEBUG)
else:
germinate_logging(logging.INFO)
g = Germinator(options.arch)
archive = germinate.archive.TagFile(
options.dist, options.components, options.arch,
options.mirrors, source_mirrors=options.source_mirrors,
installer_packages=options.installer, cleanup=options.cleanup)
g.parse_archive(archive)
if os.path.isfile("hints"):
with open("hints") as hints:
g.parse_hints(hints)
try:
structure = SeedStructure(options.release, options.seeds, options.bzr)
for seed_package in options.seed_packages:
parent, pkg = seed_package.split('/')
structure.add(pkg, [" * " + pkg], parent)
g.plant_seeds(structure)
except SeedError:
sys.exit(1)
try:
with Seed(options.seeds, options.release, "blacklist",
options.bzr) as blacklist:
g.parse_blacklist(structure, blacklist)
except SeedError:
pass
g.grow(structure)
g.add_extras(structure)
if options.want_rdepends:
g.reverse_depends(structure)
for seedname in structure.names + ["extra"]:
g.write_full_list(structure, seedname, seedname)
g.write_seed_list(structure, seedname + ".seed", seedname)
g.write_seed_recommends_list(structure,
seedname + ".seed-recommends", seedname)
g.write_depends_list(structure, seedname + ".depends", seedname)
g.write_build_depends_list(structure,
seedname + ".build-depends", seedname)
if seedname != "extra" and seedname in structure:
structure.write_seed_text(seedname + ".seedtext", seedname)
g.write_sources_list(structure, seedname + ".sources", seedname)
g.write_build_sources_list(structure,
seedname + ".build-sources", seedname)
g.write_all_list(structure, "all")
g.write_all_source_list(structure, "all.sources")
g.write_supported_list(structure, "%s+build-depends" % structure.supported)
g.write_supported_source_list(
structure, "%s+build-depends.sources" % structure.supported)
g.write_all_extra_list(structure, "all+extra")
g.write_all_extra_source_list(structure, "all+extra.sources")
g.write_provides_list(structure, "provides")
structure.write("structure")
structure.write_dot("structure.dot")
if os.path.exists("rdepends"):
shutil.rmtree("rdepends")
if options.want_rdepends:
os.mkdir("rdepends")
os.mkdir(os.path.join("rdepends", "ALL"))
for pkg in g.get_all(structure):
dirname = os.path.join("rdepends", g.get_source(pkg))
if not os.path.exists(dirname):
os.mkdir(dirname)
g.write_rdepend_list(structure, os.path.join(dirname, pkg), pkg)
os.symlink(os.path.join("..", g.get_source(pkg), pkg),
os.path.join("rdepends", "ALL", pkg))
g.write_blacklisted(structure, "blacklisted")
return 0
def main(argv):
options = parse_options(argv)
if options.verbose:
germinate_logging(logging.DEBUG)
else:
germinate_logging(logging.INFO)
g = Germinator(options.arch)
g._always_follow_build_depends = options.always_follow_build_depends
archive = germinate.archive.TagFile(options.dist,
options.components,
options.arch,
options.mirrors,
source_mirrors=options.source_mirrors,
installer_packages=options.installer,
cleanup=options.cleanup)
g.parse_archive(archive)
if os.path.isfile("hints"):
with open("hints") as hints:
g.parse_hints(hints)
try:
structure = SeedStructure(options.release, options.seeds, options.vcs)
for seed_package in options.seed_packages:
parent, pkg = seed_package.split('/')
structure.add(pkg, [" * " + pkg], parent)
g.plant_seeds(structure)
except SeedError:
sys.exit(1)
try:
with Seed(options.seeds, options.release, "blacklist",
options.vcs) as blacklist:
g.parse_blacklist(structure, blacklist)
except SeedError:
pass
g.grow(structure)
g.add_extras(structure)
if options.want_rdepends:
g.reverse_depends(structure)
for seedname in structure.names + ["extra"]:
g.write_full_list(structure, seedname, seedname)
g.write_seed_list(structure, seedname + ".seed", seedname)
g.write_seed_recommends_list(structure, seedname + ".seed-recommends",
seedname)
g.write_depends_list(structure, seedname + ".depends", seedname)
g.write_build_depends_list(structure, seedname + ".build-depends",
seedname)
g.write_snap_list(structure, seedname + ".snaps", seedname)
if seedname != "extra" and seedname in structure:
structure.write_seed_text(seedname + ".seedtext", seedname)
g.write_sources_list(structure, seedname + ".sources", seedname)
g.write_build_sources_list(structure, seedname + ".build-sources",
seedname)
g.write_all_list(structure, "all")
g.write_all_source_list(structure, "all.sources")
g.write_all_snap_list(structure, "all.snaps")
g.write_supported_list(structure, "%s+build-depends" % structure.supported)
g.write_supported_source_list(
structure, "%s+build-depends.sources" % structure.supported)
g.write_all_extra_list(structure, "all+extra")
g.write_all_extra_source_list(structure, "all+extra.sources")
g.write_provides_list(structure, "provides")
structure.write("structure")
structure.write_dot("structure.dot")
if os.path.exists("rdepends"):
shutil.rmtree("rdepends")
if options.want_rdepends:
os.mkdir("rdepends")
os.mkdir(os.path.join("rdepends", "ALL"))
for pkg in g.get_all(structure):
dirname = os.path.join("rdepends", g.get_source(pkg))
if not os.path.exists(dirname):
os.mkdir(dirname)
g.write_rdepend_list(structure, os.path.join(dirname, pkg), pkg)
os.symlink(os.path.join("..", g.get_source(pkg), pkg),
os.path.join("rdepends", "ALL", pkg))
g.write_blacklisted(structure, "blacklisted")
return 0
def test_parse_archive(self):
"""Germinator.parse_archive successfully parses a simple archive."""
self.addSource("warty",
"main",
"hello",
"1.0-1", ["hello", "hello-dependency"],
fields={"Maintainer": "Test Person <*****@*****.**>"})
self.addPackage("warty",
"main",
"i386",
"hello",
"1.0-1",
fields={
"Maintainer": "Test Person <*****@*****.**>",
"Depends": "hello-dependency",
})
self.addPackage("warty",
"main",
"i386",
"hello-dependency",
"1.0-1",
fields={
"Source": "hello",
"Multi-Arch": "foreign"
})
self.addSeed("ubuntu.warty", "supported")
self.addSeedPackage("ubuntu.warty", "supported", "hello")
germinator = Germinator("i386")
archive = TagFile("warty", "main", "i386",
"file://%s" % self.archive_dir)
germinator.parse_archive(archive)
self.assertIn("hello", germinator._sources)
self.assertEqual(
{
"Maintainer": u"Test Person <*****@*****.**>",
"Version": "1.0-1",
"Build-Depends": [],
"Build-Depends-Indep": [],
"Build-Depends-Arch": [],
"Binaries": ["hello", "hello-dependency"],
}, germinator._sources["hello"])
self.assertIn("hello", germinator._packages)
self.assertEqual(
{
"Section": "",
"Version": "1.0-1",
"Maintainer": u"Test Person <*****@*****.**>",
"Essential": "",
"Pre-Depends": [],
"Built-Using": [],
"Depends": [[("hello-dependency", "", "")]],
"Recommends": [],
"Size": 0,
"Installed-Size": 0,
"Source": "hello",
"Provides": [],
"Kernel-Version": "",
"Multi-Arch": "none",
}, germinator._packages["hello"])
self.assertEqual("deb", germinator._packagetype["hello"])
self.assertIn("hello-dependency", germinator._packages)
self.assertEqual(
{
"Section": "",
"Version": "1.0-1",
"Maintainer": u"",
"Essential": "",
"Pre-Depends": [],
"Built-Using": [],
"Depends": [],
"Recommends": [],
"Size": 0,
"Installed-Size": 0,
"Source": "hello",
"Provides": [],
"Kernel-Version": "",
"Multi-Arch": "foreign",
}, germinator._packages["hello-dependency"])
self.assertEqual("deb", germinator._packagetype["hello-dependency"])
self.assertEqual({}, germinator._provides)
def main(argv):
options, args = parse_options(argv)
if not os.path.exists('debian/control'):
error_exit('must be run from the top level of a source package')
this_source = None
with open('debian/control') as control:
for line in control:
if line.startswith('Source:'):
this_source = line[7:].strip()
break
elif line == '':
break
if this_source is None:
error_exit('cannot find Source: in debian/control')
if not this_source.endswith('-meta'):
error_exit('source package name must be *-meta')
metapackage = this_source[:-5]
print("[info] Initialising %s-* package lists update..." % metapackage)
config = SafeConfigParser()
with open('update.cfg') as config_file:
try:
# >= 3.2
config.read_file(config_file)
except AttributeError:
# < 3.2
config.readfp(config_file)
if len(args) > 0:
dist = args[0]
else:
dist = config.get('DEFAULT', 'dist')
seeds = config.get(dist, 'seeds').split()
try:
output_seeds = config.get(dist, 'output_seeds').split()
except NoOptionError:
output_seeds = list(seeds)
architectures = config.get(dist, 'architectures').split()
try:
archive_base_default = config.get(dist, 'archive_base/default')
archive_base_default = re.split(r'[, ]+', archive_base_default)
except (NoSectionError, NoOptionError):
archive_base_default = None
archive_base = {}
for arch in architectures:
try:
archive_base[arch] = config.get(dist, 'archive_base/%s' % arch)
archive_base[arch] = re.split(r'[, ]+', archive_base[arch])
except (NoSectionError, NoOptionError):
if archive_base_default is not None:
archive_base[arch] = archive_base_default
else:
error_exit('no archive_base configured for %s' % arch)
if options.bzr and config.has_option("%s/bzr" % dist, 'seed_base'):
seed_base = config.get("%s/bzr" % dist, 'seed_base')
else:
seed_base = config.get(dist, 'seed_base')
seed_base = re.split(r'[, ]+', seed_base)
if options.bzr and config.has_option("%s/bzr" % dist, 'seed_dist'):
seed_dist = config.get("%s/bzr" % dist, 'seed_dist')
elif config.has_option(dist, 'seed_dist'):
seed_dist = config.get(dist, 'seed_dist')
else:
seed_dist = dist
if config.has_option(dist, 'dists'):
dists = config.get(dist, 'dists').split()
else:
dists = [dist]
try:
archive_exceptions = config.get(dist, 'archive_base/exceptions').split()
except (NoSectionError, NoOptionError):
archive_exceptions = []
components = config.get(dist, 'components').split()
def seed_packages(germinator_method, structure, seed_name):
if config.has_option(dist, "seed_map/%s" % seed_name):
mapped_seeds = config.get(dist, "seed_map/%s" % seed_name).split()
else:
mapped_seeds = []
task_seeds_re = re.compile('^Task-Seeds:\s*(.*)', re.I)
with structure[seed_name] as seed:
for line in seed:
task_seeds_match = task_seeds_re.match(line)
if task_seeds_match is not None:
mapped_seeds = task_seeds_match.group(1).split()
break
if seed_name not in mapped_seeds:
mapped_seeds.append(seed_name)
packages = []
for mapped_seed in mapped_seeds:
packages.extend(germinator_method(structure, mapped_seed))
return packages
def metapackage_name(structure, seed_name):
if config.has_option(dist, "metapackage_map/%s" % seed_name):
return config.get(dist, "metapackage_map/%s" % seed_name)
else:
task_meta_re = re.compile('^Task-Metapackage:\s*(.*)', re.I)
with structure[seed_name] as seed:
for line in seed:
task_meta_match = task_meta_re.match(line)
if task_meta_match is not None:
return task_meta_match.group(1)
return "%s-%s" % (metapackage, seed_name)
debootstrap_version_file = 'debootstrap-version'
def get_debootstrap_version():
version_cmd = subprocess.Popen(
['dpkg-query', '-W', '--showformat', '${Version}', 'debootstrap'],
stdout=subprocess.PIPE, universal_newlines=True)
version, _ = version_cmd.communicate()
if not version:
error_exit('debootstrap does not appear to be installed')
return version
def debootstrap_packages(arch):
env = dict(os.environ)
if 'PATH' in env:
env['PATH'] = '/usr/sbin:/sbin:%s' % env['PATH']
else:
env['PATH'] = '/usr/sbin:/sbin:/usr/bin:/bin'
debootstrap = subprocess.Popen(
['debootstrap', '--arch', arch,
'--components', ','.join(components),
'--print-debs', dist, 'debootstrap-dir', archive_base[arch][0]],
stdout=subprocess.PIPE, env=env, stderr=subprocess.PIPE,
universal_newlines=True)
(debootstrap_stdout, debootstrap_stderr) = debootstrap.communicate()
if debootstrap.returncode != 0:
error_exit('Unable to retrieve package list from debootstrap; '
'stdout: %s\nstderr: %s' %
(debootstrap_stdout, debootstrap_stderr))
# sometimes debootstrap gives empty packages / multiple separators
packages = [pkg for pkg in debootstrap_stdout.split() if pkg]
return sorted(packages)
def check_debootstrap_version():
if os.path.exists(debootstrap_version_file):
with open(debootstrap_version_file) as debootstrap:
old_debootstrap_version = debootstrap.read().strip()
debootstrap_version = get_debootstrap_version()
failed = subprocess.call(
['dpkg', '--compare-versions',
debootstrap_version, 'ge', old_debootstrap_version])
if failed:
error_exit('Installed debootstrap is older than in the '
'previous version! (%s < %s)' %
(debootstrap_version, old_debootstrap_version))
def update_debootstrap_version():
with open(debootstrap_version_file, 'w') as debootstrap:
debootstrap.write(get_debootstrap_version() + '\n')
def format_changes(items):
by_arch = defaultdict(set)
for pkg, arch in items:
by_arch[pkg].add(arch)
all_pkgs = sorted(by_arch)
chunks = []
for pkg in all_pkgs:
arches = by_arch[pkg]
if set(architectures) - arches:
# only some architectures
chunks.append('%s [%s]' % (pkg, ' '.join(sorted(arches))))
else:
# all architectures
chunks.append(pkg)
return ', '.join(chunks)
germinate_logging(logging.DEBUG)
check_debootstrap_version()
additions = defaultdict(list)
removals = defaultdict(list)
moves = defaultdict(list)
metapackage_map = {}
for architecture in architectures:
print("[%s] Downloading available package lists..." % architecture)
germinator = Germinator(architecture)
archive = germinate.archive.TagFile(
dists, components, architecture,
archive_base[architecture], source_mirrors=archive_base_default,
cleanup=True, archive_exceptions=archive_exceptions)
germinator.parse_archive(archive)
debootstrap_base = set(debootstrap_packages(architecture))
print("[%s] Loading seed lists..." % architecture)
try:
structure = SeedStructure(seed_dist, seed_base, options.bzr)
germinator.plant_seeds(structure, seeds=seeds)
except SeedError:
sys.exit(1)
print("[%s] Merging seeds with available package lists..." %
architecture)
for seed_name in output_seeds:
meta_name = metapackage_name(structure, seed_name)
metapackage_map[seed_name] = meta_name
output_filename = os.path.join(
options.outdir, '%s-%s' % (seed_name, architecture))
old_list = None
if os.path.exists(output_filename):
with open(output_filename) as output:
old_list = set(map(str.strip, output.readlines()))
os.rename(output_filename, output_filename + '.old')
# work on the depends
new_list = []
packages = seed_packages(germinator.get_seed_entries,
structure, seed_name)
for package in packages:
if package == meta_name:
print("%s/%s: Skipping package %s (metapackage)" %
(seed_name, architecture, package))
elif (seed_name == 'minimal' and
package not in debootstrap_base):
print("%s/%s: Skipping package %s (package not in "
"debootstrap)" % (seed_name, architecture, package))
elif germinator.is_essential(package):
print("%s/%s: Skipping package %s (essential)" %
(seed_name, architecture, package))
else:
new_list.append(package)
new_list.sort()
with open(output_filename, 'w') as output:
for package in new_list:
output.write(package)
output.write('\n')
# work on the recommends
old_recommends_list = None
new_recommends_list = []
packages = seed_packages(germinator.get_seed_recommends_entries,
structure, seed_name)
for package in packages:
if package == meta_name:
print("%s/%s: Skipping package %s (metapackage)" %
(seed_name, architecture, package))
continue
if seed_name == 'minimal' and package not in debootstrap_base:
print("%s/%s: Skipping package %s (package not in "
"debootstrap)" % (seed_name, architecture, package))
else:
new_recommends_list.append(package)
new_recommends_list.sort()
seed_name_recommends = '%s-recommends' % seed_name
output_recommends_filename = os.path.join(
options.outdir, '%s-%s' % (seed_name_recommends, architecture))
if os.path.exists(output_recommends_filename):
with open(output_recommends_filename) as output:
old_recommends_list = set(
map(str.strip, output.readlines()))
os.rename(
output_recommends_filename,
output_recommends_filename + '.old')
with open(output_recommends_filename, 'w') as output:
for package in new_recommends_list:
output.write(package)
output.write('\n')
# Calculate deltas
merged = defaultdict(int)
recommends_merged = defaultdict(int)
if old_list is not None:
for package in new_list:
merged[package] += 1
for package in old_list:
merged[package] -= 1
if old_recommends_list is not None:
for package in new_recommends_list:
recommends_merged[package] += 1
for package in old_recommends_list:
recommends_merged[package] -= 1
mergeditems = sorted(merged.items())
for package, value in mergeditems:
#print(package, value)
if value == 1:
if recommends_merged.get(package, 0) == -1:
moves[package].append([seed_name, architecture])
recommends_merged[package] += 1
else:
additions[package].append([seed_name, architecture])
elif value == -1:
if recommends_merged.get(package, 0) == 1:
moves[package].append([seed_name_recommends,
architecture])
recommends_merged[package] -= 1
else:
removals[package].append([seed_name, architecture])
mergedrecitems = sorted(recommends_merged.items())
for package, value in mergedrecitems:
#print(package, value)
if value == 1:
additions[package].append([seed_name_recommends,
architecture])
elif value == -1:
removals[package].append([seed_name_recommends,
architecture])
with open('metapackage-map', 'w') as metapackage_map_file:
for seed_name in output_seeds:
print(seed_name, metapackage_map[seed_name],
file=metapackage_map_file)
if not options.nodch and (additions or removals or moves):
dch_help = subprocess.Popen(['dch', '--help'], stdout=subprocess.PIPE,
universal_newlines=True)
try:
have_U = '-U' in dch_help.stdout.read()
finally:
if dch_help.stdout:
dch_help.stdout.close()
dch_help.wait()
if have_U:
subprocess.check_call(['dch', '-iU', 'Refreshed dependencies'])
else:
subprocess.check_call(['dch', '-i', 'Refreshed dependencies'])
changes = []
for package in sorted(additions):
changes.append('Added %s to %s' %
(package, format_changes(additions[package])))
for package in sorted(removals):
changes.append('Removed %s from %s' %
(package, format_changes(removals[package])))
for package in sorted(moves):
# TODO: We should really list where it moved from as well, but
# that gets wordy very quickly, and at the moment this is only
# implemented for depends->recommends or vice versa. In future,
# using this for moves between seeds might also be useful.
changes.append('Moved %s to %s' %
(package, format_changes(moves[package])))
for change in changes:
print(change)
subprocess.check_call(['dch', '-a', change])
update_debootstrap_version()
else:
if not options.nodch:
print("No changes found")
return 0