def test_add(self):
entry_path = os.path.join(self.temp_dir, "entry")
data = b"test\n"
with mkfile(entry_path, mode="wb") as entry:
entry.write(data)
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
checksum_file.add("entry")
self.assertEqual({"entry": hashlib.md5(data).hexdigest()},
checksum_file.entries)
def test_add_updated_mtime(self):
# Adding an existing file with an mtime newer than that of the
# checksums file causes its checksum to be updated.
path = os.path.join(self.temp_dir, "entry")
with mkfile(path) as entry:
pass
checksum_file = ChecksumFile(self.config,
self.temp_dir,
"MD5SUMS",
hashlib.md5,
sign=False)
checksum_file.add("entry")
checksum_file.write()
self.rewind_mtime(checksum_file.path)
with mkfile(path) as entry:
print("mtime", end="", file=entry)
checksum_file.add("entry")
self.assertEqual(
hashlib.md5(b"mtime").hexdigest(), checksum_file.entries["entry"])
def test_select_proxy(self):
self.assertIsNone(_select_proxy(self.config, "any-caller"))
with mkfile(self.config_path) as f:
print("test1\thttp://foo.example.org:3128/", file=f)
print("test2\thttp://bar.example.org:3128/", file=f)
self.assertEqual("http://foo.example.org:3128/",
_select_proxy(self.config, "test1"))
self.assertEqual("http://bar.example.org:3128/",
_select_proxy(self.config, "test2"))
self.assertIsNone(_select_proxy(self.config, "other-caller"))
def test_check_manifest_unknown_file(self):
config = Config(read=False)
config.root = self.use_temp_dir()
manifest = os.path.join(self.temp_dir, "www", "simple", ".manifest")
with mkfile(manifest) as f:
print(
"ubuntu\tprecise\t/precise/ubuntu-12.04.2-desktop-i386.iso\t"
"726970368",
file=f)
self.assertRaises(UnknownManifestFile, check_manifest, config)
def test_remove(self):
entry_path = os.path.join(self.temp_dir, "entry")
data = "test\n"
with mkfile(entry_path) as entry:
print(data, end="", file=entry)
self.create_checksum_files(["entry"])
checksum_files = self.cls(self.config, self.temp_dir)
checksum_files.read()
checksum_files.remove("entry")
self.assertChecksumsEqual({}, checksum_files)
def test_context_manager(self):
for name in "1", "2":
entry_path = os.path.join(self.temp_dir, name)
with mkfile(entry_path) as entry:
print(name, end="", file=entry)
md5sums_path = os.path.join(self.temp_dir, "MD5SUMS")
with mkfile(md5sums_path) as md5sums:
subprocess.call(["md5sum", "-b", "1", "2"],
stdout=md5sums,
cwd=self.temp_dir)
with ChecksumFile(self.config,
self.temp_dir,
"MD5SUMS",
hashlib.md5,
sign=False) as checksum_file:
self.assertCountEqual(["1", "2"], checksum_file.entries)
checksum_file.remove("1")
with open(md5sums_path) as md5sums:
self.assertEqual("%s *2\n" % hashlib.md5(b"2").hexdigest(),
md5sums.read())
def test_merge(self):
old_dir = os.path.join(self.temp_dir, "old")
os.mkdir(old_dir)
entry_path = os.path.join(self.temp_dir, "entry")
with mkfile(entry_path) as entry:
print("data", end="", file=entry)
shutil.copy(entry_path, os.path.join(old_dir, "entry"))
self.create_checksum_files(["entry"], directory=old_dir)
checksum_files = self.cls(self.config, self.temp_dir)
checksum_files.merge([old_dir], "entry", ["entry"])
self.assertChecksumsEqual({"entry": b"data"}, checksum_files)
def test_arches_override(self):
# If ARCHES is set in the environment, it overrides
# etc/default-arches.
os.environ["CDIMAGE_ROOT"] = self.use_temp_dir()
os.environ["ARCHES"] = "amd64"
os.environ.pop("CPUARCHES", None)
etc_dir = os.path.join(self.temp_dir, "etc")
with mkfile(os.path.join(etc_dir, "config")) as f:
print(dedent("""\
#! /bin/sh
PROJECT=ubuntu
DIST=raring
"""),
file=f)
with mkfile(os.path.join(etc_dir, "default-arches")) as f:
print("*\tdaily-live\traring\tamd64 amd64+mac i386", file=f)
config = Config(IMAGE_TYPE="daily-live")
self.assertEqual("daily-live", config["IMAGE_TYPE"])
self.assertEqual("amd64", config["ARCHES"])
self.assertEqual("amd64", config["CPUARCHES"])
def check_manifest_pass(self):
config = Config(read=False)
config.root = self.use_temp_dir()
manifest = os.path.join(self.temp_dir, "www", "simple", ".manifest")
with mkfile(manifest) as f:
print(
"ubuntu\tprecise\t/precise/ubuntu-12.04.2-desktop-i386.iso\t"
"726970368",
file=f)
touch(
os.path.join(self.temp_dir, "www", "simple", "precise",
"ubuntu-12.04.2-desktop-i386.iso"))
def test_merge_ignores_stale_checksums(self):
old_dir = os.path.join(self.temp_dir, "old")
with mkfile(os.path.join(old_dir, "MD5SUMS")) as old_md5sums:
print("checksum *entry", file=old_md5sums)
entry_path = os.path.join(self.temp_dir, "entry")
touch(entry_path)
next_minute = time.time() + 60
os.utime(entry_path, (next_minute, next_minute))
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
checksum_file.merge([old_dir], "entry", ["entry"])
self.assertEqual({}, checksum_file.entries)
def test_read_shell(self):
os.environ["CDIMAGE_ROOT"] = self.use_temp_dir()
with mkfile(os.path.join(self.temp_dir, "etc", "config")) as f:
print(dedent("""\
#! /bin/sh
PROJECT=ubuntu
CAPPROJECT=Ubuntu
"""),
file=f)
config = Config()
self.assertEqual("ubuntu", config["PROJECT"])
self.assertEqual("Ubuntu", config["CAPPROJECT"])
self.assertNotIn("DEBUG", config)
def send_mail_to_file(self, path, subject, generator, recipients, body,
dry_run=False):
with mkfile(path) as f:
print("To: %s" % ", ".join(recipients), file=f)
print("Subject: %s" % subject, file=f)
print("X-Generated-By: %s" % generator, file=f)
print("", file=f)
if isinstance(body, text_file_type):
for line in body:
print(line.rstrip("\n"), file=f)
else:
for line in body.splitlines():
print(line, file=f)
def test_add_existing(self):
# Attempting to add an existing file that is not newer than the
# checksums file has no effect. (Use .remove() first to overwrite
# an existing checksum.)
entry_path = os.path.join(self.temp_dir, "entry")
data = "test\n"
with mkfile(entry_path) as entry:
entry.write(data)
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
checksum_file.entries["entry"] = ""
checksum_file.add("entry")
self.assertEqual("", checksum_file.entries["entry"])
def test_context_manager(self):
for name in "1", "2":
entry_path = os.path.join(self.temp_dir, name)
with mkfile(entry_path) as entry:
print(name, end="", file=entry)
self.create_checksum_files(["1", "2"])
with self.cls(self.config, self.temp_dir,
sign=False) as checksum_files:
self.assertChecksumsEqual({"1": b"1", "2": b"2"}, checksum_files)
checksum_files.remove("1")
with open(os.path.join(self.temp_dir, "MD5SUMS-metalink")) as md5sums:
self.assertEqual("%s *2\n" % hashlib.md5(b"2").hexdigest(),
md5sums.read())
def test_merge_all(self):
old_dir = os.path.join(self.temp_dir, "old")
old_iso_i386_path = os.path.join(old_dir, "foo-i386.iso")
with mkfile(old_iso_i386_path) as old_iso_i386:
print("foo-i386.iso", end="", file=old_iso_i386)
old_metalink_i386_path = os.path.join(old_dir, "foo-i386.metalink")
with mkfile(old_metalink_i386_path) as old_metalink_i386:
print("foo-i386.metalink", end="", file=old_metalink_i386)
self.create_checksum_files(["foo-i386.metalink"], directory=old_dir)
metalink_amd64_path = os.path.join(self.temp_dir, "foo-amd64.metalink")
with mkfile(metalink_amd64_path) as metalink_amd64:
print("foo-amd64.metalink", end="", file=metalink_amd64)
touch(os.path.join(self.temp_dir, "foo-amd64.list"))
shutil.copy(old_metalink_i386_path,
os.path.join(self.temp_dir, "foo-i386.metalink"))
checksum_files = self.cls(self.config, self.temp_dir)
checksum_files.merge_all([old_dir])
self.assertChecksumsEqual(
{
"foo-amd64.metalink": b"foo-amd64.metalink",
"foo-i386.metalink": b"foo-i386.metalink",
}, checksum_files)
def test_read_shell_config(self):
os.environ["ONE"] = "one"
config_path = os.path.join(self.temp_dir, "config")
with mkfile(config_path) as config:
print(dedent("""\
ONE="$ONE two three"
TWO=two
THREE=three"""),
file=config)
config_dict = dict(
osextras.read_shell_config(config_path, ["ONE", "TWO"]))
self.assertEqual("one two three", config_dict["ONE"])
self.assertEqual("two", config_dict["TWO"])
self.assertNotIn("three", config_dict)
def test_read(self):
with mkfile(os.path.join(self.temp_dir, "MD5SUMS")) as md5sums:
print(dedent("""\
checksum one-path
checksum *another-path
"""),
file=md5sums)
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
checksum_file.read()
self.assertEqual({
"one-path": "checksum",
"another-path": "checksum"
}, checksum_file.entries)
def test_write(self):
checksum_files = self.cls(self.config, self.temp_dir, sign=False)
for name in "1", "2":
entry_path = os.path.join(self.temp_dir, name)
with mkfile(entry_path) as entry:
print(name, end="", file=entry)
checksum_files.add(name)
checksum_files.write()
for cf in checksum_files.checksum_files:
self.assertEqual(
0,
subprocess.call([
self.files_and_commands[cf.name], "-c", "--status", cf.name
],
cwd=self.temp_dir))
def test_update_tasks_sends_mail(self, mock_send_mail):
original_call = subprocess.call
def call_side_effect(command, *args, **kwargs):
if (len(command) >= 4 and command[:2] == ["diff", "-u"] and
"stdout" in kwargs):
old = os.path.basename(command[2])
new = os.path.basename(command[3])
original_call(
["printf", "%s\\n", "--- %s" % old], *args, **kwargs)
original_call(
["printf", "%s\\n", "+++ %s" % new], *args, **kwargs)
return 1
else:
return original_call(command, *args, **kwargs)
self.write_ubuntu_structure()
self.config["PROJECT"] = "ubuntu"
self.config["CAPPROJECT"] = "Ubuntu"
self.config["DIST"] = "raring"
self.config["IMAGE_TYPE"] = "daily-live"
output_dir = os.path.join(
self.temp_dir, "scratch", "ubuntu", "raring", "daily-live",
"tasks")
touch(os.path.join(output_dir, "required"))
touch(os.path.join(output_dir, "minimal"))
touch(os.path.join(output_dir, "standard"))
touch(os.path.join("%s-previous" % output_dir, "minimal"))
touch(os.path.join("%s-previous" % output_dir, "standard"))
task_mail_path = os.path.join(self.temp_dir, "etc", "task-mail")
with mkfile(task_mail_path) as task_mail:
print("*****@*****.**", file=task_mail)
mock_send_mail.side_effect = partial(
self.send_mail_to_file, os.path.join(self.temp_dir, "mail"))
output = GerminateOutput(self.config, self.temp_dir)
with mock.patch("subprocess.call", side_effect=call_side_effect):
output.update_tasks("20130319")
with open(os.path.join(self.temp_dir, "mail")) as mail:
self.assertEqual(dedent("""\
To: [email protected]
Subject: Task changes for Ubuntu daily-live/raring on 20130319
X-Generated-By: update-tasks
--- minimal
+++ minimal
--- standard
+++ standard
"""), mail.read())
def test_send_mail_from_file(self, mock_popen):
path = os.path.join(self.temp_dir, "body")
with mkfile(path) as body:
print("Body", file=body)
print("Text", file=body)
with open(path) as body:
send_mail("Test subject", "test_notify", ["*****@*****.**"], body)
expected_command = [
"mail",
"-s",
"Test subject",
"-a",
"X-Generated-By: test_notify",
"*****@*****.**",
]
mock_popen.assert_called_once_with(expected_command, stdin=body)
def test_get_mirrors(self):
config = Config(read=False)
config.root = self.use_temp_dir()
production_path = os.path.join(self.temp_dir, "production",
"trigger-mirrors")
os.makedirs(os.path.dirname(production_path))
with mkfile(production_path) as production:
print("sync x.example.org", file=production)
print("async other.example.org", file=production)
print("sync y.example.org z.example.org", file=production)
self.assertEqual(["x.example.org", "y.example.org", "z.example.org"],
_get_mirrors(config))
self.configure_triggers()
self.assertEqual(["foo", "bar"], _get_mirrors(self.config))
self.config["UBUNTU_DEFAULTS_LOCALE"] = "zh_CN"
self.assertEqual(["strix.canonical.com"], _get_mirrors(self.config))
def test_common_initrd_packages(self):
self.write_ubuntu_structure()
manifest_path = os.path.join(
self.temp_dir, "ftp", "dists", "raring", "main", "installer-i386",
"current", "images", "MANIFEST.udebs")
with mkfile(manifest_path) as manifest:
print(dedent("""\
cdrom/initrd.gz
\tanna 1.45ubuntu1 i386
\tcdrom-detect 1.43ubuntu1 all
netboot/netboot.tar.gz
\tanna 1.45ubuntu1 i386
\tnet-retriever 1.32ubuntu1 i386"""), file=manifest)
self.config["DIST"] = "raring"
output = GerminateOutput(self.config, self.temp_dir)
self.assertEqual(set(["anna"]), output.common_initrd_packages("i386"))
def test_get_mirrors_async(self):
config = Config(read=False)
config.root = self.use_temp_dir()
production_path = os.path.join(self.temp_dir, "production",
"trigger-mirrors")
with mkfile(production_path) as production:
print("sync x.example.org", file=production)
print("async a.example.org b.example.org", file=production)
print("sync y.example.org z.example.org", file=production)
print("async c.example.org", file=production)
self.assertEqual(["a.example.org", "b.example.org", "c.example.org"],
_get_mirrors_async(config))
self.configure_triggers()
self.assertEqual(["foo-async", "bar-async"],
_get_mirrors_async(self.config))
self.config["UBUNTU_DEFAULTS_LOCALE"] = "zh_CN"
self.assertEqual([], _get_mirrors_async(self.config))
def test_task_headers(self):
self.write_ubuntu_structure()
seedtext_path = os.path.join(self.temp_dir, "i386", "desktop.seedtext")
with mkfile(seedtext_path) as seedtext:
print(dedent("""\
Task-Per-Derivative: 1
Task-Key: ubuntu-desktop
Task-Seeds: desktop-common
= Seed text starts here ="""), file=seedtext)
output = GerminateOutput(self.config, self.temp_dir)
expected = {
"per-derivative": "1",
"key": "ubuntu-desktop",
"seeds": "desktop-common",
}
self.assertEqual(expected, output.task_headers("i386", "desktop"))
self.assertEqual({}, output.task_headers("i386", "missing"))
def write_seed_output(self, arch, seed, packages):
"""Write a simplified Germinate output file, enough for testing."""
with mkfile(os.path.join(self.temp_dir, arch, seed)) as f:
why = "Ubuntu.Trusty %s seed" % seed
pkg_len = max(len("Package"), max(map(len, packages)))
src_len = max(len("Source"), max(map(len, packages)))
why_len = len(why)
print("%-*s | %-*s | %-*s |" %
(pkg_len, "Package", src_len, "Source", why_len, "Why"),
file=f)
print(("-" * pkg_len) + "-+-" + ("-" * src_len) + "-+-" +
("-" * why_len) + "-+",
file=f)
for pkg in packages:
print("%-*s | %-*s | %-*s |" %
(pkg_len, pkg, src_len, pkg, why_len, why),
file=f)
print(("-" * (pkg_len + src_len + why_len + 6)) + "-+", file=f)
print("%*s |" % (pkg_len + src_len + why_len + 6, ""), file=f)
def test_add_updated_ctime(self):
# Adding an existing file with a ctime newer than that of the
# checksums file causes its checksum to be updated.
path = os.path.join(self.temp_dir, "entry")
with mkfile(path) as entry:
print("ctime", end="", file=entry)
checksum_file = ChecksumFile(self.config,
self.temp_dir,
"MD5SUMS",
hashlib.md5,
sign=False)
checksum_file.entries["entry"] = ""
checksum_file.changed = True
checksum_file.write()
# We can simulate a ctime change by rewinding the mtime of both
# entry and the checksums file.
self.rewind_mtime(checksum_file.path)
self.rewind_mtime(path)
checksum_file.add("entry")
self.assertEqual(
hashlib.md5(b"ctime").hexdigest(), checksum_file.entries["entry"])
def test_send_mail_dry_run_from_file(self):
path = os.path.join(self.temp_dir, "body")
with mkfile(path) as body:
print("Body", file=body)
print("Text", file=body)
self.capture_logging()
with open(path) as body:
send_mail("Test subject",
"test_notify", ["*****@*****.**"],
body,
dry_run=True)
self.assertLogEqual([
"Would send mail to: [email protected]",
"",
"Subject: Test subject",
"X-Generated-By: test_notify",
"",
"Body",
"Text",
"",
])
def test_write(self):
checksum_file = ChecksumFile(self.config,
self.temp_dir,
"MD5SUMS",
hashlib.md5,
sign=False)
for name in "1", "2":
entry_path = os.path.join(self.temp_dir, name)
with mkfile(entry_path) as entry:
print(name, end="", file=entry)
checksum_file.add(name)
checksum_file.write()
with open(checksum_file.path) as md5sums:
expected = dedent("""\
%s *1
%s *2
""") % (hashlib.md5(b"1").hexdigest(),
hashlib.md5(b"2").hexdigest())
self.assertEqual(expected, md5sums.read())
self.assertEqual(
0,
subprocess.call(["md5sum", "-c", "--status", "MD5SUMS"],
cwd=self.temp_dir))
def write_structure(self, seed_inherit):
with mkfile(os.path.join(self.temp_dir, "STRUCTURE")) as structure:
for seed, inherit in seed_inherit:
print("%s: %s" % (seed, " ".join(inherit)), file=structure)
def test_write_tasks_project(self):
self.write_ubuntu_structure()
for arch in "amd64", "i386":
seed_dir = os.path.join(self.temp_dir, arch)
self.write_seed_output(arch, "required", ["base-files-%s" % arch])
self.write_seed_output(arch, "minimal", ["adduser-%s" % arch])
self.write_seed_output(arch, "desktop", ["xterm", "firefox"])
self.write_seed_output(arch, "live", ["xterm"])
with mkfile(os.path.join(
seed_dir, "minimal.seedtext")) as seedtext:
print("Task-Seeds: required", file=seedtext)
with mkfile(os.path.join(
seed_dir, "desktop.seedtext")) as seedtext:
print("Task-Per-Derivative: 1", file=seedtext)
with mkfile(os.path.join(seed_dir, "live.seedtext")) as seedtext:
print("Task-Per-Derivative: 1", file=seedtext)
self.config["DIST"] = "raring"
self.config["ARCHES"] = "amd64 i386"
self.config["IMAGE_TYPE"] = "daily-live"
self.config["CDIMAGE_LIVE"] = "1"
output = GerminateOutput(self.config, self.temp_dir)
output.write_tasks_project("ubuntu")
output_dir = os.path.join(
self.temp_dir, "scratch", "ubuntu", "raring", "daily-live",
"tasks")
self.assertCountEqual([
"required", "minimal", "desktop", "live",
"override.amd64", "override.i386",
"important.amd64", "important.i386",
"MASTER",
], os.listdir(output_dir))
with open(os.path.join(output_dir, "required")) as f:
self.assertEqual(
dedent("""\
#ifdef ARCH_amd64
base-files-amd64
#endif /* ARCH_amd64 */
#ifdef ARCH_i386
base-files-i386
#endif /* ARCH_i386 */
"""),
f.read())
with open(os.path.join(output_dir, "minimal")) as f:
self.assertEqual(
dedent("""\
#ifdef ARCH_amd64
adduser-amd64
#endif /* ARCH_amd64 */
#ifdef ARCH_i386
adduser-i386
#endif /* ARCH_i386 */
"""),
f.read())
with open(os.path.join(output_dir, "desktop")) as f:
self.assertEqual(
dedent("""\
#ifdef ARCH_amd64
firefox
xterm
#endif /* ARCH_amd64 */
#ifdef ARCH_i386
firefox
xterm
#endif /* ARCH_i386 */
"""),
f.read())
with open(os.path.join(output_dir, "live")) as f:
self.assertEqual(
dedent("""\
#ifdef ARCH_amd64
xterm
#endif /* ARCH_amd64 */
#ifdef ARCH_i386
xterm
#endif /* ARCH_i386 */
"""),
f.read())
with open(os.path.join(output_dir, "override.amd64")) as f:
self.assertEqual(
dedent("""\
adduser-amd64 Task minimal
base-files-amd64 Task minimal
firefox Task ubuntu-desktop
xterm Task ubuntu-desktop, ubuntu-live
"""),
f.read())
with open(os.path.join(output_dir, "override.i386")) as f:
self.assertEqual(
dedent("""\
adduser-i386 Task minimal
base-files-i386 Task minimal
firefox Task ubuntu-desktop
xterm Task ubuntu-desktop, ubuntu-live
"""),
f.read())
with open(os.path.join(output_dir, "important.amd64")) as f:
self.assertEqual("adduser-amd64\nbase-files-amd64\n", f.read())
with open(os.path.join(output_dir, "important.i386")) as f:
self.assertEqual("adduser-i386\nbase-files-i386\n", f.read())
with open(os.path.join(output_dir, "MASTER")) as f:
self.assertEqual("#include <ubuntu/raring/ship-live>\n", f.read())