def test_differ_in_content_returns_false_for_identical_files(self):
# Identical files do not differ.
self.useTempDir()
text = self.factory.getUniqueString()
write_file('one', text)
write_file('other', text)
self.assertFalse(differ_in_content('one', 'other'))
def testReplaceUpdatedHtpasswd(self):
"""Test that the htpasswd file is only replaced if it changes."""
FILE_CONTENT = "Kneel before Zod!"
# The publisher Config object does not have an interface, so we
# need to remove the security wrapper.
pub_config = getPubConfig(self.ppa)
filename = os.path.join(pub_config.htaccessroot, ".htpasswd")
# Write out a dummy .htpasswd
ensure_directory_exists(pub_config.htaccessroot)
write_file(filename, FILE_CONTENT)
# Write the same contents in a temp file.
fd, temp_filename = tempfile.mkstemp(dir=pub_config.htaccessroot)
file = os.fdopen(fd, "w")
file.write(FILE_CONTENT)
file.close()
# Replacement should not happen.
script = self.getScript()
self.assertFalse(
script.replaceUpdatedHtpasswd(self.ppa, temp_filename))
# Writing a different .htpasswd should see it get replaced.
write_file(filename, "Come to me, son of Jor-El!")
self.assertTrue(
script.replaceUpdatedHtpasswd(self.ppa, temp_filename))
os.remove(filename)
def createEmptyPocketRequest(self, distroseries, pocket, comp):
"""Creates empty files for a release pocket and distroseries"""
suite = distroseries.getSuite(pocket)
# Create empty override lists.
needed_paths = [
(comp, ),
("extra", comp),
(comp, "src"),
]
for sub_comp in self.publisher.subcomponents:
needed_paths.append((comp, sub_comp))
for path in needed_paths:
write_file(
os.path.join(self._config.overrideroot,
".".join(("override", suite) + path)), "")
# Create empty file lists.
def touch_list(*parts):
write_file(
os.path.join(self._config.overrideroot,
"_".join((suite, ) + parts)), "")
touch_list(comp, "source")
arch_tags = [
a.architecturetag for a in distroseries.enabled_architectures
]
for arch in arch_tags:
# Touch more file lists for the archs.
touch_list(comp, "binary-" + arch)
for sub_comp in self.publisher.subcomponents:
touch_list(comp, sub_comp, "binary-" + arch)
def createEmptyPocketRequest(self, distroseries, pocket, comp):
"""Creates empty files for a release pocket and distroseries"""
suite = distroseries.getSuite(pocket)
# Create empty override lists.
needed_paths = [
(comp,),
("extra", comp),
(comp, "src"),
]
for sub_comp in self.publisher.subcomponents:
needed_paths.append((comp, sub_comp))
for path in needed_paths:
write_file(os.path.join(
self._config.overrideroot,
".".join(("override", suite) + path)), "")
# Create empty file lists.
def touch_list(*parts):
write_file(os.path.join(
self._config.overrideroot,
"_".join((suite, ) + parts)), "")
touch_list(comp, "source")
arch_tags = [
a.architecturetag for a in distroseries.enabled_architectures]
for arch in arch_tags:
# Touch more file lists for the archs.
touch_list(comp, "binary-" + arch)
for sub_comp in self.publisher.subcomponents:
touch_list(comp, sub_comp, "binary-" + arch)
def test_differ_in_content_returns_false_for_identical_files(self):
# Identical files do not differ.
self.useTempDir()
text = self.factory.getUniqueString()
write_file('one', text)
write_file('other', text)
self.assertFalse(differ_in_content('one', 'other'))
def test_triggers_store_uploads(self):
# The upload processor triggers store uploads if appropriate.
self.pushConfig("snappy",
store_url="http://sca.example/",
store_upload_url="http://updown.example/")
self.switchToAdmin()
self.snap.store_series = self.factory.makeSnappySeries(
usable_distro_series=[self.snap.distro_series])
self.snap.store_name = self.snap.name
self.snap.store_upload = True
self.snap.store_secrets = {"root": Macaroon().serialize()}
Store.of(self.snap).flush()
self.switchToUploader()
self.assertFalse(self.build.verifySuccessfulUpload())
upload_dir = os.path.join(self.incoming_folder, "test",
str(self.build.id), "ubuntu")
write_file(os.path.join(upload_dir, "wget_0_all.snap"), "snap")
handler = UploadHandler.forProcessor(self.uploadprocessor,
self.incoming_folder, "test",
self.build)
result = handler.processSnap(self.log)
self.assertEqual(
UploadStatusEnum.ACCEPTED, result,
"Snap upload failed\nGot: %s" % self.log.getLogBuffer())
self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
self.assertTrue(self.build.verifySuccessfulUpload())
self.assertEqual(1, len(list(self.build.store_upload_jobs)))
def testReplaceUpdatedHtpasswd(self):
"""Test that the htpasswd file is only replaced if it changes."""
FILE_CONTENT = "Kneel before Zod!"
# The publisher Config object does not have an interface, so we
# need to remove the security wrapper.
pub_config = getPubConfig(self.ppa)
filename = os.path.join(pub_config.htaccessroot, ".htpasswd")
# Write out a dummy .htpasswd
ensure_directory_exists(pub_config.htaccessroot)
write_file(filename, FILE_CONTENT)
# Write the same contents in a temp file.
fd, temp_filename = tempfile.mkstemp(dir=pub_config.htaccessroot)
file = os.fdopen(fd, "w")
file.write(FILE_CONTENT)
file.close()
# Replacement should not happen.
script = self.getScript()
self.assertFalse(script.replaceUpdatedHtpasswd(self.ppa,
temp_filename))
# Writing a different .htpasswd should see it get replaced.
write_file(filename, "Come to me, son of Jor-El!")
self.assertTrue(script.replaceUpdatedHtpasswd(self.ppa, temp_filename))
os.remove(filename)
def test_urlfetch_does_not_support_file_urls_by_default(self):
"""urlfetch() does not support file urls by default."""
test_path = self.useFixture(TempDir()).join('file')
write_file(test_path, '')
url = 'file://' + test_path
e = self.assertRaises(InvalidSchema, urlfetch, url)
self.assertEqual("No connection adapters were found for '%s'" % url,
str(e))
def test_present_executable(self):
temp_dir = self.makeTemporaryDirectory()
bin_dir = os.path.join(temp_dir, "bin")
program = os.path.join(bin_dir, "program")
write_file(program, "")
os.chmod(program, 0o755)
self.useFixture(EnvironmentVariable("PATH", bin_dir))
self.assertTrue(find_on_path("program"))
def test_signFile_absolute_outside_archive(self):
filename = os.path.join(self.temp_dir, "signme")
write_file(filename, "sign this")
signer = ISignableArchive(self.archive)
self.assertTrue(signer.can_sign)
self.assertRaises(AssertionError, signer.signFile, self.suite,
filename)
def test_signFile_absolute_within_archive(self):
filename = os.path.join(self.archive_root, "signme")
write_file(filename, "sign this")
signer = ISignableArchive(self.archive)
self.assertTrue(signer.can_sign)
signer.signFile(self.suite, filename)
self.assertTrue(os.path.exists(filename + ".gpg"))
def test_urlfetch_supports_file_urls_if_allow_file(self):
"""urlfetch() supports file urls if explicitly asked to do so."""
test_path = self.useFixture(TempDir()).join('file')
write_file(test_path, 'Success.')
url = 'file://' + test_path
self.assertThat(
urlfetch(url, allow_file=True),
MatchesStructure(status_code=Equals(200),
headers=ContainsDict(
{'Content-Length': Equals(8)}),
content=Equals('Success.')))
def test_signFile_runs_parts(self):
filename = os.path.join(self.archive_root, "signme")
write_file(filename, "sign this")
signer = ISignableArchive(self.archive)
self.assertTrue(signer.can_sign)
signer.signFile(self.suite, filename)
self.assertThat(
"%s.gpg" % filename,
FileContains(
"detached signature of %s (%s, %s/%s)\n" %
(filename, self.archive_root, self.distro.name, self.suite)))
def writeMarkerFile(self, file_path):
"""Create a marker file at location `file_path`.
An arbitrary string is written to the file, and flushed to the
filesystem. Any surrounding directories are created as needed.
:param file_path: Full path to a file: optional directory prefix
followed by required file name.
:return: The arbitrary string that is also in the file.
"""
marker_contents = self.factory.getUniqueString()
write_file(file_path, marker_contents)
return marker_contents
def test_sorts_by_mtime(self):
# Files are sorted by ascending mtime.
root = self.useFixture(TempDir())
file_paths = [root.join(str(name)) for name in range(3)]
now = time.time()
for i, path in enumerate(file_paths):
write_file(path, '%s\n' % i)
os.utime(path, (now - i, now - i))
contents = []
for fd, _ in get_files_to_parse(file_paths):
fd.seek(0)
contents.append(fd.read())
self.assertEqual(['2\n', '1\n', '0\n'], contents)
def writeMarkerFile(self, file_path):
"""Create a marker file at location `file_path`.
An arbitrary string is written to the file, and flushed to the
filesystem. Any surrounding directories are created as needed.
:param file_path: Full path to a file: optional directory prefix
followed by required file name.
:return: The arbitrary string that is also in the file.
"""
marker_contents = self.factory.getUniqueString()
write_file(file_path, marker_contents)
return marker_contents
def test_removes_only_stale_files(self):
# removeStaleOutputs removes only stale germinate output files.
self.setUpDistroAndScript()
series_name = self.distroseries[0].name
seed_old_file = "old_flavour_%s_i386" % series_name
seed_new_file = "new_flavour_%s_i386" % series_name
other_file = "other-file"
output = partial(os.path.join, self.script.config.germinateroot)
for base in (seed_old_file, seed_new_file, other_file):
write_file(output(base), "")
self.script.removeStaleOutputs(series_name, set([seed_new_file]))
self.assertFalse(os.path.exists(output(seed_old_file)))
self.assertTrue(os.path.exists(output(seed_new_file)))
self.assertTrue(os.path.exists(output(other_file)))
def test_requires_snap(self):
# The upload processor fails if the upload does not contain any
# .snap files.
self.assertFalse(self.build.verifySuccessfulUpload())
upload_dir = os.path.join(self.incoming_folder, "test",
str(self.build.id), "ubuntu")
write_file(os.path.join(upload_dir, "wget_0_all.manifest"), "manifest")
handler = UploadHandler.forProcessor(self.uploadprocessor,
self.incoming_folder, "test",
self.build)
result = handler.processSnap(self.log)
self.assertEqual(UploadStatusEnum.REJECTED, result)
self.assertIn("ERROR Build did not produce any snap packages.",
self.log.getLogBuffer())
self.assertFalse(self.build.verifySuccessfulUpload())
def createDeb(self, filename, data_format):
"""Return the contents of a dummy .deb file."""
tempdir = self.makeTemporaryDirectory()
members = [
"debian-binary",
"control.tar.gz",
"data.tar.%s" % data_format,
]
for member in members:
write_file(os.path.join(tempdir, member), "")
retcode = subprocess.call(
["ar", "rc", filename] + members, cwd=tempdir)
self.assertEqual(0, retcode)
with open(os.path.join(tempdir, filename)) as f:
return f.read()
def createDeb(self, filename, data_format):
"""Return the contents of a dummy .deb file."""
tempdir = self.makeTemporaryDirectory()
members = [
"debian-binary",
"control.tar.gz",
"data.tar.%s" % data_format,
]
for member in members:
write_file(os.path.join(tempdir, member), "")
retcode = subprocess.call(["ar", "rc", filename] + members,
cwd=tempdir)
self.assertEqual(0, retcode)
with open(os.path.join(tempdir, filename)) as f:
return f.read()
def test_sets_build_and_state(self):
# The upload processor uploads files and sets the correct status.
self.assertFalse(self.build.verifySuccessfulUpload())
upload_dir = os.path.join(self.incoming_folder, "test",
str(self.build.id), "ubuntu")
write_file(os.path.join(upload_dir, "wget_0_all.snap"), "snap")
write_file(os.path.join(upload_dir, "wget_0_all.manifest"), "manifest")
handler = UploadHandler.forProcessor(self.uploadprocessor,
self.incoming_folder, "test",
self.build)
result = handler.processSnap(self.log)
self.assertEqual(
UploadStatusEnum.ACCEPTED, result,
"Snap upload failed\nGot: %s" % self.log.getLogBuffer())
self.assertEqual(BuildStatus.FULLYBUILT, self.build.status)
self.assertTrue(self.build.verifySuccessfulUpload())
def test_sign_with_signing_key(self):
filename = os.path.join(getPubConfig(self.archive).archiveroot, "file")
write_file(filename, "contents")
self.assertIsNone(self.archive.signing_key)
self.useFixture(InProcessKeyServerFixture()).start()
key_path = os.path.join(gpgkeysdir, '*****@*****.**')
yield IArchiveSigningKey(self.archive).setSigningKey(
key_path, async_keyserver=True)
self.assertIsNotNone(self.archive.signing_key)
custom_processor = CustomUpload()
custom_processor.sign(self.archive, "suite", filename)
with open(filename) as cleartext_file:
cleartext = cleartext_file.read()
with open("%s.gpg" % filename) as signature_file:
signature = getUtility(IGPGHandler).getVerifiedSignature(
cleartext, signature_file.read())
self.assertEqual(self.archive.signing_key.fingerprint,
signature.fingerprint)
def test_signRepository_runs_parts(self):
suite_dir = os.path.join(self.archive_root, "dists", self.suite)
release_path = os.path.join(suite_dir, "Release")
write_file(release_path, "Release contents")
signer = ISignableArchive(self.archive)
self.assertTrue(signer.can_sign)
self.assertContentEqual(["Release.gpg", "InRelease"],
signer.signRepository(self.suite))
self.assertThat(
os.path.join(suite_dir, "Release.gpg"),
FileContains("detached signature of %s (%s, %s/%s)\n" %
(release_path, self.archive_root, self.distro.name,
self.suite)))
self.assertThat(
os.path.join(suite_dir, "InRelease"),
FileContains("clear signature of %s (%s, %s/%s)\n" %
(release_path, self.archive_root, self.distro.name,
self.suite)))
def test_sign_with_external_run_parts(self):
self.enableRunParts(distribution_name=self.distro.name)
archiveroot = getPubConfig(self.archive).archiveroot
filename = os.path.join(archiveroot, "file")
write_file(filename, "contents")
self.assertIsNone(self.archive.signing_key)
run_parts_fixture = self.useFixture(
MonkeyPatch("lp.archivepublisher.archivesigningkey.run_parts",
FakeMethod()))
custom_processor = CustomUpload()
custom_processor.sign(self.archive, "suite", filename)
args, kwargs = run_parts_fixture.new_value.calls[0]
self.assertEqual((self.distro.name, "sign.d"), args)
self.assertThat(
kwargs["env"],
MatchesDict({
"ARCHIVEROOT": Equals(archiveroot),
"INPUT_PATH": Equals(filename),
"OUTPUT_PATH": Equals("%s.gpg" % filename),
"MODE": Equals("detached"),
"DISTRIBUTION": Equals(self.distro.name),
"SUITE": Equals("suite"),
}))
def test_user_defined_fields(self):
series = self.factory.makeDistroSeries()
archive_root = self.useTempDir()
sphandler = SourcePackageHandler(
series.distribution.name, archive_root,
PackagePublishingPocket.RELEASE, None)
dsc_contents = {
"Format": "3.0 (quilt)",
"Source": "foo",
"Binary": "foo",
"Architecture": "all arm64",
"Version": "1.0-1",
"Maintainer": "Foo Bar <*****@*****.**>",
"Files": "xxx 000 foo_1.0-1.dsc",
"Build-Indep-Architecture": "amd64",
"Directory": "pool/main/f/foo",
"Package": "foo",
"Component": "main",
"Section": "misc",
}
sp_data = SourcePackageData(**dsc_contents)
self.assertEqual(
[["Build-Indep-Architecture", "amd64"]], sp_data._user_defined)
sp_data.archive_root = archive_root
sp_data.dsc = ""
sp_data.copyright = ""
sp_data.urgency = "low"
sp_data.changelog = None
sp_data.changelog_entry = None
sp_data.date_uploaded = UTC_NOW
# We don't need a real .dsc here.
write_file(
os.path.join(archive_root, "pool/main/f/foo/foo_1.0-1.dsc"), "x")
spr = sphandler.createSourcePackageRelease(sp_data, series)
self.assertIsNotNone(spr)
self.assertEqual(
[["Build-Indep-Architecture", "amd64"]], spr.user_defined_fields)
def test_user_defined_fields(self):
das = self.factory.makeDistroArchSeries()
archive_root = self.useTempDir()
sphandler = SourcePackageHandler(
das.distroseries.distribution.name, archive_root,
PackagePublishingPocket.RELEASE, None)
bphandler = BinaryPackageHandler(
sphandler, archive_root, PackagePublishingPocket.RELEASE)
spr = self.factory.makeSourcePackageRelease(
distroseries=das.distroseries)
deb_contents = {
"Package": "foo",
"Installed-Size": "0",
"Maintainer": "Foo Bar <*****@*****.**>",
"Section": "misc",
"Architecture": "amd64",
"Version": "1.0-1",
"Filename": "pool/main/f/foo/foo_1.0-1_amd64.deb",
"Component": "main",
"Size": "0",
"MD5sum": "0" * 32,
"Description": "",
"Summary": "",
"Priority": "extra",
"Python-Version": "2.7",
}
bp_data = BinaryPackageData(**deb_contents)
self.assertEqual([["Python-Version", "2.7"]], bp_data._user_defined)
bp_data.archive_root = archive_root
# We don't need a real .deb here.
write_file(
os.path.join(archive_root, "pool/main/f/foo/foo_1.0-1_amd64.deb"),
"x")
bpr = bphandler.createBinaryPackage(bp_data, spr, das, "amd64")
self.assertIsNotNone(bpr)
self.assertEqual([["Python-Version", "2.7"]], bpr.user_defined_fields)
def test_signRepository_honours_pubconf(self):
pubconf = getPubConfig(self.archive)
pubconf.distsroot = self.makeTemporaryDirectory()
suite_dir = os.path.join(pubconf.distsroot, self.suite)
release_path = os.path.join(suite_dir, "Release")
write_file(release_path, "Release contents")
signer = ISignableArchive(self.archive)
self.assertTrue(signer.can_sign)
self.assertRaises(AssertionError, signer.signRepository, self.suite)
self.assertContentEqual(["Release.gpg", "InRelease"],
signer.signRepository(self.suite,
pubconf=pubconf))
self.assertThat(
os.path.join(suite_dir, "Release.gpg"),
FileContains("detached signature of %s (%s, %s/%s)\n" %
(release_path, self.archive_root, self.distro.name,
self.suite)))
self.assertThat(
os.path.join(suite_dir, "InRelease"),
FileContains("clear signature of %s (%s, %s/%s)\n" %
(release_path, self.archive_root, self.distro.name,
self.suite)))
def fake_overrides(script, distroseries):
"""Fake overrides files so `script` can run `apt-ftparchive`."""
components = ['main', 'restricted', 'universe', 'multiverse']
architectures = script.getArchs(distroseries.name)
suffixes = components + ['extra.' + component for component in components]
for suffix in suffixes:
write_file(os.path.join(
script.config.overrideroot,
"override.%s.%s" % (distroseries.name, suffix)), "")
for component in components:
write_file(os.path.join(
script.config.overrideroot,
"%s_%s_source" % (distroseries.name, component)), "")
for arch in architectures:
write_file(os.path.join(
script.config.overrideroot,
"%s_%s_binary-%s" % (distroseries.name, component, arch)), "")
def test_present_not_executable(self):
temp_dir = self.makeTemporaryDirectory()
bin_dir = os.path.join(temp_dir, "bin")
write_file(os.path.join(bin_dir, "program"), "")
self.useFixture(EnvironmentVariable("PATH", bin_dir))
self.assertFalse(find_on_path("program"))
def touch_list(*parts):
write_file(
os.path.join(self._config.overrideroot,
"_".join((suite, ) + parts)), "")
def setUpKeyAndCert(self):
self.key = os.path.join(self.uefi_dir, "uefi.key")
self.cert = os.path.join(self.uefi_dir, "uefi.crt")
write_file(self.key, "")
write_file(self.cert, "")
def test_differ_in_content_returns_true_if_one_file_does_not_exist(self):
# A nonexistent file differs from an existing one.
self.useTempDir()
write_file('one', self.factory.getUniqueString())
self.assertTrue(differ_in_content('one', 'other'))
def test_differ_in_content_returns_true_for_differing_files(self):
# Files with different contents differ.
self.useTempDir()
write_file('one', self.factory.getUniqueString())
write_file('other', self.factory.getUniqueString())
self.assertTrue(differ_in_content('one', 'other'))
def test_differ_in_content_returns_true_if_one_file_does_not_exist(self):
# A nonexistent file differs from an existing one.
self.useTempDir()
write_file('one', self.factory.getUniqueString())
self.assertTrue(differ_in_content('one', 'other'))
def test_write_file(self):
directory = self.makeTemporaryDirectory()
filename = os.path.join(directory, 'filename')
content = self.getUniqueString()
write_file(filename, content)
self.assertThat(filename, FileContains(content))
def touch_list(*parts):
write_file(os.path.join(
self._config.overrideroot,
"_".join((suite, ) + parts)), "")
def __call__(self, *args, **kwargs):
super(FakeMethodCallLog, self).__call__(*args, **kwargs)
description = args[0]
cmdl = args[1]
self.callers[description] += 1
if description == "UEFI signing":
filename = cmdl[-1]
if filename.endswith(".efi"):
write_file(filename + ".signed", "")
elif description == "UEFI keygen":
write_file(self.upload.uefi_key, "")
write_file(self.upload.uefi_cert, "")
elif description == "Kmod signing":
filename = cmdl[-1]
if filename.endswith(".ko.sig"):
write_file(filename, "")
elif description == "Kmod keygen cert":
write_file(self.upload.kmod_x509, "")
elif description == "Kmod keygen key":
write_file(self.upload.kmod_pem, "")
elif description == "Opal signing":
filename = cmdl[-1]
if filename.endswith(".opal.sig"):
write_file(filename, "")
elif description == "Opal keygen cert":
write_file(self.upload.opal_x509, "")
elif description == "Opal keygen key":
write_file(self.upload.opal_pem, "")
else:
raise AssertionError("unknown command executed cmd=(%s)" %
" ".join(cmdl))
return 0
def test_write_file(self):
directory = self.makeTemporaryDirectory()
filename = os.path.join(directory, 'filename')
content = self.getUniqueString()
write_file(filename, content)
self.assertThat(filename, FileContains(content))
def test_differ_in_content_returns_true_for_differing_files(self):
# Files with different contents differ.
self.useTempDir()
write_file('one', self.factory.getUniqueString())
write_file('other', self.factory.getUniqueString())
self.assertTrue(differ_in_content('one', 'other'))
