def test_unpack_dsc_with_vendor(self):
# Some source packages unpack differently depending on dpkg's idea
# of the "vendor", and in extreme cases may even fail with some
# vendors. gina always sets the vendor to the target distribution
# name to ensure that it unpacks packages as if unpacking on that
# distribution.
archive_root = self.useTempDir()
pool_dir = os.path.join(archive_root, "pool/main/f/foo")
os.makedirs(pool_dir)
# Synthesise a package that can be unpacked with DEB_VENDOR=debian
# but not with DEB_VENDOR=ubuntu.
with open(os.path.join(pool_dir, "foo_1.0.orig.tar.gz"),
"wb+") as buffer:
orig_tar = LaunchpadWriteTarFile(buffer)
orig_tar.add_directory("foo-1.0")
orig_tar.close()
buffer.seek(0)
orig_tar_contents = buffer.read()
with open(os.path.join(pool_dir, "foo_1.0-1.debian.tar.gz"),
"wb+") as buffer:
debian_tar = LaunchpadWriteTarFile(buffer)
debian_tar.add_file("debian/source/format", "3.0 (quilt)\n")
debian_tar.add_file("debian/patches/ubuntu.series",
"--- corrupt patch\n")
debian_tar.add_file("debian/rules", "")
debian_tar.close()
buffer.seek(0)
debian_tar_contents = buffer.read()
dsc_path = os.path.join(pool_dir, "foo_1.0-1.dsc")
with open(dsc_path, "w") as dsc:
dsc.write(
dedent("""\
Format: 3.0 (quilt)
Source: foo
Binary: foo
Architecture: all
Version: 1.0-1
Maintainer: Foo Bar <*****@*****.**>
Files:
%s %s foo_1.0.orig.tar.gz
%s %s foo_1.0-1.debian.tar.gz
""" % (hashlib.md5(orig_tar_contents).hexdigest(),
len(orig_tar_contents),
hashlib.md5(debian_tar_contents).hexdigest(),
len(debian_tar_contents))))
dsc_contents = parse_tagfile(dsc_path)
dsc_contents["Directory"] = pool_dir
dsc_contents["Package"] = "foo"
dsc_contents["Component"] = "main"
dsc_contents["Section"] = "misc"
sp_data = SourcePackageData(**dsc_contents)
# Unpacking this in an Ubuntu context fails.
self.assertRaises(ExecutionError, sp_data.do_package, "ubuntu",
archive_root)
# But all is well in a Debian context.
sp_data.do_package("debian", archive_root)
def test_process_package_cleans_up_after_unpack_failure(self):
archive_root = self.useTempDir()
pool_dir = os.path.join(archive_root, "pool/main/f/foo")
os.makedirs(pool_dir)
with open(os.path.join(
pool_dir, "foo_1.0.orig.tar.gz"), "wb+") as buffer:
orig_tar = LaunchpadWriteTarFile(buffer)
orig_tar.add_directory("foo-1.0")
orig_tar.close()
buffer.seek(0)
orig_tar_contents = buffer.read()
with open(os.path.join(
pool_dir, "foo_1.0-1.debian.tar.gz"), "wb+") as buffer:
debian_tar = LaunchpadWriteTarFile(buffer)
debian_tar.add_file("debian/source/format", "3.0 (quilt)\n")
debian_tar.add_file("debian/patches/series", "--- corrupt patch\n")
debian_tar.add_file("debian/rules", "")
debian_tar.close()
buffer.seek(0)
debian_tar_contents = buffer.read()
dsc_path = os.path.join(pool_dir, "foo_1.0-1.dsc")
with open(dsc_path, "w") as dsc:
dsc.write(dedent("""\
Format: 3.0 (quilt)
Source: foo
Binary: foo
Architecture: all
Version: 1.0-1
Maintainer: Foo Bar <*****@*****.**>
Files:
%s %s foo_1.0.orig.tar.gz
%s %s foo_1.0-1.debian.tar.gz
""" % (
hashlib.md5(orig_tar_contents).hexdigest(),
len(orig_tar_contents),
hashlib.md5(debian_tar_contents).hexdigest(),
len(debian_tar_contents))))
dsc_contents = parse_tagfile(dsc_path)
dsc_contents["Directory"] = pool_dir
dsc_contents["Package"] = "foo"
dsc_contents["Component"] = "main"
dsc_contents["Section"] = "misc"
sp_data = SourcePackageData(**dsc_contents)
unpack_tmpdir = self.makeTemporaryDirectory()
with EnvironmentVariableFixture("TMPDIR", unpack_tmpdir):
# Force tempfile to recheck TMPDIR.
tempfile.tempdir = None
try:
self.assertRaises(
ExecutionError,
sp_data.process_package, "ubuntu", archive_root)
finally:
# Force tempfile to recheck TMPDIR for future tests.
tempfile.tempdir = None
self.assertEqual([], os.listdir(unpack_tmpdir))
def test_unpack_dsc_with_vendor(self):
# Some source packages unpack differently depending on dpkg's idea
# of the "vendor", and in extreme cases may even fail with some
# vendors. gina always sets the vendor to the target distribution
# name to ensure that it unpacks packages as if unpacking on that
# distribution.
archive_root = self.useTempDir()
pool_dir = os.path.join(archive_root, "pool/main/f/foo")
os.makedirs(pool_dir)
# Synthesise a package that can be unpacked with DEB_VENDOR=debian
# but not with DEB_VENDOR=ubuntu.
with open(os.path.join(pool_dir, "foo_1.0.orig.tar.gz"), "wb+") as buffer:
orig_tar = LaunchpadWriteTarFile(buffer)
orig_tar.add_directory("foo-1.0")
orig_tar.close()
buffer.seek(0)
orig_tar_contents = buffer.read()
with open(os.path.join(pool_dir, "foo_1.0-1.debian.tar.gz"), "wb+") as buffer:
debian_tar = LaunchpadWriteTarFile(buffer)
debian_tar.add_file("debian/source/format", "3.0 (quilt)\n")
debian_tar.add_file("debian/patches/ubuntu.series", "--- corrupt patch\n")
debian_tar.add_file("debian/rules", "")
debian_tar.close()
buffer.seek(0)
debian_tar_contents = buffer.read()
dsc_path = os.path.join(pool_dir, "foo_1.0-1.dsc")
with open(dsc_path, "w") as dsc:
dsc.write(
dedent(
"""\
Format: 3.0 (quilt)
Source: foo
Binary: foo
Architecture: all
Version: 1.0-1
Maintainer: Foo Bar <*****@*****.**>
Files:
%s %s foo_1.0.orig.tar.gz
%s %s foo_1.0-1.debian.tar.gz
"""
% (
hashlib.md5(orig_tar_contents).hexdigest(),
len(orig_tar_contents),
hashlib.md5(debian_tar_contents).hexdigest(),
len(debian_tar_contents),
)
)
)
dsc_contents = parse_tagfile(dsc_path)
dsc_contents["Directory"] = pool_dir
dsc_contents["Package"] = "foo"
dsc_contents["Component"] = "main"
dsc_contents["Section"] = "misc"
sp_data = SourcePackageData(**dsc_contents)
# Unpacking this in an Ubuntu context fails.
self.assertRaises(ExecutionError, sp_data.do_package, "ubuntu", archive_root)
# But all is well in a Debian context.
sp_data.do_package("debian", archive_root)
class TestDdtpTarball(TestCaseWithFactory):
layer = ZopelessDatabaseLayer
def setUp(self):
super(TestDdtpTarball, self).setUp()
self.temp_dir = self.makeTemporaryDirectory()
self.distro = self.factory.makeDistribution()
db_pubconf = getUtility(IPublisherConfigSet).getByDistribution(
self.distro)
db_pubconf.root_dir = unicode(self.temp_dir)
self.archive = self.factory.makeArchive(distribution=self.distro,
purpose=ArchivePurpose.PRIMARY)
self.suite = "distroseries"
# CustomUpload.installFiles requires a umask of 0o022.
old_umask = os.umask(0o022)
self.addCleanup(os.umask, old_umask)
def openArchive(self, version):
self.path = os.path.join(self.temp_dir,
"translations_main_%s.tar.gz" % version)
self.buffer = open(self.path, "wb")
self.tarfile = LaunchpadWriteTarFile(self.buffer)
def process(self):
self.tarfile.close()
self.buffer.close()
DdtpTarballUpload().process(self.archive, self.path, self.suite)
def getTranslationsPath(self, filename):
pubconf = getPubConfig(self.archive)
return os.path.join(pubconf.archiveroot, "dists", self.suite, "main",
"i18n", filename)
def test_basic(self):
# Processing a simple correct tar file works.
self.openArchive("20060728")
self.tarfile.add_file("i18n/Translation-de", b"")
self.process()
self.assertTrue(
os.path.exists(self.getTranslationsPath("Translation-de")))
def test_ignores_empty_directories(self):
# Empty directories in the tarball are not extracted.
self.openArchive("20060728")
self.tarfile.add_file("i18n/Translation-de", b"")
self.tarfile.add_directory("i18n/foo")
self.process()
self.assertTrue(
os.path.exists(self.getTranslationsPath("Translation-de")))
self.assertFalse(os.path.exists(self.getTranslationsPath("foo")))
def test_partial_update(self):
# If a DDTP tarball only contains a subset of published translation
# files, these are updated and the rest are left untouched.
self.openArchive("20060728")
self.tarfile.add_file("i18n/Translation-bn", b"bn")
self.tarfile.add_file("i18n/Translation-ca", b"ca")
self.process()
with open(self.getTranslationsPath("Translation-bn")) as bn_file:
self.assertEqual("bn", bn_file.read())
with open(self.getTranslationsPath("Translation-ca")) as ca_file:
self.assertEqual("ca", ca_file.read())
self.openArchive("20060817")
self.tarfile.add_file("i18n/Translation-bn", b"new bn")
self.process()
with open(self.getTranslationsPath("Translation-bn")) as bn_file:
self.assertEqual("new bn", bn_file.read())
with open(self.getTranslationsPath("Translation-ca")) as ca_file:
self.assertEqual("ca", ca_file.read())
def test_breaks_hard_links(self):
# Our archive uses dsync to replace identical files with hard links
# in order to save some space. tarfile.extract overwrites
# pre-existing files rather than creating new files and moving them
# into place, so making this work requires special care. Test that
# that care has been taken.
self.openArchive("20060728")
self.tarfile.add_file("i18n/Translation-ca", b"")
self.process()
ca = self.getTranslationsPath("Translation-ca")
bn = self.getTranslationsPath("Translation-bn")
os.link(ca, bn)
self.assertTrue(os.path.exists(bn))
self.assertEqual(2, os.stat(bn).st_nlink)
self.assertEqual(2, os.stat(ca).st_nlink)
self.openArchive("20060817")
self.tarfile.add_file("i18n/Translation-bn", b"break hard link")
self.process()
with open(bn) as bn_file:
self.assertEqual("break hard link", bn_file.read())
with open(ca) as ca_file:
self.assertEqual("", ca_file.read())
self.assertEqual(1, os.stat(bn).st_nlink)
self.assertEqual(1, os.stat(ca).st_nlink)
def test_parsePath_handles_underscore_in_directory(self):
# parsePath is not misled by an underscore in the directory name.
self.assertEqual(
# XXX cjwatson 2012-07-03: .tar.gz is not stripped off the end
# of the version due to something of an ambiguity in the design;
# how should translations_main_1.0.1.tar.gz be parsed? In
# practice this doesn't matter because DdtpTarballUpload never
# uses the version for anything.
("translations", "main", "1.tar.gz"),
DdtpTarballUpload.parsePath(
"/dir_with_underscores/translations_main_1.tar.gz"))
def test_getSeriesKey_extracts_component(self):
# getSeriesKey extracts the component from an upload's filename.
self.openArchive("20060728")
self.assertEqual("main", DdtpTarballUpload.getSeriesKey(self.path))
def test_getSeriesKey_returns_None_on_mismatch(self):
# getSeriesKey returns None if the filename does not match the
# expected pattern.
self.assertIsNone(DdtpTarballUpload.getSeriesKey("argh_1.0.jpg"))
def test_getSeriesKey_refuses_names_with_wrong_number_of_fields(self):
# getSeriesKey requires exactly three fields.
self.assertIsNone(DdtpTarballUpload.getSeriesKey("package_1.0.tar.gz"))
self.assertIsNone(
DdtpTarballUpload.getSeriesKey("one_two_three_four_5.tar.gz"))
class TestDdtpTarball(TestCase):
def setUp(self):
super(TestDdtpTarball, self).setUp()
self.temp_dir = self.makeTemporaryDirectory()
self.pubconf = FakeConfig(self.temp_dir)
self.suite = "distroseries"
# CustomUpload.installFiles requires a umask of 022.
old_umask = os.umask(022)
self.addCleanup(os.umask, old_umask)
def openArchive(self, version):
self.path = os.path.join(
self.temp_dir, "translations_main_%s.tar.gz" % version)
self.buffer = open(self.path, "wb")
self.archive = LaunchpadWriteTarFile(self.buffer)
def process(self):
self.archive.close()
self.buffer.close()
process_ddtp_tarball(self.pubconf, self.path, self.suite)
def getTranslationsPath(self, filename):
return os.path.join(
self.temp_dir, "dists", self.suite, "main", "i18n", filename)
def test_basic(self):
# Processing a simple correct tar file works.
self.openArchive("20060728")
self.archive.add_file("i18n/Translation-de", "")
self.process()
self.assertTrue(os.path.exists(
self.getTranslationsPath("Translation-de")))
def test_ignores_empty_directories(self):
# Empty directories in the tarball are not extracted.
self.openArchive("20060728")
self.archive.add_file("i18n/Translation-de", "")
self.archive.add_directory("i18n/foo")
self.process()
self.assertTrue(os.path.exists(
self.getTranslationsPath("Translation-de")))
self.assertFalse(os.path.exists(self.getTranslationsPath("foo")))
def test_partial_update(self):
# If a DDTP tarball only contains a subset of published translation
# files, these are updated and the rest are left untouched.
self.openArchive("20060728")
self.archive.add_file("i18n/Translation-bn", "bn")
self.archive.add_file("i18n/Translation-ca", "ca")
self.process()
with open(self.getTranslationsPath("Translation-bn")) as bn_file:
self.assertEqual("bn", bn_file.read())
with open(self.getTranslationsPath("Translation-ca")) as ca_file:
self.assertEqual("ca", ca_file.read())
self.openArchive("20060817")
self.archive.add_file("i18n/Translation-bn", "new bn")
self.process()
with open(self.getTranslationsPath("Translation-bn")) as bn_file:
self.assertEqual("new bn", bn_file.read())
with open(self.getTranslationsPath("Translation-ca")) as ca_file:
self.assertEqual("ca", ca_file.read())
def test_breaks_hard_links(self):
# Our archive uses dsync to replace identical files with hard links
# in order to save some space. tarfile.extract overwrites
# pre-existing files rather than creating new files and moving them
# into place, so making this work requires special care. Test that
# that care has been taken.
self.openArchive("20060728")
self.archive.add_file("i18n/Translation-ca", "")
self.process()
ca = self.getTranslationsPath("Translation-ca")
bn = self.getTranslationsPath("Translation-bn")
os.link(ca, bn)
self.assertTrue(os.path.exists(bn))
self.assertEqual(2, os.stat(bn).st_nlink)
self.assertEqual(2, os.stat(ca).st_nlink)
self.openArchive("20060817")
self.archive.add_file("i18n/Translation-bn", "break hard link")
self.process()
with open(bn) as bn_file:
self.assertEqual("break hard link", bn_file.read())
with open(ca) as ca_file:
self.assertEqual("", ca_file.read())
self.assertEqual(1, os.stat(bn).st_nlink)
self.assertEqual(1, os.stat(ca).st_nlink)
def test_parsePath_handles_underscore_in_directory(self):
# parsePath is not misled by an underscore in the directory name.
self.assertEqual(
# XXX cjwatson 2012-07-03: .tar.gz is not stripped off the end
# of the version due to something of an ambiguity in the design;
# how should translations_main_1.0.1.tar.gz be parsed? In
# practice this doesn't matter because DdtpTarballUpload never
# uses the version for anything.
("translations", "main", "1.tar.gz"),
DdtpTarballUpload.parsePath(
"/dir_with_underscores/translations_main_1.tar.gz"))
def test_getSeriesKey_extracts_component(self):
# getSeriesKey extracts the component from an upload's filename.
self.openArchive("20060728")
self.assertEqual("main", DdtpTarballUpload.getSeriesKey(self.path))
def test_getSeriesKey_returns_None_on_mismatch(self):
# getSeriesKey returns None if the filename does not match the
# expected pattern.
self.assertIsNone(DdtpTarballUpload.getSeriesKey("argh_1.0.jpg"))
def test_getSeriesKey_refuses_names_with_wrong_number_of_fields(self):
# getSeriesKey requires exactly three fields.
self.assertIsNone(DdtpTarballUpload.getSeriesKey("package_1.0.tar.gz"))
self.assertIsNone(DdtpTarballUpload.getSeriesKey(
"one_two_three_four_5.tar.gz"))
