def sync_local_mirror(config, multipidfile_state):
if config["CDIMAGE_NOSYNC"]:
return
capproject = config.capproject
sync_lock = os.path.join(config.root, "etc", ".lock-archive-sync")
if not multipidfile_state:
log_marker("Syncing %s mirror" % capproject)
# Acquire lock to allow parallel builds to ensure a consistent
# archive.
try:
subprocess.check_call(["lockfile", "-r", "4", sync_lock])
except subprocess.CalledProcessError:
logger.error("Couldn't acquire archive sync lock!")
raise
try:
anonftpsync(config)
finally:
osextras.unlink_force(sync_lock)
else:
log_marker(
"Parallel build; waiting for %s mirror to sync" % capproject)
try:
subprocess.check_call(["lockfile", "-8", "-r", "450", sync_lock])
except subprocess.CalledProcessError:
logger.error("Timed out waiting for archive sync lock!")
raise
osextras.unlink_force(sync_lock)
def _write(self, pids):
# Must be called within context manager lock.
assert os.path.exists(self.lock_path), (
"Called _write on %s without locking!" % self)
if pids:
with open(self.path, "w") as fd:
for pid in sorted(pids):
print(pid, file=fd)
else:
osextras.unlink_force(self.path)
def sign_cdimage(config, path):
if not can_sign(config):
return False
with open(path, "rb") as infile:
with open("%s.gpg" % path, "wb") as outfile:
try:
subprocess.check_call(_signing_command(config),
stdin=infile,
stdout=outfile)
except subprocess.CalledProcessError:
osextras.unlink_force("%s.gpg" % path)
raise
return True
def lock_build_image_set(config):
project = config.project
if config["UBUNTU_DEFAULTS_LOCALE"] == "zh_CN":
project = "ubuntu-chinese-edition"
if config.distribution == "ubuntu":
full_series = config.series
else:
full_series = "%s-%s" % (config.distribution, config.series)
lock_path = os.path.join(
config.root, "etc", ".lock-build-image-set-%s-%s-%s" %
(project, full_series, config.image_type))
try:
subprocess.check_call(["lockfile", "-l", "7200", "-r", "0", lock_path])
except subprocess.CalledProcessError:
logger.error("Another image set is already building!")
raise
try:
yield
finally:
osextras.unlink_force(lock_path)
def publish_source(self, date):
for i in count(1):
in_prefix = "%s-src-%d" % (self.config.series, i)
out_prefix = "%s-src-%d" % (self.config.series, i)
source_dir = os.path.join(self.image_output, "src")
source_prefix = os.path.join(source_dir, in_prefix)
target_dir = os.path.join(self.publish_base, date, "source")
target_prefix = os.path.join(target_dir, out_prefix)
if not os.path.exists("%s.raw" % source_prefix):
break
logger.info("Publishing source %d ..." % i)
osextras.ensuredir(target_dir)
shutil.move("%s.raw" % source_prefix, "%s.iso" % target_prefix)
shutil.move("%s.list" % source_prefix, "%s.list" % target_prefix)
with ChecksumFileSet(
self.config, target_dir, sign=False) as checksum_files:
checksum_files.remove("%s.iso" % out_prefix)
# Jigdo integration
if os.path.exists("%s.jigdo" % source_prefix):
logger.info("Publishing source %d jigdo ..." % i)
shutil.move(
"%s.jigdo" % source_prefix, "%s.jigdo" % target_prefix)
shutil.move(
"%s.template" % source_prefix,
"%s.template" % target_prefix)
else:
logger.warning("No jigdo for source %d!" % i)
osextras.unlink_force("%s.jigdo" % target_prefix)
osextras.unlink_force("%s.template" % target_prefix)
# zsync metafiles
if self.try_zsyncmake and osextras.find_on_path("zsyncmake"):
logger.info("Making source %d zsync metafile ..." % i)
osextras.unlink_force("%s.iso.zsync" % target_prefix)
zsyncmake(
"%s.iso" % target_prefix, "%s.iso.zsync" % target_prefix,
"%s.iso" % out_prefix)
yield os.path.join(
self.project, self.image_type, "%s-src" % self.config.series)
def anonftpsync(config):
env = dict(os.environ)
for key, value in _anonftpsync_options(config).items():
env[key] = value
target = os.path.join(config.root, "ftp")
fqdn = socket.getfqdn()
lock_base = "Archive-Update-in-Progress-%s" % fqdn
lock = os.path.join(target, lock_base)
pkglist = "--include-from=" + config["RSYNC_PKGLIST_PATH"]
if subprocess.call(
["lockfile", "-!", "-l", "43200", "-r", "0", lock]) == 0:
raise Exception(
"%s is unable to start rsync; lock file exists." % fqdn)
try:
log_path = os.path.join(config.root, "log", "rsync.log")
osextras.ensuredir(os.path.dirname(log_path))
with open(log_path, "w") as log:
command_base = [
"rsync", "--recursive", "--links", "--hard-links", "--times",
"--verbose", "--stats", "--chmod=Dg+s,g+rwX",
pkglist,
"--exclude", lock_base,
"--exclude", "project/trace/%s" % fqdn,
]
exclude = env.get("RSYNC_EXCLUDE", "").split()
include = env.get("RSYNC_INCLUDE", "").split()
source_target = ["%s/" % env["RSYNC_SRC"], "%s/" % target]
subprocess.call(
command_base + [
"--exclude", "Packages*", "--exclude", "Sources*",
"--exclude", "Release*", "--exclude", "InRelease",
"--include", "i18n/by-hash/**", "--exclude", "i18n/*",
] + include + exclude + source_target,
stdout=log, stderr=subprocess.STDOUT, env=env)
# Second pass to update metadata and clean up old files.
subprocess.call(
command_base + [
"--delay-updates", "--delete", "--delete-after",
] + include + exclude + source_target,
stdout=log, stderr=subprocess.STDOUT, env=env)
# Delete dangling symlinks.
for dirpath, _, filenames in os.walk(target):
for filename in filenames:
path = os.path.join(dirpath, filename)
if os.path.islink(path) and not os.path.exists(path):
os.unlink(path)
trace_dir = os.path.join(target, "project", "trace")
osextras.ensuredir(trace_dir)
with open(os.path.join(trace_dir, fqdn), "w") as trace:
subprocess.check_call(["date", "-u"], stdout=trace)
# Note: if you don't have savelog, use any other log rotation
# facility, or comment this out, the log will simply be overwritten
# each time.
with open("/dev/null", "w") as devnull:
subprocess.call(
["savelog", log_path],
stdout=devnull, stderr=subprocess.STDOUT)
finally:
osextras.unlink_force(lock)
def test_unlink_file_missing(self):
path = os.path.join(self.temp_dir, "file")
osextras.unlink_force(path)
self.assertFalse(os.path.exists(path))
def test_unlink_file_present(self):
path = os.path.join(self.temp_dir, "file")
touch(path)
osextras.unlink_force(path)
self.assertFalse(os.path.exists(path))
def build_ubuntu_defaults_locale(config):
locale = config["UBUNTU_DEFAULTS_LOCALE"]
if locale != "zh_CN":
raise UnknownLocale(
"UBUNTU_DEFAULTS_LOCALE='%s' not currently supported!" % locale)
series = config["DIST"]
if series < "oneiric":
# Original hack: repack an existing image.
iso = config["ISO"]
if not iso:
raise Exception(
"Pass ISO=<path to Ubuntu image> in the environment.")
scratch = os.path.join(
config.root, "scratch", "ubuntu-chinese-edition", series.name)
bsdtar_tree = os.path.join(scratch, "bsdtar-tree")
log_marker("Unpacking")
if os.path.isdir(bsdtar_tree):
subprocess.check_call(["chmod", "-R", "+w", bsdtar_tree])
osextras.mkemptydir(bsdtar_tree)
subprocess.check_call(["bsdtar", "-xf", iso, "-C", bsdtar_tree])
subprocess.check_call(["chmod", "-R", "+w", bsdtar_tree])
log_marker("Transforming (robots in disguise)")
with open(os.path.join(bsdtar_tree, "isolinux", "lang"), "w") as lang:
print(locale, file=lang)
subprocess.call([
"mkisofs",
"-r", "-V", "Ubuntu Chinese %s i386" % series.version,
"-o", os.path.join(scratch, os.path.basename(iso)),
"-cache-inodes", "-J", "-l",
"-b", "isolinux/isolinux.bin", "-c", "isolinux/boot.cat",
"-no-emul-boot", "-boot-load-size", "4", "-boot-info-table",
bsdtar_tree,
])
iso_prefix = iso.rsplit(".", 1)[0]
scratch_prefix = os.path.join(
scratch, os.path.basename(iso).rsplit(".", 1)[0])
for ext in "list", "manifest":
if os.path.exists("%s.%s" % (iso_prefix, ext)):
shutil.copy2(
"%s.%s" % (iso_prefix, ext),
"%s.%s" % (scratch_prefix, ext))
else:
osextras.unlink_force("%s.%s" % (scratch_prefix, ext))
else:
log_marker("Downloading live filesystem images")
download_live_filesystems(config)
scratch = live_output_directory(config)
for entry in os.listdir(scratch):
if "." in entry:
os.rename(
os.path.join(scratch, entry),
os.path.join(scratch, "%s-desktop-%s" % (series, entry)))
pi_makelist = os.path.join(
config.root, "debian-cd", "tools", "pi-makelist")
for entry in os.listdir(scratch):
if entry.endswith(".iso"):
entry_path = os.path.join(scratch, entry)
list_path = "%s.list" % entry_path.rsplit(".", 1)[0]
with open(list_path, "w") as list_file:
subprocess.check_call(
[pi_makelist, entry_path], stdout=list_file)
def test_unlink_file_missing(self):
path = os.path.join(self.temp_dir, "file")
osextras.unlink_force(path)
self.assertFalse(os.path.exists(path))
def test_unlink_file_present(self):
path = os.path.join(self.temp_dir, "file")
touch(path)
osextras.unlink_force(path)
self.assertFalse(os.path.exists(path))
def __exit__(self, unused_exc_type, unused_exc_value, unused_exc_tb):
osextras.unlink_force(self.lock_path)
def publish(self, date):
self.new_publish_dir(date)
published = []
self.checksum_dirs = []
if not self.config["CDIMAGE_ONLYSOURCE"]:
for arch in self.config.arches:
published.extend(
list(self.publish_binary(self.publish_type, arch, date)))
if self.project == "edubuntu" and self.publish_type == "server":
for arch in self.config.arches:
published.extend(
list(self.publish_binary("serveraddon", arch, date)))
published.extend(list(self.publish_source(date)))
if not published:
logger.warning("No CDs produced!")
return
target_dir = os.path.join(self.publish_base, date)
source_report = os.path.join(
self.britney_report, "%s_probs.html" % self.config.series)
target_report = os.path.join(target_dir, "report.html")
if (self.config["CDIMAGE_INSTALL_BASE"] and
os.path.exists(source_report)):
shutil.copy2(source_report, target_report)
else:
osextras.unlink_force(target_report)
if not self.config["CDIMAGE_ONLYSOURCE"]:
checksum_directory(
self.config, target_dir, old_directories=self.checksum_dirs,
map_expr=r"s/\.\(img\|img\.gz\|iso\|iso\.gz\|tar\.gz\)$/.raw/")
subprocess.check_call(
[os.path.join(self.config.root, "bin", "make-web-indices"),
target_dir, self.config.series, "daily"])
target_dir_source = os.path.join(target_dir, "source")
if os.path.isdir(target_dir_source):
checksum_directory(
self.config, target_dir_source,
old_directories=[os.path.join(self.image_output, "src")],
map_expr=r"s/\.\(img\|img\.gz\|iso\|iso\.gz\|tar\.gz\)$/.raw/")
subprocess.check_call(
[os.path.join(self.config.root, "bin", "make-web-indices"),
target_dir_source, self.config.series, "daily"])
if (self.image_type.endswith("-live") or
self.image_type.endswith("dvd")):
# Create and publish metalink files.
md5sums_metalink = os.path.join(target_dir, "MD5SUMS-metalink")
md5sums_metalink_gpg = os.path.join(
target_dir, "MD5SUMS-metalink.gpg")
osextras.unlink_force(md5sums_metalink)
osextras.unlink_force(md5sums_metalink_gpg)
basedir, reldir = self.metalink_dirs(date)
if subprocess.call([
os.path.join(self.config.root, "bin", "make-metalink"),
basedir, self.config.series, reldir, "cdimage.ubuntu.com",
]) == 0:
metalink_checksum_directory(self.config, target_dir)
else:
for name in os.listdir(target_dir):
if name.endswith(".metalink"):
osextras.unlink_force(os.path.join(target_dir, name))
publish_current = os.path.join(self.publish_base, "current")
osextras.unlink_force(publish_current)
os.symlink(date, publish_current)
manifest_lock = os.path.join(
self.config.root, "etc", ".lock-manifest-daily")
try:
subprocess.check_call(["lockfile", "-r", "4", manifest_lock])
except subprocess.CalledProcessError:
logger.error("Couldn't acquire manifest-daily lock!")
raise
try:
manifest_daily = os.path.join(
self.tree.directory, ".manifest-daily")
with AtomicFile(manifest_daily) as manifest_daily_file:
for line in self.tree.manifest():
print(line, file=manifest_daily_file)
os.chmod(
manifest_daily, os.stat(manifest_daily).st_mode | stat.S_IWGRP)
# Create timestamps for this run.
# TODO cjwatson 20120807: Shouldn't these be in www/full
# rather than www/full[/project]?
trace_dir = os.path.join(self.full_tree, ".trace")
osextras.ensuredir(trace_dir)
fqdn = socket.getfqdn()
with open(os.path.join(trace_dir, fqdn), "w") as trace_file:
subprocess.check_call(["date", "-u"], stdout=trace_file)
finally:
osextras.unlink_force(manifest_lock)
subprocess.check_call([
os.path.join(self.config.root, "bin", "post-qa"), date,
] + published)
def publish_binary(self, publish_type, arch, date):
in_prefix = "%s-%s-%s" % (self.config.series, publish_type, arch)
out_prefix = "%s-%s-%s" % (self.config.series, publish_type, arch)
source_dir = os.path.join(self.image_output, arch)
source_prefix = os.path.join(source_dir, in_prefix)
target_dir = os.path.join(self.publish_base, date)
target_prefix = os.path.join(target_dir, out_prefix)
if not os.path.exists("%s.raw" % source_prefix):
logger.warning("No %s image for %s!" % (publish_type, arch))
for name in osextras.listdir_force(target_dir):
if name.startswith("%s." % out_prefix):
os.unlink(os.path.join(target_dir, name))
return
logger.info("Publishing %s ..." % arch)
osextras.ensuredir(target_dir)
extension = self.detect_image_extension(source_prefix)
shutil.move(
"%s.raw" % source_prefix, "%s.%s" % (target_prefix, extension))
if os.path.exists("%s.list" % source_prefix):
shutil.move("%s.list" % source_prefix, "%s.list" % target_prefix)
self.checksum_dirs.append(source_dir)
with ChecksumFileSet(
self.config, target_dir, sign=False) as checksum_files:
checksum_files.remove("%s.%s" % (out_prefix, extension))
# Jigdo integration
if os.path.exists("%s.jigdo" % source_prefix):
logger.info("Publishing %s jigdo ..." % arch)
shutil.move("%s.jigdo" % source_prefix, "%s.jigdo" % target_prefix)
shutil.move(
"%s.template" % source_prefix, "%s.template" % target_prefix)
if self.jigdo_ports(arch):
self.replace_jigdo_mirror(
"%s.jigdo" % target_prefix,
"http://archive.ubuntu.com/ubuntu",
"http://ports.ubuntu.com/ubuntu-ports")
else:
osextras.unlink_force("%s.jigdo" % target_prefix)
osextras.unlink_force("%s.template" % target_prefix)
# Live filesystem manifests
if os.path.exists("%s.manifest" % source_prefix):
logger.info("Publishing %s live manifest ..." % arch)
shutil.move(
"%s.manifest" % source_prefix, "%s.manifest" % target_prefix)
else:
osextras.unlink_force("%s.manifest" % target_prefix)
if (self.config["CDIMAGE_SQUASHFS_BASE"] and
os.path.exists("%s.squashfs" % source_prefix)):
logger.info("Publishing %s squashfs ..." % arch)
shutil.move(
"%s.squashfs" % source_prefix, "%s.squashfs" % target_prefix)
else:
osextras.unlink_force("%s.squashfs" % target_prefix)
# Flashable Android boot images
if os.path.exists("%s.bootimg" % source_prefix):
logger.info("Publishing %s abootimg bootloader images ..." % arch)
shutil.move(
"%s.bootimg" % source_prefix, "%s.bootimg" % target_prefix)
# zsync metafiles
if self.try_zsyncmake and osextras.find_on_path("zsyncmake"):
logger.info("Making %s zsync metafile ..." % arch)
osextras.unlink_force("%s.%s.zsync" % (target_prefix, extension))
zsyncmake(
"%s.%s" % (target_prefix, extension),
"%s.%s.zsync" % (target_prefix, extension),
"%s.%s" % (out_prefix, extension))
size = os.stat("%s.%s" % (target_prefix, extension)).st_size
if size > self.size_limit_extension(extension):
with open("%s.OVERSIZED" % target_prefix, "a"):
pass
else:
osextras.unlink_force("%s.OVERSIZED" % target_prefix)
yield os.path.join(self.project, self.image_type_dir, in_prefix)
