def germinate_project(self, project):
osextras.mkemptydir(self.output_dir(project))
for arch in self.config.arches:
logger.info("Germinating for %s/%s ..." %
(self.config.series, arch))
self.germinate_arch(project, arch)
def zsyncmake(infile, outfile, url):
command = ["zsyncmake"]
if infile.endswith(".gz"):
command.append("-Z")
command.extend(["-o", outfile, "-u", url, infile])
if subprocess.call(command) != 0:
logger.info("Trying again with block size 2048 ...")
command[1:1] = ["-b", "2048"]
subprocess.check_call(command)
def live_build_finished(arch, full_name, machine, status, text_status,
lp_build=None):
timestamp = time.strftime("%F %T")
logger.info("%s on %s finished at %s (%s)" % (
full_name, machine, timestamp, text_status))
tracker_set_rebuild_status(config, [0, 1, 2], 3, arch)
if status == 0:
successful.add(arch)
if arch == "amd64" and "amd64+mac" in config.arches:
successful.add("amd64+mac")
else:
live_build_notify_failure(config, arch, lp_build=lp_build)
def _trigger_mirror(config, key, user, host, background=False):
logger.info("%s:" % host)
command = [
"ssh",
"-i",
key,
"-o",
"StrictHostKeyChecking no",
"-o",
"BatchMode yes",
"%s@%s" % (user, host),
_trigger_command(config),
]
if background:
subprocess.Popen(command)
else:
subprocess.call(command)
def publish_source(self, date):
for i in count(1):
in_prefix = "%s-src-%d" % (self.config.series, i)
out_prefix = "%s-src-%d" % (self.config.series, i)
source_dir = os.path.join(self.image_output, "src")
source_prefix = os.path.join(source_dir, in_prefix)
target_dir = os.path.join(self.publish_base, date, "source")
target_prefix = os.path.join(target_dir, out_prefix)
if not os.path.exists("%s.raw" % source_prefix):
break
logger.info("Publishing source %d ..." % i)
osextras.ensuredir(target_dir)
shutil.move("%s.raw" % source_prefix, "%s.iso" % target_prefix)
shutil.move("%s.list" % source_prefix, "%s.list" % target_prefix)
with ChecksumFileSet(
self.config, target_dir, sign=False) as checksum_files:
checksum_files.remove("%s.iso" % out_prefix)
# Jigdo integration
if os.path.exists("%s.jigdo" % source_prefix):
logger.info("Publishing source %d jigdo ..." % i)
shutil.move(
"%s.jigdo" % source_prefix, "%s.jigdo" % target_prefix)
shutil.move(
"%s.template" % source_prefix,
"%s.template" % target_prefix)
else:
logger.warning("No jigdo for source %d!" % i)
osextras.unlink_force("%s.jigdo" % target_prefix)
osextras.unlink_force("%s.template" % target_prefix)
# zsync metafiles
if self.try_zsyncmake and osextras.find_on_path("zsyncmake"):
logger.info("Making source %d zsync metafile ..." % i)
osextras.unlink_force("%s.iso.zsync" % target_prefix)
zsyncmake(
"%s.iso" % target_prefix, "%s.iso.zsync" % target_prefix,
"%s.iso" % out_prefix)
yield os.path.join(
self.project, self.image_type, "%s-src" % self.config.series)
def log_marker(message):
logger.info("===== %s =====" % message)
logger.info(time.strftime("%a %b %e %H:%M:%S UTC %Y", time.gmtime()))
def run_live_builds(config):
builds = {}
lp_builds = []
for arch in config.arches:
if arch == "amd64+mac":
# Use normal amd64 live image on amd64+mac.
continue
full_name = live_build_full_name(config, arch)
timestamp = time.strftime("%F %T")
lp, lp_livefs = get_lp_livefs(config, arch)
if lp_livefs is None:
machine = live_builder(config, arch)
else:
machine = "Launchpad"
logger.info(
"%s on %s starting at %s" % (full_name, machine, timestamp))
tracker_set_rebuild_status(config, [0, 1], 2, arch)
if lp_livefs is not None:
lp_kwargs = live_build_lp_kwargs(config, lp, lp_livefs, arch)
lp_build = lp_livefs.requestBuild(**lp_kwargs)
logger.info("%s: %s" % (full_name, lp_build.web_link))
lp_builds.append((lp_build, arch, full_name, machine, None))
else:
proc = subprocess.Popen(live_build_command(config, arch))
builds[proc.pid] = (proc, arch, full_name, machine)
successful = set()
def live_build_finished(arch, full_name, machine, status, text_status,
lp_build=None):
timestamp = time.strftime("%F %T")
logger.info("%s on %s finished at %s (%s)" % (
full_name, machine, timestamp, text_status))
tracker_set_rebuild_status(config, [0, 1, 2], 3, arch)
if status == 0:
successful.add(arch)
if arch == "amd64" and "amd64+mac" in config.arches:
successful.add("amd64+mac")
else:
live_build_notify_failure(config, arch, lp_build=lp_build)
while builds or lp_builds:
# Check for non-Launchpad build results.
if builds:
pid, status = os.waitpid(0, os.WNOHANG)
if pid and pid in builds:
_, arch, full_name, machine = builds.pop(pid)
live_build_finished(
arch, full_name, machine, status,
"success" if status == 0 else "failed")
# Check for Launchpad build results.
pending_lp_builds = []
for lp_item in lp_builds:
lp_build, arch, full_name, machine, log_timeout = lp_item
lp_build.lp_refresh()
if lp_build.buildstate in (
"Needs building", "Currently building", "Uploading build"):
pending_lp_builds.append(lp_item)
elif lp_build.buildstate == "Successfully built":
live_build_finished(
arch, full_name, machine, 0, lp_build.buildstate,
lp_build=lp_build)
elif (lp_build.build_log_url is None and
(log_timeout is None or time.time() < log_timeout)):
# Wait up to five minutes for Launchpad to fetch the build
# log from the slave. We need a timeout since in rare cases
# this might fail.
if log_timeout is None:
log_timeout = time.time() + 300
pending_lp_builds.append(
(lp_build, arch, full_name, machine, log_timeout))
else:
live_build_finished(
arch, full_name, machine, 1, lp_build.buildstate,
lp_build=lp_build)
lp_builds = pending_lp_builds
if lp_builds:
# Wait a while before polling Launchpad again. If a
# non-Launchpad build completes in the meantime, it will
# interrupt this sleep with SIGCHLD.
time.sleep(15)
if not successful:
raise LiveBuildsFailed("No live filesystem builds succeeded.")
return successful
def germinate_arch(self, project, arch):
cpuarch = arch.split("+")[0]
for dist in self.germinate_dists:
for suffix in (
"binary-%s/Packages.gz" % cpuarch,
"source/Sources.gz",
"debian-installer/binary-%s/Packages.gz" % cpuarch,
):
files = [
"dists/%s/%s/%s" % (dist, component, suffix)
for component in self.components
]
if self.config["LOCAL"]:
files.append("%s/dists/%s/local/%s" %
(self.config["LOCALDEBS"], dist, suffix))
self.make_index(project, arch, files[0], files)
arch_output_dir = os.path.join(self.output_dir(project), arch)
osextras.mkemptydir(arch_output_dir)
if (self.config["GERMINATE_HINTS"]
and os.path.isfile(self.config["GERMINATE_HINTS"])):
shutil.copy2(self.config["GERMINATE_HINTS"],
os.path.join(arch_output_dir, "hints"))
command = [
self.germinate_path,
"--seed-source",
",".join(self.seed_sources(project)),
"--mirror",
"file://%s/" % self.output_dir(project),
"--seed-dist",
self.seed_dist(project),
"--dist",
",".join(self.germinate_dists),
"--arch",
cpuarch,
"--components",
"main",
"--no-rdepends",
]
if self.use_bzr:
command.append("--bzr")
if self.config.image_type == "source":
command.append("--always-follow-build-depends")
proxy_check_call(self.config,
"germinate",
command,
cwd=arch_output_dir)
output_structure = os.path.join(self.output_dir(project), "STRUCTURE")
shutil.copy2(os.path.join(arch_output_dir, "structure"),
output_structure)
if self.config.series == "breezy":
# Unfortunately, we now need a second germinate run to figure
# out the dependencies of language packs and the like.
extras = []
with open(os.path.join(arch_output_dir, "ship.acsets"),
"w") as ship_acsets:
output = GerminateOutput(self.config, output_structure)
for pkg in output.seed_packages(arch, "ship.seed"):
extras.append("desktop/%s" % pkg)
print(pkg, file=ship_acsets)
if extras:
logger.info(
"Re-germinating for %s/%s language pack dependencies ..." %
(self.config.series, arch))
command.extend(["--seed-packages", ",".join(extras)])
proxy_check_call(self.config,
"germinate",
command,
cwd=arch_output_dir)
def send_mail(subject, generator, recipients, body, dry_run=False):
if dry_run:
logger.info("Would send mail to: %s" % ", ".join(recipients))
logger.info("")
logger.info("Subject: %s" % subject)
logger.info("X-Generated-By: %s" % generator)
logger.info("")
if isinstance(body, text_file_type):
for line in body:
logger.info(line.rstrip("\n"))
else:
for line in body.splitlines():
logger.info(line)
logger.info("")
else:
command = [
"mail", "-s", subject, "-a", "X-Generated-By: %s" % generator]
command.extend(recipients)
if isinstance(body, text_file_type):
mailer = subprocess.Popen(command, stdin=body)
else:
mailer = subprocess.Popen(command, stdin=subprocess.PIPE)
if bytes is not str and isinstance(body, str):
body = body.encode()
mailer.stdin.write(body)
mailer.stdin.close()
mailer.wait()
def publish_binary(self, publish_type, arch, date):
in_prefix = "%s-%s-%s" % (self.config.series, publish_type, arch)
out_prefix = "%s-%s-%s" % (self.config.series, publish_type, arch)
source_dir = os.path.join(self.image_output, arch)
source_prefix = os.path.join(source_dir, in_prefix)
target_dir = os.path.join(self.publish_base, date)
target_prefix = os.path.join(target_dir, out_prefix)
if not os.path.exists("%s.raw" % source_prefix):
logger.warning("No %s image for %s!" % (publish_type, arch))
for name in osextras.listdir_force(target_dir):
if name.startswith("%s." % out_prefix):
os.unlink(os.path.join(target_dir, name))
return
logger.info("Publishing %s ..." % arch)
osextras.ensuredir(target_dir)
extension = self.detect_image_extension(source_prefix)
shutil.move(
"%s.raw" % source_prefix, "%s.%s" % (target_prefix, extension))
if os.path.exists("%s.list" % source_prefix):
shutil.move("%s.list" % source_prefix, "%s.list" % target_prefix)
self.checksum_dirs.append(source_dir)
with ChecksumFileSet(
self.config, target_dir, sign=False) as checksum_files:
checksum_files.remove("%s.%s" % (out_prefix, extension))
# Jigdo integration
if os.path.exists("%s.jigdo" % source_prefix):
logger.info("Publishing %s jigdo ..." % arch)
shutil.move("%s.jigdo" % source_prefix, "%s.jigdo" % target_prefix)
shutil.move(
"%s.template" % source_prefix, "%s.template" % target_prefix)
if self.jigdo_ports(arch):
self.replace_jigdo_mirror(
"%s.jigdo" % target_prefix,
"http://archive.ubuntu.com/ubuntu",
"http://ports.ubuntu.com/ubuntu-ports")
else:
osextras.unlink_force("%s.jigdo" % target_prefix)
osextras.unlink_force("%s.template" % target_prefix)
# Live filesystem manifests
if os.path.exists("%s.manifest" % source_prefix):
logger.info("Publishing %s live manifest ..." % arch)
shutil.move(
"%s.manifest" % source_prefix, "%s.manifest" % target_prefix)
else:
osextras.unlink_force("%s.manifest" % target_prefix)
if (self.config["CDIMAGE_SQUASHFS_BASE"] and
os.path.exists("%s.squashfs" % source_prefix)):
logger.info("Publishing %s squashfs ..." % arch)
shutil.move(
"%s.squashfs" % source_prefix, "%s.squashfs" % target_prefix)
else:
osextras.unlink_force("%s.squashfs" % target_prefix)
# Flashable Android boot images
if os.path.exists("%s.bootimg" % source_prefix):
logger.info("Publishing %s abootimg bootloader images ..." % arch)
shutil.move(
"%s.bootimg" % source_prefix, "%s.bootimg" % target_prefix)
# zsync metafiles
if self.try_zsyncmake and osextras.find_on_path("zsyncmake"):
logger.info("Making %s zsync metafile ..." % arch)
osextras.unlink_force("%s.%s.zsync" % (target_prefix, extension))
zsyncmake(
"%s.%s" % (target_prefix, extension),
"%s.%s.zsync" % (target_prefix, extension),
"%s.%s" % (out_prefix, extension))
size = os.stat("%s.%s" % (target_prefix, extension)).st_size
if size > self.size_limit_extension(extension):
with open("%s.OVERSIZED" % target_prefix, "a"):
pass
else:
osextras.unlink_force("%s.OVERSIZED" % target_prefix)
yield os.path.join(self.project, self.image_type_dir, in_prefix)
