def perform_copy(context: Context, input: str = None, stage: str = None, output: str = None, recursive: bool = False) -> None: input = context.input(input, allow_none=True) stage = context.stage(stage, create_parents=True, allow_none=True) output = context.output(output, create_parents=True, allow_none=True) paths = [path for path in [input, stage, output] if path is not None] if len(paths) != 2: raise Exception("copy step expects two of {input, stage, output}!") source, dest = paths project.log("copy", context.namepath(source), "to", context.namepath(dest)) if recursive: if os.path.isdir(dest): if os.listdir(dest): raise Exception( "cannot recursively copy into directory with existing contents" ) for rel in os.listdir(source): shutil.copytree(os.path.join(source, rel), os.path.join(dest, rel), copy_function=util.copy3, symlinks=True) else: if os.path.exists(dest): raise Exception("cannot recursively copy over existing file") shutil.copytree(source, dest, copy_function=util.copy3, symlinks=True) else: util.copy3(source, dest)
def upload(bindir: str, branch_config: aptbranch.Config) -> None: branch = branch_config.name keyid = branch_config.signing_key files = os.listdir(bindir) project.log("upload", "preparing uploads...") with tempfile.TemporaryDirectory() as tempdir: uploads = {} debs = [] if any(file.endswith(".aci") for file in files): project.log("upload", "preparing upload of acis...") for file in files: path = os.path.join(bindir, file) if not os.path.isfile(path): raise Exception("not a normal file: %s" % file) if file.endswith(".aci"): upload_aci(path, uploads, keyid) elif file.endswith(".deb"): debs.append(path) project.log("upload", "preparing upload of debs...") upload_apt(debs, uploads, keyid, tempdir) project.log("upload", "performing", len(uploads), "uploads to", branch) perform_uploads(uploads, branch_config) project.log("upload", "upload to", branch, "complete!")
def final_tgz(context: Context, control: dict): project.log("tar", "creating", context.project.pkgbase, "version", context.project.full_version) subprocess.check_call([ "tar", "-C", context.outputdir, "-czf", context.project.get_output_path(context.branch), "--" ] + os.listdir(context.outputdir))
def perform_upstream(context: Context, upstream: str, version: str, stage: str): upstream = upstream.replace("%", version) project.log("upstream", "importing", upstream) input = context.upstream(upstream) stage = context.stage(stage.replace("%", version), create_parents=True) shutil.copyfile(input, stage)
def perform_remove(context: Context, stage: str, recursive: bool = False): project.log("remove", "removing directory" if recursive else "removing file", stage) path = context.stage(stage) if recursive: shutil.rmtree(path) else: os.remove(path)
def perform_debclean(context: Context, stage: str, options: list) -> None: if not options: raise Exception( "expected at least one option to be specified for debclean") for option in options: if option not in debclean.DEBCLEAN_OPTIONS: raise Exception("invalid debclean option: %s" % option) project.log("debclean", "performing cleaning with options:", *options) rootfs = context.stage(stage, require_existence=True) for opt in options: debclean.DEBCLEAN_OPTIONS[opt](rootfs)
def final_aci(context: Context, control: dict) -> None: project.log("acbuild", context.project.pkgbase, "version", context.project.full_version) with acbuild.Build(context.branch) as b: b.set_name("homeworld.mit.edu/" + context.project.pkgbase) if "set-exec" in control: b.set_exec(*control["set-exec"].split(" ")) b.label_add("version", context.project.full_version) for x in os.listdir(context.outputdir): b.copy_in(os.path.join(context.outputdir, x), "/" + x) for port in control.get("ports", []): b.port_add(port["name"], port["protocol"], port["port"]) b.write(context.project.get_output_path(context.branch))
def perform_debinstall(context: Context, packages: list, stage: str): # TODO: move as many checks as possible into schema validation if not packages: raise Exception( "expected at least one package to be specified for debinstall") project.log("debinstall", "installing", len(packages), "debian packages") rootfs = context.stage(stage, require_existence=True) chroot_base = ["fakeroot", "fakechroot", "chroot", rootfs] subprocess.check_call(chroot_base + ["apt-get", "update"]) subprocess.check_call(chroot_base + ["apt-get", "install", "-y", "--"] + packages) debclean.clean_apt_files(rootfs)
def prepare_exec_source_target(context: Context, kind: str, input: str, stage: str, output: str): source = context.input(input, allow_none=True) dest = context.output(output, create_parents=True, allow_none=True) if stage is not None: if dest is None: dest = context.stage(stage, create_parents=True, allow_none=True) elif source is None: source = context.stage(stage, require_existence=True, allow_none=True) else: raise Exception( kind + " step expects at most two of {input, stage, output}") if source is not None and dest is not None: project.log(kind, "generating", context.namepath(dest), "from", context.namepath(source)) elif source is not None: project.log(kind, "processing", context.namepath(source)) elif dest is not None: project.log(kind, "generating", context.namepath(dest)) else: project.log(kind, "running snippet") if source is not None: with open(source, "r") as f: source = f.read() return source, dest
def final_deb(context: Context, control: dict) -> None: project.log("debuild", context.project.pkgbase, "version", context.project.full_version) depends = control.get("depends", []) install_scripts = control.get("install-scripts", {}) install_scripts = { key: context.input(value) for key, value in install_scripts.items() } debuild.perform_debuild(context.outputdir, context.project.pkgbase, context.project.full_version, context.project.change_date, depends, context.project.get_output_path(context.branch), install_scripts)
def perform_mkdir(context: Context, stage: str = None, output: str = None, recursive: bool = False) -> None: if stage is not None: assert output is None project.log("mkdir", stage) target = context.stage(stage) else: assert output is not None, "schema should ensure this" target = context.output(output) project.log("mkdir", context.namepath(output)) if recursive: os.makedirs(target) else: os.mkdir(target)
def perform_fakechroot_clean(context: Context, stage: str) -> None: """cleans up any symbolic links pointing with absolute paths to the build directory itself""" project.log("fakechroot-clean", "cleaning up directory:", stage) rootfs = context.stage(stage, require_existence=True) for root, dirs, files in os.walk(rootfs): for f in files: path = os.path.join(root, f) if not os.path.islink(path): continue full_link = os.readlink(path) if not os.path.isabs(full_link): continue rootrel = os.path.relpath(full_link, rootfs) if rootrel.split("/")[0] == "..": # doesn't point within the rootfs; nothing to do continue os.remove(path) os.symlink(os.path.join("/", rootrel), path)
def perform_upstream_extract(context: Context, upstream: str, version: str, stage: str, focus: str = None) -> None: upstream = upstream.replace("%", version) project.log("upstream", "unpacking archive", upstream) tarfile = context.upstream(upstream) targetdir = context.stage(stage.replace("%", version)) if not os.path.isdir(targetdir): os.makedirs(targetdir) if focus is not None: focus = os.path.normpath(focus.replace("%", version)) strip_count = focus.strip("/").count("/") + 1 subprocess.check_call([ "tar", "-C", targetdir, "-xf", tarfile, "--strip-components=%d" % strip_count, focus ]) else: subprocess.check_call(["tar", "-C", targetdir, "-xf", tarfile])
def perform_debremove(context: Context, packages: list, stage: str, force_remove_essential: bool = False, force_depends: bool = False, no_triggers: bool = False): if not packages: raise Exception( "expected at least one package to be specified for debremove") project.log("debremove", "removing", len(packages), "debian packages") args = ["fakeroot", "fakechroot", "dpkg", "--purge"] args += ["--root=" + context.stage(stage, require_existence=True)] if force_remove_essential: args += ["--force-remove-essential"] if force_depends: args += ["--force-depends"] if no_triggers: args += ["--no-triggers"] args += packages subprocess.check_call(args)
def perform_aci_unpack(context: Context, name: str, version: str, stage: str = None, output: str = None): if [stage, output].count(None) != 1: raise Exception( "aci-unpack expects exactly one of {stage, output} to be specified" ) if stage is None: targetdir = context.output(output, create_parents=True) else: targetdir = context.stage(stage, create_parents=True) project.log("aci", "unpacking container rootfs from", name, "version", version, "to", context.namepath(targetdir)) if not os.path.isdir(targetdir): os.makedirs(targetdir) aci = project.get_output_path_for_aci(context.branch, name, version) subprocess.check_call( ["tar", "-C", targetdir, "-xf", aci, "--strip-components=1", "rootfs"])
def perform_go_build(context: Context, version: str, stage: str, sources_input: list = (), packages: list = (), gopath: str = "go", no_cgo: bool = False, ldflags: str = None): if not sources_input and not packages: raise Exception( "go-build expects at least one source file or package to build") project.log("go", "(%s)" % version, "compiling output", stage) env = { "GOPATH": context.stage(gopath, require_existence=True), "CGO_ENABLED": "0" if no_cgo else "1" } output = context.stage(stage, create_parents=True) sources = [context.input(source) for source in sources_input] + list(packages) gobuild.build(context.branch, version, sources, output, env, ldflags)
def perform_debootstrap(context: Context, release: str, version: str, stage: str, extra: list = ()): project.log("debootstrap", "bootstrapping debian release", release, "at version", version, "with", len(extra), "extra packages") # NOTE: if this fails with a fakechroot error, that probably means that you should update the version of debian that # you're bootstrapping. args = [ "fakeroot", "fakechroot", "debootstrap", "--components=main", "--variant=minbase" ] if extra: args += ["--include=" + ",".join(extra)] args += [ release, context.stage(stage, create_parents=True), "http://snapshot.debian.org/archive/debian/%s/" % version ] subprocess.check_call(args)
def perform_go_build(context: Context, version: str, stage: str, packages: list = (), gopath: str = "go", no_cgo: bool = False, ldflags: str = None): if not packages: raise Exception("go-build expects at least one package to build") newpath = [] for segment in gopath.split(":"): if os.path.isabs(segment): newpath.append(segment) else: newpath.append(context.stage(segment, require_existence=True)) project.log("go", "(%s)" % version, "compiling output", stage) env = {"GOPATH": ":".join(newpath), "CGO_ENABLED": "0" if no_cgo else "1"} output = context.stage(stage, create_parents=True) gobuild.build(context.branch, version, list(packages), output, env, ldflags)
def perform_acbuild(context: Context, name: str, stage: str, exec: str = None, copy: list = (), env: dict = None, mounts: dict = None, labels: dict = None, ports: list = ()): project.log("acbuild", "building container", name) with acbuild.Build(context.branch) as build: build.set_name(name) if exec is not None: build.set_exec(exec) for copyentry in copy: output, input, stageent = copyentry["output"], copyentry.get( "input"), copyentry.get("stage") if [input, stageent].count(None) != 1: raise Exception( "acbuild/copy must have either input or stage, not both!") if input is not None: build.copy_in(context.input(input), output) else: build.copy_in(context.stage(stageent), output) if env is not None: for key, value in env.items(): build.env_add(key, value) if mounts is not None: for key, value in mounts.items(): build.mount_add(key, value) if labels is not None: for key, value in labels.items(): build.label_add(key, value) if ports is not None: for port in ports: build.port_add(port["name"], port["protocol"], port["port"]) build.write(context.stage(stage))
def perform_go_prepare(context: Context, version: str, stage: str): project.log("go", "(%s)" % version, "preparing for external build") gobuild.unpack(context.branch, version, context.stage(stage))
def perform_debug_shell(context: Context) -> None: project.log("debug", "launching shell") subprocess.check_call(["bash"], cwd=context.stagedir) project.log("debug", "closed shell")