def get_projects(): """get projects for index page""" projects = [] # FIXME: use os.walk, silly pants for project in os.listdir(config.build_directory()): projects.append({"name": project, "url": None, "date": None, "builds": []}) projectpath = config.build_directory(project) for branch in os.listdir(projectpath): branchpath = os.path.join(projectpath, branch) for system in os.listdir(branchpath): data = get_build_data(project, branch, system, "current") if not data: continue if not projects[-1]["date"] or data["idate"] > projects[-1]["date"]: projects[-1]["date"] = data["idate"] if "upstream" in data: projects[-1]["url"] = data["upstream"] projects[-1]["builds"].append(data) if projects[-1]["date"]: projects[-1]["builds"] = sorted(projects[-1]["builds"], key=lambda k: k["date"], reverse=True) else: projects.remove(projects[-1]) projects = sorted(projects, key=lambda k: k["date"], reverse=True) return projects
def delete_build(project, branch="", system="", fsdate=""): """delete build""" # pylint: disable=too-many-branches validate_build(project, branch, system) buildpath = config.build_directory(project, branch, system) if fsdate: currentpath = os.path.join(buildpath, "current") if fsdate == "current": if os.path.lexists(currentpath): fsdate = os.readlink(currentpath) else: abort(404, "Current build does not exist") buildpath = os.path.join(buildpath, fsdate) parentpath, _ = os.path.split(buildpath) if not os.path.isdir(buildpath): return False shutil.rmtree(buildpath) if fsdate and os.path.lexists(currentpath): current = os.readlink(currentpath) if current == fsdate: latest = os.path.basename(sorted(os.listdir(parentpath))[0]) if os.path.lexists(currentpath): os.unlink(currentpath) if latest != "current": os.symlink(latest, currentpath) while parentpath != config.build_directory(): if os.path.isdir(parentpath) and not os.listdir(parentpath): os.rmdir(parentpath) parentpath, _ = os.path.split(parentpath) return True
def get_build_file(project=None, branch=None, system=None, fsdate=None, bfile=None): """get file for build""" validate_build(project, branch, system) ext = os.path.splitext(bfile)[1] path = config.build_directory(project, branch, system, fsdate) if not os.path.exists(path): abort(404, "Build does not exist.") if bfile == "status.svg": response.set_header("Cache-control", "no-cache") response.set_header("Pragma", "no-cache") if not failure_for_build(project, branch, system, fsdate): return static_file("ok.svg", root=rootpath("media", "status")) return static_file("fail.svg", root=rootpath("media", "status")) elif ext == ".zip": return static_file(bfile, root=rootpath(path)) elif ext == ".bz2": return static_file(bfile, root=rootpath(path)) elif ext == ".txt": response.content_type = "text/plain" path = rootpath(path, bfile.replace(".txt", ".bz2")) if os.path.exists(path): return bz2.BZ2File(path).read() abort(404, "No such file.")
def save_build(project, branch, system, data): """save build to disk""" # pylint: disable=too-many-locals validate_build(project, branch, system) if not data: raise ValueError("build should have data") metadata = metadata_for_build(project, branch, system, "current") if "commit" in metadata and metadata["commit"] == data["commit"]: if not data["force"]: print("This commit is already built") return delete_build(project, branch, system, "current") date = datetime.utcnow() fsdate = date.strftime("%Y%m%d%H%M%S") buildpath = config.build_directory(project, branch, system) if not os.path.isdir(buildpath): os.makedirs(buildpath) currentpath = os.path.join(buildpath, "current") buildpath = os.path.join(buildpath, fsdate) if not os.path.isdir(buildpath): os.makedirs(buildpath) metadata["date"] = date.isoformat() metadata["client"] = data["client"] metadata["commit"] = data["commit"] metadata["description"] = data["description"] metadata["upstream"] = data["upstream"] posthook = {} posthook["github"] = check_github_posthook(data, metadata) for key, value in data.items(): if key not in STUSKEYS: continue metadata[key] = {"status": value["status"]} if "log" in value and value["log"]: text = remove_control_characters(b64decode(value["log"].encode("UTF-8")).decode("UTF-8")) buildlog = bz2.compress(text.encode("UTF-8")) with open(os.path.join(buildpath, "{}-log.bz2".format(key)), "wb") as fle: fle.write(buildlog) if "zip" in value and value["zip"]: buildzip = b64decode(value["zip"].encode("UTF-8")) with open(os.path.join(buildpath, "{}.zip".format(key)), "wb") as fle: fle.write(buildzip) with open(os.path.join(buildpath, "metadata.bz2"), "wb") as fle: if config.config["github"] and posthook["github"]: handle_github(project, branch, system, fsdate, metadata) fle.write(bz2.compress(json.dumps(metadata).encode("UTF-8"))) if os.path.lexists(currentpath): os.unlink(currentpath) os.symlink(fsdate, currentpath) print("[SAVED] {}".format(project))
def parse_links_for_build(project, branch, system, fsdate, metadata): """get status links array for build""" systempath = config.build_directory(project, branch, system, fsdate) for key in STUSKEYS: if os.path.exists(os.path.join(systempath, "{}-log.bz2".format(key))): metadata[key]["url"] = quote("/build/{}/{}/{}/{}/{}-log.txt".format(project, branch, system, fsdate, key)) else: metadata[key]["url"] = "#"
def metadata_for_build(project, branch, system, fsdate): """get metadata for build""" metadata = {} path = config.build_directory(project, branch, system, fsdate, "metadata.bz2") if os.path.exists(path): try: bz2data = bz2.BZ2File(path).read() except EOFError: bz2data = None if bz2data: metadata = json.loads(bz2data.decode("UTF-8")) return metadata
def get_build_data(project, branch, system, fsdate, get_history=True, in_metadata=None): """get data for build""" # pylint: disable=too-many-arguments metadata = {} if in_metadata: for key, value in in_metadata.items(): metadata[key] = value else: metadata = metadata_for_build(project, branch, system, fsdate) if not metadata: return date = date_for_metadata(metadata) if not date: return for key in STUSKEYS: if key not in metadata: metadata[key] = {"status": -1} metadata["project"] = project metadata["fsdate"] = fsdate metadata["idate"] = date metadata["fdate"] = date.strftime("%Y-%m-%d %H:%M") metadata["system"] = system metadata["branch"] = branch metadata["systemimage"] = icon_for_system(system) metadata["statusimage"] = status_image_link_for_build(project, branch, system, fsdate) parse_status_for_metadata(metadata) parse_links_for_build(project, branch, system, fsdate, metadata) if get_history: metadata["history"] = [] systempath = config.build_directory(project, branch, system) current = os.readlink(os.path.join(systempath, "current")) for old_fsdate in sorted(os.listdir(systempath), reverse=True): if old_fsdate == fsdate or old_fsdate == current or old_fsdate == "current": continue old = get_build_data(project, branch, system, old_fsdate, get_history=False) if not old: continue metadata["history"].append(old) return metadata
def cook_recipe(recipe): """prepare && cook recipe""" # pylint: disable=too-many-branches logging.info("Building %s from %s", recipe["name"], recipe["source"]) logging.debug(recipe["build"]) if "test" in recipe: logging.debug(recipe["test"]) if "package" in recipe: logging.debug(recipe["package"]) os.chdir(STARTDIR) projectdir = config.build_directory(recipe["name"]) pkgdir = os.path.join(projectdir, "pkg") srcdir = os.path.join(projectdir, "src") if "build_in_srcdir" in recipe and recipe["build_in_srcdir"]: builddir = srcdir else: builddir = os.path.join(projectdir, "build") import socket result = { "client": socket.gethostname(), "build": {"status": -1}, "test": {"status": -1}, "package": {"status": -1}, "analyze": {"status": -1}, } if config.config.get("force"): result["force"] = True if os.path.exists(os.path.join(srcdir, ".buildhck_built")): os.remove(os.path.join(srcdir, ".buildhck_built")) if "upstream" in recipe and recipe["upstream"]: result["upstream"] = recipe["upstream"] if "github" in recipe and recipe["github"]: result["github"] = recipe["github"] s_mkdir(projectdir) send_build = True try: perform_recipe(recipe, srcdir, builddir, pkgdir, result) except CookException as exc: logging.error("%s build failed :: %s", recipe["name"], str(exc)) except RecipeException as exc: logging.error("%s recipe error :: %s", recipe["name"], str(exc)) send_build = False except DownloadException as exc: logging.error("%s download failed :: %s", recipe["name"], str(exc)) send_build = False except NothingToDoException: send_build = False if send_build: upload_build(recipe, result, srcdir) # cleanup build and pkg directory if config.config.get("cleanup"): cleanup_build(srcdir, builddir, pkgdir) os.chdir(STARTDIR)
def build_exists(project, branch="", system="", fsdate=""): """check if build dir exists""" validate_build(project, branch, system) buildpath = config.build_directory(project, branch, system, fsdate) return os.path.exists(buildpath)