Example #1
0
def get_projects():
    """get projects for index page"""
    projects = []
    # FIXME: use os.walk, silly pants
    for project in os.listdir(config.build_directory()):
        projects.append({"name": project, "url": None, "date": None, "builds": []})
        projectpath = config.build_directory(project)
        for branch in os.listdir(projectpath):
            branchpath = os.path.join(projectpath, branch)
            for system in os.listdir(branchpath):
                data = get_build_data(project, branch, system, "current")
                if not data:
                    continue
                if not projects[-1]["date"] or data["idate"] > projects[-1]["date"]:
                    projects[-1]["date"] = data["idate"]
                    if "upstream" in data:
                        projects[-1]["url"] = data["upstream"]
                projects[-1]["builds"].append(data)
        if projects[-1]["date"]:
            projects[-1]["builds"] = sorted(projects[-1]["builds"], key=lambda k: k["date"], reverse=True)
        else:
            projects.remove(projects[-1])

    projects = sorted(projects, key=lambda k: k["date"], reverse=True)
    return projects
Example #2
0
def delete_build(project, branch="", system="", fsdate=""):
    """delete build"""
    # pylint: disable=too-many-branches

    validate_build(project, branch, system)

    buildpath = config.build_directory(project, branch, system)
    if fsdate:
        currentpath = os.path.join(buildpath, "current")
        if fsdate == "current":
            if os.path.lexists(currentpath):
                fsdate = os.readlink(currentpath)
            else:
                abort(404, "Current build does not exist")
        buildpath = os.path.join(buildpath, fsdate)
    parentpath, _ = os.path.split(buildpath)
    if not os.path.isdir(buildpath):
        return False
    shutil.rmtree(buildpath)

    if fsdate and os.path.lexists(currentpath):
        current = os.readlink(currentpath)
        if current == fsdate:
            latest = os.path.basename(sorted(os.listdir(parentpath))[0])
            if os.path.lexists(currentpath):
                os.unlink(currentpath)
            if latest != "current":
                os.symlink(latest, currentpath)

    while parentpath != config.build_directory():
        if os.path.isdir(parentpath) and not os.listdir(parentpath):
            os.rmdir(parentpath)
        parentpath, _ = os.path.split(parentpath)

    return True
Example #3
0
def get_build_file(project=None, branch=None, system=None, fsdate=None, bfile=None):
    """get file for build"""
    validate_build(project, branch, system)

    ext = os.path.splitext(bfile)[1]
    path = config.build_directory(project, branch, system, fsdate)

    if not os.path.exists(path):
        abort(404, "Build does not exist.")

    if bfile == "status.svg":
        response.set_header("Cache-control", "no-cache")
        response.set_header("Pragma", "no-cache")
        if not failure_for_build(project, branch, system, fsdate):
            return static_file("ok.svg", root=rootpath("media", "status"))
        return static_file("fail.svg", root=rootpath("media", "status"))
    elif ext == ".zip":
        return static_file(bfile, root=rootpath(path))
    elif ext == ".bz2":
        return static_file(bfile, root=rootpath(path))
    elif ext == ".txt":
        response.content_type = "text/plain"
        path = rootpath(path, bfile.replace(".txt", ".bz2"))
        if os.path.exists(path):
            return bz2.BZ2File(path).read()

    abort(404, "No such file.")
Example #4
0
def save_build(project, branch, system, data):
    """save build to disk"""
    # pylint: disable=too-many-locals
    validate_build(project, branch, system)
    if not data:
        raise ValueError("build should have data")

    metadata = metadata_for_build(project, branch, system, "current")
    if "commit" in metadata and metadata["commit"] == data["commit"]:
        if not data["force"]:
            print("This commit is already built")
            return
        delete_build(project, branch, system, "current")

    date = datetime.utcnow()
    fsdate = date.strftime("%Y%m%d%H%M%S")
    buildpath = config.build_directory(project, branch, system)
    if not os.path.isdir(buildpath):
        os.makedirs(buildpath)
    currentpath = os.path.join(buildpath, "current")
    buildpath = os.path.join(buildpath, fsdate)
    if not os.path.isdir(buildpath):
        os.makedirs(buildpath)

    metadata["date"] = date.isoformat()
    metadata["client"] = data["client"]
    metadata["commit"] = data["commit"]
    metadata["description"] = data["description"]
    metadata["upstream"] = data["upstream"]

    posthook = {}
    posthook["github"] = check_github_posthook(data, metadata)

    for key, value in data.items():
        if key not in STUSKEYS:
            continue

        metadata[key] = {"status": value["status"]}

        if "log" in value and value["log"]:
            text = remove_control_characters(b64decode(value["log"].encode("UTF-8")).decode("UTF-8"))
            buildlog = bz2.compress(text.encode("UTF-8"))
            with open(os.path.join(buildpath, "{}-log.bz2".format(key)), "wb") as fle:
                fle.write(buildlog)

        if "zip" in value and value["zip"]:
            buildzip = b64decode(value["zip"].encode("UTF-8"))
            with open(os.path.join(buildpath, "{}.zip".format(key)), "wb") as fle:
                fle.write(buildzip)

    with open(os.path.join(buildpath, "metadata.bz2"), "wb") as fle:
        if config.config["github"] and posthook["github"]:
            handle_github(project, branch, system, fsdate, metadata)
        fle.write(bz2.compress(json.dumps(metadata).encode("UTF-8")))
        if os.path.lexists(currentpath):
            os.unlink(currentpath)
        os.symlink(fsdate, currentpath)
        print("[SAVED] {}".format(project))
Example #5
0
def parse_links_for_build(project, branch, system, fsdate, metadata):
    """get status links array for build"""
    systempath = config.build_directory(project, branch, system, fsdate)

    for key in STUSKEYS:
        if os.path.exists(os.path.join(systempath, "{}-log.bz2".format(key))):
            metadata[key]["url"] = quote("/build/{}/{}/{}/{}/{}-log.txt".format(project, branch, system, fsdate, key))
        else:
            metadata[key]["url"] = "#"
Example #6
0
def metadata_for_build(project, branch, system, fsdate):
    """get metadata for build"""
    metadata = {}
    path = config.build_directory(project, branch, system, fsdate, "metadata.bz2")
    if os.path.exists(path):
        try:
            bz2data = bz2.BZ2File(path).read()
        except EOFError:
            bz2data = None
        if bz2data:
            metadata = json.loads(bz2data.decode("UTF-8"))
    return metadata
Example #7
0
def get_build_data(project, branch, system, fsdate, get_history=True, in_metadata=None):
    """get data for build"""
    # pylint: disable=too-many-arguments

    metadata = {}
    if in_metadata:
        for key, value in in_metadata.items():
            metadata[key] = value
    else:
        metadata = metadata_for_build(project, branch, system, fsdate)
    if not metadata:
        return
    date = date_for_metadata(metadata)
    if not date:
        return

    for key in STUSKEYS:
        if key not in metadata:
            metadata[key] = {"status": -1}

    metadata["project"] = project
    metadata["fsdate"] = fsdate
    metadata["idate"] = date
    metadata["fdate"] = date.strftime("%Y-%m-%d %H:%M")
    metadata["system"] = system
    metadata["branch"] = branch
    metadata["systemimage"] = icon_for_system(system)
    metadata["statusimage"] = status_image_link_for_build(project, branch, system, fsdate)

    parse_status_for_metadata(metadata)
    parse_links_for_build(project, branch, system, fsdate, metadata)

    if get_history:
        metadata["history"] = []
        systempath = config.build_directory(project, branch, system)
        current = os.readlink(os.path.join(systempath, "current"))
        for old_fsdate in sorted(os.listdir(systempath), reverse=True):
            if old_fsdate == fsdate or old_fsdate == current or old_fsdate == "current":
                continue
            old = get_build_data(project, branch, system, old_fsdate, get_history=False)
            if not old:
                continue
            metadata["history"].append(old)

    return metadata
Example #8
0
def cook_recipe(recipe):
    """prepare && cook recipe"""
    # pylint: disable=too-many-branches

    logging.info("Building %s from %s", recipe["name"], recipe["source"])
    logging.debug(recipe["build"])
    if "test" in recipe:
        logging.debug(recipe["test"])
    if "package" in recipe:
        logging.debug(recipe["package"])

    os.chdir(STARTDIR)
    projectdir = config.build_directory(recipe["name"])

    pkgdir = os.path.join(projectdir, "pkg")
    srcdir = os.path.join(projectdir, "src")

    if "build_in_srcdir" in recipe and recipe["build_in_srcdir"]:
        builddir = srcdir
    else:
        builddir = os.path.join(projectdir, "build")

    import socket

    result = {
        "client": socket.gethostname(),
        "build": {"status": -1},
        "test": {"status": -1},
        "package": {"status": -1},
        "analyze": {"status": -1},
    }

    if config.config.get("force"):
        result["force"] = True
        if os.path.exists(os.path.join(srcdir, ".buildhck_built")):
            os.remove(os.path.join(srcdir, ".buildhck_built"))

    if "upstream" in recipe and recipe["upstream"]:
        result["upstream"] = recipe["upstream"]

    if "github" in recipe and recipe["github"]:
        result["github"] = recipe["github"]

    s_mkdir(projectdir)
    send_build = True
    try:
        perform_recipe(recipe, srcdir, builddir, pkgdir, result)
    except CookException as exc:
        logging.error("%s build failed :: %s", recipe["name"], str(exc))
    except RecipeException as exc:
        logging.error("%s recipe error :: %s", recipe["name"], str(exc))
        send_build = False
    except DownloadException as exc:
        logging.error("%s download failed :: %s", recipe["name"], str(exc))
        send_build = False
    except NothingToDoException:
        send_build = False

    if send_build:
        upload_build(recipe, result, srcdir)

    # cleanup build and pkg directory
    if config.config.get("cleanup"):
        cleanup_build(srcdir, builddir, pkgdir)

    os.chdir(STARTDIR)
Example #9
0
def build_exists(project, branch="", system="", fsdate=""):
    """check if build dir exists"""
    validate_build(project, branch, system)
    buildpath = config.build_directory(project, branch, system, fsdate)
    return os.path.exists(buildpath)