def updateGit(proj: str, ref: str, log: Callable[[str], None]):
successful = True
try:
log(">>> git fetch --all\n")
rv = runSubprocess(["git", "fetch", "--all"],
log,
cwd=getProjPath(proj))
if rv != 0:
raise Exception(rv)
log(">>> git reset --hard " + ref + "\n")
rv = runSubprocess(["git", "reset", "--hard", ref],
log,
cwd=getProjPath(proj))
if rv != 0:
raise Exception(rv)
log(">>> git diff --stat " + ref + "~1 " + ref + "\n")
rv = runSubprocess(["git", "diff", "--stat=100", ref + "~1", ref],
log,
cwd=getProjPath(proj))
if rv != 0:
raise Exception(rv)
except Exception as e:
successful = False
log("git operations failed: " + str(e) + "\n")
return successful
def doCompile(proj: str, buildPath: str, cfg: dict,
log: Callable[[str], None]) -> bool:
successful = True
copyFolderStructure(getProjPath(proj), buildPath)
main = cfg.get("main", None)
if main:
cmd = [
"latexmk",
"-interaction=nonstopmode",
# "-gg",
"-file-line-error",
"-outdir=" + buildPath,
"-pdf",
main + ".tex"
]
env = {
"max_print_line": "100",
"error_line": "254",
"half_error_line": "238"
}
log(">>> " + (" ".join(cmd)) + "\n")
rv = runSubprocess(cmd, log, cwd=getProjPath(proj), env=env)
if rv != 0:
log("latexmk failed: " + str(rv) + "\n")
successful = False
else:
log("Missing 'main' in config")
successful = False
return successful
def npm(proj: str, buildPath: str, cfg: dict, log: Callable[[str],
None]) -> bool:
successful = True
root = cfg.get("root", "")
output = cfg.get("output", "")
env = cfg.get("env", {})
cwd = getProjPath(proj) + "/" + root
if output:
try:
log(">>> yarn install\n")
rv = runSubprocess(["yarn", "install"], log, cwd=cwd, env=env)
if rv != 0:
raise Exception(rv)
log(">>> yarn build\n")
rv = runSubprocess(["yarn", "build"], log, cwd=cwd, env=env)
if rv != 0:
raise Exception(rv)
log(">>> creating output archive...\n")
shutil.make_archive(buildPath + "/output",
"zip",
root_dir=cwd,
base_dir="./" + output)
except Exception as e:
successful = False
log("yarn failed: " + str(e) + "\n")
else:
successful = False
log("Missing 'output' in config")
return successful
def get_builds(proj):
if not os.path.isfile(getProjPath(proj) + "/.ci.json"):
return "Not found", 404
if os.path.exists(getBuildPath(proj)):
dirs = [
entry for entry in os.listdir(getBuildPath(proj))
if entry != "latest" and os.path.isdir(getBuildPath(proj, entry))
]
data = []
for ref in dirs:
toAdd = {
"commit": git.getCommitDetails(proj, ref),
"build": compile.getStatus(proj, ref)
}
if ((proj, ref) in compile.q):
toAdd["build"]["status"] = "queued"
data.append(toAdd)
return json.dumps({
"list": data,
"language": getConfig(proj).get("language", None),
"id": git.repos[proj]["github"],
"latest": parseRef(proj, "latest")
}), {
"Content-Type": "application/json"
}
else:
return json.dumps({
"list": [],
"language": None,
"latest": ""
}), {
"Content-Type": "application/json"
}
from threading import Timer
import flask_sse
import compile, git
from utils import getBuildPath, getProjPath, parseRef, getConfig
SECRET = os.environ.get('SECRET', "").encode("utf-8")
PASSWORD = os.environ.get('PASSWORD', "")
JWT_SECRET = os.environ.get('JWT_SECRET', "secret")
PROJECTS = json.loads("[]" if os.environ.get('PROJECTS', None) is None else (
"[" +
",".join(['"' + x + '"'
for x in os.environ.get('PROJECTS').split(",")]) + "]"))
for p in PROJECTS:
git.getRepo(p, getProjPath(p))
class StringConverter(BaseConverter):
def __init__(self, url_map, exc="."):
super(StringConverter, self).__init__(url_map)
self.exc = list(exc)
def to_python(self, value):
if any(x not in value for x in self.exc):
return value
raise ValidationError()
def to_url(self, value):
return value and 'yes' or 'no'
def compile(proj: str, ref: str, channel: Channel) -> None:
if not os.path.exists(getBuildPath(proj, ref)):
os.makedirs(getBuildPath(proj, ref))
with open(getBuildPath(proj, ref) + "/.log", 'w', 1) as logFile:
def log(s):
logFile.write(s)
channel.publish(proj, {"event": "log", "ref": ref[:7], "data": s})
try:
timeStart = time.time()
updateStatus(proj, ref, channel, "pending", (timeStart, None))
print(">> Started: " + time.strftime("%c"))
log(">> Started: " + time.strftime("%c") + "\n")
successful = True
successfulGit = updateGit(proj, ref, log)
successful = successfulGit
shutil.copy2(
getProjPath(proj) + "/.ci.json",
getBuildPath(proj, ref) + "/.ci.json")
cfg = getConfig(proj, ref)
if successful:
successfulCfg = True
lang = cfg.get("language", None)
if not lang:
successfulCfg = False
successful = successfulCfg
if successful:
if not os.path.exists(getBuildPath(proj)):
log("creating " + getBuildPath(proj))
os.makedirs(getBuildPath(proj))
successfulCompile = compileLang[lang](proj,
getBuildPath(
proj, ref), cfg,
log)
successful = successfulCompile
else:
log("not compiling" + "\n")
stats = {} #type: Dict[str, Union[str, Any]]
if successful:
if "stats" in cfg:
if cfg["language"] == "latex" and "counts" in cfg["stats"]:
(success,
counts) = latex.count(getProjPath(proj),
getBuildPath(proj, ref),
cfg["main"] + ".tex")
if success:
stats["counts"] = counts
else:
stats["counts"] = False
print(">> Finished " + ref)
log((">>" if successful else ">!") + " Finished: " +
time.strftime("%X") + " " + ref + "\n")
except Exception as e:
successful = False
log(">! " + str(e))
print(">> Error: " + ref)
traceback.print_exc()
updateStatus(
proj, ref, channel, "success" if successful else "error",
(timeStart, time.time() - timeStart), "Git stage failed"
if not successfulGit else "Config error" if not successfulCfg else
"Compile stage failed" if not successfulCompile else None, stats)
symlink_force(ref, getBuildPath(proj, "latest"))