def render(run, report_dir):
temporary_report_dir = litani.get_report_data_dir() / str(uuid.uuid4())
temporary_report_dir.mkdir(parents=True)
old_report_dir_path = litani.get_report_dir().resolve()
old_report_dir = litani.ExpireableDirectory(old_report_dir_path)
artifact_dir = temporary_report_dir / "artifacts"
shutil.copytree(litani.get_artifacts_dir(), artifact_dir)
render_artifact_indexes(artifact_dir)
template_dir = pathlib.Path(__file__).parent.parent / "templates"
env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(template_dir)))
svgs = render_runtimes(run, env, temporary_report_dir)
litani_report_archive_path = os.getenv("LITANI_REPORT_ARCHIVE_PATH")
dash_templ = env.get_template("dashboard.jinja.html")
page = dash_templ.render(
run=run,
svgs=svgs,
litani_hash=get_git_hash(),
litani_version=litani.VERSION,
litani_report_archive_path=litani_report_archive_path)
with litani.atomic_write(temporary_report_dir / "index.html") as handle:
print(page, file=handle)
with litani.atomic_write(temporary_report_dir / litani.RUN_FILE) as handle:
print(json.dumps(run, indent=2), file=handle)
pipe_templ = env.get_template("pipeline.jinja.html")
for pipe in run["pipelines"]:
page = pipe_templ.render(run=run, pipe=pipe)
with litani.atomic_write(temporary_report_dir / pipe["url"]) as handle:
print(page, file=handle)
temp_symlink_dir = report_dir.with_name(report_dir.name +
str(uuid.uuid4()))
os.symlink(temporary_report_dir, temp_symlink_dir)
os.rename(temp_symlink_dir, report_dir)
# Release lock so that other processes can read from this directory
new_report_dir = litani.LockableDirectory(report_dir.resolve())
new_report_dir.release()
if old_report_dir_path.exists():
old_report_dir.expire()
litani.unlink_expired()
def render_artifact_indexes(artifact_dir, env):
def dirs_needing_indexes():
for root, dirs, fyles in os.walk(artifact_dir):
if "index.html" not in fyles:
yield pathlib.Path(root), dirs, fyles
index_templ = env.get_template("file-list.jinja.html")
for dyr, dirs, files in dirs_needing_indexes():
page = index_templ.render(file_list=sorted(files),
dir_list=sorted(dirs),
root=str(dyr))
with litani.atomic_write(dyr / "index.html") as handle:
print(page, file=handle)
def render(run, report_dir):
artifacts_dst = report_dir / "artifacts"
if artifacts_dst.exists():
shutil.rmtree(report_dir / "artifacts")
shutil.copytree(litani.get_artifacts_dir(), report_dir / "artifacts")
render_artifact_indexes(artifacts_dst)
template_dir = pathlib.Path(__file__).parent.parent / "templates"
env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(template_dir)))
svgs = render_runtimes(run, env, report_dir)
dash_templ = env.get_template("dashboard.jinja.html")
page = dash_templ.render(run=run, svgs=svgs)
with litani.atomic_write(report_dir / "index.html") as handle:
print(page, file=handle)
pipe_templ = env.get_template("pipeline.jinja.html")
for pipe in run["pipelines"]:
page = pipe_templ.render(run=run, pipe=pipe)
with litani.atomic_write(report_dir / pipe["url"]) as handle:
print(page, file=handle)
def render_artifact_indexes(artifact_dir):
def dirs_needing_indexes():
for root, dirs, fyles in os.walk(artifact_dir):
if "index.html" not in fyles:
yield pathlib.Path(root), dirs, fyles
template_dir = pathlib.Path(__file__).parent.parent / "templates"
env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(template_dir)))
index_templ = env.get_template("file-list.jinja.html")
for dyr, dirs, files in dirs_needing_indexes():
page = index_templ.render(file_list=sorted(files),
dir_list=sorted(dirs),
root=str(dyr))
with litani.atomic_write(dyr / "index.html") as handle:
print(page, file=handle)
async def add_job(job_dict):
cache_file = litani.get_cache_dir() / litani.CACHE_FILE
with open(cache_file) as handle:
cache_contents = json.load(handle)
if job_dict["ci_stage"] not in cache_contents["stages"]:
valid_stages = "', '".join(cache_contents["stages"])
logging.error(
"Invalid stage name '%s' was provided, possible "
"stage names are: '%s'", job_dict["ci_stage"], valid_stages)
sys.exit(1)
jobs_dir = litani.get_cache_dir() / litani.JOBS_DIR
jobs_dir.mkdir(exist_ok=True, parents=True)
if job_dict["phony_outputs"]:
if not job_dict["outputs"]:
job_dict["outputs"] = job_dict["phony_outputs"]
else:
for phony_output in job_dict["phony_outputs"]:
if phony_output not in job_dict["outputs"]:
job_dict["outputs"].append(phony_output)
if "func" in job_dict:
job_dict.pop("func")
job_id = str(uuid.uuid4())
job_dict["job_id"] = job_id
job_dict["status_file"] = str(
litani.get_status_dir() / ("%s.json" % job_id))
logging.debug("Adding job: %s", json.dumps(job_dict, indent=2))
for key in _PRIVATE_JOB_FIELDS:
if key not in job_dict:
raise AssertionError(f"Key {key} missing from job definition")
with litani.atomic_write(jobs_dir / ("%s.json" % job_id)) as handle:
print(json.dumps(job_dict, indent=2), file=handle)
def render(self, gnu_file, out_file=None):
if not self.should_render():
raise UserWarning(
"Should not call Gnuplot.render() if should_render() is False")
cmd = ["gnuplot"]
with subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True) as proc:
out, _ = proc.communicate(input=gnu_file)
if proc.returncode:
logging.error("Failed to render gnuplot file:")
logging.error(gnu_file)
self._should_render = False
return
lines = [l for l in out.splitlines() if "<?xml version" not in l]
if out_file:
with litani.atomic_write(out_file) as handle:
print("\n".join(lines), file=handle)
return lines
def render(run, report_dir, pipeline_depgraph_renderer):
temporary_report_dir = litani.get_report_data_dir() / str(uuid.uuid4())
temporary_report_dir.mkdir(parents=True)
old_report_dir_path = litani.get_report_dir().resolve()
old_report_dir = litani.ExpireableDirectory(old_report_dir_path)
artifact_dir = temporary_report_dir / "artifacts"
shutil.copytree(litani.get_artifacts_dir(), artifact_dir)
template_dir = pathlib.Path(__file__).parent.parent / "templates"
env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(template_dir)),
autoescape=jinja2.select_autoescape(
enabled_extensions=('html'),
default_for_string=True))
render_artifact_indexes(artifact_dir, env)
gnuplot = Gnuplot()
svgs = get_dashboard_svgs(run, env, gnuplot)
litani_report_archive_path = os.getenv("LITANI_REPORT_ARCHIVE_PATH")
dash_templ = env.get_template("dashboard.jinja.html")
page = dash_templ.render(
run=run,
svgs=svgs,
litani_hash=get_git_hash(),
litani_version=litani.VERSION,
litani_report_archive_path=litani_report_archive_path,
summary=get_summary(run))
with litani.atomic_write(temporary_report_dir / "index.html") as handle:
print(page, file=handle)
with litani.atomic_write(temporary_report_dir / litani.RUN_FILE) as handle:
print(json.dumps(run, indent=2), file=handle)
pipe_templ = env.get_template("pipeline.jinja.html")
for pipe in run["pipelines"]:
pipeline_depgraph_renderer.render(render_root=temporary_report_dir,
pipe_url=pathlib.Path(pipe["url"]),
pipe=pipe)
for stage in pipe["ci_stages"]:
for job in stage["jobs"]:
if JobOutcomeTableRenderer.should_render(job):
JobOutcomeTableRenderer.render(
temporary_report_dir / pipe["url"], env, job)
if MemoryTraceRenderer.should_render(job):
MemoryTraceRenderer.render(
temporary_report_dir / pipe["url"], env, job, gnuplot)
pipe_page = pipe_templ.render(run=run, pipe=pipe)
with litani.atomic_write(temporary_report_dir / pipe["url"] /
"index.html") as handle:
print(pipe_page, file=handle)
temp_symlink_dir = report_dir.with_name(report_dir.name +
str(uuid.uuid4()))
os.symlink(temporary_report_dir, temp_symlink_dir)
os.rename(temp_symlink_dir, report_dir)
# Release lock so that other processes can read from this directory
new_report_dir = litani.LockableDirectory(report_dir.resolve())
new_report_dir.release()
if old_report_dir_path.exists():
old_report_dir.expire()
litani.unlink_expired()
def __call__(self, run):
temporary_report_dir = litani.get_report_data_dir() / str(uuid.uuid4())
temporary_report_dir.mkdir(parents=True)
old_report_dir_path = litani.get_report_dir().resolve()
old_report_dir = litani.ExpireableDirectory(old_report_dir_path)
artifact_dir = temporary_report_dir / "artifacts"
shutil.copytree(litani.get_artifacts_dir(), artifact_dir)
template_dir = pathlib.Path(__file__).parent.parent / "templates"
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(str(template_dir)),
autoescape=jinja2.select_autoescape(enabled_extensions=('html'),
default_for_string=True))
render_artifact_indexes(artifact_dir, env)
gnuplot = Gnuplot()
svgs = get_dashboard_svgs(run, env, gnuplot)
litani_report_archive_path = os.getenv("LITANI_REPORT_ARCHIVE_PATH")
with litani.atomic_write(temporary_report_dir /
litani.RUN_FILE) as handle:
print(json.dumps(run, indent=2), file=handle)
front_page_outputs = {}
pipe_templ = env.get_template("pipeline.jinja.html")
for pipe in run["pipelines"]:
self.pipeline_depgraph_renderer.render(
render_root=temporary_report_dir,
pipe_url=pathlib.Path(pipe["url"]),
pipe=pipe)
for stage in pipe["ci_stages"]:
for job in stage["jobs"]:
if JobOutcomeTableRenderer.should_render(job):
JobOutcomeTableRenderer.render(
temporary_report_dir / pipe["url"], env, job)
if MemoryTraceRenderer.should_render(job):
MemoryTraceRenderer.render(
temporary_report_dir / pipe["url"], env, job,
gnuplot)
tags = job["wrapper_arguments"]["tags"]
description = job["wrapper_arguments"]["description"]
if tags and "front-page-text" in tags:
if "stdout" in job and job["stdout"]:
front_page_outputs[description] = job
pipe_page = pipe_templ.render(run=run, pipe=pipe)
with litani.atomic_write(temporary_report_dir / pipe["url"] /
"index.html") as handle:
print(pipe_page, file=handle)
if "end_time" in run:
s = datetime.datetime.strptime(run["start_time"],
litani.TIME_FORMAT_R)
e = datetime.datetime.strptime(run["end_time"],
litani.TIME_FORMAT_R)
runtime = (e - s).seconds
run["__duration_str"] = s_to_hhmmss(runtime)
dash_templ = env.get_template("dashboard.jinja.html")
page = dash_templ.render(
run=run,
svgs=svgs,
litani_version=litani.VERSION,
litani_report_archive_path=litani_report_archive_path,
summary=get_summary(run),
front_page_outputs=front_page_outputs)
with litani.atomic_write(temporary_report_dir /
"index.html") as handle:
print(page, file=handle)
locked_dir = litani.LockableDirectory(temporary_report_dir)
temp_symlink_dir = self.report_dir.with_name(self.report_dir.name +
str(uuid.uuid4()))
os.symlink(temporary_report_dir, temp_symlink_dir)
os.rename(temp_symlink_dir, self.report_dir)
locked_dir.release()
try:
old_report_dir.expire()
except FileNotFoundError:
pass
litani.unlink_expired()
