def _get_recent_run_data(weeks,
group_field,
project=None,
additional_filters=None):
"""Get all the data from the time period and aggregate the results"""
data = {
"passed": {},
"skipped": {},
"error": {},
"failed": {},
"xfailed": {},
"xpassed": {}
}
delta = timedelta(weeks=weeks).total_seconds()
current_time = time.time()
time_period_in_sec = current_time - delta
# create filters for start time and that the group_field exists
filters = [
f"start_time>{datetime.utcfromtimestamp(time_period_in_sec)}",
f"{group_field}@y"
]
if additional_filters:
filters.extend(additional_filters.split(","))
if project:
filters.append(f"project_id={project}")
# generate the group field
group_field = string_to_column(group_field, Run)
# create the query
query = session.query(
group_field,
func.sum(Run.summary["failures"].cast(Float)),
func.sum(Run.summary["errors"].cast(Float)),
func.sum(Run.summary["skips"].cast(Float)),
func.sum(Run.summary["tests"].cast(Float)),
func.sum(Run.summary["xpassed"].cast(Float)),
func.sum(Run.summary["xfailed"].cast(Float)),
).group_by(group_field)
# filter the query
query = apply_filters(query, filters, Run)
# make the query
query_data = query.all()
# parse the data
for group, failed, error, skipped, total, xpassed, xfailed in query_data:
# convert all data to percentages
data["failed"][group] = int(round((failed / total) * 100.0))
data["error"][group] = int(round((error / total) * 100.0))
data["skipped"][group] = int(round((skipped / total) * 100.0))
data["xpassed"][group] = int(round((xpassed or 0.0 / total) * 100.0))
data["xfailed"][group] = int(round((xfailed or 0.0 / total) * 100.0))
data["passed"][group] = int(
100 - (data["failed"][group] + data["error"][group] +
data["skipped"][group] + data["xfailed"][group] +
data["xpassed"][group]))
return data
def get_result_summary(source=None,
env=None,
job_name=None,
project=None,
additional_filters=None):
"""Get a summary of results"""
summary = {
"error": 0,
"skipped": 0,
"failed": 0,
"passed": 0,
"total": 0,
"xfailed": 0,
"xpassed": 0,
}
query = session.query(
func.sum(Run.summary["errors"].cast(Integer)),
func.sum(Run.summary["skips"].cast(Integer)),
func.sum(Run.summary["failures"].cast(Integer)),
func.sum(Run.summary["tests"].cast(Integer)),
func.sum(Run.summary["xfailures"].cast(Integer)),
func.sum(Run.summary["xpasses"].cast(Integer)),
)
# parse any filters
filters = []
if source:
filters.append(f"source={source}")
if env:
filters.append(f"env={env}")
if job_name:
filters.append(f"metadata.jenkins.job_name={job_name}")
if project:
filters.append(f"project_id={project}")
if additional_filters:
filters.extend(additional_filters.split(","))
# TODO: implement some page size here?
if filters:
query = apply_filters(query, filters, Run)
# get the total number
query_data = query.all()
# parse the data
for error, skipped, failed, total, xfailed, xpassed in query_data:
summary["error"] += error
summary["skipped"] += skipped
summary["failed"] += failed
summary["total"] += total
summary["xfailed"] += xfailed or 0.0
summary["xpassed"] += xpassed or 0.0
summary["passed"] += total - (error + skipped + failed + xpassed +
xfailed)
return summary
def _create_result(tar,
run_id,
result,
artifacts,
project_id=None,
metadata=None):
"""Create a result with artifacts, used in the archive importer"""
old_id = None
result_id = result.get("id")
if is_uuid(result_id):
result_record = session.query(Result).get(result_id)
else:
result_record = None
if result_record:
result_record.run_id = run_id
else:
old_id = result["id"]
if "id" in result:
result.pop("id")
result["run_id"] = run_id
if project_id:
result["project_id"] = project_id
if metadata:
result["metadata"] = result.get("metadata", {})
result["metadata"].update(metadata)
result["env"] = result.get("metadata", {}).get("env")
result["component"] = result.get("metadata", {}).get("component")
result_record = Result.from_dict(**result)
session.add(result_record)
session.commit()
result = result_record.to_dict()
for artifact in artifacts:
session.add(
Artifact(
filename=artifact.name.split("/")[-1],
result_id=result["id"],
data={
"contentType": "text/plain",
"resultId": result["id"]
},
content=tar.extractfile(artifact).read(),
))
session.commit()
return old_id
def _get_builds(job_name, builds, project=None, additional_filters=None):
filters = [
f"metadata.jenkins.job_name={job_name}",
"metadata.jenkins.build_number@y"
]
if additional_filters:
filters.extend(additional_filters.split(","))
if project:
filters.append(f"project_id={project}")
# generate the group_field
group_field = string_to_column("metadata.jenkins.build_number", Run)
# get the runs on which to run the aggregation, we select from a subset of runs to improve
# performance, otherwise we'd be aggregating over ALL runs
heatmap_run_limit = int((HEATMAP_RUN_LIMIT / HEATMAP_MAX_BUILDS) * builds)
sub_query = (apply_filters(Run.query, filters, Run).order_by(
desc("start_time")).limit(heatmap_run_limit).subquery())
# create the query
query = (session.query(
func.min(Run.start_time).label("min_start_time"),
group_field.cast(Integer).label("build_number"),
).select_entity_from(sub_query).group_by("build_number").order_by(
desc("min_start_time")))
# add filters to the query
query = apply_filters(query, filters, Run)
# make the query
builds = [build for build in query.limit(builds)]
min_start_times = [build.min_start_time for build in builds]
if min_start_times:
min_start_time = min(build.min_start_time for build in builds)
else:
min_start_time = None
return min_start_time, [str(build.build_number) for build in builds]
def run_archive_import(import_):
"""Import a test run from an Ibutsu archive file"""
# Update the status of the import
import_record = Import.query.get(str(import_["id"]))
metadata = {}
if import_record.data.get("metadata"):
# metadata is expected to be a json dict
metadata = import_record.data["metadata"]
_update_import_status(import_record, "running")
# Fetch the file contents
import_file = ImportFile.query.filter(
ImportFile.import_id == import_["id"]).first()
if not import_file:
_update_import_status(import_record, "error")
return
# First open the tarball and pull in the results
run = None
run_artifacts = []
results = []
result_artifacts = {}
start_time = None
file_object = BytesIO(import_file.content)
with tarfile.open(fileobj=file_object) as tar:
for member in tar.getmembers():
# We don't care about directories, skip them
if member.isdir():
continue
# Grab the run id
run_id, rest = member.name.split("/", 1)
if "/" not in rest:
if member.name.endswith("run.json"):
run = json.loads(tar.extractfile(member).read())
else:
run_artifacts.append(member)
continue
result_id, file_name = rest.split("/")
if member.name.endswith("result.json"):
result = json.loads(tar.extractfile(member).read())
result_start_time = result.get("start_time")
if not start_time or start_time > result_start_time:
start_time = result_start_time
results.append(result)
else:
try:
result_artifacts[result_id].append(member)
except KeyError:
result_artifacts[result_id] = [member]
run_dict = run or {
"duration": 0,
"summary": {
"errors": 0,
"failures": 0,
"skips": 0,
"xfailures": 0,
"xpasses": 0,
"tests": 0,
},
}
# patch things up a bit, if necessary
run_dict["metadata"] = run_dict.get("metadata", {})
run_dict["metadata"].update(metadata)
_populate_metadata(run_dict, import_record)
_populate_created_times(run_dict, start_time)
# If this run has a valid ID, check if this run exists
if is_uuid(run_dict.get("id")):
run = session.query(Run).get(run_dict["id"])
if run:
run.update(run_dict)
else:
run = Run.from_dict(**run_dict)
session.add(run)
session.commit()
import_record.run_id = run.id
import_record.data["run_id"] = [run.id]
# Loop through any artifacts associated with the run and upload them
for artifact in run_artifacts:
session.add(
Artifact(
filename=artifact.name.split("/")[-1],
run_id=run.id,
data={
"contentType": "text/plain",
"runId": run.id
},
content=tar.extractfile(artifact).read(),
))
# Now loop through all the results, and create or update them
for result in results:
artifacts = result_artifacts.get(result["id"], [])
_create_result(
tar,
run.id,
result,
artifacts,
project_id=run_dict.get("project_id")
or import_record.data.get("project_id"),
metadata=metadata,
)
# Update the import record
_update_import_status(import_record, "done")
if run:
update_run.delay(run.id)
def _get_heatmap(job_name,
builds,
group_field,
count_skips,
project=None,
additional_filters=None):
"""Get Jenkins Heatmap Data."""
# Get the distinct builds that exist in the DB
min_start_time, builds = _get_builds(job_name, builds, project,
additional_filters)
# Create the filters for the query
filters = [
f"metadata.jenkins.job_name={job_name}",
f"metadata.jenkins.build_number*{';'.join(builds)}",
f"{group_field}@y",
]
if additional_filters:
filters.extend(additional_filters.split(","))
if min_start_time:
filters.append(f"start_time){min_start_time}")
if project:
filters.append(f"project_id={project}")
# generate the group_fields
group_field = string_to_column(group_field, Run)
job_name = string_to_column("metadata.jenkins.job_name", Run)
build_number = string_to_column("metadata.jenkins.build_number", Run)
annotations = string_to_column("metadata.annotations", Run)
# create the base query
query = session.query(
Run.id.label("run_id"),
annotations.label("annotations"),
group_field.label("group_field"),
job_name.label("job_name"),
build_number.label("build_number"),
func.sum(Run.summary["failures"].cast(Float)).label("failures"),
func.sum(Run.summary["errors"].cast(Float)).label("errors"),
func.sum(Run.summary["skips"].cast(Float)).label("skips"),
func.sum(Run.summary["xfailures"].cast(Float)).label("xfailures"),
func.sum(Run.summary["xpasses"].cast(Float)).label("xpasses"),
func.sum(Run.summary["tests"].cast(Float)).label("total"),
).group_by(group_field, job_name, build_number, Run.id)
# add filters to the query
query = apply_filters(query, filters, Run)
# convert the base query to a sub query
subquery = query.subquery()
# create the main query (this allows us to do math on the SQL side)
if count_skips:
passes = subquery.c.total - (subquery.c.errors + subquery.c.skips +
subquery.c.failures + subquery.c.xpasses +
subquery.c.xfailures)
else:
passes = subquery.c.total - (subquery.c.errors + subquery.c.failures +
subquery.c.xpasses + subquery.c.xfailures)
query = session.query(
subquery.c.group_field,
subquery.c.build_number,
subquery.c.run_id,
subquery.c.annotations,
# handle potential division by 0 errors, if the total is 0, set the pass_percent to 0
case(
[
(subquery.c.total == 0, 0),
],
else_=(100 * passes / subquery.c.total),
).label("pass_percent"),
)
# parse the data for the frontend
query_data = query.all()
data = {datum.group_field: [] for datum in query_data}
for datum in query_data:
data[datum.group_field].append([
round(datum.pass_percent, 2), datum.run_id, datum.annotations,
datum.build_number
])
# compute the slope for each component
data_with_slope = data.copy()
for key, value in data.items():
slope_info = _calculate_slope([v[0] for v in value])
data_with_slope[key].insert(0, [slope_info, 0])
return data_with_slope, builds
def _get_jenkins_aggregation(filters=None,
project=None,
page=1,
page_size=25,
run_limit=None):
"""Get a list of Jenkins jobs"""
offset = (page * page_size) - page_size
# first create the filters
query_filters = [
"metadata.jenkins.build_number@y", "metadata.jenkins.job_name@y"
]
if filters:
for idx, filter in enumerate(filters):
if "job_name" in filter or "build_number" in filter:
filters[idx] = f"metadata.jenkins.{filter}"
query_filters.extend(filters)
if project:
query_filters.append(f"project_id={project}")
filters = query_filters
# generate the group_fields
job_name = string_to_column("metadata.jenkins.job_name", Run)
build_number = string_to_column("metadata.jenkins.build_number", Run)
build_url = string_to_column("metadata.jenkins.build_url", Run)
env = string_to_column("env", Run)
# get the runs on which to run the aggregation, we select from a subset of runs to improve
# performance, otherwise we'd be aggregating over ALL runs
sub_query = (apply_filters(Run.query, filters, Run).order_by(
desc("start_time")).limit(run_limit or JJV_RUN_LIMIT).subquery())
# create the base query
query = (session.query(
job_name.label("job_name"),
build_number.label("build_number"),
func.min(build_url.cast(Text)).label("build_url"),
func.min(env).label("env"),
func.min(Run.source).label("source"),
func.sum(Run.summary["xfailures"].cast(Integer)).label("xfailures"),
func.sum(Run.summary["xpasses"].cast(Integer)).label("xpasses"),
func.sum(Run.summary["failures"].cast(Integer)).label("failures"),
func.sum(Run.summary["errors"].cast(Integer)).label("errors"),
func.sum(Run.summary["skips"].cast(Integer)).label("skips"),
func.sum(Run.summary["tests"].cast(Integer)).label("tests"),
func.min(Run.start_time).label("min_start_time"),
func.max(Run.start_time).label("max_start_time"),
func.sum(Run.duration).label("total_execution_time"),
func.max(Run.duration).label("max_duration"),
).select_entity_from(sub_query).group_by(job_name, build_number).order_by(
desc("max_start_time")))
# apply filters to the query
query = apply_filters(query, filters, Run)
# apply pagination and get data
query_data = query.offset(offset).limit(page_size).all()
# parse the data for the frontend
data = {
"jobs": [],
"pagination": {
"page": page,
"pageSize": page_size,
"totalItems": query.count(), # TODO: examine performance here
},
}
for datum in query_data:
data["jobs"].append({
"_id":
f"{datum.job_name}-{datum.build_number}",
"build_number":
datum.build_number,
"build_url":
datum.build_url,
"duration":
(datum.max_start_time.timestamp() -
datum.min_start_time.timestamp()) + datum.max_duration, # noqa
"env":
datum.env,
"job_name":
datum.job_name,
"source":
datum.source,
"start_time":
datum.min_start_time,
"summary": {
"xfailures":
datum.xfailures,
"xpasses":
datum.xpasses,
"errors":
datum.errors,
"failures":
datum.failures,
"skips":
datum.skips,
"tests":
datum.tests,
"passes":
datum.tests - (datum.errors + datum.failures + datum.skips),
},
"total_execution_time":
datum.total_execution_time,
})
return data