def retrieve(self, request, project, jm, pk=None):
"""
GET method implementation for detail view
Return a single job with log_references and
artifact names and links to the artifact blobs.
"""
obj = jm.get_job(pk)
if obj:
job = obj[0]
job["resource_uri"] = reverse("jobs-detail",
kwargs={"project": jm.project, "pk": job["id"]})
job["logs"] = jm.get_log_references(pk)
# make artifact ids into uris
with ArtifactsModel(project) as artifacts_model:
artifact_refs = artifacts_model.get_job_artifact_references(pk)
job["artifacts"] = []
for art in artifact_refs:
ref = reverse("artifact-detail",
kwargs={"project": jm.project, "pk": art["id"]})
art["resource_uri"] = ref
job["artifacts"].append(art)
option_collections = jm.refdata_model.get_all_option_collections()
job["platform_option"] = get_option(job, option_collections)
return Response(job)
else:
return Response("No job with id: {0}".format(pk), 404)
def list(self, request, project, jm):
"""
GET method implementation for list view
Optional paramters (default):
- offset (0)
- count (10)
- return_type (dict)
"""
filter = UrlQueryFilter(request.query_params)
offset = int(filter.pop("offset", 0))
count = min(int(filter.pop("count", 10)), 2000)
return_type = filter.pop("return_type", "dict").lower()
exclusion_profile = filter.pop("exclusion_profile", "default")
visibility = filter.pop("visibility", "included")
if exclusion_profile in ('false', 'null'):
exclusion_profile = None
results = jm.get_job_list(offset, count, conditions=filter.conditions,
exclusion_profile=exclusion_profile,
visibility=visibility)
if results:
option_collections = jm.refdata_model.get_all_option_collections()
for job in results:
job["platform_option"] = get_option(job, option_collections)
response_body = dict(meta={"repository": project}, results=[])
if results and return_type == "list":
response_body["job_property_names"] = results[0].keys()
results = [job.values() for job in results]
response_body["results"] = results
response_body["meta"].update(offset=offset, count=count)
return Response(response_body)
def retrieve(self, request, project, jm, pk=None):
"""
GET method implementation for detail view
Return a single job with log_references and
artifact names and links to the artifact blobs.
"""
obj = jm.get_job(pk)
if obj:
job = obj[0]
job["resource_uri"] = reverse("jobs-detail",
kwargs={"project": jm.project, "pk": job["id"]})
job["logs"] = jm.get_log_references(pk)
# make artifact ids into uris
with ArtifactsModel(project) as artifacts_model:
artifact_refs = artifacts_model.get_job_artifact_references(pk)
job["artifacts"] = []
for art in artifact_refs:
ref = reverse("artifact-detail",
kwargs={"project": jm.project, "pk": art["id"]})
art["resource_uri"] = ref
job["artifacts"].append(art)
option_collections = jm.refdata_model.get_all_option_collections()
job["platform_option"] = get_option(job, option_collections)
return Response(job)
else:
return Response("No job with id: {0}".format(pk), 404)
def list(self, request, project, jm):
"""
GET method implementation for list view
Optional paramters (default):
- offset (0)
- count (10)
- return_type (dict)
"""
filter = UrlQueryFilter(request.query_params)
offset = int(filter.pop("offset", 0))
count = min(int(filter.pop("count", 10)), 2000)
return_type = filter.pop("return_type", "dict").lower()
exclusion_profile = filter.pop("exclusion_profile", "default")
visibility = filter.pop("visibility", "included")
if exclusion_profile in ('false', 'null'):
exclusion_profile = None
results = jm.get_job_list(offset, count, conditions=filter.conditions,
exclusion_profile=exclusion_profile,
visibility=visibility)
if results:
option_collections = jm.refdata_model.get_all_option_collections()
for job in results:
job["platform_option"] = get_option(job, option_collections)
response_body = dict(meta={"repository": project}, results=[])
if results and return_type == "list":
response_body["job_property_names"] = results[0].keys()
results = [job.values() for job in results]
response_body["results"] = results
response_body["meta"].update(offset=offset, count=count)
return Response(response_body)
def list(self, request, project, jm):
"""
GET method implementation for list view
"""
filter = UrlQueryFilter(request.QUERY_PARAMS)
offset = filter.pop("offset", 0)
count = min(int(filter.pop("count", 10)), 1000)
full = filter.pop('full', 'true').lower() == 'true'
objs = jm.get_job_list(offset, count, full, filter.conditions)
if objs:
option_collections = jm.refdata_model.get_all_option_collections()
for job in objs:
job["platform_option"] = get_option(job, option_collections)
return Response(objs)
def list(self, request, project, jm):
"""
GET method implementation for list view
"""
filter = UrlQueryFilter(request.QUERY_PARAMS)
offset = filter.pop("offset", 0)
count = min(int(filter.pop("count", 10)), 1000)
full = filter.pop('full', 'true').lower() == 'true'
objs = jm.get_job_list(offset, count, full, filter.conditions)
if objs:
option_collections = jm.refdata_model.get_all_option_collections()
for job in objs:
job["platform_option"] = get_option(job, option_collections)
return Response(objs)
def get_resultsets_with_jobs(jm,
rs_list,
full,
filter_kwargs,
debug,
exclusion_state='included',
sort_key='push_timestamp'):
"""Convert db result of resultsets in a list to JSON"""
rs_map = {}
for rs in rs_list:
rs_map[rs["id"]] = rs
# all rs should have the revisions_uri, so add it here
rs["revisions_uri"] = reverse("resultset-revisions",
kwargs={
"project": jm.project,
"pk": rs["id"]
})
job_list = jm.get_result_set_job_list(rs_map.keys(), full,
exclusion_state, **filter_kwargs)
jobs_ungrouped = job_list.get('job_list', [])
reference_signature_names = job_list.get('reference_signature_names',
{})
option_collections = jm.refdata_model.get_all_option_collections()
rs_grouper = lambda rsg: rsg["result_set_id"]
# the main grouper for a result set is the combination of
# platform and options
platform_grouper = lambda pg: (
PLATFORM_ORDER.get(pg["platform"], 100),
OPT_ORDER.get(get_option(pg, option_collections), 100))
def get_sortable_job_symbol(symbol):
"""
Sort jobs by symbol.
Symbol could be something like 1, 2 or 3. Or A, B, C or R1, R2, R10.
So this will pad the numeric portion with 0s like R001, R010, etc.
"""
newsymbol = symbol
if symbol.isdigit():
newsymbol = symbol.zfill(3)
else:
x = re.split(r'(\d+)', symbol)
newsymbol = x[0]
if len(x) > 1:
newsymbol += x[1].zfill(3)
return newsymbol
job_group_grouper = lambda jgg: jgg["job_group_symbol"]
job_type_grouper = lambda jtg: (get_sortable_job_symbol(jtg[
'job_type_symbol']))
rs_sorted = sorted(jobs_ungrouped, key=rs_grouper)
resultsets = []
for rs_id, resultset_group in itertools.groupby(rs_sorted,
key=rs_grouper):
resultset = rs_map[rs_id]
resultsets.append(resultset)
# we found jobs for this resultset, so remove it from the map
# now that it's in the ``resultsets`` list.
# after we are done with all these jobs, whatever is in the map are
# resultsets with no jobs yet, which we add back in to the list
# of resultsets to be returned.
del (rs_map[rs_id])
job_counts = dict.fromkeys(
jm.RESULTS + jm.INCOMPLETE_STATES + ["total"], 0)
#itertools needs the elements to be sorted by the grouper
by_platform = sorted(list(resultset_group), key=platform_grouper)
platforms = []
for platform_group_name, platform_group in itertools.groupby(
by_platform, key=platform_grouper):
by_job_group = sorted(list(platform_group),
key=job_group_grouper)
platform_name = by_job_group[0]["platform"]
platform_option = option_collections[
by_job_group[0]["option_collection_hash"]]['opt']
groups = []
for jg_symbol, jg_group in itertools.groupby(
by_job_group, job_group_grouper):
by_job_type = sorted(list(jg_group), key=job_type_grouper)
job_list = []
groups.append({
"symbol": jg_symbol,
"name": by_job_type[0]["job_group_name"],
"jobs": job_list
})
# build the uri ref for each job
for job in by_job_type:
job_list.append(
get_job_value_list(job, reference_signature_names,
platform_option, jm.project,
debug))
if job["state"] == "completed":
job_counts[job["result"]] += 1
else:
job_counts[job["state"]] += 1
job_counts["total"] += 1
platforms.append({
"name": platform_name,
"option": platform_option,
"groups": groups,
})
#the unique set of results that are contained in this resultset
#can be used to determine the resultset's severity
resultset.update({
"platforms": platforms,
"job_counts": job_counts,
})
# the resultsets left in the map have no jobs, so fill in the fields
# with blanks that WOULD otherwise have been filled.
for rs in rs_map.values():
rs.update({
"platforms": [],
"job_counts":
dict.fromkeys(jm.RESULTS + jm.INCOMPLETE_STATES + ["total"],
0),
})
resultsets.append(rs)
return sorted(resultsets, key=lambda x: x[sort_key], reverse=True)
def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs):
"""Convert db result of resultsets in a list to JSON"""
# Fetch the job results all at once, then parse them out in memory.
# organize the resultsets into an obj by key for lookups
rs_map = {}
for rs in rs_list:
rs_map[rs["id"]] = rs
# all rs should have the revisions_uri, so add it here
rs["revisions_uri"] = reverse("resultset-revisions",
kwargs={"project": jm.project, "pk": rs["id"]})
jobs_ungrouped = jm.get_result_set_job_list(
rs_map.keys(),
full,
**filter_kwargs
)
option_collections = jm.refdata_model.get_all_option_collections()
rs_grouper = lambda rsg: rsg["result_set_id"]
# the main grouper for a result set is the combination of
# platform and options
platform_grouper = lambda pg: "{0} {1}".format(
pg["platform"],
get_option(pg, option_collections)
)
job_group_grouper = lambda jgg: jgg["job_group_symbol"]
job_type_grouper = lambda jtg: jtg['job_type_symbol']
rs_sorted = sorted(jobs_ungrouped, key=rs_grouper)
resultsets = []
for rs_id, resultset_group in itertools.groupby(rs_sorted, key=rs_grouper):
resultset = rs_map[rs_id]
resultsets.append(resultset)
# we found jobs for this resultset, so remove it from the map
# now that it's in the ``resultsets`` list.
# after we are done with all these jobs, whatever is in the map are
# resultsets with no jobs yet, which we add back in to the list
# of resultsets to be returned.
del(rs_map[rs_id])
job_counts = dict.fromkeys(
jm.RESULTS + jm.INCOMPLETE_STATES + ["total"], 0)
#itertools needs the elements to be sorted by the grouper
by_platform = sorted(list(resultset_group), key=platform_grouper)
platforms = []
for platform_group_name, platform_group in itertools.groupby(
by_platform,
key=platform_grouper):
by_job_group = sorted(list(platform_group), key=job_group_grouper)
platform_name = by_job_group[0]["platform"]
platform_option = option_collections[
by_job_group[0]["option_collection_hash"]]['opt']
groups = []
for jg_symbol, jg_group in itertools.groupby(
by_job_group,
job_group_grouper):
by_job_type = sorted(list(jg_group), key=job_type_grouper)
groups.append({
"symbol": jg_symbol,
"name": by_job_type[0]["job_group_name"],
"jobs": by_job_type
})
# build the uri ref for each job
for job in by_job_type:
job["id"] = job["job_id"]
del(job["job_id"])
del(job["option_collection_hash"])
job["platform_option"] = platform_option
job["resource_uri"] = reverse("jobs-detail",
kwargs={"project": jm.project, "pk": job["id"]})
if job["state"] == "completed":
job_counts[job["result"]] += 1
else:
job_counts[job["state"]] += 1
job_counts["total"] += 1
platforms.append({
"name": platform_name,
"option": platform_option,
"groups": groups,
})
#the unique set of results that are contained in this resultset
#can be used to determine the resultset's severity
resultset.update({
"platforms": platforms,
"job_counts": job_counts,
})
# the resultsets left in the map have no jobs, so fill in the fields
# with blanks that WOULD otherwise have been filled.
for rs in rs_map.values():
rs.update({
"platforms": [],
"job_counts": dict.fromkeys(
jm.RESULTS + jm.INCOMPLETE_STATES + ["total"], 0),
})
resultsets.append(rs)
return sorted(
resultsets,
key=lambda x: x["push_timestamp"],
reverse=True)
def get_resultsets_with_jobs(
jm, rs_list, full, filter_kwargs, debug, sort_key='push_timestamp'):
"""Convert db result of resultsets in a list to JSON"""
if 'result_set_ids' in filter_kwargs:
del filter_kwargs['result_set_ids']
rs_map = {}
for rs in rs_list:
rs_map[rs["id"]] = rs
# all rs should have the revisions_uri, so add it here
rs["revisions_uri"] = reverse("resultset-revisions",
kwargs={"project": jm.project, "pk": rs["id"]})
job_list = jm.get_result_set_job_list(
rs_map.keys(),
full,
**filter_kwargs
)
jobs_ungrouped = job_list.get('job_list', [])
reference_signature_names = job_list.get('reference_signature_names', {})
option_collections = jm.refdata_model.get_all_option_collections()
rs_grouper = lambda rsg: rsg["result_set_id"]
# the main grouper for a result set is the combination of
# platform and options
platform_grouper = lambda pg: (
PLATFORM_ORDER.get(pg["platform"], 100),
OPT_ORDER.get(get_option(pg, option_collections), 100)
)
def get_sortable_job_symbol(symbol):
"""
Sort jobs by symbol.
Symbol could be something like 1, 2 or 3. Or A, B, C or R1, R2, R10.
So this will pad the numeric portion with 0s like R001, R010, etc.
"""
newsymbol = symbol
if symbol.isdigit():
newsymbol = symbol.zfill(3)
else:
x = re.split('(\d+)', symbol)
newsymbol = x[0]
if len(x) > 1:
newsymbol += x[1].zfill(3)
return newsymbol
job_group_grouper = lambda jgg: jgg["job_group_symbol"]
job_type_grouper = lambda jtg: (get_sortable_job_symbol(jtg['job_type_symbol']))
rs_sorted = sorted(jobs_ungrouped, key=rs_grouper)
resultsets = []
for rs_id, resultset_group in itertools.groupby(rs_sorted, key=rs_grouper):
resultset = rs_map[rs_id]
resultsets.append(resultset)
# we found jobs for this resultset, so remove it from the map
# now that it's in the ``resultsets`` list.
# after we are done with all these jobs, whatever is in the map are
# resultsets with no jobs yet, which we add back in to the list
# of resultsets to be returned.
del(rs_map[rs_id])
job_counts = dict.fromkeys(
jm.RESULTS + jm.INCOMPLETE_STATES + ["total"], 0)
#itertools needs the elements to be sorted by the grouper
by_platform = sorted(list(resultset_group), key=platform_grouper)
platforms = []
for platform_group_name, platform_group in itertools.groupby(
by_platform,
key=platform_grouper):
by_job_group = sorted(list(platform_group), key=job_group_grouper)
platform_name = by_job_group[0]["platform"]
platform_option = option_collections[
by_job_group[0]["option_collection_hash"]]['opt']
groups = []
for jg_symbol, jg_group in itertools.groupby(
by_job_group,
job_group_grouper):
by_job_type = sorted(list(jg_group), key=job_type_grouper)
job_list = []
groups.append({
"symbol": jg_symbol,
"name": by_job_type[0]["job_group_name"],
"jobs": job_list
})
# build the uri ref for each job
for job in by_job_type:
job_list.append(
get_job_value_list(
job, reference_signature_names,
platform_option, jm.project, debug
)
)
if job["state"] == "completed":
job_counts[job["result"]] += 1
else:
job_counts[job["state"]] += 1
job_counts["total"] += 1
platforms.append({
"name": platform_name,
"option": platform_option,
"groups": groups,
})
#the unique set of results that are contained in this resultset
#can be used to determine the resultset's severity
resultset.update({
"platforms": platforms,
"job_counts": job_counts,
})
# the resultsets left in the map have no jobs, so fill in the fields
# with blanks that WOULD otherwise have been filled.
for rs in rs_map.values():
rs.update({
"platforms": [],
"job_counts": dict.fromkeys(
jm.RESULTS + jm.INCOMPLETE_STATES + ["total"], 0),
})
resultsets.append(rs)
return sorted(
resultsets,
key=lambda x: x[sort_key],
reverse=True)
