def TryChooserGetEndtoEndTimes(starttime=None,
endtime=None,
branch_name='mozilla-central'):
"""Get end to end times report for the speficied time interval and branch.
Input: starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
branch_name - branch name, default vaue is 'mozilla-central'
Output: EndtoEndTimesReport
"""
starttime, endtime = get_time_interval(starttime, endtime)
q = BuildRequestsQuery(starttime=starttime,
endtime=endtime,
branch_name=branch_name)
q_results = q.execute()
report = TryChooserEndtoEndTimesReport(starttime, endtime, branch_name)
for r in q_results:
params = dict((str(k), v) for (k, v) in dict(r).items())
br = BuildRequest(**params)
report.add_build_request(br)
return report
def GetBuilderDetailsReport(builder_name=None, starttime=None, endtime=None):
"""Get the builder details report based on statusdb for a builder in the
speficied time interval.
Input: builder_name - builder name
starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
Output: BuilderDetailsReport
"""
starttime, endtime = get_time_interval(starttime, endtime)
starttime_date = datetime.fromtimestamp(starttime)
endtime_date = datetime.fromtimestamp(endtime)
report = BuilderDetailsReport(starttime, endtime, name=builder_name)
q = BuildsQuery(builder_name=builder_name, get_builder_name=True,
starttime=starttime_date, endtime=endtime_date)
q_results = q.execute()
for r in q_results:
params = dict((str(k), v) for (k, v) in dict(r).items())
build = Build(**params)
report.add(build)
return report
def GetBuilderDetailsReport(builder_name=None, starttime=None, endtime=None):
"""Get the builder details report based on statusdb for a builder in the
speficied time interval.
Input: builder_name - builder name
starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
Output: BuilderDetailsReport
"""
starttime, endtime = get_time_interval(starttime, endtime)
starttime_date = datetime.fromtimestamp(starttime)
endtime_date = datetime.fromtimestamp(endtime)
report = BuilderDetailsReport(starttime, endtime, name=builder_name)
q = BuildsQuery(builder_name=builder_name,
get_builder_name=True,
starttime=starttime_date,
endtime=endtime_date)
q_results = q.execute()
for r in q_results:
params = dict((str(k), v) for (k, v) in dict(r).items())
build = Build(**params)
report.add(build)
return report
def GetBuildersReport(starttime=None, endtime=None,
branch_name='mozilla-central', platform=None, build_type=None,
job_type=None, detail_level='builder'):
"""Get the average time per builder report for the speficied time interval
and branch.
Input: starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
branch_name - branch name, default vaue is 'mozilla-central'
Output: BuildersReport
"""
platform = platform or []
build_type = build_type or []
job_type = job_type or []
starttime, endtime = get_time_interval(starttime, endtime)
detail_level_no = BUILDERS_DETAIL_LEVELS.index(detail_level) + 1
q = BuildersQuery(starttime, endtime, branch_name)
q_results = q.execute()
report = BuildersReport(starttime, endtime, branch_name,
detail_level=detail_level_no)
report.set_filters(dict(platform=platform, build_type=build_type,
job_type=job_type))
for r in q_results:
params = dict((str(k), v) for (k, v) in dict(r).items())
br = BuildRequest(**params)
report.add(br)
return report
def GetPushes(starttime=None, endtime=None, int_size=0, branches=None):
"""Get pushes and statistics.
Input: starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
int_size - break down results per interval (in seconds), if specified
branches - filter by list of branches, if not spefified fetches all
branches
Output: pushes report
"""
starttime, endtime = get_time_interval(starttime, endtime)
q = PushesQuery(starttime, endtime, branches)
q_results = q.execute()
report = PushesReport(starttime, endtime, int_size=int_size,
branches=branches)
for r in q_results:
branch_name = get_branch_name(r['branch'])
stime = float(r['when_timestamp'])
revision = r['revision']
push = Push(stime, branch_name, revision)
report.add(push)
return report
def GetBuilderTypeReport(starttime=None, endtime=None, buildername=None):
"""Get the average time per builder report for one builder for the
speficied time interval. The builder is specified by its buildername.
Input: starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is
not specified either)
buildername - buildername
Output: BuilderTypeReport
"""
starttime, endtime = get_time_interval(starttime, endtime)
q = BuildersTypeQuery(starttime, endtime, buildername)
q_results = q.execute()
report = BuilderTypeReport(buildername=buildername,
starttime=starttime,
endtime=endtime)
for r in q_results:
params = dict((str(k), v) for (k, v) in dict(r).items())
br = BuildRequest(**params)
report.add(br)
return report
def GetSlaveDetailsReport(slave_id=None, starttime=None, endtime=None,
int_size=0, last_int_size=0):
"""Get the slave details report for a slave in the speficied time interval.
Input: slave_id - slave id
starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
int_size - break down results per interval (in seconds), if specified
last_int_size - the length in seconds for the last time interval
for which to compute fail and busy/idle percentage.
Output: SlaveDetailsReport
"""
starttime, endtime = get_time_interval(starttime, endtime)
starttime_date = datetime.fromtimestamp(starttime)
endtime_date = datetime.fromtimestamp(endtime)
report = SlaveDetailsReport(starttime, endtime, slave_id,
int_size=int_size, last_int_size=last_int_size)
q = BuildsQuery(slave_id=slave_id, get_builder_name=True,
starttime=starttime_date, endtime=endtime_date)
q_results = q.execute()
for r in q_results:
params = dict((str(k), v) for (k, v) in dict(r).items())
build = Build(**params)
report.add(build)
if not report.name:
report.name = build.slave_name
return report
def GetEndtoEndTimes(starttime=None, endtime=None,
branch_name='mozilla-central'):
"""Get end to end times report for the speficied time interval and branch.
Input: starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
branch_name - branch name, default vaue is 'mozilla-central'
Output: EndtoEndTimesReport
"""
starttime, endtime = get_time_interval(starttime, endtime)
report = EndtoEndTimesReport(starttime, endtime, branch_name)
build_requests = GetBuildRequests(branch_name=branch_name,
starttime=starttime, endtime=endtime, changeid_all=True)
for key in build_requests:
report.add_build_request(build_requests[key])
changes = GetChanges(branch_name=branch_name, starttime=starttime,
endtime=endtime, pending_only=True)
report.parse_incomplete(changes)
return report
def GetSlavesReport(starttime=None, endtime=None, int_size=0, last_int_size=0):
"""Get the slaves report for the speficied time interval.
Input: starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
last_int_size - the length in seconds for the last time interval
for which to compute fail and busy/idle percentage.
Output: SlavesReport
"""
starttime, endtime = get_time_interval(starttime, endtime)
starttime_date = datetime.fromtimestamp(starttime)
endtime_date = datetime.fromtimestamp(endtime)
report = SlavesReport(starttime,
endtime,
int_size=int_size,
last_int_size=last_int_size)
q = BuildsQuery(starttime=starttime_date, endtime=endtime_date)
q_results = q.execute()
for r in q_results:
params = dict((str(k), v) for (k, v) in dict(r).items())
build = Build(**params)
report.add(build)
return report
def GetPushes(starttime=None, endtime=None, int_size=0, branches=None):
"""Get pushes and statistics.
Input: starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
int_size - break down results per interval (in seconds), if specified
branches - filter by list of branches, if not spefified fetches all
branches
Output: pushes report
"""
starttime, endtime = get_time_interval(starttime, endtime)
q = PushesQuery(starttime, endtime, branches)
q_results = q.execute()
report = PushesReport(starttime,
endtime,
int_size=int_size,
branches=branches)
for r in q_results:
branch_name = get_branch_name(r['branch'])
stime = float(r['when_timestamp'])
revision = r['revision']
push = Push(stime, branch_name, revision)
report.add(push)
return report
def GetBuilderTypeReport(starttime=None, endtime=None, buildername=None):
"""Get the average time per builder report for one builder for the
speficied time interval. The builder is specified by its buildername.
Input: starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is
not specified either)
buildername - buildername
Output: BuilderTypeReport
"""
starttime, endtime = get_time_interval(starttime, endtime)
q = BuildersTypeQuery(starttime, endtime, buildername)
q_results = q.execute()
report = BuilderTypeReport(buildername=buildername, starttime=starttime,
endtime=endtime)
for r in q_results:
params = dict((str(k), v) for (k, v) in dict(r).items())
br = BuildRequest(**params)
report.add(br)
return report
def GetEndtoEndTimes(starttime=None,
endtime=None,
branch_name='mozilla-central'):
"""Get end to end times report for the speficied time interval and branch.
Input: starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
branch_name - branch name, default vaue is 'mozilla-central'
Output: EndtoEndTimesReport
"""
starttime, endtime = get_time_interval(starttime, endtime)
report = EndtoEndTimesReport(starttime, endtime, branch_name)
build_requests = GetBuildRequests(branch_name=branch_name,
starttime=starttime,
endtime=endtime,
changeid_all=True)
for key in build_requests:
report.add_build_request(build_requests[key])
changes = GetChanges(branch_name=branch_name,
starttime=starttime,
endtime=endtime,
pending_only=True)
report.parse_incomplete(changes)
return report
def TryChooserGetEndtoEndTimes(starttime=None, endtime=None,
branch_name='mozilla-central'):
"""Get end to end times report for the speficied time interval and branch.
Input: starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
branch_name - branch name, default vaue is 'mozilla-central'
Output: EndtoEndTimesReport
"""
starttime, endtime = get_time_interval(starttime, endtime)
q = BuildRequestsQuery(starttime=starttime, endtime=endtime,
branch_name=branch_name)
q_results = q.execute()
report = TryChooserEndtoEndTimesReport(starttime, endtime, branch_name)
for r in q_results:
params = dict((str(k), v) for (k, v) in dict(r).items())
br = BuildRequest(**params)
report.add_build_request(br)
return report
def to_python(self, value_dict, state):
starttime, endtime = value_dict['starttime'], value_dict['endtime']
starttime, endtime = get_time_interval(starttime, endtime)
value_dict['starttime'] = starttime
value_dict['endtime'] = endtime
return value_dict
def to_python(self, value_dict, state):
starttime, endtime = value_dict["starttime"], value_dict["endtime"]
starttime, endtime = get_time_interval(starttime, endtime)
value_dict["starttime"] = starttime
value_dict["endtime"] = endtime
return value_dict
def GetWaitTimes(pool='buildpool',
mpb=15,
starttime=None,
endtime=None,
int_size=0,
maxb=0):
"""Get wait times and statistics for buildpool.
Input: pool - name of the pool (e.g. buildpool, or trybuildpool)
mpb - minutes per block, length of wait time block in minutes
starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is
not specified either)
int_size - break down results per interval (in seconds), if specified
maxb - maximum block size; for wait times larger than maxb, group
into the largest block
Output: wait times report
"""
starttime, endtime = get_time_interval(starttime, endtime)
q = WaitTimesQuery(starttime, endtime, pool)
q_results = q.execute()
report = WaitTimesReport(pool,
starttime,
endtime,
mpb=mpb,
maxb=maxb,
int_size=int_size,
masters=get_masters_for_pool(pool))
for r in q_results:
buildername = r['buildername']
# start time is changes.when_timestamp, or buildrequests.submitted_at
# if build has no changes
stime = r['when_timestamp'] or r['submitted_at']
etime = r['start_time']
platform = get_platform(buildername)
has_no_changes = not bool(r['when_timestamp'])
wt = WaitTime(stime,
etime,
platform,
buildername=buildername,
has_no_changes=has_no_changes)
report.add(wt)
return report
def GetTestRuns(starttime=None, endtime=None, int_size=0, category=None, platform=None, group=False, btype=None):
"""Get test run metrics for a given interval
Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours
current time (if starttime is not specified either)
int_size - break down results per interval (in seconds), if specified
category - filter by list of builder categories, if not specified fetches all categories
platform - filter by platform, if not specified, fetch all platforms
group - group results together by test suite if specified, otherwise fetch all builders uniquely
btype - filter by build type (opt/debug/etc), if not specified, return all
Output: pushes report
"""
starttime, endtime = get_time_interval(starttime, endtime)
q = TestRunsQuery(starttime, endtime, category=category)
q_results = q.execute()
report = TestRunsReport(starttime, endtime, category=category, platform=platform, group=group, btype=btype)
filtered_results = []
#filter by platform, build type
if platform and btype:
filtered_results = filter(lambda x: platform == get_platform(x['name']) and btype == get_build_type(x['name']), q_results)
elif platform and not btype:
filtered_results = filter(lambda x: platform == get_platform(x['name']), q_results)
elif btype and not platform:
filtered_results = filter(lambda x: btype == get_build_type(x['name']), q_results)
else:
filtered_results = list(q_results)
if group:
for test in test_suites:
grouped_results = filter(lambda p: p['name'].find(test)>-1, filtered_results)
if grouped_results:
average_total = reduce(lambda x, y: x + y, map(lambda x: x['total'], grouped_results)) / len(grouped_results)
average_test = reduce(lambda x, y: x + y, map(lambda x: x['test'], grouped_results)) / len(grouped_results)
row = { 'total': int(average_total),
'test' : int(average_test),
'ratio': average_test/average_total,
'platform': platform,
}
report.add(test, row)
else:
for r in filtered_results:
this_row = { 'total':r['total'],
'test':r['test'],
'ratio':r['test']/r['total'],
'platform': get_platform(r['name']),
}
report.add(r['name'], this_row)
return report
def GetIdleJobsReport(starttime=None, endtime=None, int_size=0):
"""Get test metrics for idlejobs jobs in a given time interval
Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours
current time (if starttime is not specified either)
int_size - break down results per interval (in seconds), if specified
Output: idlejobs report
"""
starttime, endtime = get_time_interval(starttime, endtime)
q = IdleJobsQuery(starttime, endtime)
q_results = q.execute()
report = IdleJobsReport(starttime, endtime, int_size)
for r in q_results:
this_row = {
'starttime': GetTimeStamp(r['starttime']),
'endtime' : GetTimeStamp(r['endtime']),
}
report.add(r['name'], this_row)
return report
def GetBuildersReport(starttime=None,
endtime=None,
branch_name='mozilla-central',
platform=None,
build_type=None,
job_type=None,
detail_level='builder'):
"""Get the average time per builder report for the speficied time interval
and branch.
Input: starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is not
specified either)
branch_name - branch name, default vaue is 'mozilla-central'
Output: BuildersReport
"""
platform = platform or []
build_type = build_type or []
job_type = job_type or []
starttime, endtime = get_time_interval(starttime, endtime)
detail_level_no = BUILDERS_DETAIL_LEVELS.index(detail_level) + 1
q = BuildersQuery(starttime, endtime, branch_name)
q_results = q.execute()
report = BuildersReport(starttime,
endtime,
branch_name,
detail_level=detail_level_no)
report.set_filters(
dict(platform=platform, build_type=build_type, job_type=job_type))
for r in q_results:
params = dict((str(k), v) for (k, v) in dict(r).items())
br = BuildRequest(**params)
report.add(br)
return report
def GetWaitTimes(pool='buildpool', mpb=15, starttime=None, endtime=None,
int_size=0, maxb=0):
"""Get wait times and statistics for buildpool.
Input: pool - name of the pool (e.g. buildpool, or trybuildpool)
mpb - minutes per block, length of wait time block in minutes
starttime - start time (UNIX timestamp in seconds), if not
specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified,
starttime plus 24 hours or current time (if starttime is
not specified either)
int_size - break down results per interval (in seconds), if specified
maxb - maximum block size; for wait times larger than maxb, group
into the largest block
Output: wait times report
"""
starttime, endtime = get_time_interval(starttime, endtime)
q = WaitTimesQuery(starttime, endtime, pool)
q_results = q.execute()
report = WaitTimesReport(pool, starttime, endtime, mpb=mpb, maxb=maxb,
int_size = int_size, masters=get_masters_for_pool(pool))
for r in q_results:
buildername = r['buildername']
# start time is changes.when_timestamp, or buildrequests.submitted_at
# if build has no changes
stime = r['when_timestamp'] or r['submitted_at']
etime = r['start_time']
platform = get_platform(buildername)
has_no_changes = not bool(r['when_timestamp'])
wt = WaitTime(stime, etime, platform, buildername=buildername,
has_no_changes=has_no_changes)
report.add(wt)
return report
def GetTestRuns(starttime=None,
endtime=None,
int_size=0,
category=None,
platform=None,
group=False,
btype=None):
"""Get test run metrics for a given interval
Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours
endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours
current time (if starttime is not specified either)
int_size - break down results per interval (in seconds), if specified
category - filter by list of builder categories, if not specified fetches all categories
platform - filter by platform, if not specified, fetch all platforms
group - group results together by test suite if specified, otherwise fetch all builders uniquely
btype - filter by build type (opt/debug/etc), if not specified, return all
Output: pushes report
"""
starttime, endtime = get_time_interval(starttime, endtime)
q = TestRunsQuery(starttime, endtime, category=category)
q_results = q.execute()
report = TestRunsReport(starttime,
endtime,
category=category,
platform=platform,
group=group,
btype=btype)
filtered_results = []
#filter by platform, build type
if platform and btype:
filtered_results = filter(
lambda x: platform == get_platform(x['name']) and btype ==
get_build_type(x['name']), q_results)
elif platform and not btype:
filtered_results = filter(
lambda x: platform == get_platform(x['name']), q_results)
elif btype and not platform:
filtered_results = filter(lambda x: btype == get_build_type(x['name']),
q_results)
else:
filtered_results = list(q_results)
if group:
for test in test_suites:
grouped_results = filter(lambda p: p['name'].find(test) > -1,
filtered_results)
if grouped_results:
average_total = reduce(
lambda x, y: x + y,
map(lambda x: x['total'],
grouped_results)) / len(grouped_results)
average_test = reduce(
lambda x, y: x + y,
map(lambda x: x['test'],
grouped_results)) / len(grouped_results)
row = {
'total': int(average_total),
'test': int(average_test),
'ratio': average_test / average_total,
'platform': platform,
}
report.add(test, row)
else:
for r in filtered_results:
this_row = {
'total': r['total'],
'test': r['test'],
'ratio': r['test'] / r['total'],
'platform': get_platform(r['name']),
}
report.add(r['name'], this_row)
return report
