def TryChooserGetEndtoEndTimes(starttime=None, endtime=None, branch_name='mozilla-central'): """Get end to end times report for the speficied time interval and branch. Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours or current time (if starttime is not specified either) branch_name - branch name, default vaue is 'mozilla-central' Output: EndtoEndTimesReport """ starttime, endtime = get_time_interval(starttime, endtime) q = BuildRequestsQuery(starttime=starttime, endtime=endtime, branch_name=branch_name) q_results = q.execute() report = TryChooserEndtoEndTimesReport(starttime, endtime, branch_name) for r in q_results: params = dict((str(k), v) for (k, v) in dict(r).items()) br = BuildRequest(**params) report.add_build_request(br) return report
def GetBuilderDetailsReport(builder_name=None, starttime=None, endtime=None): """Get the builder details report based on statusdb for a builder in the speficied time interval. Input: builder_name - builder name starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours or current time (if starttime is not specified either) Output: BuilderDetailsReport """ starttime, endtime = get_time_interval(starttime, endtime) starttime_date = datetime.fromtimestamp(starttime) endtime_date = datetime.fromtimestamp(endtime) report = BuilderDetailsReport(starttime, endtime, name=builder_name) q = BuildsQuery(builder_name=builder_name, get_builder_name=True, starttime=starttime_date, endtime=endtime_date) q_results = q.execute() for r in q_results: params = dict((str(k), v) for (k, v) in dict(r).items()) build = Build(**params) report.add(build) return report
def GetBuildersReport(starttime=None, endtime=None, branch_name='mozilla-central', platform=None, build_type=None, job_type=None, detail_level='builder'): """Get the average time per builder report for the speficied time interval and branch. Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours or current time (if starttime is not specified either) branch_name - branch name, default vaue is 'mozilla-central' Output: BuildersReport """ platform = platform or [] build_type = build_type or [] job_type = job_type or [] starttime, endtime = get_time_interval(starttime, endtime) detail_level_no = BUILDERS_DETAIL_LEVELS.index(detail_level) + 1 q = BuildersQuery(starttime, endtime, branch_name) q_results = q.execute() report = BuildersReport(starttime, endtime, branch_name, detail_level=detail_level_no) report.set_filters(dict(platform=platform, build_type=build_type, job_type=job_type)) for r in q_results: params = dict((str(k), v) for (k, v) in dict(r).items()) br = BuildRequest(**params) report.add(br) return report
def GetPushes(starttime=None, endtime=None, int_size=0, branches=None): """Get pushes and statistics. Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours or current time (if starttime is not specified either) int_size - break down results per interval (in seconds), if specified branches - filter by list of branches, if not spefified fetches all branches Output: pushes report """ starttime, endtime = get_time_interval(starttime, endtime) q = PushesQuery(starttime, endtime, branches) q_results = q.execute() report = PushesReport(starttime, endtime, int_size=int_size, branches=branches) for r in q_results: branch_name = get_branch_name(r['branch']) stime = float(r['when_timestamp']) revision = r['revision'] push = Push(stime, branch_name, revision) report.add(push) return report
def GetBuilderTypeReport(starttime=None, endtime=None, buildername=None): """Get the average time per builder report for one builder for the speficied time interval. The builder is specified by its buildername. Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours or current time (if starttime is not specified either) buildername - buildername Output: BuilderTypeReport """ starttime, endtime = get_time_interval(starttime, endtime) q = BuildersTypeQuery(starttime, endtime, buildername) q_results = q.execute() report = BuilderTypeReport(buildername=buildername, starttime=starttime, endtime=endtime) for r in q_results: params = dict((str(k), v) for (k, v) in dict(r).items()) br = BuildRequest(**params) report.add(br) return report
def GetSlaveDetailsReport(slave_id=None, starttime=None, endtime=None, int_size=0, last_int_size=0): """Get the slave details report for a slave in the speficied time interval. Input: slave_id - slave id starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours or current time (if starttime is not specified either) int_size - break down results per interval (in seconds), if specified last_int_size - the length in seconds for the last time interval for which to compute fail and busy/idle percentage. Output: SlaveDetailsReport """ starttime, endtime = get_time_interval(starttime, endtime) starttime_date = datetime.fromtimestamp(starttime) endtime_date = datetime.fromtimestamp(endtime) report = SlaveDetailsReport(starttime, endtime, slave_id, int_size=int_size, last_int_size=last_int_size) q = BuildsQuery(slave_id=slave_id, get_builder_name=True, starttime=starttime_date, endtime=endtime_date) q_results = q.execute() for r in q_results: params = dict((str(k), v) for (k, v) in dict(r).items()) build = Build(**params) report.add(build) if not report.name: report.name = build.slave_name return report
def GetEndtoEndTimes(starttime=None, endtime=None, branch_name='mozilla-central'): """Get end to end times report for the speficied time interval and branch. Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours or current time (if starttime is not specified either) branch_name - branch name, default vaue is 'mozilla-central' Output: EndtoEndTimesReport """ starttime, endtime = get_time_interval(starttime, endtime) report = EndtoEndTimesReport(starttime, endtime, branch_name) build_requests = GetBuildRequests(branch_name=branch_name, starttime=starttime, endtime=endtime, changeid_all=True) for key in build_requests: report.add_build_request(build_requests[key]) changes = GetChanges(branch_name=branch_name, starttime=starttime, endtime=endtime, pending_only=True) report.parse_incomplete(changes) return report
def GetSlavesReport(starttime=None, endtime=None, int_size=0, last_int_size=0): """Get the slaves report for the speficied time interval. Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours or current time (if starttime is not specified either) last_int_size - the length in seconds for the last time interval for which to compute fail and busy/idle percentage. Output: SlavesReport """ starttime, endtime = get_time_interval(starttime, endtime) starttime_date = datetime.fromtimestamp(starttime) endtime_date = datetime.fromtimestamp(endtime) report = SlavesReport(starttime, endtime, int_size=int_size, last_int_size=last_int_size) q = BuildsQuery(starttime=starttime_date, endtime=endtime_date) q_results = q.execute() for r in q_results: params = dict((str(k), v) for (k, v) in dict(r).items()) build = Build(**params) report.add(build) return report
def to_python(self, value_dict, state): starttime, endtime = value_dict['starttime'], value_dict['endtime'] starttime, endtime = get_time_interval(starttime, endtime) value_dict['starttime'] = starttime value_dict['endtime'] = endtime return value_dict
def to_python(self, value_dict, state): starttime, endtime = value_dict["starttime"], value_dict["endtime"] starttime, endtime = get_time_interval(starttime, endtime) value_dict["starttime"] = starttime value_dict["endtime"] = endtime return value_dict
def GetWaitTimes(pool='buildpool', mpb=15, starttime=None, endtime=None, int_size=0, maxb=0): """Get wait times and statistics for buildpool. Input: pool - name of the pool (e.g. buildpool, or trybuildpool) mpb - minutes per block, length of wait time block in minutes starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours or current time (if starttime is not specified either) int_size - break down results per interval (in seconds), if specified maxb - maximum block size; for wait times larger than maxb, group into the largest block Output: wait times report """ starttime, endtime = get_time_interval(starttime, endtime) q = WaitTimesQuery(starttime, endtime, pool) q_results = q.execute() report = WaitTimesReport(pool, starttime, endtime, mpb=mpb, maxb=maxb, int_size=int_size, masters=get_masters_for_pool(pool)) for r in q_results: buildername = r['buildername'] # start time is changes.when_timestamp, or buildrequests.submitted_at # if build has no changes stime = r['when_timestamp'] or r['submitted_at'] etime = r['start_time'] platform = get_platform(buildername) has_no_changes = not bool(r['when_timestamp']) wt = WaitTime(stime, etime, platform, buildername=buildername, has_no_changes=has_no_changes) report.add(wt) return report
def GetTestRuns(starttime=None, endtime=None, int_size=0, category=None, platform=None, group=False, btype=None): """Get test run metrics for a given interval Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours current time (if starttime is not specified either) int_size - break down results per interval (in seconds), if specified category - filter by list of builder categories, if not specified fetches all categories platform - filter by platform, if not specified, fetch all platforms group - group results together by test suite if specified, otherwise fetch all builders uniquely btype - filter by build type (opt/debug/etc), if not specified, return all Output: pushes report """ starttime, endtime = get_time_interval(starttime, endtime) q = TestRunsQuery(starttime, endtime, category=category) q_results = q.execute() report = TestRunsReport(starttime, endtime, category=category, platform=platform, group=group, btype=btype) filtered_results = [] #filter by platform, build type if platform and btype: filtered_results = filter(lambda x: platform == get_platform(x['name']) and btype == get_build_type(x['name']), q_results) elif platform and not btype: filtered_results = filter(lambda x: platform == get_platform(x['name']), q_results) elif btype and not platform: filtered_results = filter(lambda x: btype == get_build_type(x['name']), q_results) else: filtered_results = list(q_results) if group: for test in test_suites: grouped_results = filter(lambda p: p['name'].find(test)>-1, filtered_results) if grouped_results: average_total = reduce(lambda x, y: x + y, map(lambda x: x['total'], grouped_results)) / len(grouped_results) average_test = reduce(lambda x, y: x + y, map(lambda x: x['test'], grouped_results)) / len(grouped_results) row = { 'total': int(average_total), 'test' : int(average_test), 'ratio': average_test/average_total, 'platform': platform, } report.add(test, row) else: for r in filtered_results: this_row = { 'total':r['total'], 'test':r['test'], 'ratio':r['test']/r['total'], 'platform': get_platform(r['name']), } report.add(r['name'], this_row) return report
def GetIdleJobsReport(starttime=None, endtime=None, int_size=0): """Get test metrics for idlejobs jobs in a given time interval Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours current time (if starttime is not specified either) int_size - break down results per interval (in seconds), if specified Output: idlejobs report """ starttime, endtime = get_time_interval(starttime, endtime) q = IdleJobsQuery(starttime, endtime) q_results = q.execute() report = IdleJobsReport(starttime, endtime, int_size) for r in q_results: this_row = { 'starttime': GetTimeStamp(r['starttime']), 'endtime' : GetTimeStamp(r['endtime']), } report.add(r['name'], this_row) return report
def GetBuildersReport(starttime=None, endtime=None, branch_name='mozilla-central', platform=None, build_type=None, job_type=None, detail_level='builder'): """Get the average time per builder report for the speficied time interval and branch. Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours or current time (if starttime is not specified either) branch_name - branch name, default vaue is 'mozilla-central' Output: BuildersReport """ platform = platform or [] build_type = build_type or [] job_type = job_type or [] starttime, endtime = get_time_interval(starttime, endtime) detail_level_no = BUILDERS_DETAIL_LEVELS.index(detail_level) + 1 q = BuildersQuery(starttime, endtime, branch_name) q_results = q.execute() report = BuildersReport(starttime, endtime, branch_name, detail_level=detail_level_no) report.set_filters( dict(platform=platform, build_type=build_type, job_type=job_type)) for r in q_results: params = dict((str(k), v) for (k, v) in dict(r).items()) br = BuildRequest(**params) report.add(br) return report
def GetWaitTimes(pool='buildpool', mpb=15, starttime=None, endtime=None, int_size=0, maxb=0): """Get wait times and statistics for buildpool. Input: pool - name of the pool (e.g. buildpool, or trybuildpool) mpb - minutes per block, length of wait time block in minutes starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours or current time (if starttime is not specified either) int_size - break down results per interval (in seconds), if specified maxb - maximum block size; for wait times larger than maxb, group into the largest block Output: wait times report """ starttime, endtime = get_time_interval(starttime, endtime) q = WaitTimesQuery(starttime, endtime, pool) q_results = q.execute() report = WaitTimesReport(pool, starttime, endtime, mpb=mpb, maxb=maxb, int_size = int_size, masters=get_masters_for_pool(pool)) for r in q_results: buildername = r['buildername'] # start time is changes.when_timestamp, or buildrequests.submitted_at # if build has no changes stime = r['when_timestamp'] or r['submitted_at'] etime = r['start_time'] platform = get_platform(buildername) has_no_changes = not bool(r['when_timestamp']) wt = WaitTime(stime, etime, platform, buildername=buildername, has_no_changes=has_no_changes) report.add(wt) return report
def GetTestRuns(starttime=None, endtime=None, int_size=0, category=None, platform=None, group=False, btype=None): """Get test run metrics for a given interval Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours current time (if starttime is not specified either) int_size - break down results per interval (in seconds), if specified category - filter by list of builder categories, if not specified fetches all categories platform - filter by platform, if not specified, fetch all platforms group - group results together by test suite if specified, otherwise fetch all builders uniquely btype - filter by build type (opt/debug/etc), if not specified, return all Output: pushes report """ starttime, endtime = get_time_interval(starttime, endtime) q = TestRunsQuery(starttime, endtime, category=category) q_results = q.execute() report = TestRunsReport(starttime, endtime, category=category, platform=platform, group=group, btype=btype) filtered_results = [] #filter by platform, build type if platform and btype: filtered_results = filter( lambda x: platform == get_platform(x['name']) and btype == get_build_type(x['name']), q_results) elif platform and not btype: filtered_results = filter( lambda x: platform == get_platform(x['name']), q_results) elif btype and not platform: filtered_results = filter(lambda x: btype == get_build_type(x['name']), q_results) else: filtered_results = list(q_results) if group: for test in test_suites: grouped_results = filter(lambda p: p['name'].find(test) > -1, filtered_results) if grouped_results: average_total = reduce( lambda x, y: x + y, map(lambda x: x['total'], grouped_results)) / len(grouped_results) average_test = reduce( lambda x, y: x + y, map(lambda x: x['test'], grouped_results)) / len(grouped_results) row = { 'total': int(average_total), 'test': int(average_test), 'ratio': average_test / average_total, 'platform': platform, } report.add(test, row) else: for r in filtered_results: this_row = { 'total': r['total'], 'test': r['test'], 'ratio': r['test'] / r['total'], 'platform': get_platform(r['name']), } report.add(r['name'], this_row) return report