Ejemplo n.º 1
0
def TryChooserGetEndtoEndTimes(starttime=None,
                               endtime=None,
                               branch_name='mozilla-central'):
    """Get end to end times report for the speficied time interval and branch.

    Input: starttime - start time (UNIX timestamp in seconds), if not 
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                starttime plus 24 hours or current time (if starttime is not 
                specified either)
           branch_name - branch name, default vaue is 'mozilla-central'
    Output: EndtoEndTimesReport
    """
    starttime, endtime = get_time_interval(starttime, endtime)

    q = BuildRequestsQuery(starttime=starttime,
                           endtime=endtime,
                           branch_name=branch_name)
    q_results = q.execute()

    report = TryChooserEndtoEndTimesReport(starttime, endtime, branch_name)
    for r in q_results:
        params = dict((str(k), v) for (k, v) in dict(r).items())
        br = BuildRequest(**params)
        report.add_build_request(br)

    return report
Ejemplo n.º 2
0
def GetBuilderDetailsReport(builder_name=None, starttime=None, endtime=None):
    """Get the builder details report based on statusdb for a builder in the 
    speficied time interval.

    Input: builder_name - builder name
           starttime - start time (UNIX timestamp in seconds), if not 
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                starttime plus 24 hours or current time (if starttime is not 
                specified either)
    Output: BuilderDetailsReport
    """
    starttime, endtime = get_time_interval(starttime, endtime)
    starttime_date = datetime.fromtimestamp(starttime)
    endtime_date = datetime.fromtimestamp(endtime)

    report = BuilderDetailsReport(starttime, endtime, name=builder_name)

    q = BuildsQuery(builder_name=builder_name, get_builder_name=True,
        starttime=starttime_date, endtime=endtime_date)
    q_results = q.execute()

    for r in q_results:
        params = dict((str(k), v) for (k, v) in dict(r).items())
        build = Build(**params)
        report.add(build)

    return report
Ejemplo n.º 3
0
def GetBuilderDetailsReport(builder_name=None, starttime=None, endtime=None):
    """Get the builder details report based on statusdb for a builder in the 
    speficied time interval.

    Input: builder_name - builder name
           starttime - start time (UNIX timestamp in seconds), if not 
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                starttime plus 24 hours or current time (if starttime is not 
                specified either)
    Output: BuilderDetailsReport
    """
    starttime, endtime = get_time_interval(starttime, endtime)
    starttime_date = datetime.fromtimestamp(starttime)
    endtime_date = datetime.fromtimestamp(endtime)

    report = BuilderDetailsReport(starttime, endtime, name=builder_name)

    q = BuildsQuery(builder_name=builder_name,
                    get_builder_name=True,
                    starttime=starttime_date,
                    endtime=endtime_date)
    q_results = q.execute()

    for r in q_results:
        params = dict((str(k), v) for (k, v) in dict(r).items())
        build = Build(**params)
        report.add(build)

    return report
Ejemplo n.º 4
0
def GetBuildersReport(starttime=None, endtime=None, 
    branch_name='mozilla-central', platform=None, build_type=None, 
    job_type=None, detail_level='builder'):
    """Get the average time per builder report for the speficied time interval 
    and branch.

    Input: starttime - start time (UNIX timestamp in seconds), if not 
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                starttime plus 24 hours or current time (if starttime is not 
                specified either)
           branch_name - branch name, default vaue is 'mozilla-central'
    Output: BuildersReport
    """
    platform = platform or []
    build_type = build_type or []
    job_type = job_type or []

    starttime, endtime = get_time_interval(starttime, endtime)
    detail_level_no = BUILDERS_DETAIL_LEVELS.index(detail_level) + 1

    q = BuildersQuery(starttime, endtime, branch_name)
    q_results = q.execute()

    report = BuildersReport(starttime, endtime, branch_name, 
        detail_level=detail_level_no)
    report.set_filters(dict(platform=platform, build_type=build_type, 
        job_type=job_type))

    for r in q_results:
        params = dict((str(k), v) for (k, v) in dict(r).items())
        br = BuildRequest(**params)
        report.add(br)

    return report
Ejemplo n.º 5
0
def GetPushes(starttime=None, endtime=None, int_size=0, branches=None):
    """Get pushes and statistics.

    Input: starttime - start time (UNIX timestamp in seconds), if not
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified,
                starttime plus 24 hours or current time (if starttime is not
                specified either)
           int_size - break down results per interval (in seconds), if specified
           branches - filter by list of branches, if not spefified fetches all
                branches
    Output: pushes report
    """
    starttime, endtime = get_time_interval(starttime, endtime)

    q = PushesQuery(starttime, endtime, branches)
    q_results = q.execute()

    report = PushesReport(starttime, endtime, int_size=int_size,
        branches=branches)
    for r in q_results:
        branch_name = get_branch_name(r['branch'])
        stime = float(r['when_timestamp'])
        revision = r['revision']

        push = Push(stime, branch_name, revision)
        report.add(push)

    return report
Ejemplo n.º 6
0
def GetBuilderTypeReport(starttime=None, endtime=None, buildername=None):
    """Get the average time per builder report for one builder for the 
    speficied time interval. The builder is specified by its buildername.

    Input: starttime - start time (UNIX timestamp in seconds), if not 
                    specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                    starttime plus 24 hours or current time (if starttime is 
                    not specified either)
           buildername - buildername
    Output: BuilderTypeReport
    """
    starttime, endtime = get_time_interval(starttime, endtime)

    q = BuildersTypeQuery(starttime, endtime, buildername)
    q_results = q.execute()

    report = BuilderTypeReport(buildername=buildername,
                               starttime=starttime,
                               endtime=endtime)
    for r in q_results:
        params = dict((str(k), v) for (k, v) in dict(r).items())
        br = BuildRequest(**params)
        report.add(br)

    return report
Ejemplo n.º 7
0
def GetSlaveDetailsReport(slave_id=None, starttime=None, endtime=None,
    int_size=0, last_int_size=0):
    """Get the slave details report for a slave in the speficied time interval.

    Input: slave_id - slave id
           starttime - start time (UNIX timestamp in seconds), if not 
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                starttime plus 24 hours or current time (if starttime is not 
                specified either)
           int_size - break down results per interval (in seconds), if specified
           last_int_size - the length in seconds for the last time interval 
                for which to compute fail and busy/idle percentage.
    Output: SlaveDetailsReport
    """
    starttime, endtime = get_time_interval(starttime, endtime)
    starttime_date = datetime.fromtimestamp(starttime)
    endtime_date = datetime.fromtimestamp(endtime)

    report = SlaveDetailsReport(starttime, endtime, slave_id, 
        int_size=int_size, last_int_size=last_int_size)

    q = BuildsQuery(slave_id=slave_id, get_builder_name=True,
        starttime=starttime_date, endtime=endtime_date)
    q_results = q.execute()

    for r in q_results:
        params = dict((str(k), v) for (k, v) in dict(r).items())
        build = Build(**params)
        report.add(build)
        if not report.name:
            report.name = build.slave_name

    return report
Ejemplo n.º 8
0
def GetEndtoEndTimes(starttime=None, endtime=None,
    branch_name='mozilla-central'):
    """Get end to end times report for the speficied time interval and branch.

    Input: starttime - start time (UNIX timestamp in seconds), if not 
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                starttime plus 24 hours or current time (if starttime is not 
                specified either)
           branch_name - branch name, default vaue is 'mozilla-central'
    Output: EndtoEndTimesReport
    """
    starttime, endtime = get_time_interval(starttime, endtime)

    report = EndtoEndTimesReport(starttime, endtime, branch_name)

    build_requests = GetBuildRequests(branch_name=branch_name, 
        starttime=starttime, endtime=endtime, changeid_all=True)
    for key in build_requests:
        report.add_build_request(build_requests[key])

    changes = GetChanges(branch_name=branch_name, starttime=starttime, 
        endtime=endtime, pending_only=True)
    report.parse_incomplete(changes)

    return report
Ejemplo n.º 9
0
def GetSlavesReport(starttime=None, endtime=None, int_size=0, last_int_size=0):
    """Get the slaves report for the speficied time interval.

    Input: starttime - start time (UNIX timestamp in seconds), if not 
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                starttime plus 24 hours or current time (if starttime is not 
                specified either)
           last_int_size - the length in seconds for the last time interval 
                for which to compute fail and busy/idle percentage.
    Output: SlavesReport
    """
    starttime, endtime = get_time_interval(starttime, endtime)
    starttime_date = datetime.fromtimestamp(starttime)
    endtime_date = datetime.fromtimestamp(endtime)

    report = SlavesReport(starttime,
                          endtime,
                          int_size=int_size,
                          last_int_size=last_int_size)

    q = BuildsQuery(starttime=starttime_date, endtime=endtime_date)
    q_results = q.execute()

    for r in q_results:
        params = dict((str(k), v) for (k, v) in dict(r).items())
        build = Build(**params)
        report.add(build)

    return report
def GetPushes(starttime=None, endtime=None, int_size=0, branches=None):
    """Get pushes and statistics.

    Input: starttime - start time (UNIX timestamp in seconds), if not
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified,
                starttime plus 24 hours or current time (if starttime is not
                specified either)
           int_size - break down results per interval (in seconds), if specified
           branches - filter by list of branches, if not spefified fetches all
                branches
    Output: pushes report
    """
    starttime, endtime = get_time_interval(starttime, endtime)

    q = PushesQuery(starttime, endtime, branches)
    q_results = q.execute()

    report = PushesReport(starttime,
                          endtime,
                          int_size=int_size,
                          branches=branches)
    for r in q_results:
        branch_name = get_branch_name(r['branch'])
        stime = float(r['when_timestamp'])
        revision = r['revision']

        push = Push(stime, branch_name, revision)
        report.add(push)

    return report
Ejemplo n.º 11
0
def GetBuilderTypeReport(starttime=None, endtime=None, buildername=None):
    """Get the average time per builder report for one builder for the 
    speficied time interval. The builder is specified by its buildername.

    Input: starttime - start time (UNIX timestamp in seconds), if not 
                    specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                    starttime plus 24 hours or current time (if starttime is 
                    not specified either)
           buildername - buildername
    Output: BuilderTypeReport
    """
    starttime, endtime = get_time_interval(starttime, endtime)

    q = BuildersTypeQuery(starttime, endtime, buildername)
    q_results = q.execute()

    report = BuilderTypeReport(buildername=buildername, starttime=starttime, 
        endtime=endtime)
    for r in q_results:
        params = dict((str(k), v) for (k, v) in dict(r).items())
        br = BuildRequest(**params)
        report.add(br)

    return report
Ejemplo n.º 12
0
def GetEndtoEndTimes(starttime=None,
                     endtime=None,
                     branch_name='mozilla-central'):
    """Get end to end times report for the speficied time interval and branch.

    Input: starttime - start time (UNIX timestamp in seconds), if not 
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                starttime plus 24 hours or current time (if starttime is not 
                specified either)
           branch_name - branch name, default vaue is 'mozilla-central'
    Output: EndtoEndTimesReport
    """
    starttime, endtime = get_time_interval(starttime, endtime)

    report = EndtoEndTimesReport(starttime, endtime, branch_name)

    build_requests = GetBuildRequests(branch_name=branch_name,
                                      starttime=starttime,
                                      endtime=endtime,
                                      changeid_all=True)
    for key in build_requests:
        report.add_build_request(build_requests[key])

    changes = GetChanges(branch_name=branch_name,
                         starttime=starttime,
                         endtime=endtime,
                         pending_only=True)
    report.parse_incomplete(changes)

    return report
Ejemplo n.º 13
0
def TryChooserGetEndtoEndTimes(starttime=None, endtime=None, 
    branch_name='mozilla-central'):
    """Get end to end times report for the speficied time interval and branch.

    Input: starttime - start time (UNIX timestamp in seconds), if not 
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                starttime plus 24 hours or current time (if starttime is not 
                specified either)
           branch_name - branch name, default vaue is 'mozilla-central'
    Output: EndtoEndTimesReport
    """
    starttime, endtime = get_time_interval(starttime, endtime)

    q = BuildRequestsQuery(starttime=starttime, endtime=endtime, 
            branch_name=branch_name)
    q_results = q.execute()

    report = TryChooserEndtoEndTimesReport(starttime, endtime, branch_name)
    for r in q_results:
        params = dict((str(k), v) for (k, v) in dict(r).items())
        br = BuildRequest(**params)
        report.add_build_request(br)

    return report
    def to_python(self, value_dict, state):
        starttime, endtime = value_dict['starttime'], value_dict['endtime']
        starttime, endtime = get_time_interval(starttime, endtime)

        value_dict['starttime'] = starttime
        value_dict['endtime'] = endtime

        return value_dict
Ejemplo n.º 15
0
    def to_python(self, value_dict, state):
        starttime, endtime = value_dict["starttime"], value_dict["endtime"]
        starttime, endtime = get_time_interval(starttime, endtime)

        value_dict["starttime"] = starttime
        value_dict["endtime"] = endtime

        return value_dict
def GetWaitTimes(pool='buildpool',
                 mpb=15,
                 starttime=None,
                 endtime=None,
                 int_size=0,
                 maxb=0):
    """Get wait times and statistics for buildpool.

    Input: pool - name of the pool (e.g. buildpool, or trybuildpool)
           mpb - minutes per block, length of wait time block in minutes
           starttime - start time (UNIX timestamp in seconds), if not 
                    specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                    starttime plus 24 hours or current time (if starttime is 
                    not specified either)
           int_size - break down results per interval (in seconds), if specified
           maxb - maximum block size; for wait times larger than maxb, group 
                    into the largest block
    Output: wait times report
    """
    starttime, endtime = get_time_interval(starttime, endtime)

    q = WaitTimesQuery(starttime, endtime, pool)
    q_results = q.execute()

    report = WaitTimesReport(pool,
                             starttime,
                             endtime,
                             mpb=mpb,
                             maxb=maxb,
                             int_size=int_size,
                             masters=get_masters_for_pool(pool))
    for r in q_results:
        buildername = r['buildername']
        # start time is changes.when_timestamp, or buildrequests.submitted_at
        # if build has no changes
        stime = r['when_timestamp'] or r['submitted_at']
        etime = r['start_time']

        platform = get_platform(buildername)
        has_no_changes = not bool(r['when_timestamp'])

        wt = WaitTime(stime,
                      etime,
                      platform,
                      buildername=buildername,
                      has_no_changes=has_no_changes)
        report.add(wt)

    return report
Ejemplo n.º 17
0
def GetTestRuns(starttime=None, endtime=None, int_size=0, category=None, platform=None, group=False, btype=None):
    """Get test run metrics for a given interval
    Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours
                     current time (if starttime is not specified either)
           int_size - break down results per interval (in seconds), if specified
           category - filter by list of builder categories, if not specified fetches all categories
           platform - filter by platform, if not specified, fetch all platforms
           group    - group results together by test suite if specified, otherwise fetch all builders uniquely
           btype    - filter by build type (opt/debug/etc), if not specified, return all
    Output: pushes report
    """
    starttime, endtime = get_time_interval(starttime, endtime)
    q = TestRunsQuery(starttime, endtime, category=category)
    q_results = q.execute()
    report = TestRunsReport(starttime, endtime, category=category, platform=platform, group=group, btype=btype)
    filtered_results = []
    #filter by platform, build type
    if platform and btype:
        filtered_results = filter(lambda x: platform == get_platform(x['name']) and btype == get_build_type(x['name']), q_results)
    elif platform and not btype:
        filtered_results = filter(lambda x: platform == get_platform(x['name']), q_results)
    elif btype and not platform:
        filtered_results = filter(lambda x: btype == get_build_type(x['name']), q_results)
    else:
        filtered_results = list(q_results)

    if group:
        for test in test_suites:
            grouped_results = filter(lambda p: p['name'].find(test)>-1, filtered_results)
            if grouped_results:
                average_total   = reduce(lambda x, y: x + y, map(lambda x: x['total'], grouped_results)) / len(grouped_results)
                average_test    = reduce(lambda x, y: x + y, map(lambda x: x['test'], grouped_results)) / len(grouped_results)
                row = { 'total': int(average_total),
                        'test' : int(average_test),
                        'ratio': average_test/average_total,
                        'platform': platform,
                      }
                report.add(test, row)
    else:
        for r in filtered_results:
            this_row = { 'total':r['total'],
                         'test':r['test'],
                         'ratio':r['test']/r['total'],
                         'platform': get_platform(r['name']),
                       }
            report.add(r['name'], this_row)

    return report
Ejemplo n.º 18
0
def GetIdleJobsReport(starttime=None, endtime=None, int_size=0):
    """Get test metrics for idlejobs jobs in a given time interval
    Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours
                     current time (if starttime is not specified either)
           int_size - break down results per interval (in seconds), if specified
    Output: idlejobs report
    """
    starttime, endtime = get_time_interval(starttime, endtime)
    q = IdleJobsQuery(starttime, endtime)
    q_results = q.execute()
    report = IdleJobsReport(starttime, endtime, int_size)

    for r in q_results:
            this_row = {
                         'starttime': GetTimeStamp(r['starttime']),
                         'endtime'  : GetTimeStamp(r['endtime']),
                       }
            report.add(r['name'], this_row)
    return report
Ejemplo n.º 19
0
def GetBuildersReport(starttime=None,
                      endtime=None,
                      branch_name='mozilla-central',
                      platform=None,
                      build_type=None,
                      job_type=None,
                      detail_level='builder'):
    """Get the average time per builder report for the speficied time interval 
    and branch.

    Input: starttime - start time (UNIX timestamp in seconds), if not 
                specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                starttime plus 24 hours or current time (if starttime is not 
                specified either)
           branch_name - branch name, default vaue is 'mozilla-central'
    Output: BuildersReport
    """
    platform = platform or []
    build_type = build_type or []
    job_type = job_type or []

    starttime, endtime = get_time_interval(starttime, endtime)
    detail_level_no = BUILDERS_DETAIL_LEVELS.index(detail_level) + 1

    q = BuildersQuery(starttime, endtime, branch_name)
    q_results = q.execute()

    report = BuildersReport(starttime,
                            endtime,
                            branch_name,
                            detail_level=detail_level_no)
    report.set_filters(
        dict(platform=platform, build_type=build_type, job_type=job_type))

    for r in q_results:
        params = dict((str(k), v) for (k, v) in dict(r).items())
        br = BuildRequest(**params)
        report.add(br)

    return report
Ejemplo n.º 20
0
def GetWaitTimes(pool='buildpool', mpb=15, starttime=None, endtime=None,
    int_size=0, maxb=0):
    """Get wait times and statistics for buildpool.

    Input: pool - name of the pool (e.g. buildpool, or trybuildpool)
           mpb - minutes per block, length of wait time block in minutes
           starttime - start time (UNIX timestamp in seconds), if not 
                    specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, 
                    starttime plus 24 hours or current time (if starttime is 
                    not specified either)
           int_size - break down results per interval (in seconds), if specified
           maxb - maximum block size; for wait times larger than maxb, group 
                    into the largest block
    Output: wait times report
    """
    starttime, endtime = get_time_interval(starttime, endtime)

    q = WaitTimesQuery(starttime, endtime, pool)
    q_results = q.execute()

    report = WaitTimesReport(pool, starttime, endtime, mpb=mpb, maxb=maxb, 
        int_size = int_size, masters=get_masters_for_pool(pool))
    for r in q_results:
        buildername = r['buildername']
        # start time is changes.when_timestamp, or buildrequests.submitted_at 
        # if build has no changes
        stime = r['when_timestamp'] or r['submitted_at']
        etime = r['start_time']

        platform = get_platform(buildername)
        has_no_changes = not bool(r['when_timestamp'])

        wt = WaitTime(stime, etime, platform, buildername=buildername,
                      has_no_changes=has_no_changes)
        report.add(wt)

    return report
def GetTestRuns(starttime=None,
                endtime=None,
                int_size=0,
                category=None,
                platform=None,
                group=False,
                btype=None):
    """Get test run metrics for a given interval
    Input: starttime - start time (UNIX timestamp in seconds), if not specified, endtime minus 24 hours
           endtime - end time (UNIX timestamp in seconds), if not specified, starttime plus 24 hours
                     current time (if starttime is not specified either)
           int_size - break down results per interval (in seconds), if specified
           category - filter by list of builder categories, if not specified fetches all categories
           platform - filter by platform, if not specified, fetch all platforms
           group    - group results together by test suite if specified, otherwise fetch all builders uniquely
           btype    - filter by build type (opt/debug/etc), if not specified, return all
    Output: pushes report
    """
    starttime, endtime = get_time_interval(starttime, endtime)
    q = TestRunsQuery(starttime, endtime, category=category)
    q_results = q.execute()
    report = TestRunsReport(starttime,
                            endtime,
                            category=category,
                            platform=platform,
                            group=group,
                            btype=btype)
    filtered_results = []
    #filter by platform, build type
    if platform and btype:
        filtered_results = filter(
            lambda x: platform == get_platform(x['name']) and btype ==
            get_build_type(x['name']), q_results)
    elif platform and not btype:
        filtered_results = filter(
            lambda x: platform == get_platform(x['name']), q_results)
    elif btype and not platform:
        filtered_results = filter(lambda x: btype == get_build_type(x['name']),
                                  q_results)
    else:
        filtered_results = list(q_results)

    if group:
        for test in test_suites:
            grouped_results = filter(lambda p: p['name'].find(test) > -1,
                                     filtered_results)
            if grouped_results:
                average_total = reduce(
                    lambda x, y: x + y,
                    map(lambda x: x['total'],
                        grouped_results)) / len(grouped_results)
                average_test = reduce(
                    lambda x, y: x + y,
                    map(lambda x: x['test'],
                        grouped_results)) / len(grouped_results)
                row = {
                    'total': int(average_total),
                    'test': int(average_test),
                    'ratio': average_test / average_total,
                    'platform': platform,
                }
                report.add(test, row)
    else:
        for r in filtered_results:
            this_row = {
                'total': r['total'],
                'test': r['test'],
                'ratio': r['test'] / r['total'],
                'platform': get_platform(r['name']),
            }
            report.add(r['name'], this_row)

    return report