Beispiel #1
0
    def wrapper_generator(*args, **kwargs):
        generator_obj = generator(*args, **kwargs)
        readonly_connection.connection().set_django_connection()
        try:
            first_value = generator_obj.next()
        finally:
            readonly_connection.connection().unset_django_connection()
        yield first_value

        while True:
            yield generator_obj.next()
Beispiel #2
0
 def get_num_groups(self, query, group_by):
     """
     Returns the number of distinct groups for the given query grouped by the
     fields in group_by.
     """
     sql, params = self._get_num_groups_sql(query, group_by)
     cursor = readonly_connection.connection().cursor()
     cursor.execute(sql, params)
     return self._cursor_rowcount(cursor)
Beispiel #3
0
 def get_num_groups(self, query, group_by):
     """
     Returns the number of distinct groups for the given query grouped by the
     fields in group_by.
     """
     sql, params = self._get_num_groups_sql(query, group_by)
     cursor = readonly_connection.connection().cursor()
     cursor.execute(sql, params)
     return self._cursor_rowcount(cursor)
Beispiel #4
0
 def _custom_select_query(self, query_set, selects):
     query_selects, where, params = query_set._get_sql_clause()
     if query_set._distinct:
         distinct = 'DISTINCT '
     else:
         distinct = ''
     sql_query = 'SELECT ' + distinct + ','.join(selects) + where
     cursor = readonly_connection.connection().cursor()
     cursor.execute(sql_query, params)
     return cursor.fetchall()
Beispiel #5
0
 def execute_group_query(self, query, group_by):
     """
     Performs the given query grouped by the fields in group_by with the
     given query's extra select fields added.  Returns a list of dicts, where
     each dict corresponds to single row and contains a key for each grouped
     field as well as all of the extra select fields.
     """
     sql, params = self._get_group_query_sql(query, group_by)
     cursor = readonly_connection.connection().cursor()
     cursor.execute(sql, params)
     field_names = self._get_column_names(cursor)
     row_dicts = [dict(zip(field_names, row)) for row in cursor.fetchall()]
     return row_dicts
Beispiel #6
0
    def _custom_select_query(self, query_set, selects):
        sql, params = query_set.query.as_sql()
        from_ = sql[sql.find(' FROM'):]

        if query_set.query.distinct:
            distinct = 'DISTINCT '
        else:
            distinct = ''

        sql_query = ('SELECT ' + distinct + ','.join(selects) + from_)
        cursor = readonly_connection.connection().cursor()
        cursor.execute(sql_query, params)
        return cursor.fetchall()
Beispiel #7
0
 def execute_group_query(self, query, group_by):
     """
     Performs the given query grouped by the fields in group_by with the
     given query's extra select fields added.  Returns a list of dicts, where
     each dict corresponds to single row and contains a key for each grouped
     field as well as all of the extra select fields.
     """
     sql, params = self._get_group_query_sql(query, group_by)
     cursor = readonly_connection.connection().cursor()
     cursor.execute(sql, params)
     field_names = self._get_column_names(cursor)
     row_dicts = [dict(zip(field_names, row)) for row in cursor.fetchall()]
     return row_dicts
Beispiel #8
0
    def _custom_select_query(self, query_set, selects):
        sql, params = query_set.query.as_sql()
        from_ = sql[sql.find(" FROM") :]

        if query_set.query.distinct:
            distinct = "DISTINCT "
        else:
            distinct = ""

        sql_query = "SELECT " + distinct + ",".join(selects) + from_
        cursor = readonly_connection.connection().cursor()
        cursor.execute(sql_query, params)
        return cursor.fetchall()
Beispiel #9
0
 def execute_group_query(self, query, group_by, extra_select_fields=[]):
     """
     Performs the given query grouped by the fields in group_by with the
     given extra select fields added.  extra_select_fields should be a dict
     mapping field alias to field SQL.  Usually, the extra fields will use
     group aggregation functions.  Returns a list of dicts, where each dict
     corresponds to single row and contains a key for each grouped field as
     well as all of the extra select fields.
     """
     sql, params = self._get_group_query_sql(query, group_by,
                                             extra_select_fields)
     cursor = readonly_connection.connection().cursor()
     cursor.execute(sql, params)
     field_names = self._get_column_names(cursor)
     row_dicts = [dict(zip(field_names, row)) for row in cursor.fetchall()]
     return row_dicts
Beispiel #10
0
 def _query_pivot_table(self, base_objects_by_id, pivot_table,
                        pivot_from_field, pivot_to_field):
     """
     @param id_list list of IDs of self.model objects to include
     @param pivot_table the name of the pivot table
     @param pivot_from_field a field name on pivot_table referencing
     self.model
     @param pivot_to_field a field name on pivot_table referencing the
     related model.
     @returns pivot list of IDs (base_id, related_id)
     """
     query = """
     SELECT %(from_field)s, %(to_field)s
     FROM %(table)s
     WHERE %(from_field)s IN (%(id_list)s)
     """ % dict(from_field=pivot_from_field,
                to_field=pivot_to_field,
                table=pivot_table,
                id_list=','.join(str(id_) for id_
                                 in base_objects_by_id.iterkeys()))
     cursor = readonly_connection.connection().cursor()
     cursor.execute(query)
     return cursor.fetchall()
Beispiel #11
0
def execute_query_with_param(query, param):
    cursor = readonly_connection.connection().cursor()
    cursor.execute(query, param)
    return cursor.fetchall()
Beispiel #12
0
def _create_metrics_plot_helper(plot_info, extra_text=None):
    """
    Create a metrics plot of the given plot data.
    plot_info: a MetricsPlot object.
    extra_text: text to show at the uppper-left of the graph

    TODO(showard): move some/all of this logic into methods on MetricsPlot
    """
    query = plot_info.query_dict['__main__']
    cursor = readonly_connection.connection().cursor()
    cursor.execute(query)

    if not cursor.rowcount:
        raise NoDataError('query did not return any data')
    rows = cursor.fetchall()
    # "transpose" rows, so columns[0] is all the values from the first column,
    # etc.
    columns = zip(*rows)

    plots = []
    labels = [str(label) for label in columns[0]]
    needs_resort = (cursor.description[0][0] == 'kernel')

    # Collect all the data for the plot
    col = 1
    while col < len(cursor.description):
        y = columns[col]
        label = cursor.description[col][0]
        col += 1
        if (col < len(cursor.description)
                and 'errors-' + label == cursor.description[col][0]):
            errors = columns[col]
            col += 1
        else:
            errors = None
        if needs_resort:
            y = _resort(labels, y)
            if errors:
                errors = _resort(labels, errors)

        x = [index for index, value in enumerate(y) if value is not None]
        if not x:
            raise NoDataError('No data for series ' + label)
        y = [y[i] for i in x]
        if errors:
            errors = [errors[i] for i in x]
        plots.append({'label': label, 'x': x, 'y': y, 'errors': errors})

    if needs_resort:
        labels = _resort(labels, labels)

    # Normalize the data if necessary
    normalize_to = plot_info.normalize_to
    if normalize_to == 'first' or normalize_to.startswith('x__'):
        if normalize_to != 'first':
            baseline = normalize_to[3:]
            try:
                baseline_index = labels.index(baseline)
            except ValueError:
                raise ValidationError(
                    {'Normalize': 'Invalid baseline %s' % baseline})
        for plot in plots:
            if normalize_to == 'first':
                plot_index = 0
            else:
                try:
                    plot_index = plot['x'].index(baseline_index)
                # if the value is not found, then we cannot normalize
                except ValueError:
                    raise ValidationError({
                        'Normalize': ('%s does not have a value for %s' %
                                      (plot['label'], normalize_to[3:]))
                    })
            base_values = [plot['y'][plot_index]] * len(plot['y'])
            if plot['errors']:
                base_errors = [plot['errors'][plot_index]] * len(
                    plot['errors'])
            plot['y'], plot['errors'] = _normalize(plot['y'], plot['errors'],
                                                   base_values, None
                                                   or base_errors)

    elif normalize_to.startswith('series__'):
        base_series = normalize_to[8:]
        _normalize_to_series(plots, base_series)

    # Call the appropriate function to draw the line or bar plot
    if plot_info.is_line:
        figure, area_data = _create_line(plots, labels, plot_info)
    else:
        figure, area_data = _create_bar(plots, labels, plot_info)

    # TODO(showard): extract these magic numbers to named constants
    if extra_text:
        text_y = .95 - .0075 * len(plots)
        figure.text(.1, text_y, extra_text, size='xx-small')

    return (figure, area_data)
Beispiel #13
0
 def wrapper_method(*args, **kwargs):
     readonly_connection.connection().set_django_connection()
     try:
         return method(*args, **kwargs)
     finally:
         readonly_connection.connection().unset_django_connection()
Beispiel #14
0
def _create_qual_histogram_helper(plot_info, extra_text=None):
    """\
    Create a machine qualification histogram of the given data.

    plot_info: a QualificationHistogram
    extra_text: text to show at the upper-left of the graph

    TODO(showard): move much or all of this into methods on
    QualificationHistogram
    """
    cursor = readonly_connection.connection().cursor()
    cursor.execute(plot_info.query)

    if not cursor.rowcount:
        raise NoDataError('query did not return any data')

    # Lists to store the plot data.
    # hist_data store tuples of (hostname, pass_rate) for machines that have
    #     pass rates between 0 and 100%, exclusive.
    # no_tests is a list of machines that have run none of the selected tests
    # no_pass is a list of machines with 0% pass rate
    # perfect is a list of machines with a 100% pass rate
    hist_data = []
    no_tests = []
    no_pass = []
    perfect = []

    # Construct the lists of data to plot
    for hostname, total, good in cursor.fetchall():
        if total == 0:
            no_tests.append(hostname)
            continue

        if good == 0:
            no_pass.append(hostname)
        elif good == total:
            perfect.append(hostname)
        else:
            percentage = 100.0 * good / total
            hist_data.append((hostname, percentage))

    interval = plot_info.interval
    bins = range(0, 100, interval)
    if bins[-1] != 100:
        bins.append(bins[-1] + interval)

    figure, height = _create_figure(_SINGLE_PLOT_HEIGHT)
    subplot = figure.add_subplot(1, 1, 1)

    # Plot the data and get all the bars plotted
    _,_, bars = subplot.hist([data[1] for data in hist_data],
                         bins=bins, align='left')
    bars += subplot.bar([-interval], len(no_pass),
                    width=interval, align='center')
    bars += subplot.bar([bins[-1]], len(perfect),
                    width=interval, align='center')
    bars += subplot.bar([-3 * interval], len(no_tests),
                    width=interval, align='center')

    buckets = [(bin, min(bin + interval, 100)) for bin in bins[:-1]]
    # set the x-axis range to cover all the normal bins plus the three "special"
    # ones - N/A (3 intervals left), 0% (1 interval left) ,and 100% (far right)
    subplot.set_xlim(-4 * interval, bins[-1] + interval)
    subplot.set_xticks([-3 * interval, -interval] + bins + [100 + interval])
    subplot.set_xticklabels(['N/A', '0%'] +
                        ['%d%% - <%d%%' % bucket for bucket in buckets] +
                        ['100%'], rotation=90, size='small')

    # Find the coordinates on the image for each bar
    x = []
    y = []
    for bar in bars:
        x.append(bar.get_x())
        y.append(bar.get_height())
    f = subplot.plot(x, y, linestyle='None')[0]
    upper_left_coords = f.get_transform().transform(zip(x, y))
    bottom_right_coords = f.get_transform().transform(
        [(x_val + interval, 0) for x_val in x])

    # Set the title attributes
    titles = ['%d%% - <%d%%: %d machines' % (bucket[0], bucket[1], y_val)
              for bucket, y_val in zip(buckets, y)]
    titles.append('0%%: %d machines' % len(no_pass))
    titles.append('100%%: %d machines' % len(perfect))
    titles.append('N/A: %d machines' % len(no_tests))

    # Get the hostnames for each bucket in the histogram
    names_list = [_get_hostnames_in_bucket(hist_data, bucket)
                  for bucket in buckets]
    names_list += [no_pass, perfect]

    if plot_info.filter_string:
        plot_info.filter_string += ' AND '

    # Construct the list of drilldown parameters to be passed when the user
    # clicks on the bar.
    params = []
    for names in names_list:
        if names:
            hostnames = ','.join(_quote(hostname) for hostname in names)
            hostname_filter = 'hostname IN (%s)' % hostnames
            full_filter = plot_info.filter_string + hostname_filter
            params.append({'type': 'normal',
                           'filterString': full_filter})
        else:
            params.append({'type': 'empty'})

    params.append({'type': 'not_applicable',
                   'hosts': '<br />'.join(no_tests)})

    area_data = [dict(left=ulx, top=height - uly,
                      right=brx, bottom=height - bry,
                      title=title, callback=plot_info.drilldown_callback,
                      callback_arguments=param_dict)
                 for (ulx, uly), (brx, bry), title, param_dict
                 in zip(upper_left_coords, bottom_right_coords, titles, params)]

    # TODO(showard): extract these magic numbers to named constants
    if extra_text:
        figure.text(.1, .95, extra_text, size='xx-small')

    return (figure, area_data)
Beispiel #15
0
def _create_metrics_plot_helper(plot_info, extra_text=None):
    """
    Create a metrics plot of the given plot data.
    plot_info: a MetricsPlot object.
    extra_text: text to show at the uppper-left of the graph

    TODO(showard): move some/all of this logic into methods on MetricsPlot
    """
    query = plot_info.query_dict['__main__']
    cursor = readonly_connection.connection().cursor()
    cursor.execute(query)

    if not cursor.rowcount:
        raise NoDataError('query did not return any data')
    rows = cursor.fetchall()
    # "transpose" rows, so columns[0] is all the values from the first column,
    # etc.
    columns = zip(*rows)

    plots = []
    labels = [str(label) for label in columns[0]]
    needs_resort = (cursor.description[0][0] == 'kernel')

    # Collect all the data for the plot
    col = 1
    while col < len(cursor.description):
        y = columns[col]
        label = cursor.description[col][0]
        col += 1
        if (col < len(cursor.description) and
            'errors-' + label == cursor.description[col][0]):
            errors = columns[col]
            col += 1
        else:
            errors = None
        if needs_resort:
            y = _resort(labels, y)
            if errors:
                errors = _resort(labels, errors)

        x = [index for index, value in enumerate(y) if value is not None]
        if not x:
            raise NoDataError('No data for series ' + label)
        y = [y[i] for i in x]
        if errors:
            errors = [errors[i] for i in x]
        plots.append({
            'label': label,
            'x': x,
            'y': y,
            'errors': errors
        })

    if needs_resort:
        labels = _resort(labels, labels)

    # Normalize the data if necessary
    normalize_to = plot_info.normalize_to
    if normalize_to == 'first' or normalize_to.startswith('x__'):
        if normalize_to != 'first':
            baseline = normalize_to[3:]
            try:
                baseline_index = labels.index(baseline)
            except ValueError:
                raise ValidationError({
                    'Normalize' : 'Invalid baseline %s' % baseline
                    })
        for plot in plots:
            if normalize_to == 'first':
                plot_index = 0
            else:
                try:
                    plot_index = plot['x'].index(baseline_index)
                # if the value is not found, then we cannot normalize
                except ValueError:
                    raise ValidationError({
                        'Normalize' : ('%s does not have a value for %s'
                                       % (plot['label'], normalize_to[3:]))
                        })
            base_values = [plot['y'][plot_index]] * len(plot['y'])
            if plot['errors']:
                base_errors = [plot['errors'][plot_index]] * len(plot['errors'])
            plot['y'], plot['errors'] = _normalize(plot['y'], plot['errors'],
                                                   base_values,
                                                   None or base_errors)

    elif normalize_to.startswith('series__'):
        base_series = normalize_to[8:]
        _normalize_to_series(plots, base_series)

    # Call the appropriate function to draw the line or bar plot
    if plot_info.is_line:
        figure, area_data = _create_line(plots, labels, plot_info)
    else:
        figure, area_data = _create_bar(plots, labels, plot_info)

    # TODO(showard): extract these magic numbers to named constants
    if extra_text:
        text_y = .95 - .0075 * len(plots)
        figure.text(.1, text_y, extra_text, size='xx-small')

    return (figure, area_data)
Beispiel #16
0
def _create_qual_histogram_helper(plot_info, extra_text=None):
    """\
    Create a machine qualification histogram of the given data.

    plot_info: a QualificationHistogram
    extra_text: text to show at the upper-left of the graph

    TODO(showard): move much or all of this into methods on
    QualificationHistogram
    """
    cursor = readonly_connection.connection().cursor()
    cursor.execute(plot_info.query)

    if not cursor.rowcount:
        raise NoDataError('query did not return any data')

    # Lists to store the plot data.
    # hist_data store tuples of (hostname, pass_rate) for machines that have
    #     pass rates between 0 and 100%, exclusive.
    # no_tests is a list of machines that have run none of the selected tests
    # no_pass is a list of machines with 0% pass rate
    # perfect is a list of machines with a 100% pass rate
    hist_data = []
    no_tests = []
    no_pass = []
    perfect = []

    # Construct the lists of data to plot
    for hostname, total, good in cursor.fetchall():
        if total == 0:
            no_tests.append(hostname)
            continue

        if good == 0:
            no_pass.append(hostname)
        elif good == total:
            perfect.append(hostname)
        else:
            percentage = 100.0 * good / total
            hist_data.append((hostname, percentage))

    interval = plot_info.interval
    bins = range(0, 100, interval)
    if bins[-1] != 100:
        bins.append(bins[-1] + interval)

    figure, height = _create_figure(_SINGLE_PLOT_HEIGHT)
    subplot = figure.add_subplot(1, 1, 1)

    # Plot the data and get all the bars plotted
    _, _, bars = subplot.hist([data[1] for data in hist_data],
                              bins=bins,
                              align='left')
    bars += subplot.bar([-interval],
                        len(no_pass),
                        width=interval,
                        align='center')
    bars += subplot.bar([bins[-1]],
                        len(perfect),
                        width=interval,
                        align='center')
    bars += subplot.bar([-3 * interval],
                        len(no_tests),
                        width=interval,
                        align='center')

    buckets = [(bin, min(bin + interval, 100)) for bin in bins[:-1]]
    # set the x-axis range to cover all the normal bins plus the three "special"
    # ones - N/A (3 intervals left), 0% (1 interval left) ,and 100% (far right)
    subplot.set_xlim(-4 * interval, bins[-1] + interval)
    subplot.set_xticks([-3 * interval, -interval] + bins + [100 + interval])
    subplot.set_xticklabels(['N/A', '0%'] +
                            ['%d%% - <%d%%' % bucket
                             for bucket in buckets] + ['100%'],
                            rotation=90,
                            size='small')

    # Find the coordinates on the image for each bar
    x = []
    y = []
    for bar in bars:
        x.append(bar.get_x())
        y.append(bar.get_height())
    f = subplot.plot(x, y, linestyle='None')[0]
    upper_left_coords = f.get_transform().transform(zip(x, y))
    bottom_right_coords = f.get_transform().transform([(x_val + interval, 0)
                                                       for x_val in x])

    # Set the title attributes
    titles = [
        '%d%% - <%d%%: %d machines' % (bucket[0], bucket[1], y_val)
        for bucket, y_val in zip(buckets, y)
    ]
    titles.append('0%%: %d machines' % len(no_pass))
    titles.append('100%%: %d machines' % len(perfect))
    titles.append('N/A: %d machines' % len(no_tests))

    # Get the hostnames for each bucket in the histogram
    names_list = [
        _get_hostnames_in_bucket(hist_data, bucket) for bucket in buckets
    ]
    names_list += [no_pass, perfect]

    if plot_info.filter_string:
        plot_info.filter_string += ' AND '

    # Construct the list of drilldown parameters to be passed when the user
    # clicks on the bar.
    params = []
    for names in names_list:
        if names:
            hostnames = ','.join(_quote(hostname) for hostname in names)
            hostname_filter = 'hostname IN (%s)' % hostnames
            full_filter = plot_info.filter_string + hostname_filter
            params.append({'type': 'normal', 'filterString': full_filter})
        else:
            params.append({'type': 'empty'})

    params.append({'type': 'not_applicable', 'hosts': '<br />'.join(no_tests)})

    area_data = [
        dict(left=ulx,
             top=height - uly,
             right=brx,
             bottom=height - bry,
             title=title,
             callback=plot_info.drilldown_callback,
             callback_arguments=param_dict)
        for (ulx, uly), (brx, bry), title, param_dict in zip(
            upper_left_coords, bottom_right_coords, titles, params)
    ]

    # TODO(showard): extract these magic numbers to named constants
    if extra_text:
        figure.text(.1, .95, extra_text, size='xx-small')

    return (figure, area_data)
Beispiel #17
0
def execute_query_with_param(query, param):
    cursor = readonly_connection.connection().cursor()
    cursor.execute(query, param)
    return cursor.fetchall()