Example #1
0
    def create_aggregates(self, filterPath, channel, majorVersion, byDateType, payload, buildId, revision):
        # Aggregate histograms
        for name, values in payload.get('histograms', {}).iteritems():
            filePath = (channel, majorVersion, name, byDateType)

            # Find cache set for filePath
            cacheSet = self.cache.setdefault(filePath, {})
            # Find aggregator for filter path
            aggregator = cacheSet.get(filterPath, None)
            if aggregator == None:
                aggregator = HistogramAggregator()
                cacheSet[filterPath] = aggregator

            # Aggregate values
            aggregator.merge(values + [1], buildId, revision)

        # Aggregate simple measurements
        for name, value in payload.get('simpleMeasurements', {}).iteritems():
            # Handle cases where the value is a dictionary of simple measures
            if type(value) == dict:
                for subName, subValue in value.iteritems():
                    self.aggregate_simple_measure(channel, majorVersion, filterPath,
                                                  name + "_" + str(subName),
                                                  byDateType, subValue)
            else:
                self.aggregate_simple_measure(channel, majorVersion, filterPath,
                                              name, byDateType, value)
 def merge_blob(self, filePath, blob):
     existing_blob = self.cache.setdefault(filePath, {})
     for filterPath, dump in blob.iteritems():
         aggregator = existing_blob.get(filterPath, None)
         if aggregator is None:
             aggregator = HistogramAggregator(**dump)
             existing_blob[filterPath] = aggregator
         else:
             aggregator.merge(**dump)
Example #3
0
    def aggregate_simple_measure(self, channel, majorVersion, filterPath, name,
                                 byDateType, value):
        # Sanity check value
        if type(value) not in (int, long, float):
            log("%s is not a value type for simpleMeasurements \"%s\"",
                type(value), name)
            return

        bucket = simple_measures_buckets[1]
        values = [0] * (len(bucket) + 6)
        for i in reversed(range(0, len(bucket))):
            if value >= bucket[i]:
                values[i] = 1
                break

        log_val = math.log(math.fabs(value) + 1)
        values[-6] = value                # sum
        values[-5] = log_val              # log_sum
        values[-4] = log_val * log_val    # log_sum_squares
        values[-3] = 0                    # sum_squares_lo
        values[-2] = 0                    # sum_squares_hi
        values[-1] = 1                    # count


        filePath = (channel, majorVersion, "SIMPLE_MEASURES_" + name.upper(), byDateType)

        # Find cache set for filePath
        cacheSet = self.cache.setdefault(filePath, {})
        # Find aggregator for filter path
        aggregator = cacheSet.get(filterPath, None)
        if aggregator == None:
            aggregator = HistogramAggregator()
            cacheSet[filterPath] = aggregator

        # Aggregate values
        aggregator.merge(values, "0", "simple-measures-hack")