def create_aggregates(self, filterPath, channel, majorVersion, byDateType,
payload, buildId, revision):
# Aggregate histograms
for name, values in payload.get('histograms', {}).iteritems():
filePath = (channel, majorVersion, name, byDateType)
# Find cache set for filePath
cacheSet = self.cache.setdefault(filePath, {})
# Find aggregator for filter path
aggregator = cacheSet.get(filterPath, None)
if aggregator == None:
aggregator = HistogramAggregator()
cacheSet[filterPath] = aggregator
# Aggregate values
aggregator.merge(values + [1], buildId, revision)
# Aggregate simple measurements
for name, value in payload.get('simpleMeasurements', {}).iteritems():
# Handle cases where the value is a dictionary of simple measures
if type(value) == dict:
for subName, subValue in value.iteritems():
self.aggregate_simple_measure(channel, majorVersion,
filterPath,
name + "_" + str(subName),
byDateType, subValue)
else:
self.aggregate_simple_measure(channel, majorVersion,
filterPath, name, byDateType,
value)
def create_aggregates(self, filterPath, channel, majorVersion, byDateType, payload, buildId, revision):
# Aggregate histograms
for name, values in payload.get('histograms', {}).iteritems():
filePath = (channel, majorVersion, name, byDateType)
# Find cache set for filePath
cacheSet = self.cache.setdefault(filePath, {})
# Find aggregator for filter path
aggregator = cacheSet.get(filterPath, None)
if aggregator == None:
aggregator = HistogramAggregator()
cacheSet[filterPath] = aggregator
# Aggregate values
aggregator.merge(values + [1], buildId, revision)
# Aggregate simple measurements
for name, value in payload.get('simpleMeasurements', {}).iteritems():
# Handle cases where the value is a dictionary of simple measures
if type(value) == dict:
for subName, subValue in value.iteritems():
self.aggregate_simple_measure(channel, majorVersion, filterPath,
name + "_" + str(subName),
byDateType, subValue)
else:
self.aggregate_simple_measure(channel, majorVersion, filterPath,
name, byDateType, value)
def merge_blob(self, filePath, blob):
existing_blob = self.cache.setdefault(filePath, {})
for filterPath, dump in blob.iteritems():
aggregator = existing_blob.get(filterPath, None)
if aggregator is None:
aggregator = HistogramAggregator(**dump)
existing_blob[filterPath] = aggregator
else:
aggregator.merge(**dump)
def merge_blob(self, filePath, blob):
existing_blob = self.cache.setdefault(filePath, {})
for filterPath, dump in blob.iteritems():
aggregator = existing_blob.get(filterPath, None)
if aggregator is None:
aggregator = HistogramAggregator(**dump)
existing_blob[filterPath] = aggregator
else:
aggregator.merge(**dump)
def aggregate_simple_measure(self, channel, majorVersion, filterPath, name,
byDateType, value):
# Sanity check value
if type(value) not in (int, long, float):
log("%s is not a value type for simpleMeasurements \"%s\"",
type(value), name)
return
bucket = simple_measures_buckets[1]
values = [0] * (len(bucket) + 6)
for i in reversed(range(0, len(bucket))):
if value >= bucket[i]:
values[i] = 1
break
log_val = math.log(math.fabs(value) + 1)
values[-6] = value # sum
values[-5] = log_val # log_sum
values[-4] = log_val * log_val # log_sum_squares
values[-3] = 0 # sum_squares_lo
values[-2] = 0 # sum_squares_hi
values[-1] = 1 # count
filePath = (channel, majorVersion, "SIMPLE_MEASURES_" + name.upper(),
byDateType)
# Find cache set for filePath
cacheSet = self.cache.setdefault(filePath, {})
# Find aggregator for filter path
aggregator = cacheSet.get(filterPath, None)
if aggregator == None:
aggregator = HistogramAggregator()
cacheSet[filterPath] = aggregator
# Aggregate values
aggregator.merge(values, "0", "simple-measures-hack")
def aggregate_simple_measure(self, channel, majorVersion, filterPath, name,
byDateType, value):
# Sanity check value
if type(value) not in (int, long, float):
log("%s is not a value type for simpleMeasurements \"%s\"",
type(value), name)
return
bucket = simple_measures_buckets[1]
values = [0] * (len(bucket) + 6)
for i in reversed(range(0, len(bucket))):
if value >= bucket[i]:
values[i] = 1
break
log_val = math.log(math.fabs(value) + 1)
values[-6] = value # sum
values[-5] = log_val # log_sum
values[-4] = log_val * log_val # log_sum_squares
values[-3] = 0 # sum_squares_lo
values[-2] = 0 # sum_squares_hi
values[-1] = 1 # count
filePath = (channel, majorVersion, "SIMPLE_MEASURES_" + name.upper(), byDateType)
# Find cache set for filePath
cacheSet = self.cache.setdefault(filePath, {})
# Find aggregator for filter path
aggregator = cacheSet.get(filterPath, None)
if aggregator == None:
aggregator = HistogramAggregator()
cacheSet[filterPath] = aggregator
# Aggregate values
aggregator.merge(values, "0", "simple-measures-hack")