def process_message(message):
(dimensions, metric_name, region, tenant_id, time_stamp, value,
value_meta) = parse_measurement_message(message)
tags = dimensions
tags[u'_tenant_id'] = tenant_id
tags[u'_region'] = region
if not value_meta:
value_meta_str = u'"{}"'
else:
value_meta_str = line_utils.escape_value(json.dumps(value_meta, ensure_ascii=False))
key_values = [line_utils.escape_tag(metric_name)]
# tags should be sorted client-side to take load off server
for key in sorted(tags.keys()):
key_tag = line_utils.escape_tag(key)
value_tag = line_utils.escape_tag(tags[key])
key_values.append(key_tag + u'=' + value_tag)
key_values = u','.join(key_values)
value_field = u'value={}'.format(value)
value_meta_field = u'value_meta=' + value_meta_str
data = key_values + u' ' + value_field + u',' + \
value_meta_field + u' ' + str(int(time_stamp))
LOG.debug(data)
return data
def test_parse_measurement_message(self):
message = Mock()
message.value.return_value = """{
"metric": {
"name": "metric_name",
"timestamp": "metric_timestamp",
"value": "0.0",
"value_meta": {
},
"dimensions": {}
},
"meta": {
"region": "meta_region",
"tenantId": "meta_tenantId"
}
}"""
data = utils.parse_measurement_message(message)
self.assertEqual(data[0], {})
self.assertEqual(data[1], 'metric_name')
self.assertEqual(data[2], 'meta_region')
self.assertEqual(data[3], 'meta_tenantId')
self.assertEqual(data[4], 'metric_timestamp')
self.assertEqual(data[5], 0.0)
self.assertEqual(data[6], {})
def process_message(self, message):
(dimensions, metric_name, region, tenant_id, time_stamp, value,
value_meta) = parse_measurement_message(message)
tags = dimensions
tags[u'_tenant_id'] = tenant_id
tags[u'_region'] = region
if not value_meta:
value_meta_str = u'"{}"'
else:
value_meta_str = line_utils.escape_value(
json.dumps(value_meta, ensure_ascii=False))
key_values = [line_utils.escape_tag(metric_name)]
# tags should be sorted client-side to take load off server
for key in sorted(tags.keys()):
key_tag = line_utils.escape_tag(key)
value_tag = line_utils.escape_tag(tags[key])
key_values.append(key_tag + u'=' + value_tag)
key_values = u','.join(key_values)
value_field = u'value={}'.format(value)
value_meta_field = u'value_meta=' + value_meta_str
data = key_values + u' ' + value_field + u',' + value_meta_field + u' ' + str(
int(time_stamp))
LOG.debug(data)
return data
def process_message(self, message):
(dimensions, metric_name, region, tenant_id, time_stamp, value,
value_meta) = parse_measurement_message(message)
tags = dimensions
tags['_tenant_id'] = tenant_id.encode('utf8')
tags['_region'] = region.encode('utf8')
ts = time_stamp / 1000.0
data = {
"measurement":
metric_name.encode('utf8'),
"time":
datetime.fromtimestamp(
ts, tz=pytz.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
"fields": {
"value":
value,
"value_meta":
json.dumps(value_meta, ensure_ascii=False).encode('utf8')
},
"tags":
tags
}
LOG.debug(data)
return data
def test_parse_measurement_message(self):
message = Mock()
message.message.value = """{
"metric": {
"name": "metric_name",
"timestamp": "metric_timestamp",
"value": "0.0",
"value_meta": {
},
"dimensions": {}
},
"meta": {
"region": "meta_region",
"tenantId": "meta_tenantId"
}
}"""
data = utils.parse_measurement_message(message)
self.assertEqual(data[0], {})
self.assertEqual(data[1], 'metric_name')
self.assertEqual(data[2], 'meta_region')
self.assertEqual(data[3], 'meta_tenantId')
self.assertEqual(data[4], 'metric_timestamp')
self.assertEqual(data[5], 0.0)
self.assertEqual(data[6], {})
def process_message(self, message):
(dimensions, metric_name, region, tenant_id, time_stamp, value,
value_meta) = parse_measurement_message(message)
metric_hash, metric_map = create_metric_hash(metric_name, dimensions)
measurement = (tenant_id.encode('utf8'), region.encode('utf8'),
metric_hash, time_stamp, value,
json.dumps(value_meta,
ensure_ascii=False).encode('utf8'))
LOG.debug(measurement)
return MetricMeasurementInfo(tenant_id.encode('utf8'),
region.encode('utf8'), metric_hash,
metric_map, measurement)
def process_message(self, message):
(dimensions, metric_name, region, tenant_id, time_stamp, value,
value_meta) = parse_measurement_message(message)
tags = dimensions
tags['_tenant_id'] = tenant_id.encode('utf8')
tags['_region'] = region.encode('utf8')
ts = time_stamp / 1000.0
data = {"measurement": metric_name.encode('utf8'),
"time": datetime.fromtimestamp(ts, tz=pytz.utc).strftime(
'%Y-%m-%dT%H:%M:%S.%fZ'),
"fields": {
"value": value,
"value_meta": json.dumps(value_meta,
ensure_ascii=False).encode('utf8')
},
"tags": tags}
LOG.debug(data)
return data
def process_message(self, message):
(dimensions, metric_name, region, tenant_id, time_stamp, value,
value_meta) = parse_measurement_message(message)
metric_hash, metric_map = create_metric_hash(metric_name,
dimensions)
measurement = (tenant_id.encode('utf8'),
region.encode('utf8'),
metric_hash,
time_stamp,
value,
json.dumps(value_meta, ensure_ascii=False).encode(
'utf8'))
LOG.debug(measurement)
return MetricMeasurementInfo(
tenant_id.encode('utf8'),
region.encode('utf8'),
metric_hash,
metric_map,
measurement)
def process_message(self, message):
(dimensions, metric_name, region, tenant_id, time_stamp, value,
value_meta) = parse_measurement_message(message)
with self._lock:
dim_names = []
dim_list = []
for name in sorted(dimensions.iterkeys()):
dim_list.append('%s\t%s' % (name, dimensions[name]))
dim_names.append(name)
hash_string = '%s\0%s\0%s\0%s' % (region, tenant_id, metric_name,
'\0'.join(dim_list))
metric_id = hashlib.sha1(hash_string.encode('utf8')).hexdigest()
metric = Metric(id=metric_id,
region=region,
tenant_id=tenant_id,
name=metric_name,
dimension_list=dim_list,
dimension_names=dim_names,
time_stamp=time_stamp,
value=value,
value_meta=json.dumps(value_meta,
ensure_ascii=False))
id_bytes = bytearray.fromhex(metric.id)
if self._metric_id_cache.get(metric.id, None):
measurement_bound_stmt = self._measurement_update_stmt.bind(
(self._retention, metric.value, metric.value_meta,
id_bytes, metric.time_stamp))
self._metric_batch.add_measurement_query(
measurement_bound_stmt)
metric_update_bound_stmt = self._metric_update_stmt.bind(
(self._retention, id_bytes, metric.time_stamp,
metric.region, metric.tenant_id, metric.name,
metric.dimension_list, metric.dimension_names))
self._metric_batch.add_metric_query(metric_update_bound_stmt)
return metric
self._metric_id_cache[metric.id] = metric.id
metric_insert_bound_stmt = self._metric_insert_stmt.bind(
(self._retention, id_bytes, metric.time_stamp,
metric.time_stamp, metric.region, metric.tenant_id,
metric.name, metric.dimension_list, metric.dimension_names))
self._metric_batch.add_metric_query(metric_insert_bound_stmt)
for dim in metric.dimension_list:
(name, value) = dim.split('\t')
dim_key = self._get_dimnesion_key(metric.region,
metric.tenant_id, name,
value)
if not self._dimension_cache.get(dim_key, None):
dimension_bound_stmt = self._dimension_stmt.bind(
(metric.region, metric.tenant_id, name, value))
self._metric_batch.add_dimension_query(
dimension_bound_stmt)
self._dimension_cache[dim_key] = dim_key
metric_dim_key = self._get_metric_dimnesion_key(
metric.region, metric.tenant_id, metric.name, name, value)
if not self._metric_dimension_cache.get(metric_dim_key, None):
dimension_metric_bound_stmt = self._dimension_metric_stmt.bind(
(metric.region, metric.tenant_id, name, value,
metric.name))
self._metric_batch.add_dimension_metric_query(
dimension_metric_bound_stmt)
metric_dimension_bound_stmt = self._metric_dimension_stmt.bind(
(metric.region, metric.tenant_id, metric.name, name,
value))
self._metric_batch.add_metric_dimension_query(
metric_dimension_bound_stmt)
self._metric_dimension_cache[
metric_dim_key] = metric_dim_key
measurement_insert_bound_stmt = self._measurement_insert_stmt.bind(
(self._retention, metric.value, metric.value_meta,
metric.region, metric.tenant_id, metric.name,
metric.dimension_list, id_bytes, metric.time_stamp))
self._metric_batch.add_measurement_query(
measurement_insert_bound_stmt)
return metric
def process_message(self, message):
(dimensions, metric_name, region, tenant_id, time_stamp, value,
value_meta) = parse_measurement_message(message)
with self._lock:
dim_names = []
dim_list = []
for name in sorted(dimensions.iterkeys()):
dim_list.append('%s\t%s' % (name, dimensions[name]))
dim_names.append(name)
hash_string = '%s\0%s\0%s\0%s' % (region, tenant_id, metric_name, '\0'.join(dim_list))
metric_id = hashlib.sha1(hash_string.encode('utf8')).hexdigest()
metric = Metric(id=metric_id,
region=region,
tenant_id=tenant_id,
name=metric_name,
dimension_list=dim_list,
dimension_names=dim_names,
time_stamp=time_stamp,
value=value,
value_meta=json.dumps(value_meta, ensure_ascii=False))
id_bytes = bytearray.fromhex(metric.id)
if self._metric_id_cache.get(metric.id, None):
measurement_bound_stmt = self._measurement_update_stmt.bind((self._retention,
metric.value,
metric.value_meta,
id_bytes,
metric.time_stamp))
self._metric_batch.add_measurement_query(measurement_bound_stmt)
metric_update_bound_stmt = self._metric_update_stmt.bind((self._retention,
metric.time_stamp,
metric.region,
metric.tenant_id,
metric.name,
metric.dimension_list,
metric.dimension_names))
self._metric_batch.add_metric_query(metric_update_bound_stmt)
return metric
self._metric_id_cache[metric.id] = metric.id
metric_insert_bound_stmt = self._metric_insert_stmt.bind((self._retention,
id_bytes,
metric.time_stamp,
metric.time_stamp,
metric.region,
metric.tenant_id,
metric.name,
metric.dimension_list,
metric.dimension_names))
self._metric_batch.add_metric_query(metric_insert_bound_stmt)
for dim in metric.dimension_list:
(name, value) = dim.split('\t')
dim_key = self._get_dimnesion_key(metric.region, metric.tenant_id, name, value)
if not self._dimension_cache.get(dim_key, None):
dimension_bound_stmt = self._dimension_stmt.bind((metric.region,
metric.tenant_id,
name,
value))
self._metric_batch.add_dimension_query(dimension_bound_stmt)
self._dimension_cache[dim_key] = dim_key
metric_dim_key = self._get_metric_dimnesion_key(
metric.region, metric.tenant_id, metric.name, name, value)
if not self._metric_dimension_cache.get(metric_dim_key, None):
dimension_metric_bound_stmt = self._dimension_metric_stmt.bind(
(metric.region, metric.tenant_id, name, value, metric.name))
self._metric_batch.add_dimension_metric_query(dimension_metric_bound_stmt)
metric_dimension_bound_stmt = self._metric_dimension_stmt.bind(
(metric.region, metric.tenant_id, metric.name, name, value))
self._metric_batch.add_metric_dimension_query(metric_dimension_bound_stmt)
self._metric_dimension_cache[metric_dim_key] = metric_dim_key
measurement_insert_bound_stmt = self._measurement_insert_stmt.bind(
(self._retention,
metric.value,
metric.value_meta,
metric.region,
metric.tenant_id,
metric.name,
metric.dimension_list,
id_bytes,
metric.time_stamp))
self._metric_batch.add_measurement_query(measurement_insert_bound_stmt)
return metric
