def process_message(message): (dimensions, metric_name, region, tenant_id, time_stamp, value, value_meta) = parse_measurement_message(message) tags = dimensions tags[u'_tenant_id'] = tenant_id tags[u'_region'] = region if not value_meta: value_meta_str = u'"{}"' else: value_meta_str = line_utils.escape_value(json.dumps(value_meta, ensure_ascii=False)) key_values = [line_utils.escape_tag(metric_name)] # tags should be sorted client-side to take load off server for key in sorted(tags.keys()): key_tag = line_utils.escape_tag(key) value_tag = line_utils.escape_tag(tags[key]) key_values.append(key_tag + u'=' + value_tag) key_values = u','.join(key_values) value_field = u'value={}'.format(value) value_meta_field = u'value_meta=' + value_meta_str data = key_values + u' ' + value_field + u',' + \ value_meta_field + u' ' + str(int(time_stamp)) LOG.debug(data) return data
def test_parse_measurement_message(self): message = Mock() message.value.return_value = """{ "metric": { "name": "metric_name", "timestamp": "metric_timestamp", "value": "0.0", "value_meta": { }, "dimensions": {} }, "meta": { "region": "meta_region", "tenantId": "meta_tenantId" } }""" data = utils.parse_measurement_message(message) self.assertEqual(data[0], {}) self.assertEqual(data[1], 'metric_name') self.assertEqual(data[2], 'meta_region') self.assertEqual(data[3], 'meta_tenantId') self.assertEqual(data[4], 'metric_timestamp') self.assertEqual(data[5], 0.0) self.assertEqual(data[6], {})
def process_message(self, message): (dimensions, metric_name, region, tenant_id, time_stamp, value, value_meta) = parse_measurement_message(message) tags = dimensions tags[u'_tenant_id'] = tenant_id tags[u'_region'] = region if not value_meta: value_meta_str = u'"{}"' else: value_meta_str = line_utils.escape_value( json.dumps(value_meta, ensure_ascii=False)) key_values = [line_utils.escape_tag(metric_name)] # tags should be sorted client-side to take load off server for key in sorted(tags.keys()): key_tag = line_utils.escape_tag(key) value_tag = line_utils.escape_tag(tags[key]) key_values.append(key_tag + u'=' + value_tag) key_values = u','.join(key_values) value_field = u'value={}'.format(value) value_meta_field = u'value_meta=' + value_meta_str data = key_values + u' ' + value_field + u',' + value_meta_field + u' ' + str( int(time_stamp)) LOG.debug(data) return data
def process_message(self, message): (dimensions, metric_name, region, tenant_id, time_stamp, value, value_meta) = parse_measurement_message(message) tags = dimensions tags['_tenant_id'] = tenant_id.encode('utf8') tags['_region'] = region.encode('utf8') ts = time_stamp / 1000.0 data = { "measurement": metric_name.encode('utf8'), "time": datetime.fromtimestamp( ts, tz=pytz.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ'), "fields": { "value": value, "value_meta": json.dumps(value_meta, ensure_ascii=False).encode('utf8') }, "tags": tags } LOG.debug(data) return data
def test_parse_measurement_message(self): message = Mock() message.message.value = """{ "metric": { "name": "metric_name", "timestamp": "metric_timestamp", "value": "0.0", "value_meta": { }, "dimensions": {} }, "meta": { "region": "meta_region", "tenantId": "meta_tenantId" } }""" data = utils.parse_measurement_message(message) self.assertEqual(data[0], {}) self.assertEqual(data[1], 'metric_name') self.assertEqual(data[2], 'meta_region') self.assertEqual(data[3], 'meta_tenantId') self.assertEqual(data[4], 'metric_timestamp') self.assertEqual(data[5], 0.0) self.assertEqual(data[6], {})
def process_message(self, message): (dimensions, metric_name, region, tenant_id, time_stamp, value, value_meta) = parse_measurement_message(message) metric_hash, metric_map = create_metric_hash(metric_name, dimensions) measurement = (tenant_id.encode('utf8'), region.encode('utf8'), metric_hash, time_stamp, value, json.dumps(value_meta, ensure_ascii=False).encode('utf8')) LOG.debug(measurement) return MetricMeasurementInfo(tenant_id.encode('utf8'), region.encode('utf8'), metric_hash, metric_map, measurement)
def process_message(self, message): (dimensions, metric_name, region, tenant_id, time_stamp, value, value_meta) = parse_measurement_message(message) tags = dimensions tags['_tenant_id'] = tenant_id.encode('utf8') tags['_region'] = region.encode('utf8') ts = time_stamp / 1000.0 data = {"measurement": metric_name.encode('utf8'), "time": datetime.fromtimestamp(ts, tz=pytz.utc).strftime( '%Y-%m-%dT%H:%M:%S.%fZ'), "fields": { "value": value, "value_meta": json.dumps(value_meta, ensure_ascii=False).encode('utf8') }, "tags": tags} LOG.debug(data) return data
def process_message(self, message): (dimensions, metric_name, region, tenant_id, time_stamp, value, value_meta) = parse_measurement_message(message) metric_hash, metric_map = create_metric_hash(metric_name, dimensions) measurement = (tenant_id.encode('utf8'), region.encode('utf8'), metric_hash, time_stamp, value, json.dumps(value_meta, ensure_ascii=False).encode( 'utf8')) LOG.debug(measurement) return MetricMeasurementInfo( tenant_id.encode('utf8'), region.encode('utf8'), metric_hash, metric_map, measurement)
def process_message(self, message): (dimensions, metric_name, region, tenant_id, time_stamp, value, value_meta) = parse_measurement_message(message) with self._lock: dim_names = [] dim_list = [] for name in sorted(dimensions.iterkeys()): dim_list.append('%s\t%s' % (name, dimensions[name])) dim_names.append(name) hash_string = '%s\0%s\0%s\0%s' % (region, tenant_id, metric_name, '\0'.join(dim_list)) metric_id = hashlib.sha1(hash_string.encode('utf8')).hexdigest() metric = Metric(id=metric_id, region=region, tenant_id=tenant_id, name=metric_name, dimension_list=dim_list, dimension_names=dim_names, time_stamp=time_stamp, value=value, value_meta=json.dumps(value_meta, ensure_ascii=False)) id_bytes = bytearray.fromhex(metric.id) if self._metric_id_cache.get(metric.id, None): measurement_bound_stmt = self._measurement_update_stmt.bind( (self._retention, metric.value, metric.value_meta, id_bytes, metric.time_stamp)) self._metric_batch.add_measurement_query( measurement_bound_stmt) metric_update_bound_stmt = self._metric_update_stmt.bind( (self._retention, id_bytes, metric.time_stamp, metric.region, metric.tenant_id, metric.name, metric.dimension_list, metric.dimension_names)) self._metric_batch.add_metric_query(metric_update_bound_stmt) return metric self._metric_id_cache[metric.id] = metric.id metric_insert_bound_stmt = self._metric_insert_stmt.bind( (self._retention, id_bytes, metric.time_stamp, metric.time_stamp, metric.region, metric.tenant_id, metric.name, metric.dimension_list, metric.dimension_names)) self._metric_batch.add_metric_query(metric_insert_bound_stmt) for dim in metric.dimension_list: (name, value) = dim.split('\t') dim_key = self._get_dimnesion_key(metric.region, metric.tenant_id, name, value) if not self._dimension_cache.get(dim_key, None): dimension_bound_stmt = self._dimension_stmt.bind( (metric.region, metric.tenant_id, name, value)) self._metric_batch.add_dimension_query( dimension_bound_stmt) self._dimension_cache[dim_key] = dim_key metric_dim_key = self._get_metric_dimnesion_key( metric.region, metric.tenant_id, metric.name, name, value) if not self._metric_dimension_cache.get(metric_dim_key, None): dimension_metric_bound_stmt = self._dimension_metric_stmt.bind( (metric.region, metric.tenant_id, name, value, metric.name)) self._metric_batch.add_dimension_metric_query( dimension_metric_bound_stmt) metric_dimension_bound_stmt = self._metric_dimension_stmt.bind( (metric.region, metric.tenant_id, metric.name, name, value)) self._metric_batch.add_metric_dimension_query( metric_dimension_bound_stmt) self._metric_dimension_cache[ metric_dim_key] = metric_dim_key measurement_insert_bound_stmt = self._measurement_insert_stmt.bind( (self._retention, metric.value, metric.value_meta, metric.region, metric.tenant_id, metric.name, metric.dimension_list, id_bytes, metric.time_stamp)) self._metric_batch.add_measurement_query( measurement_insert_bound_stmt) return metric
def process_message(self, message): (dimensions, metric_name, region, tenant_id, time_stamp, value, value_meta) = parse_measurement_message(message) with self._lock: dim_names = [] dim_list = [] for name in sorted(dimensions.iterkeys()): dim_list.append('%s\t%s' % (name, dimensions[name])) dim_names.append(name) hash_string = '%s\0%s\0%s\0%s' % (region, tenant_id, metric_name, '\0'.join(dim_list)) metric_id = hashlib.sha1(hash_string.encode('utf8')).hexdigest() metric = Metric(id=metric_id, region=region, tenant_id=tenant_id, name=metric_name, dimension_list=dim_list, dimension_names=dim_names, time_stamp=time_stamp, value=value, value_meta=json.dumps(value_meta, ensure_ascii=False)) id_bytes = bytearray.fromhex(metric.id) if self._metric_id_cache.get(metric.id, None): measurement_bound_stmt = self._measurement_update_stmt.bind((self._retention, metric.value, metric.value_meta, id_bytes, metric.time_stamp)) self._metric_batch.add_measurement_query(measurement_bound_stmt) metric_update_bound_stmt = self._metric_update_stmt.bind((self._retention, metric.time_stamp, metric.region, metric.tenant_id, metric.name, metric.dimension_list, metric.dimension_names)) self._metric_batch.add_metric_query(metric_update_bound_stmt) return metric self._metric_id_cache[metric.id] = metric.id metric_insert_bound_stmt = self._metric_insert_stmt.bind((self._retention, id_bytes, metric.time_stamp, metric.time_stamp, metric.region, metric.tenant_id, metric.name, metric.dimension_list, metric.dimension_names)) self._metric_batch.add_metric_query(metric_insert_bound_stmt) for dim in metric.dimension_list: (name, value) = dim.split('\t') dim_key = self._get_dimnesion_key(metric.region, metric.tenant_id, name, value) if not self._dimension_cache.get(dim_key, None): dimension_bound_stmt = self._dimension_stmt.bind((metric.region, metric.tenant_id, name, value)) self._metric_batch.add_dimension_query(dimension_bound_stmt) self._dimension_cache[dim_key] = dim_key metric_dim_key = self._get_metric_dimnesion_key( metric.region, metric.tenant_id, metric.name, name, value) if not self._metric_dimension_cache.get(metric_dim_key, None): dimension_metric_bound_stmt = self._dimension_metric_stmt.bind( (metric.region, metric.tenant_id, name, value, metric.name)) self._metric_batch.add_dimension_metric_query(dimension_metric_bound_stmt) metric_dimension_bound_stmt = self._metric_dimension_stmt.bind( (metric.region, metric.tenant_id, metric.name, name, value)) self._metric_batch.add_metric_dimension_query(metric_dimension_bound_stmt) self._metric_dimension_cache[metric_dim_key] = metric_dim_key measurement_insert_bound_stmt = self._measurement_insert_stmt.bind( (self._retention, metric.value, metric.value_meta, metric.region, metric.tenant_id, metric.name, metric.dimension_list, id_bytes, metric.time_stamp)) self._metric_batch.add_measurement_query(measurement_insert_bound_stmt) return metric