def update_timeseries_metadata(session,
datacenter,
starttime,
endtime,
device_type_units={}):
with database.session() as db_session:
datacenter_metadata = get_datacenter_metadata(db_session, datacenter)
logger.debug('update_timeseries_metadata original metadata: %s',
datacenter_metadata)
time_interval = datacenter_metadata['time_interval']
for device_type, device_type_metadata in six.iteritems(
datacenter_metadata['device_types']):
for measurement, measurement_metadata in six.iteritems(
device_type_metadata):
response = list_timeseries(
session, {
'where': {
'starttime': starttime,
'endtime': endtime
},
'group_by': ['time(%ss)' % time_interval],
'order_by': ['time'],
'aggregation': 'mean',
'datacenter': datacenter,
'device_type': {
device_type: {
measurement: []
}
}
},
time_precision=None,
convert_timestamp=True,
format_timestamp=False,
device_type_units=device_type_units,
result_as_dataframe=True)
response = response.dropna()
result = response.values
mean = result.mean()
deviation = result.std()
assert not np.isnan(mean)
assert not np.isnan(deviation)
time_offset = datetime.timedelta(seconds=time_interval)
differentiation_response = (response.shift(-1, time_offset) -
response)
differentiation_response = differentiation_response.dropna()
differentiation_result = differentiation_response.values
differentiation_mean = differentiation_result.mean()
differentiation_deviation = differentiation_result.std()
assert not np.isnan(differentiation_mean)
assert not np.isnan(differentiation_deviation)
measurement_metadata['attribute']['mean'] = mean
measurement_metadata['attribute']['deviation'] = deviation
measurement_metadata['attribute']['differentiation_mean'] = (
differentiation_mean)
measurement_metadata['attribute']['differentiation_deviation'] = (
differentiation_deviation)
logger.debug('updated datacenter metadata: %s', datacenter_metadata)
with database.session() as db_session:
set_datacenter_metadata(db_session, datacenter, datacenter_metadata)
def create_timeseries(session,
data,
tags,
time_precision=None,
convert_timestamp=True,
device_type_units={},
measurement_callback=None,
tags_callback=None):
logger.debug('create timeseries tags: %s', tags)
datacenter = tags.pop('datacenter')
device_types = tags.pop('device_type', {})
with database.session() as db_session:
(device_type_mapping, device_type_types, device_type_patterns,
device_type_unit_converters) = get_device_type_infos(
db_session, datacenter, device_types, device_type_units, False)
logger.debug(
'device_type_mapping %s device_type_types %s '
'time_precision %s '
'device_type_patterns %s convert_timestamp %s device_type_units %s',
device_type_mapping, device_type_types, time_precision,
device_type_patterns, convert_timestamp, device_type_units)
return create_timeseries_internal(
session,
data,
datacenter,
tags,
time_precision=time_precision,
convert_timestamp=convert_timestamp,
device_type_mapping=device_type_mapping,
device_type_types=device_type_types,
device_type_patterns=device_type_patterns,
device_type_unit_converters=device_type_unit_converters,
measurement_callback=measurement_callback,
tags_callback=tags_callback)
def update_test_result_status(datacenter, test_result, status):
logger.debug('update %s test result %s status %s', datacenter, test_result,
status)
with database.session() as session:
test_result_db = session.query(models.TestResult).filter_by(
datacenter_name=datacenter, name=test_result).first()
if not test_result_db:
datacenter_db = session.query(
models.Datacenter).filter_by(name=datacenter).first()
test_result_db = models.TestResult(name=test_result)
datacenter_db.test_results.append(test_result_db)
test_result_db.status = status
def delete_timeseries(session, tags):
logger.debug('delete timeseries tags: %s', tags)
datacenter = tags.pop('datacenter')
device_types = tags.pop('device_type')
with database.session() as db_session:
(device_type_mapping, device_type_types, device_type_patterns,
device_type_unit_converters) = get_device_type_infos(
db_session, datacenter, device_types, {}, False)
logger.debug('device_type_mapping %s', device_type_mapping)
delete_timeseries_internal(session,
datacenter,
device_type_mapping=device_type_mapping)
def save_test_result(datacenter, test_result, result):
logger.debug('save %s test result %s', datacenter, test_result)
device_type_mapping = result.get('device_type_mapping', {})
device_type_types = result.get('device_type_types', {})
statistics = result.get('statistics', {})
model = result.get('model')
model_type = result.get('model_type')
with database.influx_session(dataframe=True) as session:
if 'predictions' in result:
predictions = result['predictions']
logger.debug('save predictions %s', predictions.columns)
timeseries.create_test_result_timeseries(
session,
predictions, {
'datacenter': datacenter,
'reference': test_result
},
'prediction',
device_type_mapping=device_type_mapping,
device_type_types=device_type_types)
if 'expectations' in result:
expectations = result['expectations']
logger.debug('save expectations: %s', expectations.columns)
timeseries.create_test_result_timeseries(
session,
expectations, {
'datacenter': datacenter,
'reference': test_result
},
'expectation',
device_type_mapping=device_type_mapping,
device_type_types=device_type_types)
with database.session() as session:
logger.debug('save statistics: %s', statistics)
properties = {
'device_type_mapping': device_type_mapping,
'device_type_types': device_type_types,
'statistics': {
'.'.join(column): value
for column, value in six.iteritems(statistics)
},
'model': model,
'model_type': model_type
}
test_result_db = session.query(models.TestResult).filter_by(
datacenter_name=datacenter, name=test_result).first()
if not test_result_db:
datacenter_db = session.query(
models.Datacenter).filter_by(name=datacenter).first()
test_result_db = models.TestResult(name=test_result)
datacenter_db.test_results.append(test_result_db)
test_result_db.properties = properties
def list_timeseries(session,
data,
time_precision=None,
convert_timestamp=False,
format_timestamp=True,
device_type_units={},
result_as_dataframe=None,
measurement_callback=None,
data_callback=None):
logger.debug('timeseries data: %s', data)
datacenter = data.pop('datacenter')
device_types = data.pop('device_type', {})
with database.session() as db_session:
(device_type_mapping, device_type_types, device_type_patterns,
device_type_unit_converters) = get_device_type_infos(
db_session, datacenter, device_types, device_type_units)
logger.debug(
'device_type_mapping %s device_type_types %s '
'time_precision %s device_type_unit_converters %s'
'device_type_patterns %s convert_timestamp %s format_timestamp %s'
'result_as_dataframe %s', device_type_mapping, device_type_types,
time_precision, device_type_unit_converters, device_type_patterns,
convert_timestamp, format_timestamp, result_as_dataframe)
return list_timeseries_internal(
session,
data,
datacenter,
time_precision=time_precision,
convert_timestamp=convert_timestamp,
format_timestamp=format_timestamp,
device_type_mapping=device_type_mapping,
device_type_types=device_type_types,
device_type_patterns=device_type_patterns,
device_type_unit_converters=device_type_unit_converters,
result_as_dataframe=result_as_dataframe,
measurement_callback=measurement_callback,
data_callback=data_callback)
def get_metadata(self):
with database.session() as session:
return timeseries.get_datacenter_metadata(
session, self.datacenter
)