def setUp(self):
"""
Insert series.
"""
series = Series(ENTITY, METRIC)
series.add_samples(Sample(VALUE, datetime.now()))
self._series_service = SeriesService(self.connection)
self._series_service.insert(series)
time.sleep(self.wait_time)
def setUp(self):
"""
Insert series.
"""
series = Series(ENTITY, METRIC)
series.add_samples(Sample(VALUE, datetime.now()))
self._series_service = SeriesService(self.connection)
self._series_service.insert(series)
time.sleep(self.wait_time)
class TestSQLService(ServiceTestBase):
def setUp(self):
"""
Insert series.
"""
series = Series(ENTITY, METRIC)
series.add_samples(Sample(VALUE, datetime.now()))
self._series_service = SeriesService(self.connection)
self._series_service.insert(series)
time.sleep(self.wait_time)
def test_query(self):
result = self.service.query(QUERY)
# print(result)
self.assertIsNotNone(result)
self.assertEqual((1, 7), result.shape)
self.assertTrue(isinstance(result.at[0, 'datetime'], str))
self.assertEqual(METRIC, result.at[0, 'metric'])
self.assertEqual(ENTITY, result.at[0, 'entity'])
self.assertEqual(VALUE, result.at[0, 'value'])
class TestSQLService(ServiceTestBase):
def setUp(self):
"""
Insert series.
"""
series = Series(ENTITY, METRIC)
series.add_samples(Sample(VALUE, datetime.now()))
self._series_service = SeriesService(self.connection)
self._series_service.insert(series)
time.sleep(self.wait_time)
def test_query(self):
result = self.service.query(QUERY)
# print(result)
self.assertIsNotNone(result)
self.assertEqual((1, 7), result.shape)
self.assertTrue(isinstance(result.at[0, 'datetime'], str))
self.assertEqual(METRIC, result.at[0, 'metric'])
self.assertEqual(ENTITY, result.at[0, 'entity'])
self.assertEqual(VALUE, result.at[0, 'value'])
def main(filename,
atsd_url,
username,
password,
stat_as_tag,
entity_name=None,
timestamp=None):
if timestamp is None:
timestamp = datetime.now()
conn = connect_url(atsd_url, username, password)
series_service = SeriesService(conn)
with open(filename) as f:
entries = json.load(f)
for entry in entries:
benchmark_name_split = entry['benchmark'].split('.')
entity = benchmark_name_split[
-2] if entity_name is None else entity_name
metric_prefix = 'jmh.' + entry['mode'] + '.' + entry[
'primaryMetric']['scoreUnit']
tags = {'method': benchmark_name_split[-1]}
if not stat_as_tag:
metric = metric_prefix + '.avg'
else:
metric = metric_prefix
tags['stat'] = 'avg'
series_service.insert(
Series(entity, metric,
[Sample(entry['primaryMetric']['score'], timestamp)],
tags))
for key, value in entry['primaryMetric']['scorePercentiles'].items(
):
if key == '0.0':
stat = 'min'
elif key == '100.0':
stat = 'max'
else:
stat = key
if not stat_as_tag:
metric = metric_prefix + '.' + stat
else:
metric = metric_prefix
tags['stat'] = stat
series_service.insert(
Series(entity, metric, [Sample(value, timestamp)], tags))
from datetime import datetime
from atsd_client import connect, connect_url
from atsd_client.utils import print_tags
from atsd_client.models import SeriesQuery, SeriesFilter, EntityFilter, DateFilter, ControlFilter
from atsd_client.services import MetricsService, SeriesService
'''
Retrieve series for a given metric, for each series fetch first and last value with corresponding dates.
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Initialize services
svc = SeriesService(connection)
metric_service = MetricsService(connection)
# set metric name
metric_name = "ca.daily.reservoir_storage_af"
# print header
print(
'entity,entityLabel,seriesTags,firstValueDate,firstValue,lastValueDate,lastValue'
)
# query series with current metric for all entities with meta information in ascending order to get first value
sf = SeriesFilter(metric=metric_name)
ef = EntityFilter(entity='*')
df = DateFilter(start_date="1970-01-01T00:00:00Z", end_date=datetime.now())
cf = ControlFilter(limit=1, add_meta=True, direction="ASC")
STAGE_2_DURATION_MIN, STAGE_2_DURATION_MAX = 5, 10
STAGE_3_DURATION_MIN, STAGE_3_DURATION_MAX = 16, 24
# value caching: enabled when INTERVAL_MINUTES > 0
THRESHOLD_PERCENT = 1
INTERVAL_MINUTES = 10
SAVE_AS_COMMANDS = True
connection = connect_url('https://atsd_hostname:8443', 'user', 'password')
# configuration parameters: end
entities_service = EntitiesService(connection)
metrics_service = MetricsService(connection)
svc = SeriesService(connection)
def positive_spline(diff_value, start_value, t, l, x):
return diff_value * (((x - t).total_seconds() / l.total_seconds()) ** 3) + start_value
def positive_inv_spline(diff_value, start_value, t, l, x):
return diff_value * (1 + (((x - t).total_seconds() / l.total_seconds()) - 1) ** 3) + start_value
def negative_spline(diff_value, end_value, t, l, x):
return diff_value * ((1 - ((x - t).total_seconds() / l.total_seconds())) ** 3) + end_value
def linear(diff_value, start_value, t, l, x):
from datetime import datetime
from atsd_client import connect, connect_url
from atsd_client.models import SeriesQuery, SeriesFilter, EntityFilter, DateFilter
from atsd_client.services import MetricsService, SeriesService
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# set minimum window size to make decision
min_window_size = 5
# 10 days interval in milliseconds
interval_ms = 10 * 24 * 60 * 60 * 1000
svc = SeriesService(connection)
metric_service = MetricsService(connection)
# set metric and entity
metric = 'ca.daily.reservoir_storage_af'
entity = 'ca.oro'
print('Metric: %s\nEntity: %s\n' % (metric, entity))
# prepare query to retrieve data
sf = SeriesFilter(metric=metric, exact_match=True)
ef = EntityFilter(entity=entity)
df = DateFilter(start_date="1970-01-01T00:00:00Z", end_date=datetime.now())
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df)
series, = svc.query(query)
'''
Load all series values that are non-positive for the specified metric.
Optionally, if deleteValues parameter is set replace these values with NaN.
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# disable deleting inappropriate values
deleteValues = False
# specify metric name
metric_name = "ca.daily.reservoir_storage_af"
svc = SeriesService(connection)
metrics_service = MetricsService(connection)
# query series with current metric and all entities
sf = SeriesFilter(metric=metric_name)
ef = EntityFilter(entity='*')
df = DateFilter(start_date="1970-01-01T00:00:00Z", end_date=datetime.now())
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df)
series = svc.query(query)
if deleteValues:
print('Inappropriate values to be deleted.\n')
else:
print('Leave as is inappropriate values.\n')
print('metric,entity,tags,data')
Delete data for a given series with tags for the specified date interval.
'''
# Connect to an ATSD server
connection = connect_url('https://atsd_hostname:8443', 'user', 'password')
# set series
metric = 'm-to-delete'
entity = 'e-to-delete'
tags = {'tag_key_1': 'tag_value_1', 'tag_key_2': 'tag_value_2'}
# specify date interval
startDate = "2018-10-01T00:00:00Z"
endDate = "2018-10-02T00:00:00Z"
series_service = SeriesService(connection)
# query the series to be deleted, use exactMatch to exclude not specified tags
sf = SeriesFilter(metric=metric, tags=tags, exact_match=True)
ef = EntityFilter(entity=entity)
df = DateFilter(start_date=startDate, end_date=endDate)
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df)
series_list = series_service.query(query)
if len(series_list) > 1:
raise Exception('There are multiple series meet the requirements')
series = series_list[0]
# check data existence
if len(series.data) == 0:
# connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Series filter
metric = 'm-to-delete'
entity = 'e-to-delete'
tags = {'tag_key_1': 'tag_value_1', 'tag_key_2': 'tag_value_2'}
# Specify date interval
startDate = "2018-10-01T00:00:00Z"
endDate = "2018-10-02T00:00:00Z"
# Exclude samples with NaN values (NaN represents deleted values)
expr = '!Float.isNaN(value)'
series_service = SeriesService(connection)
# Query the series to be deleted, use exactMatch to exclude not specified tags
sf = SeriesFilter(metric=metric, tags=tags, exact_match=True)
ef = EntityFilter(entity=entity)
df = DateFilter(start_date=startDate, end_date=endDate)
vf = SampleFilter(expr)
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df, sample_filter=vf)
series_list = series_service.query(query)
if len(series_list) > 1:
raise Exception('There are multiple series meet the requirements')
series = series_list[0]
# Check data existence
from atsd_client import connect, connect_url
from atsd_client.models import SeriesFilter, EntityFilter, DateFilter, SeriesQuery, Sample
from atsd_client.services import SeriesService
'''
Load data for one year and insert the data with multiple year shift.
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Initialize services
svc = SeriesService(connection)
# specify years to increase
year_count = 5
# specify metric and entity names
metric_name = 'sml.power-consumed'
entity_name = '*'
# specify date filter
start_date = '2018-01-01T00:00:00Z'
end_date = '2019-01-01T00:00:01Z'
# prepare series_query and execute it
sf = SeriesFilter(metric=metric_name)
ef = EntityFilter(entity=entity_name)
df = DateFilter(start_date=start_date, end_date=end_date)
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df)
series_list = svc.query(query)
from atsd_client import connect, connect_url
from atsd_client.models import SeriesFilter, EntityFilter, DateFilter, SeriesQuery, Sample
from atsd_client.services import SeriesService
'''
Load data for one year and insert the data with multiple year shift.
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Initialize services
svc = SeriesService(connection)
# specify years to increase
year_count = 5
# specify metric and entity names
metric_name = 'sml.power-consumed'
entity_name = '*'
# specify date filter
start_date = '2018-01-01T00:00:00Z'
end_date = '2019-01-01T00:00:01Z'
# prepare series_query and execute it
sf = SeriesFilter(metric=metric_name)
ef = EntityFilter(entity=entity_name)
df = DateFilter(start_date=start_date, end_date=end_date)
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df)
series_list = svc.query(query)
for series in series_list:
query-cpu_busy-nurswgvml007-all-DETAIL.json
query-cpu_busy-nurswgvml007-all-PERCENTILE_90+DELTA.json
query-disk_used-nurswgvml006-all-all.json
query-disk_used-nurswgvml006-all-DETAIL.json
query-disk_used-nurswgvml006-all-MIN+MAX.json
query-disk_used-nurswgvml006-all-MIN.json
query-disk_used-nurswgvml006-all-WTAVG.json
query-log_event_counter-nurswgvml007-command=com.axibase.tsd.Server;level=INFO;logger=com.axibase.tsd.service.config.ServerPropertiesReader-all.json
rate-cpu_busy-nurswgvml007-all-DETAIL.json
group-disk_used-nurswgvml006-all-DETAIL.json
rate+group-collectd.cpu.busy-nurswghbs001-all-MIN.json
'''
# Connect to an ATSD server
connection = connect_url('https://atsd_hostname:8443', 'user', 'password')
svc = SeriesService(connection)
# set start_date and end_date
start_date = '2018-05-07T07:00:00Z'
end_date = '2018-05-08T08:00:00Z'
# list all json files from the current directory
files = [
f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.json')
]
for filename in files:
# parse filename to get series information
query, metric_name, entity_name, tags, aggregate_types = splitext(
filename)[0].split('-')
from atsd_client import connect, connect_url
from atsd_client.models import SeriesFilter, EntityFilter, DateFilter, SeriesQuery, ForecastFilter
from atsd_client.services import SeriesService
'''
Monitor data availability. The inputs stored in a data-availability.csv table.
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Initialize services
svc = SeriesService(connection)
filename = 'data-availability.csv'
with open(filename) as fp:
line = fp.readline()
while line:
# skip commented lines
if line.startswith('#'):
line = fp.readline()
continue
metric_name, entity_name, interval, end_date, forecast_name, comments = line.split(',')
count, unit = interval.split('-')
sf = SeriesFilter(metric=metric_name)
ef = EntityFilter(entity=entity_name)
import matplotlib.pyplot as plt
from TDB.hw import forecast
# В файле connection.properties нужно указать хост, имя и пароль от ATSD
connection = connect();
# Указываем интересующие нас метрику и сущность, для которых будет строиться прогноз
sf = SeriesFilter(metric="direct.queries")
ef = EntityFilter(entity="duckduckgo")
# Выбираем начальную дату для запроса
start_date = "2018-11-03T02:59:00Z"
# Инициализируем временной фильтр, конечная дата = начальная дата + 3 месяца
df = DateFilter(interval={"count": 3, "unit": "MONTH"}, start_date=start_date)
# Формируем series-запрос
query_data = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df)
svc = SeriesService(connection)
# Загружаем указанный ряд
series, = svc.query(query_data)
"""
Визуализируем исторические и спрогнозированные данные
"""
print(series)
start = len(series.values())
end = start + 25
plt.subplot(111)
plt.plot(series.values(), label="History", marker=".")
plt.plot(range(start, end), forecast(series.values(), 7, 25)[-25:], label="Forecast", marker=".")
plt.legend(bbox_to_anchor=(1, 0.14), loc=1, borderaxespad=0.1)
plt.grid(True)
plt.show()
from atsd_client import connect, connect_url
from atsd_client.utils import print_tags
from atsd_client.models import SeriesQuery, SeriesFilter, EntityFilter, DateFilter, ControlFilter
from atsd_client.services import MetricsService, SeriesService
'''
Retrieve series for a given metric, for each series fetch first and last value with corresponding dates.
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Initialize services
svc = SeriesService(connection)
metric_service = MetricsService(connection)
# set metric name
metric_name = "ca.daily.reservoir_storage_af"
# print header
print('entity,entityLabel,seriesTags,firstValueDate,firstValue,lastValueDate,lastValue')
# query series with current metric for all entities with meta information in ascending order to get first value
sf = SeriesFilter(metric=metric_name)
ef = EntityFilter(entity='*')
df = DateFilter(start_date="1970-01-01T00:00:00Z", end_date=datetime.now())
cf = ControlFilter(limit=1, add_meta=True, direction="ASC")
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df, control_filter=cf)
series_list_asc = svc.query(query)
if entity_service.get(source_entity) is None:
logging.warning("'" + source_entity + "' entity does not exist")
exit(1)
if entity_service.get(dst_entity) is None:
logging.warning("'" + dst_entity + "' entity does not exist")
exit(1)
metric_service = MetricsService(connection)
if metric_service.get(metric) is None:
logging.warning("'" + metric + "' metric does not exist")
exit(1)
series_service = SeriesService(connection)
def insert_or_warning(series_to_insert):
if not dry_run:
series_service.insert(series_to_insert)
else:
logging.warning("Dry run enabled, series are not inserted.")
dst_entity_filter = EntityFilter(dst_entity)
dst_date_filter = DateFilter(start_date, 'now')
series_filter = SeriesFilter(metric, tag_expression=tag_expression)
limit_control = ControlFilter(limit=1, direction="ASC")
sample_filter = SampleFilter("!Double.isNaN(value)")
dst_series_query = SeriesQuery(series_filter, dst_entity_filter, dst_date_filter,
from prettytable import PrettyTable
from atsd_client import connect, connect_url
from atsd_client.models import SeriesQuery, SeriesFilter, EntityFilter, DateFilter, ControlFilter, to_iso
from atsd_client.services import MetricsService, SeriesService
'''
Find series with data older than `now - (metric.retention_days + grace_interval_days)`.
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Initialize services
svc = SeriesService(connection)
metric_service = MetricsService(connection)
metric_expression = "enabled AND persistent AND retentionDays > 0"
metric_list = metric_service.list(expression=metric_expression)
series_count = 0
# ATSD expired data removal schedule frequency, default is one day
grace_interval_days = 1
t = PrettyTable(['Metric', 'Entity', 'Tags', 'Retention Days', 'Threshold', 'Presented Sample Date'])
for metric in metric_list:
# calculate datetime before which there is data
threshold = datetime.now() - timedelta(days=metric.retention_days + grace_interval_days)
# query series with current metric and all entities from the beginning up to threshold
# enough to get at least one value, limit set to 1
from atsd_client import connect_url
from atsd_client.services import EntitiesService, SeriesService
from atsd_client.models import SeriesDeleteQuery
'''
Delete series for all metrics for the specified entity with names starting with the specified prefix.
'''
# Connect to an ATSD server
conn = connect_url('https://atsd_hostname:8443', 'user', 'password')
# Set query
entity = "entity"
metric_expr = "name LIKE 'me*'"
entities_service = EntitiesService(conn)
series_service = SeriesService(conn)
# Query all metrics for entity
metrics = entities_service.metrics(entity=entity, expression=metric_expr)
if not metrics:
print("No metrics are found for entity " + entity)
else:
# Delete series for each metric
for metric in metrics:
query = SeriesDeleteQuery(entity=entity, metric=metric.name, exact_match=False)
# Uncomment next line to delete series
#response = series_service.delete(query)
#print(response)
self.splines.append([t0, t1, metric, partial(spline_builder, t0, t1 - t0)])
def get_spline(self, metric, t):
for [s, e, m, f] in self.splines:
if s <= t < e and m == metric:
return f
return None
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Initialize services
entities_service = EntitiesService(connection)
svc = SeriesService(connection)
procedures = [['Heating', 10], ['Mixing', 5], ['Brewing', 40], ['Cooling', 15], ['Inactive', 5]]
metrics = [['axi.temperature', {
'Heating': partial(positive_spline, 55, 30),
'Mixing': partial(positive_inv_spline, 5, 85),
'Brewing': partial(negative_spline, 10, 80),
'Cooling': partial(negative_spline, 30, 50),
'Inactive': partial(linear, -20, 50),
}], ['axi.pressure', {
'Heating': partial(positive_spline, 12, 10),
'Mixing': partial(linear, 0, 22),
'Brewing': partial(positive_spline, 3, 22),
'Cooling': partial(negative_spline, 15, 10),
'Inactive': partial(linear, 0, 10),
query-disk_used-nurswgvml006-all-DETAIL.json
query-disk_used-nurswgvml006-all-MIN+MAX.json
query-disk_used-nurswgvml006-all-MIN.json
query-disk_used-nurswgvml006-all-WTAVG.json
query-log_event_counter-nurswgvml007-command=com.axibase.tsd.Server;level=INFO;logger=com.axibase.tsd.service.config.ServerPropertiesReader-all.json
rate-cpu_busy-nurswgvml007-all-DETAIL.json
group-disk_used-nurswgvml006-all-DETAIL.json
rate+group-collectd.cpu.busy-nurswghbs001-all-MIN.json
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Initialize services
svc = SeriesService(connection)
# set start_date and end_date
start_date = '2018-05-07T07:00:00Z'
end_date = '2018-05-08T08:00:00Z'
# list all json files from the current directory
files = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.json')]
for filename in files:
# parse filename to get series information
query, metric_name, entity_name, tags, aggregate_types = splitext(filename)[0].split('-')
# prepare tags
exact_match = tags != 'all'
if exact_match:
from atsd_client.models import SeriesDeleteQuery
'''
Delete series for all metrics for the specified entity with names starting with the specified prefix.
'''
# Connect to ATSD server
# connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Set query
entity = "entity"
metric_expr = "name LIKE 'me*'"
# Initialize services
entities_service = EntitiesService(connection)
series_service = SeriesService(connection)
# Query all metrics for entity
metrics = entities_service.metrics(entity=entity, expression=metric_expr)
if not metrics:
print("No metrics are found for entity " + entity)
else:
# Delete series for each metric
for metric in metrics:
query = SeriesDeleteQuery(entity=entity,
metric=metric.name,
exact_match=False)
print("deleting ", entity, metric.name)
# Uncomment next line to delete series
# response = series_service.delete(query)
from datetime import datetime
from atsd_client import connect, connect_url
from atsd_client.models import SeriesFilter, EntityFilter, DateFilter, SeriesQuery
from atsd_client.services import SeriesService, MetricsService
'''
Copy data from one metric to the new one.
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Initialize services
svc = SeriesService(connection)
metrics_service = MetricsService(connection)
# specify source and destination metric names
metric_src = 'metric_src'
metric_dst = 'metric_dst'
# copy series with all entities, specific entity name can be set instead
entity = '*'
# query series with required metric and all entities
sf = SeriesFilter(metric=metric_src)
ef = EntityFilter(entity='*')
df = DateFilter(start_date='1970-01-01T00:00:00Z', end_date=datetime.now())
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df)
series = svc.query(query)
