entities_list = eg_service.get_entities(entity_group_name, tags=tag_expression)
# exclude entities that have no required tags
entities = [entity for entity in entities_list if entity.tags]
entities_service = EntitiesService(connection)
# prepare property query
ef = EntityFilter('entity')
df = DateFilter(start_date="1970-01-01T00:00:00Z", end_date=datetime.now())
property_query = PropertiesQuery(entity_filter=ef,
date_filter=df,
type=property_type)
print('entity_name,entity_label,tags')
for entity in entities:
pretty_tags = print_tags(entity.tags)
for key in entity.tags:
entity.tags[key] = ''
# set actual entity and execute property query
property_query.set_entity_filter(EntityFilter(entity.name))
properties_list = properties_service.query(property_query)
# set entity tags from property tags
if properties_list:
property_tags = properties_list[0].tags
for key, value in six.iteritems(property_tags):
entity.tags['env.%s' % key] = value
print('%s,%s,%s' % (entity.name, print_str(entity.label), pretty_tags))
# Uncomment next line to delete tags
# entities_service.update(entity)
# Connect to an ATSD server
connection = connect_url('https://atsd_hostname:8443', 'user', 'password')
# set grace_interval to one day
grace_interval_minutes = 24 * 60
# query entities with last_insert_date
min_insert_date = "1970-01-01T00:00:00.000Z"
# calculate the upper boundary for the allowed last_insert_date values excluding grace_interval
max_insert_date = datetime.now() - timedelta(seconds=grace_interval_minutes *
60)
entities_service = EntitiesService(connection)
metrics_service = MetricsService(connection)
# query entities that have name started with 06
entities = entities_service.list(expression="name LIKE '06*'")
print('metric, entity, tags, last_insert_date')
for entity in entities:
# query all metrics for each entity
metrics = entities_service.metrics(entity)
for metric in metrics:
# query all series for each metric and entity
series = metrics_service.series(metric,
entity,
min_insert_date=min_insert_date,
max_insert_date=max_insert_date)
for s in series:
print("%s,%s,%s,%s" %
(s.metric, s.entity, print_tags(s.tags), s.last_insert_date))
tf = TransformationFilter()
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df)
if args.aggregate_period > 0:
tf.set_aggregate(
Aggregate(period={'count': args.aggregate_period, 'unit': TimeUnit.MINUTE}, types=[AggregateType.MEDIAN]))
if args.interpolate_period > 0:
tf.set_interpolate(Interpolate(period={'count': args.interpolate_period, 'unit': TimeUnit.MINUTE},
function=InterpolateFunction.LINEAR))
query.set_transformation_filter(tf)
series_list = svc.query(query)
for series in series_list:
metric_id = '- %s %s' % (series.metric, print_tags(series.tags))
log('\t' + metric_id)
# exclude empty series for specific tags
if len(series.data) > 2:
ts = {int(sample.t / 1000): sample.v for sample in series.data}
detector = AnomalyDetector(ts, score_threshold=args.min_score)
anomalies = []
for anomaly in detector.get_anomalies():
if time.mktime(now.timetuple()) - args.last_hours * 3600 <= anomaly.exact_timestamp:
anomalies.append(anomaly)
if anomalies:
message.append(metric_id)
for anomaly in anomalies:
series_list_desc = svc.query(query)
for series_asc in series_list_asc:
if len(series_asc.data) > 0:
# get corresponding descending series and remove series from desc list
index_series_desc = -1
for idx, sd in enumerate(series_list_desc):
if sd.entity == series_asc.entity and sd.metric == series_asc.entity and sd.tags == series_asc.tags:
index_series_desc = idx
break
series_desc = series_list_desc.pop(index_series_desc)
# get label from meta information
label = series_asc.meta['entity'].label if series_asc.meta[
'entity'].label is not None else ''
# get first and last samples in series to output
print('%s,%s,%s,%s,%s,%s,%s' %
(series_asc.entity, label, print_tags(series_asc.tags),
series_asc.get_first_value_date(), series_asc.get_first_value(),
series_desc.get_first_value_date(),
series_desc.get_first_value()))
# print remaining series that are not in ascending list
for series_desc in series_list_desc:
label = series_desc.meta['entity'].label if series_desc.meta[
'entity'].label is not None else ''
print('%s,%s,%s,%s,%s,%s,%s' %
(series_desc.entity, label, print_tags(series_desc.tags), '', '',
series_desc.get_first_value_date(), series_desc.get_first_value()))
series_list_asc = svc.query(query)
# change filter to get last value and query series, descending order set by default
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df, control_filter=ControlFilter(limit=1))
series_list_desc = svc.query(query)
for series_asc in series_list_asc:
if len(series_asc.data) > 0:
# get corresponding descending series and remove series from desc list
index_series_desc = -1
for idx, sd in enumerate(series_list_desc):
if sd.entity == series_asc.entity and sd.metric == series_asc.entity and sd.tags == series_asc.tags:
index_series_desc = idx
break
series_desc = series_list_desc.pop(index_series_desc)
# get label from meta information
label = series_asc.meta['entity'].label if series_asc.meta['entity'].label is not None else ''
# get first and last samples in series to output
print('%s,%s,%s,%s,%s,%s,%s' % (series_asc.entity, label, print_tags(series_asc.tags),
series_asc.get_first_value_date(), series_asc.get_first_value(),
series_desc.get_first_value_date(), series_desc.get_first_value()))
# print remaining series that are not in ascending list
for series_desc in series_list_desc:
label = series_desc.meta['entity'].label if series_desc.meta['entity'].label is not None else ''
print('%s,%s,%s,%s,%s,%s,%s' % (series_desc.entity, label, print_tags(series_desc.tags),
'', '',
series_desc.get_first_value_date(), series_desc.get_first_value()))
# Connect to an ATSD server
connection = connect_url('https://atsd_hostname:8443', 'user', 'password')
# set export parameters
start_date = "2018-05-01T00:00:00Z"
end_date = "now"
type = "logger"
source = "com.axibase.tsd.service.search.searchserviceimpl"
entity = "7eacbc4723ed"
message_service = MessageService(connection)
ef = EntityFilter(entity=entity)
df = DateFilter(start_date=start_date, end_date=end_date)
query = MessageQuery(entity_filter=ef,
date_filter=df,
type=type,
source=source)
messages = message_service.query(query)
with open('export.csv', 'w') as f:
print('date, entity, type, source, severity, tags, message', file=f)
for message in messages:
# make message body single line
msg = message.message.replace("\n", r"\n").replace("\t", r"\t")
print('%s, %s, %s, %s, %s, %s, %s' %
(message.date, message.entity, message.type, message.source,
message.severity, print_tags(message.tags), msg),
file=f)
# Query the series to be deleted, use exactMatch=False to include all tags
sf = SeriesFilter(metric=metric, exact_match=False)
ef = EntityFilter(entity=entity)
df = DateFilter(start_date=startDate, end_date=endDate)
vf = SampleFilter(expr)
query = SeriesQuery(series_filter=sf,
entity_filter=ef,
date_filter=df,
sample_filter=vf)
series_list = series_service.query(query)
if len(series_list) == 0:
print("No series are found")
else:
for series in series_list:
# Replace value of samples with NaN
if len(series.data) == 0:
print("Skip series with no data in the interval ", series.metric,
series.entity, series.tags)
continue
print("Deleting %s values for %s, %s, %s" %
(len(series.data), series.metric, series.entity,
utils.print_tags(series.tags)))
for sample in series.data:
print("- Deleting %s, %s " % (sample.get_date(), sample.v))
sample.v = None
series.aggregate = None
# Uncomment next line to delete
# series_service.insert(series)
# Exclude samples with NaN values (NaN represents deleted values)
expr = '!Float.isNaN(value)'
series_service = SeriesService(connection)
# Query the series to be deleted, use exactMatch=False to include all tags
sf = SeriesFilter(metric=metric, exact_match=False)
ef = EntityFilter(entity=entity)
df = DateFilter(start_date=startDate, end_date=endDate)
vf = SampleFilter(expr)
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df, sample_filter=vf)
series_list = series_service.query(query)
if len(series_list) == 0:
print("No series are found")
else:
for series in series_list:
# Replace value of samples with NaN
if len(series.data) == 0:
print("Skip series with no data in the interval ", series.metric, series.entity, series.tags)
continue
print("Deleting %s values for %s, %s, %s" % (
len(series.data), series.metric, series.entity, utils.print_tags(series.tags)))
for sample in series.data:
print("- Deleting %s, %s " % (sample.get_date(), sample.v))
sample.v = None
series.aggregate = None
# Uncomment next line to delete
# series_service.insert(series)
from atsd_client.utils import print_tags, print_str
'''
Delete entity tags with names starting with the specified prefix from entities that belongs specific entity group.
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Initialize services
# set the name of entity_group and prefix for tag key
entity_group_name = 'docker-images'
tag_expression = 'env.*'
eg_service = EntityGroupsService(connection)
entities_service = EntitiesService(connection)
entities_list = eg_service.get_entities(entity_group_name, tags=tag_expression)
# exclude entities that have no required tags
entities = [entity for entity in entities_list if entity.tags]
print('entity_name,entity_label,tags')
for entity in entities:
pretty_tags = print_tags(entity.tags)
for key in entity.tags:
entity.tags[key] = ''
print('%s,%s,%s' % (entity.name, print_str(entity.label), pretty_tags))
# Uncomment next line to delete tags
# entities_service.update(entity)
'''
Locate high-cardinality series that have tags more than specified cardinality.
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# Initialize services
metrics_service = MetricsService(connection)
# query all metrics that have last_insert_date, i.e. series
metrics = metrics_service.list(min_insert_date="1970-01-01T00:00:00.000Z")
# set cardinality
cardinality = 8
series_count = 0
print('metric,entity,tags,last_insert_date')
for metric in metrics:
# query series list for each metric
series_list = metrics_service.series(metric)
for s in series_list:
# check tags cardinality for each series in list
if len(s.tags) > cardinality:
series_count += 1
print("%s,%s,%s,%s" % (s.metric, s.entity, print_tags(s.tags), s.last_insert_date))
print("Number of series that have more than %d tags combinations is %d " % (cardinality, series_count))
Export messages from ATSD into CSV-file using specified start_date, end_date, type, source and entity.
'''
# Connect to ATSD server
#connection = connect('/path/to/connection.properties')
connection = connect_url('https://atsd_hostname:8443', 'username', 'password')
# set export parameters
start_date = "2018-05-01T00:00:00Z"
end_date = "now"
type = "logger"
source = "com.axibase.tsd.service.search.searchserviceimpl"
entity = "7eacbc4723ed"
message_service = MessageService(connection)
ef = EntityFilter(entity=entity)
df = DateFilter(start_date=start_date, end_date=end_date)
query = MessageQuery(entity_filter=ef, date_filter=df, type=type, source=source)
messages = message_service.query(query)
with open('export.csv', 'w') as f:
print('date, entity, type, source, severity, tags, message', file=f)
for message in messages:
# make message body single line
msg = message.message.replace("\n", r"\n").replace("\t", r"\t")
print('%s, %s, %s, %s, %s, %s, %s' % (
message.date, message.entity, message.type, message.source, message.severity, print_tags(message.tags), msg),
file=f)
tf = TransformationFilter()
query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df)
if args.aggregate_period > 0:
tf.set_aggregate(
Aggregate(period={'count': args.aggregate_period, 'unit': TimeUnit.MINUTE}, types=[AggregateType.MEDIAN]))
if args.interpolate_period > 0:
tf.set_interpolate(Interpolate(period={'count': args.interpolate_period, 'unit': TimeUnit.MINUTE},
function=InterpolateFunction.LINEAR))
query.set_transformation_filter(tf)
series_list = svc.query(query)
for series in series_list:
metric_id = '- %s %s' % (series.metric, print_tags(series.tags))
log('\t' + metric_id)
# exclude empty series for specific tags
if len(series.data) > 2:
ts = {int(sample.t / 1000): sample.v for sample in series.data}
detector = AnomalyDetector(ts, score_threshold=args.min_score)
anomalies = []
for anomaly in detector.get_anomalies():
if time.mktime(now.timetuple()) - args.last_hours * 3600 <= anomaly.exact_timestamp:
anomalies.append(anomaly)
if anomalies:
message.append(metric_id)
for anomaly in anomalies: