def get_schema(self, get_stats=False): connection = atsd_client.connect_url( self.url, self.configuration.get('username'), self.configuration.get('password'), verify=self.configuration.get('trust_certificate', False), timeout=self.configuration.get('timeout', 600)) metrics = MetricsService(connection) ml = metrics.list( expression=self.configuration.get('expression', None), minInsertDate=self.configuration.get('min_insert_date', None), limit=self.configuration.get('limit', 5000)) metrics_list = [i.name.encode('utf-8') for i in ml] metrics_list.append('atsd_series') schema = {} default_columns = [ 'entity', 'datetime', 'time', 'metric', 'value', 'text', 'tags', 'entity.tags', 'metric.tags' ] for table_name in metrics_list: schema[table_name] = { 'name': "'{}'".format(table_name), 'columns': default_columns } values = schema.values() return values
def get_schema(self, get_stats=False): connection = atsd_client.connect_url( self.url, self.configuration.get("username"), self.configuration.get("password"), verify=self.configuration.get("trust_certificate", False), timeout=self.configuration.get("timeout", 600), ) metrics = MetricsService(connection) ml = metrics.list( expression=self.configuration.get("expression", None), minInsertDate=self.configuration.get("min_insert_date", None), limit=self.configuration.get("limit", 5000), ) metrics_list = [i.name.encode("utf-8") for i in ml] metrics_list.append("atsd_series") schema = {} default_columns = [ "entity", "datetime", "time", "metric", "value", "text", "tags", "entity.tags", "metric.tags", ] for table_name in metrics_list: schema[table_name] = { "name": "'{}'".format(table_name), "columns": default_columns, } values = list(schema.values()) return values
def get_schema(self, get_stats=False): connection = atsd_client.connect_url(self.url, self.configuration.get('username'), self.configuration.get('password'), verify=self.configuration.get('trust_certificate', False), timeout=self.configuration.get('timeout', 600)) metrics = MetricsService(connection) ml = metrics.list(expression=self.configuration.get('expression', None), minInsertDate=self.configuration.get('min_insert_date', None), limit=self.configuration.get('limit', 5000)) metrics_list = [i.name.encode('utf-8') for i in ml] metrics_list.append('atsd_series') schema = {} default_columns = ['entity', 'datetime', 'time', 'metric', 'value', 'text', 'tags', 'entity.tags', 'metric.tags'] for table_name in metrics_list: schema[table_name] = {'name': "'{}'".format(table_name), 'columns': default_columns} values = schema.values() return values
from atsd_client import connect, connect_url from atsd_client.models import SeriesQuery, SeriesFilter, EntityFilter, DateFilter, ControlFilter, to_iso from atsd_client.services import MetricsService, SeriesService ''' Find series with data older than `now - (metric.retention_days + grace_interval_days)`. ''' # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # Initialize services svc = SeriesService(connection) metric_service = MetricsService(connection) metric_expression = "enabled AND persistent AND retentionDays > 0" metric_list = metric_service.list(expression=metric_expression) series_count = 0 # ATSD expired data removal schedule frequency, default is one day grace_interval_days = 1 t = PrettyTable(['Metric', 'Entity', 'Tags', 'Retention Days', 'Threshold', 'Presented Sample Date']) for metric in metric_list: # calculate datetime before which there is data threshold = datetime.now() - timedelta(days=metric.retention_days + grace_interval_days) # query series with current metric and all entities from the beginning up to threshold # enough to get at least one value, limit set to 1 sf = SeriesFilter(metric=metric.name)
from atsd_client import connect_url from atsd_client.services import MetricsService ''' Locate a collection of metrics that have no last_insert_date. ''' # Connect to an ATSD server connection = connect_url('https://atsd_hostname:8443', 'user', 'password') metric_service = MetricsService(connection) # query entities without last_insert_date metric_list = metric_service.list(max_insert_date="1970-01-01T00:00:00.000Z") metrics_count = 0 print('metric_name') for metric in metric_list: if metric.enabled and metric.persistent \ and metric.retention_days == 0 and metric.series_retention_days == 0: metrics_count += 1 print(metric.name) print("\nMetrics count without last insert date is %d." % metrics_count)
from atsd_client.services import MetricsService, EntitiesService ''' Locate lagging series among all the series that differ only in tags (have the same metric and entity) during the grace interval. ''' # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # set grace interval in hours grace_interval_hours = 1 entities_service = EntitiesService(connection) metrics_service = MetricsService(connection) # query all entities that have last_insert_date, i.e. series entities = entities_service.list(expression="name LIKE '06*'", min_insert_date="1970-01-01T00:00:00.000Z") print('metric,entity,tags,last_insert_date') for entity in entities: # query all metrics for each entity metrics = entities_service.metrics(entity.name) for m in metrics: # query series list for each metric and entity series_list = metrics_service.series(m.name, entity.name) # for each list with more than 1 series if len(series_list) > 1: # set lower limit to maximum last insert date over series list minus one hour lower_limit_date = max(s.last_insert_date for s in series_list) - timedelta(
from atsd_client import connect, connect_url from atsd_client.services import MetricsService ''' Locate a collection of metrics that have been created after specified date. ''' # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # Initialize services metrics_service = MetricsService(connection) # query all metrics created after specified_date metric_list = metrics_service.list(expression="createdDate > '2018-05-16T00:00:00Z'") print('metric_name') for metric in metric_list: print(metric.name)
STAGE_3_DURATION_MIN, STAGE_3_DURATION_MAX = 16, 24 # value caching: enabled when INTERVAL_MINUTES > 0 THRESHOLD_PERCENT = 1 INTERVAL_MINUTES = 10 SAVE_AS_COMMANDS = True # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # configuration parameters: end entities_service = EntitiesService(connection) metrics_service = MetricsService(connection) svc = SeriesService(connection) def positive_spline(diff_value, start_value, t, l, x): return diff_value * (((x - t).total_seconds() / l.total_seconds()) ** 3) + start_value def positive_inv_spline(diff_value, start_value, t, l, x): return diff_value * (1 + (((x - t).total_seconds() / l.total_seconds()) - 1) ** 3) + start_value def negative_spline(diff_value, end_value, t, l, x): return diff_value * ((1 - ((x - t).total_seconds() / l.total_seconds())) ** 3) + end_value
Locate a collection of entities (docker hosts in this cases) that have not inserted data for more than 7 days. Delete related entities (docker containers, images, network, volumes). Delete docker host entities. ''' # Uncomment the next two lines to set custom local timezone # os.environ['TZ'] = 'Europe/London' # time.tzset() tags_printer = pprint.PrettyPrinter(indent=4) # Connect to an ATSD server connection = connect_url('https://atsd_hostname:8443', 'user', 'password') entity_service = EntitiesService(connection) metric_service = MetricsService(connection) # select all entities that collect this metric # this metric is collected by docker hosts docker_hosts = metric_service.series('docker.cpu.sum.usage.total.percent') print("Docker hosts found: " + str(len(docker_hosts))) for docker_host_series in docker_hosts: print("--------------") # get minutes since last insert elapsed_minutes = docker_host_series.get_elapsed_minutes() entity_filter = "lower(tags.docker-host) = lower('" + docker_host_series.entity + "')" # find related entities, which tag value equals docker host
from atsd_client import connect, connect_url from atsd_client.services import MetricsService from atsd_client.utils import print_tags ''' Locate high-cardinality series that have tags more than specified cardinality. ''' # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # Initialize services metrics_service = MetricsService(connection) # query all metrics that have last_insert_date, i.e. series metrics = metrics_service.list(min_insert_date="1970-01-01T00:00:00.000Z") # set cardinality cardinality = 8 series_count = 0 print('metric,entity,tags,last_insert_date') for metric in metrics: # query series list for each metric series_list = metrics_service.series(metric) for s in series_list: # check tags cardinality for each series in list if len(s.tags) > cardinality: series_count += 1 print("%s,%s,%s,%s" % (s.metric, s.entity, print_tags(s.tags), s.last_insert_date))
Delete Docker host entities. ''' # Uncomment the next two lines to set custom local timezone # os.environ['TZ'] = 'Europe/London' # time.tzset() tags_printer = pprint.PrettyPrinter(indent=4) # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # Initialize services entity_service = EntitiesService(connection) metric_service = MetricsService(connection) # select all entities that collect this metric # this metric is collected by docker hosts docker_hosts = metric_service.series('docker.cpu.sum.usage.total.percent') print("Docker hosts found: " + str(len(docker_hosts))) for docker_host_series in docker_hosts: print("--------------") # get minutes since last insert elapsed_minutes = docker_host_series.get_elapsed_minutes() entity_filter = "lower(tags.docker-host) = lower('" + docker_host_series.entity + "')" # find related entities, which tag value equals docker host
''' Locate series that have no data during the actual time interval (grace_interval) using an expression filter for entity. ''' # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # set grace_interval to one day grace_interval_minutes = 24 * 60 # query entities with last_insert_date min_insert_date = "1970-01-01T00:00:00.000Z" # calculate the upper boundary for the allowed last_insert_date values excluding grace_interval max_insert_date = datetime.now() - timedelta(seconds=grace_interval_minutes * 60) entities_service = EntitiesService(connection) metrics_service = MetricsService(connection) # query entities that have name started with 06 entities = entities_service.list(expression="name LIKE '06*'") print('metric, entity, tags, last_insert_date') for entity in entities: # query all metrics for each entity metrics = entities_service.metrics(entity) for metric in metrics: # query all series for each metric and entity series = metrics_service.series(metric, entity, min_insert_date=min_insert_date, max_insert_date=max_insert_date) for s in series: print("%s,%s,%s,%s" % (s.metric, s.entity, print_tags(s.tags), s.last_insert_date))
from atsd_client import connect, connect_url from atsd_client.models import SeriesFilter, EntityFilter, DateFilter, SeriesQuery from atsd_client.services import MetricsService, SeriesService ''' Retrieve metrics with 'frequency' tag. Load series for this metric. Print values that violate frequency (but not data gaps if the gap is a multiple of the frequency period). ''' # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # Initialize services svc = SeriesService(connection) metric_service = MetricsService(connection) # query metrics with not empty frequency tag, include all metric tags to response metric_list = metric_service.list(expression='tags.frequency != ""', tags='*') def resolve_frequency(frequency): """ Transform metric frequency tag into relativedelta instance """ if frequency in ['Daily (D)', 'Daily', 'Daily, 7-Day', 'Daily, Close']: return relativedelta(days=1) elif frequency in ['Weekly', 'Weekly, As of Monday', 'Weekly, As of Wednesday', 'Weekly, As of Thursday', 'Weekly, Ending Monday', 'Weekly, Ending Wednesday', 'Weekly, Ending Thursday', 'Weekly, Ending Friday', 'Weekly, Ending Saturday', ]: return relativedelta(weeks=1)
Print entities of that the series. ''' # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # specify metric name metric_name = "ca.daily.reservoir_storage_af" # set lower boundary for lastInsertDate min_insert_date = "2018-05-01T00:00:00.000Z" # set grace interval in hours for 14 days grace_interval_hours = 24 * 14 entities_service = EntitiesService(connection) metrics_service = MetricsService(connection) # query required metric meta data metric = metrics_service.get(metric_name) if metric is None: print('No metric with name %s' % metric_name) sys.exit() elif metric.last_insert_date is None: print('No data for metric name %s' % metric_name) sys.exit() # calculate the upper boundary for the allowed last_insert_date values excluding grace interval max_insert_date = metric.last_insert_date - timedelta( seconds=grace_interval_hours * 3600) # query series list for the metric
batch_size = int(args.batch_size) connection = connect('/path/to/connection.properties') entity_service = EntitiesService(connection) if entity_service.get(source_entity) is None: logging.warning("'" + source_entity + "' entity does not exist") exit(1) if entity_service.get(dst_entity) is None: logging.warning("'" + dst_entity + "' entity does not exist") exit(1) metric_service = MetricsService(connection) if metric_service.get(metric) is None: logging.warning("'" + metric + "' metric does not exist") exit(1) series_service = SeriesService(connection) def insert_or_warning(series_to_insert): if not dry_run: series_service.insert(series_to_insert) else: logging.warning("Dry run enabled, series are not inserted.")
Locate series that have no data during the actual time interval (grace_interval) using an expression filter for entity. ''' # Connect to an ATSD server connection = connect_url('https://atsd_hostname:8443', 'user', 'password') # set grace_interval to one day grace_interval_minutes = 24 * 60 # query entities with last_insert_date min_insert_date = "1970-01-01T00:00:00.000Z" # calculate the upper boundary for the allowed last_insert_date values excluding grace_interval max_insert_date = datetime.now() - timedelta(seconds=grace_interval_minutes * 60) entities_service = EntitiesService(connection) metrics_service = MetricsService(connection) # query entities that have name started with 06 entities = entities_service.list(expression="name LIKE '06*'") print('metric, entity, tags, last_insert_date') for entity in entities: # query all metrics for each entity metrics = entities_service.metrics(entity) for metric in metrics: # query all series for each metric and entity series = metrics_service.series(metric, entity, min_insert_date=min_insert_date, max_insert_date=max_insert_date) for s in series:
from datetime import datetime from dateutil.relativedelta import relativedelta from atsd_client import connect_url from atsd_client.models import SeriesFilter, EntityFilter, DateFilter, SeriesQuery from atsd_client.services import MetricsService, SeriesService ''' Retrieve metrics with 'frequency' tag. Load series for this metric. Print values that violate frequency (but not data gaps if the gap is a multiple of the frequency period). ''' # Connect to an ATSD server connection = connect_url('https://atsd_hostname:8443', 'user', 'password') svc = SeriesService(connection) metric_service = MetricsService(connection) # query metrics with not empty frequency tag, include all metric tags to response metric_list = metric_service.list(expression='tags.frequency != ""', tags='*') def resolve_frequency(frequency): """ Transform metric frequency tag into relativedelta instance """ if frequency in ['Daily (D)', 'Daily', 'Daily, 7-Day', 'Daily, Close']: return relativedelta(days=1) elif frequency in [ 'Weekly', 'Weekly, As of Monday', 'Weekly, As of Wednesday',
Print entities of that the series. ''' # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # specify metric name metric_name = "ca.daily.reservoir_storage_af" # set lower boundary for lastInsertDate min_insert_date = "2018-05-01T00:00:00.000Z" # set grace interval in hours for 14 days grace_interval_hours = 24 * 14 entities_service = EntitiesService(connection) metrics_service = MetricsService(connection) # query required metric meta data metric = metrics_service.get(metric_name) if metric is None: print('No metric with name %s' % metric_name) sys.exit() elif metric.last_insert_date is None: print('No data for metric name %s' % metric_name) sys.exit() # calculate the upper boundary for the allowed last_insert_date values excluding grace interval max_insert_date = metric.last_insert_date - timedelta(seconds=grace_interval_hours * 3600) # query series list for the metric series_list = metrics_service.series(metric, min_insert_date=min_insert_date, max_insert_date=max_insert_date)
from atsd_client import connect, connect_url from atsd_client.services import MetricsService ''' Locate a collection of metrics that have no last_insert_date. ''' # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # Initialize services metric_service = MetricsService(connection) # query entities without last_insert_date metric_list = metric_service.list(max_insert_date="1970-01-01T00:00:00.000Z") metrics_count = 0 print('metric_name') for metric in metric_list: if metric.enabled and metric.persistent \ and metric.retention_days == 0 and metric.series_retention_days == 0: metrics_count += 1 print(metric.name) print("\nMetrics count without last insert date is %d." % metrics_count)
Load all series values that are non-positive for the specified metric. Optionally, if deleteValues parameter is set replace these values with NaN. ''' # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # disable deleting inappropriate values deleteValues = False # specify metric name metric_name = "ca.daily.reservoir_storage_af" svc = SeriesService(connection) metrics_service = MetricsService(connection) # query series with current metric and all entities sf = SeriesFilter(metric=metric_name) ef = EntityFilter(entity='*') df = DateFilter(start_date="1970-01-01T00:00:00Z", end_date=datetime.now()) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df) series = svc.query(query) if deleteValues: print('Inappropriate values to be deleted.\n') else: print('Leave as is inappropriate values.\n') print('metric,entity,tags,data') for s in series:
STAGE_1_DURATION_MIN, STAGE_1_DURATION_MAX = 4, 8 STAGE_2_DURATION_MIN, STAGE_2_DURATION_MAX = 5, 10 STAGE_3_DURATION_MIN, STAGE_3_DURATION_MAX = 16, 24 # value caching: enabled when INTERVAL_MINUTES > 0 THRESHOLD_PERCENT = 1 INTERVAL_MINUTES = 10 SAVE_AS_COMMANDS = True connection = connect_url('https://atsd_hostname:8443', 'user', 'password') # configuration parameters: end entities_service = EntitiesService(connection) metrics_service = MetricsService(connection) svc = SeriesService(connection) def positive_spline(diff_value, start_value, t, l, x): return diff_value * (((x - t).total_seconds() / l.total_seconds()) ** 3) + start_value def positive_inv_spline(diff_value, start_value, t, l, x): return diff_value * (1 + (((x - t).total_seconds() / l.total_seconds()) - 1) ** 3) + start_value def negative_spline(diff_value, end_value, t, l, x): return diff_value * ((1 - ((x - t).total_seconds() / l.total_seconds())) ** 3) + end_value
from atsd_client import connect_url from atsd_client.services import MetricsService, EntitiesService ''' Locate lagging series among all the series that differ only in tags (have the same metric and entity) during the grace interval. ''' # Connect to an ATSD server connection = connect_url('https://atsd_hostname:8443', 'user', 'password') # set grace interval in hours grace_interval_hours = 1 entities_service = EntitiesService(connection) metrics_service = MetricsService(connection) # query all entities that have last_insert_date, i.e. series entities = entities_service.list(expression="name LIKE '06*'", min_insert_date="1970-01-01T00:00:00.000Z") print('metric,entity,tags,last_insert_date') for entity in entities: # query all metrics for each entity metrics = entities_service.metrics(entity.name) for m in metrics: # query series list for each metric and entity series_list = metrics_service.series(m.name, entity.name) # for each list with more than 1 series if len(series_list) > 1: # calculate maximum of all last_insert_date's in list and subtract 1 hour # it will be lower limit date to compare
from datetime import datetime from atsd_client import connect, connect_url from atsd_client.models import SeriesFilter, EntityFilter, DateFilter, SeriesQuery from atsd_client.services import SeriesService, MetricsService ''' Copy data from one metric to the new one. ''' # Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # Initialize services svc = SeriesService(connection) metrics_service = MetricsService(connection) # specify source and destination metric names metric_src = 'metric_src' metric_dst = 'metric_dst' # copy series with all entities, specific entity name can be set instead entity = '*' # query series with required metric and all entities sf = SeriesFilter(metric=metric_src) ef = EntityFilter(entity='*') df = DateFilter(start_date='1970-01-01T00:00:00Z', end_date=datetime.now()) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df) series = svc.query(query)