def test_aggregate_series(self): val = random.randint(0, VALUE - 1) insert_series_sample(self.service, val, val + 1) time.sleep(WAIT_TIME) now = datetime.now() sf = SeriesFilter(metric=METRIC, tags={TAG: [TAG_VALUE]}) ef = EntityFilter(entity=ENTITY) df = DateFilter(start_date=now - timedelta(hours=1), end_date=now) aggr = Aggregate(period={ 'count': 10, 'unit': TimeUnit.SECOND }, types=[AggregateType.MAX, AggregateType.MIN]) tf = TransformationFilter(aggregate=aggr) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df, transformation_filter=tf) series = self.service.query(query) self.assertEqual(len(series), 2) if series[0].aggregate['type'] == 'MAX': max = series[0].get_last_value() min = series[1].get_last_value() else: min = series[0].get_last_value() max = series[1].get_last_value() self.assertGreaterEqual(max, min)
def test_value_filter(self): """ Check value filter. """ insert_series_sample(self.service, None, 2, 3) time.sleep(WAIT_TIME) sf = SeriesFilter(metric=METRIC, tags={TAG: TAG_VALUE}, exact_match=True) ef = EntityFilter(entity=ENTITY) df = DateFilter(start_date='now - 1 * MINUTE', end_date="now") vf = ValueFilter('Float.isNaN(value) OR value=2') query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df, value_filter=vf) series = self.service.query(query) # print(series) self.assertIsNotNone(series) self.assertEqual(1, len(series)) s = series[0] self.assertIsInstance(s, Series) self.assertEqual(2, len(s.data)) self.assertEqual(None, s.get_first_value()) self.assertEqual(2, s.get_last_value())
def test_insert_retrieve_versioning(self): test_status = 'pyapi.status' now = datetime.now() series = Series(ENTITY, VERSION_METRIC) val = random.randint(0, VALUE - 1) series.add_samples( Sample(value=val, time=now - timedelta(seconds=2), version={'status': test_status})) sf = SeriesFilter(metric=VERSION_METRIC) ef = EntityFilter(entity=ENTITY) df = DateFilter(start_date=now - timedelta(hours=1), end_date=now) vf = VersioningFilter(versioned=True) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df, versioning_filter=vf) successful = self.service.insert(series) time.sleep(WAIT_TIME) series, = self.service.query(query) # print(series) last_sample = series.data[-1] self.assertTrue(successful) self.assertEqual(last_sample.v, val) self.assertIsNotNone(last_sample.version) self.assertEqual(last_sample.version['status'], test_status)
def test_forecast(self): now = datetime.now() sf = SeriesFilter(metric=METRIC, tags={TAG: [TAG_VALUE]}, type=models.SeriesType.FORECAST) ef = EntityFilter(entity=ENTITY) df = DateFilter(start_date=now - timedelta(hours=1), end_date=now) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df) series = self.svc.query(query) self.assertEqual(series[0].type, models.SeriesType.FORECAST)
def test_group(self): time.sleep(1) insert_series_sample(self.svc, VALUE - 1) time.sleep(WAIT_TIME) now = datetime.now() sf = SeriesFilter(metric=METRIC, tags={TAG: [TAG_VALUE]}) ef = EntityFilter(entity=ENTITY) df = DateFilter(start_date=now - timedelta(hours=1), end_date=now) tf = TransformationFilter(group=Group(type=AggregateType.COUNT, period={'count': 1, 'unit': TimeUnit.SECOND})) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df, transformation_filter=tf) series = self.svc.query(query) self.assertEqual(series[0].get_last_value(), 1)
def test_rate(self): v1 = 5 v2 = 3 insert_series_sample(self.svc, v1, v2) time.sleep(WAIT_TIME + 2) now = datetime.now() sf = SeriesFilter(metric=METRIC, tags={TAG: [TAG_VALUE]}) ef = EntityFilter(entity=ENTITY) df = DateFilter(start_date=now - timedelta(hours=1), end_date=now) tf = TransformationFilter(rate=Rate(counter=False)) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df, transformation_filter=tf) series = self.svc.query(query) self.assertEqual(int(series[0].get_last_value()), v2 - v1)
def test_insert_retrieve_series(self): val = random.randint(0, VALUE - 1) insert_series_sample(self.service, val) time.sleep(WAIT_TIME + 2) now = datetime.now() sf = SeriesFilter(metric=METRIC, tags={TAG: [TAG_VALUE]}) ef = EntityFilter(entity=ENTITY) df = DateFilter(start_date=now - timedelta(hours=1), end_date=now) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df) series = self.service.query(query) # for s in series: # print(s) self.assertIsNotNone(series) self.assertGreater(len(series), 0) s = series[0] self.assertIsInstance(s, Series) self.assertGreater(len(s.data), 0) self.assertEqual(s.get_last_value(), val)
# Connect to ATSD server #connection = connect('/path/to/connection.properties') connection = connect_url('https://atsd_hostname:8443', 'username', 'password') # disable deleting inappropriate values deleteValues = False # specify metric name metric_name = "ca.daily.reservoir_storage_af" svc = SeriesService(connection) metrics_service = MetricsService(connection) # query series with current metric and all entities sf = SeriesFilter(metric=metric_name) ef = EntityFilter(entity='*') df = DateFilter(start_date="1970-01-01T00:00:00Z", end_date=datetime.now()) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df) series = svc.query(query) if deleteValues: print('Inappropriate values to be deleted.\n') else: print('Leave as is inappropriate values.\n') print('metric,entity,tags,data') for s in series: # filter non-positive values s.data = [ sample for sample in s.data if sample.v is not None and sample.v <= 0
# Connect to an ATSD server connection = connect_url('https://atsd_hostname:8443', 'user', 'password') # set series metric = 'm-to-delete' entity = 'e-to-delete' tags = {'tag_key_1': 'tag_value_1', 'tag_key_2': 'tag_value_2'} # specify date interval startDate = "2018-10-01T00:00:00Z" endDate = "2018-10-02T00:00:00Z" series_service = SeriesService(connection) # query the series to be deleted, use exactMatch to exclude not specified tags sf = SeriesFilter(metric=metric, tags=tags, exact_match=True) ef = EntityFilter(entity=entity) df = DateFilter(start_date=startDate, end_date=endDate) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df) series_list = series_service.query(query) if len(series_list) > 1: raise Exception('There are multiple series meet the requirements') series = series_list[0] # check data existence if len(series.data) == 0: print('No data in required interval') else: # replace value of samples with nan
# Series filter metric = args.metric entity = args.entity # Date interval startDate = args.start endDate = args.end # Exclude samples with NaN values (NaN represents deleted values) expr = '!Float.isNaN(value)' series_service = SeriesService(connection) # Query the series to be deleted, use exactMatch=False to include all tags sf = SeriesFilter(metric=metric, exact_match=False) ef = EntityFilter(entity=entity) df = DateFilter(start_date=startDate, end_date=endDate) vf = SampleFilter(expr) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df, sample_filter=vf) series_list = series_service.query(query) if len(series_list) == 0: print("No series are found") else: for series in series_list: # Replace value of samples with NaN if len(series.data) == 0:
logging.warning("'" + metric + "' metric does not exist") exit(1) series_service = SeriesService(connection) def insert_or_warning(series_to_insert): if not dry_run: series_service.insert(series_to_insert) else: logging.warning("Dry run enabled, series are not inserted.") dst_entity_filter = EntityFilter(dst_entity) dst_date_filter = DateFilter(start_date, 'now') series_filter = SeriesFilter(metric, tag_expression=tag_expression) limit_control = ControlFilter(limit=1, direction="ASC") sample_filter = SampleFilter("!Double.isNaN(value)") dst_series_query = SeriesQuery(series_filter, dst_entity_filter, dst_date_filter, control_filter=limit_control, sample_filter=sample_filter) dst_series = series_service.query(dst_series_query) if no_data(dst_series): logging.warning("No destination series found for '%s' entity, '%s' metric" % (dst_entity, metric)) exit(1) def err(tags, time=None, entity=source_entity): error_msg = "No series found for '" + entity + "' entity, '" + metric + "' metric and '" + str(tags) + "'" if time is not None: error_msg += " before " + str(time)
key for key in dir(AggregateType) if not key.startswith('_') ] else: aggregate_types = aggregate_types.split('+') # try to retrieve series from the previous query expected_series = [] with open(filename) as fp: line = fp.readline() while line: series = Series.from_dict(ast.literal_eval(line)) expected_series.extend(series) line = fp.readline() # prepare series query sf = SeriesFilter(metric=metric_name, tags=tags, exact_match=exact_match) ef = EntityFilter(entity=entity_name) df = DateFilter(start_date=start_date, end_date=end_date) aggregate = Aggregate(period={ 'count': 7, 'unit': TimeUnit.MINUTE }, threshold={ 'min': 10, 'max': 90 }, types=aggregate_types, order=1) tf = TransformationFilter(aggregate=aggregate) # add rate and group to the transformation filter if specified instead of query
filename = 'data-availability.csv' with open(filename) as fp: line = fp.readline() while line: # skip commented lines if line.startswith('#'): line = fp.readline() continue metric_name, entity_name, interval, end_date, forecast_name, comments = line.split( ',') count, unit = interval.split('-') sf = SeriesFilter(metric=metric_name) ef = EntityFilter(entity=entity_name) df = DateFilter(end_date=end_date, interval={ 'count': count, 'unit': unit }) ff = None if forecast_name: sf.set_type('FORECAST') if forecast_name != '-': ff = ForecastFilter(forecast_name=forecast_name) query = SeriesQuery(series_filter=sf, entity_filter=ef,
filename = 'data-availability.csv' with open(filename) as fp: line = fp.readline() while line: # skip commented lines if line.startswith('#'): line = fp.readline() continue metric_name, entity_name, interval, end_date, forecast_name, comments = line.split(',') count, unit = interval.split('-') sf = SeriesFilter(metric=metric_name) ef = EntityFilter(entity=entity_name) df = DateFilter(end_date=end_date, interval={'count': count, 'unit': unit}) ff = None if forecast_name: sf.set_type('FORECAST') if forecast_name != '-': ff = ForecastFilter(forecast_name=forecast_name) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df, forecast_filter=ff) series = svc.query(query) if series: if not series[0].data: print('No data for: %s' % line)
''' # Connect to an ATSD server connection = connect_url('https://atsd_hostname:8443', 'user', 'password') svc = SeriesService(connection) metrics_service = MetricsService(connection) # specify source and destination metric names metric_src = 'metric_src' metric_dst = 'metric_dst' # copy series with all entities, specific entity name can be set instead entity = '*' # query series with required metric and all entities sf = SeriesFilter(metric=metric_src) ef = EntityFilter(entity='*') df = DateFilter(start_date='1970-01-01T00:00:00Z', end_date=datetime.now()) query = SeriesQuery(series_filter=sf, entity_filter=ef, date_filter=df) series = svc.query(query) # copy metric meta information metric = metrics_service.get(metric_src) metric.name = metric_dst metrics_service.create_or_replace(metric) for s in series: s.metric = metric_dst s.aggregate = None svc.insert(s)