def _store_unaggregated_timeseries(self, metrics_and_data, version=3): """Store unaggregated timeseries. :param metrics_and_data: A list of (metric, serialized_data) tuples :param version: Storage engine data format version """ utils.parallel_map( utils.return_none_on_failure( self._store_unaggregated_timeseries_unbatched), ((metric, data, version) for metric, data in metrics_and_data))
def add_measures_batch(self, metrics_and_measures): """Add a batch of measures for some metrics. :param metrics_and_measures: A dict where keys are metric objects and values are a list of :py:class:`gnocchi.incoming.Measure`. """ utils.parallel_map( self._store_new_measures, ((metric, self._encode_measures(measures)) for metric, measures in six.iteritems(metrics_and_measures)))
def add_measures_batch(self, metrics_and_measures): """Add a batch of measures for some metrics. :param metrics_and_measures: A dict where keys are metrics and value are measure. """ utils.parallel_map( self._store_new_measures, ((metric, self._encode_measures(measures)) for metric, measures in six.iteritems(metrics_and_measures)))
def _delete_metric_splits(self, metrics_keys_aggregations, version=3): """Delete splits of metrics. :param metrics_keys_aggregations: A dict where keys are `storage.Metric` and values are lists of (key, aggregation) tuples. """ utils.parallel_map( utils.return_none_on_failure(self._delete_metric_splits_unbatched), ((metric, key, aggregation) for metric, keys_and_aggregations in six.iteritems(metrics_keys_aggregations) for key, aggregation in keys_and_aggregations))
def test_parallel_map_four(self): utils.parallel_map.MAX_WORKERS = 4 starmap = itertools.starmap with mock.patch("itertools.starmap") as sm: sm.side_effect = starmap self.assertEqual([1, 2, 3], utils.parallel_map(lambda x: x, [[1], [2], [3]])) sm.assert_not_called()
def _map_add_measures(bound_timeserie): # NOTE (gordc): bound_timeserie is entire set of # unaggregated measures matching largest # granularity. the following takes only the points # affected by new measures for specific granularity tstamp = max(bound_timeserie.first, measures['timestamps'][0]) new_first_block_timestamp = bound_timeserie.first_block_timestamp() computed_points['number'] = len(bound_timeserie) for d in definition: ts = bound_timeserie.group_serie( d.granularity, carbonara.round_timestamp(tstamp, d.granularity)) utils.parallel_map( self._add_measures, ((aggregation, d, metric, ts, current_first_block_timestamp, new_first_block_timestamp) for aggregation in agg_methods))
def _store_metric_splits(self, metrics_keys_aggregations_data_offset, version=3): """Store metric splits. Store a bunch of splits for some metrics. :param metrics_keys_aggregations_data_offset: A dict where keys are `storage.Metric` and values are a list of (key, aggregation, data, offset) tuples. :param version: Storage engine format version. """ utils.parallel_map( self._store_metric_splits_unbatched, ((metric, key, aggregation, data, offset, version) for metric, keys_aggregations_data_offset in six.iteritems(metrics_keys_aggregations_data_offset) for key, aggregation, data, offset in keys_aggregations_data_offset))
def _get_splits(self, metrics_aggregations_keys, version=3): results = collections.defaultdict( lambda: collections.defaultdict(list)) for metric, aggregation, split in utils.parallel_map( lambda m, k, a, v: (m, a, self._get_splits_unbatched(m, k, a, v)), # noqa ((metric, key, aggregation, version) for metric, aggregations_and_keys in six.iteritems(metrics_aggregations_keys) for aggregation, keys in six.iteritems(aggregations_and_keys) for key in keys)): results[metric][aggregation].append(split) return results
def _get_measures_timeserie(self, metric, aggregation, granularity, from_timestamp=None, to_timestamp=None): # Find the number of point for d in metric.archive_policy.definition: if d.granularity == granularity: points = d.points break else: raise GranularityDoesNotExist(metric, granularity) all_keys = None try: all_keys = self._list_split_keys_for_metric( metric, aggregation, granularity) except MetricDoesNotExist: for d in metric.archive_policy.definition: if d.granularity == granularity: return carbonara.AggregatedTimeSerie( sampling=granularity, aggregation_method=aggregation, max_size=d.points) raise GranularityDoesNotExist(metric, granularity) if from_timestamp: from_timestamp = carbonara.SplitKey.from_timestamp_and_sampling( from_timestamp, granularity) if to_timestamp: to_timestamp = carbonara.SplitKey.from_timestamp_and_sampling( to_timestamp, granularity) timeseries = list( filter( lambda x: x is not None, utils.parallel_map( self._get_measures_and_unserialize, ((metric, key, aggregation) for key in sorted(all_keys) if ((not from_timestamp or key >= from_timestamp) and ( not to_timestamp or key <= to_timestamp)))))) return carbonara.AggregatedTimeSerie.from_timeseries( sampling=granularity, aggregation_method=aggregation, timeseries=timeseries, max_size=points)
def _get_or_create_unaggregated_timeseries(self, metrics, version=3): """Get the unaggregated timeserie of metrics. If the metrics does not exist, it is created. :param metrics: A list of metrics. :param version: The storage format version number. """ return dict( six.moves.zip( metrics, utils.parallel_map( utils.return_none_on_failure( self._get_or_create_unaggregated_timeseries_unbatched), ((metric, version) for metric in metrics))))
def get_aggregated_measures(self, metric, aggregations, from_timestamp=None, to_timestamp=None): """Get aggregated measures from a metric. :param metric: The metric measured. :param aggregations: The aggregations to retrieve. :param from timestamp: The timestamp to get the measure from. :param to timestamp: The timestamp to get the measure to. """ keys = self._list_split_keys(metric, aggregations) timeseries = utils.parallel_map( self._get_measures_timeserie, ((metric, agg, keys[agg], from_timestamp, to_timestamp) for agg in aggregations)) return { agg: ts.fetch(from_timestamp, to_timestamp) for agg, ts in six.moves.zip(aggregations, timeseries) }
def get_measures(self, metric, from_timestamp=None, to_timestamp=None, aggregation='mean', granularity=None, resample=None): """Get a measure to a metric. :param metric: The metric measured. :param from timestamp: The timestamp to get the measure from. :param to timestamp: The timestamp to get the measure to. :param aggregation: The type of aggregation to retrieve. :param granularity: The granularity to retrieve. :param resample: The granularity to resample to. """ if aggregation not in metric.archive_policy.aggregation_methods: if granularity is None: granularity = metric.archive_policy.definition[0].granularity raise AggregationDoesNotExist(metric, aggregation, granularity) if granularity is None: agg_timeseries = utils.parallel_map( self._get_measures_timeserie, ((metric, aggregation, ap.granularity, from_timestamp, to_timestamp) for ap in reversed(metric.archive_policy.definition))) else: agg_timeseries = [ self._get_measures_timeserie(metric, aggregation, granularity, from_timestamp, to_timestamp) ] if resample and granularity: agg_timeseries = list( map(lambda agg: agg.resample(resample), agg_timeseries)) return list( itertools.chain(*[ ts.fetch(from_timestamp, to_timestamp) for ts in agg_timeseries ]))
def get_measures(self, metric, granularities, from_timestamp=None, to_timestamp=None, aggregation='mean', resample=None): """Get a measure to a metric. :param metric: The metric measured. :param granularities: The granularities to retrieve. :param from timestamp: The timestamp to get the measure from. :param to timestamp: The timestamp to get the measure to. :param aggregation: The type of aggregation to retrieve. :param resample: The granularity to resample to. """ aggregations = [] for g in sorted(granularities, reverse=True): agg = metric.archive_policy.get_aggregation(aggregation, g) if agg is None: raise AggregationDoesNotExist(metric, aggregation, g) aggregations.append(agg) agg_timeseries = utils.parallel_map( self._get_measures_timeserie, ((metric, ag, from_timestamp, to_timestamp) for ag in aggregations)) if resample: agg_timeseries = list( map(lambda agg: agg.resample(resample), agg_timeseries)) return list( itertools.chain(*[ ts.fetch(from_timestamp, to_timestamp) for ts in agg_timeseries ]))
def search_value(self, metrics, query, from_timestamp=None, to_timestamp=None, aggregation='mean', granularity=None): """Search for an aggregated value that realizes a predicate. :param metrics: The list of metrics to look into. :param query: The query being sent. :param from_timestamp: The timestamp to get the measure from. :param to_timestamp: The timestamp to get the measure to. :param aggregation: The type of aggregation to retrieve. :param granularity: The granularity to retrieve. """ granularity = granularity or [] predicate = MeasureQuery(query) results = utils.parallel_map( self._find_measure, [(metric, aggregation, gran, predicate, from_timestamp, to_timestamp) for metric in metrics for gran in granularity or ( defin.granularity for defin in metric.archive_policy.definition)]) result = collections.defaultdict(list) for r in results: for metric, metric_result in six.iteritems(r): result[metric].extend(metric_result) # Sort the result for metric, r in six.iteritems(result): # Sort by timestamp asc, granularity desc r.sort(key=lambda t: (t[0], -t[1])) return result
def get_measures(storage, references, operations, from_timestamp=None, to_timestamp=None, granularities=None, needed_overlap=100.0, fill=None): """Get aggregated measures of multiple entities. :param storage: The storage driver. :param metrics_and_aggregations: List of metric+agg_method tuple measured to aggregate. :param from timestamp: The timestamp to get the measure from. :param to timestamp: The timestamp to get the measure to. :param granularities: The granularities to retrieve. :param fill: The value to use to fill in missing data in series. """ if granularities is None: all_granularities = ( definition.granularity for ref in references for definition in ref.metric.archive_policy.definition ) # granularities_in_common granularities = [ g for g, occurrence in six.iteritems( collections.Counter(all_granularities)) if occurrence == len(references) ] if not granularities: raise exceptions.UnAggregableTimeseries( list((ref.name, ref.aggregation) for ref in references), 'No granularity match') references_with_missing_granularity = [] for ref in references: if (ref.aggregation not in ref.metric.archive_policy.aggregation_methods): raise gnocchi_storage.AggregationDoesNotExist( ref.metric, ref.aggregation, # Use the first granularity, that should be good enough since # they are all missing anyway ref.metric.archive_policy.definition[0].granularity) available_granularities = [ d.granularity for d in ref.metric.archive_policy.definition ] for g in granularities: if g not in available_granularities: references_with_missing_granularity.append( (ref.name, ref.aggregation, g)) break if references_with_missing_granularity: raise exceptions.UnAggregableTimeseries( references_with_missing_granularity, "Granularities are missing") tss = utils.parallel_map(_get_measures_timeserie, [(storage, ref, g, from_timestamp, to_timestamp) for ref in references for g in granularities]) return aggregated(tss, operations, from_timestamp, to_timestamp, needed_overlap, fill)
def get_measures(storage, references, operations, from_timestamp=None, to_timestamp=None, granularities=None, needed_overlap=100.0, fill=None): """Get aggregated measures of multiple entities. :param storage: The storage driver. :param metrics_and_aggregations: List of metric+agg_method tuple measured to aggregate. :param from timestamp: The timestamp to get the measure from. :param to timestamp: The timestamp to get the measure to. :param granularities: The granularities to retrieve. :param fill: The value to use to fill in missing data in series. """ if granularities is None: all_granularities = ( definition.granularity for ref in references for definition in ref.metric.archive_policy.definition) # granularities_in_common granularities = [ g for g, occurrence in six.iteritems( collections.Counter(all_granularities)) if occurrence == len(references) ] if not granularities: raise exceptions.UnAggregableTimeseries( list((ref.name, ref.aggregation) for ref in references), 'No granularity match') references_with_missing_granularity = [] for ref in references: if (ref.aggregation not in ref.metric.archive_policy.aggregation_methods): raise gnocchi_storage.AggregationDoesNotExist( ref.metric, ref.aggregation, # Use the first granularity, that should be good enough since # they are all missing anyway ref.metric.archive_policy.definition[0].granularity) available_granularities = [ d.granularity for d in ref.metric.archive_policy.definition ] for g in granularities: if g not in available_granularities: references_with_missing_granularity.append( (ref.name, ref.aggregation, g)) break if references_with_missing_granularity: raise exceptions.UnAggregableTimeseries( references_with_missing_granularity, "Granularities are missing") tss = utils.parallel_map(_get_measures_timeserie, [(storage, ref, g, from_timestamp, to_timestamp) for ref in references for g in granularities]) return aggregated(tss, operations, from_timestamp, to_timestamp, needed_overlap, fill)
def get_measures(storage, metrics_and_aggregations, operations, from_timestamp=None, to_timestamp=None, granularity=None, needed_overlap=100.0, fill=None, ref_identifier="id"): """Get aggregated measures of multiple entities. :param storage: The storage driver. :param metrics_and_aggregations: List of metric+agg_method tuple measured to aggregate. :param from timestamp: The timestamp to get the measure from. :param to timestamp: The timestamp to get the measure to. :param granularity: The granularity to retrieve. :param fill: The value to use to fill in missing data in series. """ references_with_missing_granularity = [] for (metric, aggregation) in metrics_and_aggregations: if aggregation not in metric.archive_policy.aggregation_methods: raise gnocchi_storage.AggregationDoesNotExist(metric, aggregation) if granularity is not None: for d in metric.archive_policy.definition: if d.granularity == granularity: break else: references_with_missing_granularity.append( (getattr(metric, ref_identifier), aggregation)) if references_with_missing_granularity: raise exceptions.UnAggregableTimeseries( references_with_missing_granularity, "granularity '%d' is missing" % utils.timespan_total_seconds(granularity)) if granularity is None: granularities = (definition.granularity for (metric, aggregation) in metrics_and_aggregations for definition in metric.archive_policy.definition) granularities_in_common = [ g for g, occurrence in six.iteritems( collections.Counter(granularities)) if occurrence == len(metrics_and_aggregations) ] if not granularities_in_common: raise exceptions.UnAggregableTimeseries( list((str(getattr(m, ref_identifier)), a) for (m, a) in metrics_and_aggregations), 'No granularity match') else: granularities_in_common = [granularity] tss = utils.parallel_map( _get_measures_timeserie, [(storage, metric, aggregation, ref_identifier, g, from_timestamp, to_timestamp) for (metric, aggregation) in metrics_and_aggregations for g in granularities_in_common]) return aggregated(tss, operations, from_timestamp, to_timestamp, needed_overlap, fill)
def test_parallel_map_four(self): utils.parallel_map.NUM_WORKERS = 4 self.assertEqual([1, 2, 3], utils.parallel_map(lambda x: x, [[1], [2], [3]]))
def _get_measures(self, metric, keys, aggregation, version=3): return utils.parallel_map(self._get_measures_unbatched, ((metric, key, aggregation, version) for key in keys))
def _delete_metric_splits(self, metric, keys_and_aggregations, version=3): utils.parallel_map( utils.return_none_on_failure(self._delete_metric_splits_unbatched), ((metric, key, aggregation) for key, aggregation in keys_and_aggregations))