#!/usr/bin/env python # encoding: utf-8 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import sys from moztelemetry.spark import get_pings from moztelemetry.histogram import cached_exponential_buckets from collections import defaultdict # Simple measurement and count histogram labels simple_measures_labels = cached_exponential_buckets(1, 30000, 50) count_histogram_labels = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 23, 25, 27, 29, 31, 34, 37, 40, 43, 46, 50, 54, 58, 63, 68, 74, 80, 86, 93, 101, 109, 118, 128, 138, 149, 161, 174, 188, 203, 219, 237, 256, 277, 299, 323, 349, 377, 408, 441, 477, 516, 558, 603, 652, 705, 762, 824, 891, 963, 1041, 1125, 1216, 1315, 1422, 1537, 1662, 1797, 1943, 2101, 2271, 2455, 2654, 2869, 3102, 3354, 3626, 3920, 4238, 4582, 4954, 5356, 5791, 6261, 6769, 7318, 7912, 8554, 9249, 10000] def aggregate_metrics(sc, channels, submission_date, fraction=1): """ Returns the build-id and submission date aggregates for a given submission date. :param sc: A SparkContext instance :param channel: Either the name of a channel or a list/tuple of names :param submission-date: The submission date for which the data will be aggregated :param fraction: An approximative fraction of submissions to consider for aggregation """ if not isinstance(channels, (tuple, list)): channels = [channels] channels = set(channels) rdds = [get_pings(sc, channel=ch, submission_date=submission_date, doc_type="saved_session", schema="v4", fraction=fraction) for ch in channels]
#!/usr/bin/env python # encoding: utf-8 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from moztelemetry.spark import get_pings from moztelemetry.histogram import cached_exponential_buckets from collections import defaultdict # Simple measurement and count histogram labels simple_measures_labels = cached_exponential_buckets(1, 30000, 50) count_histogram_labels = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 23, 25, 27, 29, 31, 34, 37, 40, 43, 46, 50, 54, 58, 63, 68, 74, 80, 86, 93, 101, 109, 118, 128, 138, 149, 161, 174, 188, 203, 219, 237, 256, 277, 299, 323, 349, 377, 408, 441, 477, 516, 558, 603, 652, 705, 762, 824, 891, 963, 1041, 1125, 1216, 1315, 1422, 1537, 1662, 1797, 1943, 2101, 2271, 2455, 2654, 2869, 3102, 3354, 3626, 3920, 4238, 4582, 4954, 5356, 5791, 6261, 6769, 7318, 7912, 8554, 9249, 10000] def aggregate_metrics(sc, channels, submission_date, fraction=1): """ Returns the build-id and submission date aggregates for a given submission date. :param sc: A SparkContext instance :param channel: Either the name of a channel or a list/tuple of names :param submission-date: The submission date for which the data will be aggregated :param fraction: An approximative fraction of submissions to consider for aggregation """ if not isinstance(channels, (tuple, list)): channels = [channels] channels = set(channels) rdds = [get_pings(sc, channel=ch, submission_date=submission_date, doc_type="saved_session", schema="v4", fraction=fraction) for ch in channels] pings = reduce(lambda x, y: x.union(y), rdds) return _aggregate_metrics(pings)
#!/usr/bin/env python # encoding: utf-8 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import sys from moztelemetry.dataset import Dataset from moztelemetry.histogram import cached_exponential_buckets from collections import defaultdict # Simple measurement, count histogram, and numeric scalars labels & prefixes SIMPLE_MEASURES_LABELS = cached_exponential_buckets(1, 30000, 50) COUNT_HISTOGRAM_LABELS = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 23, 25, 27, 29, 31, 34, 37, 40, 43, 46, 50, 54, 58, 63, 68, 74, 80, 86, 93, 101, 109, 118, 128, 138, 149, 161, 174, 188, 203, 219, 237, 256, 277, 299, 323, 349, 377, 408, 441, 477, 516, 558, 603, 652, 705, 762, 824, 891, 963, 1041, 1125, 1216, 1315, 1422, 1537, 1662, 1797, 1943, 2101, 2271, 2455, 2654, 2869, 3102, 3354, 3626, 3920, 4238, 4582, 4954, 5356, 5791, 6261, 6769, 7318, 7912, 8554, 9249, 10000 ] NUMERIC_SCALARS_LABELS = COUNT_HISTOGRAM_LABELS SIMPLE_MEASURES_PREFIX = 'SIMPLE_MEASURES' COUNT_HISTOGRAM_PREFIX = '[[COUNT]]' NUMERIC_SCALARS_PREFIX = 'SCALARS' SCALAR_MEASURE_MAP = {
#!/usr/bin/env python # encoding: utf-8 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from collections import defaultdict from moztelemetry.dataset import Dataset from moztelemetry.histogram import cached_exponential_buckets # Simple measurement, count histogram, and numeric scalars labels & prefixes SIMPLE_MEASURES_LABELS = cached_exponential_buckets(1, 30000, 50) COUNT_HISTOGRAM_LABELS = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 23, 25, 27, 29, 31, 34, 37, 40, 43, 46, 50, 54, 58, 63, 68, 74, 80, 86, 93, 101, 109, 118, 128, 138, 149, 161, 174, 188, 203, 219, 237, 256, 277, 299, 323, 349, 377, 408, 441, 477, 516, 558, 603, 652, 705, 762, 824, 891, 963, 1041, 1125, 1216, 1315, 1422, 1537, 1662, 1797, 1943, 2101, 2271, 2455, 2654, 2869, 3102, 3354, 3626, 3920, 4238, 4582, 4954, 5356, 5791, 6261, 6769, 7318, 7912, 8554, 9249, 10000, ] NUMERIC_SCALARS_LABELS = COUNT_HISTOGRAM_LABELS SIMPLE_MEASURES_PREFIX = 'SIMPLE_MEASURES' COUNT_HISTOGRAM_PREFIX = '[[COUNT]]' NUMERIC_SCALARS_PREFIX = 'SCALARS' SCALAR_MEASURE_MAP = { SIMPLE_MEASURES_PREFIX: SIMPLE_MEASURES_LABELS, COUNT_HISTOGRAM_PREFIX: COUNT_HISTOGRAM_LABELS,