def __init__(self, test_name, test_description=None):
# oc will automatically search for the ENV VAR 'APPLICATIONINSIGHTS_CONNECTION_STRING'
self.exporter = metrics_exporter.new_metrics_exporter()
self.stats = stats_module.stats
self.view_manager = self.stats.view_manager
self.stats_recorder = self.stats.stats_recorder
self.azure_logger = get_azure_logger(test_name)
self.name = test_name
self.desc = test_description
events_measure_name = "The number of events handled by " + self.name
events_measure_desc = "The number of events handled by " + self.desc if self.desc else None
memory_measure_name = "memory usage percentage for " + self.name
memory_measure_desc = "memory usage percentage for " + self.desc if self.desc else None
cpu_measure_name = "cpu usage percentage for " + self.name
cpu_measure_desc = "cpu usage percentage for " + self.desc if self.desc else None
error_measure_name = "error count for " + self.name
error_measure_desc = "The number of errors happened while running the test for " + self.desc if self.desc else None
self.events_measure = measure_module.MeasureInt(
events_measure_name, events_measure_desc, "events")
self.memory_measure = measure_module.MeasureFloat(
memory_measure_name, memory_measure_desc)
self.cpu_measure = measure_module.MeasureFloat(cpu_measure_name,
cpu_measure_desc)
self.error_measure = measure_module.MeasureInt(error_measure_name,
error_measure_desc)
self.events_measure_view = view_module.View(
events_measure_name, events_measure_desc, [], self.events_measure,
aggregation_module.SumAggregation())
self.memory_measure_view = view_module.View(
memory_measure_name, memory_measure_desc, [], self.memory_measure,
aggregation_module.LastValueAggregation())
self.cpu_measure_view = view_module.View(
cpu_measure_name, cpu_measure_desc, [], self.cpu_measure,
aggregation_module.LastValueAggregation())
self.error_measure_view = view_module.View(
error_measure_name, error_measure_desc, [], self.error_measure,
aggregation_module.CountAggregation())
self.view_manager.register_view(self.events_measure_view)
self.view_manager.register_view(self.memory_measure_view)
self.view_manager.register_view(self.cpu_measure_view)
self.view_manager.register_view(self.error_measure_view)
self.mmap = self.stats_recorder.new_measurement_map()
def track_metric(self, metric_name, metric_value):
if self.appinsights_key:
print("Tracking metric:" + metric_name + ", Value: " +
str(metric_value))
if not metric_name in self.metrics:
metrics_measure = measure_module.MeasureInt(
metric_name, metric_name, metric_name)
metrics_view = view_module.View(
metric_name,
metric_name,
[],
metrics_measure,
aggregation_module.LastValueAggregation(
value=metric_value),
)
view_manager.register_view(metrics_view)
mmap = stats_recorder.new_measurement_map()
tmap = tag_map_module.TagMap()
self.metrics[metric_name] = {
"measure": metrics_measure,
"measurement_map": mmap,
"tag_map": tmap,
}
measure = self.metrics[metric_name]["measure"]
mmap = self.metrics[metric_name]["measurement_map"]
tmap = self.metrics[metric_name]["tag_map"]
print("Putting metric:" + metric_name + ", Value: " +
str(metric_value))
mmap.measure_int_put(measure, metric_value)
mmap.record(tmap)
def track_metric(self, metric_name, metric_value):
try:
if self.appinsights_key:
if not metric_name in self.metrics:
metrics_measure = measure_module.MeasureInt(
metric_name, metric_name, metric_name)
metrics_view = view_module.View(
metric_name,
metric_name,
[],
metrics_measure,
aggregation_module.LastValueAggregation(
value=metric_value),
)
self.view_manager.register_view(metrics_view)
mmap = self.stats_recorder.new_measurement_map()
tmap = tag_map_module.TagMap()
self.metrics[metric_name] = {
"measure": metrics_measure,
"measurement_map": mmap,
"tag_map": tmap,
}
measure = self.metrics[metric_name]["measure"]
mmap = self.metrics[metric_name]["measurement_map"]
tmap = self.metrics[metric_name]["tag_map"]
mmap.measure_int_put(measure, metric_value)
mmap.record(tmap)
except Exception as e:
print("Exception when tracking a metric:")
print(e)
def track_metric(self, metric_name, metric_value):
if (self.appinsights_key):
print("Tracking metric:" + metric_name + ", Value: " +
str(metric_value))
if (not metric_name in self.metrics):
metrics_measure = measure_module.MeasureInt(
metric_name, metric_name, metric_name)
metrics_view = view_module.View(
metric_name, metric_name, [], metrics_measure,
aggregation_module.LastValueAggregation(
value=metric_value))
view_manager.register_view(metrics_view)
mmap = stats_recorder.new_measurement_map()
tmap = tag_map_module.TagMap()
self.metrics[metric_name] = {
'measure': metrics_measure,
'measurement_map': mmap,
'tag_map': tmap
}
measure = self.metrics[metric_name]['measure']
mmap = self.metrics[metric_name]['measurement_map']
tmap = self.metrics[metric_name]['tag_map']
print("Putting metric:" + metric_name + ", Value: " +
str(metric_value))
mmap.measure_int_put(measure, metric_value)
mmap.record(tmap)
def test_constructor_defaults(self):
name = "testName"
description = "testMeasure"
measure = measure_module.MeasureInt(name=name, description=description)
self.assertEqual(None, measure.unit)
def track_metric(self, metric_name, metric_value):
try:
if (self.appinsights_key):
if (not metric_name in self.metrics):
metrics_measure = measure_module.MeasureInt(
metric_name, metric_name, metric_name)
metrics_view = view_module.View(
metric_name, metric_name, [], metrics_measure,
aggregation_module.LastValueAggregation(
value=metric_value))
self.view_manager.register_view(metrics_view)
mmap = self.stats_recorder.new_measurement_map()
tmap = tag_map_module.TagMap()
self.metrics[metric_name] = {
'measure': metrics_measure,
'measurement_map': mmap,
'tag_map': tmap
}
measure = self.metrics[metric_name]['measure']
mmap = self.metrics[metric_name]['measurement_map']
tmap = self.metrics[metric_name]['tag_map']
mmap.measure_int_put(measure, metric_value)
mmap.record(tmap)
except Exception as e:
print('Exception when tracking a metric:')
print(e)
def test_stats_record_async(self):
# We are using sufix in order to prevent cached objects
sufix = str(os.getpid())
tag_key = "SampleKeyAsyncTest%s" % sufix
measure_name = "SampleMeasureNameAsyncTest%s" % sufix
measure_description = "SampleDescriptionAsyncTest%s" % sufix
view_name = "SampleViewNameAsyncTest%s" % sufix
view_description = "SampleViewDescriptionAsyncTest%s" % sufix
FRONTEND_KEY_ASYNC = tag_key_module.TagKey(tag_key)
VIDEO_SIZE_MEASURE_ASYNC = measure_module.MeasureInt(
measure_name, measure_description, "By")
VIDEO_SIZE_VIEW_NAME_ASYNC = view_name
VIDEO_SIZE_DISTRIBUTION_ASYNC =\
aggregation_module.DistributionAggregation(
[0.0, 16.0 * MiB, 256.0 * MiB]
)
VIDEO_SIZE_VIEW_ASYNC = view_module.View(
VIDEO_SIZE_VIEW_NAME_ASYNC, view_description, [FRONTEND_KEY_ASYNC],
VIDEO_SIZE_MEASURE_ASYNC, VIDEO_SIZE_DISTRIBUTION_ASYNC)
stats = stats_module.Stats()
view_manager = stats.view_manager
stats_recorder = stats.stats_recorder
exporter = stackdriver.new_stats_exporter(
stackdriver.Options(project_id=PROJECT))
view_manager.register_exporter(exporter)
# Register view.
view_manager.register_view(VIDEO_SIZE_VIEW_ASYNC)
# Sleep for [0, 10] milliseconds to fake work.
time.sleep(random.randint(1, 10) / 1000.0)
# Process video.
# Record the processed video size.
tag_value = tag_value_module.TagValue("1200")
tag_map = tag_map_module.TagMap()
tag_map.insert(FRONTEND_KEY_ASYNC, tag_value)
measure_map = stats_recorder.new_measurement_map()
measure_map.measure_int_put(VIDEO_SIZE_MEASURE_ASYNC, 25 * MiB)
measure_map.record(tag_map)
@retry(wait_fixed=RETRY_WAIT_PERIOD,
stop_max_attempt_number=RETRY_MAX_ATTEMPT)
def get_metric_descriptors(self, exporter, view_description):
name = exporter.client.project_path(PROJECT)
list_metrics_descriptors = exporter.client.list_metric_descriptors(
name)
element = next((element for element in list_metrics_descriptors
if element.description == view_description), None)
self.assertIsNotNone(element)
self.assertEqual(element.description, view_description)
self.assertEqual(element.unit, "By")
get_metric_descriptors(self, exporter, view_description)
def test_constructor_explicit(self):
name = "testName"
description = "testMeasure"
unit = "testUnit"
measure = measure_module.MeasureInt(name=name, description=description, unit=unit)
self.assertEqual("testName", measure.name)
self.assertEqual("testMeasure", measure.description)
self.assertEqual("testUnit", measure.unit)
def test_prometheus_stats(self):
method_key = tag_key_module.TagKey("method")
request_count_measure = measure_module.MeasureInt(
"request_count", "number of requests", "1")
request_count_view_name = "request_count_view"
count_agg = aggregation_module.CountAggregation()
request_count_view = view_module.View(
request_count_view_name,
"number of requests broken down by methods", [method_key],
request_count_measure, count_agg)
stats = stats_module.stats
view_manager = stats.view_manager
stats_recorder = stats.stats_recorder
exporter = prometheus.new_stats_exporter(
prometheus.Options(namespace="opencensus", port=9303))
view_manager.register_exporter(exporter)
view_manager.register_view(request_count_view)
time.sleep(random.randint(1, 10) / 1000.0)
method_value_1 = tag_value_module.TagValue("some method")
tag_map_1 = tag_map_module.TagMap()
tag_map_1.insert(method_key, method_value_1)
measure_map_1 = stats_recorder.new_measurement_map()
measure_map_1.measure_int_put(request_count_measure, 1)
measure_map_1.record(tag_map_1)
method_value_2 = tag_value_module.TagValue("some other method")
tag_map_2 = tag_map_module.TagMap()
tag_map_2.insert(method_key, method_value_2)
measure_map_2 = stats_recorder.new_measurement_map()
measure_map_2.measure_int_put(request_count_measure, 1)
measure_map_2.record(tag_map_2)
measure_map_2.record(tag_map_2)
if sys.version_info > (3, 0):
import urllib.request
contents = urllib.request.urlopen(
"http://localhost:9303/metrics").read()
else:
import urllib2
contents = urllib2.urlopen("http://localhost:9303/metrics").read()
self.assertIn(b'# TYPE opencensus_request_count_view_total counter',
contents)
self.assertIn(
b'opencensus_request_count_view_total'
b'{method="some method"} 1.0', contents)
self.assertIn(
b'opencensus_request_count_view_total'
b'{method="some other method"} 2.0', contents)
def test_prometheus_stats(self):
import random
import time
import sys
from opencensus.stats import aggregation as aggregation_module
from opencensus.stats.exporters import prometheus_exporter as prometheus
from opencensus.stats import measure as measure_module
from opencensus.stats import stats as stats_module
from opencensus.stats import view as view_module
from opencensus.tags import tag_key as tag_key_module
from opencensus.tags import tag_map as tag_map_module
from opencensus.tags import tag_value as tag_value_module
MiB = 1 << 20
FRONTEND_KEY = tag_key_module.TagKey("my.org/keys/frontend")
VIDEO_SIZE_MEASURE = measure_module.MeasureInt(
"my.org/measures/video_size", "size of processed videos", "By")
VIDEO_SIZE_VIEW_NAME = "my.org/views/video_size"
VIDEO_SIZE_DISTRIBUTION = aggregation_module.CountAggregation(
256.0 * MiB)
VIDEO_SIZE_VIEW = view_module.View(VIDEO_SIZE_VIEW_NAME,
"processed video size over time",
[FRONTEND_KEY],
VIDEO_SIZE_MEASURE,
VIDEO_SIZE_DISTRIBUTION)
stats = stats_module.Stats()
view_manager = stats.view_manager
stats_recorder = stats.stats_recorder
exporter = prometheus.new_stats_exporter(prometheus.Options(namespace="opencensus", port=9303))
view_manager.register_exporter(exporter)
view_manager.register_view(VIDEO_SIZE_VIEW)
time.sleep(random.randint(1, 10) / 1000.0)
tag_value = tag_value_module.TagValue(str(random.randint(1, 10000)))
tag_map = tag_map_module.TagMap()
tag_map.insert(FRONTEND_KEY, tag_value)
measure_map = stats_recorder.new_measurement_map()
measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 25 * MiB)
measure_map.record(tag_map)
if sys.version_info > (3, 0):
import urllib.request
contents = urllib.request.urlopen("http://localhost:9303/metrics").read()
else:
import urllib2
contents = urllib2.urlopen("http://localhost:9303/metrics").read()
self.assertIn(b'# TYPE opencensus_my.org/views/video_size counter', contents)
self.assertIn(b'opencensus_my.org/views/video_size 268435456.0', contents)
def test_stats_record_sync(self):
# We are using sufix in order to prevent cached objects
sufix = str(os.getgid())
tag_key = "SampleKeySyncTest%s" % sufix
measure_name = "SampleMeasureNameSyncTest%s" % sufix
measure_description = "SampleDescriptionSyncTest%s" % sufix
view_name = "SampleViewNameSyncTest%s" % sufix
view_description = "SampleViewDescriptionSyncTest%s" % sufix
FRONTEND_KEY = tag_key_module.TagKey(tag_key)
VIDEO_SIZE_MEASURE = measure_module.MeasureInt(measure_name,
measure_description,
"By")
VIDEO_SIZE_VIEW_NAME = view_name
VIDEO_SIZE_DISTRIBUTION = aggregation_module.DistributionAggregation(
[0.0, 16.0 * MiB, 256.0 * MiB])
VIDEO_SIZE_VIEW = view_module.View(VIDEO_SIZE_VIEW_NAME,
view_description, [FRONTEND_KEY],
VIDEO_SIZE_MEASURE,
VIDEO_SIZE_DISTRIBUTION)
stats = stats_module.stats
view_manager = stats.view_manager
stats_recorder = stats.stats_recorder
client = monitoring_v3.MetricServiceClient()
exporter = stackdriver.StackdriverStatsExporter(
options=stackdriver.Options(project_id=PROJECT), client=client)
view_manager.register_exporter(exporter)
# Register view.
view_manager.register_view(VIDEO_SIZE_VIEW)
# Sleep for [0, 10] milliseconds to fake work.
time.sleep(random.randint(1, 10) / 1000.0)
# Process video.
# Record the processed video size.
tag_value = tag_value_module.TagValue("1200")
tag_map = tag_map_module.TagMap()
tag_map.insert(FRONTEND_KEY, tag_value)
measure_map = stats_recorder.new_measurement_map()
measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 25 * MiB)
measure_map.record(tag_map)
exporter.export_metrics(stats_module.stats.get_metrics())
# Sleep for [0, 10] milliseconds to fake wait.
time.sleep(random.randint(1, 10) / 1000.0)
self.check_sd_md(exporter, view_description)
def create_measurement_view(self, measurement_name):
"creates a measurement and a view"
tg_key = tag_key_module.TagKey("TEST_ID")
measurement = measure_module.MeasureInt(
f"gw_m_{measurement_name}_response",
"response time of the home page", "s")
view_name = f"views_{measurement_name}_response"
aggregation = aggregation_module.LastValueAggregation()
view = view_module.View(view_name,
f"glasswall {measurement_name} response time",
[tg_key], measurement, aggregation)
# Register view.
self.view_manager.register_view(view)
return measurement
def test_record_with_attachment(self):
boundaries = [1, 2, 3]
distribution = {1: "test"}
distribution_aggregation = aggregation_module.DistributionAggregation(
boundaries=boundaries, distribution=distribution)
name = "testName"
description = "testMeasure"
unit = "testUnit"
measure = measure_module.MeasureInt(name=name,
description=description,
unit=unit)
description = "testMeasure"
columns = ["key1", "key2"]
view = view_module.View(name=name,
description=description,
columns=columns,
measure=measure,
aggregation=distribution_aggregation)
start_time = datetime.utcnow()
attachments = {"One": "one", "Two": "two"}
end_time = datetime.utcnow()
view_data = view_data_module.ViewData(view=view,
start_time=start_time,
end_time=end_time)
context = mock.Mock
context.map = {'key1': 'val1', 'key2': 'val2'}
time = datetime.utcnow().isoformat() + 'Z'
value = 1
view_data.record(context=context,
value=value,
timestamp=time,
attachments=attachments)
tag_values = view_data.get_tag_values(tags=context.map,
columns=view.columns)
tuple_vals = tuple(tag_values)
self.assertEqual(['val1', 'val2'], tag_values)
self.assertIsNotNone(view_data.tag_value_aggregation_data_map)
self.assertTrue(tuple_vals in view_data.tag_value_aggregation_data_map)
self.assertIsNotNone(
view_data.tag_value_aggregation_data_map[tuple_vals])
self.assertEqual(
attachments, view_data.tag_value_aggregation_data_map[tuple_vals].
exemplars[1].attachments)
def test_stats_record_async(self):
# We are using sufix in order to prevent cached objects
sufix = str(os.getpid())
tag_key = "SampleKeyAsyncTest%s" % sufix
measure_name = "SampleMeasureNameAsyncTest%s" % sufix
measure_description = "SampleDescriptionAsyncTest%s" % sufix
view_name = "SampleViewNameAsyncTest%s" % sufix
view_description = "SampleViewDescriptionAsyncTest%s" % sufix
FRONTEND_KEY_ASYNC = tag_key_module.TagKey(tag_key)
VIDEO_SIZE_MEASURE_ASYNC = measure_module.MeasureInt(
measure_name, measure_description, "By")
VIDEO_SIZE_VIEW_NAME_ASYNC = view_name
VIDEO_SIZE_DISTRIBUTION_ASYNC =\
aggregation_module.DistributionAggregation(
[0.0, 16.0 * MiB, 256.0 * MiB]
)
VIDEO_SIZE_VIEW_ASYNC = view_module.View(
VIDEO_SIZE_VIEW_NAME_ASYNC, view_description, [FRONTEND_KEY_ASYNC],
VIDEO_SIZE_MEASURE_ASYNC, VIDEO_SIZE_DISTRIBUTION_ASYNC)
stats = stats_module.stats
view_manager = stats.view_manager
stats_recorder = stats.stats_recorder
exporter = stackdriver.new_stats_exporter(
stackdriver.Options(project_id=PROJECT))
view_manager.register_exporter(exporter)
# Register view.
view_manager.register_view(VIDEO_SIZE_VIEW_ASYNC)
# Sleep for [0, 10] milliseconds to fake work.
time.sleep(random.randint(1, 10) / 1000.0)
# Process video.
# Record the processed video size.
tag_value = tag_value_module.TagValue("1200")
tag_map = tag_map_module.TagMap()
tag_map.insert(FRONTEND_KEY_ASYNC, tag_value)
measure_map = stats_recorder.new_measurement_map()
measure_map.measure_int_put(VIDEO_SIZE_MEASURE_ASYNC, 25 * MiB)
measure_map.record(tag_map)
# Give the exporter thread enough time to export exactly once
time.sleep(transport.DEFAULT_INTERVAL * 1.5)
self.check_sd_md(exporter, view_description)
def __init__(self, export_metrics=False):
self.nb_check_requests = measure_module.MeasureInt(
"nb_check_requests",
"number of dates check requests for all weeks", "nb")
self.nb_check_requests_view = view_module.View(
"nb_check_requests view",
"number of dates check requests for all weeks", [],
self.nb_check_requests, aggregation_module.CountAggregation())
view_manager.register_view(self.nb_check_requests_view)
self.mmap = stats_recorder.new_measurement_map()
self.tmap = tag_map_module.TagMap()
if export_metrics:
exporter = metrics_exporter.new_metrics_exporter(
connection_string=azure_insights.connection_string)
view_manager.register_exporter(exporter)
def __init__(self, dialog_id: str = None):
super(BookingDialog, self).__init__(dialog_id
or BookingDialog.__name__)
self.add_dialog(TextPrompt(TextPrompt.__name__))
self.add_dialog(ConfirmPrompt(ConfirmPrompt.__name__))
self.add_dialog(DateResolverDialog(DateResolverDialog.__name__))
self.add_dialog(
WaterfallDialog(
WaterfallDialog.__name__,
[
self.destination_step,
self.origin_step,
self.from_date_step,
self.to_date_step,
self.budget_step,
self.confirm_step,
self.final_step,
],
))
self.initial_dialog_id = WaterfallDialog.__name__
self.logger = None
self.stats = stats_module.stats
self.view_manager = self.stats.view_manager
self.stats_recorder = self.stats.stats_recorder
self.bot_measure = measure_module.MeasureInt("botdefects",
"number of bot defects",
"botdefects")
self.bot_view = view_module.View("defect view",
"number of bot defects", [],
self.bot_measure,
aggregation_module.CountAggregation())
self.view_manager.register_view(self.bot_view)
self.mmap = self.stats_recorder.new_measurement_map()
self.tmap = tag_map_module.TagMap()
self.metrics_exporter = None
self.message_history = set()
from flask import Blueprint, request
from recidiviz.big_query import view_update_manager
from recidiviz.utils import monitoring
from recidiviz.utils.auth import authenticate_request
from recidiviz.utils.environment import GCP_PROJECT_STAGING
from recidiviz.utils.metadata import local_project_id_override
from recidiviz.utils.params import get_bool_param_value
from recidiviz.validation.checks.check_resolver import checker_for_validation
from recidiviz.validation.configured_validations import get_all_validations, STATES_TO_VALIDATE
from recidiviz.validation.validation_models import DataValidationJob, DataValidationJobResult
from recidiviz.validation.views import view_config
m_failed_validations = measure.MeasureInt("validation/num_failures",
"The number of failed validations",
"1")
failed_validations_view = view.View(
"recidiviz/validation/num_failures", "The sum of failed validations", [
monitoring.TagKey.REGION, monitoring.TagKey.VALIDATION_CHECK_TYPE,
monitoring.TagKey.VALIDATION_VIEW_ID
], m_failed_validations, aggregation.SumAggregation())
monitoring.register_views([failed_validations_view])
validation_manager_blueprint = Blueprint('validation_manager', __name__)
@validation_manager_blueprint.route('/validate')
@authenticate_request
# limitations under the License.
import time
from opencensus.ext.azure import metrics_exporter
from opencensus.stats import aggregation as aggregation_module
from opencensus.stats import measure as measure_module
from opencensus.stats import stats as stats_module
from opencensus.stats import view as view_module
from opencensus.tags import tag_map as tag_map_module
stats = stats_module.stats
view_manager = stats.view_manager
stats_recorder = stats.stats_recorder
CARROTS_MEASURE = measure_module.MeasureInt("carrots", "number of carrots",
"carrots")
CARROTS_VIEW = view_module.View("carrots_view", "number of carrots", [],
CARROTS_MEASURE,
aggregation_module.CountAggregation())
def main():
# Enable metrics
# Set the interval in seconds in which you want to send metrics
# TODO: you need to specify the instrumentation key in a connection string
# and place it in the APPLICATIONINSIGHTS_CONNECTION_STRING
# environment variable.
exporter = metrics_exporter.new_metrics_exporter()
view_manager.register_exporter(exporter)
view_manager.register_view(CARROTS_VIEW)
ExportBigQueryViewConfig,
ExportViewCollectionConfig,
)
from recidiviz.metrics.export.optimized_metric_big_query_view_exporter import (
OptimizedMetricBigQueryViewExporter, )
from recidiviz.metrics.export.optimized_metric_big_query_view_export_validator import (
OptimizedMetricBigQueryViewExportValidator, )
from recidiviz.metrics.export.view_export_cloud_task_manager import (
ViewExportCloudTaskManager, )
from recidiviz.utils import metadata, monitoring
from recidiviz.utils.auth.gae import requires_gae_auth
from recidiviz.utils.params import get_str_param_value
m_failed_metric_export_validation = measure.MeasureInt(
"bigquery/metric_view_export_manager/metric_view_export_validation_failure",
"Counted every time a set of exported metric views fails validation",
"1",
)
failed_metric_export_validation_view = opencensus_view.View(
"bigquery/metric_view_export_manager/num_metric_view_export_validation_failure",
"The sum of times a set of exported metric views fails validation",
[monitoring.TagKey.REGION, monitoring.TagKey.METRIC_VIEW_EXPORT_NAME],
m_failed_metric_export_validation,
aggregation.SumAggregation(),
)
m_failed_metric_export_job = measure.MeasureInt(
"bigquery/metric_view_export_manager/metric_view_export_job_failure",
"Counted every time a set of exported metric views fails for non-validation reasons",
"1",
def __init__(self, name, description, unit,
tags: List[tag_key_module.TagKey]):
self._measure = measure_module.MeasureInt(name, description, unit)
self._view = view.View(name, description, tags, self.measure,
aggregation.LastValueAggregation())
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
# =============================================================================
"""Contains logic related to EntityEnums."""
import re
from typing import Dict, Optional, Type, TypeVar
from aenum import Enum, EnumMeta
from opencensus.stats import aggregation, measure, view
from recidiviz.common.str_field_utils import normalize
from recidiviz.utils import monitoring
from recidiviz.utils.types import ClsT
m_enum_errors = measure.MeasureInt("converter/enum_error_count",
"The number of enum errors", "1")
enum_errors_view = view.View(
"recidiviz/converter/enum_error_count",
"The sum of enum errors",
[monitoring.TagKey.REGION, monitoring.TagKey.ENTITY_TYPE],
m_enum_errors,
aggregation.SumAggregation(),
)
monitoring.register_views([enum_errors_view])
class EnumParsingError(Exception):
"""Raised if an MappableEnum can't be built from the provided string."""
def __init__(self, cls: type, string_to_parse: str):
msg = "Could not parse {0} when building {1}".format(
string_to_parse, cls)
"""Contains logic to match database entities with ingested entities."""
from abc import abstractmethod
from typing import List, Generic
from opencensus.stats import measure, view, aggregation
from recidiviz.persistence.database.session import Session
from recidiviz.persistence.entity.entities import EntityPersonType
from recidiviz.persistence.entity_matching.entity_matching_types import \
MatchedEntities
from recidiviz.utils import monitoring
m_matching_errors = measure.MeasureInt(
'persistence/entity_matching/error_count',
'Number of EntityMatchingErrors thrown for a specific entity type', '1')
matching_errors_by_entity_view = view.View(
'recidiviz/persistence/entity_matching/error_count',
'Sum of the errors in the entit matching layer, by entity',
[monitoring.TagKey.REGION, monitoring.TagKey.ENTITY_TYPE],
m_matching_errors,
aggregation.SumAggregation())
monitoring.register_views([matching_errors_by_entity_view])
class BaseEntityMatcher(Generic[EntityPersonType]):
"""Base class for all entity matchers."""
from config import connection_string
from datetime import datetime
from opencensus.ext.azure import metrics_exporter
from opencensus.stats import aggregation as aggregation_module
from opencensus.stats import measure as measure_module
from opencensus.stats import stats as stats_module
from opencensus.stats import view as view_module
from opencensus.tags import tag_map as tag_map_module
stats = stats_module.stats
view_manager = stats.view_manager
stats_recorder = stats.stats_recorder
prompt_measure = measure_module.MeasureInt("prompts", "number of prompts",
"prompts")
prompt_view = view_module.View("prompt view", "number of prompts", [],
prompt_measure,
aggregation_module.CountAggregation())
view_manager.register_view(prompt_view)
mmap = stats_recorder.new_measurement_map()
tmap = tag_map_module.TagMap()
exporter = metrics_exporter.new_metrics_exporter(
connection_string=connection_string)
view_manager.register_exporter(exporter)
def prompt():
input("Press enter.")
mmap.measure_int_put(prompt_measure, 1)
def __init__(self, name, description, unit, tags: List[str]):
self._measure = measure_module.MeasureInt(name, description, unit)
tags = [tag_key_module.TagKey(tag) for tag in tags]
self._view = View(name, description, tags, self.measure,
aggregation.LastValueAggregation())
import time
from opencensus.ext.prometheus import stats_exporter as prometheus
from opencensus.stats import aggregation as aggregation_module
from opencensus.stats import measure as measure_module
from opencensus.stats import stats as stats_module
from opencensus.stats import view as view_module
from opencensus.tags import tag_key as tag_key_module
from opencensus.tags import tag_map as tag_map_module
from opencensus.tags import tag_value as tag_value_module
from pprint import pprint
MiB = 1 << 20
FRONTEND_KEY = tag_key_module.TagKey("myorg_keys_frontend")
VIDEO_SIZE_MEASURE = measure_module.MeasureInt("myorg_measures_video_size",
"size of processed videos",
"By")
VIDEO_SIZE_VIEW_NAME = "myorg_views_video_size"
VIDEO_SIZE_DISTRIBUTION = aggregation_module.DistributionAggregation(
[0.0, 16.0 * MiB, 256.0 * MiB])
VIDEO_SIZE_VIEW = view_module.View(VIDEO_SIZE_VIEW_NAME,
"processed video size over time",
[FRONTEND_KEY], VIDEO_SIZE_MEASURE,
VIDEO_SIZE_DISTRIBUTION)
def main():
stats = stats_module.Stats()
view_manager = stats.view_manager
stats_recorder = stats.stats_recorder
# 2. Metrics
#Initialization
from datetime import datetime
from opencensus.stats import aggregation as aggregation_module
from opencensus.stats import measure as measure_module
from opencensus.stats import stats as stats_module
from opencensus.stats import view as view_module
from opencensus.tags import tag_map as tag_map_module
stats = stats_module.stats
view_manager = stats.view_manager
stats_recorder = stats.stats_recorder
prompt_measure = measure_module.MeasureInt("rcs_log_num", "number of rcs_logs",
"logs")
prompt_view = view_module.View("rcs_log_view_3", "number of logs", [],
prompt_measure,
aggregation_module.SumAggregation())
view_manager.register_view(prompt_view)
mmap = stats_recorder.new_measurement_map()
tmap = tag_map_module.TagMap()
# Register the metrics exporter
##customMetrics
## | where name =='prompt view'
from opencensus.ext.azure import metrics_exporter
exporter = metrics_exporter.new_metrics_exporter(
connection_string='InstrumentationKey=<Your Key>')
from recidiviz.persistence import persistence
from recidiviz.persistence import datastore_ingest_info
from recidiviz.persistence.database.schema_utils import SchemaType
from recidiviz.persistence.database.sqlalchemy_database_key import SQLAlchemyDatabaseKey
from recidiviz.persistence.datastore_ingest_info import BatchIngestInfoData
from recidiviz.persistence.ingest_info_validator import ingest_info_validator
from recidiviz.utils import monitoring, regions
from recidiviz.utils.auth.gae import requires_gae_auth
FAILED_TASK_THRESHOLD = 0.1
batch_blueprint = Blueprint("batch", __name__)
m_batch_count = measure.MeasureInt(
"persistence/batch_persistence/batch_count",
"The count of batch persistence calls",
"1",
)
count_view = view.View(
"recidiviz/persistence/batch_persistence/batch_count",
"The sum of batch persistence calls that occurred",
[monitoring.TagKey.REGION, monitoring.TagKey.STATUS, monitoring.TagKey.PERSISTED],
m_batch_count,
aggregation.SumAggregation(),
)
monitoring.register_views([count_view])
class BatchPersistError(Exception):
"""Raised when there was an error with batch persistence."""
from recidiviz.persistence.database.session_factory import SessionFactory
from recidiviz.persistence.database.base_schema import JailsBase
from recidiviz.persistence.database.schema_utils import \
schema_base_for_system_level
from recidiviz.persistence.entity.county import entities as county_entities
from recidiviz.persistence.ingest_info_validator import ingest_info_validator
from recidiviz.persistence.database.schema.county import dao as county_dao
from recidiviz.persistence.entity_matching import entity_matching
from recidiviz.persistence.entity_validator import entity_validator
from recidiviz.persistence.database import database
from recidiviz.persistence.ingest_info_converter import ingest_info_converter
from recidiviz.persistence.ingest_info_converter.base_converter import \
IngestInfoConversionResult
from recidiviz.utils import environment, monitoring
m_people = measure.MeasureInt("persistence/num_people",
"The number of people persisted", "1")
m_aborts = measure.MeasureInt("persistence/num_aborts",
"The number of aborted writes", "1")
m_errors = measure.MeasureInt("persistence/num_errors", "The number of errors",
"1")
people_persisted_view = view.View(
"recidiviz/persistence/num_people", "The sum of people persisted",
[monitoring.TagKey.REGION, monitoring.TagKey.PERSISTED], m_people,
aggregation.SumAggregation())
aborted_writes_view = view.View(
"recidiviz/persistence/num_aborts",
"The sum of aborted writes to persistence",
[monitoring.TagKey.REGION, monitoring.TagKey.REASON], m_aborts,
aggregation.SumAggregation())
errors_persisted_view = view.View(
"recidiviz/persistence/num_errors",
# limitations under the License.
import time
import random
from opencensus.stats import aggregation as aggregation_module
from opencensus.stats.exporters import stackdriver_exporter as stackdriver
from opencensus.stats import measure as measure_module
from opencensus.stats import stats as stats_module
from opencensus.stats import view as view_module
from opencensus.tags import tag_key as tag_key_module
from opencensus.tags import tag_map as tag_map_module
from opencensus.tags import tag_value as tag_value_module
MiB = 1 << 20
FRONTEND_KEY = tag_key_module.TagKey("my.org/keys/frontend")
VIDEO_SIZE_MEASURE = measure_module.MeasureInt(
"my.org/measure/video_size_test2", "size of processed videos", "By")
VIDEO_SIZE_VIEW_NAME = "my.org/views/video_size_test2"
VIDEO_SIZE_DISTRIBUTION = aggregation_module.DistributionAggregation(
[0.0, 16.0 * MiB, 256.0 * MiB])
VIDEO_SIZE_VIEW = view_module.View(VIDEO_SIZE_VIEW_NAME,
"processed video size over time",
[FRONTEND_KEY], VIDEO_SIZE_MEASURE,
VIDEO_SIZE_DISTRIBUTION)
stats = stats_module.Stats()
view_manager = stats.view_manager
stats_recorder = stats.stats_recorder
exporter = stackdriver.new_stats_exporter(
stackdriver.Options(project_id="opencenus-node"))
view_manager.register_exporter(exporter)
# =============================================================================
"""Controller for updating raw state data latest views in BQ."""
import logging
from opencensus.stats import measure, view as opencensus_view, aggregation
from recidiviz.utils import monitoring
from recidiviz.big_query.big_query_client import BigQueryClient
from recidiviz.ingest.direct.controllers.direct_ingest_raw_file_import_manager import DirectIngestRegionRawFileConfig, \
DirectIngestRawFileConfig
from recidiviz.ingest.direct.controllers.direct_ingest_big_query_view_types import \
DirectIngestRawDataTableLatestView
m_failed_latest_views_update = measure.MeasureInt(
"ingest/direct/controllers/direct_ingest_raw_data_table_latest_view_updater/update_views_for_state_failure",
"Counted every time updating views for state fails", "1")
failed_latest_view_updates_view = opencensus_view.View(
"ingest/direct/controllers/direct_ingest_raw_data_table_latest_view_updater/num_update_views_for_state_failure",
"The sum of times a view failed to update",
[monitoring.TagKey.CREATE_UPDATE_RAW_DATA_LATEST_VIEWS_FILE_TAG],
m_failed_latest_views_update, aggregation.SumAggregation())
monitoring.register_views([failed_latest_view_updates_view])
class DirectIngestRawDataTableLatestViewUpdater:
"""Controller for updating raw state data latest views in BQ."""
def __init__(self,
state_code: str,
