batch_blueprint = Blueprint("batch", __name__)
m_batch_count = measure.MeasureInt(
"persistence/batch_persistence/batch_count",
"The count of batch persistence calls",
"1",
)
count_view = view.View(
"recidiviz/persistence/batch_persistence/batch_count",
"The sum of batch persistence calls that occurred",
[monitoring.TagKey.REGION, monitoring.TagKey.STATUS, monitoring.TagKey.PERSISTED],
m_batch_count,
aggregation.SumAggregation(),
)
monitoring.register_views([count_view])
class BatchPersistError(Exception):
"""Raised when there was an error with batch persistence."""
def __init__(self, region: str, scrape_type: ScrapeType):
msg_template = "Error when running '{}' for region {}"
msg = msg_template.format(scrape_type, region)
super().__init__(msg)
class DatastoreError(Exception):
"""Raised when there was an error with Datastore."""
def __init__(self, region: str, call_type: str):
)
from recidiviz.utils import monitoring
m_duration_s = measure.MeasureFloat(
"function_duration", "The time it took for this function to run", "s"
)
duration_distribution_view = view.View(
"recidiviz/function_durations",
"The distribution of the function durations",
[monitoring.TagKey.REGION, monitoring.TagKey.FUNCTION],
m_duration_s,
aggregation.DistributionAggregation(monitoring.exponential_buckets(0.1, 5, 10)),
)
monitoring.register_views([duration_distribution_view])
# Contains a list of all the addresses of all of the functions in our stack that are currently being timed. Used to
# detect recursion.
stack: ContextVar[List[int]] = ContextVar("stack", default=[])
def span(func: Callable) -> Callable:
"""Creates a new span for this function in the trace.
This allows us to visualize how much of the processing time of a given request is spent inside of this function
without relying on log entries. Additionally the duration of the function call is recorded as a metric.
"""
@wraps(func)
def run_inside_new_span(*args: Any, **kwargs: Any) -> None:
from recidiviz.validation.configured_validations import get_all_validations, STATES_TO_VALIDATE
from recidiviz.validation.validation_models import DataValidationJob, DataValidationJobResult
from recidiviz.validation.views import view_config
m_failed_validations = measure.MeasureInt("validation/num_failures",
"The number of failed validations",
"1")
failed_validations_view = view.View(
"recidiviz/validation/num_failures", "The sum of failed validations", [
monitoring.TagKey.REGION, monitoring.TagKey.VALIDATION_CHECK_TYPE,
monitoring.TagKey.VALIDATION_VIEW_ID
], m_failed_validations, aggregation.SumAggregation())
monitoring.register_views([failed_validations_view])
validation_manager_blueprint = Blueprint('validation_manager', __name__)
@validation_manager_blueprint.route('/validate')
@authenticate_request
def handle_validation_request():
"""API endpoint to service data validation requests."""
should_update_views = get_bool_param_value('should_update_views',
request.args,
default=False)
failed_validations = execute_validation(
should_update_views=should_update_views)
return _readable_response(failed_validations), HTTPStatus.OK
from opencensus.stats import aggregation, measure, view
from recidiviz.common.str_field_utils import normalize
from recidiviz.utils import monitoring
from recidiviz.utils.types import ClsT
m_enum_errors = measure.MeasureInt("converter/enum_error_count",
"The number of enum errors", "1")
enum_errors_view = view.View(
"recidiviz/converter/enum_error_count",
"The sum of enum errors",
[monitoring.TagKey.REGION, monitoring.TagKey.ENTITY_TYPE],
m_enum_errors,
aggregation.SumAggregation(),
)
monitoring.register_views([enum_errors_view])
class EnumParsingError(Exception):
"""Raised if an MappableEnum can't be built from the provided string."""
def __init__(self, cls: type, string_to_parse: str):
msg = "Could not parse {0} when building {1}".format(
string_to_parse, cls)
self.entity_type = cls
super().__init__(msg)
class EntityEnumMeta(EnumMeta):
"""Metaclass for mappable enums."""
# pylint doesn't understand |cls| as |self|:
m_failed_metric_export_job = measure.MeasureInt(
"bigquery/metric_view_export_manager/metric_view_export_job_failure",
"Counted every time a set of exported metric views fails for non-validation reasons",
"1",
)
failed_metric_export_view = opencensus_view.View(
"bigquery/metric_view_export_manager/num_metric_view_export_job_failure",
"The sum of times a set of exported metric views fails to export for non-validation reasons",
[monitoring.TagKey.REGION, monitoring.TagKey.METRIC_VIEW_EXPORT_NAME],
m_failed_metric_export_job,
aggregation.SumAggregation(),
)
monitoring.register_views(
[failed_metric_export_validation_view, failed_metric_export_view])
export_blueprint = Blueprint("export", __name__)
@export_blueprint.route("/create_metric_view_data_export_task")
@requires_gae_auth
def create_metric_view_data_export_task() -> Tuple[str, HTTPStatus]:
"""Queues a task to export data in BigQuery metric views to cloud storage buckets.
Example:
export/create_metric_view_data_export_task?export_job_filter=US_ID
URL parameters:
export_job_filter: (string) Kind of jobs to initiate export for. Can either be an export_name (e.g. LANTERN)
or a state_code (e.g. US_ND)
Args:
from recidiviz.ingest.direct.controllers.direct_ingest_raw_file_import_manager import DirectIngestRegionRawFileConfig, \
DirectIngestRawFileConfig
from recidiviz.ingest.direct.controllers.direct_ingest_big_query_view_types import \
DirectIngestRawDataTableLatestView
m_failed_latest_views_update = measure.MeasureInt(
"ingest/direct/controllers/direct_ingest_raw_data_table_latest_view_updater/update_views_for_state_failure",
"Counted every time updating views for state fails", "1")
failed_latest_view_updates_view = opencensus_view.View(
"ingest/direct/controllers/direct_ingest_raw_data_table_latest_view_updater/num_update_views_for_state_failure",
"The sum of times a view failed to update",
[monitoring.TagKey.CREATE_UPDATE_RAW_DATA_LATEST_VIEWS_FILE_TAG],
m_failed_latest_views_update, aggregation.SumAggregation())
monitoring.register_views([failed_latest_view_updates_view])
class DirectIngestRawDataTableLatestViewUpdater:
"""Controller for updating raw state data latest views in BQ."""
def __init__(self,
state_code: str,
project_id: str,
bq_client: BigQueryClient,
dry_run: bool = False):
self.state_code = state_code
self.project_id = project_id
self.bq_client = bq_client
self.dry_run = dry_run
self.raw_file_region_config = DirectIngestRegionRawFileConfig(
state_code)
from recidiviz.persistence.entity_matching.entity_matching_types import \
MatchedEntities
from recidiviz.utils import monitoring
m_matching_errors = measure.MeasureInt(
'persistence/entity_matching/error_count',
'Number of EntityMatchingErrors thrown for a specific entity type', '1')
matching_errors_by_entity_view = view.View(
'recidiviz/persistence/entity_matching/error_count',
'Sum of the errors in the entit matching layer, by entity',
[monitoring.TagKey.REGION, monitoring.TagKey.ENTITY_TYPE],
m_matching_errors,
aggregation.SumAggregation())
monitoring.register_views([matching_errors_by_entity_view])
class BaseEntityMatcher(Generic[EntityPersonType]):
"""Base class for all entity matchers."""
@abstractmethod
def run_match(self, session: Session, region_code: str,
ingested_people: List[EntityPersonType]) \
-> MatchedEntities:
"""
Attempts to match all people from |ingested_people| with corresponding
people in our database for the given |region|. Returns an
MatchedEntities object that contains the results of matching.
"""
"1")
people_persisted_view = view.View(
"recidiviz/persistence/num_people", "The sum of people persisted",
[monitoring.TagKey.REGION, monitoring.TagKey.PERSISTED], m_people,
aggregation.SumAggregation())
aborted_writes_view = view.View(
"recidiviz/persistence/num_aborts",
"The sum of aborted writes to persistence",
[monitoring.TagKey.REGION, monitoring.TagKey.REASON], m_aborts,
aggregation.SumAggregation())
errors_persisted_view = view.View(
"recidiviz/persistence/num_errors",
"The sum of errors in the persistence layer",
[monitoring.TagKey.REGION, monitoring.TagKey.ERROR], m_errors,
aggregation.SumAggregation())
monitoring.register_views(
[people_persisted_view, aborted_writes_view, errors_persisted_view])
ERROR_THRESHOLD = 0.5
def infer_release_on_open_bookings(region_code: str,
last_ingest_time: datetime.datetime,
custody_status: CustodyStatus):
"""
Look up all open bookings whose last_seen_time is earlier than the
provided last_ingest_time in the provided region, update those
bookings to have an inferred release date equal to the provided
last_ingest_time.
Args:
region_code: the region_code
"recidiviz/persistence/num_errors",
"The sum of errors in the persistence layer",
[monitoring.TagKey.REGION, monitoring.TagKey.ERROR],
m_errors,
aggregation.SumAggregation(),
)
retried_transactions_view = view.View(
"recidiviz/persistence/num_transaction_retries",
"The total number of transaction retries",
[monitoring.TagKey.REGION],
m_retries,
aggregation.SumAggregation(),
)
monitoring.register_views([
people_persisted_view,
aborted_writes_view,
errors_persisted_view,
retried_transactions_view,
])
OVERALL_THRESHOLD = "overall_threshold"
ENUM_THRESHOLD = "enum_threshold"
ENTITY_MATCHING_THRESHOLD = "entity_matching_threshold"
DATABASE_INVARIANT_THRESHOLD = "database_invariant_threshold"
SYSTEM_TYPE_TO_ERROR_THRESHOLD: Dict[SystemLevel, Dict[str, float]] = {
SystemLevel.COUNTY: {
OVERALL_THRESHOLD: 0.5,
ENUM_THRESHOLD: 0.5,
ENTITY_MATCHING_THRESHOLD: 0.0,
DATABASE_INVARIANT_THRESHOLD: 0.0,
},
from recidiviz.ingest.scrape import sessions
from recidiviz.ingest.scrape.task_params import QueueRequest
from recidiviz.utils import monitoring, regions
from recidiviz.utils.auth.gae import requires_gae_auth
m_tasks = measure.MeasureInt("ingest/scrape/task_count",
"The count of scrape tasks that occurred", "1")
task_view = view.View(
"recidiviz/ingest/scrape/task_count",
"The sum of scrape tasks that occurred",
[monitoring.TagKey.REGION, monitoring.TagKey.STATUS],
m_tasks,
aggregation.SumAggregation(),
)
monitoring.register_views([task_view])
class RequestProcessingError(Exception):
"""Exception containing the request that failed to process"""
def __init__(self, region: str, task: str, queue_request: QueueRequest):
request_string = pprint.pformat(queue_request.to_serializable())
msg = "Error when running '{}' for '{}' with request:\n{}".format(
task, region, request_string)
super().__init__(msg)
worker = Blueprint("worker", __name__)
# NB: Region is part of the url so that request logs can be filtered on it.
m_failed_engine_initialization = measure.MeasureInt(
"persistence/database/sqlalchemy_engine_initialization_failures",
"Counts the number of engine initialization failures",
"1",
)
failed_engine_initialization_view = view.View(
"recidiviz/persistence/database/sqlalchemy_engine_initialization_failures",
"Sum of all engine initialization failures",
[monitoring.TagKey.SCHEMA_TYPE, monitoring.TagKey.DATABASE_NAME],
m_failed_engine_initialization,
aggregation.SumAggregation(),
)
monitoring.register_views([failed_engine_initialization_view])
class SQLAlchemyEngineManager:
"""A class to manage all SQLAlchemy Engines for our database instances."""
_engine_for_database: Dict[SQLAlchemyDatabaseKey, Engine] = {}
@classmethod
def init_engine_for_postgres_instance(
cls,
database_key: SQLAlchemyDatabaseKey,
db_url: URL,
) -> Engine:
"""Initializes a sqlalchemy Engine object for the given Postgres database /
schema and caches it for future use."""
)
sftp_attempts_view = view.View(
"recidiviz/ingest/sftp/attempts",
"The sum of attempts that were made for SFTP",
[monitoring.TagKey.REGION, monitoring.TagKey.SFTP_TASK_TYPE],
m_sftp_attempts,
aggregation.SumAggregation(),
)
sftp_errors_view = view.View(
"recidiviz/ingest/sftp/errors",
"The sum of errors that were made for SFTP",
[monitoring.TagKey.REGION, monitoring.TagKey.SFTP_TASK_TYPE],
m_sftp_errors,
aggregation.SumAggregation(),
)
monitoring.register_views([sftp_attempts_view, sftp_errors_view])
direct_ingest_control = Blueprint("direct_ingest_control", __name__)
@direct_ingest_control.route("/normalize_raw_file_path")
@requires_gae_auth
def normalize_raw_file_path() -> Tuple[str, HTTPStatus]:
"""Called from a Cloud Function when a new file is added to a bucket that is configured to rename files but not
ingest them. For example, a bucket that is being used for automatic data transfer testing.
"""
# The bucket name for the file to normalize
bucket = get_str_param_value("bucket", request.args)
# The relative path to the file, not including the bucket name
relative_file_path = get_str_param_value("relative_file_path",
request.args,
