batch_blueprint = Blueprint("batch", __name__) m_batch_count = measure.MeasureInt( "persistence/batch_persistence/batch_count", "The count of batch persistence calls", "1", ) count_view = view.View( "recidiviz/persistence/batch_persistence/batch_count", "The sum of batch persistence calls that occurred", [monitoring.TagKey.REGION, monitoring.TagKey.STATUS, monitoring.TagKey.PERSISTED], m_batch_count, aggregation.SumAggregation(), ) monitoring.register_views([count_view]) class BatchPersistError(Exception): """Raised when there was an error with batch persistence.""" def __init__(self, region: str, scrape_type: ScrapeType): msg_template = "Error when running '{}' for region {}" msg = msg_template.format(scrape_type, region) super().__init__(msg) class DatastoreError(Exception): """Raised when there was an error with Datastore.""" def __init__(self, region: str, call_type: str):
) from recidiviz.utils import monitoring m_duration_s = measure.MeasureFloat( "function_duration", "The time it took for this function to run", "s" ) duration_distribution_view = view.View( "recidiviz/function_durations", "The distribution of the function durations", [monitoring.TagKey.REGION, monitoring.TagKey.FUNCTION], m_duration_s, aggregation.DistributionAggregation(monitoring.exponential_buckets(0.1, 5, 10)), ) monitoring.register_views([duration_distribution_view]) # Contains a list of all the addresses of all of the functions in our stack that are currently being timed. Used to # detect recursion. stack: ContextVar[List[int]] = ContextVar("stack", default=[]) def span(func: Callable) -> Callable: """Creates a new span for this function in the trace. This allows us to visualize how much of the processing time of a given request is spent inside of this function without relying on log entries. Additionally the duration of the function call is recorded as a metric. """ @wraps(func) def run_inside_new_span(*args: Any, **kwargs: Any) -> None:
from recidiviz.validation.configured_validations import get_all_validations, STATES_TO_VALIDATE from recidiviz.validation.validation_models import DataValidationJob, DataValidationJobResult from recidiviz.validation.views import view_config m_failed_validations = measure.MeasureInt("validation/num_failures", "The number of failed validations", "1") failed_validations_view = view.View( "recidiviz/validation/num_failures", "The sum of failed validations", [ monitoring.TagKey.REGION, monitoring.TagKey.VALIDATION_CHECK_TYPE, monitoring.TagKey.VALIDATION_VIEW_ID ], m_failed_validations, aggregation.SumAggregation()) monitoring.register_views([failed_validations_view]) validation_manager_blueprint = Blueprint('validation_manager', __name__) @validation_manager_blueprint.route('/validate') @authenticate_request def handle_validation_request(): """API endpoint to service data validation requests.""" should_update_views = get_bool_param_value('should_update_views', request.args, default=False) failed_validations = execute_validation( should_update_views=should_update_views) return _readable_response(failed_validations), HTTPStatus.OK
from opencensus.stats import aggregation, measure, view from recidiviz.common.str_field_utils import normalize from recidiviz.utils import monitoring from recidiviz.utils.types import ClsT m_enum_errors = measure.MeasureInt("converter/enum_error_count", "The number of enum errors", "1") enum_errors_view = view.View( "recidiviz/converter/enum_error_count", "The sum of enum errors", [monitoring.TagKey.REGION, monitoring.TagKey.ENTITY_TYPE], m_enum_errors, aggregation.SumAggregation(), ) monitoring.register_views([enum_errors_view]) class EnumParsingError(Exception): """Raised if an MappableEnum can't be built from the provided string.""" def __init__(self, cls: type, string_to_parse: str): msg = "Could not parse {0} when building {1}".format( string_to_parse, cls) self.entity_type = cls super().__init__(msg) class EntityEnumMeta(EnumMeta): """Metaclass for mappable enums.""" # pylint doesn't understand |cls| as |self|:
m_failed_metric_export_job = measure.MeasureInt( "bigquery/metric_view_export_manager/metric_view_export_job_failure", "Counted every time a set of exported metric views fails for non-validation reasons", "1", ) failed_metric_export_view = opencensus_view.View( "bigquery/metric_view_export_manager/num_metric_view_export_job_failure", "The sum of times a set of exported metric views fails to export for non-validation reasons", [monitoring.TagKey.REGION, monitoring.TagKey.METRIC_VIEW_EXPORT_NAME], m_failed_metric_export_job, aggregation.SumAggregation(), ) monitoring.register_views( [failed_metric_export_validation_view, failed_metric_export_view]) export_blueprint = Blueprint("export", __name__) @export_blueprint.route("/create_metric_view_data_export_task") @requires_gae_auth def create_metric_view_data_export_task() -> Tuple[str, HTTPStatus]: """Queues a task to export data in BigQuery metric views to cloud storage buckets. Example: export/create_metric_view_data_export_task?export_job_filter=US_ID URL parameters: export_job_filter: (string) Kind of jobs to initiate export for. Can either be an export_name (e.g. LANTERN) or a state_code (e.g. US_ND) Args:
from recidiviz.ingest.direct.controllers.direct_ingest_raw_file_import_manager import DirectIngestRegionRawFileConfig, \ DirectIngestRawFileConfig from recidiviz.ingest.direct.controllers.direct_ingest_big_query_view_types import \ DirectIngestRawDataTableLatestView m_failed_latest_views_update = measure.MeasureInt( "ingest/direct/controllers/direct_ingest_raw_data_table_latest_view_updater/update_views_for_state_failure", "Counted every time updating views for state fails", "1") failed_latest_view_updates_view = opencensus_view.View( "ingest/direct/controllers/direct_ingest_raw_data_table_latest_view_updater/num_update_views_for_state_failure", "The sum of times a view failed to update", [monitoring.TagKey.CREATE_UPDATE_RAW_DATA_LATEST_VIEWS_FILE_TAG], m_failed_latest_views_update, aggregation.SumAggregation()) monitoring.register_views([failed_latest_view_updates_view]) class DirectIngestRawDataTableLatestViewUpdater: """Controller for updating raw state data latest views in BQ.""" def __init__(self, state_code: str, project_id: str, bq_client: BigQueryClient, dry_run: bool = False): self.state_code = state_code self.project_id = project_id self.bq_client = bq_client self.dry_run = dry_run self.raw_file_region_config = DirectIngestRegionRawFileConfig( state_code)
from recidiviz.persistence.entity_matching.entity_matching_types import \ MatchedEntities from recidiviz.utils import monitoring m_matching_errors = measure.MeasureInt( 'persistence/entity_matching/error_count', 'Number of EntityMatchingErrors thrown for a specific entity type', '1') matching_errors_by_entity_view = view.View( 'recidiviz/persistence/entity_matching/error_count', 'Sum of the errors in the entit matching layer, by entity', [monitoring.TagKey.REGION, monitoring.TagKey.ENTITY_TYPE], m_matching_errors, aggregation.SumAggregation()) monitoring.register_views([matching_errors_by_entity_view]) class BaseEntityMatcher(Generic[EntityPersonType]): """Base class for all entity matchers.""" @abstractmethod def run_match(self, session: Session, region_code: str, ingested_people: List[EntityPersonType]) \ -> MatchedEntities: """ Attempts to match all people from |ingested_people| with corresponding people in our database for the given |region|. Returns an MatchedEntities object that contains the results of matching. """
"1") people_persisted_view = view.View( "recidiviz/persistence/num_people", "The sum of people persisted", [monitoring.TagKey.REGION, monitoring.TagKey.PERSISTED], m_people, aggregation.SumAggregation()) aborted_writes_view = view.View( "recidiviz/persistence/num_aborts", "The sum of aborted writes to persistence", [monitoring.TagKey.REGION, monitoring.TagKey.REASON], m_aborts, aggregation.SumAggregation()) errors_persisted_view = view.View( "recidiviz/persistence/num_errors", "The sum of errors in the persistence layer", [monitoring.TagKey.REGION, monitoring.TagKey.ERROR], m_errors, aggregation.SumAggregation()) monitoring.register_views( [people_persisted_view, aborted_writes_view, errors_persisted_view]) ERROR_THRESHOLD = 0.5 def infer_release_on_open_bookings(region_code: str, last_ingest_time: datetime.datetime, custody_status: CustodyStatus): """ Look up all open bookings whose last_seen_time is earlier than the provided last_ingest_time in the provided region, update those bookings to have an inferred release date equal to the provided last_ingest_time. Args: region_code: the region_code
"recidiviz/persistence/num_errors", "The sum of errors in the persistence layer", [monitoring.TagKey.REGION, monitoring.TagKey.ERROR], m_errors, aggregation.SumAggregation(), ) retried_transactions_view = view.View( "recidiviz/persistence/num_transaction_retries", "The total number of transaction retries", [monitoring.TagKey.REGION], m_retries, aggregation.SumAggregation(), ) monitoring.register_views([ people_persisted_view, aborted_writes_view, errors_persisted_view, retried_transactions_view, ]) OVERALL_THRESHOLD = "overall_threshold" ENUM_THRESHOLD = "enum_threshold" ENTITY_MATCHING_THRESHOLD = "entity_matching_threshold" DATABASE_INVARIANT_THRESHOLD = "database_invariant_threshold" SYSTEM_TYPE_TO_ERROR_THRESHOLD: Dict[SystemLevel, Dict[str, float]] = { SystemLevel.COUNTY: { OVERALL_THRESHOLD: 0.5, ENUM_THRESHOLD: 0.5, ENTITY_MATCHING_THRESHOLD: 0.0, DATABASE_INVARIANT_THRESHOLD: 0.0, },
from recidiviz.ingest.scrape import sessions from recidiviz.ingest.scrape.task_params import QueueRequest from recidiviz.utils import monitoring, regions from recidiviz.utils.auth.gae import requires_gae_auth m_tasks = measure.MeasureInt("ingest/scrape/task_count", "The count of scrape tasks that occurred", "1") task_view = view.View( "recidiviz/ingest/scrape/task_count", "The sum of scrape tasks that occurred", [monitoring.TagKey.REGION, monitoring.TagKey.STATUS], m_tasks, aggregation.SumAggregation(), ) monitoring.register_views([task_view]) class RequestProcessingError(Exception): """Exception containing the request that failed to process""" def __init__(self, region: str, task: str, queue_request: QueueRequest): request_string = pprint.pformat(queue_request.to_serializable()) msg = "Error when running '{}' for '{}' with request:\n{}".format( task, region, request_string) super().__init__(msg) worker = Blueprint("worker", __name__) # NB: Region is part of the url so that request logs can be filtered on it.
m_failed_engine_initialization = measure.MeasureInt( "persistence/database/sqlalchemy_engine_initialization_failures", "Counts the number of engine initialization failures", "1", ) failed_engine_initialization_view = view.View( "recidiviz/persistence/database/sqlalchemy_engine_initialization_failures", "Sum of all engine initialization failures", [monitoring.TagKey.SCHEMA_TYPE, monitoring.TagKey.DATABASE_NAME], m_failed_engine_initialization, aggregation.SumAggregation(), ) monitoring.register_views([failed_engine_initialization_view]) class SQLAlchemyEngineManager: """A class to manage all SQLAlchemy Engines for our database instances.""" _engine_for_database: Dict[SQLAlchemyDatabaseKey, Engine] = {} @classmethod def init_engine_for_postgres_instance( cls, database_key: SQLAlchemyDatabaseKey, db_url: URL, ) -> Engine: """Initializes a sqlalchemy Engine object for the given Postgres database / schema and caches it for future use."""
) sftp_attempts_view = view.View( "recidiviz/ingest/sftp/attempts", "The sum of attempts that were made for SFTP", [monitoring.TagKey.REGION, monitoring.TagKey.SFTP_TASK_TYPE], m_sftp_attempts, aggregation.SumAggregation(), ) sftp_errors_view = view.View( "recidiviz/ingest/sftp/errors", "The sum of errors that were made for SFTP", [monitoring.TagKey.REGION, monitoring.TagKey.SFTP_TASK_TYPE], m_sftp_errors, aggregation.SumAggregation(), ) monitoring.register_views([sftp_attempts_view, sftp_errors_view]) direct_ingest_control = Blueprint("direct_ingest_control", __name__) @direct_ingest_control.route("/normalize_raw_file_path") @requires_gae_auth def normalize_raw_file_path() -> Tuple[str, HTTPStatus]: """Called from a Cloud Function when a new file is added to a bucket that is configured to rename files but not ingest them. For example, a bucket that is being used for automatic data transfer testing. """ # The bucket name for the file to normalize bucket = get_str_param_value("bucket", request.args) # The relative path to the file, not including the bucket name relative_file_path = get_str_param_value("relative_file_path", request.args,