def set_headers(response: Response) -> Response:
if not in_development():
response.headers[
"Strict-Transport-Security"] = "max-age=63072000" # max age of 2 years
response.headers["Content-Security-Policy"] = "frame-ancestors 'none'"
response.headers["X-Frame-Options"] = "DENY"
return response
def get_batch_ids(
self,
state_code: StateCode,
*,
override_fs: Optional[GCSFileSystem] = None) -> List[str]:
"""Returns a sorted list of batch id numbers from the a specific state bucket from GCS"""
if in_development():
project_id = GCP_PROJECT_STAGING
else:
project_id = metadata.project_id()
if override_fs is None:
buckets = self.monthly_reports_gcsfs.ls_with_blob_prefix(
bucket_name=f"{project_id}-report-html",
blob_prefix=state_code.value,
)
else:
buckets = override_fs.ls_with_blob_prefix(
bucket_name=f"{project_id}-report-html",
blob_prefix=state_code.value,
)
files = [file for file in buckets if isinstance(file, GcsfsFilePath)]
batch_ids = list(
{batch_id.blob_name.split("/")[1]
for batch_id in files})
batch_ids.sort(reverse=True)
return batch_ids
def auth_and_call(*args: Any, **kwargs: Any) -> Any:
"""Authenticates the inbound request and delegates.
Args:
*args: args to the function
**kwargs: keyword args to the function
Returns:
The output of the function, if successfully authenticated.
An error or redirect response, otherwise.
"""
if in_development():
# Bypass GAE auth check in development.
return func(*args, **kwargs)
is_cron = request.headers.get("X-Appengine-Cron")
is_task = request.headers.get("X-AppEngine-QueueName")
incoming_app_id = request.headers.get("X-Appengine-Inbound-Appid")
jwt = request.headers.get("x-goog-iap-jwt-assertion")
project_id = metadata.project_id()
project_number = metadata.project_number()
if is_cron:
logging.info("Requester is one of our cron jobs, proceeding.")
elif is_task:
logging.info("Requester is the taskqueue, proceeding.")
elif incoming_app_id:
# Check whether this is an intra-app call from our GAE service
logging.info("Requester authenticated as app-id: [%s].", incoming_app_id)
if incoming_app_id == project_id:
logging.info("Authenticated intra-app call, proceeding.")
else:
logging.info("App ID is [%s], not allowed - exiting.", incoming_app_id)
return (
"Failed: Unauthorized external request.",
HTTPStatus.UNAUTHORIZED,
)
elif jwt:
(
user_id,
user_email,
error_str,
) = validate_jwt.validate_iap_jwt_from_app_engine(
jwt, project_number, project_id
)
logging.info("Requester authenticated as [%s] ([%s]).", user_id, user_email)
if error_str:
logging.info("Error validating user credentials: [%s].", error_str)
return ("Error: %s" % error_str, HTTPStatus.UNAUTHORIZED)
else:
return ("Failed: Unauthorized external request.", HTTPStatus.UNAUTHORIZED)
# If we made it this far, client is authorized - run the decorated func
return func(*args, **kwargs)
def runtime_env_vars() -> Tuple[str, HTTPStatus]:
if in_development():
env_string = "development"
elif in_gcp_staging():
env_string = "staging"
elif in_gcp_production():
env_string = "production"
else:
env_string = "unknown"
return f'window.RUNTIME_GCP_ENVIRONMENT="{env_string}";', HTTPStatus.OK
def set_headers(response: Response) -> Response:
if not in_development():
response.headers[
"Strict-Transport-Security"] = "max-age=63072000" # max age of 2 years
response.headers["Content-Security-Policy"] = "frame-ancestors 'none'"
response.headers["X-Frame-Options"] = "DENY"
# Recidiviz-specific version header
response.headers["X-Recidiviz-Current-Version"] = os.getenv(
"CURRENT_GIT_SHA", "")
return response
def _fetch_etl_view_ids() -> Tuple[str, HTTPStatus]:
override_project_id: Optional[str] = None
if in_development():
override_project_id = GCP_PROJECT_STAGING
return (
jsonify(
[builder.view_id for builder in CASE_TRIAGE_EXPORTED_VIEW_BUILDERS]
+ list(
get_importable_csvs(override_project_id=override_project_id).keys()
)
),
HTTPStatus.OK,
)
def get_local_file(file_path: GcsfsFilePath) -> str:
"""
Helper function for supporting local development flows.
When in development environments, we fetch file contents from `recidiviz/case_triage/local/gcs`
In Google Cloud environments, we delegate to Cloud Storage
"""
if in_development():
return Path(os.path.join(local_path, "gcs",
file_path.abs_path())).read_text()
gcs_fs = GcsfsFactory.build()
return gcs_fs.download_as_string(file_path)
def start_timers(self) -> None:
"""Starts store refresh timers for all stores that are a subclass of the AdminPanelStore class."""
if in_gcp() or in_development():
stores_with_timers = [
self.ingest_metadata_store,
self.validation_metadata_store,
self.ingest_data_freshness_store,
self.validation_status_store,
]
for store in stores_with_timers:
RepeatedTimer(15 * 60,
store.recalculate_store,
run_immediately=True).start()
def get_local_secret(secret_name: str) -> Optional[str]:
"""
Helper function for supporting local development flows.
When in development environments, we fetch file contents from `recidiviz/case_triage/local/gsm`
In Google Cloud environments, we delegate to Secrets Manager
"""
if in_development():
try:
return Path(os.path.join(local_path, "gsm",
secret_name)).read_text()
except OSError:
logging.error("Couldn't locate secret %s", secret_name)
return None
return get_secret(secret_name)
def _get_operations_db_metadata(
state_code: StateCode,
ingest_db_name: str) -> Dict[str, Union[int, Optional[datetime]]]:
"""Returns the following dictionary with information about the operations database for the state:
{
unprocessedFilesRaw: <int>
unprocessedFilesIngestView: <int>
dateOfEarliestUnprocessedIngestView: <datetime>
}
If running locally, this does not hit the live DB instance and only returns fake data.
"""
if in_development():
return {
"unprocessedFilesRaw": -1,
"unprocessedFilesIngestView": -2,
"dateOfEarliestUnprocessedIngestView": datetime(2021, 4, 28),
}
file_metadata_manager = PostgresDirectIngestFileMetadataManager(
region_code=state_code.value,
ingest_database_name=ingest_db_name,
)
try:
# Raw files are processed in the primary instance, not secondary
num_unprocessed_raw_files = (
file_metadata_manager.get_num_unprocessed_raw_files())
except DirectIngestInstanceError as _:
num_unprocessed_raw_files = 0
return {
"unprocessedFilesRaw":
num_unprocessed_raw_files,
"unprocessedFilesIngestView":
file_metadata_manager.get_num_unprocessed_ingest_files(),
"dateOfEarliestUnprocessedIngestView":
file_metadata_manager.get_date_of_earliest_unprocessed_ingest_file(
),
}
from recidiviz.utils.environment import in_development, in_test
from recidiviz.utils.flask_exception import FlaskException
from recidiviz.utils.timer import RepeatedTimer
# Flask setup
static_folder = os.path.abspath(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"../../frontends/case-triage/build/",
))
app = Flask(__name__, static_folder=static_folder)
app.secret_key = get_local_secret("case_triage_secret_key")
CSRFProtect(app)
if in_development():
db_url = local_postgres_helpers.postgres_db_url_from_env_vars()
else:
db_url = SQLAlchemyEngineManager.get_server_postgres_instance_url(
schema_type=SchemaType.CASE_TRIAGE)
app.config["SESSION_COOKIE_HTTPONLY"] = True
app.config["SESSION_COOKIE_SECURE"] = True
app.config["SESSION_COOKIE_SAMESITE"] = "Strict"
setup_scoped_sessions(app, db_url)
# Auth setup
def on_successful_authorization(_payload: Dict[str, str], token: str) -> None:
"""
Memoize the user's info (email_address, picture, etc) into our session
def __init__(self) -> None:
if in_development():
with local_project_id_override(GCP_PROJECT_STAGING):
self._initialize_stores()
elif in_gcp():
self._initialize_stores()
def _generate_emails(state_code_str: str) -> Tuple[str, HTTPStatus]:
try:
data = request.json
state_code = StateCode(state_code_str)
if state_code not in EMAIL_STATE_CODES:
raise ValueError("State code is invalid for PO monthly reports")
# TODO(#7790): Support more email types.
report_type = ReportType(data.get("reportType"))
if report_type != ReportType.POMonthlyReport:
raise ValueError(f"{report_type.value} is not a valid ReportType")
test_address = data.get("testAddress")
region_code = data.get("regionCode")
message_body_override = data.get("messageBodyOverride")
email_allowlist = data.get("emailAllowlist")
validate_email_address(test_address)
if email_allowlist is not None:
for recipient_email in email_allowlist:
validate_email_address(recipient_email)
except (ValueError, JSONDecodeError) as error:
logging.error(error)
return str(error), HTTPStatus.BAD_REQUEST
if test_address == "":
test_address = None
if region_code not in REGION_CODES:
region_code = None
try:
batch_id = generate_batch_id()
if in_development():
with local_project_id_override(GCP_PROJECT_STAGING):
result: MultiRequestResult[str, str] = data_retrieval.start(
state_code=state_code,
report_type=report_type,
batch_id=batch_id,
test_address=test_address,
region_code=region_code,
email_allowlist=email_allowlist,
message_body_override=message_body_override,
)
else:
result = data_retrieval.start(
state_code=state_code,
report_type=report_type,
batch_id=batch_id,
test_address=test_address,
region_code=region_code,
email_allowlist=email_allowlist,
message_body_override=message_body_override,
)
except InvalidRegionCodeException:
return "Invalid region code provided", HTTPStatus.BAD_REQUEST
new_batch_text = f"New batch started for {state_code} and {report_type}. Batch id = {batch_id}."
test_address_text = (
f"Emails generated for test address: {test_address}" if test_address else ""
)
counts_text = f"Successfully generated {len(result.successes)} email(s)"
success_text = f"{new_batch_text} {test_address_text} {counts_text}."
if result.failures and not result.successes:
return (
f"{success_text}"
f" Failed to generate all emails. Retry the request again."
), HTTPStatus.INTERNAL_SERVER_ERROR
if result.failures:
return (
f"{success_text}"
f" Failed to generate {len(result.failures)} email(s): {', '.join(result.failures)}"
), HTTPStatus.MULTI_STATUS
return (
jsonify(
{
"batchId": batch_id,
"statusText": f"{success_text}",
}
),
HTTPStatus.OK,
)
