def test_clear(self):
secrets.set_secret('top_track', 'An Eagle In Your Mind')
secrets.set_secret('solid_track', 'Kaini Industries')
secrets.clear_secrets()
assert secrets.get_secret('top_track') is None
assert secrets.get_secret('solid_track') is None
def for_region(cls, region_code: str) -> "SftpAuth":
prefix = f"{region_code}_sftp"
host = secrets.get_secret(f"{prefix}_host")
if host is None:
raise ValueError(f"Unable to find host name for secret key {prefix}_host")
username = secrets.get_secret(f"{prefix}_username")
password = secrets.get_secret(f"{prefix}_password")
connection_options = SftpAuth.set_up_connection_options(prefix, host)
return SftpAuth(host, username, password, connection_options)
def _create_engine():
db_user = secrets.get_secret('us_ma_middlesex_db_user')
db_password = secrets.get_secret('us_ma_middlesex_db_password')
db_name = secrets.get_secret('us_ma_middlesex_db_name')
cloudsql_instance_id = secrets.get_secret('us_ma_middlesex_instance_id')
return sqlalchemy.create_engine(
sqlalchemy.engine.url.URL(
drivername=_DATABASE_TYPE,
username=db_user,
password=db_password,
database=db_name,
query={'host': '/cloudsql/{}'.format(cloudsql_instance_id)}))
def _get_server_postgres_instance_url(
cls, *, db_user_key: str, db_password_key: str, db_name_key: str,
cloudsql_instance_id_key: str) -> str:
db_user = secrets.get_secret(db_user_key)
db_password = secrets.get_secret(db_password_key)
db_name = secrets.get_secret(db_name_key)
cloudsql_instance_id = secrets.get_secret(cloudsql_instance_id_key)
sqlalchemy_url = ('postgresql://{db_user}:{db_password}@/{db_name}'
'?host=/cloudsql/{cloudsql_instance_id}').format(
db_user=db_user,
db_password=db_password,
db_name=db_name,
cloudsql_instance_id=cloudsql_instance_id)
return sqlalchemy_url
def get_proxies(use_test=False):
"""Retrieves proxy username/pass from environment variables
Retrieves proxy information to use in requests to third-party
services. If not in production environment, defaults to test proxy
credentials (so problems during test runs don't risk our main proxy
IP's reputation).
Args:
use_test: (bool) Use test proxy credentials, not prod
Returns:
Proxies dict for requests library, in the form:
{'<protocol>': '<http://<proxy creds>@<proxy url>'}
Raises:
Exception: General exception, since scraper cannot
proceed without this
"""
if not environment.in_gae() or use_test:
return None
user_var = 'proxy_user'
pass_var = 'proxy_password'
proxy_url = secrets.get_secret("proxy_url")
if proxy_url is None:
raise Exception("No proxy url")
# On the proxy side, a random ip is chosen for a session it has not seen
# so collisions can still happen so we increase the integer to reduce the
# odds.
base_proxy_user = secrets.get_secret(user_var)
proxy_user = PROXY_USER_TEMPLATE.format(
base_proxy_user, random.random())
proxy_password = secrets.get_secret(pass_var)
if (base_proxy_user is None) or (proxy_password is None):
raise Exception("No proxy user/pass")
proxy_credentials = proxy_user + ":" + proxy_password
proxy_request_url = 'http://' + proxy_credentials + "@" + proxy_url
proxies = {'http': proxy_request_url, 'https': proxy_request_url}
return proxies
def _get_db_password(cls, *, database_key: SQLAlchemyDatabaseKey) -> str:
secret_key = f"{cls._secret_manager_prefix(database_key)}_db_password"
password = secrets.get_secret(secret_key)
if password is None:
raise ValueError(
f"Unable to retrieve database password for key [{secret_key}]")
return password
def _get_db_host(cls, *, database_key: SQLAlchemyDatabaseKey) -> str:
secret_key = f"{cls._secret_manager_prefix(database_key)}_db_host"
host = secrets.get_secret(secret_key)
if host is None:
raise ValueError(
f"Unable to retrieve database host for key [{secret_key}]")
return host
def get_stripped_cloudsql_instance_id(
cls, schema_type: SchemaType
) -> Optional[str]:
"""The full instance id stored in secrets has the form project_id:zone:instance_id.
This returns just the final instance_id, for example, 'dev-data' or 'prod-state-data'. If a key is not
configured for the given schema, returns None.
Should be used when using the sqladmin_client().
"""
instance_id_key = cls.get_cloudsql_instance_id_key(schema_type)
if instance_id_key is None:
return None
instance_id_full = secrets.get_secret(instance_id_key)
if instance_id_full is None:
raise ValueError(
f"Unable to retrieve instance id for schema type [{schema_type}]"
)
# Remove Project ID and Zone information from Cloud SQL instance ID.
# Expected format "project_id:zone:instance_id"
instance_id = instance_id_full.split(":")[-1]
return instance_id
def get_db_user(cls, schema_type: SchemaType) -> str:
user = secrets.get_secret(cls.get_db_user_key(schema_type))
if user is None:
raise ValueError(
f"Unable to retrieve database user for schema type [{schema_type}]"
)
return user
def get_db_readonly_password(cls, schema_type: SchemaType) -> str:
password = secrets.get_secret(cls.get_db_readonly_password_key(schema_type))
if password is None:
raise ValueError(
f"Unable to retrieve database readonly password for schema type [{schema_type}]"
)
return password
def get_db_name(cls, schema_type: SchemaType) -> str:
db_name = secrets.get_secret(cls.get_db_name_key(schema_type))
if db_name is None:
raise ValueError(
f"Unable to retrieve database name for schema type [{schema_type}]"
)
return db_name
def get_db_host(cls, schema_type: SchemaType) -> str:
host = secrets.get_secret(cls.get_db_host_key(schema_type))
if host is None:
raise ValueError(
f"Unable to retrieve database host for schema type [{schema_type}]"
)
return host
def _get_db_user(cls, *, database_key: SQLAlchemyDatabaseKey) -> str:
secret_key = f"{cls._secret_manager_prefix(database_key)}_db_user"
user = secrets.get_secret(secret_key)
if user is None:
raise ValueError(
f"Unable to retrieve database user for key [{secret_key}]")
return user
def get_headers():
"""Retrieves headers (e.g., user agent string) from environment
variables
Retrieves user agent string information to use in requests to
third-party services.
Args:
N/A
Returns:
Headers dict for the requests library, in the form:
{'User-Agent': '<user agent string>'}
Raises:
Exception: General exception, since scraper cannot
proceed without this
"""
in_prod = environment.in_gae()
if not in_prod:
user_agent_string = ('For any issues, concerns, or rate constraints,'
'e-mail [email protected]')
else:
user_agent_string = secrets.get_secret("user_agent")
if not user_agent_string:
raise Exception("No user agent string")
headers = {'User-Agent': user_agent_string}
return headers
def _get_sendgrid_api_key(self) -> str:
"""Returns the value for the SendGrid API key from the secrets manager."""
sendgrid_api_value = secrets.get_secret(_SENDGRID_API_KEY)
if not sendgrid_api_value:
raise ValueError(
f"Could not get secret value for `Sendgrid API Key`. Provided with "
f"key={_SENDGRID_API_KEY}")
return sendgrid_api_value
def get_cdn_static_ip() -> str:
cdn_ip = secrets.get_secret(_PO_REPORT_CDN_STATIC_IP_KEY)
if not cdn_ip:
raise ValueError(
f'Could not get secret value for `CDN static IP`. Provided with '
f'key={_PO_REPORT_CDN_STATIC_IP_KEY}')
return cdn_ip
def _get_server_postgres_instance_url(cls, *, schema_type: SchemaType) -> str:
instance_id_key = cls.get_cloudsql_instance_id_key(schema_type)
if instance_id_key is None:
raise ValueError(f'Instance id is not configured for schema type [{schema_type}]')
db_user = secrets.get_secret(f'{cls._SCHEMA_TO_SECRET_MANAGER_PREFIX[schema_type]}_db_user')
db_password = secrets.get_secret(f'{cls._SCHEMA_TO_SECRET_MANAGER_PREFIX[schema_type]}_db_password')
db_name = cls.get_db_name(schema_type)
cloudsql_instance_id = secrets.get_secret(instance_id_key)
sqlalchemy_url = ('postgresql://{db_user}:{db_password}@/{db_name}'
'?host=/cloudsql/{cloudsql_instance_id}').format(
db_user=db_user,
db_password=db_password,
db_name=db_name,
cloudsql_instance_id=cloudsql_instance_id)
return sqlalchemy_url
def cloudsql_instance_id() -> str:
"""Get the Cloud SQL instance ID."""
instance_id_full = secrets.get_secret('cloudsql_instance_id')
# Remove Project ID and Zone information from Cloud SQL instance ID.
# Expected format "project_id:zone:instance_id"
instance_id = instance_id_full.split(':')[-1]
return instance_id
def get_data_discovery_cache() -> redis.Redis:
"""
Returns a client for the data discovery Redis instance.
Redis commands can be issued directly to this client and all connection handling is done under inside `redis.Redis`.
Idle connections will be closed by `redis.Redis` automatically.
To get query cached data discovery information from the cache, you may want to provide this `Redis` instance
to a `RedisCommunicator`, `DataDiscoveryArgsFactory`, or `SingleIngestFileParquetCache` class.
"""
if not in_gcp():
return redis.Redis()
redis_host = get_secret("data_discovery_redis_host")
redis_port = get_secret("data_discovery_redis_port")
if redis_host and redis_port:
return redis.Redis(
host=redis_host,
port=int(redis_port),
)
raise ValueError("Cannot find data discovery redis secrets")
def test_get_not_in_cache_error(self, mock_project_id: MagicMock) -> None:
mock_project_id.return_value = "test-project"
mock_client = Mock()
mock_client.secret_version_path.return_value = "test-project.top_track.latest"
mock_client.access_secret_version.side_effect = Exception(
"Something bad happened")
with patch(
"google.cloud.secretmanager_v1beta1.SecretManagerServiceClient",
return_value=mock_client,
):
actual = secrets.get_secret("top_track")
assert actual is None
def __init__(self) -> None:
domain = secrets.get_secret("auth0_api_domain")
client_id = secrets.get_secret("auth0_api_client_id")
client_secret = secrets.get_secret("auth0_api_client_secret")
if not domain:
raise ValueError(
"Missing required domain for Auth0 Management API")
if not client_id:
raise ValueError(
"Missing required client_id for Auth0 Management API")
if not client_secret:
raise ValueError(
"Missing required client_secret for Auth0 Management API")
self._domain = domain
self._client_id = client_id
self._client_secret = client_secret
self.audience = f"https://{self._domain}/api/v2/"
self.client = Auth0(self._domain, self.access_token)
def test_get_not_in_cache_not_found(self,
mock_project_id: MagicMock) -> None:
mock_project_id.return_value = "test-project"
mock_client = Mock()
mock_client.secret_version_path.return_value = "test-project.top_track.latest"
mock_client.access_secret_version.side_effect = exceptions.NotFound(
"Could not find it")
with patch(
"google.cloud.secretmanager_v1beta1.SecretManagerServiceClient",
return_value=mock_client,
):
actual = secrets.get_secret("top_track")
assert actual is None
def test_get_not_in_cache_bad_payload(self,
mock_project_id: MagicMock) -> None:
mock_project_id.return_value = "test-project"
mock_client = Mock()
mock_client.secret_version_path.return_value = "test-project.top_track.latest"
mock_client.access_secret_version.return_value = (
service_pb2.AccessSecretVersionResponse(payload=None))
with patch(
"google.cloud.secretmanager_v1beta1.SecretManagerServiceClient",
return_value=mock_client,
):
actual = secrets.get_secret("top_track")
assert actual is None
def get_stripped_cloudql_instance_id(cls, schema_type: SchemaType) -> str:
"""The full instance id stored in secrets has the form
project_id:zone:instance_id. This returns just the final instance_id
For example, 'dev-data' or 'prod-state-data'.
Should be used when using the sqladmin_client().
"""
instance_id_key = cls.get_cloudql_instance_id_key(schema_type)
instance_id_full = secrets.get_secret(instance_id_key)
# Remove Project ID and Zone information from Cloud SQL instance ID.
# Expected format "project_id:zone:instance_id"
instance_id = instance_id_full.split(':')[-1]
return instance_id
def get_local_secret(secret_name: str) -> Optional[str]:
"""
Helper function for supporting local development flows.
When in development environments, we fetch file contents from `recidiviz/case_triage/local/gsm`
In Google Cloud environments, we delegate to Secrets Manager
"""
if in_development():
try:
return Path(os.path.join(local_path, "gsm",
secret_name)).read_text()
except OSError:
logging.error("Couldn't locate secret %s", secret_name)
return None
return get_secret(secret_name)
def test_get_not_in_cache(self, mock_project_id):
mock_project_id.return_value = "test-project"
payload = SecretPayload(data=bytes("Olson".encode("UTF-8")))
mock_client = Mock()
mock_client.secret_version_path.return_value = "test-project.top_track.latest"
mock_client.access_secret_version.return_value = (
service_pb2.AccessSecretVersionResponse(payload=payload)
)
with patch(
"google.cloud.secretmanager_v1beta1.SecretManagerServiceClient",
return_value=mock_client,
):
actual = secrets.get_secret("top_track")
assert actual == "Olson"
def _get_full_cloudsql_instance_id(cls, schema_type: SchemaType) -> str:
"""Rerturns the full instance id stored in secrets with the form
project_id:region:instance_id.
For example:
recidiviz-staging:us-east1:dev-state-data
"""
instance_id_key = cls._get_cloudsql_instance_id_key(schema_type)
instance_id_full = secrets.get_secret(instance_id_key)
if instance_id_full is None:
raise ValueError(
f"Unable to retrieve instance id for schema type [{schema_type}]"
)
return instance_id_full
def set_up_connection_options(prefix: str, host: str) -> CnOpts:
connection_options = CnOpts()
try:
connection_options.get_hostkey(host)
except SSHException as s:
hostkey = secrets.get_secret(f"{prefix}_hostkey")
if hostkey is None:
raise ValueError(
f"Unable to find hostkey for secret key {prefix}_hostkey"
) from s
hostkeyEntry = HostKeyEntry.from_line(hostkey)
if hostkeyEntry:
key = hostkeyEntry.key
name, keytype, _ = hostkey.split(" ")
connection_options.hostkeys.add(name, keytype, key)
else:
raise ValueError(
f"Unable to add hostkey to connection_options for secret key {prefix}_hostkey"
) from s
return connection_options
def get_server_postgres_instance_url(cls, *, schema_type: SchemaType) -> str:
instance_id_key = cls.get_cloudsql_instance_id_key(schema_type)
if instance_id_key is None:
raise ValueError(
f"Instance id is not configured for schema type [{schema_type}]"
)
db_user = cls.get_db_user(schema_type)
db_password = cls.get_db_password(schema_type)
db_name = cls.get_db_name(schema_type)
cloudsql_instance_id = secrets.get_secret(instance_id_key)
sqlalchemy_url = (
"postgresql://{db_user}:{db_password}@/{db_name}"
"?host=/cloudsql/{cloudsql_instance_id}"
).format(
db_user=db_user,
db_password=db_password,
db_name=db_name,
cloudsql_instance_id=cloudsql_instance_id,
)
return sqlalchemy_url
def get_server_postgres_instance_url(
cls, *, database_key: SQLAlchemyDatabaseKey) -> URL:
schema_type = database_key.schema_type
instance_id_key = cls._get_cloudsql_instance_id_key(schema_type)
if instance_id_key is None:
raise ValueError(
f"Instance id is not configured for schema type [{schema_type}]"
)
db_user = cls._get_db_user(database_key=database_key)
db_password = cls._get_db_password(database_key=database_key)
db_name = database_key.db_name
cloudsql_instance_id = secrets.get_secret(instance_id_key)
db_name = database_key.db_name
url = URL.create(
drivername="postgresql",
username=db_user,
password=db_password,
database=db_name,
query={"host": f"/cloudsql/{cloudsql_instance_id}"},
)
return url