def test_clear(self): secrets.set_secret('top_track', 'An Eagle In Your Mind') secrets.set_secret('solid_track', 'Kaini Industries') secrets.clear_secrets() assert secrets.get_secret('top_track') is None assert secrets.get_secret('solid_track') is None
def for_region(cls, region_code: str) -> "SftpAuth": prefix = f"{region_code}_sftp" host = secrets.get_secret(f"{prefix}_host") if host is None: raise ValueError(f"Unable to find host name for secret key {prefix}_host") username = secrets.get_secret(f"{prefix}_username") password = secrets.get_secret(f"{prefix}_password") connection_options = SftpAuth.set_up_connection_options(prefix, host) return SftpAuth(host, username, password, connection_options)
def _create_engine(): db_user = secrets.get_secret('us_ma_middlesex_db_user') db_password = secrets.get_secret('us_ma_middlesex_db_password') db_name = secrets.get_secret('us_ma_middlesex_db_name') cloudsql_instance_id = secrets.get_secret('us_ma_middlesex_instance_id') return sqlalchemy.create_engine( sqlalchemy.engine.url.URL( drivername=_DATABASE_TYPE, username=db_user, password=db_password, database=db_name, query={'host': '/cloudsql/{}'.format(cloudsql_instance_id)}))
def _get_server_postgres_instance_url( cls, *, db_user_key: str, db_password_key: str, db_name_key: str, cloudsql_instance_id_key: str) -> str: db_user = secrets.get_secret(db_user_key) db_password = secrets.get_secret(db_password_key) db_name = secrets.get_secret(db_name_key) cloudsql_instance_id = secrets.get_secret(cloudsql_instance_id_key) sqlalchemy_url = ('postgresql://{db_user}:{db_password}@/{db_name}' '?host=/cloudsql/{cloudsql_instance_id}').format( db_user=db_user, db_password=db_password, db_name=db_name, cloudsql_instance_id=cloudsql_instance_id) return sqlalchemy_url
def get_proxies(use_test=False): """Retrieves proxy username/pass from environment variables Retrieves proxy information to use in requests to third-party services. If not in production environment, defaults to test proxy credentials (so problems during test runs don't risk our main proxy IP's reputation). Args: use_test: (bool) Use test proxy credentials, not prod Returns: Proxies dict for requests library, in the form: {'<protocol>': '<http://<proxy creds>@<proxy url>'} Raises: Exception: General exception, since scraper cannot proceed without this """ if not environment.in_gae() or use_test: return None user_var = 'proxy_user' pass_var = 'proxy_password' proxy_url = secrets.get_secret("proxy_url") if proxy_url is None: raise Exception("No proxy url") # On the proxy side, a random ip is chosen for a session it has not seen # so collisions can still happen so we increase the integer to reduce the # odds. base_proxy_user = secrets.get_secret(user_var) proxy_user = PROXY_USER_TEMPLATE.format( base_proxy_user, random.random()) proxy_password = secrets.get_secret(pass_var) if (base_proxy_user is None) or (proxy_password is None): raise Exception("No proxy user/pass") proxy_credentials = proxy_user + ":" + proxy_password proxy_request_url = 'http://' + proxy_credentials + "@" + proxy_url proxies = {'http': proxy_request_url, 'https': proxy_request_url} return proxies
def _get_db_password(cls, *, database_key: SQLAlchemyDatabaseKey) -> str: secret_key = f"{cls._secret_manager_prefix(database_key)}_db_password" password = secrets.get_secret(secret_key) if password is None: raise ValueError( f"Unable to retrieve database password for key [{secret_key}]") return password
def _get_db_host(cls, *, database_key: SQLAlchemyDatabaseKey) -> str: secret_key = f"{cls._secret_manager_prefix(database_key)}_db_host" host = secrets.get_secret(secret_key) if host is None: raise ValueError( f"Unable to retrieve database host for key [{secret_key}]") return host
def get_stripped_cloudsql_instance_id( cls, schema_type: SchemaType ) -> Optional[str]: """The full instance id stored in secrets has the form project_id:zone:instance_id. This returns just the final instance_id, for example, 'dev-data' or 'prod-state-data'. If a key is not configured for the given schema, returns None. Should be used when using the sqladmin_client(). """ instance_id_key = cls.get_cloudsql_instance_id_key(schema_type) if instance_id_key is None: return None instance_id_full = secrets.get_secret(instance_id_key) if instance_id_full is None: raise ValueError( f"Unable to retrieve instance id for schema type [{schema_type}]" ) # Remove Project ID and Zone information from Cloud SQL instance ID. # Expected format "project_id:zone:instance_id" instance_id = instance_id_full.split(":")[-1] return instance_id
def get_db_user(cls, schema_type: SchemaType) -> str: user = secrets.get_secret(cls.get_db_user_key(schema_type)) if user is None: raise ValueError( f"Unable to retrieve database user for schema type [{schema_type}]" ) return user
def get_db_readonly_password(cls, schema_type: SchemaType) -> str: password = secrets.get_secret(cls.get_db_readonly_password_key(schema_type)) if password is None: raise ValueError( f"Unable to retrieve database readonly password for schema type [{schema_type}]" ) return password
def get_db_name(cls, schema_type: SchemaType) -> str: db_name = secrets.get_secret(cls.get_db_name_key(schema_type)) if db_name is None: raise ValueError( f"Unable to retrieve database name for schema type [{schema_type}]" ) return db_name
def get_db_host(cls, schema_type: SchemaType) -> str: host = secrets.get_secret(cls.get_db_host_key(schema_type)) if host is None: raise ValueError( f"Unable to retrieve database host for schema type [{schema_type}]" ) return host
def _get_db_user(cls, *, database_key: SQLAlchemyDatabaseKey) -> str: secret_key = f"{cls._secret_manager_prefix(database_key)}_db_user" user = secrets.get_secret(secret_key) if user is None: raise ValueError( f"Unable to retrieve database user for key [{secret_key}]") return user
def get_headers(): """Retrieves headers (e.g., user agent string) from environment variables Retrieves user agent string information to use in requests to third-party services. Args: N/A Returns: Headers dict for the requests library, in the form: {'User-Agent': '<user agent string>'} Raises: Exception: General exception, since scraper cannot proceed without this """ in_prod = environment.in_gae() if not in_prod: user_agent_string = ('For any issues, concerns, or rate constraints,' 'e-mail [email protected]') else: user_agent_string = secrets.get_secret("user_agent") if not user_agent_string: raise Exception("No user agent string") headers = {'User-Agent': user_agent_string} return headers
def _get_sendgrid_api_key(self) -> str: """Returns the value for the SendGrid API key from the secrets manager.""" sendgrid_api_value = secrets.get_secret(_SENDGRID_API_KEY) if not sendgrid_api_value: raise ValueError( f"Could not get secret value for `Sendgrid API Key`. Provided with " f"key={_SENDGRID_API_KEY}") return sendgrid_api_value
def get_cdn_static_ip() -> str: cdn_ip = secrets.get_secret(_PO_REPORT_CDN_STATIC_IP_KEY) if not cdn_ip: raise ValueError( f'Could not get secret value for `CDN static IP`. Provided with ' f'key={_PO_REPORT_CDN_STATIC_IP_KEY}') return cdn_ip
def _get_server_postgres_instance_url(cls, *, schema_type: SchemaType) -> str: instance_id_key = cls.get_cloudsql_instance_id_key(schema_type) if instance_id_key is None: raise ValueError(f'Instance id is not configured for schema type [{schema_type}]') db_user = secrets.get_secret(f'{cls._SCHEMA_TO_SECRET_MANAGER_PREFIX[schema_type]}_db_user') db_password = secrets.get_secret(f'{cls._SCHEMA_TO_SECRET_MANAGER_PREFIX[schema_type]}_db_password') db_name = cls.get_db_name(schema_type) cloudsql_instance_id = secrets.get_secret(instance_id_key) sqlalchemy_url = ('postgresql://{db_user}:{db_password}@/{db_name}' '?host=/cloudsql/{cloudsql_instance_id}').format( db_user=db_user, db_password=db_password, db_name=db_name, cloudsql_instance_id=cloudsql_instance_id) return sqlalchemy_url
def cloudsql_instance_id() -> str: """Get the Cloud SQL instance ID.""" instance_id_full = secrets.get_secret('cloudsql_instance_id') # Remove Project ID and Zone information from Cloud SQL instance ID. # Expected format "project_id:zone:instance_id" instance_id = instance_id_full.split(':')[-1] return instance_id
def get_data_discovery_cache() -> redis.Redis: """ Returns a client for the data discovery Redis instance. Redis commands can be issued directly to this client and all connection handling is done under inside `redis.Redis`. Idle connections will be closed by `redis.Redis` automatically. To get query cached data discovery information from the cache, you may want to provide this `Redis` instance to a `RedisCommunicator`, `DataDiscoveryArgsFactory`, or `SingleIngestFileParquetCache` class. """ if not in_gcp(): return redis.Redis() redis_host = get_secret("data_discovery_redis_host") redis_port = get_secret("data_discovery_redis_port") if redis_host and redis_port: return redis.Redis( host=redis_host, port=int(redis_port), ) raise ValueError("Cannot find data discovery redis secrets")
def test_get_not_in_cache_error(self, mock_project_id: MagicMock) -> None: mock_project_id.return_value = "test-project" mock_client = Mock() mock_client.secret_version_path.return_value = "test-project.top_track.latest" mock_client.access_secret_version.side_effect = Exception( "Something bad happened") with patch( "google.cloud.secretmanager_v1beta1.SecretManagerServiceClient", return_value=mock_client, ): actual = secrets.get_secret("top_track") assert actual is None
def __init__(self) -> None: domain = secrets.get_secret("auth0_api_domain") client_id = secrets.get_secret("auth0_api_client_id") client_secret = secrets.get_secret("auth0_api_client_secret") if not domain: raise ValueError( "Missing required domain for Auth0 Management API") if not client_id: raise ValueError( "Missing required client_id for Auth0 Management API") if not client_secret: raise ValueError( "Missing required client_secret for Auth0 Management API") self._domain = domain self._client_id = client_id self._client_secret = client_secret self.audience = f"https://{self._domain}/api/v2/" self.client = Auth0(self._domain, self.access_token)
def test_get_not_in_cache_not_found(self, mock_project_id: MagicMock) -> None: mock_project_id.return_value = "test-project" mock_client = Mock() mock_client.secret_version_path.return_value = "test-project.top_track.latest" mock_client.access_secret_version.side_effect = exceptions.NotFound( "Could not find it") with patch( "google.cloud.secretmanager_v1beta1.SecretManagerServiceClient", return_value=mock_client, ): actual = secrets.get_secret("top_track") assert actual is None
def test_get_not_in_cache_bad_payload(self, mock_project_id: MagicMock) -> None: mock_project_id.return_value = "test-project" mock_client = Mock() mock_client.secret_version_path.return_value = "test-project.top_track.latest" mock_client.access_secret_version.return_value = ( service_pb2.AccessSecretVersionResponse(payload=None)) with patch( "google.cloud.secretmanager_v1beta1.SecretManagerServiceClient", return_value=mock_client, ): actual = secrets.get_secret("top_track") assert actual is None
def get_stripped_cloudql_instance_id(cls, schema_type: SchemaType) -> str: """The full instance id stored in secrets has the form project_id:zone:instance_id. This returns just the final instance_id For example, 'dev-data' or 'prod-state-data'. Should be used when using the sqladmin_client(). """ instance_id_key = cls.get_cloudql_instance_id_key(schema_type) instance_id_full = secrets.get_secret(instance_id_key) # Remove Project ID and Zone information from Cloud SQL instance ID. # Expected format "project_id:zone:instance_id" instance_id = instance_id_full.split(':')[-1] return instance_id
def get_local_secret(secret_name: str) -> Optional[str]: """ Helper function for supporting local development flows. When in development environments, we fetch file contents from `recidiviz/case_triage/local/gsm` In Google Cloud environments, we delegate to Secrets Manager """ if in_development(): try: return Path(os.path.join(local_path, "gsm", secret_name)).read_text() except OSError: logging.error("Couldn't locate secret %s", secret_name) return None return get_secret(secret_name)
def test_get_not_in_cache(self, mock_project_id): mock_project_id.return_value = "test-project" payload = SecretPayload(data=bytes("Olson".encode("UTF-8"))) mock_client = Mock() mock_client.secret_version_path.return_value = "test-project.top_track.latest" mock_client.access_secret_version.return_value = ( service_pb2.AccessSecretVersionResponse(payload=payload) ) with patch( "google.cloud.secretmanager_v1beta1.SecretManagerServiceClient", return_value=mock_client, ): actual = secrets.get_secret("top_track") assert actual == "Olson"
def _get_full_cloudsql_instance_id(cls, schema_type: SchemaType) -> str: """Rerturns the full instance id stored in secrets with the form project_id:region:instance_id. For example: recidiviz-staging:us-east1:dev-state-data """ instance_id_key = cls._get_cloudsql_instance_id_key(schema_type) instance_id_full = secrets.get_secret(instance_id_key) if instance_id_full is None: raise ValueError( f"Unable to retrieve instance id for schema type [{schema_type}]" ) return instance_id_full
def set_up_connection_options(prefix: str, host: str) -> CnOpts: connection_options = CnOpts() try: connection_options.get_hostkey(host) except SSHException as s: hostkey = secrets.get_secret(f"{prefix}_hostkey") if hostkey is None: raise ValueError( f"Unable to find hostkey for secret key {prefix}_hostkey" ) from s hostkeyEntry = HostKeyEntry.from_line(hostkey) if hostkeyEntry: key = hostkeyEntry.key name, keytype, _ = hostkey.split(" ") connection_options.hostkeys.add(name, keytype, key) else: raise ValueError( f"Unable to add hostkey to connection_options for secret key {prefix}_hostkey" ) from s return connection_options
def get_server_postgres_instance_url(cls, *, schema_type: SchemaType) -> str: instance_id_key = cls.get_cloudsql_instance_id_key(schema_type) if instance_id_key is None: raise ValueError( f"Instance id is not configured for schema type [{schema_type}]" ) db_user = cls.get_db_user(schema_type) db_password = cls.get_db_password(schema_type) db_name = cls.get_db_name(schema_type) cloudsql_instance_id = secrets.get_secret(instance_id_key) sqlalchemy_url = ( "postgresql://{db_user}:{db_password}@/{db_name}" "?host=/cloudsql/{cloudsql_instance_id}" ).format( db_user=db_user, db_password=db_password, db_name=db_name, cloudsql_instance_id=cloudsql_instance_id, ) return sqlalchemy_url
def get_server_postgres_instance_url( cls, *, database_key: SQLAlchemyDatabaseKey) -> URL: schema_type = database_key.schema_type instance_id_key = cls._get_cloudsql_instance_id_key(schema_type) if instance_id_key is None: raise ValueError( f"Instance id is not configured for schema type [{schema_type}]" ) db_user = cls._get_db_user(database_key=database_key) db_password = cls._get_db_password(database_key=database_key) db_name = database_key.db_name cloudsql_instance_id = secrets.get_secret(instance_id_key) db_name = database_key.db_name url = URL.create( drivername="postgresql", username=db_user, password=db_password, database=db_name, query={"host": f"/cloudsql/{cloudsql_instance_id}"}, ) return url