def _extract_approle(settings: BaseSettings) -> Optional[Approle]:
"""Extract Approle information from environment or from BaseSettings.Config"""
_vault_role_id: Optional[str] = None
_vault_secret_id: Optional[SecretStr] = None
# Load from BaseSettings.Config
if getattr(settings.__config__, "vault_role_id", None) is not None:
_vault_role_id = settings.__config__.vault_role_id # type: ignore
if getattr(settings.__config__, "vault_secret_id", None) is not None:
if isinstance(settings.__config__.vault_secret_id,
SecretStr): # type: ignore
_vault_secret_id = settings.__config__.vault_secret_id # type: ignore
else:
_vault_secret_id = SecretStr(
settings.__config__.vault_secret_id) # type: ignore
# Load (and eventually override) from environment
if "VAULT_ROLE_ID" in os.environ:
_vault_role_id = os.environ["VAULT_ROLE_ID"]
if "VAULT_SECRET_ID" in os.environ:
_vault_secret_id = SecretStr(os.environ["VAULT_SECRET_ID"])
if _vault_role_id is not None and _vault_secret_id is not None:
return Approle(role_id=_vault_role_id, secret_id=_vault_secret_id)
return None
def from_legacy_config(other: SovereignConfig) -> "SovereignConfigv2":
new_templates = dict()
for version, templates in other.templates.items():
specs = list()
for type, path in templates.items():
if isinstance(path, str):
specs.append(
TemplateSpecification(
type=type, spec=Loadable.from_legacy_fmt(path)))
else:
# Just in case? Although this shouldn't happen
specs.append(TemplateSpecification(type=type, spec=path))
new_templates[str(version)] = specs
return SovereignConfigv2(
sources=other.sources,
templates=new_templates,
source_config=SourcesConfiguration(
refresh_rate=other.sources_refresh_rate,
cache_strategy=other.cache_strategy,
),
modifiers=other.modifiers,
global_modifiers=other.global_modifiers,
template_context=ContextConfiguration(
context=ContextConfiguration.context_from_legacy(
other.template_context),
refresh=other.refresh_context,
refresh_rate=other.context_refresh_rate,
),
matching=NodeMatching(
enabled=other.node_matching,
source_key=other.source_match_key,
node_key=other.node_match_key,
),
authentication=AuthConfiguration(
enabled=other.auth_enabled,
auth_passwords=SecretStr(other.auth_passwords),
encryption_key=SecretStr(other.encryption_key),
),
logging=LoggingConfiguration(
application_logs=ApplicationLogConfiguration(
enabled=other.enable_application_logs, ),
access_logs=AccessLogConfiguration(
enabled=other.enable_access_logs,
log_fmt=other.log_fmt,
ignore_empty_fields=other.ignore_empty_log_fields,
),
),
statsd=other.statsd,
sentry_dsn=SecretStr(other.sentry_dsn),
debug=other.debug_enabled,
legacy_fields=LegacyConfig(
regions=other.regions,
eds_priority_matrix=other.eds_priority_matrix,
dns_hard_fail=other.dns_hard_fail,
environment=other.environment,
),
)
class SecretSettings(BaseSettings):
na_postgresdsn: PostgresDsn
jp_postgresdsn: PostgresDsn
rayshift_api_key: SecretStr = SecretStr("")
github_webhook_secret: SecretStr = SecretStr("")
redisdsn: RedisDsn
class Config:
env_file = ".env"
secrets_dir = "secrets"
async def _get_data(self, endpoint, jq_filter):
"""
Basic data retrieval function
- builds the full url
- retrieves built headers
- calls a fetch and returns filtered data or an error
"""
full_url = f'{self.baseroute}/{endpoint}'
api_key = self.authentication.api_key
if not self.authentication.api_secret:
self.authentication.api_secret = SecretStr('')
api_secret: str = self.authentication.api_secret.get_secret_value()
username = self.authentication.username
timestamp = int(datetime.datetime.now().timestamp())
headers = build_headers(api_key, api_secret, username, str(timestamp))
async with ClientSession(headers=headers) as session:
data = await fetch(full_url, session)
try:
return transform_with_jq(data['content'], jq_filter)
except ValueError:
# This follows the HTTP connector's behavior for a similar situation
LOGGER.error('Could not transform the data using %s as filter',
jq_filter)
raise
except ScriptRuntimeError:
LOGGER.error('Could not transform the data using %s as filter',
jq_filter)
raise
async def test_authentication_wrong_password(
async_client: AsyncClient,
test_user: schemas.UserDB,
) -> None:
actual = await crud_user.authenticate(
test_user.email, SecretStr(TEST_USER_PASSWORD + "xxx"))
assert not actual
async def test_authentication_with_not_existed_user(
async_client: AsyncClient,
test_user: schemas.UserDB,
) -> None:
actual = await crud_user.authenticate(test_user.email + "xxx",
SecretStr(TEST_USER_PASSWORD))
assert not actual
async def verify_and_process(request, sso_type: str):
"""
Fetches user details and returns a login token.
If user does not have an account, it will be created.
:param request: starlette request object
:param sso_type: one of supported types - google/facebook/linkedin.
"""
sso_client = LoginSSOFactory.get_client(sso_type)
user_details = await sso_client.verify(request)
try:
AccountProcessor.get_user(user_details['email'])
existing_user = True
except DoesNotExist:
existing_user = False
user_details['password'] = SecretStr(Utility.generate_password())
user_details['account'] = user_details['email']
if existing_user:
AccountProcessor.get_user_details(user_details['email'])
else:
await AccountProcessor.account_setup(user_details)
tmp_token = Utility.generate_token(user_details['email'])
await AccountProcessor.confirm_email(tmp_token)
access_token = Authentication.create_access_token(data={"sub": user_details["email"]})
return existing_user, user_details, access_token
def get_connection_params(self, *, database=None):
conv = pymysql.converters.conversions.copy()
conv[246] = float
con_params = {
'host':
self.host,
'user':
self.user,
'password':
self.password.get_secret_value()
if self.password else SecretStr('').get_secret_value(),
'port':
self.port,
'database':
database,
'charset':
self.charset,
'connect_timeout':
self.connect_timeout,
'conv':
conv,
'cursorclass':
pymysql.cursors.DictCursor,
}
# remove None values
return {k: v for k, v in con_params.items() if v is not None}
def test_refresh_oauth_token(
req_mock,
get_identifier,
is_closed,
close,
connect,
retrieve_data,
snowflake_connector_oauth,
snowflake_datasource,
):
cm = SnowflakeConnector.get_snowflake_connection_manager()
# Expired JWT
snowflake_connector_oauth.user_tokens_keeper.access_token = SecretStr(
jwt.encode({'exp': datetime.now() - timedelta(hours=24)},
key='supersecret'))
req_mock.return_value.status_code = 201
req_mock.return_value.ok = False
req_mock.return_value.return_value = {
'access_token': 'token',
'refresh_token': 'token'
}
try:
snowflake_connector_oauth._retrieve_data(snowflake_datasource)
assert req_mock.call_count == 1
except Exception as e:
assert str(e) == 'HTTP Error 401: Unauthorized'
assert False
else:
assert True
finally:
cm.force_clean()
req_mock.reset_mock()
# Invalid JWT
snowflake_connector_oauth.user_tokens_keeper.access_token = SecretStr(
'PLOP')
try:
snowflake_connector_oauth._retrieve_data(snowflake_datasource)
assert req_mock.call_count == 1
except Exception as e:
assert str(e) == 'HTTP Error 401: Unauthorized'
assert False
else:
assert True
finally:
cm.force_clean()
class AuthConfiguration(BaseSettings):
enabled: bool = False
auth_passwords: SecretStr = SecretStr("")
encryption_key: SecretStr = SecretStr("")
class Config:
fields = {
"enabled": {
"env": "SOVEREIGN_AUTH_ENABLED"
},
"auth_passwords": {
"env": "SOVEREIGN_AUTH_PASSWORDS"
},
"encryption_key": {
"env": "SOVEREIGN_ENCRYPTION_KEY"
},
}
async def test_authentication_success(
async_client: AsyncClient,
test_user: schemas.UserDB,
) -> None:
actual = await crud_user.authenticate(test_user.email,
SecretStr(TEST_USER_PASSWORD))
assert actual
assert actual == test_user
def _decrypt_secrets(self, conf_dict: Dict[str, Any]):
from hummingbot.client.config.security import Security # avoids circular import
for attr, value in conf_dict.items():
attr_type = self._hb_config.__fields__[attr].type_
if attr_type == SecretStr:
decrypted_value = Security.secrets_manager.decrypt_secret_value(
attr, value.get_secret_value())
conf_dict[attr] = SecretStr(decrypted_value)
def test_prometheus_configuration(
app_config: ApplicationConfig, client, auth_required: bool
):
if auth_required:
app_config.secrets.prometheus_username = SecretStr("username")
app_config.secrets.prometheus_password = SecretStr("password")
credentials = base64.b64encode("username:password".encode("UTF-8")).decode(
"UTF-8"
)
response = client.get("/metrics")
assert response.status_code == 401
response = client.get(
"/metrics", headers=dict(Authorization=f"Basic {credentials}")
)
assert response.status_code == 200
else:
assert client.get("/metrics").status_code == 200
class Settings(BaseSettings):
jwt_secret: SecretStr = SecretStr(secrets.token_urlsafe(16))
admin_secret: SecretStr
local_path: Optional[str] = None
gcs_path: Optional[GCSPath] = None
class Config:
case_sensitive = False
env_prefix = "UPLDR_"
def plugin_configs_plain_to_secret(
plugin_configs: Dict[str, Dict[str, Any]]) -> None:
"""
In-place modification of secret values in `plugin_configs` str -> SecretStr.
"""
for plugin_name, plugin_config in plugin_configs.items():
for key, value in plugin_config.items():
if key in ("key", "password", "token") and isinstance(
value, str):
plugin_configs[plugin_name][key] = SecretStr(value)
class _ElasticsearchSection(Settings):
host: str = "localhost"
port: int = 9200
user: str = "elastic"
password: SecretStr = SecretStr("")
ca_crt_path: Path
timeout: float = 10.0
retry_on_timeout: bool = True
max_retries: int = 5
http_compress: bool = True
def test_account_setup_user_info(self):
account = {
"account": "Test_Account",
"bot": "Test",
"first_name": "Test_First",
"last_name": "Test_Last",
"password": SecretStr("Welcome@1"),
}
with pytest.raises(AssertionError):
AccountProcessor.account_setup(account_setup=account,
user="******")
class DBSettings(BaseSettings):
mongo_host: str = 'localhost'
mongo_port: int = 27017
mongo_username: str = 'modelci'
mongo_password: SecretStr = SecretStr('modelci@2020')
mongo_db: str = 'modelci'
mongo_auth_source: str = 'modelci'
auth_mechanism: str = 'SCRAM-SHA-256'
class Config:
env_file = Path(__file__).absolute().parent / '.env'
def snowflake_connector_oauth(mocker):
user_tokens_keeper = mocker.Mock(
access_token=SecretStr(OAUTH_ACCESS_TOKEN),
refresh_token=SecretStr(OAUTH_REFRESH_TOKEN),
update_tokens=mocker.Mock(),
)
sso_credentials_keeper = mocker.Mock(
client_id=OAUTH_CLIENT_ID,
client_secret=SecretStr(OAUTH_CLIENT_SECRET))
return SnowflakeConnector(
name='test_name',
authentication_method=AuthenticationMethod.OAUTH,
user='******',
password='******',
account='test_account',
token_endpoint=OAUTH_TOKEN_ENDPOINT,
token_endpoint_content_type=OAUTH_TOKEN_ENDPOINT_CONTENT_TYPE,
user_tokens_keeper=user_tokens_keeper,
sso_credentials_keeper=sso_credentials_keeper,
default_warehouse='default_wh',
)
class TestAppSettings(AppSettings):
debug: bool = True
title: str = "Test FastAPI example application"
secret_key: SecretStr = SecretStr("test_secret")
database_url: PostgresDsn
max_connection_count: int = 5
min_connection_count: int = 5
logging_level: int = logging.DEBUG
def validate_postgres_conn(cls, v: Optional[str],
values: Dict[str, Any]) -> str:
if isinstance(v, str):
return v
password: SecretStr = values.get("POSTGRES_PASSWORD", SecretStr(""))
return "{scheme}://{user}:{password}@{host}/{db}".format(
scheme="postgresql+asyncpg",
user=values.get("POSTGRES_USER"),
password=password.get_secret_value(),
host=values.get("POSTGRES_HOST"),
db=values.get("POSTGRES_DB"),
)
class DevAppSettings(AppSettings):
"""Application settings with override params for dev environment."""
# fastapi.applications.FastAPI initializer kwargs
debug: bool = True
title: str = "Dev Mirumon Service"
# Auth settings
secret_key: SecretStr = SecretStr("dev-secret-key")
shared_key: SecretStr = SecretStr("dev-shared-key")
# Infrastructure settings
postgres_dsn: PostgresDsn = (
"postgresql://*****:*****@localhost/postgres" # type:ignore
)
rabbit_dsn: AnyUrl = "amqp://*****:*****@localhost" # type:ignore
redis_dsn: RedisDsn = "redis://*****:*****@localhost/0" # type: ignore
class Config(AppSettings.Config):
env_file = ".env"
def test_secrets_path_url(tmp_path):
(tmp_path / 'foo').write_text('http://www.example.com')
(tmp_path / 'bar').write_text('snap')
class Settings(BaseSettings):
foo: HttpUrl
bar: SecretStr
class Config:
secrets_dir = tmp_path
assert Settings().dict() == {'foo': 'http://www.example.com', 'bar': SecretStr('snap')}
def _extract_vault_token(settings: BaseSettings) -> Optional[SecretStr]:
"""Extract Vault token from environment, from .vault-token file or from BaseSettings.Config"""
_vault_token: SecretStr
if "VAULT_TOKEN" in os.environ:
_vault_token = SecretStr(os.environ["VAULT_TOKEN"])
return _vault_token
with suppress(FileNotFoundError):
with open(Path.home() / ".vault-token") as token_file:
_vault_token = SecretStr(token_file.read().strip())
return _vault_token
if getattr(settings.__config__, "vault_token", None) is not None:
if isinstance(settings.__config__.vault_token,
SecretStr): # type: ignore
_vault_token = settings.__config__.vault_token # type: ignore
else:
_vault_token = SecretStr(
settings.__config__.vault_token) # type: ignore
return _vault_token
return None
def test_account_setup_user_info(self):
account = {
"account": "Test_Account",
"bot": "Test",
"first_name": "Test_First",
"last_name": "Test_Last",
"password": SecretStr("Welcome@1"),
}
with pytest.raises(AppException):
loop = asyncio.new_event_loop()
loop.run_until_complete(
AccountProcessor.account_setup(account_setup=account,
user="******"))
class TestAppSettings(AppSettings):
"""Application settings with override params for test environment."""
# fastapi.applications.FastAPI initializer kwargs
debug: bool = False
title: str = "Test Mirumon Service"
# Timeout settings
rest_max_response_time: float = 4.0
event_timeout: int = 4
# Auth settings
secret_key: SecretStr = SecretStr("test-secret-key")
shared_key: SecretStr = SecretStr("test-shared-key")
# Infrastructure settings
postgres_dsn: PostgresDsn = "postgresql://*****:*****@localhost/postgres" # type: ignore # noqa: E501
rabbit_dsn: AnyUrl = "amqp://*****:*****@localhost" # type: ignore
redis_dsn: RedisDsn = "redis://user@localhost/0" # type: ignore
class Config(AppSettings.Config):
env_file = ".env.test"
def _retrieve_data(self, data_source):
connection = pyhdb.connect(
self.host,
self.port,
self.user,
self.password.get_secret_value()
if self.password else SecretStr('').get_secret_value(),
)
df = pandas_read_sql(data_source.query, con=connection)
connection.close()
return df
class InfluxDBConfig(BaseModel):
'''
InfluxDB configuration.
'''
#: Whether to enable pushing to InfluxDB
enable: bool = False
#: URL to connect to
url: str = 'http://localhost:8086'
#: Token that allows access
token: SecretStr = SecretStr('')
#: Organization to use
org: str = 'rctmon'
#: Database to use
bucket: str = 'rctmon'
async def get_test_user() -> schemas.UserDB:
faker = Faker()
profile = faker.profile()
TEST_USER_EMAIL = profile.get("mail", None)
TEST_USER_USERNAME = profile.get("username", None)
user_db = await crud_user.get_user_by_email(email=TEST_USER_EMAIL)
if user_db is None:
user_in = schemas.UserCreate(
username=TEST_USER_USERNAME,
email=TEST_USER_EMAIL,
password=SecretStr(TEST_USER_PASSWORD),
)
user_id = await crud_user.create(payload=user_in)
user_db = await crud_user.get(user_id)
return user_db
def test_account_setup(self):
account = {
"account": "Test_Account",
"bot": "Test",
"email": "*****@*****.**",
"first_name": "Test_First",
"last_name": "Test_Last",
"password": SecretStr("Welcome@1"),
}
actual = AccountProcessor.account_setup(account_setup=account,
user="******")
assert actual["role"] == "admin"
assert actual["_id"]
assert actual["account"]
assert actual["bot"]