def test_get_secrets_default(monkeypatch):
"""
Test getting a secret falls back to provided default with the file being there.
"""
the_secret = "Shhhh"
the_secret_name = "SECRET"
secret = getsecret(the_secret_name, the_secret)
assert the_secret == secret
def test_get_secrets(monkeypatch):
"""Test getting a secret from the special /run/secrets directory."""
the_secret = "Shhhh"
the_secret_name = "SECRET"
open_mock = unittest.mock.mock_open(read_data=the_secret)
monkeypatch.setattr("builtins.open", open_mock)
secret = getsecret(the_secret_name, "Not the secret")
assert the_secret == secret
open_mock.assert_called_once_with(Path("/run/secrets") / the_secret_name, "r")
def connect(
log_level: Union[str, None] = None,
db_port: Union[int, None] = None,
db_user: Union[str, None] = None,
db_pass: Union[str, None] = None,
db_host: Union[str, None] = None,
db_connection_pool_size: Union[int, None] = None,
db_connection_pool_overflow: Union[int, None] = None,
redis_host: Union[str, None] = None,
redis_port: Union[int, None] = None,
redis_password: Union[str, None] = None,
conn: Union[Connection, None] = None,
) -> Connection:
"""
Connects flowmachine to a database, and performs initial set-up routines.
You may provide a Settings object here, which can specify the database
you wish to connect to, logging behaviour, available tables and so on.
Parameters
----------
log_level : str, default "error"
Level to log at
db_port : int, default 9000
Port number to connect to flowdb
db_user : str, default "analyst"
Name of user to connect to flowdb as
db_pass : str, default "foo"
Password to connect to flowdb
db_host : str, default "localhost"
Hostname of flowdb server
db_connection_pool_size : int, default 5
Default number of database connections to use
db_connection_pool_overflow : int, default 1
Number of extra database connections to allow
redis_host : str, default "localhost"
Hostname for redis server.
redis_port : int, default 6379
Port the redis server is available on
redis_password : str, default "fm_redis"
Password for the redis instance
conn : flowmachine.core.Connection
Optionally provide an existing Connection object to use, overriding any the db options specified here.
Returns
-------
Connection
Notes
-----
All parameters can also be provided as environment variables, named the same
but in uppercase, e.g. `env LOG_LEVEL=error` instead of `connect(log_level="error")`.
If a parameter is provided, and an environment variable is set,
then the provided value is used. If neither is provided, the defaults as given
in the docstring are used.
Parameters can _also_ be set using Docker secrets, in which case a file with the name
of the parameter in upper case should be present at /run/secrets/THE_PARAM.
If a secret is available, the secret takes precedence over both the environment variable, and
the default.
"""
log_level = (getsecret("LOG_LEVEL", os.getenv("LOG_LEVEL", "error"))
if log_level is None else log_level)
db_port = int(
getsecret("FLOWDB_PORT", os.getenv("FLOWDB_PORT", 9000)
) if db_port is None else db_port)
db_user = (getsecret("FLOWDB_USER", os.getenv("FLOWDB_USER", "analyst"))
if db_user is None else db_user)
db_pass = (getsecret("FLOWDB_PASS", os.getenv("FLOWDB_PASS"))
if db_pass is None else db_pass)
db_host = (getsecret("FLOWDB_HOST", os.getenv("FLOWDB_HOST", "localhost"))
if db_host is None else db_host)
db_connection_pool_size = (int(
getsecret("DB_CONNECTION_POOL_SIZE",
os.getenv("DB_CONNECTION_POOL_SIZE", 5)))
if db_connection_pool_size is None else
db_connection_pool_size)
db_connection_pool_overflow = int(
getsecret("DB_CONNECTION_POOL_OVERFLOW",
os.getenv("DB_CONNECTION_POOL_OVERFLOW", 1)) if
db_connection_pool_overflow is None else db_connection_pool_overflow)
redis_host = (getsecret("REDIS_HOST", os.getenv("REDIS_HOST", "localhost"))
if redis_host is None else redis_host)
redis_port = int(
getsecret("REDIS_PORT", os.getenv("REDIS_PORT", 6379)
) if redis_port is None else redis_port)
redis_pw = (getsecret("REDIS_PASSWORD_FILE", os.getenv("REDIS_PASSWORD"))
if redis_password is None else redis_password)
if db_pass is None:
raise ValueError(
"You must provide a secret named FLOWDB_PASS, set an environment variable named FLOWDB_PASS, or provide a db_pass argument."
)
if redis_pw is None:
raise ValueError(
"You must provide a secret named REDIS_PASSWORD_FILE, set an environment variable named REDIS_PASSWORD, or provide a redis_password argument."
)
try:
Query.connection
warnings.warn("FlowMachine already started. Ignoring.")
except AttributeError:
_init_logging(log_level)
if conn is None:
conn = Connection(
host=db_host,
port=db_port,
user=db_user,
password=db_pass,
database="flowdb",
pool_size=db_connection_pool_size,
overflow=db_connection_pool_overflow,
)
Query.connection = conn
Query.redis = redis.StrictRedis(host=redis_host,
port=redis_port,
password=redis_pw)
_start_threadpool(thread_pool_size=db_connection_pool_size)
print(f"FlowMachine version: {flowmachine.__version__}")
print(
f"Flowdb running on: {db_host}:{db_port}/flowdb (connecting user: {db_user})"
)
return Query.connection
def connect(
log_level: Union[str, None] = None,
write_log_file: Union[bool, None] = None,
db_port: Union[int, None] = None,
db_user: Union[str, None] = None,
db_pw: Union[str, None] = None,
db_host: Union[str, None] = None,
db_name: Union[str, None] = None,
db_connection_pool_size: Union[int, None] = None,
db_connection_pool_overflow: Union[int, None] = None,
redis_host: Union[str, None] = None,
redis_port: Union[int, None] = None,
redis_password: Union[str, None] = None,
conn: Union[Connection, None] = None,
) -> Connection:
"""
Connects flowmachine to a database, and performs initial set-up routines.
You may provide a Settings object here, which can specify the database
you wish to connect to, logging behaviour, available tables and so on.
Parameters
----------
log_level : str, default "error"
Level to log at
write_log_file : bool, default False
If True, logging output will be written to the file 'flowmachine-debug.log'
in the directory '/var/log/flowmachine/' (or the directory given by the
environment variable 'LOG_DIRECTORY' if it is set). Log files are rotated
at midnight.
db_port : int, default 9000
Port number to connect to flowdb
db_user : str, default "analyst"
Name of user to connect to flowdb as
db_pw : str, default "foo"
Password to connect to flowdb
db_host : str, default "localhost"
Hostname of flowdb server
db_name : str, default "flowdb"
Name of database to connect to.
db_connection_pool_size : int, default 5
Default number of database connections to use
db_connection_pool_overflow : int, default 1
Number of extra database connections to allow
redis_host : str, default "localhost"
Hostname for redis server.
redis_port : int, default 6379
Port the redis server is available on
conn : flowmachine.core.Connection
Optionally provide an existing Connection object to use, overriding any the db options specified here.
Returns
-------
Connection
Notes
-----
All parameters can also be provided as environment variables, named the same
but in uppercase, e.g. `env LOG_LEVEL=error` instead of `connect(log_level="error")`.
If a parameter is provided, and an environment variable is set,
then the provided value is used. If neither is provided, the defaults as given
in the docstring are used.
Parameters can _also_ be set using Docker secrets, in which case a file with the name
of the parameter in upper case should be present at /run/secrets/THE_PARAM.
If a secret is available, the secret takes precedence over both the environment variable, and
the default.
"""
log_level = (getsecret("LOG_LEVEL", os.getenv("LOG_LEVEL", "error"))
if log_level is None else log_level)
write_log_file = (("TRUE" == getsecret(
"WRITE_LOG_FILE", os.getenv("WRITE_LOG_FILE", "FALSE")).upper())
if write_log_file is None else write_log_file)
db_port = int(
getsecret("FLOWDB_PORT", os.getenv("FLOWDB_PORT", 9000)
) if db_port is None else db_port)
db_user = (getsecret("DB_USER", os.getenv("DB_USER", "analyst"))
if db_user is None else db_user)
db_pw = getsecret("DB_PW", os.getenv("DB_PW",
"foo")) if db_pw is None else db_pw
db_host = (getsecret("DB_HOST", os.getenv("DB_HOST", "localhost"))
if db_host is None else db_host)
db_name = (getsecret("DB_NAME", os.getenv("DB_NAME", "flowdb"))
if db_name is None else db_name)
db_connection_pool_size = (int(
getsecret("DB_CONNECTION_POOL_SIZE",
os.getenv("DB_CONNECTION_POOL_SIZE", 5)))
if db_connection_pool_size is None else
db_connection_pool_size)
db_connection_pool_overflow = int(
getsecret("DB_CONNECTION_POOL_OVERFLOW",
os.getenv("DB_CONNECTION_POOL_OVERFLOW", 1)) if
db_connection_pool_overflow is None else db_connection_pool_overflow)
redis_host = (getsecret("REDIS_HOST", os.getenv("REDIS_HOST", "localhost"))
if redis_host is None else redis_host)
redis_port = int(
getsecret("REDIS_PORT", os.getenv("REDIS_PORT", 6379)
) if redis_port is None else redis_port)
redis_pw = (getsecret("REDIS_PASSWORD_FILE",
os.getenv("REDIS_PASSWORD", "fm_redis"))
if redis_password is None else redis_password)
try:
Query.connection
warnings.warn("FlowMachine already started. Ignoring.")
except AttributeError:
_init_logging(log_level, write_log_file)
if conn is None:
conn = Connection(
db_port,
db_user,
db_pw,
db_host,
db_name,
db_connection_pool_size,
db_connection_pool_overflow,
)
Query.connection = conn
Query.redis = redis.StrictRedis(host=redis_host,
port=redis_port,
password=redis_pw)
_start_threadpool(db_connection_pool_size)
print(f"FlowMachine version: {flowmachine.__version__}")
print(
f"Flowdb running on: {db_host}:{db_port}/{db_name} (connecting user: {db_user})"
)
return Query.connection