def load(
username: str,
password: str,
host: str,
port: int,
db_name: str,
collection: str,
azure: bool,
filename: typing.Optional[str],
update: bool,
progress: bool,
production: bool,
) -> None:
coords = database.DatabaseCoordinates(username=username,
password=password,
host=host,
port=port,
production=production)
reader: transform.ExtractProtocol
if azure:
LOGGER.info(f'Loading data from Azure into {db_name}.{collection}')
azure_coords = transform.AzureCoordinates.from_env()
reader = transform.AzureExtractReader(azure_coords)
elif filename:
LOGGER.info(
f'Loading data from {filename} into {db_name}.{collection}')
reader = transform.LocalExtractReader(filename)
else:
LOGGER.error('Must supply a filename or use azure')
raise ValueError('Must supply a filename or use azure')
data = transform.transform(reader, progress)
database.load(coords, db_name, collection, data, update)
LOGGER.info(f'Finished loading data')
def database_coordinates(username: str, password: str, host: str, port: int,
production: bool) -> database.DatabaseCoordinates:
return database.DatabaseCoordinates(
username=username,
password=password,
host=host,
port=port,
production=production,
)
App = flask.Flask(__name__)
DEFAULT_ETL_SECRET_KEY = 'no key'
App.config.update(
dict(SECRET_KEY=os.environ.get('ETL_SECRET_KEY',
DEFAULT_ETL_SECRET_KEY), ))
DATABASE_COORDS = database.DatabaseCoordinates(
username=os.environ.get(database.EnvVariables.username.value,
default=database.EnvDefaults.username.value),
password=os.environ.get(database.EnvVariables.password.value,
default=database.EnvDefaults.password.value),
host=os.environ.get(database.EnvVariables.host.value,
default=database.EnvDefaults.host.value),
port=int(
os.environ.get(database.EnvVariables.port.value,
default=database.EnvDefaults.port.value)),
production=bool(
os.environ.get(database.EnvVariables.production.value,
default=database.EnvDefaults.production.value)),
)
DATABASE_NAME = os.environ.get(database.EnvVariables.database.value,
default=database.EnvDefaults.database.value)
COLLECTION = os.environ.get(database.EnvVariables.collection.value,
default=database.EnvDefaults.collection.value)
def _run_tl_and_verify() -> None: