class DbtRunner(Runner):
def __init__(self,
elt_context: ELTContext,
dbt_service: DbtService = None):
self.context = elt_context
self.dbt_service = dbt_service or DbtService(elt_context.project)
self.connection_service = ConnectionService(elt_context)
@property
def project(self):
return self.context.project
def run(self, session, dry_run=False, models=None):
# we should probably refactor this part to have an ELTContext object already
# filled with the each plugins' configuration so we don't have to query
# multiple times for the same data.
settings_service = PluginSettingsService(self.project)
try:
load = self.connection_service.load_params()
analyze = self.connection_service.analyze_params()
env = {
# inject the inferred 'schemas' from the ELTContext
"MELTANO_LOAD_SCHEMA": load["schema"],
"MELTANO_ANALYZE_SCHEMA": analyze["schema"],
"DBT_TARGET": self.connection_service.dialect,
# inject the extractor & loader configuration as ENV variables.
# that means dbt will have access to all the configuration of
# the extractor and loader
**settings_service.as_env(session, self.context.extractor.ref),
**settings_service.as_env(session, self.context.loader.ref),
}
except Exception as e:
logging.debug("Could not inject environment to dbt.")
logging.debug(
f"Could not hydrate ENV from the EltContext: {str(e)}")
raise e
# Get an asyncio event loop and use it to run the dbt commands
loop = asyncio.get_event_loop()
loop.run_until_complete(self.dbt_service.deps())
if models is not None:
models = models.replace("-", "_")
if dry_run:
loop.run_until_complete(self.dbt_service.compile(models, env=env))
else:
loop.run_until_complete(self.dbt_service.run(models, env=env))
def test_params(
self,
project_add_service,
elt_context_builder,
session,
tap,
loader,
load_params,
analyze_params,
):
project_add_service.add(PluginType.LOADERS, loader)
elt_context = (elt_context_builder.with_extractor(
tap.name).with_loader(loader).context(session))
subject = ConnectionService(elt_context)
assert subject.load_params() == load_params
assert subject.analyze_params() == analyze_params
def test_params_with_transform(
self,
project_add_service,
elt_context_builder,
session,
tap,
dbt,
loader,
analyze_params,
):
try:
project_add_service.add(PluginType.LOADERS, loader)
except PluginAlreadyAddedException:
pass
elt_context = (
elt_context_builder.with_session(session).with_extractor(
tap.name).with_loader(loader).with_transform("run").context())
subject = ConnectionService(elt_context)
assert subject.analyze_params() == analyze_params
def get_db_engine(self, extractor, loader, transform):
project = Project.find()
context = (ELTContextBuilder(project).with_session(
db.session).with_extractor(extractor).with_loader(
loader).with_transform(transform).context())
connection_service = ConnectionService(context)
engine_hooks = []
dialect = connection_service.dialect
engine_uri = None
params = None
try:
params = connection_service.analyze_params()
engine_uri = connection_service.analyze_uri()
if dialect not in ENABLED_DIALECTS:
raise UnsupportedConnectionDialect(dialect)
except DialectNotSupportedError:
raise UnsupportedConnectionDialect(dialect)
if dialect == "postgres":
def set_connection_schema(raw, conn):
schema = params["schema"]
with raw.cursor() as cursor:
res = cursor.execute(f"SET search_path TO {schema};")
logging.debug(f"Connection schema set to {schema}")
engine_hooks.append(lambda engine: listen(engine, "first_connect",
set_connection_schema))
engine = sqlalchemy.create_engine(engine_uri)
for hook in engine_hooks:
hook(engine)
return engine
def __init__(
self,
elt_context: ELTContext,
config_service: ConfigService = None,
connection_service: ConnectionService = None,
**config,
):
self.context = elt_context
self.config = config
self.config_service = config_service or ConfigService(
elt_context.project)
self.connection_service = connection_service or ConnectionService(
elt_context)
self.tap_config_dir = Path(
config.get("tap_config_dir", "/etc/singer/tap"))
self.target_config_dir = Path(
config.get("target_config_dir", "/etc/singer/target"))
def __init__(self,
elt_context: ELTContext,
dbt_service: DbtService = None):
self.context = elt_context
self.dbt_service = dbt_service or DbtService(elt_context.project)
self.connection_service = ConnectionService(elt_context)