class DbtRunner(Runner): def __init__(self, elt_context: ELTContext, dbt_service: DbtService = None): self.context = elt_context self.dbt_service = dbt_service or DbtService(elt_context.project) self.connection_service = ConnectionService(elt_context) @property def project(self): return self.context.project def run(self, session, dry_run=False, models=None): # we should probably refactor this part to have an ELTContext object already # filled with the each plugins' configuration so we don't have to query # multiple times for the same data. settings_service = PluginSettingsService(self.project) try: load = self.connection_service.load_params() analyze = self.connection_service.analyze_params() env = { # inject the inferred 'schemas' from the ELTContext "MELTANO_LOAD_SCHEMA": load["schema"], "MELTANO_ANALYZE_SCHEMA": analyze["schema"], "DBT_TARGET": self.connection_service.dialect, # inject the extractor & loader configuration as ENV variables. # that means dbt will have access to all the configuration of # the extractor and loader **settings_service.as_env(session, self.context.extractor.ref), **settings_service.as_env(session, self.context.loader.ref), } except Exception as e: logging.debug("Could not inject environment to dbt.") logging.debug( f"Could not hydrate ENV from the EltContext: {str(e)}") raise e # Get an asyncio event loop and use it to run the dbt commands loop = asyncio.get_event_loop() loop.run_until_complete(self.dbt_service.deps()) if models is not None: models = models.replace("-", "_") if dry_run: loop.run_until_complete(self.dbt_service.compile(models, env=env)) else: loop.run_until_complete(self.dbt_service.run(models, env=env))
def test_params( self, project_add_service, elt_context_builder, session, tap, loader, load_params, analyze_params, ): project_add_service.add(PluginType.LOADERS, loader) elt_context = (elt_context_builder.with_extractor( tap.name).with_loader(loader).context(session)) subject = ConnectionService(elt_context) assert subject.load_params() == load_params assert subject.analyze_params() == analyze_params
def test_params_with_transform( self, project_add_service, elt_context_builder, session, tap, dbt, loader, analyze_params, ): try: project_add_service.add(PluginType.LOADERS, loader) except PluginAlreadyAddedException: pass elt_context = ( elt_context_builder.with_session(session).with_extractor( tap.name).with_loader(loader).with_transform("run").context()) subject = ConnectionService(elt_context) assert subject.analyze_params() == analyze_params
def get_db_engine(self, extractor, loader, transform): project = Project.find() context = (ELTContextBuilder(project).with_session( db.session).with_extractor(extractor).with_loader( loader).with_transform(transform).context()) connection_service = ConnectionService(context) engine_hooks = [] dialect = connection_service.dialect engine_uri = None params = None try: params = connection_service.analyze_params() engine_uri = connection_service.analyze_uri() if dialect not in ENABLED_DIALECTS: raise UnsupportedConnectionDialect(dialect) except DialectNotSupportedError: raise UnsupportedConnectionDialect(dialect) if dialect == "postgres": def set_connection_schema(raw, conn): schema = params["schema"] with raw.cursor() as cursor: res = cursor.execute(f"SET search_path TO {schema};") logging.debug(f"Connection schema set to {schema}") engine_hooks.append(lambda engine: listen(engine, "first_connect", set_connection_schema)) engine = sqlalchemy.create_engine(engine_uri) for hook in engine_hooks: hook(engine) return engine
def __init__( self, elt_context: ELTContext, config_service: ConfigService = None, connection_service: ConnectionService = None, **config, ): self.context = elt_context self.config = config self.config_service = config_service or ConfigService( elt_context.project) self.connection_service = connection_service or ConnectionService( elt_context) self.tap_config_dir = Path( config.get("tap_config_dir", "/etc/singer/tap")) self.target_config_dir = Path( config.get("target_config_dir", "/etc/singer/target"))
def __init__(self, elt_context: ELTContext, dbt_service: DbtService = None): self.context = elt_context self.dbt_service = dbt_service or DbtService(elt_context.project) self.connection_service = ConnectionService(elt_context)