def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'):
from dbt.config import Project, Profile, RuntimeConfig
from dbt.utils import parse_cli_vars
from copy import deepcopy
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
if not isinstance(project, Project):
project = Project.from_project_config(deepcopy(project), packages)
if not isinstance(profile, Profile):
profile = Profile.from_raw_profile_info(deepcopy(profile),
project.profile_name,
cli_vars)
args = Obj()
args.vars = repr(cli_vars)
return RuntimeConfig.from_parts(
project=project,
profile=profile,
args=args
)
def load_config(self):
# we've written our profile and project. Now we want to instantiate a
# fresh adapter for the tests.
# it's important to use a different connection handle here so
# we don't look into an incomplete transaction
kwargs = {
'profile': None,
'profile_dir': DBT_CONFIG_DIR,
'target': None,
}
config = RuntimeConfig.from_args(TestArgs(kwargs))
adapter = get_adapter(config)
adapter.cleanup_connections()
self.adapter_type = adapter.type()
self.adapter = adapter
self.config = config
self._drop_schemas()
self._create_schemas()
def generate_parse_exposure(
exposure: ParsedExposure,
config: RuntimeConfig,
manifest: MacroManifest,
package_name: str,
) -> Dict[str, Any]:
project = config.load_dependencies()[package_name]
return {
'ref': ExposureRefResolver(
None,
exposure,
project,
manifest,
),
'source': ExposureSourceResolver(
None,
exposure,
project,
manifest,
)
}
def load_config(self):
# we've written our profile and project. Now we want to instantiate a
# fresh adapter for the tests.
# it's important to use a different connection handle here so
# we don't look into an incomplete transaction
kwargs = {
'profile': None,
'profiles_dir': self.test_root_dir,
'target': None,
}
config = RuntimeConfig.from_args(TestArgs(kwargs))
adapter = get_adapter(config)
adapter.cleanup_connections()
self.adapter_type = adapter.type()
self.adapter = adapter
self.config = config
self._drop_schemas()
self._create_schemas()
def load_all(
cls,
root_config: RuntimeConfig,
internal_manifest: Optional[Manifest],
macro_hook: Callable[[Manifest], Any],
) -> Manifest:
with PARSING_STATE:
projects = root_config.load_dependencies()
v1_configs = []
for project in projects.values():
if project.config_version == 1:
v1_configs.append(f'\n\n - {project.project_name}')
if v1_configs:
deprecations.warn('dbt-project-yaml-v1',
project_names=''.join(v1_configs))
loader = cls(root_config, projects, macro_hook)
loader.load(internal_manifest=internal_manifest)
loader.write_parse_results()
manifest = loader.create_manifest()
_check_manifest(manifest, root_config)
manifest.build_flat_graph()
return manifest
def load_all(
cls,
root_config: RuntimeConfig,
macro_manifest: MacroManifest,
macro_hook: Callable[[AnyManifest], Any],
) -> Manifest:
with PARSING_STATE:
start_load_all = time.perf_counter()
projects = root_config.load_dependencies()
loader = cls(root_config, projects, macro_hook)
loader.load(macro_manifest=macro_manifest)
loader.write_parse_results()
manifest = loader.create_manifest()
_check_manifest(manifest, root_config)
manifest.build_flat_graph()
loader._perf_info.load_all_elapsed = (time.perf_counter() -
start_load_all)
loader.track_project_load()
return manifest
def _warn_for_unused_resource_config_paths(manifest: Manifest,
config: RuntimeConfig) -> None:
resource_fqns: Mapping[str, PathSet] = manifest.get_resource_fqns()
disabled_fqns: PathSet = frozenset(tuple(n.fqn) for n in manifest.disabled)
config.warn_for_unused_resource_config_paths(resource_fqns, disabled_fqns)
def invoke_dbt(parsed):
task = None
cfg = None
log_cache_events(getattr(parsed, 'log_cache_events', False))
try:
if parsed.which in {'deps', 'clean'}:
# deps doesn't need a profile, so don't require one.
cfg = Project.from_current_directory(getattr(parsed, 'vars', '{}'))
elif parsed.which != 'debug':
# for debug, we will attempt to load the various configurations as
# part of the task, so just leave cfg=None.
cfg = RuntimeConfig.from_args(parsed)
except DbtProjectError as e:
logger.info("Encountered an error while reading the project:")
logger.info(dbt.compat.to_string(e))
dbt.tracking.track_invalid_invocation(config=cfg,
args=parsed,
result_type=e.result_type)
return None
except DbtProfileError as e:
logger.info("Encountered an error while reading profiles:")
logger.info(" ERROR {}".format(str(e)))
all_profiles = read_profiles(parsed.profiles_dir).keys()
if len(all_profiles) > 0:
logger.info("Defined profiles:")
for profile in all_profiles:
logger.info(" - {}".format(profile))
else:
logger.info("There are no profiles defined in your "
"profiles.yml file")
logger.info(PROFILES_HELP_MESSAGE)
dbt.tracking.track_invalid_invocation(config=cfg,
args=parsed,
result_type=e.result_type)
return None
flags.NON_DESTRUCTIVE = getattr(parsed, 'non_destructive', False)
flags.USE_CACHE = getattr(parsed, 'use_cache', True)
arg_drop_existing = getattr(parsed, 'drop_existing', False)
arg_full_refresh = getattr(parsed, 'full_refresh', False)
if arg_drop_existing:
dbt.deprecations.warn('drop-existing')
flags.FULL_REFRESH = True
elif arg_full_refresh:
flags.FULL_REFRESH = True
logger.debug("running dbt with arguments %s", parsed)
task = parsed.cls(args=parsed, config=cfg)
return task, cfg
def __init__(self, args):
self.args = args
self.config = RuntimeConfig.from_args(args)
def __init__(self, args):
self.args = args
self.config = RuntimeConfig.from_args(args)
self.target_path = self.config.target_path
self.run_results = self._get_run_results()
def __init__(self, args):
self.args = args
self.config = RuntimeConfig.from_args(args)
self.model_path = self.config.source_paths[0]
self.target_path = self.config.target_path
self.manifest = self._get_manifest()