def collect_parts(
cls: Type['RuntimeConfig'], args: Any
) -> Tuple[Project, Profile]:
# profile_name from the project
project_root = args.project_dir if args.project_dir else os.getcwd()
version_check = getattr(args, 'version_check', False)
partial = Project.partial_load(
project_root,
verify_version=version_check
)
# build the profile using the base renderer and the one fact we know
cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, 'vars', '{}'))
profile_renderer = ProfileRenderer(generate_base_context(cli_vars))
profile_name = partial.render_profile_name(profile_renderer)
profile = cls._get_rendered_profile(
args, profile_renderer, profile_name
)
# get a new renderer using our target information and render the
# project
ctx = generate_target_context(profile, cli_vars)
project_renderer = DbtProjectYamlRenderer(ctx)
project = partial.render(project_renderer)
return (project, profile)
def get_credentials(profile_yml):
"Render a YAML string profiles.yml into credentials"
dicty_thing = load_yaml_text(profile_yml)
renderer = dbt.config.renderer.ProfileRenderer(generate_base_context({}))
profile = dbt.config.Profile.from_raw_profiles(dicty_thing, 'default',
renderer)
return profile.credentials
def profile_from_dict(profile, profile_name, cli_vars='{}'):
from dbt.config import Profile, ConfigRenderer
from dbt.context.base import generate_base_context
from dbt.utils import parse_cli_vars
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
renderer = ConfigRenderer(generate_base_context(cli_vars))
return Profile.from_raw_profile_info(
profile,
profile_name,
renderer,
)
def _choose_target_name(self, profile_name: str):
has_raw_profile = (self.raw_profile_data is not None
and profile_name in self.raw_profile_data)
if not has_raw_profile:
return None
# mypy appeasement, we checked just above
assert self.raw_profile_data is not None
raw_profile = self.raw_profile_data[profile_name]
renderer = ProfileRenderer(generate_base_context(self.cli_vars))
target_name, _ = Profile.render_profile(raw_profile=raw_profile,
profile_name=profile_name,
target_override=getattr(
self.args, 'target', None),
renderer=renderer)
return target_name
def reparse_macros(self):
internal_package_names = get_adapter_package_names(
self.root_project.credentials.type
)
macro_resolver = MacroResolver(
self.manifest.macros,
self.root_project.project_name,
internal_package_names
)
base_ctx = generate_base_context({})
for macro in self.manifest.macros.values():
possible_macro_calls = statically_extract_macro_calls(macro.macro_sql, base_ctx)
for macro_name in possible_macro_calls:
# adapter.dispatch calls can generate a call with the same name as the macro
# it ought to be an adapter prefix (postgres_) or default_
if macro_name == macro.name:
continue
dep_macro_id = macro_resolver.get_macro_id(macro.package_name, macro_name)
if dep_macro_id:
macro.depends_on.add_macro(dep_macro_id) # will check for dupes
def validate_connection(cls, target_dict):
"""Validate a connection dictionary. On error, raises a DbtConfigError.
"""
target_name = 'test'
# make a fake profile that we can parse
profile_data = {
'outputs': {
target_name: target_dict,
},
}
# this will raise a DbtConfigError on failure
profile = Profile.from_raw_profile_info(
raw_profile=profile_data,
profile_name='',
target_override=target_name,
renderer=ProfileRenderer(generate_base_context({})),
)
result = cls.attempt_connection(profile)
if result is not None:
raise dbt.exceptions.DbtProfileError(
result, result_type='connection_failure')
def _load_profile(self):
if not os.path.exists(self.profile_path):
self.profile_fail_details = FILE_NOT_FOUND
self.messages.append(MISSING_PROFILE_MESSAGE.format(
path=self.profile_path, url=ProfileConfigDocs
))
return red('ERROR not found')
try:
raw_profile_data = load_yaml_text(
dbt.clients.system.load_file_contents(self.profile_path)
)
except Exception:
pass # we'll report this when we try to load the profile for real
else:
if isinstance(raw_profile_data, dict):
self.raw_profile_data = raw_profile_data
profile_errors = []
profile_names = self._choose_profile_names()
renderer = ProfileRenderer(generate_base_context(self.cli_vars))
for profile_name in profile_names:
try:
profile: Profile = QueryCommentedProfile.render_from_args(
self.args, renderer, profile_name
)
except dbt.exceptions.DbtConfigError as exc:
profile_errors.append(str(exc))
else:
if len(profile_names) == 1:
# if a profile was specified, set it on the task
self.target_name = self._choose_target_name(profile_name)
self.profile = profile
if profile_errors:
self.profile_fail_details = '\n\n'.join(profile_errors)
return red('ERROR invalid')
return green('OK found and valid')
def _load_project(self):
if not os.path.exists(self.project_path):
self.project_fail_details = FILE_NOT_FOUND
return red('ERROR not found')
if self.profile is None:
ctx = generate_base_context(self.cli_vars)
else:
ctx = generate_target_context(self.profile, self.cli_vars)
renderer = DbtProjectYamlRenderer(ctx)
try:
self.project = Project.from_project_root(
self.project_dir,
renderer,
verify_version=getattr(self.args, 'version_check', False),
)
except dbt.exceptions.DbtConfigError as exc:
self.project_fail_details = str(exc)
return red('ERROR invalid')
return green('OK found and valid')
def _choose_profile_names(self) -> Optional[List[str]]:
project_profile: Optional[str] = None
if os.path.exists(self.project_path):
try:
partial = Project.partial_load(
os.path.dirname(self.project_path),
verify_version=getattr(self.args, 'version_check', False),
)
renderer = DbtProjectYamlRenderer(
generate_base_context(self.cli_vars)
)
project_profile = partial.render_profile_name(renderer)
except dbt.exceptions.DbtProjectError:
pass
args_profile: Optional[str] = getattr(self.args, 'profile', None)
try:
return [Profile.pick_profile_name(args_profile, project_profile)]
except dbt.exceptions.DbtConfigError:
pass
# try to guess
profiles = []
if self.raw_profile_data:
profiles = [k for k in self.raw_profile_data if k != 'config']
if project_profile is None:
self.messages.append('Could not load dbt_project.yml')
elif len(profiles) == 0:
self.messages.append('The profiles.yml has no profiles')
elif len(profiles) == 1:
self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0]))
else:
self.messages.append(MULTIPLE_PROFILE_MESSAGE.format(
'\n'.join(' - {}'.format(o) for o in profiles)
))
return profiles
def get_project(git_path):
ctx = generate_base_context({})
renderer = DbtProjectYamlRenderer(ctx)
return Project.from_project_root(git_path, renderer)
def test_base_context():
ctx = base.generate_base_context({})
assert_has_keys(REQUIRED_BASE_KEYS, MAYBE_KEYS, ctx)
