def from_args(cls, args):
"""Given arguments, read in dbt_project.yml from the current directory,
read in packages.yml if it exists, and use them to find the profile to
load.
:param args argparse.Namespace: The arguments as parsed from the cli.
:raises DbtProjectError: If the project is invalid or missing.
:raises DbtProfileError: If the profile is invalid or missing.
:raises ValidationException: If the cli variables are invalid.
"""
cli_vars = parse_cli_vars(getattr(args, 'vars', '{}'))
# build the project and read in packages.yml
project = Project.from_current_directory(cli_vars)
# build the profile
profile = Profile.from_args(
args=args,
project_profile_name=project.profile_name,
cli_vars=cli_vars
)
return cls.from_parts(
project=project,
profile=profile,
args=args
)
def from_project_root(cls, project_root, cli_vars):
"""Create a project from a root directory. Reads in dbt_project.yml and
packages.yml, if it exists.
:param project_root str: The path to the project root to load.
:raises DbtProjectError: If the project is missing or invalid, or if
the packages file exists and is invalid.
:returns Project: The project, with defaults populated.
"""
project_root = os.path.normpath(project_root)
project_yaml_filepath = os.path.join(project_root, 'dbt_project.yml')
# get the project.yml contents
if not path_exists(project_yaml_filepath):
raise DbtProjectError(
'no dbt_project.yml found at expected path {}'.format(
project_yaml_filepath))
if isinstance(cli_vars, compat.basestring):
cli_vars = parse_cli_vars(cli_vars)
renderer = ConfigRenderer(cli_vars)
project_dict = _load_yaml(project_yaml_filepath)
rendered_project = renderer.render_project(project_dict)
rendered_project['project-root'] = project_root
packages_dict = package_data_from_root(project_root)
return cls.from_project_config(rendered_project, packages_dict)
def from_args(cls, args, project_profile_name=None, cli_vars=None):
"""Given the raw profiles as read from disk and the name of the desired
profile if specified, return the profile component of the runtime
config.
:param args argparse.Namespace: The arguments as parsed from the cli.
:param cli_vars dict: The command-line variables passed as arguments,
as a dict.
:param project_profile_name Optional[str]: The profile name, if
specified in a project.
:raises DbtProjectError: If there is no profile name specified in the
project or the command line arguments, or if the specified profile
is not found
:raises DbtProfileError: If the profile is invalid or missing, or the
target could not be found.
:returns Profile: The new Profile object.
"""
if cli_vars is None:
cli_vars = parse_cli_vars(getattr(args, 'vars', '{}'))
threads_override = getattr(args, 'threads', None)
target_override = getattr(args, 'target', None)
raw_profiles = read_profile(args.profiles_dir)
profile_name = cls.pick_profile_name(args.profile,
project_profile_name)
return cls.from_raw_profiles(
raw_profiles=raw_profiles,
profile_name=profile_name,
cli_vars=cli_vars,
target_override=target_override,
threads_override=threads_override
)
def from_project_root(cls, project_root, cli_vars):
"""Create a project from a root directory. Reads in dbt_project.yml and
packages.yml, if it exists.
:param project_root str: The path to the project root to load.
:raises DbtProjectError: If the project is missing or invalid, or if
the packages file exists and is invalid.
:returns Project: The project, with defaults populated.
"""
project_root = os.path.normpath(project_root)
project_yaml_filepath = os.path.join(project_root, 'dbt_project.yml')
# get the project.yml contents
if not path_exists(project_yaml_filepath):
raise DbtProjectError(
'no dbt_project.yml found at expected path {}'
.format(project_yaml_filepath)
)
if isinstance(cli_vars, compat.basestring):
cli_vars = parse_cli_vars(cli_vars)
renderer = ConfigRenderer(cli_vars)
project_dict = _load_yaml(project_yaml_filepath)
rendered_project = renderer.render_project(project_dict)
rendered_project['project-root'] = project_root
packages_dict = package_data_from_root(project_root)
return cls.from_project_config(rendered_project, packages_dict)
def from_parts(
cls,
project: Project,
profile: Profile,
args: Any,
dependencies: Optional[Mapping[str, 'RuntimeConfig']] = None,
) -> 'RuntimeConfig':
"""Instantiate a RuntimeConfig from its components.
:param profile: A parsed dbt Profile.
:param project: A parsed dbt Project.
:param args: The parsed command-line arguments.
:returns RuntimeConfig: The new configuration.
"""
quoting: Dict[str, Any] = (get_relation_class_by_name(
profile.credentials.type).get_default_quote_policy().replace_dict(
project.quoting)).to_dict()
cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, 'vars', '{}'))
return cls(
project_name=project.project_name,
version=project.version,
project_root=project.project_root,
source_paths=project.source_paths,
macro_paths=project.macro_paths,
data_paths=project.data_paths,
test_paths=project.test_paths,
analysis_paths=project.analysis_paths,
docs_paths=project.docs_paths,
target_path=project.target_path,
snapshot_paths=project.snapshot_paths,
clean_targets=project.clean_targets,
log_path=project.log_path,
modules_path=project.modules_path,
quoting=quoting,
models=project.models,
on_run_start=project.on_run_start,
on_run_end=project.on_run_end,
seeds=project.seeds,
snapshots=project.snapshots,
dbt_version=project.dbt_version,
packages=project.packages,
query_comment=project.query_comment,
sources=project.sources,
vars=project.vars,
config_version=project.config_version,
profile_name=profile.profile_name,
target_name=profile.target_name,
config=profile.config,
threads=profile.threads,
credentials=profile.credentials,
args=args,
cli_vars=cli_vars,
dependencies=dependencies,
)
def project_from_dict(project, profile, packages=None, cli_vars='{}'):
from dbt.context.target import generate_target_context
from dbt.config import Project, ConfigRenderer
from dbt.utils import parse_cli_vars
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
renderer = ConfigRenderer(generate_target_context(profile, cli_vars))
project_root = project.pop('project-root', os.getcwd())
return Project.render_from_dict(project_root, project, packages, renderer)
def from_parts(
cls,
project: Project,
profile: Profile,
args: Any,
dependencies: Optional[Mapping[str, 'RuntimeConfig']] = None,
) -> 'RuntimeConfig':
"""Instantiate a RuntimeConfig from its components.
:param profile: Ignored.
:param project: A parsed dbt Project.
:param args: The parsed command-line arguments.
:returns RuntimeConfig: The new configuration.
"""
cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, 'vars', '{}'))
return cls(
project_name=project.project_name,
version=project.version,
project_root=project.project_root,
source_paths=project.source_paths,
macro_paths=project.macro_paths,
data_paths=project.data_paths,
test_paths=project.test_paths,
analysis_paths=project.analysis_paths,
docs_paths=project.docs_paths,
target_path=project.target_path,
snapshot_paths=project.snapshot_paths,
clean_targets=project.clean_targets,
log_path=project.log_path,
modules_path=project.modules_path,
quoting=project.quoting, # we never use this anyway.
models=project.models,
on_run_start=project.on_run_start,
on_run_end=project.on_run_end,
seeds=project.seeds,
snapshots=project.snapshots,
dbt_version=project.dbt_version,
packages=project.packages,
query_comment=project.query_comment,
sources=project.sources,
vars=project.vars,
config_version=project.config_version,
profile_name='',
target_name='',
config=UnsetConfig(),
threads=getattr(args, 'threads', 1),
credentials=UnsetCredentials(),
args=args,
cli_vars=cli_vars,
dependencies=dependencies,
)
def profile_from_dict(profile, profile_name, cli_vars='{}'):
from dbt.config import Profile, ConfigRenderer
from dbt.context.base import generate_base_context
from dbt.utils import parse_cli_vars
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
renderer = ConfigRenderer(generate_base_context(cli_vars))
return Profile.from_raw_profile_info(
profile,
profile_name,
renderer,
)
def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'):
from dbt.config import Project, Profile, RuntimeConfig
from dbt.utils import parse_cli_vars
from copy import deepcopy
if not isinstance(project, Project):
project = Project.from_project_config(deepcopy(project), packages)
if not isinstance(profile, Profile):
profile = Profile.from_raw_profile_info(deepcopy(profile),
project.profile_name)
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
return RuntimeConfig.from_parts(project=project,
profile=profile,
cli_vars=cli_vars)
def __init__(self, project_name, version, project_root, source_paths,
macro_paths, data_paths, test_paths, analysis_paths,
docs_paths, target_path, snapshot_paths, clean_targets,
log_path, modules_path, quoting, models, on_run_start,
on_run_end, seeds, snapshots, dbt_version, profile_name,
target_name, config, threads, credentials, packages,
query_comment, args):
# 'vars'
self.args = args
self.cli_vars = parse_cli_vars(getattr(args, 'vars', '{}'))
# 'project'
Project.__init__(
self,
project_name=project_name,
version=version,
project_root=project_root,
profile_name=profile_name,
source_paths=source_paths,
macro_paths=macro_paths,
data_paths=data_paths,
test_paths=test_paths,
analysis_paths=analysis_paths,
docs_paths=docs_paths,
target_path=target_path,
snapshot_paths=snapshot_paths,
clean_targets=clean_targets,
log_path=log_path,
modules_path=modules_path,
quoting=quoting,
models=models,
on_run_start=on_run_start,
on_run_end=on_run_end,
seeds=seeds,
snapshots=snapshots,
dbt_version=dbt_version,
packages=packages,
query_comment=query_comment,
)
# 'profile'
Profile.__init__(self,
profile_name=profile_name,
target_name=target_name,
config=config,
threads=threads,
credentials=credentials)
self.validate()
def __init__(self, project_name, version, project_root, source_paths,
macro_paths, data_paths, test_paths, analysis_paths,
docs_paths, target_path, clean_targets, log_path,
modules_path, quoting, models, on_run_start, on_run_end,
archive, seeds, dbt_version, profile_name, target_name,
config, threads, credentials, packages, args):
# 'vars'
self.args = args
self.cli_vars = parse_cli_vars(getattr(args, 'vars', '{}'))
# 'project'
Project.__init__(
self,
project_name=project_name,
version=version,
project_root=project_root,
profile_name=profile_name,
source_paths=source_paths,
macro_paths=macro_paths,
data_paths=data_paths,
test_paths=test_paths,
analysis_paths=analysis_paths,
docs_paths=docs_paths,
target_path=target_path,
clean_targets=clean_targets,
log_path=log_path,
modules_path=modules_path,
quoting=quoting,
models=models,
on_run_start=on_run_start,
on_run_end=on_run_end,
archive=archive,
seeds=seeds,
dbt_version=dbt_version,
packages=packages
)
# 'profile'
Profile.__init__(
self,
profile_name=profile_name,
target_name=target_name,
config=config,
threads=threads,
credentials=credentials
)
self.validate()
def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'):
from dbt.config import Project, Profile, RuntimeConfig
from dbt.utils import parse_cli_vars
from copy import deepcopy
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
if not isinstance(project, Project):
project = Project.from_project_config(deepcopy(project), packages)
if not isinstance(profile, Profile):
profile = Profile.from_raw_profile_info(deepcopy(profile),
project.profile_name,
cli_vars)
args = Obj()
args.vars = repr(cli_vars)
return RuntimeConfig.from_parts(
project=project,
profile=profile,
args=args
)
def collect_parts(cls: Type['RuntimeConfig'],
args: Any) -> Tuple[Project, Profile]:
# profile_name from the project
project_root = args.project_dir if args.project_dir else os.getcwd()
partial = Project.partial_load(project_root)
# build the profile using the base renderer and the one fact we know
cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, 'vars', '{}'))
profile_renderer = ProfileRenderer(generate_base_context(cli_vars))
profile_name = partial.render_profile_name(profile_renderer)
profile = cls._get_rendered_profile(args, profile_renderer,
profile_name)
# get a new renderer using our target information and render the
# project
ctx = generate_target_context(profile, cli_vars)
project_renderer = DbtProjectYamlRenderer(ctx, partial.config_version)
project = partial.render(project_renderer)
return (project, profile)
def from_args(cls, args):
"""Given arguments, read in dbt_project.yml from the current directory,
read in packages.yml if it exists, and use them to find the profile to
load.
:param args argparse.Namespace: The arguments as parsed from the cli.
:raises DbtProjectError: If the project is invalid or missing.
:raises DbtProfileError: If the profile is invalid or missing.
:raises ValidationException: If the cli variables are invalid.
"""
cli_vars = parse_cli_vars(getattr(args, 'vars', '{}'))
# build the project and read in packages.yml
project = Project.from_current_directory(cli_vars)
# build the profile
profile = Profile.from_args(args=args,
project_profile_name=project.profile_name,
cli_vars=cli_vars)
return cls.from_parts(project=project, profile=profile, args=args)
def handle_request(self) -> Result:
if self.real_task is None:
raise InternalException(
'CLI task is in a bad state: handle_request called with no '
'real_task set!')
# It's important to update cli_vars here, because set_config()'s
# `self.config` is before the fork(), so it would alter the behavior of
# future calls.
# read any cli vars we got and use it to update cli_vars
self.config.cli_vars.update(
parse_cli_vars(getattr(self.args, 'vars', '{}')))
# If this changed the vars, rewrite args.vars to reflect our merged
# vars and reload the manifest.
dumped = yaml.safe_dump(self.config.cli_vars)
if dumped != self.args.vars:
self.real_task.args.vars = dumped
if isinstance(self.real_task, RemoteManifestMethod):
self.real_task.manifest = get_full_manifest(self.config)
# we parsed args from the cli, so we're set on that front
return self.real_task.handle_request()
