def from_args(cls, args): """Given arguments, read in dbt_project.yml from the current directory, read in packages.yml if it exists, and use them to find the profile to load. :param args argparse.Namespace: The arguments as parsed from the cli. :raises DbtProjectError: If the project is invalid or missing. :raises DbtProfileError: If the profile is invalid or missing. :raises ValidationException: If the cli variables are invalid. """ cli_vars = parse_cli_vars(getattr(args, 'vars', '{}')) # build the project and read in packages.yml project = Project.from_current_directory(cli_vars) # build the profile profile = Profile.from_args( args=args, project_profile_name=project.profile_name, cli_vars=cli_vars ) return cls.from_parts( project=project, profile=profile, args=args )
def from_project_root(cls, project_root, cli_vars): """Create a project from a root directory. Reads in dbt_project.yml and packages.yml, if it exists. :param project_root str: The path to the project root to load. :raises DbtProjectError: If the project is missing or invalid, or if the packages file exists and is invalid. :returns Project: The project, with defaults populated. """ project_root = os.path.normpath(project_root) project_yaml_filepath = os.path.join(project_root, 'dbt_project.yml') # get the project.yml contents if not path_exists(project_yaml_filepath): raise DbtProjectError( 'no dbt_project.yml found at expected path {}'.format( project_yaml_filepath)) if isinstance(cli_vars, compat.basestring): cli_vars = parse_cli_vars(cli_vars) renderer = ConfigRenderer(cli_vars) project_dict = _load_yaml(project_yaml_filepath) rendered_project = renderer.render_project(project_dict) rendered_project['project-root'] = project_root packages_dict = package_data_from_root(project_root) return cls.from_project_config(rendered_project, packages_dict)
def from_args(cls, args, project_profile_name=None, cli_vars=None): """Given the raw profiles as read from disk and the name of the desired profile if specified, return the profile component of the runtime config. :param args argparse.Namespace: The arguments as parsed from the cli. :param cli_vars dict: The command-line variables passed as arguments, as a dict. :param project_profile_name Optional[str]: The profile name, if specified in a project. :raises DbtProjectError: If there is no profile name specified in the project or the command line arguments, or if the specified profile is not found :raises DbtProfileError: If the profile is invalid or missing, or the target could not be found. :returns Profile: The new Profile object. """ if cli_vars is None: cli_vars = parse_cli_vars(getattr(args, 'vars', '{}')) threads_override = getattr(args, 'threads', None) target_override = getattr(args, 'target', None) raw_profiles = read_profile(args.profiles_dir) profile_name = cls.pick_profile_name(args.profile, project_profile_name) return cls.from_raw_profiles( raw_profiles=raw_profiles, profile_name=profile_name, cli_vars=cli_vars, target_override=target_override, threads_override=threads_override )
def from_project_root(cls, project_root, cli_vars): """Create a project from a root directory. Reads in dbt_project.yml and packages.yml, if it exists. :param project_root str: The path to the project root to load. :raises DbtProjectError: If the project is missing or invalid, or if the packages file exists and is invalid. :returns Project: The project, with defaults populated. """ project_root = os.path.normpath(project_root) project_yaml_filepath = os.path.join(project_root, 'dbt_project.yml') # get the project.yml contents if not path_exists(project_yaml_filepath): raise DbtProjectError( 'no dbt_project.yml found at expected path {}' .format(project_yaml_filepath) ) if isinstance(cli_vars, compat.basestring): cli_vars = parse_cli_vars(cli_vars) renderer = ConfigRenderer(cli_vars) project_dict = _load_yaml(project_yaml_filepath) rendered_project = renderer.render_project(project_dict) rendered_project['project-root'] = project_root packages_dict = package_data_from_root(project_root) return cls.from_project_config(rendered_project, packages_dict)
def from_parts( cls, project: Project, profile: Profile, args: Any, dependencies: Optional[Mapping[str, 'RuntimeConfig']] = None, ) -> 'RuntimeConfig': """Instantiate a RuntimeConfig from its components. :param profile: A parsed dbt Profile. :param project: A parsed dbt Project. :param args: The parsed command-line arguments. :returns RuntimeConfig: The new configuration. """ quoting: Dict[str, Any] = (get_relation_class_by_name( profile.credentials.type).get_default_quote_policy().replace_dict( project.quoting)).to_dict() cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, 'vars', '{}')) return cls( project_name=project.project_name, version=project.version, project_root=project.project_root, source_paths=project.source_paths, macro_paths=project.macro_paths, data_paths=project.data_paths, test_paths=project.test_paths, analysis_paths=project.analysis_paths, docs_paths=project.docs_paths, target_path=project.target_path, snapshot_paths=project.snapshot_paths, clean_targets=project.clean_targets, log_path=project.log_path, modules_path=project.modules_path, quoting=quoting, models=project.models, on_run_start=project.on_run_start, on_run_end=project.on_run_end, seeds=project.seeds, snapshots=project.snapshots, dbt_version=project.dbt_version, packages=project.packages, query_comment=project.query_comment, sources=project.sources, vars=project.vars, config_version=project.config_version, profile_name=profile.profile_name, target_name=profile.target_name, config=profile.config, threads=profile.threads, credentials=profile.credentials, args=args, cli_vars=cli_vars, dependencies=dependencies, )
def project_from_dict(project, profile, packages=None, cli_vars='{}'): from dbt.context.target import generate_target_context from dbt.config import Project, ConfigRenderer from dbt.utils import parse_cli_vars if not isinstance(cli_vars, dict): cli_vars = parse_cli_vars(cli_vars) renderer = ConfigRenderer(generate_target_context(profile, cli_vars)) project_root = project.pop('project-root', os.getcwd()) return Project.render_from_dict(project_root, project, packages, renderer)
def from_parts( cls, project: Project, profile: Profile, args: Any, dependencies: Optional[Mapping[str, 'RuntimeConfig']] = None, ) -> 'RuntimeConfig': """Instantiate a RuntimeConfig from its components. :param profile: Ignored. :param project: A parsed dbt Project. :param args: The parsed command-line arguments. :returns RuntimeConfig: The new configuration. """ cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, 'vars', '{}')) return cls( project_name=project.project_name, version=project.version, project_root=project.project_root, source_paths=project.source_paths, macro_paths=project.macro_paths, data_paths=project.data_paths, test_paths=project.test_paths, analysis_paths=project.analysis_paths, docs_paths=project.docs_paths, target_path=project.target_path, snapshot_paths=project.snapshot_paths, clean_targets=project.clean_targets, log_path=project.log_path, modules_path=project.modules_path, quoting=project.quoting, # we never use this anyway. models=project.models, on_run_start=project.on_run_start, on_run_end=project.on_run_end, seeds=project.seeds, snapshots=project.snapshots, dbt_version=project.dbt_version, packages=project.packages, query_comment=project.query_comment, sources=project.sources, vars=project.vars, config_version=project.config_version, profile_name='', target_name='', config=UnsetConfig(), threads=getattr(args, 'threads', 1), credentials=UnsetCredentials(), args=args, cli_vars=cli_vars, dependencies=dependencies, )
def profile_from_dict(profile, profile_name, cli_vars='{}'): from dbt.config import Profile, ConfigRenderer from dbt.context.base import generate_base_context from dbt.utils import parse_cli_vars if not isinstance(cli_vars, dict): cli_vars = parse_cli_vars(cli_vars) renderer = ConfigRenderer(generate_base_context(cli_vars)) return Profile.from_raw_profile_info( profile, profile_name, renderer, )
def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'): from dbt.config import Project, Profile, RuntimeConfig from dbt.utils import parse_cli_vars from copy import deepcopy if not isinstance(project, Project): project = Project.from_project_config(deepcopy(project), packages) if not isinstance(profile, Profile): profile = Profile.from_raw_profile_info(deepcopy(profile), project.profile_name) if not isinstance(cli_vars, dict): cli_vars = parse_cli_vars(cli_vars) return RuntimeConfig.from_parts(project=project, profile=profile, cli_vars=cli_vars)
def __init__(self, project_name, version, project_root, source_paths, macro_paths, data_paths, test_paths, analysis_paths, docs_paths, target_path, snapshot_paths, clean_targets, log_path, modules_path, quoting, models, on_run_start, on_run_end, seeds, snapshots, dbt_version, profile_name, target_name, config, threads, credentials, packages, query_comment, args): # 'vars' self.args = args self.cli_vars = parse_cli_vars(getattr(args, 'vars', '{}')) # 'project' Project.__init__( self, project_name=project_name, version=version, project_root=project_root, profile_name=profile_name, source_paths=source_paths, macro_paths=macro_paths, data_paths=data_paths, test_paths=test_paths, analysis_paths=analysis_paths, docs_paths=docs_paths, target_path=target_path, snapshot_paths=snapshot_paths, clean_targets=clean_targets, log_path=log_path, modules_path=modules_path, quoting=quoting, models=models, on_run_start=on_run_start, on_run_end=on_run_end, seeds=seeds, snapshots=snapshots, dbt_version=dbt_version, packages=packages, query_comment=query_comment, ) # 'profile' Profile.__init__(self, profile_name=profile_name, target_name=target_name, config=config, threads=threads, credentials=credentials) self.validate()
def __init__(self, project_name, version, project_root, source_paths, macro_paths, data_paths, test_paths, analysis_paths, docs_paths, target_path, clean_targets, log_path, modules_path, quoting, models, on_run_start, on_run_end, archive, seeds, dbt_version, profile_name, target_name, config, threads, credentials, packages, args): # 'vars' self.args = args self.cli_vars = parse_cli_vars(getattr(args, 'vars', '{}')) # 'project' Project.__init__( self, project_name=project_name, version=version, project_root=project_root, profile_name=profile_name, source_paths=source_paths, macro_paths=macro_paths, data_paths=data_paths, test_paths=test_paths, analysis_paths=analysis_paths, docs_paths=docs_paths, target_path=target_path, clean_targets=clean_targets, log_path=log_path, modules_path=modules_path, quoting=quoting, models=models, on_run_start=on_run_start, on_run_end=on_run_end, archive=archive, seeds=seeds, dbt_version=dbt_version, packages=packages ) # 'profile' Profile.__init__( self, profile_name=profile_name, target_name=target_name, config=config, threads=threads, credentials=credentials ) self.validate()
def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'): from dbt.config import Project, Profile, RuntimeConfig from dbt.utils import parse_cli_vars from copy import deepcopy if not isinstance(cli_vars, dict): cli_vars = parse_cli_vars(cli_vars) if not isinstance(project, Project): project = Project.from_project_config(deepcopy(project), packages) if not isinstance(profile, Profile): profile = Profile.from_raw_profile_info(deepcopy(profile), project.profile_name, cli_vars) args = Obj() args.vars = repr(cli_vars) return RuntimeConfig.from_parts( project=project, profile=profile, args=args )
def collect_parts(cls: Type['RuntimeConfig'], args: Any) -> Tuple[Project, Profile]: # profile_name from the project project_root = args.project_dir if args.project_dir else os.getcwd() partial = Project.partial_load(project_root) # build the profile using the base renderer and the one fact we know cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, 'vars', '{}')) profile_renderer = ProfileRenderer(generate_base_context(cli_vars)) profile_name = partial.render_profile_name(profile_renderer) profile = cls._get_rendered_profile(args, profile_renderer, profile_name) # get a new renderer using our target information and render the # project ctx = generate_target_context(profile, cli_vars) project_renderer = DbtProjectYamlRenderer(ctx, partial.config_version) project = partial.render(project_renderer) return (project, profile)
def from_args(cls, args): """Given arguments, read in dbt_project.yml from the current directory, read in packages.yml if it exists, and use them to find the profile to load. :param args argparse.Namespace: The arguments as parsed from the cli. :raises DbtProjectError: If the project is invalid or missing. :raises DbtProfileError: If the profile is invalid or missing. :raises ValidationException: If the cli variables are invalid. """ cli_vars = parse_cli_vars(getattr(args, 'vars', '{}')) # build the project and read in packages.yml project = Project.from_current_directory(cli_vars) # build the profile profile = Profile.from_args(args=args, project_profile_name=project.profile_name, cli_vars=cli_vars) return cls.from_parts(project=project, profile=profile, args=args)
def handle_request(self) -> Result: if self.real_task is None: raise InternalException( 'CLI task is in a bad state: handle_request called with no ' 'real_task set!') # It's important to update cli_vars here, because set_config()'s # `self.config` is before the fork(), so it would alter the behavior of # future calls. # read any cli vars we got and use it to update cli_vars self.config.cli_vars.update( parse_cli_vars(getattr(self.args, 'vars', '{}'))) # If this changed the vars, rewrite args.vars to reflect our merged # vars and reload the manifest. dumped = yaml.safe_dump(self.config.cli_vars) if dumped != self.args.vars: self.real_task.args.vars = dumped if isinstance(self.real_task, RemoteManifestMethod): self.real_task.manifest = get_full_manifest(self.config) # we parsed args from the cli, so we're set on that front return self.real_task.handle_request()