def _load_profile(self): if not os.path.exists(self.profile_path): self.profile_fail_details = FILE_NOT_FOUND self.messages.append(MISSING_PROFILE_MESSAGE.format( path=self.profile_path, url=ProfileConfigDocs )) return red('ERROR not found') try: raw_profile_data = load_yaml_text( dbt.clients.system.load_file_contents(self.profile_path) ) except Exception: pass # we'll report this when we try to load the profile for real else: if isinstance(raw_profile_data, dict): self.raw_profile_data = raw_profile_data self.profile_name = self._choose_profile_name() self.target_name = self._choose_target_name() try: self.profile = Profile.from_args(self.args, self.profile_name, self.cli_vars) except dbt.exceptions.DbtConfigError as exc: self.profile_fail_details = str(exc) return red('ERROR invalid') return green('OK found and valid')
def _choose_profile_name(self): assert self.project or self.project_fail_details, \ '_load_project() required' project_profile = None if self.project: project_profile = self.project.profile_name args_profile = getattr(self.args, 'profile', None) try: return Profile.pick_profile_name(args_profile, project_profile) except dbt.exceptions.DbtConfigError: pass # try to guess if self.raw_profile_data: profiles = [k for k in self.raw_profile_data if k != 'config'] if len(profiles) == 0: self.messages.append('The profiles.yml has no profiles') elif len(profiles) == 1: self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0])) return profiles[0] else: self.messages.append(MULTIPLE_PROFILE_MESSAGE.format( '\n'.join(' - {}'.format(o) for o in profiles) )) return None
def _load_profile(self): if not os.path.exists(self.profile_path): self.profile_fail_details = FILE_NOT_FOUND self.messages.append( MISSING_PROFILE_MESSAGE.format(path=self.profile_path, url=ProfileConfigDocs)) return red('ERROR not found') try: raw_profile_data = load_yaml_text( dbt.clients.system.load_file_contents(self.profile_path)) except Exception: pass # we'll report this when we try to load the profile for real else: if isinstance(raw_profile_data, dict): self.raw_profile_data = raw_profile_data self.profile_name = self._choose_profile_name() self.target_name = self._choose_target_name() try: self.profile = Profile.from_args(self.args, self.profile_name) except dbt.exceptions.DbtConfigError as exc: self.profile_fail_details = str(exc) return red('ERROR invalid') return green('OK found and valid')
def _choose_profile_name(self): assert self.project or self.project_fail_details, \ '_load_project() required' project_profile = None if self.project: project_profile = self.project.profile_name args_profile = getattr(self.args, 'profile', None) try: return Profile.pick_profile_name(args_profile, project_profile) except dbt.exceptions.DbtConfigError: pass # try to guess if self.raw_profile_data: profiles = [k for k in self.raw_profile_data if k != 'config'] if len(profiles) == 0: self.messages.append('The profiles.yml has no profiles') elif len(profiles) == 1: self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0])) return profiles[0] else: self.messages.append( MULTIPLE_PROFILE_MESSAGE.format('\n'.join( ' - {}'.format(o) for o in profiles))) return None
def _choose_target_name(self): has_raw_profile = (self.raw_profile_data and self.profile_name and self.profile_name in self.raw_profile_data) if has_raw_profile: raw_profile = self.raw_profile_data[self.profile_name] target_name, _ = Profile.render_profile( raw_profile, self.profile_name, getattr(self.args, 'target', None), self.cli_vars) return target_name return None
def _choose_target_name(self): has_raw_profile = (self.raw_profile_data and self.profile_name and self.profile_name in self.raw_profile_data) if has_raw_profile: raw_profile = self.raw_profile_data[self.profile_name] target_name, _ = Profile.render_profile( raw_profile, self.profile_name, getattr(self.args, 'target', None), self.cli_vars ) return target_name return None
def profile_from_dict(profile, profile_name, cli_vars='{}'): from dbt.config import Profile, ConfigRenderer from dbt.context.base import generate_base_context from dbt.utils import parse_cli_vars if not isinstance(cli_vars, dict): cli_vars = parse_cli_vars(cli_vars) renderer = ConfigRenderer(generate_base_context(cli_vars)) return Profile.from_raw_profile_info( profile, profile_name, renderer, )
def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'): from dbt.config import Project, Profile, RuntimeConfig from dbt.utils import parse_cli_vars from copy import deepcopy if not isinstance(project, Project): project = Project.from_project_config(deepcopy(project), packages) if not isinstance(profile, Profile): profile = Profile.from_raw_profile_info(deepcopy(profile), project.profile_name) if not isinstance(cli_vars, dict): cli_vars = parse_cli_vars(cli_vars) return RuntimeConfig.from_parts(project=project, profile=profile, cli_vars=cli_vars)
def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'): from dbt.config import Project, Profile, RuntimeConfig from dbt.utils import parse_cli_vars from copy import deepcopy if not isinstance(cli_vars, dict): cli_vars = parse_cli_vars(cli_vars) if not isinstance(project, Project): project = Project.from_project_config(deepcopy(project), packages) if not isinstance(profile, Profile): profile = Profile.from_raw_profile_info(deepcopy(profile), project.profile_name, cli_vars) args = Obj() args.vars = repr(cli_vars) return RuntimeConfig.from_parts( project=project, profile=profile, args=args )
def _choose_target_name(self, profile_name: str): has_raw_profile = (self.raw_profile_data is not None and profile_name in self.raw_profile_data) if not has_raw_profile: return None # mypy appeasement, we checked just above assert self.raw_profile_data is not None raw_profile = self.raw_profile_data[profile_name] renderer = ProfileRenderer(generate_base_context(self.cli_vars)) target_name, _ = Profile.render_profile(raw_profile=raw_profile, profile_name=profile_name, target_override=getattr( self.args, 'target', None), renderer=renderer) return target_name
def main(): args = sys.argv[1:] parsed = dbt.parse_args(args) project = Project.from_args(parsed) profile = Profile.from_args(parsed, project.profile_name) """ due to dbt's usage of popping out values from the profile dictionary we need to parse the yaml again popped values include type, threads """ profile_yaml = read_profile(PROFILES_DIR) db_type = profile_yaml[profile.profile_name]['outputs'][ profile.target_name]['type'] parser = ArgumentParser() parser.add_argument('cmd', help="Option to perform (prepare, run, teardown)") parser.add_argument('--projdir', help="Project directory path", default=os.path.curdir) parser.add_argument( '--sqldump', help="SQL dump file path to create tables", default="{}/db/redshift/00_campaign_r/V1.0__campaign_r.sql".format( os.path.curdir)) parser.add_argument('--dataset', help="Dataset and test directory path", default="{}/unittest/AttributionAssisted".format( os.path.curdir)) args = parser.parse_args() db_schema_r_conn = connect_db(map_db_type(db_type, profile), '{}_r'.format(profile.credentials['schema'])) db_schema_conn = connect_db(map_db_type(db_type, profile), profile.credentials['schema']) if args.cmd == 'prepare': prepare_data(db_schema_r_conn, args.dataset, args.sqldump) elif args.cmd == 'run': test_exec(db_schema_r_conn, db_schema_conn, args.dataset, args.sqldump, args.projdir) elif args.cmd == 'cleanup': pass
def validate_connection(cls, target_dict): """Validate a connection dictionary. On error, raises a DbtConfigError. """ target_name = 'test' # make a fake profile that we can parse profile_data = { 'outputs': { target_name: target_dict, }, } # this will raise a DbtConfigError on failure profile = Profile.from_raw_profile_info( raw_profile=profile_data, profile_name='', target_override=target_name, renderer=ProfileRenderer(generate_base_context({})), ) result = cls.attempt_connection(profile) if result is not None: raise dbt.exceptions.DbtProfileError( result, result_type='connection_failure')
def _choose_profile_names(self) -> Optional[List[str]]: project_profile: Optional[str] = None if os.path.exists(self.project_path): try: partial = Project.partial_load( os.path.dirname(self.project_path), verify_version=getattr(self.args, 'version_check', False), ) renderer = DbtProjectYamlRenderer( generate_base_context(self.cli_vars) ) project_profile = partial.render_profile_name(renderer) except dbt.exceptions.DbtProjectError: pass args_profile: Optional[str] = getattr(self.args, 'profile', None) try: return [Profile.pick_profile_name(args_profile, project_profile)] except dbt.exceptions.DbtConfigError: pass # try to guess profiles = [] if self.raw_profile_data: profiles = [k for k in self.raw_profile_data if k != 'config'] if project_profile is None: self.messages.append('Could not load dbt_project.yml') elif len(profiles) == 0: self.messages.append('The profiles.yml has no profiles') elif len(profiles) == 1: self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0])) else: self.messages.append(MULTIPLE_PROFILE_MESSAGE.format( '\n'.join(' - {}'.format(o) for o in profiles) )) return profiles