def _load_profile(self):
if not os.path.exists(self.profile_path):
self.profile_fail_details = FILE_NOT_FOUND
self.messages.append(MISSING_PROFILE_MESSAGE.format(
path=self.profile_path, url=ProfileConfigDocs
))
return red('ERROR not found')
try:
raw_profile_data = load_yaml_text(
dbt.clients.system.load_file_contents(self.profile_path)
)
except Exception:
pass # we'll report this when we try to load the profile for real
else:
if isinstance(raw_profile_data, dict):
self.raw_profile_data = raw_profile_data
self.profile_name = self._choose_profile_name()
self.target_name = self._choose_target_name()
try:
self.profile = Profile.from_args(self.args, self.profile_name,
self.cli_vars)
except dbt.exceptions.DbtConfigError as exc:
self.profile_fail_details = str(exc)
return red('ERROR invalid')
return green('OK found and valid')
def _choose_profile_name(self):
assert self.project or self.project_fail_details, \
'_load_project() required'
project_profile = None
if self.project:
project_profile = self.project.profile_name
args_profile = getattr(self.args, 'profile', None)
try:
return Profile.pick_profile_name(args_profile, project_profile)
except dbt.exceptions.DbtConfigError:
pass
# try to guess
if self.raw_profile_data:
profiles = [k for k in self.raw_profile_data if k != 'config']
if len(profiles) == 0:
self.messages.append('The profiles.yml has no profiles')
elif len(profiles) == 1:
self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0]))
return profiles[0]
else:
self.messages.append(MULTIPLE_PROFILE_MESSAGE.format(
'\n'.join(' - {}'.format(o) for o in profiles)
))
return None
def _load_profile(self):
if not os.path.exists(self.profile_path):
self.profile_fail_details = FILE_NOT_FOUND
self.messages.append(
MISSING_PROFILE_MESSAGE.format(path=self.profile_path,
url=ProfileConfigDocs))
return red('ERROR not found')
try:
raw_profile_data = load_yaml_text(
dbt.clients.system.load_file_contents(self.profile_path))
except Exception:
pass # we'll report this when we try to load the profile for real
else:
if isinstance(raw_profile_data, dict):
self.raw_profile_data = raw_profile_data
self.profile_name = self._choose_profile_name()
self.target_name = self._choose_target_name()
try:
self.profile = Profile.from_args(self.args, self.profile_name)
except dbt.exceptions.DbtConfigError as exc:
self.profile_fail_details = str(exc)
return red('ERROR invalid')
return green('OK found and valid')
def _choose_profile_name(self):
assert self.project or self.project_fail_details, \
'_load_project() required'
project_profile = None
if self.project:
project_profile = self.project.profile_name
args_profile = getattr(self.args, 'profile', None)
try:
return Profile.pick_profile_name(args_profile, project_profile)
except dbt.exceptions.DbtConfigError:
pass
# try to guess
if self.raw_profile_data:
profiles = [k for k in self.raw_profile_data if k != 'config']
if len(profiles) == 0:
self.messages.append('The profiles.yml has no profiles')
elif len(profiles) == 1:
self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0]))
return profiles[0]
else:
self.messages.append(
MULTIPLE_PROFILE_MESSAGE.format('\n'.join(
' - {}'.format(o) for o in profiles)))
return None
def _choose_target_name(self):
has_raw_profile = (self.raw_profile_data and self.profile_name
and self.profile_name in self.raw_profile_data)
if has_raw_profile:
raw_profile = self.raw_profile_data[self.profile_name]
target_name, _ = Profile.render_profile(
raw_profile, self.profile_name,
getattr(self.args, 'target', None), self.cli_vars)
return target_name
return None
def _choose_target_name(self):
has_raw_profile = (self.raw_profile_data and self.profile_name and
self.profile_name in self.raw_profile_data)
if has_raw_profile:
raw_profile = self.raw_profile_data[self.profile_name]
target_name, _ = Profile.render_profile(
raw_profile, self.profile_name,
getattr(self.args, 'target', None), self.cli_vars
)
return target_name
return None
def profile_from_dict(profile, profile_name, cli_vars='{}'):
from dbt.config import Profile, ConfigRenderer
from dbt.context.base import generate_base_context
from dbt.utils import parse_cli_vars
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
renderer = ConfigRenderer(generate_base_context(cli_vars))
return Profile.from_raw_profile_info(
profile,
profile_name,
renderer,
)
def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'):
from dbt.config import Project, Profile, RuntimeConfig
from dbt.utils import parse_cli_vars
from copy import deepcopy
if not isinstance(project, Project):
project = Project.from_project_config(deepcopy(project), packages)
if not isinstance(profile, Profile):
profile = Profile.from_raw_profile_info(deepcopy(profile),
project.profile_name)
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
return RuntimeConfig.from_parts(project=project,
profile=profile,
cli_vars=cli_vars)
def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'):
from dbt.config import Project, Profile, RuntimeConfig
from dbt.utils import parse_cli_vars
from copy import deepcopy
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
if not isinstance(project, Project):
project = Project.from_project_config(deepcopy(project), packages)
if not isinstance(profile, Profile):
profile = Profile.from_raw_profile_info(deepcopy(profile),
project.profile_name,
cli_vars)
args = Obj()
args.vars = repr(cli_vars)
return RuntimeConfig.from_parts(
project=project,
profile=profile,
args=args
)
def _choose_target_name(self, profile_name: str):
has_raw_profile = (self.raw_profile_data is not None
and profile_name in self.raw_profile_data)
if not has_raw_profile:
return None
# mypy appeasement, we checked just above
assert self.raw_profile_data is not None
raw_profile = self.raw_profile_data[profile_name]
renderer = ProfileRenderer(generate_base_context(self.cli_vars))
target_name, _ = Profile.render_profile(raw_profile=raw_profile,
profile_name=profile_name,
target_override=getattr(
self.args, 'target', None),
renderer=renderer)
return target_name
def main():
args = sys.argv[1:]
parsed = dbt.parse_args(args)
project = Project.from_args(parsed)
profile = Profile.from_args(parsed, project.profile_name)
"""
due to dbt's usage of popping out values from the profile dictionary we need to parse the yaml again
popped values include type, threads
"""
profile_yaml = read_profile(PROFILES_DIR)
db_type = profile_yaml[profile.profile_name]['outputs'][
profile.target_name]['type']
parser = ArgumentParser()
parser.add_argument('cmd',
help="Option to perform (prepare, run, teardown)")
parser.add_argument('--projdir',
help="Project directory path",
default=os.path.curdir)
parser.add_argument(
'--sqldump',
help="SQL dump file path to create tables",
default="{}/db/redshift/00_campaign_r/V1.0__campaign_r.sql".format(
os.path.curdir))
parser.add_argument('--dataset',
help="Dataset and test directory path",
default="{}/unittest/AttributionAssisted".format(
os.path.curdir))
args = parser.parse_args()
db_schema_r_conn = connect_db(map_db_type(db_type, profile),
'{}_r'.format(profile.credentials['schema']))
db_schema_conn = connect_db(map_db_type(db_type, profile),
profile.credentials['schema'])
if args.cmd == 'prepare':
prepare_data(db_schema_r_conn, args.dataset, args.sqldump)
elif args.cmd == 'run':
test_exec(db_schema_r_conn, db_schema_conn, args.dataset, args.sqldump,
args.projdir)
elif args.cmd == 'cleanup':
pass
def validate_connection(cls, target_dict):
"""Validate a connection dictionary. On error, raises a DbtConfigError.
"""
target_name = 'test'
# make a fake profile that we can parse
profile_data = {
'outputs': {
target_name: target_dict,
},
}
# this will raise a DbtConfigError on failure
profile = Profile.from_raw_profile_info(
raw_profile=profile_data,
profile_name='',
target_override=target_name,
renderer=ProfileRenderer(generate_base_context({})),
)
result = cls.attempt_connection(profile)
if result is not None:
raise dbt.exceptions.DbtProfileError(
result, result_type='connection_failure')
def _choose_profile_names(self) -> Optional[List[str]]:
project_profile: Optional[str] = None
if os.path.exists(self.project_path):
try:
partial = Project.partial_load(
os.path.dirname(self.project_path),
verify_version=getattr(self.args, 'version_check', False),
)
renderer = DbtProjectYamlRenderer(
generate_base_context(self.cli_vars)
)
project_profile = partial.render_profile_name(renderer)
except dbt.exceptions.DbtProjectError:
pass
args_profile: Optional[str] = getattr(self.args, 'profile', None)
try:
return [Profile.pick_profile_name(args_profile, project_profile)]
except dbt.exceptions.DbtConfigError:
pass
# try to guess
profiles = []
if self.raw_profile_data:
profiles = [k for k in self.raw_profile_data if k != 'config']
if project_profile is None:
self.messages.append('Could not load dbt_project.yml')
elif len(profiles) == 0:
self.messages.append('The profiles.yml has no profiles')
elif len(profiles) == 1:
self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0]))
else:
self.messages.append(MULTIPLE_PROFILE_MESSAGE.format(
'\n'.join(' - {}'.format(o) for o in profiles)
))
return profiles
