def invoke_dbt(parsed):
task = None
cfg = None
try:
if parsed.which in {'deps', 'clean'}:
# deps doesn't need a profile, so don't require one.
cfg = Project.from_current_directory()
elif parsed.which != 'debug':
# for debug, we will attempt to load the various configurations as
# part of the task, so just leave cfg=None.
cfg = RuntimeConfig.from_args(parsed)
except DbtProjectError as e:
logger.info("Encountered an error while reading the project:")
logger.info(dbt.compat.to_string(e))
all_profiles = read_profiles(parsed.profiles_dir).keys()
if len(all_profiles) > 0:
logger.info("Defined profiles:")
for profile in all_profiles:
logger.info(" - {}".format(profile))
else:
logger.info("There are no profiles defined in your "
"profiles.yml file")
logger.info(PROFILES_HELP_MESSAGE)
dbt.tracking.track_invalid_invocation(config=cfg,
args=parsed,
result_type=e.result_type)
return None
except DbtProfileError as e:
logger.info("Encountered an error while reading profiles:")
logger.info(" ERROR {}".format(str(e)))
dbt.tracking.track_invalid_invocation(config=cfg,
args=parsed,
result_type=e.result_type)
return None
flags.NON_DESTRUCTIVE = getattr(parsed, 'non_destructive', False)
arg_drop_existing = getattr(parsed, 'drop_existing', False)
arg_full_refresh = getattr(parsed, 'full_refresh', False)
if arg_drop_existing:
dbt.deprecations.warn('drop-existing')
flags.FULL_REFRESH = True
elif arg_full_refresh:
flags.FULL_REFRESH = True
logger.debug("running dbt with arguments %s", parsed)
task = parsed.cls(args=parsed, config=cfg)
return task, cfg
def get_project(git_path):
phony_profile = Profile.from_raw_profiles(raw_profiles=PHONY_PROFILE,
profile_name='hubcap',
renderer=ProfileRenderer({}))
ctx = generate_target_context(phony_profile, cli_vars={})
renderer = DbtProjectYamlRenderer(ctx)
return Project.from_project_root(git_path, renderer)
def _fetch_metadata(self, project) -> ProjectPackageMetadata:
path = self._checkout()
if self.revision == 'master' and self.warn_unpinned:
warn_or_error(
'The git package "{}" is not pinned.\n\tThis can introduce '
'breaking changes into your project without warning!\n\nSee {}'
.format(self.git, PIN_PACKAGE_URL),
log_fmt=printer.yellow('WARNING: {}'))
loaded = Project.from_project_root(path, {})
return ProjectPackageMetadata.from_project(loaded)
def _fetch_metadata(self, project, renderer) -> ProjectPackageMetadata:
path = self._checkout()
if self.unpinned_msg() and self.warn_unpinned:
warn_or_error(
'The git package "{}" \n\tis {}.\n\tThis can introduce '
'breaking changes into your project without warning!\n\nSee {}'
.format(self.git, self.unpinned_msg(), PIN_PACKAGE_URL),
log_fmt=ui.yellow('WARNING: {}'))
loaded = Project.from_project_root(path, renderer)
return ProjectPackageMetadata.from_project(loaded)
def project_from_dict(project, profile, packages=None, cli_vars='{}'):
from dbt.context.target import generate_target_context
from dbt.config import Project, ConfigRenderer
from dbt.utils import parse_cli_vars
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
renderer = ConfigRenderer(generate_target_context(profile, cli_vars))
project_root = project.pop('project-root', os.getcwd())
return Project.render_from_dict(project_root, project, packages, renderer)
def _load_project(self):
if not os.path.exists(self.project_path):
self.project_fail_details = FILE_NOT_FOUND
return red('ERROR not found')
try:
self.project = Project.from_current_directory(self.cli_vars)
except dbt.exceptions.DbtConfigError as exc:
self.project_fail_details = str(exc)
return red('ERROR invalid')
return green('OK found and valid')
def _load_project(self):
if not os.path.exists(self.project_path):
self.project_fail_details = FILE_NOT_FOUND
return red('ERROR not found')
try:
self.project = Project.from_current_directory(self.cli_vars)
except dbt.exceptions.DbtConfigError as exc:
self.project_fail_details = str(exc)
return red('ERROR invalid')
return green('OK found and valid')
def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'):
from dbt.config import Project, Profile, RuntimeConfig
from dbt.utils import parse_cli_vars
from copy import deepcopy
if not isinstance(project, Project):
project = Project.from_project_config(deepcopy(project), packages)
if not isinstance(profile, Profile):
profile = Profile.from_raw_profile_info(deepcopy(profile),
project.profile_name)
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
return RuntimeConfig.from_parts(project=project,
profile=profile,
cli_vars=cli_vars)
def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'):
from dbt.config import Project, Profile, RuntimeConfig
from dbt.utils import parse_cli_vars
from copy import deepcopy
if not isinstance(cli_vars, dict):
cli_vars = parse_cli_vars(cli_vars)
if not isinstance(project, Project):
project = Project.from_project_config(deepcopy(project), packages)
if not isinstance(profile, Profile):
profile = Profile.from_raw_profile_info(deepcopy(profile),
project.profile_name,
cli_vars)
args = Obj()
args.vars = repr(cli_vars)
return RuntimeConfig.from_parts(
project=project,
profile=profile,
args=args
)
def main():
args = sys.argv[1:]
parsed = dbt.parse_args(args)
project = Project.from_args(parsed)
profile = Profile.from_args(parsed, project.profile_name)
"""
due to dbt's usage of popping out values from the profile dictionary we need to parse the yaml again
popped values include type, threads
"""
profile_yaml = read_profile(PROFILES_DIR)
db_type = profile_yaml[profile.profile_name]['outputs'][
profile.target_name]['type']
parser = ArgumentParser()
parser.add_argument('cmd',
help="Option to perform (prepare, run, teardown)")
parser.add_argument('--projdir',
help="Project directory path",
default=os.path.curdir)
parser.add_argument(
'--sqldump',
help="SQL dump file path to create tables",
default="{}/db/redshift/00_campaign_r/V1.0__campaign_r.sql".format(
os.path.curdir))
parser.add_argument('--dataset',
help="Dataset and test directory path",
default="{}/unittest/AttributionAssisted".format(
os.path.curdir))
args = parser.parse_args()
db_schema_r_conn = connect_db(map_db_type(db_type, profile),
'{}_r'.format(profile.credentials['schema']))
db_schema_conn = connect_db(map_db_type(db_type, profile),
profile.credentials['schema'])
if args.cmd == 'prepare':
prepare_data(db_schema_r_conn, args.dataset, args.sqldump)
elif args.cmd == 'run':
test_exec(db_schema_r_conn, db_schema_conn, args.dataset, args.sqldump,
args.projdir)
elif args.cmd == 'cleanup':
pass
def _load_project(self):
if not os.path.exists(self.project_path):
self.project_fail_details = FILE_NOT_FOUND
return red('ERROR not found')
if self.profile is None:
ctx = generate_base_context(self.cli_vars)
else:
ctx = generate_target_context(self.profile, self.cli_vars)
renderer = DbtProjectYamlRenderer(ctx)
try:
self.project = Project.from_project_root(
self.project_dir,
renderer,
verify_version=getattr(self.args, 'version_check', False),
)
except dbt.exceptions.DbtConfigError as exc:
self.project_fail_details = str(exc)
return red('ERROR invalid')
return green('OK found and valid')
def _choose_profile_names(self) -> Optional[List[str]]:
project_profile: Optional[str] = None
if os.path.exists(self.project_path):
try:
partial = Project.partial_load(
os.path.dirname(self.project_path),
verify_version=getattr(self.args, 'version_check', False),
)
renderer = DbtProjectYamlRenderer(
generate_base_context(self.cli_vars)
)
project_profile = partial.render_profile_name(renderer)
except dbt.exceptions.DbtProjectError:
pass
args_profile: Optional[str] = getattr(self.args, 'profile', None)
try:
return [Profile.pick_profile_name(args_profile, project_profile)]
except dbt.exceptions.DbtConfigError:
pass
# try to guess
profiles = []
if self.raw_profile_data:
profiles = [k for k in self.raw_profile_data if k != 'config']
if project_profile is None:
self.messages.append('Could not load dbt_project.yml')
elif len(profiles) == 0:
self.messages.append('The profiles.yml has no profiles')
elif len(profiles) == 1:
self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0]))
else:
self.messages.append(MULTIPLE_PROFILE_MESSAGE.format(
'\n'.join(' - {}'.format(o) for o in profiles)
))
return profiles
def get_project(git_path):
ctx = generate_base_context({})
renderer = DbtProjectYamlRenderer(ctx)
return Project.from_project_root(git_path, renderer)