def dbt_url_provider(profile, target):
raw_profiles = read_profile(PROFILES_DIR)
profile = Profile.from_raw_profiles(raw_profiles=raw_profiles,
profile_name=profile,
cli_vars={},
target_override=target,
threads_override=1)
credentials = profile.credentials
return f"redshift+psycopg2://{credentials.user}:{credentials.password}@{credentials.host}:{credentials.port}/{credentials.database}"
def read_profiles(profiles_dir=None):
"""This is only used for some error handling"""
if profiles_dir is None:
profiles_dir = PROFILES_DIR
raw_profiles = read_profile(profiles_dir)
if raw_profiles is None:
profiles = {}
else:
profiles = {k: v for (k, v) in raw_profiles.items() if k != 'config'}
return profiles
def main():
args = sys.argv[1:]
parsed = dbt.parse_args(args)
project = Project.from_args(parsed)
profile = Profile.from_args(parsed, project.profile_name)
"""
due to dbt's usage of popping out values from the profile dictionary we need to parse the yaml again
popped values include type, threads
"""
profile_yaml = read_profile(PROFILES_DIR)
db_type = profile_yaml[profile.profile_name]['outputs'][
profile.target_name]['type']
parser = ArgumentParser()
parser.add_argument('cmd',
help="Option to perform (prepare, run, teardown)")
parser.add_argument('--projdir',
help="Project directory path",
default=os.path.curdir)
parser.add_argument(
'--sqldump',
help="SQL dump file path to create tables",
default="{}/db/redshift/00_campaign_r/V1.0__campaign_r.sql".format(
os.path.curdir))
parser.add_argument('--dataset',
help="Dataset and test directory path",
default="{}/unittest/AttributionAssisted".format(
os.path.curdir))
args = parser.parse_args()
db_schema_r_conn = connect_db(map_db_type(db_type, profile),
'{}_r'.format(profile.credentials['schema']))
db_schema_conn = connect_db(map_db_type(db_type, profile),
profile.credentials['schema'])
if args.cmd == 'prepare':
prepare_data(db_schema_r_conn, args.dataset, args.sqldump)
elif args.cmd == 'run':
test_exec(db_schema_r_conn, db_schema_conn, args.dataset, args.sqldump,
args.projdir)
elif args.cmd == 'cleanup':
pass
def create_dbt_profile(
self,
extra_targets: Optional[dict[str, Any]] = None,
) -> Profile:
"""Create a dbt Profile with any added extra targets.
Extra targets are appended under the given profile_name. If no profile is found
with the given profile_name it is created with any extra_targets.
Returns:
A dbt profile for the task represented by this configuration.
"""
if self.profiles_dir is not None:
raw_profiles = read_profile(self.profiles_dir)
else:
raw_profiles = {}
if extra_targets:
profile = raw_profiles.setdefault(self.profile_name, {})
outputs = profile.setdefault("outputs", {})
outputs.setdefault("target", self.target)
profile["outputs"] = {**outputs, **extra_targets}
user_config = raw_profiles.get("config", {})
profile = Profile.from_raw_profile_info(
raw_profile=raw_profiles.get(
self.profile_name, {}
), # Let dbt handle missing profile errors.
profile_name=self.profile_name,
renderer=self.profile_renderer,
user_config=user_config,
target_override=self.target,
threads_override=self.threads,
)
profile.profile_env_vars = self.profile_renderer.ctx_obj.env_vars
return profile