def is_opted_out():
profiles = project.read_profiles()
if profiles is None or profiles.get("config") is None:
return False
elif profiles['config'].get("send_anonymous_usage_stats") == False:
return True
else:
return False
def invoke_dbt(parsed):
task = None
proj = None
try:
proj = project.read_project('dbt_project.yml',
parsed.profiles_dir,
validate=False,
profile_to_load=parsed.profile)
proj.validate()
except project.DbtProjectError as e:
logger.info("Encountered an error while reading the project:")
logger.info(" ERROR {}".format(str(e)))
logger.info("Did you set the correct --profile? Using: {}".format(
parsed.profile))
logger.info("Valid profiles:")
all_profiles = project.read_profiles(parsed.profiles_dir).keys()
for profile in all_profiles:
logger.info(" - {}".format(profile))
dbt.tracking.track_invalid_invocation(project=proj,
args=parsed,
result_type="invalid_profile",
result=str(e))
return None
if parsed.target is not None:
targets = proj.cfg.get('outputs', {}).keys()
if parsed.target in targets:
proj.cfg['target'] = parsed.target
else:
logger.info("Encountered an error while reading the project:")
logger.info(" ERROR Specified target {} is not a valid option "
"for profile {}".format(parsed.target,
proj.profile_to_load))
logger.info("Valid targets are: {}".format(targets))
dbt.tracking.track_invalid_invocation(project=proj,
args=parsed,
result_type="invalid_target",
result="target not found")
return None
log_dir = proj.get('log-path', 'logs')
logger.debug("running dbt with arguments %s", parsed)
task = parsed.cls(args=parsed, project=proj)
return task, proj
config = read_config('config/prod.yml')
PROFILE = sys.argv[1]
BASTION_SERVER = config['bastion']
cwd = os.path.dirname(os.path.realpath(__file__))
program_name = "analyze-vacuum-schema.py"
PROG = os.path.join(cwd, program_name)
LOG_DIR = os.path.join(cwd, "logs")
if not os.path.exists(LOG_DIR):
os.makedirs(LOG_DIR)
all_profiles = read_profiles()
p = Project({}, all_profiles, [PROFILE])
env = p.run_environment()
ssh_config = paramiko.SSHConfig()
user_config_file = os.path.expanduser("~/.ssh/config")
with open(user_config_file) as f:
ssh_config.parse(f)
user_config = ssh_config.lookup(BASTION_SERVER)
hostname = user_config['hostname']
username = user_config['user']
def get_schemas(conn_string):
schemas_query = "select nspname from pg_namespace where nspname not like 'pg%'"
def invoke_dbt(parsed):
task = None
proj = None
adapter_cache.reset()
try:
proj = project.read_project('dbt_project.yml',
parsed.profiles_dir,
validate=False,
profile_to_load=parsed.profile,
args=parsed)
proj.validate()
except project.DbtProjectError as e:
logger.info("Encountered an error while reading the project:")
logger.info(dbt.compat.to_string(e))
all_profiles = project.read_profiles(parsed.profiles_dir).keys()
if len(all_profiles) > 0:
logger.info("Defined profiles:")
for profile in all_profiles:
logger.info(" - {}".format(profile))
else:
logger.info("There are no profiles defined in your "
"profiles.yml file")
logger.info(PROFILES_HELP_MESSAGE)
dbt.tracking.track_invalid_invocation(project=proj,
args=parsed,
result_type="invalid_profile",
result=str(e))
return None
except project.DbtProfileError as e:
logger.info("Encountered an error while reading profiles:")
logger.info(" ERROR {}".format(str(e)))
dbt.tracking.track_invalid_invocation(project=proj,
args=parsed,
result_type="invalid_profile",
result=str(e))
return None
if parsed.target is not None:
targets = proj.cfg.get('outputs', {}).keys()
if parsed.target in targets:
proj.cfg['target'] = parsed.target
else:
logger.info("Encountered an error while reading the project:")
logger.info(" ERROR Specified target {} is not a valid option "
"for profile {}".format(parsed.target,
proj.profile_to_load))
logger.info("Valid targets are: {}".format(', '.join(targets)))
dbt.tracking.track_invalid_invocation(project=proj,
args=parsed,
result_type="invalid_target",
result="target not found")
return None
flags.NON_DESTRUCTIVE = getattr(proj.args, 'non_destructive', False)
flags.FULL_REFRESH = getattr(proj.args, 'full_refresh', False)
logger.debug("running dbt with arguments %s", parsed)
task = parsed.cls(args=parsed, project=proj)
return task, proj
def handle(args):
p = argparse.ArgumentParser(prog='dbt: data build tool',
formatter_class=argparse.RawTextHelpFormatter)
p.add_argument('--version',
action='version',
version=dbt.version.get_version_information(),
help="Show version information")
subs = p.add_subparsers()
base_subparser = argparse.ArgumentParser(add_help=False)
base_subparser.add_argument('--profile',
default=["user"],
nargs='+',
type=str,
help='Which profile to load')
base_subparser.add_argument(
'--target',
default=None,
type=str,
help='Which run-target to load for the given profile')
sub = subs.add_parser('init', parents=[base_subparser])
sub.add_argument('project_name', type=str, help='Name of the new project')
sub.set_defaults(cls=init_task.InitTask, which='init')
sub = subs.add_parser('clean', parents=[base_subparser])
sub.set_defaults(cls=clean_task.CleanTask, which='clean')
sub = subs.add_parser('compile', parents=[base_subparser])
sub.add_argument('--dry',
action='store_true',
help="Compile 'dry run' models")
sub.set_defaults(cls=compile_task.CompileTask, which='compile')
sub = subs.add_parser('debug', parents=[base_subparser])
sub.set_defaults(cls=debug_task.DebugTask, which='debug')
sub = subs.add_parser('deps', parents=[base_subparser])
sub.set_defaults(cls=deps_task.DepsTask, which='deps')
sub = subs.add_parser('run', parents=[base_subparser])
sub.add_argument('--dry', action='store_true', help="'dry run' models")
sub.add_argument(
'--models',
required=False,
nargs='+',
help=
"Specify the models to run. All models depending on these models will also be run"
)
sub.set_defaults(cls=run_task.RunTask, which='run')
sub = subs.add_parser('seed', parents=[base_subparser])
sub.add_argument('--drop-existing',
action='store_true',
help="Drop existing seed tables and recreate them")
sub.set_defaults(cls=seed_task.SeedTask, which='seed')
sub = subs.add_parser('test', parents=[base_subparser])
sub.add_argument('--skip-test-creates',
action='store_true',
help="Don't create temporary views to validate model SQL")
sub.add_argument('--validate',
action='store_true',
help='Run constraint validations from schema.yml files')
sub.set_defaults(cls=test_task.TestTask, which='test')
if len(args) == 0: return p.print_help()
parsed = p.parse_args(args)
task = None
proj = None
if parsed.which == 'init':
# bypass looking for a project file if we're running `dbt init`
task = parsed.cls(args=parsed)
elif os.path.isfile('dbt_project.yml'):
try:
proj = project.read_project('dbt_project.yml',
validate=False).with_profiles(
parsed.profile)
proj.validate()
except project.DbtProjectError as e:
print("Encountered an error while reading the project:")
print(" ERROR {}".format(str(e)))
print("Did you set the correct --profile? Using: {}".format(
parsed.profile))
all_profiles = project.read_profiles().keys()
profiles_string = "\n".join([" - " + key for key in all_profiles])
print("Valid profiles:\n{}".format(profiles_string))
dbt.tracking.track_invalid_invocation(
project=proj,
args=parsed,
result_type="invalid_profile",
result=str(e))
return None
if parsed.target is not None:
targets = proj.cfg.get('outputs', {}).keys()
if parsed.target in targets:
proj.cfg['run-target'] = parsed.target
else:
print("Encountered an error while reading the project:")
print(
" ERROR Specified target {} is not a valid option for profile {}"
.format(parsed.target, parsed.profile))
print("Valid targets are: {}".format(targets))
dbt.tracking.track_invalid_invocation(
project=proj,
args=parsed,
result_type="invalid_target",
result="target not found")
return None
log_dir = proj.get('log-path', 'logs')
logger = getLogger(log_dir, __name__)
logger.info("running dbt with arguments %s", parsed)
task = parsed.cls(args=parsed, project=proj)
else:
raise RuntimeError(
"dbt must be run from a project root directory with a dbt_project.yml file"
)
dbt.tracking.track_invocation_start(project=proj, args=parsed)
try:
task.run()
dbt.tracking.track_invocation_end(project=proj,
args=parsed,
result_type="ok",
result=None)
except Exception as e:
dbt.tracking.track_invocation_end(project=proj,
args=parsed,
result_type="error",
result=str(e))
raise
def handle(args):
p = argparse.ArgumentParser(prog='dbt: data build tool')
subs = p.add_subparsers()
base_subparser = argparse.ArgumentParser(add_help=False)
base_subparser.add_argument('--profile', default=["user"], nargs='+', type=str, help='Which profile to load')
base_subparser.add_argument('--target', default=None, type=str, help='Which run-target to load for the given profile')
sub = subs.add_parser('init', parents=[base_subparser])
sub.add_argument('project_name', type=str, help='Name of the new project')
sub.set_defaults(cls=init_task.InitTask, which='init')
sub = subs.add_parser('clean', parents=[base_subparser])
sub.set_defaults(cls=clean_task.CleanTask, which='clean')
sub = subs.add_parser('compile', parents=[base_subparser])
sub.set_defaults(cls=compile_task.CompileTask, which='compile')
sub = subs.add_parser('debug', parents=[base_subparser])
sub.set_defaults(cls=debug_task.DebugTask, which='debug')
sub = subs.add_parser('deps', parents=[base_subparser])
sub.set_defaults(cls=deps_task.DepsTask, which='deps')
sub = subs.add_parser('run', parents=[base_subparser])
sub.add_argument('--models', required=False, nargs='+', help="Specify the models to run. All models depending on these models will also be run")
sub.set_defaults(cls=run_task.RunTask, which='run')
sub = subs.add_parser('seed', parents=[base_subparser])
sub.add_argument('--drop-existing', action='store_true', help="Drop existing seed tables and recreate them")
sub.set_defaults(cls=seed_task.SeedTask, which='seed')
sub = subs.add_parser('test', parents=[base_subparser])
sub.add_argument('--skip-test-creates', action='store_true', help="Don't create temporary views to validate model SQL")
sub.add_argument('--validate', action='store_true', help='Run constraint validations from schema.yml files')
sub.set_defaults(cls=test_task.TestTask, which='test')
if len(args) == 0: return p.print_help()
parsed = p.parse_args(args)
if parsed.which == 'init':
# bypass looking for a project file if we're running `dbt init`
parsed.cls(args=parsed).run()
elif os.path.isfile('dbt_project.yml'):
try:
proj = project.read_project('dbt_project.yml', validate=False).with_profiles(parsed.profile)
proj.validate()
except project.DbtProjectError as e:
print("Encountered an error while reading the project:")
print(" ERROR {}".format(str(e)))
print("Did you set the correct --profile? Using: {}".format(parsed.profile))
all_profiles = project.read_profiles().keys()
profiles_string = "\n".join([" - " + key for key in all_profiles])
print("Valid profiles:\n{}".format(profiles_string))
return
if parsed.target is not None:
targets = proj.cfg.get('outputs', {}).keys()
if parsed.target in targets:
proj.cfg['run-target'] = parsed.target
else:
print("Encountered an error while reading the project:")
print(" ERROR Specified target {} is not a valid option for profile {}".format(parsed.target, parsed.profile))
print("Valid targets are: {}".format(targets))
return
parsed.cls(args=parsed, project=proj).run()
else:
raise RuntimeError("dbt must be run from a project root directory with a dbt_project.yml file")
def invoke_dbt(parsed):
task = None
proj = None
try:
proj = project.read_project(
'dbt_project.yml',
parsed.profiles_dir,
validate=False,
profile_to_load=parsed.profile,
args=parsed
)
proj.validate()
except project.DbtProjectError as e:
logger.info("Encountered an error while reading the project:")
logger.info(dbt.compat.to_string(e))
all_profiles = project.read_profiles(parsed.profiles_dir).keys()
if len(all_profiles) > 0:
logger.info("Defined profiles:")
for profile in all_profiles:
logger.info(" - {}".format(profile))
else:
logger.info("There are no profiles defined in your "
"profiles.yml file")
logger.info(PROFILES_HELP_MESSAGE)
dbt.tracking.track_invalid_invocation(
project=proj,
args=parsed,
result_type="invalid_profile")
return None
except project.DbtProfileError as e:
logger.info("Encountered an error while reading profiles:")
logger.info(" ERROR {}".format(str(e)))
dbt.tracking.track_invalid_invocation(
project=proj,
args=parsed,
result_type="invalid_profile")
return None
if parsed.target is not None:
targets = proj.cfg.get('outputs', {}).keys()
if parsed.target in targets:
proj.cfg['target'] = parsed.target
# make sure we update the target if this is overriden on the cli
proj.compile_and_update_target()
else:
logger.info("Encountered an error while reading the project:")
logger.info(" ERROR Specified target {} is not a valid option "
"for profile {}"
.format(parsed.target, proj.profile_to_load))
logger.info("Valid targets are: {}".format(
', '.join(targets)))
dbt.tracking.track_invalid_invocation(
project=proj,
args=parsed,
result_type="invalid_target")
return None
proj.log_warnings()
flags.NON_DESTRUCTIVE = getattr(proj.args, 'non_destructive', False)
arg_drop_existing = getattr(proj.args, 'drop_existing', False)
arg_full_refresh = getattr(proj.args, 'full_refresh', False)
if arg_drop_existing:
dbt.deprecations.warn('drop-existing')
flags.FULL_REFRESH = True
elif arg_full_refresh:
flags.FULL_REFRESH = True
logger.debug("running dbt with arguments %s", parsed)
task = parsed.cls(args=parsed, project=proj)
return task, proj
