def run_schema_validations(self):
project = read_project('dbt_project.yml')
args = FakeArgs()
test_task = TestTask(args, project)
print(project)
return test_task.run()
def __pull_deps_recursive(self, repos, processed_repos=None, i=0):
if processed_repos is None:
processed_repos = set()
for repo_string in repos:
repo, branch = self.__split_at_branch(repo_string)
repo_folder = folder_from_git_remote(repo)
try:
if repo_folder in processed_repos:
logger.info(
"skipping already processed dependency {}".format(
repo_folder))
else:
dep_folder = self.__pull_repo(repo, branch)
dep_project = project.read_project(
os.path.join(self.project['modules-path'], dep_folder,
'dbt_project.yml'),
self.project.profiles_dir,
profile_to_load=self.project.profile_to_load)
processed_repos.add(dep_folder)
self.__pull_deps_recursive(dep_project['repositories'],
processed_repos, i + 1)
except IOError as e:
if e.errno == errno.ENOENT:
logger.info("'{}' is not a valid dbt project - "
"dbt_project.yml not found".format(repo))
exit(1)
else:
raise e
def run_data_validations(self):
project = read_project('dbt_project.yml')
args = FakeArgs()
args.data = True
test_task = TestTask(args, project)
return test_task.run()
def invoke_dbt(parsed):
task = None
proj = None
try:
proj = project.read_project('dbt_project.yml',
parsed.profiles_dir,
validate=False,
profile_to_load=parsed.profile)
proj.validate()
except project.DbtProjectError as e:
logger.info("Encountered an error while reading the project:")
logger.info(" ERROR {}".format(str(e)))
logger.info("Did you set the correct --profile? Using: {}".format(
parsed.profile))
logger.info("Valid profiles:")
all_profiles = project.read_profiles(parsed.profiles_dir).keys()
for profile in all_profiles:
logger.info(" - {}".format(profile))
dbt.tracking.track_invalid_invocation(project=proj,
args=parsed,
result_type="invalid_profile",
result=str(e))
return None
if parsed.target is not None:
targets = proj.cfg.get('outputs', {}).keys()
if parsed.target in targets:
proj.cfg['target'] = parsed.target
else:
logger.info("Encountered an error while reading the project:")
logger.info(" ERROR Specified target {} is not a valid option "
"for profile {}".format(parsed.target,
proj.profile_to_load))
logger.info("Valid targets are: {}".format(targets))
dbt.tracking.track_invalid_invocation(project=proj,
args=parsed,
result_type="invalid_target",
result="target not found")
return None
log_dir = proj.get('log-path', 'logs')
logger.debug("running dbt with arguments %s", parsed)
task = parsed.cls(args=parsed, project=proj)
return task, proj
def __pull_deps_recursive(self, repos, processed_repos = set()):
for repo in repos:
repo_folder = folder_from_git_remote(repo)
try:
if repo_folder in processed_repos:
print("skipping already processed dependency {}".format(repo_folder))
else:
dep_folder = self.__pull_repo(repo)
dep_project = project.read_project(
os.path.join(self.project['modules-path'],
dep_folder,
'dbt_project.yml')
)
processed_repos.add(dep_folder)
self.__pull_deps_recursive(dep_project['repositories'], processed_repos)
except IOError as e:
if e.errno == errno.ENOENT:
print("'{}' is not a valid dbt project - dbt_project.yml not found".format(repo))
exit(1)
else:
raise e
def run_schema_and_assert(self, include, exclude, expected_tests):
self.use_profile('postgres')
self.use_default_project()
self.project = read_project('dbt_project.yml')
self.run_sql_file(
"test/integration/007_graph_selection_tests/seed.sql")
self.run_dbt(["deps"])
results = self.run_dbt()
self.assertEqual(len(results), 5)
args = FakeArgs()
args.models = include
args.exclude = exclude
test_task = TestTask(args, self.project)
test_results = test_task.run()
ran_tests = sorted([test.node.get('name') for test in test_results])
expected_sorted = sorted(expected_tests)
self.assertEqual(ran_tests, expected_sorted)
def main(args):
if os.path.isfile('dbt_project.yml'):
proj = project.read_project('dbt_project.yml')
else:
proj = project.default_project()
p = argparse.ArgumentParser(prog='dbt: data build tool')
subs = p.add_subparsers()
sub = subs.add_parser('clean')
sub.set_defaults(cls=clean_task.CleanTask)
sub = subs.add_parser('run')
sub.set_defaults(cls=run_task.RunTask)
sub = subs.add_parser('compile')
sub.set_defaults(cls=compile_task.CompileTask)
sub = subs.add_parser('debug')
sub.set_defaults(cls=debug_task.DebugTask)
parsed = p.parse_args(args)
parsed.cls(args=parsed, project=proj).run()
def setUp(self):
DBTIntegrationTest.setUp(self)
self.use_default_project()
self.project = read_project('dbt_project.yml')
def invoke_dbt(parsed):
task = None
proj = None
adapter_cache.reset()
try:
proj = project.read_project('dbt_project.yml',
parsed.profiles_dir,
validate=False,
profile_to_load=parsed.profile,
args=parsed)
proj.validate()
except project.DbtProjectError as e:
logger.info("Encountered an error while reading the project:")
logger.info(dbt.compat.to_string(e))
all_profiles = project.read_profiles(parsed.profiles_dir).keys()
if len(all_profiles) > 0:
logger.info("Defined profiles:")
for profile in all_profiles:
logger.info(" - {}".format(profile))
else:
logger.info("There are no profiles defined in your "
"profiles.yml file")
logger.info(PROFILES_HELP_MESSAGE)
dbt.tracking.track_invalid_invocation(project=proj,
args=parsed,
result_type="invalid_profile",
result=str(e))
return None
except project.DbtProfileError as e:
logger.info("Encountered an error while reading profiles:")
logger.info(" ERROR {}".format(str(e)))
dbt.tracking.track_invalid_invocation(project=proj,
args=parsed,
result_type="invalid_profile",
result=str(e))
return None
if parsed.target is not None:
targets = proj.cfg.get('outputs', {}).keys()
if parsed.target in targets:
proj.cfg['target'] = parsed.target
else:
logger.info("Encountered an error while reading the project:")
logger.info(" ERROR Specified target {} is not a valid option "
"for profile {}".format(parsed.target,
proj.profile_to_load))
logger.info("Valid targets are: {}".format(', '.join(targets)))
dbt.tracking.track_invalid_invocation(project=proj,
args=parsed,
result_type="invalid_target",
result="target not found")
return None
flags.NON_DESTRUCTIVE = getattr(proj.args, 'non_destructive', False)
flags.FULL_REFRESH = getattr(proj.args, 'full_refresh', False)
logger.debug("running dbt with arguments %s", parsed)
task = parsed.cls(args=parsed, project=proj)
return task, proj
def handle(args):
p = argparse.ArgumentParser(prog='dbt: data build tool',
formatter_class=argparse.RawTextHelpFormatter)
p.add_argument('--version',
action='version',
version=dbt.version.get_version_information(),
help="Show version information")
subs = p.add_subparsers()
base_subparser = argparse.ArgumentParser(add_help=False)
base_subparser.add_argument('--profile',
default=["user"],
nargs='+',
type=str,
help='Which profile to load')
base_subparser.add_argument(
'--target',
default=None,
type=str,
help='Which run-target to load for the given profile')
sub = subs.add_parser('init', parents=[base_subparser])
sub.add_argument('project_name', type=str, help='Name of the new project')
sub.set_defaults(cls=init_task.InitTask, which='init')
sub = subs.add_parser('clean', parents=[base_subparser])
sub.set_defaults(cls=clean_task.CleanTask, which='clean')
sub = subs.add_parser('compile', parents=[base_subparser])
sub.add_argument('--dry',
action='store_true',
help="Compile 'dry run' models")
sub.set_defaults(cls=compile_task.CompileTask, which='compile')
sub = subs.add_parser('debug', parents=[base_subparser])
sub.set_defaults(cls=debug_task.DebugTask, which='debug')
sub = subs.add_parser('deps', parents=[base_subparser])
sub.set_defaults(cls=deps_task.DepsTask, which='deps')
sub = subs.add_parser('run', parents=[base_subparser])
sub.add_argument('--dry', action='store_true', help="'dry run' models")
sub.add_argument(
'--models',
required=False,
nargs='+',
help=
"Specify the models to run. All models depending on these models will also be run"
)
sub.set_defaults(cls=run_task.RunTask, which='run')
sub = subs.add_parser('seed', parents=[base_subparser])
sub.add_argument('--drop-existing',
action='store_true',
help="Drop existing seed tables and recreate them")
sub.set_defaults(cls=seed_task.SeedTask, which='seed')
sub = subs.add_parser('test', parents=[base_subparser])
sub.add_argument('--skip-test-creates',
action='store_true',
help="Don't create temporary views to validate model SQL")
sub.add_argument('--validate',
action='store_true',
help='Run constraint validations from schema.yml files')
sub.set_defaults(cls=test_task.TestTask, which='test')
if len(args) == 0: return p.print_help()
parsed = p.parse_args(args)
task = None
proj = None
if parsed.which == 'init':
# bypass looking for a project file if we're running `dbt init`
task = parsed.cls(args=parsed)
elif os.path.isfile('dbt_project.yml'):
try:
proj = project.read_project('dbt_project.yml',
validate=False).with_profiles(
parsed.profile)
proj.validate()
except project.DbtProjectError as e:
print("Encountered an error while reading the project:")
print(" ERROR {}".format(str(e)))
print("Did you set the correct --profile? Using: {}".format(
parsed.profile))
all_profiles = project.read_profiles().keys()
profiles_string = "\n".join([" - " + key for key in all_profiles])
print("Valid profiles:\n{}".format(profiles_string))
dbt.tracking.track_invalid_invocation(
project=proj,
args=parsed,
result_type="invalid_profile",
result=str(e))
return None
if parsed.target is not None:
targets = proj.cfg.get('outputs', {}).keys()
if parsed.target in targets:
proj.cfg['run-target'] = parsed.target
else:
print("Encountered an error while reading the project:")
print(
" ERROR Specified target {} is not a valid option for profile {}"
.format(parsed.target, parsed.profile))
print("Valid targets are: {}".format(targets))
dbt.tracking.track_invalid_invocation(
project=proj,
args=parsed,
result_type="invalid_target",
result="target not found")
return None
log_dir = proj.get('log-path', 'logs')
logger = getLogger(log_dir, __name__)
logger.info("running dbt with arguments %s", parsed)
task = parsed.cls(args=parsed, project=proj)
else:
raise RuntimeError(
"dbt must be run from a project root directory with a dbt_project.yml file"
)
dbt.tracking.track_invocation_start(project=proj, args=parsed)
try:
task.run()
dbt.tracking.track_invocation_end(project=proj,
args=parsed,
result_type="ok",
result=None)
except Exception as e:
dbt.tracking.track_invocation_end(project=proj,
args=parsed,
result_type="error",
result=str(e))
raise
def handle(args):
p = argparse.ArgumentParser(prog='dbt: data build tool')
subs = p.add_subparsers()
base_subparser = argparse.ArgumentParser(add_help=False)
base_subparser.add_argument('--profile', default=["user"], nargs='+', type=str, help='Which profile to load')
base_subparser.add_argument('--target', default=None, type=str, help='Which run-target to load for the given profile')
sub = subs.add_parser('init', parents=[base_subparser])
sub.add_argument('project_name', type=str, help='Name of the new project')
sub.set_defaults(cls=init_task.InitTask, which='init')
sub = subs.add_parser('clean', parents=[base_subparser])
sub.set_defaults(cls=clean_task.CleanTask, which='clean')
sub = subs.add_parser('compile', parents=[base_subparser])
sub.set_defaults(cls=compile_task.CompileTask, which='compile')
sub = subs.add_parser('debug', parents=[base_subparser])
sub.set_defaults(cls=debug_task.DebugTask, which='debug')
sub = subs.add_parser('deps', parents=[base_subparser])
sub.set_defaults(cls=deps_task.DepsTask, which='deps')
sub = subs.add_parser('run', parents=[base_subparser])
sub.add_argument('--models', required=False, nargs='+', help="Specify the models to run. All models depending on these models will also be run")
sub.set_defaults(cls=run_task.RunTask, which='run')
sub = subs.add_parser('seed', parents=[base_subparser])
sub.add_argument('--drop-existing', action='store_true', help="Drop existing seed tables and recreate them")
sub.set_defaults(cls=seed_task.SeedTask, which='seed')
sub = subs.add_parser('test', parents=[base_subparser])
sub.add_argument('--skip-test-creates', action='store_true', help="Don't create temporary views to validate model SQL")
sub.add_argument('--validate', action='store_true', help='Run constraint validations from schema.yml files')
sub.set_defaults(cls=test_task.TestTask, which='test')
if len(args) == 0: return p.print_help()
parsed = p.parse_args(args)
if parsed.which == 'init':
# bypass looking for a project file if we're running `dbt init`
parsed.cls(args=parsed).run()
elif os.path.isfile('dbt_project.yml'):
try:
proj = project.read_project('dbt_project.yml', validate=False).with_profiles(parsed.profile)
proj.validate()
except project.DbtProjectError as e:
print("Encountered an error while reading the project:")
print(" ERROR {}".format(str(e)))
print("Did you set the correct --profile? Using: {}".format(parsed.profile))
all_profiles = project.read_profiles().keys()
profiles_string = "\n".join([" - " + key for key in all_profiles])
print("Valid profiles:\n{}".format(profiles_string))
return
if parsed.target is not None:
targets = proj.cfg.get('outputs', {}).keys()
if parsed.target in targets:
proj.cfg['run-target'] = parsed.target
else:
print("Encountered an error while reading the project:")
print(" ERROR Specified target {} is not a valid option for profile {}".format(parsed.target, parsed.profile))
print("Valid targets are: {}".format(targets))
return
parsed.cls(args=parsed, project=proj).run()
else:
raise RuntimeError("dbt must be run from a project root directory with a dbt_project.yml file")
def invoke_dbt(parsed):
task = None
proj = None
try:
proj = project.read_project(
'dbt_project.yml',
parsed.profiles_dir,
validate=False,
profile_to_load=parsed.profile,
args=parsed
)
proj.validate()
except project.DbtProjectError as e:
logger.info("Encountered an error while reading the project:")
logger.info(dbt.compat.to_string(e))
all_profiles = project.read_profiles(parsed.profiles_dir).keys()
if len(all_profiles) > 0:
logger.info("Defined profiles:")
for profile in all_profiles:
logger.info(" - {}".format(profile))
else:
logger.info("There are no profiles defined in your "
"profiles.yml file")
logger.info(PROFILES_HELP_MESSAGE)
dbt.tracking.track_invalid_invocation(
project=proj,
args=parsed,
result_type="invalid_profile")
return None
except project.DbtProfileError as e:
logger.info("Encountered an error while reading profiles:")
logger.info(" ERROR {}".format(str(e)))
dbt.tracking.track_invalid_invocation(
project=proj,
args=parsed,
result_type="invalid_profile")
return None
if parsed.target is not None:
targets = proj.cfg.get('outputs', {}).keys()
if parsed.target in targets:
proj.cfg['target'] = parsed.target
# make sure we update the target if this is overriden on the cli
proj.compile_and_update_target()
else:
logger.info("Encountered an error while reading the project:")
logger.info(" ERROR Specified target {} is not a valid option "
"for profile {}"
.format(parsed.target, proj.profile_to_load))
logger.info("Valid targets are: {}".format(
', '.join(targets)))
dbt.tracking.track_invalid_invocation(
project=proj,
args=parsed,
result_type="invalid_target")
return None
proj.log_warnings()
flags.NON_DESTRUCTIVE = getattr(proj.args, 'non_destructive', False)
arg_drop_existing = getattr(proj.args, 'drop_existing', False)
arg_full_refresh = getattr(proj.args, 'full_refresh', False)
if arg_drop_existing:
dbt.deprecations.warn('drop-existing')
flags.FULL_REFRESH = True
elif arg_full_refresh:
flags.FULL_REFRESH = True
logger.debug("running dbt with arguments %s", parsed)
task = parsed.cls(args=parsed, project=proj)
return task, proj
def setUp(self):
DBTIntegrationTest.setUp(self)
self.use_default_project()
self.project = read_project('dbt_project.yml')
self.expected_ephemeral = []
