def run_schema_validations(self): project = read_project('dbt_project.yml') args = FakeArgs() test_task = TestTask(args, project) print(project) return test_task.run()
def __pull_deps_recursive(self, repos, processed_repos=None, i=0): if processed_repos is None: processed_repos = set() for repo_string in repos: repo, branch = self.__split_at_branch(repo_string) repo_folder = folder_from_git_remote(repo) try: if repo_folder in processed_repos: logger.info( "skipping already processed dependency {}".format( repo_folder)) else: dep_folder = self.__pull_repo(repo, branch) dep_project = project.read_project( os.path.join(self.project['modules-path'], dep_folder, 'dbt_project.yml'), self.project.profiles_dir, profile_to_load=self.project.profile_to_load) processed_repos.add(dep_folder) self.__pull_deps_recursive(dep_project['repositories'], processed_repos, i + 1) except IOError as e: if e.errno == errno.ENOENT: logger.info("'{}' is not a valid dbt project - " "dbt_project.yml not found".format(repo)) exit(1) else: raise e
def run_data_validations(self): project = read_project('dbt_project.yml') args = FakeArgs() args.data = True test_task = TestTask(args, project) return test_task.run()
def invoke_dbt(parsed): task = None proj = None try: proj = project.read_project('dbt_project.yml', parsed.profiles_dir, validate=False, profile_to_load=parsed.profile) proj.validate() except project.DbtProjectError as e: logger.info("Encountered an error while reading the project:") logger.info(" ERROR {}".format(str(e))) logger.info("Did you set the correct --profile? Using: {}".format( parsed.profile)) logger.info("Valid profiles:") all_profiles = project.read_profiles(parsed.profiles_dir).keys() for profile in all_profiles: logger.info(" - {}".format(profile)) dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_profile", result=str(e)) return None if parsed.target is not None: targets = proj.cfg.get('outputs', {}).keys() if parsed.target in targets: proj.cfg['target'] = parsed.target else: logger.info("Encountered an error while reading the project:") logger.info(" ERROR Specified target {} is not a valid option " "for profile {}".format(parsed.target, proj.profile_to_load)) logger.info("Valid targets are: {}".format(targets)) dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_target", result="target not found") return None log_dir = proj.get('log-path', 'logs') logger.debug("running dbt with arguments %s", parsed) task = parsed.cls(args=parsed, project=proj) return task, proj
def __pull_deps_recursive(self, repos, processed_repos = set()): for repo in repos: repo_folder = folder_from_git_remote(repo) try: if repo_folder in processed_repos: print("skipping already processed dependency {}".format(repo_folder)) else: dep_folder = self.__pull_repo(repo) dep_project = project.read_project( os.path.join(self.project['modules-path'], dep_folder, 'dbt_project.yml') ) processed_repos.add(dep_folder) self.__pull_deps_recursive(dep_project['repositories'], processed_repos) except IOError as e: if e.errno == errno.ENOENT: print("'{}' is not a valid dbt project - dbt_project.yml not found".format(repo)) exit(1) else: raise e
def run_schema_and_assert(self, include, exclude, expected_tests): self.use_profile('postgres') self.use_default_project() self.project = read_project('dbt_project.yml') self.run_sql_file( "test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(["deps"]) results = self.run_dbt() self.assertEqual(len(results), 5) args = FakeArgs() args.models = include args.exclude = exclude test_task = TestTask(args, self.project) test_results = test_task.run() ran_tests = sorted([test.node.get('name') for test in test_results]) expected_sorted = sorted(expected_tests) self.assertEqual(ran_tests, expected_sorted)
def main(args): if os.path.isfile('dbt_project.yml'): proj = project.read_project('dbt_project.yml') else: proj = project.default_project() p = argparse.ArgumentParser(prog='dbt: data build tool') subs = p.add_subparsers() sub = subs.add_parser('clean') sub.set_defaults(cls=clean_task.CleanTask) sub = subs.add_parser('run') sub.set_defaults(cls=run_task.RunTask) sub = subs.add_parser('compile') sub.set_defaults(cls=compile_task.CompileTask) sub = subs.add_parser('debug') sub.set_defaults(cls=debug_task.DebugTask) parsed = p.parse_args(args) parsed.cls(args=parsed, project=proj).run()
def setUp(self): DBTIntegrationTest.setUp(self) self.use_default_project() self.project = read_project('dbt_project.yml')
def invoke_dbt(parsed): task = None proj = None adapter_cache.reset() try: proj = project.read_project('dbt_project.yml', parsed.profiles_dir, validate=False, profile_to_load=parsed.profile, args=parsed) proj.validate() except project.DbtProjectError as e: logger.info("Encountered an error while reading the project:") logger.info(dbt.compat.to_string(e)) all_profiles = project.read_profiles(parsed.profiles_dir).keys() if len(all_profiles) > 0: logger.info("Defined profiles:") for profile in all_profiles: logger.info(" - {}".format(profile)) else: logger.info("There are no profiles defined in your " "profiles.yml file") logger.info(PROFILES_HELP_MESSAGE) dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_profile", result=str(e)) return None except project.DbtProfileError as e: logger.info("Encountered an error while reading profiles:") logger.info(" ERROR {}".format(str(e))) dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_profile", result=str(e)) return None if parsed.target is not None: targets = proj.cfg.get('outputs', {}).keys() if parsed.target in targets: proj.cfg['target'] = parsed.target else: logger.info("Encountered an error while reading the project:") logger.info(" ERROR Specified target {} is not a valid option " "for profile {}".format(parsed.target, proj.profile_to_load)) logger.info("Valid targets are: {}".format(', '.join(targets))) dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_target", result="target not found") return None flags.NON_DESTRUCTIVE = getattr(proj.args, 'non_destructive', False) flags.FULL_REFRESH = getattr(proj.args, 'full_refresh', False) logger.debug("running dbt with arguments %s", parsed) task = parsed.cls(args=parsed, project=proj) return task, proj
def handle(args): p = argparse.ArgumentParser(prog='dbt: data build tool', formatter_class=argparse.RawTextHelpFormatter) p.add_argument('--version', action='version', version=dbt.version.get_version_information(), help="Show version information") subs = p.add_subparsers() base_subparser = argparse.ArgumentParser(add_help=False) base_subparser.add_argument('--profile', default=["user"], nargs='+', type=str, help='Which profile to load') base_subparser.add_argument( '--target', default=None, type=str, help='Which run-target to load for the given profile') sub = subs.add_parser('init', parents=[base_subparser]) sub.add_argument('project_name', type=str, help='Name of the new project') sub.set_defaults(cls=init_task.InitTask, which='init') sub = subs.add_parser('clean', parents=[base_subparser]) sub.set_defaults(cls=clean_task.CleanTask, which='clean') sub = subs.add_parser('compile', parents=[base_subparser]) sub.add_argument('--dry', action='store_true', help="Compile 'dry run' models") sub.set_defaults(cls=compile_task.CompileTask, which='compile') sub = subs.add_parser('debug', parents=[base_subparser]) sub.set_defaults(cls=debug_task.DebugTask, which='debug') sub = subs.add_parser('deps', parents=[base_subparser]) sub.set_defaults(cls=deps_task.DepsTask, which='deps') sub = subs.add_parser('run', parents=[base_subparser]) sub.add_argument('--dry', action='store_true', help="'dry run' models") sub.add_argument( '--models', required=False, nargs='+', help= "Specify the models to run. All models depending on these models will also be run" ) sub.set_defaults(cls=run_task.RunTask, which='run') sub = subs.add_parser('seed', parents=[base_subparser]) sub.add_argument('--drop-existing', action='store_true', help="Drop existing seed tables and recreate them") sub.set_defaults(cls=seed_task.SeedTask, which='seed') sub = subs.add_parser('test', parents=[base_subparser]) sub.add_argument('--skip-test-creates', action='store_true', help="Don't create temporary views to validate model SQL") sub.add_argument('--validate', action='store_true', help='Run constraint validations from schema.yml files') sub.set_defaults(cls=test_task.TestTask, which='test') if len(args) == 0: return p.print_help() parsed = p.parse_args(args) task = None proj = None if parsed.which == 'init': # bypass looking for a project file if we're running `dbt init` task = parsed.cls(args=parsed) elif os.path.isfile('dbt_project.yml'): try: proj = project.read_project('dbt_project.yml', validate=False).with_profiles( parsed.profile) proj.validate() except project.DbtProjectError as e: print("Encountered an error while reading the project:") print(" ERROR {}".format(str(e))) print("Did you set the correct --profile? Using: {}".format( parsed.profile)) all_profiles = project.read_profiles().keys() profiles_string = "\n".join([" - " + key for key in all_profiles]) print("Valid profiles:\n{}".format(profiles_string)) dbt.tracking.track_invalid_invocation( project=proj, args=parsed, result_type="invalid_profile", result=str(e)) return None if parsed.target is not None: targets = proj.cfg.get('outputs', {}).keys() if parsed.target in targets: proj.cfg['run-target'] = parsed.target else: print("Encountered an error while reading the project:") print( " ERROR Specified target {} is not a valid option for profile {}" .format(parsed.target, parsed.profile)) print("Valid targets are: {}".format(targets)) dbt.tracking.track_invalid_invocation( project=proj, args=parsed, result_type="invalid_target", result="target not found") return None log_dir = proj.get('log-path', 'logs') logger = getLogger(log_dir, __name__) logger.info("running dbt with arguments %s", parsed) task = parsed.cls(args=parsed, project=proj) else: raise RuntimeError( "dbt must be run from a project root directory with a dbt_project.yml file" ) dbt.tracking.track_invocation_start(project=proj, args=parsed) try: task.run() dbt.tracking.track_invocation_end(project=proj, args=parsed, result_type="ok", result=None) except Exception as e: dbt.tracking.track_invocation_end(project=proj, args=parsed, result_type="error", result=str(e)) raise
def handle(args): p = argparse.ArgumentParser(prog='dbt: data build tool') subs = p.add_subparsers() base_subparser = argparse.ArgumentParser(add_help=False) base_subparser.add_argument('--profile', default=["user"], nargs='+', type=str, help='Which profile to load') base_subparser.add_argument('--target', default=None, type=str, help='Which run-target to load for the given profile') sub = subs.add_parser('init', parents=[base_subparser]) sub.add_argument('project_name', type=str, help='Name of the new project') sub.set_defaults(cls=init_task.InitTask, which='init') sub = subs.add_parser('clean', parents=[base_subparser]) sub.set_defaults(cls=clean_task.CleanTask, which='clean') sub = subs.add_parser('compile', parents=[base_subparser]) sub.set_defaults(cls=compile_task.CompileTask, which='compile') sub = subs.add_parser('debug', parents=[base_subparser]) sub.set_defaults(cls=debug_task.DebugTask, which='debug') sub = subs.add_parser('deps', parents=[base_subparser]) sub.set_defaults(cls=deps_task.DepsTask, which='deps') sub = subs.add_parser('run', parents=[base_subparser]) sub.add_argument('--models', required=False, nargs='+', help="Specify the models to run. All models depending on these models will also be run") sub.set_defaults(cls=run_task.RunTask, which='run') sub = subs.add_parser('seed', parents=[base_subparser]) sub.add_argument('--drop-existing', action='store_true', help="Drop existing seed tables and recreate them") sub.set_defaults(cls=seed_task.SeedTask, which='seed') sub = subs.add_parser('test', parents=[base_subparser]) sub.add_argument('--skip-test-creates', action='store_true', help="Don't create temporary views to validate model SQL") sub.add_argument('--validate', action='store_true', help='Run constraint validations from schema.yml files') sub.set_defaults(cls=test_task.TestTask, which='test') if len(args) == 0: return p.print_help() parsed = p.parse_args(args) if parsed.which == 'init': # bypass looking for a project file if we're running `dbt init` parsed.cls(args=parsed).run() elif os.path.isfile('dbt_project.yml'): try: proj = project.read_project('dbt_project.yml', validate=False).with_profiles(parsed.profile) proj.validate() except project.DbtProjectError as e: print("Encountered an error while reading the project:") print(" ERROR {}".format(str(e))) print("Did you set the correct --profile? Using: {}".format(parsed.profile)) all_profiles = project.read_profiles().keys() profiles_string = "\n".join([" - " + key for key in all_profiles]) print("Valid profiles:\n{}".format(profiles_string)) return if parsed.target is not None: targets = proj.cfg.get('outputs', {}).keys() if parsed.target in targets: proj.cfg['run-target'] = parsed.target else: print("Encountered an error while reading the project:") print(" ERROR Specified target {} is not a valid option for profile {}".format(parsed.target, parsed.profile)) print("Valid targets are: {}".format(targets)) return parsed.cls(args=parsed, project=proj).run() else: raise RuntimeError("dbt must be run from a project root directory with a dbt_project.yml file")
def invoke_dbt(parsed): task = None proj = None try: proj = project.read_project( 'dbt_project.yml', parsed.profiles_dir, validate=False, profile_to_load=parsed.profile, args=parsed ) proj.validate() except project.DbtProjectError as e: logger.info("Encountered an error while reading the project:") logger.info(dbt.compat.to_string(e)) all_profiles = project.read_profiles(parsed.profiles_dir).keys() if len(all_profiles) > 0: logger.info("Defined profiles:") for profile in all_profiles: logger.info(" - {}".format(profile)) else: logger.info("There are no profiles defined in your " "profiles.yml file") logger.info(PROFILES_HELP_MESSAGE) dbt.tracking.track_invalid_invocation( project=proj, args=parsed, result_type="invalid_profile") return None except project.DbtProfileError as e: logger.info("Encountered an error while reading profiles:") logger.info(" ERROR {}".format(str(e))) dbt.tracking.track_invalid_invocation( project=proj, args=parsed, result_type="invalid_profile") return None if parsed.target is not None: targets = proj.cfg.get('outputs', {}).keys() if parsed.target in targets: proj.cfg['target'] = parsed.target # make sure we update the target if this is overriden on the cli proj.compile_and_update_target() else: logger.info("Encountered an error while reading the project:") logger.info(" ERROR Specified target {} is not a valid option " "for profile {}" .format(parsed.target, proj.profile_to_load)) logger.info("Valid targets are: {}".format( ', '.join(targets))) dbt.tracking.track_invalid_invocation( project=proj, args=parsed, result_type="invalid_target") return None proj.log_warnings() flags.NON_DESTRUCTIVE = getattr(proj.args, 'non_destructive', False) arg_drop_existing = getattr(proj.args, 'drop_existing', False) arg_full_refresh = getattr(proj.args, 'full_refresh', False) if arg_drop_existing: dbt.deprecations.warn('drop-existing') flags.FULL_REFRESH = True elif arg_full_refresh: flags.FULL_REFRESH = True logger.debug("running dbt with arguments %s", parsed) task = parsed.cls(args=parsed, project=proj) return task, proj
def setUp(self): DBTIntegrationTest.setUp(self) self.use_default_project() self.project = read_project('dbt_project.yml') self.expected_ephemeral = []