def is_opted_out(): profiles = project.read_profiles() if profiles is None or profiles.get("config") is None: return False elif profiles['config'].get("send_anonymous_usage_stats") == False: return True else: return False
def invoke_dbt(parsed): task = None proj = None try: proj = project.read_project('dbt_project.yml', parsed.profiles_dir, validate=False, profile_to_load=parsed.profile) proj.validate() except project.DbtProjectError as e: logger.info("Encountered an error while reading the project:") logger.info(" ERROR {}".format(str(e))) logger.info("Did you set the correct --profile? Using: {}".format( parsed.profile)) logger.info("Valid profiles:") all_profiles = project.read_profiles(parsed.profiles_dir).keys() for profile in all_profiles: logger.info(" - {}".format(profile)) dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_profile", result=str(e)) return None if parsed.target is not None: targets = proj.cfg.get('outputs', {}).keys() if parsed.target in targets: proj.cfg['target'] = parsed.target else: logger.info("Encountered an error while reading the project:") logger.info(" ERROR Specified target {} is not a valid option " "for profile {}".format(parsed.target, proj.profile_to_load)) logger.info("Valid targets are: {}".format(targets)) dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_target", result="target not found") return None log_dir = proj.get('log-path', 'logs') logger.debug("running dbt with arguments %s", parsed) task = parsed.cls(args=parsed, project=proj) return task, proj
config = read_config('config/prod.yml') PROFILE = sys.argv[1] BASTION_SERVER = config['bastion'] cwd = os.path.dirname(os.path.realpath(__file__)) program_name = "analyze-vacuum-schema.py" PROG = os.path.join(cwd, program_name) LOG_DIR = os.path.join(cwd, "logs") if not os.path.exists(LOG_DIR): os.makedirs(LOG_DIR) all_profiles = read_profiles() p = Project({}, all_profiles, [PROFILE]) env = p.run_environment() ssh_config = paramiko.SSHConfig() user_config_file = os.path.expanduser("~/.ssh/config") with open(user_config_file) as f: ssh_config.parse(f) user_config = ssh_config.lookup(BASTION_SERVER) hostname = user_config['hostname'] username = user_config['user'] def get_schemas(conn_string): schemas_query = "select nspname from pg_namespace where nspname not like 'pg%'"
def invoke_dbt(parsed): task = None proj = None adapter_cache.reset() try: proj = project.read_project('dbt_project.yml', parsed.profiles_dir, validate=False, profile_to_load=parsed.profile, args=parsed) proj.validate() except project.DbtProjectError as e: logger.info("Encountered an error while reading the project:") logger.info(dbt.compat.to_string(e)) all_profiles = project.read_profiles(parsed.profiles_dir).keys() if len(all_profiles) > 0: logger.info("Defined profiles:") for profile in all_profiles: logger.info(" - {}".format(profile)) else: logger.info("There are no profiles defined in your " "profiles.yml file") logger.info(PROFILES_HELP_MESSAGE) dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_profile", result=str(e)) return None except project.DbtProfileError as e: logger.info("Encountered an error while reading profiles:") logger.info(" ERROR {}".format(str(e))) dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_profile", result=str(e)) return None if parsed.target is not None: targets = proj.cfg.get('outputs', {}).keys() if parsed.target in targets: proj.cfg['target'] = parsed.target else: logger.info("Encountered an error while reading the project:") logger.info(" ERROR Specified target {} is not a valid option " "for profile {}".format(parsed.target, proj.profile_to_load)) logger.info("Valid targets are: {}".format(', '.join(targets))) dbt.tracking.track_invalid_invocation(project=proj, args=parsed, result_type="invalid_target", result="target not found") return None flags.NON_DESTRUCTIVE = getattr(proj.args, 'non_destructive', False) flags.FULL_REFRESH = getattr(proj.args, 'full_refresh', False) logger.debug("running dbt with arguments %s", parsed) task = parsed.cls(args=parsed, project=proj) return task, proj
def handle(args): p = argparse.ArgumentParser(prog='dbt: data build tool', formatter_class=argparse.RawTextHelpFormatter) p.add_argument('--version', action='version', version=dbt.version.get_version_information(), help="Show version information") subs = p.add_subparsers() base_subparser = argparse.ArgumentParser(add_help=False) base_subparser.add_argument('--profile', default=["user"], nargs='+', type=str, help='Which profile to load') base_subparser.add_argument( '--target', default=None, type=str, help='Which run-target to load for the given profile') sub = subs.add_parser('init', parents=[base_subparser]) sub.add_argument('project_name', type=str, help='Name of the new project') sub.set_defaults(cls=init_task.InitTask, which='init') sub = subs.add_parser('clean', parents=[base_subparser]) sub.set_defaults(cls=clean_task.CleanTask, which='clean') sub = subs.add_parser('compile', parents=[base_subparser]) sub.add_argument('--dry', action='store_true', help="Compile 'dry run' models") sub.set_defaults(cls=compile_task.CompileTask, which='compile') sub = subs.add_parser('debug', parents=[base_subparser]) sub.set_defaults(cls=debug_task.DebugTask, which='debug') sub = subs.add_parser('deps', parents=[base_subparser]) sub.set_defaults(cls=deps_task.DepsTask, which='deps') sub = subs.add_parser('run', parents=[base_subparser]) sub.add_argument('--dry', action='store_true', help="'dry run' models") sub.add_argument( '--models', required=False, nargs='+', help= "Specify the models to run. All models depending on these models will also be run" ) sub.set_defaults(cls=run_task.RunTask, which='run') sub = subs.add_parser('seed', parents=[base_subparser]) sub.add_argument('--drop-existing', action='store_true', help="Drop existing seed tables and recreate them") sub.set_defaults(cls=seed_task.SeedTask, which='seed') sub = subs.add_parser('test', parents=[base_subparser]) sub.add_argument('--skip-test-creates', action='store_true', help="Don't create temporary views to validate model SQL") sub.add_argument('--validate', action='store_true', help='Run constraint validations from schema.yml files') sub.set_defaults(cls=test_task.TestTask, which='test') if len(args) == 0: return p.print_help() parsed = p.parse_args(args) task = None proj = None if parsed.which == 'init': # bypass looking for a project file if we're running `dbt init` task = parsed.cls(args=parsed) elif os.path.isfile('dbt_project.yml'): try: proj = project.read_project('dbt_project.yml', validate=False).with_profiles( parsed.profile) proj.validate() except project.DbtProjectError as e: print("Encountered an error while reading the project:") print(" ERROR {}".format(str(e))) print("Did you set the correct --profile? Using: {}".format( parsed.profile)) all_profiles = project.read_profiles().keys() profiles_string = "\n".join([" - " + key for key in all_profiles]) print("Valid profiles:\n{}".format(profiles_string)) dbt.tracking.track_invalid_invocation( project=proj, args=parsed, result_type="invalid_profile", result=str(e)) return None if parsed.target is not None: targets = proj.cfg.get('outputs', {}).keys() if parsed.target in targets: proj.cfg['run-target'] = parsed.target else: print("Encountered an error while reading the project:") print( " ERROR Specified target {} is not a valid option for profile {}" .format(parsed.target, parsed.profile)) print("Valid targets are: {}".format(targets)) dbt.tracking.track_invalid_invocation( project=proj, args=parsed, result_type="invalid_target", result="target not found") return None log_dir = proj.get('log-path', 'logs') logger = getLogger(log_dir, __name__) logger.info("running dbt with arguments %s", parsed) task = parsed.cls(args=parsed, project=proj) else: raise RuntimeError( "dbt must be run from a project root directory with a dbt_project.yml file" ) dbt.tracking.track_invocation_start(project=proj, args=parsed) try: task.run() dbt.tracking.track_invocation_end(project=proj, args=parsed, result_type="ok", result=None) except Exception as e: dbt.tracking.track_invocation_end(project=proj, args=parsed, result_type="error", result=str(e)) raise
def handle(args): p = argparse.ArgumentParser(prog='dbt: data build tool') subs = p.add_subparsers() base_subparser = argparse.ArgumentParser(add_help=False) base_subparser.add_argument('--profile', default=["user"], nargs='+', type=str, help='Which profile to load') base_subparser.add_argument('--target', default=None, type=str, help='Which run-target to load for the given profile') sub = subs.add_parser('init', parents=[base_subparser]) sub.add_argument('project_name', type=str, help='Name of the new project') sub.set_defaults(cls=init_task.InitTask, which='init') sub = subs.add_parser('clean', parents=[base_subparser]) sub.set_defaults(cls=clean_task.CleanTask, which='clean') sub = subs.add_parser('compile', parents=[base_subparser]) sub.set_defaults(cls=compile_task.CompileTask, which='compile') sub = subs.add_parser('debug', parents=[base_subparser]) sub.set_defaults(cls=debug_task.DebugTask, which='debug') sub = subs.add_parser('deps', parents=[base_subparser]) sub.set_defaults(cls=deps_task.DepsTask, which='deps') sub = subs.add_parser('run', parents=[base_subparser]) sub.add_argument('--models', required=False, nargs='+', help="Specify the models to run. All models depending on these models will also be run") sub.set_defaults(cls=run_task.RunTask, which='run') sub = subs.add_parser('seed', parents=[base_subparser]) sub.add_argument('--drop-existing', action='store_true', help="Drop existing seed tables and recreate them") sub.set_defaults(cls=seed_task.SeedTask, which='seed') sub = subs.add_parser('test', parents=[base_subparser]) sub.add_argument('--skip-test-creates', action='store_true', help="Don't create temporary views to validate model SQL") sub.add_argument('--validate', action='store_true', help='Run constraint validations from schema.yml files') sub.set_defaults(cls=test_task.TestTask, which='test') if len(args) == 0: return p.print_help() parsed = p.parse_args(args) if parsed.which == 'init': # bypass looking for a project file if we're running `dbt init` parsed.cls(args=parsed).run() elif os.path.isfile('dbt_project.yml'): try: proj = project.read_project('dbt_project.yml', validate=False).with_profiles(parsed.profile) proj.validate() except project.DbtProjectError as e: print("Encountered an error while reading the project:") print(" ERROR {}".format(str(e))) print("Did you set the correct --profile? Using: {}".format(parsed.profile)) all_profiles = project.read_profiles().keys() profiles_string = "\n".join([" - " + key for key in all_profiles]) print("Valid profiles:\n{}".format(profiles_string)) return if parsed.target is not None: targets = proj.cfg.get('outputs', {}).keys() if parsed.target in targets: proj.cfg['run-target'] = parsed.target else: print("Encountered an error while reading the project:") print(" ERROR Specified target {} is not a valid option for profile {}".format(parsed.target, parsed.profile)) print("Valid targets are: {}".format(targets)) return parsed.cls(args=parsed, project=proj).run() else: raise RuntimeError("dbt must be run from a project root directory with a dbt_project.yml file")
def invoke_dbt(parsed): task = None proj = None try: proj = project.read_project( 'dbt_project.yml', parsed.profiles_dir, validate=False, profile_to_load=parsed.profile, args=parsed ) proj.validate() except project.DbtProjectError as e: logger.info("Encountered an error while reading the project:") logger.info(dbt.compat.to_string(e)) all_profiles = project.read_profiles(parsed.profiles_dir).keys() if len(all_profiles) > 0: logger.info("Defined profiles:") for profile in all_profiles: logger.info(" - {}".format(profile)) else: logger.info("There are no profiles defined in your " "profiles.yml file") logger.info(PROFILES_HELP_MESSAGE) dbt.tracking.track_invalid_invocation( project=proj, args=parsed, result_type="invalid_profile") return None except project.DbtProfileError as e: logger.info("Encountered an error while reading profiles:") logger.info(" ERROR {}".format(str(e))) dbt.tracking.track_invalid_invocation( project=proj, args=parsed, result_type="invalid_profile") return None if parsed.target is not None: targets = proj.cfg.get('outputs', {}).keys() if parsed.target in targets: proj.cfg['target'] = parsed.target # make sure we update the target if this is overriden on the cli proj.compile_and_update_target() else: logger.info("Encountered an error while reading the project:") logger.info(" ERROR Specified target {} is not a valid option " "for profile {}" .format(parsed.target, proj.profile_to_load)) logger.info("Valid targets are: {}".format( ', '.join(targets))) dbt.tracking.track_invalid_invocation( project=proj, args=parsed, result_type="invalid_target") return None proj.log_warnings() flags.NON_DESTRUCTIVE = getattr(proj.args, 'non_destructive', False) arg_drop_existing = getattr(proj.args, 'drop_existing', False) arg_full_refresh = getattr(proj.args, 'full_refresh', False) if arg_drop_existing: dbt.deprecations.warn('drop-existing') flags.FULL_REFRESH = True elif arg_full_refresh: flags.FULL_REFRESH = True logger.debug("running dbt with arguments %s", parsed) task = parsed.cls(args=parsed, project=proj) return task, proj