def _check_python_compatibility(self, python: str) -> None:
python_version_result = subprocess.run([python, '--version'],
stdout=subprocess.PIPE)
if python_version_result.returncode != 0:
raise DbtenvError(f"Failed to run `{python}`.")
python_version_output = python_version_result.stdout.decode(
'utf-8').strip()
python_version_match = re.search(
r'(?P<major_version>\d+)\.(?P<minor_version>\d+)\.\d+',
python_version_output)
if not python_version_match:
raise DbtenvError(
f"No Python version number found in \"{python_version_output}\"."
)
python_version = python_version_match[0]
python_major_version = int(python_version_match['major_version'])
python_minor_version = int(python_version_match['minor_version'])
if self.version < Version('0.20') and (python_major_version,
python_minor_version) >= (3, 9):
raise DbtenvError(
f"Python {python_version} is being used, but dbt versions before 0.20.0 aren't compatible with Python 3.9 or above."
)
elif self.version < Version('0.15') and (
python_major_version, python_minor_version) >= (3, 8):
raise DbtenvError(
f"Python {python_version} is being used, but dbt versions before 0.15 aren't compatible with Python 3.8 or above."
)
logger.debug(f"Python {python_version} should be compatible with dbt.")
def install(self, force: bool = False) -> None:
if self.version >= Version('1.0.0'):
raise DbtenvError(
"dbtenv does not support installing dbt versions greater than or equal to 1.0.0 through Homebrew. Configure dbtenv to use pip instead."
)
if self.is_installed():
if force:
logger.info(
f"dbt {self.version.homebrew_version} is already installed with Homebrew but will be reinstalled."
)
self.uninstall(force=True)
else:
raise DbtenvError(
f"dbt {self.version.homebrew_version} is already installed with Homebrew."
)
else:
ensure_homebrew_dbt_tap()
logger.info(
f"Installing dbt {self.version.homebrew_version} with Homebrew.")
brew_args = ['install', get_dbt_version_formula(self.version)]
logger.debug(f"Running `brew` with arguments {brew_args}.")
subprocess.run(['brew', *brew_args])
# We can't rely on the Homebrew process return code to check for success/failure because a non-zero return code
# might just indicate some extraneous problem, like a failure symlinking dbt into the bin directory.
if self.is_installed():
logger.info(
f"Successfully installed dbt {self.version.homebrew_version} with Homebrew."
)
else:
raise DbtenvError(
f"Failed to install dbt {self.version.homebrew_version} with Homebrew."
)
def install_dbt(env: Environment, version: Version) -> None:
if env.primary_installer == Installer.PIP:
dbtenv.pip.PipDbt(env, version).install()
elif env.primary_installer == Installer.HOMEBREW:
dbtenv.homebrew.HomebrewDbt(env, version).install()
else:
raise DbtenvError(f"Unknown installer `{env.primary_installer}`.")
def get_installable_versions(env: Environment) -> List[Version]:
if env.primary_installer == Installer.PIP:
return dbtenv.pip.get_pypi_dbt_versions()
elif env.primary_installer == Installer.HOMEBREW:
return dbtenv.homebrew.get_homebrew_dbt_versions()
else:
raise DbtenvError(f"Unknown installer `{env.primary_installer}`.")
def get_dbt(env: Environment, version: Version) -> Dbt:
error = DbtenvError(f"No dbt {version} executable found.")
pip_dbt = None
if env.use_pip:
pip_dbt = dbtenv.pip.PipDbt(env, version)
try:
pip_dbt.get_executable(
) # Raises an appropriate error if it's not installed.
if env.primary_installer == Installer.PIP:
return pip_dbt
except DbtenvError as pip_error:
if env.installer == Installer.PIP:
raise
else:
error = pip_error
if env.use_homebrew:
homebrew_dbt = dbtenv.homebrew.HomebrewDbt(env, version)
try:
homebrew_dbt.get_executable(
) # Raises an appropriate error if it's not installed.
return homebrew_dbt
except DbtenvError as homebrew_error:
if env.installer == Installer.HOMEBREW:
raise
elif env.primary_installer == Installer.HOMEBREW:
error = homebrew_error
if pip_dbt and pip_dbt.is_installed():
return pip_dbt
raise error
def get_executable(self) -> str:
if self._executable is None:
if not os.path.isdir(self.keg_directory):
raise DbtenvError(
f"No dbt {self.version.homebrew_version} installation found in `{self.keg_directory}`."
)
dbt_path = os.path.join(self.keg_directory, 'bin/dbt')
if os.path.isfile(dbt_path):
logger.debug(f"Found dbt executable `{dbt_path}`.")
self._executable = dbt_path
else:
raise DbtenvError(
f"No dbt executable found in `{self.keg_directory}`.")
return self._executable
def ensure_dbt_is_installed(env: Environment, version: Version) -> None:
if not dbtenv.which.try_get_dbt(env, version):
if env.auto_install:
install_dbt(env, version)
else:
raise DbtenvError(
f"No dbt {version} installation found and auto-install is not enabled."
)
def get_executable(self) -> str:
if self._executable is None:
if not os.path.isdir(self.venv_directory):
raise DbtenvError(
f"No dbt {self.version.pypi_version} installation found in `{self.venv_directory}`."
)
dbt_subpath = 'Scripts\\dbt.exe' if self.env.os == 'Windows' else 'bin/dbt'
dbt_path = os.path.join(self.venv_directory, dbt_subpath)
if os.path.isfile(dbt_path):
logger.debug(f"Found dbt executable `{dbt_path}`.")
self._executable = dbt_path
else:
raise DbtenvError(
f"No dbt executable found in `{self.venv_directory}`.")
return self._executable
def _find_pip(self) -> str:
pip_subpath = 'Scripts\\pip.exe' if self.env.os == 'Windows' else 'bin/pip'
pip_path = os.path.join(self.venv_directory, pip_subpath)
if os.path.isfile(pip_path):
logger.debug(f"Found pip executable `{pip_path}`.")
return pip_path
else:
raise DbtenvError(
f"No pip executable found in `{self.venv_directory}`.")
def get_homebrew_dbt_versions() -> List[Version]:
ensure_homebrew_dbt_tap()
brew_args = ['info', '--json', 'dbt']
logger.debug(f"Running `brew` with arguments {brew_args}.")
brew_result = subprocess.run(['brew', *brew_args], stdout=subprocess.PIPE)
if brew_result.returncode != 0:
raise DbtenvError("Failed to get dbt info from Homebrew.")
formula_metadata = json.loads(brew_result.stdout)
return [
get_dbt_formula_version(formula)
for formula in formula_metadata[0]['versioned_formulae']
]
def ensure_homebrew_dbt_tap() -> None:
# While it would be simplest to just always run `brew tap dbt-labs/dbt`, we need to first check for
# an existing dbt tap because the "dbt-labs" GitHub organization used to be named "fishtown-analytics"
# so people could have a "fishtown-analytics/dbt" tap, and having multiple dbt taps causes errors.
tap_list_result = subprocess.run(['brew', 'tap'],
stdout=subprocess.PIPE,
text=True)
if not re.search(r'\b(fishtown-analytics|dbt-labs)/dbt\b',
tap_list_result.stdout):
logger.info('Adding the dbt Homebrew tap.')
tap_dbt_result = subprocess.run(['brew', 'tap', 'dbt-labs/dbt'])
if tap_dbt_result.returncode != 0:
raise DbtenvError("Failed to add the dbt Homebrew tap.")
def uninstall(self, force: bool = False) -> None:
if not self.is_installed():
raise DbtenvError(
f"No dbt {self.version.homebrew_version} installation found in `{self.keg_directory}`."
)
if force or dbtenv.string_is_true(
input(
f"Uninstall dbt {self.version.homebrew_version} from Homebrew? "
)):
brew_args = ['uninstall']
if force:
brew_args.append('--force')
brew_args.append(get_dbt_version_formula(self.version))
logger.debug(f"Running `brew` with arguments {brew_args}.")
brew_result = subprocess.run(['brew', *brew_args])
if brew_result.returncode != 0:
raise DbtenvError(
f"Failed to uninstall dbt {self.version.homebrew_version} from Homebrew."
)
self._executable = None
logger.info(
f"Successfully uninstalled dbt {self.version.homebrew_version} from Homebrew."
)
def uninstall(self, force: bool = False) -> None:
if not self.is_installed():
raise DbtenvError(
f"No dbt {self.version.pypi_version} installation found in `{self.venv_directory}`."
)
if force or dbtenv.string_is_true(
input(
f"Uninstall dbt {self.version.pypi_version} from `{self.venv_directory}`? "
)):
shutil.rmtree(self.venv_directory)
self._executable = None
logger.info(
f"Successfully uninstalled dbt {self.version.pypi_version} from `{self.venv_directory}`."
)
def execute(self, args: Args) -> None:
attempted_uninstalls = 0
if self.env.use_pip:
pip_dbt = dbtenv.pip.PipDbt(self.env, args.dbt_version)
if pip_dbt.is_installed():
pip_dbt.uninstall(force=args.force)
attempted_uninstalls += 1
if self.env.use_homebrew:
homebrew_dbt = dbtenv.homebrew.HomebrewDbt(self.env, args.dbt_version)
if homebrew_dbt.is_installed():
homebrew_dbt.uninstall(force=args.force)
attempted_uninstalls += 1
if attempted_uninstalls == 0:
raise DbtenvError(f"No dbt {args.dbt_version} installation found.")
def execute(self, args: Args) -> None:
if args.dbt_version:
version = args.dbt_version
else:
version = dbtenv.version.get_version(self.env)
logger.info(f"Using dbt {version} ({version.source_description}).")
if self.env.primary_installer == Installer.PIP:
pip_dbt = dbtenv.pip.PipDbt(self.env, version)
pip_dbt.install(force=args.force,
package_location=args.package_location,
editable=args.editable)
elif self.env.primary_installer == Installer.HOMEBREW:
homebrew_dbt = dbtenv.homebrew.HomebrewDbt(self.env, version)
homebrew_dbt.install(force=args.force)
else:
raise DbtenvError(
f"Unknown installer `{self.env.primary_installer}`.")
def install(self,
force: bool = False,
package_location: Optional[str] = None,
editable: bool = False) -> None:
if self.is_installed():
if force:
logger.info(
f"`{self.venv_directory}` already exists but will be overwritten."
)
self._executable = None
else:
raise DbtenvError(f"`{self.venv_directory}` already exists.")
python = self.env.python
self._check_python_compatibility(python)
logger.info(
f"Creating virtual environment in `{self.venv_directory}` using `{python}`."
)
venv_result = subprocess.run(
[python, '-m'
'venv', '--clear', self.venv_directory])
if venv_result.returncode != 0:
raise DbtenvError(
f"Failed to create virtual environment in `{self.venv_directory}`."
)
pip = self._find_pip()
# Upgrade pip to avoid problems with packages that might require newer pip features.
subprocess.run([pip, 'install', '--upgrade', 'pip'])
# Install wheel to avoid pip falling back to using legacy `setup.py` installs.
subprocess.run(
[pip, 'install', '--disable-pip-version-check', 'wheel'])
pip_args = ['install', '--disable-pip-version-check']
if package_location:
package_source = f"`{package_location}`"
if editable:
pip_args.append('--editable')
pip_args.append(package_location)
else:
package_source = "the Python Package Index"
if self.env.simulate_release_date:
package_metadata = get_pypi_package_metadata('dbt')
release_date = date.fromisoformat(package_metadata['releases'][
self.version.pypi_version][0]['upload_time'][:10])
logger.info(
f"Simulating release date {release_date} for dbt {self.version}."
)
class ReleaseDateFilterPyPIRequestHandler(
BaseDateFilterPyPIRequestHandler):
date = release_date
pip_filter_server = http.server.HTTPServer(
('', 0), ReleaseDateFilterPyPIRequestHandler)
pip_filter_port = pip_filter_server.socket.getsockname()[1]
threading.Thread(target=pip_filter_server.serve_forever,
daemon=True).start()
pip_args.extend([
'--index-url', f'http://localhost:{pip_filter_port}/simple'
])
elif self.version < Version('0.19.1'):
# Versions prior to 0.19.1 just specified agate>=1.6, but agate 1.6.2 introduced a dependency on PyICU
# which causes installation problems, so exclude that like versions 0.19.1 and above do.
pip_args.append('agate>=1.6,<1.6.2')
# Use correct dbt package name depending on version
if self.version < Version('1.0.0'):
pip_args.append(f'dbt=={self.version.pypi_version}')
else:
# See comment on adapter - dbt-core version coordination
# https://getdbt.slack.com/archives/C02HM9AAXL4/p1637345945047100?thread_ts=1637323222.046100&cid=C02HM9AAXL4
pip_args.append(f'dbt-core=={self.version.pypi_version}')
pip_args.append(
f'dbt-postgres~={self.version.major_minor_patch}')
pip_args.append(
f'dbt-redshift~={self.version.major_minor_patch}')
pip_args.append(
f'dbt-snowflake~={self.version.major_minor_patch}')
pip_args.append(
f'dbt-bigquery~={self.version.major_minor_patch}')
logger.info(
f"Installing dbt {self.version.pypi_version} from {package_source} into `{self.venv_directory}`."
)
logger.debug(f"Running `{pip}` with arguments {pip_args}.")
pip_result = subprocess.run([pip, *pip_args])
if pip_result.returncode != 0:
raise DbtenvError(
f"Failed to install dbt {self.version.pypi_version} from {package_source} into `{self.venv_directory}`."
)
logger.info(
f"Successfully installed dbt {self.version.pypi_version} from {package_source} into `{self.venv_directory}`."
)
def main(args: List[str] = None) -> None:
try:
if args is None:
args = sys.argv[1:]
logger.debug(f"Arguments = {args}")
env = Environment()
versions_subcommand = dbtenv.versions.VersionsSubcommand(env)
install_subcommand = dbtenv.install.InstallSubcommand(env)
version_subcommand = dbtenv.version.VersionSubcommand(env)
which_subcommand = dbtenv.which.WhichSubcommand(env)
execute_subcommand = dbtenv.execute.ExecuteSubcommand(env)
uninstall_subcommand = dbtenv.uninstall.UninstallSubcommand(env)
args_parser = build_root_args_parser(env)
subparsers = args_parser.add_subparsers(dest='subcommand',
title="Sub-commands")
common_subcommand_args_parser = build_common_args_parser(
env, dest_prefix='subcommand_')
common_install_args_parser = build_common_install_args_parser(env)
versions_subcommand.add_args_parser(subparsers,
[common_subcommand_args_parser])
install_subcommand.add_args_parser(
subparsers,
[common_subcommand_args_parser, common_install_args_parser])
version_subcommand.add_args_parser(
subparsers,
[common_subcommand_args_parser, common_install_args_parser])
which_subcommand.add_args_parser(subparsers,
[common_subcommand_args_parser])
execute_subcommand.add_args_parser(
subparsers,
[common_subcommand_args_parser, common_install_args_parser])
uninstall_subcommand.add_args_parser(subparsers,
[common_subcommand_args_parser])
parsed_args = Args()
args_parser.parse_args(args, namespace=parsed_args)
debug = parsed_args.debug or parsed_args.get('subcommand_debug')
if debug:
env.debug = debug
logger.debug(f"Parsed arguments = {parsed_args}")
quiet = parsed_args.quiet or parsed_args.get('subcommand_quiet')
if quiet:
env.quiet = quiet
installer = parsed_args.get('installer') or parsed_args.get(
'subcommand_installer')
if installer:
env.installer = installer
python = parsed_args.get('python')
if python:
env.python = python
simulate_release_date = parsed_args.get('simulate_release_date')
if simulate_release_date:
env.simulate_release_date = simulate_release_date
subcommand = parsed_args.subcommand
if not subcommand:
args_parser.print_help()
sys.exit(ExitCode.FAILURE)
if subcommand == versions_subcommand.name:
versions_subcommand.execute(parsed_args)
elif subcommand == install_subcommand.name:
install_subcommand.execute(parsed_args)
elif subcommand == version_subcommand.name:
version_subcommand.execute(parsed_args)
elif subcommand == which_subcommand.name:
which_subcommand.execute(parsed_args)
elif subcommand == execute_subcommand.name or subcommand in execute_subcommand.aliases:
execute_subcommand.execute(parsed_args)
elif subcommand == uninstall_subcommand.name:
uninstall_subcommand.execute(parsed_args)
else:
raise DbtenvError(f"Unknown sub-command `{subcommand}`.")
except DbtenvError as error:
logger.error(error)
sys.exit(ExitCode.FAILURE)
except DbtError as dbt_error:
logger.debug(dbt_error)
sys.exit(dbt_error.exit_code)
except KeyboardInterrupt:
logger.debug("Received keyboard interrupt.")
sys.exit(ExitCode.INTERRUPTED)
sys.exit(ExitCode.SUCCESS)