def test_get_application_default_credentials_path(get_config_dir):
config_path = "config_path"
get_config_dir.return_value = config_path
credentials_path = _cloud_sdk.get_application_default_credentials_path()
assert credentials_path == os.path.join(
config_path, _cloud_sdk._CREDENTIALS_FILENAME
)
def _get_explicit_environ_credentials(quota_project_id=None):
"""Gets credentials from the GOOGLE_APPLICATION_CREDENTIALS environment
variable."""
from google.auth import _cloud_sdk
cloud_sdk_adc_path = _cloud_sdk.get_application_default_credentials_path()
explicit_file = os.environ.get(environment_vars.CREDENTIALS)
_LOGGER.debug(
"Checking %s for explicit credentials as part of auth process...", explicit_file
)
if explicit_file is not None and explicit_file == cloud_sdk_adc_path:
# Cloud sdk flow calls gcloud to fetch project id, so if the explicit
# file path is cloud sdk credentials path, then we should fall back
# to cloud sdk flow, otherwise project id cannot be obtained.
_LOGGER.debug(
"Explicit credentials path %s is the same as Cloud SDK credentials path, fall back to Cloud SDK credentials flow...",
explicit_file,
)
return _get_gcloud_sdk_credentials(quota_project_id=quota_project_id)
if explicit_file is not None:
credentials, project_id = load_credentials_from_file(
os.environ[environment_vars.CREDENTIALS], quota_project_id=quota_project_id
)
return credentials, project_id
else:
return None, None
def generate_docker_args(job_mode: conf.JobMode,
args: Dict[str, Any]) -> Dict[str, Any]:
"""gemerate docker args from args and job mode"""
# Get extra dependencies in case you want to install your requirements via a
# setup.py file.
setup_extras = b.base_extras(job_mode, "setup.py", args.get("extras"))
# Google application credentials, from the CLI or from an env variable.
creds_path = conf.extract_cloud_key(args)
# Application default credentials location.
adc_loc = csdk.get_application_default_credentials_path()
adc_path = adc_loc if os.path.isfile(adc_loc) else None
# TODO we may want to take custom paths, here, in addition to detecting them.
reqs = "requirements.txt"
conda_env = "environment.yml"
# Arguments that make their way down to caliban.docker.build.build_image.
docker_args = {
"extra_dirs": args.get("dir"),
"requirements_path": reqs if os.path.exists(reqs) else None,
"conda_env_path": conda_env if os.path.exists(conda_env) else None,
"caliban_config": conf.caliban_config(),
"credentials_path": creds_path,
"adc_path": adc_path,
"setup_extras": setup_extras,
"no_cache": args.get("no_cache", False),
'build_path': os.getcwd(),
}
return docker_args
def provide_authorized_gcloud(self) -> Generator[None, None, None]:
"""
Provides a separate gcloud configuration with current credentials.
The gcloud tool allows you to login to Google Cloud only - ``gcloud auth login`` and
for the needs of Application Default Credentials ``gcloud auth application-default login``.
In our case, we want all commands to use only the credentials from ADCm so
we need to configure the credentials in gcloud manually.
"""
credentials_path = _cloud_sdk.get_application_default_credentials_path(
)
project_id = self.project_id
with ExitStack() as exit_stack:
exit_stack.enter_context(
self.provide_gcp_credential_file_as_context())
gcloud_config_tmp = exit_stack.enter_context(
tempfile.TemporaryDirectory())
exit_stack.enter_context(
patch_environ({CLOUD_SDK_CONFIG_DIR: gcloud_config_tmp}))
if CREDENTIALS in os.environ:
# This solves most cases when we are logged in using the service key in Airflow.
# Don't display stdout/stderr for security reason
check_output([
"gcloud",
"auth",
"activate-service-account",
f"--key-file={os.environ[CREDENTIALS]}",
])
elif os.path.exists(credentials_path):
# If we are logged in by `gcloud auth application-default` then we need to log in manually.
# This will make the `gcloud auth application-default` and `gcloud auth` credentials equals.
with open(credentials_path) as creds_file:
creds_content = json.loads(creds_file.read())
# Don't display stdout/stderr for security reason
check_output([
"gcloud", "config", "set", "auth/client_id",
creds_content["client_id"]
])
# Don't display stdout/stderr for security reason
check_output([
"gcloud", "config", "set", "auth/client_secret",
creds_content["client_secret"]
])
# Don't display stdout/stderr for security reason
check_output([
"gcloud",
"auth",
"activate-refresh-token",
creds_content["client_id"],
creds_content["refresh_token"],
])
if project_id:
# Don't display stdout/stderr for security reason
check_output(
["gcloud", "config", "set", "core/project", project_id])
yield
def ADCFilePath():
"""Gets the ADC default file path.
Returns:
str, The path to the default ADC file.
"""
# pylint:disable=protected-access
return _cloud_sdk.get_application_default_credentials_path()
def _get_gcloud_sdk_credentials(
target_audience: Optional[str],
) -> Optional[google_auth_credentials.Credentials]:
"""Gets the credentials and project ID from the Cloud SDK."""
from google.auth import _cloud_sdk
# Check if application default credentials exist.
credentials_filename = _cloud_sdk.get_application_default_credentials_path()
if not os.path.isfile(credentials_filename):
return None
current_credentials = _load_credentials_from_file(credentials_filename, target_audience)
return current_credentials
def _get_gcloud_sdk_credentials():
"""Gets the credentials and project ID from the Cloud SDK."""
from google.auth import _cloud_sdk
# Check if application default credentials exist.
credentials_filename = _cloud_sdk.get_application_default_credentials_path()
if not os.path.isfile(credentials_filename):
return None, None
credentials, project_id = load_credentials_from_file(credentials_filename)
if not project_id:
project_id = _cloud_sdk.get_project_id()
return credentials, project_id
def _get_gcloud_sdk_credentials():
"""Gets the credentials and project ID from the Cloud SDK."""
from google.auth import _cloud_sdk
# Check if application default credentials exist.
credentials_filename = (
_cloud_sdk.get_application_default_credentials_path())
if not os.path.isfile(credentials_filename):
return None, None
credentials, project_id = _load_credentials_from_file(
credentials_filename)
if not project_id:
project_id = _cloud_sdk.get_project_id()
return credentials, project_id
def _get_gcloud_sdk_credentials():
"""Gets the credentials and project ID from the Cloud SDK."""
from google.auth import _cloud_sdk
_LOGGER.debug("Checking Cloud SDK credentials as part of auth process...")
# Check if application default credentials exist.
credentials_filename = _cloud_sdk.get_application_default_credentials_path()
if not os.path.isfile(credentials_filename):
_LOGGER.debug("Cloud SDK credentials not found on disk; not using them")
return None, None
credentials, project_id = load_credentials_from_file(credentials_filename)
if not project_id:
project_id = _cloud_sdk.get_project_id()
return credentials, project_id
def _get_gcloud_sdk_credentials():
"""Gets the credentials and project ID from the Cloud SDK."""
# Check if application default credentials exist.
credentials_filename = (
_cloud_sdk.get_application_default_credentials_path())
if not os.path.isfile(credentials_filename):
return None, None
credentials, project_id = _load_credentials_from_file(credentials_filename)
if not project_id:
project_id = _cloud_sdk.get_project_id()
if not project_id:
_LOGGER.warning(
'No project ID could be determined from the Cloud SDK '
'configuration. Consider running `gcloud config set project` or '
'setting the %s environment variable', environment_vars.PROJECT)
return credentials, project_id
def _get_explicit_environ_credentials():
"""Gets credentials from the GOOGLE_APPLICATION_CREDENTIALS environment
variable."""
from google.auth import _cloud_sdk
cloud_sdk_adc_path = _cloud_sdk.get_application_default_credentials_path()
explicit_file = os.environ.get(environment_vars.CREDENTIALS)
if explicit_file is not None and explicit_file == cloud_sdk_adc_path:
# Cloud sdk flow calls gcloud to fetch project id, so if the explicit
# file path is cloud sdk credentials path, then we should fall back
# to cloud sdk flow, otherwise project id cannot be obtained.
return _get_gcloud_sdk_credentials()
if explicit_file is not None:
credentials, project_id = load_credentials_from_file(
os.environ[environment_vars.CREDENTIALS]
)
return credentials, project_id
else:
return None, None
def _get_gcloud_sdk_credentials():
"""Gets the credentials and project ID from the Cloud SDK."""
from google.auth import _cloud_sdk
# Check if application default credentials exist.
credentials_filename = (
_cloud_sdk.get_application_default_credentials_path())
if not os.path.isfile(credentials_filename):
return None, None
credentials, project_id = _load_credentials_from_file(
credentials_filename)
if not project_id:
project_id = _cloud_sdk.get_project_id()
if not project_id:
_LOGGER.warning(
'No project ID could be determined from the Cloud SDK '
'configuration. Consider running `gcloud config set project` or '
'setting the %s environment variable', environment_vars.PROJECT)
return credentials, project_id
def application_default_credentials_path() -> str:
"""gets gcloud default credentials path"""
return get_application_default_credentials_path()
def start_worker(self):
# type: () -> None
credential_options = []
try:
# This is the public facing API, skip if it is not available.
# (If this succeeds but the imports below fail, better to actually raise
# an error below rather than silently fail.)
# pylint: disable=unused-import
import google.auth
except ImportError:
pass
else:
from google.auth import environment_vars
from google.auth import _cloud_sdk
gcloud_cred_file = os.environ.get(
environment_vars.CREDENTIALS,
_cloud_sdk.get_application_default_credentials_path())
if os.path.exists(gcloud_cred_file):
docker_cred_file = '/docker_cred_file.json'
credential_options.extend([
'--mount',
f'type=bind,source={gcloud_cred_file},target={docker_cred_file}',
'--env',
f'{environment_vars.CREDENTIALS}={docker_cred_file}'
])
with SUBPROCESS_LOCK:
try:
_LOGGER.info('Attempting to pull image %s', self._container_image)
subprocess.check_call(['docker', 'pull', self._container_image])
except Exception:
_LOGGER.info(
'Unable to pull image %s, defaulting to local image if it exists' %
self._container_image)
self._container_id = subprocess.check_output([
'docker',
'run',
'-d',
'--network=host',
] + credential_options + [
self._container_image,
'--id=%s' % self.worker_id,
'--logging_endpoint=%s' % self.logging_api_service_descriptor().url,
'--control_endpoint=%s' % self.control_address,
'--artifact_endpoint=%s' % self.control_address,
'--provision_endpoint=%s' % self.control_address,
]).strip()
assert self._container_id is not None
while True:
status = subprocess.check_output([
'docker', 'inspect', '-f', '{{.State.Status}}', self._container_id
]).strip()
_LOGGER.info(
'Waiting for docker to start up. Current status is %s' %
status.decode('utf-8'))
if status == b'running':
_LOGGER.info(
'Docker container is running. container_id = %s, '
'worker_id = %s',
self._container_id,
self.worker_id)
break
elif status in (b'dead', b'exited'):
subprocess.call(['docker', 'container', 'logs', self._container_id])
raise RuntimeError(
'SDK failed to start. Final status is %s' %
status.decode('utf-8'))
time.sleep(1)
self._done = False
t = threading.Thread(target=self.watch_container)
t.daemon = True
t.start()
