def test_get_application_default_credentials_path(get_config_dir): config_path = "config_path" get_config_dir.return_value = config_path credentials_path = _cloud_sdk.get_application_default_credentials_path() assert credentials_path == os.path.join( config_path, _cloud_sdk._CREDENTIALS_FILENAME )
def _get_explicit_environ_credentials(quota_project_id=None): """Gets credentials from the GOOGLE_APPLICATION_CREDENTIALS environment variable.""" from google.auth import _cloud_sdk cloud_sdk_adc_path = _cloud_sdk.get_application_default_credentials_path() explicit_file = os.environ.get(environment_vars.CREDENTIALS) _LOGGER.debug( "Checking %s for explicit credentials as part of auth process...", explicit_file ) if explicit_file is not None and explicit_file == cloud_sdk_adc_path: # Cloud sdk flow calls gcloud to fetch project id, so if the explicit # file path is cloud sdk credentials path, then we should fall back # to cloud sdk flow, otherwise project id cannot be obtained. _LOGGER.debug( "Explicit credentials path %s is the same as Cloud SDK credentials path, fall back to Cloud SDK credentials flow...", explicit_file, ) return _get_gcloud_sdk_credentials(quota_project_id=quota_project_id) if explicit_file is not None: credentials, project_id = load_credentials_from_file( os.environ[environment_vars.CREDENTIALS], quota_project_id=quota_project_id ) return credentials, project_id else: return None, None
def generate_docker_args(job_mode: conf.JobMode, args: Dict[str, Any]) -> Dict[str, Any]: """gemerate docker args from args and job mode""" # Get extra dependencies in case you want to install your requirements via a # setup.py file. setup_extras = b.base_extras(job_mode, "setup.py", args.get("extras")) # Google application credentials, from the CLI or from an env variable. creds_path = conf.extract_cloud_key(args) # Application default credentials location. adc_loc = csdk.get_application_default_credentials_path() adc_path = adc_loc if os.path.isfile(adc_loc) else None # TODO we may want to take custom paths, here, in addition to detecting them. reqs = "requirements.txt" conda_env = "environment.yml" # Arguments that make their way down to caliban.docker.build.build_image. docker_args = { "extra_dirs": args.get("dir"), "requirements_path": reqs if os.path.exists(reqs) else None, "conda_env_path": conda_env if os.path.exists(conda_env) else None, "caliban_config": conf.caliban_config(), "credentials_path": creds_path, "adc_path": adc_path, "setup_extras": setup_extras, "no_cache": args.get("no_cache", False), 'build_path': os.getcwd(), } return docker_args
def provide_authorized_gcloud(self) -> Generator[None, None, None]: """ Provides a separate gcloud configuration with current credentials. The gcloud tool allows you to login to Google Cloud only - ``gcloud auth login`` and for the needs of Application Default Credentials ``gcloud auth application-default login``. In our case, we want all commands to use only the credentials from ADCm so we need to configure the credentials in gcloud manually. """ credentials_path = _cloud_sdk.get_application_default_credentials_path( ) project_id = self.project_id with ExitStack() as exit_stack: exit_stack.enter_context( self.provide_gcp_credential_file_as_context()) gcloud_config_tmp = exit_stack.enter_context( tempfile.TemporaryDirectory()) exit_stack.enter_context( patch_environ({CLOUD_SDK_CONFIG_DIR: gcloud_config_tmp})) if CREDENTIALS in os.environ: # This solves most cases when we are logged in using the service key in Airflow. # Don't display stdout/stderr for security reason check_output([ "gcloud", "auth", "activate-service-account", f"--key-file={os.environ[CREDENTIALS]}", ]) elif os.path.exists(credentials_path): # If we are logged in by `gcloud auth application-default` then we need to log in manually. # This will make the `gcloud auth application-default` and `gcloud auth` credentials equals. with open(credentials_path) as creds_file: creds_content = json.loads(creds_file.read()) # Don't display stdout/stderr for security reason check_output([ "gcloud", "config", "set", "auth/client_id", creds_content["client_id"] ]) # Don't display stdout/stderr for security reason check_output([ "gcloud", "config", "set", "auth/client_secret", creds_content["client_secret"] ]) # Don't display stdout/stderr for security reason check_output([ "gcloud", "auth", "activate-refresh-token", creds_content["client_id"], creds_content["refresh_token"], ]) if project_id: # Don't display stdout/stderr for security reason check_output( ["gcloud", "config", "set", "core/project", project_id]) yield
def ADCFilePath(): """Gets the ADC default file path. Returns: str, The path to the default ADC file. """ # pylint:disable=protected-access return _cloud_sdk.get_application_default_credentials_path()
def _get_gcloud_sdk_credentials( target_audience: Optional[str], ) -> Optional[google_auth_credentials.Credentials]: """Gets the credentials and project ID from the Cloud SDK.""" from google.auth import _cloud_sdk # Check if application default credentials exist. credentials_filename = _cloud_sdk.get_application_default_credentials_path() if not os.path.isfile(credentials_filename): return None current_credentials = _load_credentials_from_file(credentials_filename, target_audience) return current_credentials
def _get_gcloud_sdk_credentials(): """Gets the credentials and project ID from the Cloud SDK.""" from google.auth import _cloud_sdk # Check if application default credentials exist. credentials_filename = _cloud_sdk.get_application_default_credentials_path() if not os.path.isfile(credentials_filename): return None, None credentials, project_id = load_credentials_from_file(credentials_filename) if not project_id: project_id = _cloud_sdk.get_project_id() return credentials, project_id
def _get_gcloud_sdk_credentials(): """Gets the credentials and project ID from the Cloud SDK.""" from google.auth import _cloud_sdk # Check if application default credentials exist. credentials_filename = ( _cloud_sdk.get_application_default_credentials_path()) if not os.path.isfile(credentials_filename): return None, None credentials, project_id = _load_credentials_from_file( credentials_filename) if not project_id: project_id = _cloud_sdk.get_project_id() return credentials, project_id
def _get_gcloud_sdk_credentials(): """Gets the credentials and project ID from the Cloud SDK.""" from google.auth import _cloud_sdk _LOGGER.debug("Checking Cloud SDK credentials as part of auth process...") # Check if application default credentials exist. credentials_filename = _cloud_sdk.get_application_default_credentials_path() if not os.path.isfile(credentials_filename): _LOGGER.debug("Cloud SDK credentials not found on disk; not using them") return None, None credentials, project_id = load_credentials_from_file(credentials_filename) if not project_id: project_id = _cloud_sdk.get_project_id() return credentials, project_id
def _get_gcloud_sdk_credentials(): """Gets the credentials and project ID from the Cloud SDK.""" # Check if application default credentials exist. credentials_filename = ( _cloud_sdk.get_application_default_credentials_path()) if not os.path.isfile(credentials_filename): return None, None credentials, project_id = _load_credentials_from_file(credentials_filename) if not project_id: project_id = _cloud_sdk.get_project_id() if not project_id: _LOGGER.warning( 'No project ID could be determined from the Cloud SDK ' 'configuration. Consider running `gcloud config set project` or ' 'setting the %s environment variable', environment_vars.PROJECT) return credentials, project_id
def _get_explicit_environ_credentials(): """Gets credentials from the GOOGLE_APPLICATION_CREDENTIALS environment variable.""" from google.auth import _cloud_sdk cloud_sdk_adc_path = _cloud_sdk.get_application_default_credentials_path() explicit_file = os.environ.get(environment_vars.CREDENTIALS) if explicit_file is not None and explicit_file == cloud_sdk_adc_path: # Cloud sdk flow calls gcloud to fetch project id, so if the explicit # file path is cloud sdk credentials path, then we should fall back # to cloud sdk flow, otherwise project id cannot be obtained. return _get_gcloud_sdk_credentials() if explicit_file is not None: credentials, project_id = load_credentials_from_file( os.environ[environment_vars.CREDENTIALS] ) return credentials, project_id else: return None, None
def _get_gcloud_sdk_credentials(): """Gets the credentials and project ID from the Cloud SDK.""" from google.auth import _cloud_sdk # Check if application default credentials exist. credentials_filename = ( _cloud_sdk.get_application_default_credentials_path()) if not os.path.isfile(credentials_filename): return None, None credentials, project_id = _load_credentials_from_file( credentials_filename) if not project_id: project_id = _cloud_sdk.get_project_id() if not project_id: _LOGGER.warning( 'No project ID could be determined from the Cloud SDK ' 'configuration. Consider running `gcloud config set project` or ' 'setting the %s environment variable', environment_vars.PROJECT) return credentials, project_id
def application_default_credentials_path() -> str: """gets gcloud default credentials path""" return get_application_default_credentials_path()
def start_worker(self): # type: () -> None credential_options = [] try: # This is the public facing API, skip if it is not available. # (If this succeeds but the imports below fail, better to actually raise # an error below rather than silently fail.) # pylint: disable=unused-import import google.auth except ImportError: pass else: from google.auth import environment_vars from google.auth import _cloud_sdk gcloud_cred_file = os.environ.get( environment_vars.CREDENTIALS, _cloud_sdk.get_application_default_credentials_path()) if os.path.exists(gcloud_cred_file): docker_cred_file = '/docker_cred_file.json' credential_options.extend([ '--mount', f'type=bind,source={gcloud_cred_file},target={docker_cred_file}', '--env', f'{environment_vars.CREDENTIALS}={docker_cred_file}' ]) with SUBPROCESS_LOCK: try: _LOGGER.info('Attempting to pull image %s', self._container_image) subprocess.check_call(['docker', 'pull', self._container_image]) except Exception: _LOGGER.info( 'Unable to pull image %s, defaulting to local image if it exists' % self._container_image) self._container_id = subprocess.check_output([ 'docker', 'run', '-d', '--network=host', ] + credential_options + [ self._container_image, '--id=%s' % self.worker_id, '--logging_endpoint=%s' % self.logging_api_service_descriptor().url, '--control_endpoint=%s' % self.control_address, '--artifact_endpoint=%s' % self.control_address, '--provision_endpoint=%s' % self.control_address, ]).strip() assert self._container_id is not None while True: status = subprocess.check_output([ 'docker', 'inspect', '-f', '{{.State.Status}}', self._container_id ]).strip() _LOGGER.info( 'Waiting for docker to start up. Current status is %s' % status.decode('utf-8')) if status == b'running': _LOGGER.info( 'Docker container is running. container_id = %s, ' 'worker_id = %s', self._container_id, self.worker_id) break elif status in (b'dead', b'exited'): subprocess.call(['docker', 'container', 'logs', self._container_id]) raise RuntimeError( 'SDK failed to start. Final status is %s' % status.decode('utf-8')) time.sleep(1) self._done = False t = threading.Thread(target=self.watch_container) t.daemon = True t.start()