def client(self) -> hvac.Client:
"""
Return an authenticated Hashicorp Vault client
"""
_client = hvac.Client(url=self.url, **self.kwargs)
if self.auth_type == "token":
if not self.token:
raise VaultError("token cannot be None for auth_type='token'")
_client.token = self.token
elif self.auth_type == "ldap":
_client.auth.ldap.login(
username=self.username, password=self.password)
elif self.auth_type == "userpass":
_client.auth_userpass(username=self.username, password=self.password)
elif self.auth_type == "approle":
_client.auth_approle(role_id=self.role_id, secret_id=self.secret_id)
elif self.auth_type == "github":
_client.auth.github.login(token=self.token)
elif self.auth_type == "gcp":
credentials = self._get_gcp_credentials()
_client.auth.gcp.configure(credentials=credentials)
else:
raise AirflowException(f"Authentication type '{self.auth_type}' not supported")
if _client.is_authenticated():
return _client
else:
raise VaultError("Vault Authentication Error!")
def client(self):
# type: () -> hvac.Client
"""
Return an authenticated Hashicorp Vault client
"""
_client = hvac.Client(url=self.url, **self.kwargs)
if self.auth_type == "token":
if not self.token:
raise VaultError("token cannot be None for auth_type='token'")
_client.token = self.token
elif self.auth_type == "ldap":
_client.auth.ldap.login(
username=self.username, password=self.password)
elif self.auth_type == "userpass":
_client.auth_userpass(username=self.username, password=self.password)
elif self.auth_type == "approle":
_client.auth_approle(role_id=self.role_id, secret_id=self.secret_id)
elif self.auth_type == "github":
_client.auth.github.login(token=self.token)
elif self.auth_type == "gcp":
from airflow.contrib.utils.gcp_credentials_provider import (
get_credentials_and_project_id,
_get_scopes
)
scopes = _get_scopes(self.gcp_scopes)
credentials, _ = get_credentials_and_project_id(key_path=self.gcp_key_path, scopes=scopes)
_client.auth.gcp.configure(credentials=credentials)
else:
raise AirflowException("Authentication type '{}' not supported".format(self.auth_type))
if _client.is_authenticated():
return _client
else:
raise VaultError("Vault Authentication Error!")
def create_or_update_secret(
self, secret_path: str, secret: dict, method: Optional[str] = None, cas: Optional[int] = None
) -> Response:
"""
Creates or updates secret.
:param secret_path: The path of the secret.
:param secret: Secret to create or update for the path specified
:param method: Optional parameter to explicitly request a POST (create) or PUT (update) request to
the selected kv secret engine. If no argument is provided for this parameter, hvac attempts to
intelligently determine which method is appropriate. Only valid for KV engine version 1
:param cas: Set the "cas" value to use a Check-And-Set operation. If not set the write will be
allowed. If set to 0 a write will only be allowed if the key doesn't exist.
If the index is non-zero the write will only be allowed if the key's current version
matches the version specified in the cas parameter. Only valid for KV engine version 2.
:rtype: requests.Response
:return: The response of the create_or_update_secret request.
See https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v1.html
and https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v2.html for details.
"""
if self.kv_engine_version == 2 and method:
raise VaultError("The method parameter is only valid for version 1")
if self.kv_engine_version == 1 and cas:
raise VaultError("The cas parameter is only valid for version 2")
if self.kv_engine_version == 1:
response = self.client.secrets.kv.v1.create_or_update_secret(
secret_path=secret_path, secret=secret, mount_point=self.mount_point, method=method
)
else:
response = self.client.secrets.kv.v2.create_or_update_secret(
secret_path=secret_path, secret=secret, mount_point=self.mount_point, cas=cas
)
return response
def client(self) -> hvac.Client:
"""
Return an authenticated Hashicorp Vault client.
:rtype: hvac.Client
:return: Vault Client
"""
_client = hvac.Client(url=self.url, **self.kwargs)
if self.auth_type == "approle":
self._auth_approle(_client)
elif self.auth_type == "gcp":
self._auth_gcp(_client)
elif self.auth_type == "github":
self._auth_github(_client)
elif self.auth_type == "kubernetes":
self._auth_kubernetes(_client)
elif self.auth_type == "ldap":
self._auth_ldap(_client)
elif self.auth_type == "token":
_client.token = self.token
elif self.auth_type == "userpass":
self._auth_userpass(_client)
else:
raise VaultError(
f"Authentication type '{self.auth_type}' not supported")
if _client.is_authenticated():
return _client
else:
raise VaultError("Vault Authentication Error!")
def get_secret_including_metadata(
self,
secret_path: str,
secret_version: Optional[int] = None) -> Optional[dict]:
"""
Reads secret including metadata. It is only valid for KV version 2.
See https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v2.html for details.
:param secret_path: The path of the secret.
:param secret_version: Specifies the version of Secret to return. If not set, the latest
version is returned. (Can only be used in case of version 2 of KV).
:rtype: dict
:return: The key info. This is a Dict with "data" mapping keeping secret
and "metadata" mapping keeping metadata of the secret.
"""
if self.kv_engine_version == 1:
raise VaultError(
"Metadata might only be used with version 2 of the KV engine.")
try:
return self.client.secrets.kv.v2.read_secret_version(
path=secret_path,
mount_point=self.mount_point,
version=secret_version)
except InvalidPath:
self.log.debug(
"Secret not found %s with mount point %s and version %s",
secret_path,
self.mount_point,
secret_version,
)
return None
def get_secret(self, secret_path: str, secret_version: Optional[int] = None) -> Optional[dict]:
"""
Get secret value from the KV engine.
:param secret_path: The path of the secret.
:type secret_path: str
:param secret_version: Specifies the version of Secret to return. If not set, the latest
version is returned. (Can only be used in case of version 2 of KV).
:type secret_version: int
See https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v1.html
and https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v2.html for details.
:return: secret stored in the vault as a dictionary
"""
try:
if self.kv_engine_version == 1:
if secret_version:
raise VaultError("Secret version can only be used with version 2 of the KV engine")
response = self.client.secrets.kv.v1.read_secret(
path=secret_path, mount_point=self.mount_point
)
else:
response = self.client.secrets.kv.v2.read_secret_version(
path=secret_path, mount_point=self.mount_point, version=secret_version
)
except InvalidPath:
self.log.debug("Secret not found %s with mount point %s", secret_path, self.mount_point)
return None
return_data = response["data"] if self.kv_engine_version == 1 else response["data"]["data"]
return return_data
def _auth_kubernetes(self, _client: hvac.Client) -> None:
if not self.kubernetes_jwt_path:
raise VaultError(
"The kubernetes_jwt_path should be set here. This should not happen."
)
with open(self.kubernetes_jwt_path) as f:
jwt = f.read()
_client.auth_kubernetes(role=self.kubernetes_role, jwt=jwt)
def client(self) -> hvac.Client:
"""
Return an authenticated Hashicorp Vault client
"""
_client = hvac.Client(url=self.url, **self.kwargs)
if self.auth_type == "token":
if not self.token:
raise VaultError("token cannot be None for auth_type='token'")
_client.token = self.token
elif self.auth_type == "ldap":
_client.auth.ldap.login(username=self.username,
password=self.password)
elif self.auth_type == "userpass":
_client.auth_userpass(username=self.username,
password=self.password)
elif self.auth_type == "approle":
_client.auth_approle(role_id=self.role_id,
secret_id=self.secret_id)
elif self.auth_type == "kubernetes":
if not self.kubernetes_role:
raise VaultError(
"kubernetes_role cannot be None for auth_type='kubernetes'"
)
with open(self.kubernetes_jwt_path) as f:
jwt = f.read()
_client.auth_kubernetes(role=self.kubernetes_role, jwt=jwt)
elif self.auth_type == "github":
_client.auth.github.login(token=self.token)
elif self.auth_type == "gcp":
from airflow.providers.google.cloud.utils.credentials_provider import (
get_credentials_and_project_id, _get_scopes)
scopes = _get_scopes(self.gcp_scopes)
credentials, _ = get_credentials_and_project_id(
key_path=self.gcp_key_path, scopes=scopes)
_client.auth.gcp.configure(credentials=credentials)
else:
raise AirflowException(
f"Authentication type '{self.auth_type}' not supported")
if _client.is_authenticated():
return _client
else:
raise VaultError("Vault Authentication Error!")
def test_read_secrets_shows_vault_vault_error(connect_m):
'''
Test read_secrets() gracefully displays vault errors.
'''
os.environ['VAULT_ADDR'] = 'https://test.vaultserver'
connect_m.side_effect = VaultError('Boom!')
error_message = 'Vault Error: Boom!'
with pytest.raises(SystemExit, match=error_message):
vault.read_secrets('test/vault/path')
def __init__( # pylint: disable=too-many-arguments
self,
url: Optional[str] = None,
auth_type: str = 'token',
mount_point: str = "secret",
kv_engine_version: Optional[int] = None,
token: Optional[str] = None,
username: Optional[str] = None,
password: Optional[str] = None,
secret_id: Optional[str] = None,
role_id: Optional[str] = None,
kubernetes_role: Optional[str] = None,
kubernetes_jwt_path: Optional[
str] = '/var/run/secrets/kubernetes.io/serviceaccount/token',
gcp_key_path: Optional[str] = None,
gcp_scopes: Optional[str] = None,
**kwargs):
super().__init__()
if kv_engine_version and kv_engine_version not in VALID_KV_VERSIONS:
raise VaultError(
f"The version is not supported: {kv_engine_version}. "
f"It should be one of {VALID_KV_VERSIONS}")
if auth_type not in VALID_AUTH_TYPES:
raise VaultError(f"The auth_type is not supported: {auth_type}. "
f"It should be one of {VALID_AUTH_TYPES}")
if auth_type == "token" and not token:
raise VaultError(
"The 'token' authentication type requires 'token'")
if auth_type == "github" and not token:
raise VaultError(
"The 'github' authentication type requires 'token'")
if auth_type == "approle" and not role_id:
raise VaultError(
"The 'approle' authentication type requires 'role_id'")
if auth_type == "kubernetes":
if not kubernetes_role:
raise VaultError(
"The 'kubernetes' authentication type requires 'kubernetes_role'"
)
if not kubernetes_jwt_path:
raise VaultError(
"The 'kubernetes' authentication type requires 'kubernetes_jwt_path'"
)
self.kv_engine_version = kv_engine_version if kv_engine_version else 2
self.url = url
self.auth_type = auth_type
self.kwargs = kwargs
self.token = token
self.mount_point = mount_point
self.username = username
self.password = password
self.secret_id = secret_id
self.role_id = role_id
self.kubernetes_role = kubernetes_role
self.kubernetes_jwt_path = kubernetes_jwt_path
self.gcp_key_path = gcp_key_path
self.gcp_scopes = gcp_scopes
def _get_radius_parameters_from_connection(
self, radius_host: Optional[str], radius_port: Optional[int]) \
-> Tuple[Optional[str], Optional[int]]:
if not radius_port:
radius_port_str = self.connection.extra_dejson.get("radius_port")
if radius_port_str:
try:
radius_port = int(radius_port_str)
except ValueError:
raise VaultError(f"Radius port was wrong: {radius_port_str}")
if not radius_host:
radius_host = self.connection.extra_dejson.get("radius_host")
return radius_host, radius_port
def _auth_kubernetes(self, _client: hvac.Client) -> None:
if not self.kubernetes_jwt_path:
raise VaultError(
"The kubernetes_jwt_path should be set here. This should not happen."
)
with open(self.kubernetes_jwt_path) as f:
jwt = f.read().strip()
if self.auth_mount_point:
Kubernetes(_client.adapter).login(
role=self.kubernetes_role,
jwt=jwt,
mount_point=self.auth_mount_point)
else:
Kubernetes(_client.adapter).login(role=self.kubernetes_role,
jwt=jwt)
def __init__(self, obj: Dict):
self._required = obj.get('__vault_required__', True)
self._config = (self._config_scheme + obj.get('__vault__')).bind(Close)
try:
self._cache = self._request()
except VaultError as error:
if self._required is True:
raise VaultError(error)
self._cache = {}
if self._config['data_type'] == 'kv':
super().__init__(obj + self.read(obj, obj.get('__prefix__')))
elif self._config['data_type'] == 'json':
super().__init__(obj + self._cache)
else:
raise ValueError(f'only supported "kv" or "json"')
def fake_vault(path: str, mount_point: str = "secrets") -> VaultResponseDict:
vault: VaultStructure = {
"first_level_key": {
"username": "******",
"password": "******",
"complex_value": {
"inner": "value",
"int_key": 12,
"list_key": ["first", "second", "third"],
},
},
"nested/path": {"username": "******"},
}
try:
return {"data": {"data": vault[path]}}
except KeyError:
raise VaultError(f"Key {path} not found in Vault")
def get_secret_metadata(self, secret_path: str) -> Optional[dict]:
"""
Reads secret metadata (including versions) from the engine. It is only valid for KV version 2.
:param secret_path: The path of the secret.
:rtype: dict
:return: secret metadata. This is a Dict containing metadata for the secret.
See https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v2.html for details.
"""
if self.kv_engine_version == 1:
raise VaultError("Metadata might only be used with version 2 of the KV engine.")
try:
return self.client.secrets.kv.v2.read_secret_metadata(
path=secret_path, mount_point=self.mount_point
)
except InvalidPath:
self.log.debug("Secret not found %s with mount point %s", secret_path, self.mount_point)
return None
def __init__( # pylint: disable=too-many-arguments
self,
vault_conn_id: str,
auth_type: Optional[str] = None,
auth_mount_point: Optional[str] = None,
kv_engine_version: Optional[int] = None,
role_id: Optional[str] = None,
kubernetes_role: Optional[str] = None,
kubernetes_jwt_path: Optional[str] = None,
token_path: Optional[str] = None,
gcp_key_path: Optional[str] = None,
gcp_scopes: Optional[str] = None,
azure_tenant_id: Optional[str] = None,
azure_resource: Optional[str] = None,
radius_host: Optional[str] = None,
radius_port: Optional[int] = None):
super().__init__()
self.connection = self.get_connection(vault_conn_id)
if not auth_type:
auth_type = self.connection.extra_dejson.get(
'auth_type') or "token"
if not auth_mount_point:
auth_mount_point = self.connection.extra_dejson.get(
'auth_mount_point')
if not kv_engine_version:
conn_version = self.connection.extra_dejson.get(
"kv_engine_version")
try:
kv_engine_version = int(
conn_version
) if conn_version else DEFAULT_KV_ENGINE_VERSION
except ValueError:
raise VaultError(
f"The version is not an int: {conn_version}. ")
if auth_type in ["approle", "aws_iam"]:
if not role_id:
role_id = self.connection.extra_dejson.get('role_id')
azure_resource, azure_tenant_id = \
self._get_azure_parameters_from_connection(azure_resource, azure_tenant_id) \
if auth_type == 'azure' else (None, None)
gcp_key_path, gcp_keyfile_dict, gcp_scopes = \
self._get_gcp_parameters_from_connection(gcp_key_path, gcp_scopes) \
if auth_type == 'gcp' else (None, None, None)
kubernetes_jwt_path, kubernetes_role = \
self._get_kubernetes_parameters_from_connection(kubernetes_jwt_path, kubernetes_role) \
if auth_type == 'kubernetes' else (None, None)
radius_host, radius_port = self._get_radius_parameters_from_connection(radius_host, radius_port) \
if auth_type == 'radius' else (None, None)
if self.connection.conn_type == 'vault':
conn_protocol = 'http'
elif self.connection.conn_type == 'vaults':
conn_protocol = 'https'
elif self.connection.conn_type == 'http':
conn_protocol = 'http'
elif self.connection.conn_type == 'https':
conn_protocol = 'https'
else:
raise VaultError(
"The url schema must be one of ['http', 'https', 'vault', 'vaults' ]"
)
url = f"{conn_protocol}://{self.connection.host}"
if self.connection.port:
url += f":{self.connection.port}"
# Schema is really path in the Connection definition. This is pretty confusing because of URL schema
mount_point = self.connection.schema if self.connection.schema else 'secret'
self.vault_client = _VaultClient(
url=url,
auth_type=auth_type,
auth_mount_point=auth_mount_point,
mount_point=mount_point,
kv_engine_version=kv_engine_version,
token=self.connection.password,
token_path=token_path,
username=self.connection.login,
password=self.connection.password,
key_id=self.connection.login,
secret_id=self.connection.password,
role_id=role_id,
kubernetes_role=kubernetes_role,
kubernetes_jwt_path=kubernetes_jwt_path,
gcp_key_path=gcp_key_path,
gcp_keyfile_dict=gcp_keyfile_dict,
gcp_scopes=gcp_scopes,
azure_tenant_id=azure_tenant_id,
azure_resource=azure_resource,
radius_host=radius_host,
radius_secret=self.connection.password,
radius_port=radius_port)
def __init__(
self,
url: Optional[str] = None,
auth_type: str = 'token',
auth_mount_point: Optional[str] = None,
mount_point: str = "secret",
kv_engine_version: Optional[int] = None,
token: Optional[str] = None,
token_path: Optional[str] = None,
username: Optional[str] = None,
password: Optional[str] = None,
key_id: Optional[str] = None,
secret_id: Optional[str] = None,
role_id: Optional[str] = None,
kubernetes_role: Optional[str] = None,
kubernetes_jwt_path: Optional[
str] = '/var/run/secrets/kubernetes.io/serviceaccount/token',
gcp_key_path: Optional[str] = None,
gcp_keyfile_dict: Optional[dict] = None,
gcp_scopes: Optional[str] = None,
azure_tenant_id: Optional[str] = None,
azure_resource: Optional[str] = None,
radius_host: Optional[str] = None,
radius_secret: Optional[str] = None,
radius_port: Optional[int] = None,
**kwargs,
):
super().__init__()
if kv_engine_version and kv_engine_version not in VALID_KV_VERSIONS:
raise VaultError(
f"The version is not supported: {kv_engine_version}. "
f"It should be one of {VALID_KV_VERSIONS}")
if auth_type not in VALID_AUTH_TYPES:
raise VaultError(f"The auth_type is not supported: {auth_type}. "
f"It should be one of {VALID_AUTH_TYPES}")
if auth_type == "token" and not token and not token_path:
raise VaultError(
"The 'token' authentication type requires 'token' or 'token_path'"
)
if auth_type == "github" and not token and not token_path:
raise VaultError(
"The 'github' authentication type requires 'token' or 'token_path'"
)
if auth_type == "approle" and not role_id:
raise VaultError(
"The 'approle' authentication type requires 'role_id'")
if auth_type == "kubernetes":
if not kubernetes_role:
raise VaultError(
"The 'kubernetes' authentication type requires 'kubernetes_role'"
)
if not kubernetes_jwt_path:
raise VaultError(
"The 'kubernetes' authentication type requires 'kubernetes_jwt_path'"
)
if auth_type == "azure":
if not azure_resource:
raise VaultError(
"The 'azure' authentication type requires 'azure_resource'"
)
if not azure_tenant_id:
raise VaultError(
"The 'azure' authentication type requires 'azure_tenant_id'"
)
if auth_type == "radius":
if not radius_host:
raise VaultError(
"The 'radius' authentication type requires 'radius_host'")
if not radius_secret:
raise VaultError(
"The 'radius' authentication type requires 'radius_secret'"
)
self.kv_engine_version = kv_engine_version if kv_engine_version else 2
self.url = url
self.auth_type = auth_type
self.kwargs = kwargs
self.token = token
self.token_path = token_path
self.auth_mount_point = auth_mount_point
self.mount_point = mount_point
self.username = username
self.password = password
self.key_id = key_id
self.secret_id = secret_id
self.role_id = role_id
self.kubernetes_role = kubernetes_role
self.kubernetes_jwt_path = kubernetes_jwt_path
self.gcp_key_path = gcp_key_path
self.gcp_keyfile_dict = gcp_keyfile_dict
self.gcp_scopes = gcp_scopes
self.azure_tenant_id = azure_tenant_id
self.azure_resource = azure_resource
self.radius_host = radius_host
self.radius_secret = radius_secret
self.radius_port = radius_port
def __init__(
self,
vault_conn_id: str = default_conn_name,
auth_type: Optional[str] = None,
auth_mount_point: Optional[str] = None,
kv_engine_version: Optional[int] = None,
role_id: Optional[str] = None,
kubernetes_role: Optional[str] = None,
kubernetes_jwt_path: Optional[str] = None,
token_path: Optional[str] = None,
gcp_key_path: Optional[str] = None,
gcp_scopes: Optional[str] = None,
azure_tenant_id: Optional[str] = None,
azure_resource: Optional[str] = None,
radius_host: Optional[str] = None,
radius_port: Optional[int] = None,
):
super().__init__()
self.connection = self.get_connection(vault_conn_id)
if not auth_type:
auth_type = self.connection.extra_dejson.get(
'auth_type') or "token"
if not auth_mount_point:
auth_mount_point = self.connection.extra_dejson.get(
'auth_mount_point')
if not kv_engine_version:
conn_version = self.connection.extra_dejson.get(
"kv_engine_version")
try:
kv_engine_version = int(
conn_version
) if conn_version else DEFAULT_KV_ENGINE_VERSION
except ValueError:
raise VaultError(
f"The version is not an int: {conn_version}. ")
if auth_type == "approle":
if role_id:
warnings.warn(
"""The usage of role_id for AppRole authentication has been deprecated.
Please use connection login.""",
DeprecationWarning,
stacklevel=2,
)
elif self.connection.extra_dejson.get('role_id'):
role_id = self.connection.extra_dejson.get('role_id')
warnings.warn(
"""The usage of role_id in connection extra for AppRole authentication has been
deprecated. Please use connection login.""",
DeprecationWarning,
stacklevel=2,
)
elif self.connection.login:
role_id = self.connection.login
if auth_type == "aws_iam":
if not role_id:
role_id = self.connection.extra_dejson.get('role_id')
azure_resource, azure_tenant_id = (
self._get_azure_parameters_from_connection(
azure_resource, azure_tenant_id) if auth_type == 'azure' else
(None, None))
gcp_key_path, gcp_keyfile_dict, gcp_scopes = (
self._get_gcp_parameters_from_connection(gcp_key_path, gcp_scopes)
if auth_type == 'gcp' else (None, None, None))
kubernetes_jwt_path, kubernetes_role = (
self._get_kubernetes_parameters_from_connection(
kubernetes_jwt_path, kubernetes_role)
if auth_type == 'kubernetes' else (None, None))
radius_host, radius_port = (
self._get_radius_parameters_from_connection(
radius_host, radius_port) if auth_type == 'radius' else
(None, None))
if self.connection.conn_type == 'vault':
conn_protocol = 'http'
elif self.connection.conn_type == 'vaults':
conn_protocol = 'https'
elif self.connection.conn_type == 'http':
conn_protocol = 'http'
elif self.connection.conn_type == 'https':
conn_protocol = 'https'
else:
raise VaultError(
"The url schema must be one of ['http', 'https', 'vault', 'vaults' ]"
)
url = f"{conn_protocol}://{self.connection.host}"
if self.connection.port:
url += f":{self.connection.port}"
# Schema is really path in the Connection definition. This is pretty confusing because of URL schema
mount_point = self.connection.schema if self.connection.schema else 'secret'
self.vault_client = _VaultClient(
url=url,
auth_type=auth_type,
auth_mount_point=auth_mount_point,
mount_point=mount_point,
kv_engine_version=kv_engine_version,
token=self.connection.password,
token_path=token_path,
username=self.connection.login,
password=self.connection.password,
key_id=self.connection.login,
secret_id=self.connection.password,
role_id=role_id,
kubernetes_role=kubernetes_role,
kubernetes_jwt_path=kubernetes_jwt_path,
gcp_key_path=gcp_key_path,
gcp_keyfile_dict=gcp_keyfile_dict,
gcp_scopes=gcp_scopes,
azure_tenant_id=azure_tenant_id,
azure_resource=azure_resource,
radius_host=radius_host,
radius_secret=self.connection.password,
radius_port=radius_port,
)
