def login(self, username=None, password=None, client_id=None, login_realm=None):
"""Log the user in by Retrieve tokens for the user with the specified username and password.
Args:
username (str): login username
password (str): login password
client_id (str): OIDC client_id for logging into server
login_realm (str): Keycloak realm of the login user
Returns:
requests.Response: token endpoint response
Note: Login realm can be different from the realm that the client is used to manage
"""
# Get the password from vault
vault = Vault()
if username is None:
username = vault.read(self.vault_path, 'username')
password = vault.read(self.vault_path, 'password')
# Save the client_id and login realm for logging out
self.client_id = client_id or vault.read(self.vault_path, 'client_id')
self.login_realm = login_realm or vault.read(self.vault_path, 'realm')
url = '{}/realms/{}/protocol/openid-connect/token'.format(self.url_base, self.login_realm)
data = {
'grant_type': 'password',
'client_id': self.client_id,
'username': username,
'password': password,
}
response = requests.post(url, data=data, verify=self.https and self.verify_ssl)
KeyCloakError.raise_for_status(response)
self.token = response.json()
return response
def test_retry_logic_returns(self, mockClient):
instance = mockClient.return_value
v = Vault(self.cfg)
instance.read.side_effect = [
hvac.exceptions.Forbidden('Token has expired'), {
"data": {
"super": "this!"
}
}
]
with patch.object(Vault, 'login'):
self.assertEqual(v.read('secrets', 'super'), "this!")
def test_retry_logic_calls_login(self, mockClient):
instance = mockClient.return_value
v = Vault(self.cfg)
instance.read.side_effect = [
hvac.exceptions.Forbidden('Token has expired'), {
"data": {
"super": ""
}
}
]
with patch.object(Vault, 'login') as mock:
v.read('secrets', 'super')
mock.assert_called_once_with(v)
def __init__(self, config=None):
"""
Args:
config (optional[configuration.BossConfig]): Boss configuration. Defaults to loading from /etc/boss/boss.config.
"""
if config is None:
self.config = configuration.BossConfig()
else:
self.config = config
self.vault = Vault(config)
# Get the domain the endpoint lives in.
self.domain = self.config['system']['fqdn'].split('.', 1)[1]
self.iam = boto3.resource('iam', region_name=aws.get_region())
def login(self,
username=None,
password=None,
client_id=None,
login_realm=None):
"""Log the user in by Retrieve tokens for the user with the specified username and password.
Args:
username (str): login username
password (str): login password
client_id (str): OIDC client_id for logging into server
login_realm (str): Keycloak realm of the login user
Returns:
requests.Response: token endpoint response
Note: Login realm can be different from the realm that the client is used to manage
"""
# Get the password from vault
vault = Vault()
if username is None:
username = vault.read(self.vault_path, 'username')
password = vault.read(self.vault_path, 'password')
# Save the client_id and login realm for logging out
self.client_id = client_id or vault.read(self.vault_path, 'client_id')
self.login_realm = login_realm or vault.read(self.vault_path, 'realm')
url = '{}/realms/{}/protocol/openid-connect/token'.format(
self.url_base, self.login_realm)
data = {
'grant_type': 'password',
'client_id': self.client_id,
'username': username,
'password': password,
}
response = requests.post(url,
data=data,
verify=self.https and self.verify_ssl)
KeyCloakError.raise_for_status(response)
self.token = response.json()
return response
def __init__(self, config=None, region_name='us-east-1'):
"""
Args:
config (optional[configuration.BossConfig]): Boss configuration. Defaults to loading from /etc/boss/boss.config.
region_name (optional[string]): AWS region to use. Defaults to us-east-1.
"""
if config is None:
self.config = configuration.BossConfig()
else:
self.config = config
self.vault = Vault(config)
# Get the domain the endpoint lives in.
self.domain = self.config['system']['fqdn'].split('.', 1)[1]
self.iam = boto3.resource('iam', region_name=region_name)
def test_exception_if_key_dosent_exist(self, mockClient):
instance = mockClient.return_value
instance.read.return_value = {"data": {}}
v = Vault(self.cfg)
with self.assertRaises(Exception):
v.read('secrets', 'super')
def test_exception_if_read_fails(self, mockClient):
instance = mockClient.return_value
instance.read.return_value = None
v = Vault(self.cfg)
with self.assertRaises(Exception):
v.read('secrets', 'super')
def test_logout_destroys_hvac_client(self, mockClient):
v = Vault(self.cfg)
v.logout()
self.assertIsNone(v.client)
def test_exception_if_cant_auth_to_vault(self, mockClient):
instance = mockClient.return_value
instance.is_authenticated.return_value = False
with self.assertRaises(Exception):
Vault(self.cfg)
class IngestCredentials:
"""Manages temporary AWS credentials used by ingest clients.
Typical usage:
ingest_creds = IngestCredentials()
#### On POST to endpoint from ingest client.
# If supplying a custom policy:
arn = ingest_creds.create_policy(policy_doc, job_id)
# Otherwise generate the policy using ndingest.util.bossutil.generate_ingest_policy().
# Generate credentials in Vault.
ingest_creds.generate_credentials(job_id, arn)
#### On GET to endpoint from ingest client.
ingest_creds.get_credentials(job_id)
#### When ingest job complete.
ingest_creds.remove_credentials(job_id)
ingest_creds.delete_policy(job_id)
Attributes:
config (configuration.BossConfig): Boss configuration.
domain (string): Domain this class is running in. Used for naming.
iam (IAM.ServiceResource): AWS interface to IAM.
vault (bossutils.Vault): Wrapper to access Vault secret store.
"""
def __init__(self, config=None, region_name='us-east-1'):
"""
Args:
config (optional[configuration.BossConfig]): Boss configuration. Defaults to loading from /etc/boss/boss.config.
region_name (optional[string]): AWS region to use. Defaults to us-east-1.
"""
if config is None:
self.config = configuration.BossConfig()
else:
self.config = config
self.vault = Vault(config)
# Get the domain the endpoint lives in.
self.domain = self.config['system']['fqdn'].split('.', 1)[1]
self.iam = boto3.resource('iam', region_name=region_name)
def create_policy(self, policy_document, job_id, description=''):
"""Create a new IAM policy for the given ingest job.
Args:
policy_document (dict):
job_id (int): Id of ingest job used for name of Vault role.
description (optional[string]): Policy description, defaults to empty string.
Returns:
(string): New policy's ARN.
"""
sanitized_domain = self.domain.replace('.', '-')
path = IAM_PATH.format(sanitized_domain)
policy = self.iam.create_policy(
PolicyName=IAM_POLICY_NAME.format(sanitized_domain, job_id),
PolicyDocument=json.dumps(policy_document),
Path=path,
Description=description)
return policy.arn
def delete_policy(self, job_id):
"""Delete the IAM policy associated with an ingest job.
Args:
job_id (int): Id of ingest job used for name of Vault role.
Returns:
(bool): False if policy not found.
"""
sanitized_domain = self.domain.replace('.', '-')
path = IAM_PATH.format(sanitized_domain)
name = IAM_POLICY_NAME.format(sanitized_domain, job_id)
for policy in self.iam.policies.filter(Scope='Local', PathPrefix=path):
if policy.policy_name == name:
policy.delete()
return True
return False
def generate_credentials(self, job_id, iam_policy_arn):
"""Generate temporary credentials for an ingest job.
A temporary Vault role mapped to an IAM policy is created.
Args:
job_id (int): Id of ingest job used for name of Vault role.
iam_policy_arn (string): Policy associated with credentials.
Returns:
(dict): Contains keys: access_key and secret_key.
"""
# Create Vault role and associate with an IAM policy.
sanitized_domain = self.domain.replace('.', '-')
role_path = INGEST_ROLE_NAME.format(job_id)
self.vault.write(role_path, arn=iam_policy_arn)
# Generate temporary credentials for that role.
creds_path = INGEST_CREDS_NAME.format(job_id)
return self.vault.read_dict(creds_path, raw=False)
def get_credentials(self, job_id):
"""Get new temporary credentials for the given ingest job.
Before calling get_credentials(), the Vault role must be created by
generate_credentials() for the given job_id.
Args:
job_id (int): Get new credentials for this ingest job.
Returns:
(dict|None): Contains keys: access_key and secret_key.
"""
sanitized_domain = self.domain.replace('.', '-')
path = INGEST_CREDS_NAME.format(job_id)
try:
return self.vault.read_dict(path, raw=False)
except hvac.exceptions.InvalidRequest:
return None
def remove_credentials(self, job_id):
"""Revoke credentials and delete the Vault role associated with an ingest job.
This call cleans up for get_credentials() and generate_credentials().
Args:
job_id (int): Get new credentials for this ingest job.
"""
sanitized_domain = self.domain.replace('.', '-')
creds_path = INGEST_CREDS_NAME.format(job_id)
self.vault.revoke_secret_prefix(creds_path)
role_path = INGEST_ROLE_NAME.format(job_id)
self.vault.delete(role_path)
class IngestCredentials:
"""Manages temporary AWS credentials used by ingest clients.
Typical usage:
ingest_creds = IngestCredentials()
#### On POST to endpoint from ingest client.
# If supplying a custom policy:
arn = ingest_creds.create_policy(policy_doc, job_id)
# Otherwise generate the policy using ndingest.util.bossutil.generate_ingest_policy().
# Generate credentials in Vault.
ingest_creds.generate_credentials(job_id, arn)
#### On GET to endpoint from ingest client.
ingest_creds.get_credentials(job_id)
#### When ingest job complete.
ingest_creds.remove_credentials(job_id)
ingest_creds.delete_policy(job_id)
Attributes:
config (configuration.BossConfig): Boss configuration.
domain (string): Domain this class is running in. Used for naming.
iam (IAM.ServiceResource): AWS interface to IAM.
vault (bossutils.Vault): Wrapper to access Vault secret store.
"""
def __init__(self, config=None, region_name='us-east-1'):
"""
Args:
config (optional[configuration.BossConfig]): Boss configuration. Defaults to loading from /etc/boss/boss.config.
region_name (optional[string]): AWS region to use. Defaults to us-east-1.
"""
if config is None:
self.config = configuration.BossConfig()
else:
self.config = config
self.vault = Vault(config)
# Get the domain the endpoint lives in.
self.domain = self.config['system']['fqdn'].split('.', 1)[1]
self.iam = boto3.resource('iam', region_name=region_name)
def create_policy(self, policy_document, job_id, description=''):
"""Create a new IAM policy for the given ingest job.
Args:
policy_document (dict):
job_id (int): Id of ingest job used for name of Vault role.
description (optional[string]): Policy description, defaults to empty string.
Returns:
(string): New policy's ARN.
"""
sanitized_domain = self.domain.replace('.', '-')
path=IAM_PATH.format(sanitized_domain)
policy = self.iam.create_policy(
PolicyName=IAM_POLICY_NAME.format(sanitized_domain, job_id),
PolicyDocument=json.dumps(policy_document),
Path=path,
Description=description)
return policy.arn
def delete_policy(self, job_id):
"""Delete the IAM policy associated with an ingest job.
Args:
job_id (int): Id of ingest job used for name of Vault role.
Returns:
(bool): False if policy not found.
"""
sanitized_domain = self.domain.replace('.', '-')
path=IAM_PATH.format(sanitized_domain)
name = IAM_POLICY_NAME.format(sanitized_domain, job_id)
for policy in self.iam.policies.filter(Scope='Local', PathPrefix=path):
if policy.policy_name == name:
policy.delete()
return True
return False
def generate_credentials(self, job_id, iam_policy_arn):
"""Generate temporary credentials for an ingest job.
A temporary Vault role mapped to an IAM policy is created.
Args:
job_id (int): Id of ingest job used for name of Vault role.
iam_policy_arn (string): Policy associated with credentials.
Returns:
(dict): Contains keys: access_key and secret_key.
"""
# Create Vault role and associate with an IAM policy.
sanitized_domain = self.domain.replace('.', '-')
role_path = INGEST_ROLE_NAME.format(job_id)
self.vault.write(role_path, arn=iam_policy_arn)
# Generate temporary credentials for that role.
creds_path = INGEST_CREDS_NAME.format(job_id)
return self.vault.read_dict(creds_path, raw=False)
def get_credentials(self, job_id):
"""Get new temporary credentials for the given ingest job.
Before calling get_credentials(), the Vault role must be created by
generate_credentials() for the given job_id.
Args:
job_id (int): Get new credentials for this ingest job.
Returns:
(dict|None): Contains keys: access_key and secret_key.
"""
sanitized_domain = self.domain.replace('.', '-')
path = INGEST_CREDS_NAME.format(job_id)
try:
return self.vault.read_dict(path, raw=False)
except hvac.exceptions.InvalidRequest:
return None
def remove_credentials(self, job_id):
"""Revoke credentials and delete the Vault role associated with an ingest job.
This call cleans up for get_credentials() and generate_credentials().
Args:
job_id (int): Get new credentials for this ingest job.
"""
sanitized_domain = self.domain.replace('.', '-')
creds_path = INGEST_CREDS_NAME.format(job_id)
self.vault.revoke_secret_prefix(creds_path)
role_path = INGEST_ROLE_NAME.format(job_id)
self.vault.delete(role_path)
if __name__ == "__main__":
# usage (backup|restore) domain
a = sys.argv[1]
d = sys.argv[2]
with open("/etc/boss/boss.config", "w") as fh:
fh.write("""[system]
type = backup
[vault]
url = http://vault.{}:8200
token =
""".format(d))
v = Vault()
if a == "backup":
f = os.path.join(os.environ['OUTPUT1_STAGING_DIR'], 'export.json')
data = {
'policies': {},
'secrets': {},
'aws-auth': {},
'aws': {},
}
# Backup policies
for policy in v.client.list_policies():
data['policies'][policy] = v.client.read('/sys/policy/' + policy)['rules']
# Backup secrets
#!/usr/bin/env python3
# Copyright 2018 The Johns Hopkins University Applied Physics Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A simple script to connect to Vault, pull out the given
# path, and return the requested key value.
# Created for use by rds.sh
import sys
from bossutils.vault import Vault
if __name__ == '__main__':
# Usage: creds.py vault/path key
v = Vault()
p = sys.argv[1]
k = sys.argv[2]
d = v.read_dict(p)
print(d[k])