def create_transit_key(enc_key_name):
"""
Creates a new transit encryption key in Vault.
Args:
enc_key_name: The name of the Vault transit encryption key to create
"""
baseutils.retry(client.secrets.transit.create_key, enc_key_name)
def ks_enable_key_protect(cluster_name, region, key_protect_instance_guid,
key_id):
"""
Enables Key Protect on a given IKS cluster.
Key Protect will add additional security to the use of Kubernetes secrets.
If Key Protect is already enabled for the cluster, no action will be taken.
This function will wait until Key Protect is fully enabled in the environment before returning.
Args:
cluster_name: The name of the cluster to update
region: The region of the Key Protect instance
key_protect_instance_guid; The GUID of the Key Protect instance
key_id: The ID of the key inside Key Protect to use. This should be a root key
"""
cluster = baseutils.retry(ks_cluster_get,
cluster_name,
interval=30,
retry=40)
if not cluster.key_protect_enabled:
baseutils.exe_cmd(
'/usr/local/bin/ibmcloud ks key-protect-enable --cluster {cluster} --key-protect-url {kp_url} --key-protect-instance {kp_guid} --crk {key_id}'
.format(cluster=baseutils.shell_escape(cluster_name),
kp_url=baseutils.shell_escape(
'{region}.kms.cloud.ibm.com'.format(region=region)),
kp_guid=baseutils.shell_escape(key_protect_instance_guid),
key_id=baseutils.shell_escape(key_id)))
while not cluster.key_protect_enabled or cluster.master_status != 'Ready':
time.sleep(30)
cluster = baseutils.retry(ks_cluster_get,
cluster_name,
interval=30,
retry=40)
def write(path, properties):
"""
Write values to a given path.
The values are key-value pairs that should be passed as a dictionary.
Args:
path: The path to write the secrets to
properties: The properties to write as key-value pairs passed as a dictionary
"""
baseutils.retry(client.write, path=path, **properties)
def test_retry(self):
pid = os.getpid()
self.assertEqual(pid, baseutils.retry(os.getpid))
self.assertEqual(pid, baseutils.retry(os.getpid, retry=5, interval=5))
a = [1, 2]
self.assertEqual(len(a), baseutils.retry(len, a, interval=5))
try:
baseutils.retry(dict, 'value', retry=1, interval=1)
self.fail(
'baseutils.retry passed a failed attempt to create a dictionary'
)
except Exception as e:
self.assertIsInstance(e, ValueError)
def drain(node, force=True, delete_local_data=True, ignore_daemonsets=True):
"""
Drains a node in Kubernetes. The node is left in a cordoned state where new pods cannot be scheduled to it.
Args:
node: The node to be drained
force: Drain pods even if they are not managed by a ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet (Optional, default: True)
delete_local_data: Drain pods even if they use emptyDir. Data belonging to these pods will be lost (Optional, default: True)
igrnore_daemonsets: Skip pods managed by DaemonSets. These pods are undrainable. Otherwise, a daemonset pod would cause the command to fail (Optional, default: True)
"""
cmd = '{kubectl} drain {node} {force} {delete_local_data} {ignore_daemonsets}'.format(
kubectl=kubectl_binary,
node=baseutils.shell_escape(node),
force='--force' if force else '',
delete_local_data='--delete-local-data' if delete_local_data else '',
ignore_daemonsets='--ignore-daemonsets' if ignore_daemonsets else '')
baseutils.retry(baseutils.exe_cmd, cmd, interval=10, retry=6)
def encrypt_value(key_name, value):
"""
Encrypts a single value using Vault's transit encryption service.
Args:
key_name: The name of the Vault transit key to use for encryption
value: The value to encrypt
"""
return baseutils.retry(client.secrets.transit.encrypt_data, key_name, base64.b64encode(value.encode('utf-8')))['data']['ciphertext']
def decrypt_value(key_name, value):
"""
Decrypts a single value using Vault's transit encryption service.
Args:
key_name: The name of the Vault transit key to use for decryption
value: The value to decrypt
"""
return base64.b64decode(baseutils.retry(client.secrets.transit.decrypt_data, key_name, value)['data']['plaintext']).decode('utf-8')
def login(vault_url, vault_access_token):
"""
Authenticates with Vault, instantiating the long-lived client object.
This can either be called directly or you can rely on the ensure_client decorator if the credentials are passed via environment variables or files.
"""
global client
logger.info('Creating Vault client object')
client = baseutils.retry(hvac.Client, url=vault_url, token=vault_access_token, verify=get_vault_ca())
def list_keys(path):
"""
Retrieves a list of keys at a specified path in Vault.
Args:
path: The path to list the keys at, eg. secret/my/path
Returns: A list of keys at the specified path
"""
result = baseutils.retry(client.list, path=path) or {}
return result.get('data', {}).get('keys', [])
def list_releases(filter=None):
"""
Executes helm list and returns the output. A filter can be optionally specified.
Args:
filter: A regex filter that will be passed through to the "helm list <filter>" command
Returns: The output from the helm list command
"""
cmd = '{helm} list {filter}'.format(helm=helm_binary, filter=baseutils.shell_escape(filter or ''))
(rc, output) = baseutils.retry(baseutils.exe_cmd, cmd, interval=10, retry=6)
return output
def history(release):
"""
Retrieves the revision history for a specified release.
Args:
release: The name of the release to retrieve the revision history for
Returns: A list of ReleaseRevision objects
"""
cmd = '{helm} history {release} -o json'.format(helm=helm_binary, release=baseutils.shell_escape(release))
(rc, output) = baseutils.retry(baseutils.exe_cmd, cmd, interval=10, retry=6)
return ReleaseRevision.parse_release_revisions(json.loads(output))
def encrypt_dict(enc_key_name, data):
"""
Encrypts the values of a passed dictionary.
The dictionary is recursively traversed to find all values and the values are passed in batch to Vault for encryption.
It is therefore much more efficient to use this method to encrypt a dictionary with many values than individually calling encrypt for each one.
Caution: The passed dictionary is mutated.
Args:
enc_key_name: The name of the Vault transit encryption key to use for encryption
data: A dictionary of values to be encrypted. The dictionary can contain nested keys. The passed dictionary is mutated to contain the encrypted values
"""
dec_aggregated_data = _aggregate_dict_for_vault(data, 'plaintext')
try:
enc_aggregated_data = baseutils.retry(client.secrets.transit.encrypt_data, enc_key_name, None, batch_input=dec_aggregated_data)['data']['batch_results']
except Exception as e:
if str(e) == 'encryption key not found': # Create the key if it does not yet exist
create_transit_key(enc_key_name)
enc_aggregated_data = baseutils.retry(client.secrets.transit.encrypt_data, enc_key_name, None, batch_input=dec_aggregated_data)['data']['batch_results']
else:
raise
_restore_dict_from_vault(data, 'ciphertext', enc_aggregated_data)
def block_volume_list():
"""List all instances of block volumes with given command
Args: None
Returns: Returns a list of all block volumes with column headers specified in call
"""
cmd = '/usr/local/bin/ibmcloud sl block volume-list --column id --column username --column datacenter --column storage_type --column capacity_gb --column bytes_used --column ip_addr --column notes' # noqa
(rc, output) = baseutils.retry(baseutils.exe_cmd,
cmd,
interval=10,
retry=5)
block_volume_model_list = Volume.parse_volumes(output, 'block')
return block_volume_model_list
def configure_kubecfg(cluster_name):
"""
Configures the current kubecfg file (based on KUBECONFIG env var) for the specified cluster.
Args:
environment: The name of the IKS environment to configure locally
"""
cmd = '/usr/local/bin/ibmcloud ks cluster config --cluster {cluster_name}'.format(
cluster_name=baseutils.shell_escape(cluster_name))
(rc, output) = baseutils.retry(baseutils.exe_cmd,
cmd,
interval=10,
retry=6)
def decrypt_dict(enc_key_name, data):
"""
Decrypts the values of a passed dictionary.
The dictionary is recursively traversed to find all values and the values are passed in batch to Vault for decryption.
It is therefore much more efficient to use this method to decrypt a dictionary with many values than individually calling decrypt for each one.
Caution: The passed dictionary is mutated.
Args:
enc_key_name: The name of the Vault transit encryption key to use for decryption
data: A dictionary of values to be decrypted. The dictionary can contain nested keys. The passed dictionary is mutated to contain the decrypted values
"""
enc_aggregated_data = _aggregate_dict_for_vault(data, 'ciphertext')
dec_aggregated_data = baseutils.retry(client.secrets.transit.decrypt_data, enc_key_name, None, batch_input=enc_aggregated_data)['data']['batch_results']
_restore_dict_from_vault(data, 'plaintext', dec_aggregated_data)
def replace_iks_workers(cluster_name, workers):
"""
Drain and replace a series of workers, 1 at a time, in an IKS cluster.
A pod's terminationGracePeriodSeconds is obeyed during the draining of a node.
Args:
cluster_name. The name of the cluster to replace workers in or its ID
workers: A list of one or more worker objects to sequentially replace
"""
for worker in workers:
# Ensure worker still exists, for example, in case of an auto-scaling cluster
worker = baseutils.retry(ks_worker_get,
cluster_name,
worker.id,
interval=30,
retry=40)
if not worker or worker.state == 'deleted':
continue
k8s.drain(worker.private_ip)
if worker.kube_version == worker.target_version:
cmd = '/usr/local/bin/ibmcloud ks worker reload --cluster {cluster} --worker {worker} -f'.format(
cluster=baseutils.shell_escape(cluster_name),
worker=baseutils.shell_escape(worker.id))
elif 'pending' not in worker.kube_version: # pending will be in the version field if the update has already been triggered
cmd = '/usr/local/bin/ibmcloud ks worker update --cluster {cluster} --worker {worker} -f'.format(
cluster=baseutils.shell_escape(cluster_name),
worker=baseutils.shell_escape(worker.id))
try:
baseutils.retry(baseutils.exe_cmd, cmd, interval=30, retry=40)
except Exception as e:
exc_message = str(e)
if 'The specified worker has already been deleted' in exc_message:
continue
elif 'The worker node is already up to date with the latest version' not in exc_message:
# It's possible that a call to reload can return a failure from the api but still go through. Subsequent retries from the baseutils.retry
# function will then get this error message back. If the error is present, we can continue as normal.
raise
worker.state = 'reload_pending'
wait_for_worker(cluster_name, worker)
def read(path, property=None):
"""
Retrieves a value from the Vault. This is not specific to a secret engine backend and anything can be queried.
Vault secrets are returned as dictionaries. Only the actual value (data attribute of vault response) is returned, not the request metadata.
Args:
path: The path to the secret to retrieve. This must include the backend eg. secret/my/path
property: A sub-property of the secret (under the data attribute) to retrieve (Optional)
Returns: The Vault secret as a dictionary or None if the value does not exist
"""
result = baseutils.retry(client.read, path=path)
if result:
result = result.get('data', {})
if property:
result = result.get(property)
return result
def update_iks_cluster(cluster_name, kube_version):
"""
Updates the version of an IKS cluster.
Specifically, this updates the master nodes only.
The current version of the master servers is first checked and if they are already up to date, no action is taken.
Args:
cluster_name: The name of the cluster to update
kube_version: The version to upgrade Kubernetes to in the form major.minor.patch
"""
cluster = baseutils.retry(ks_cluster_get,
cluster_name,
interval=30,
retry=40)
upgrade_version = semver.parse_version_info(kube_version.split('_', 1)[0])
current_version = semver.parse_version_info(
cluster.master_kube_version.split('_', 1)[0])
if upgrade_version > current_version:
master_kube_version_prefix = '{version}_'.format(version=kube_version)
if not cluster.master_kube_version.startswith(
master_kube_version_prefix
) and 'pending' not in cluster.master_kube_version:
cmd = '/usr/local/bin/ibmcloud ks cluster master update --cluster {cluster} --kube-version {version} -f'.format(
cluster=baseutils.shell_escape(cluster_name),
version=baseutils.shell_escape(kube_version))
(rc, output) = baseutils.retry(baseutils.exe_cmd,
cmd,
interval=30,
retry=40)
while not cluster.master_kube_version.startswith(
master_kube_version_prefix
) or 'pending' in cluster.master_kube_version:
time.sleep(30)
cluster = baseutils.retry(ks_cluster_get,
cluster_name,
interval=30,
retry=40)
def get_hooks(release, resource_types=None, hook_types=None):
"""
Retrieves the Helm hooks for a deployed release as a list of resources.
Args:
release: The name of the release to retrieve the hooks for
resource_types: A list of resource types resource types to limit the returned hooks to (Optional)
hook_types: A list of hook types to limit the returned hooks to (Optional)
Returns: A list of Kubernetes resources
"""
cmd = '{helm} get hooks {release}'.format(helm=helm_binary, release=baseutils.shell_escape(release))
(rc, output) = baseutils.retry(baseutils.exe_cmd, cmd, log_level=logging.NOTSET, interval=10, retry=6) # Output can contain secrets so don't log
hooks = list(yaml.safe_load_all(output))
if resource_types:
hooks = [hook for hook in hooks if hook['kind'] in resource_types]
if hook_types:
hooks = [hook for hook in hooks if hook['metadata']['annotations']['helm.sh/hook'] in hook_types]
return hooks
def get_manifest(release, resource_type=None):
"""
Retrieves the Helm manifest for a deployed release as an array of resources.
Args:
release: The name of the release to retrieve manifest for
resource_type: Limits the returned resources to the specified resource type (Optional)
Returns: A list of manifest resources
"""
full_manifest = None
manifest = []
cmd = '{helm} get manifest {release}'.format(helm=helm_binary, release=baseutils.shell_escape(release))
(rc, output) = baseutils.retry(baseutils.exe_cmd, cmd, log_level=logging.NOTSET, interval=10, retry=6) # Output can contain secrets so don't log
full_manifest = yaml.safe_load_all(output)
for resource in full_manifest:
if resource and 'kind' in resource: # This implies the resource is a valid k8s manifest
if not resource_type or resource_type == resource['kind']:
manifest.append(resource)
return manifest
def kp_list(instance_id, name=None):
"""
List and return all keys in a Key Protect instance.
Args:
instance_id: The ID of the Key Protect instance to query
name: Filters the resultant key list to keys with a specified name (Optional)
Returns: A list of KPKey objects
"""
cmd = '/usr/local/bin/ibmcloud kp list --instance-id {instance_id} --output json'.format(
instance_id=baseutils.shell_escape(instance_id))
(rc, output) = baseutils.retry(baseutils.exe_cmd,
cmd,
interval=10,
retry=5)
keys = KPKey.parse_kp_keys(json.loads(output) or [])
if name:
keys = [key for key in keys if key.name == name]
return keys
def wait_for_worker(cluster_name, worker, seconds=5400):
"""
Waits until a worker becomes ready, typically due to a reload or new provision.
This function continually polls for the worker status and returns when the worker is Ready.
A worker can be scaled-out during the polling without issue.
The wait will time out after a specified time, resulting in an exception.
In addition to waiting for the worker, the function also waits for all kube-system pods scheduled to the node to be ready.
Args:
cluster_name: The name of the cluster that the worker belongs to
worker: The worker to wait on as an IKSWorker model
seconds: The timeout before failing a node (Optional, default: 5400)
"""
logger.info('Waiting for worker {worker_ip} to become ready'.format(
worker_ip=worker.private_ip))
with baseutils.timeout(seconds=seconds):
worker_still_exists = True # For tolerating a worker being scaled out during the reload
while worker_still_exists and (
worker.state != 'normal' or worker.status != 'Ready'
or worker.kube_version != worker.target_version
or worker.pending_operation):
time.sleep(30)
worker = baseutils.retry(ks_worker_get,
cluster_name,
worker.id,
interval=30,
retry=40)
worker_still_exists = worker and worker.state != 'deleted' and worker.state != 'provision_failed'
if worker_still_exists:
k8s.uncordon(
worker.private_ip
) # The reload should automatically uncordon but we have observed an issue where the uncordon sometimes doesn't run
logger.info(
'Worker {worker_ip} has completed reloading. Waiting for kube-system pods'
.format(worker_ip=worker.private_ip))
k8shelpers.wait_for_node_namespace_pods(worker.private_ip,
'kube-system')
logger.info('Worker {worker_ip} is ready'.format(
worker_ip=worker.private_ip))
else:
logger.info(
'Worker {worker_ip} was scaled out. Finished waiting'.format(
worker_ip=worker.private_ip))
def get(kind, namespace=None, name=None, labels=None):
"""
Retrieve one or more resources of a specific kind in Kubernetes.
An exception is thrown for invalid types or the name of a resource that does not exist.
Args:
kind: The kind of the resources, eg. deployment
namespace: The namespace of the resources. Setting to "all" triggers the flag --all-namespaces (Optional, default is as per kubecfg configuration)
name: The name of an individual resource (Optional, default: retrieve all)
labels: A label selector query to be passed to kubectl. Can either be a string of the form "label1=value1,labe2=value2" or a dictionary with "key: value" pairs (Optional)
Returns: List of dictionary resources. If name is specified, a single resource as a dictionary (this is actually defined by whether kubectl returns kind=List)
"""
if isinstance(labels, dict):
labels = ','.join('{key}={value}'.format(key=key, value=value)
for (key, value) in labels.items())
cmd = '{kubectl} get {kind} {name} {namespace} {labels} -o json'.format(
kubectl=kubectl_binary,
kind=baseutils.shell_escape(kind),
name=baseutils.shell_escape(name) if name else '',
namespace='--all-namespaces' if namespace == 'all' else
('-n {namespace}'.format(
namespace=baseutils.shell_escape(namespace)) if namespace else ''),
labels='-l {labels}'.format(
labels=baseutils.shell_escape(labels)) if labels else '')
(rc, output) = baseutils.retry(
baseutils.exe_cmd,
cmd,
log_level=logging.NOTSET if 'secret' in kind.lower() else logging.INFO,
interval=10,
retry=6)
try:
resources = json.loads(output)
except Exception:
# Sometimes the json is preceded with error messages but kubectl retries and completes. We will retry parsing after removing any error messages
output = output.splitlines()
while output and output[0].startswith('E'):
del (output[0])
output = '\n'.join(output)
resources = json.loads(output)
if resources.get('kind') == 'List':
resources = resources['items']
return resources
def kp_create(instance_id, key_name, key_material=None, standard_key=False):
"""
Create or update a key in Key Protect.
Args:
instance_id: The ID of the Key Protect instance to update
key_name: The name of the key to create
key_material: The base64 encoded value of the key (Optional, default: create a new key)
standard_key: Set True if this should be a standard key. Otherwise it will be a root key (Optional, default: False)
Returns: The ID of the new key
"""
cmd = '/usr/local/bin/ibmcloud kp create {key_name} --instance-id {instance_id} {key_material} {standard_key} --output json'.format(
key_name=baseutils.shell_escape(key_name),
instance_id=baseutils.shell_escape(instance_id),
key_material='--key-material {key_material}'.format(
key_material=baseutils.shell_escape(key_material))
if key_material else '',
standard_key='--standard-key' if standard_key else '')
(rc, output) = baseutils.retry(baseutils.exe_cmd,
cmd,
interval=10,
retry=5)
return KPKey(json.loads(output))
def sl_volume_detail(volume_id, volume_type):
"""Describe an instance of block volumes with given command
Args:
volume_id: The ID of the block volume
Returns: Returns a description of the block volume
"""
cmd = '/usr/local/bin/ibmcloud sl {volume_type} volume-detail {volume_id}'.format(
volume_type=baseutils.shell_escape(volume_type),
volume_id=baseutils.shell_escape(volume_id))
(rc, output) = baseutils.retry(baseutils.exe_cmd,
cmd,
interval=10,
retry=5)
output = output.split('\n')[1:]
volume_details = {}
for line in output:
if line:
line = re.split(r'\s\s+', line)
key = line[0].lower()
if 'active transactions' in key:
key = 'active_transactions'
else:
key = key.replace('(', '').replace(')', '').replace(' ', '_')
value = line[1]
volume_details[key] = value
volume = Volume(volume_type=volume_type)
volume.id = volume_details.get('id')
volume.name = volume_details.get('user_name')
volume.datacenter = volume_details.get('datacenter')
volume.storage_type = volume_details.get('type')
volume.capacity_gb = volume_details.get('capacity_gb')
volume.ip_addr = volume_details.get('target_ip')
volume.active_transactions = int(volume_details.get('active_transactions'))
volume.replicant_count = volume_details.get('replicant_count')
volume.bytes_used = None
volume.notes = None
return volume
def delete(path):
"""
Deletes a secret at a given path in Vault.
path: The path to delete the secrets at
"""
baseutils.retry(client.delete, path=path)
