def create_ingress(self, namespace: str, ingress: V1beta1Ingress):
'''Create an ingress in a given namespace. Raises an `ApiException` if such an ingress
already exists.'''
not_empty(namespace)
not_none(ingress)
self.extensions_v1beta1_api.create_namespaced_ingress(namespace=namespace, body=ingress)
def wait_until_deployment_available(self, namespace: str, name: str, timeout_seconds: int=60):
'''Block until the given deployment has at least one available replica (or timeout)
Return `True` if the deployment is available, `False` if a timeout occured.
'''
not_empty(namespace)
not_empty(name)
w = watch.Watch()
# Work around IncompleteRead errors resulting in ProtocolErrors - no fault of our own
start_time = int(time.time())
while (start_time + timeout_seconds) > time.time():
try:
for event in w.stream(
self.apps_api.list_namespaced_deployment,
namespace=namespace,
timeout_seconds=timeout_seconds
):
deployment_spec = event['object']
if deployment_spec is not None:
if deployment_spec.metadata.name == name:
if deployment_spec.status.available_replicas is not None \
and deployment_spec.status.available_replicas > 0:
return True
# Check explicitly if timeout occurred
if (start_time + timeout_seconds) < time.time():
return False
# Regular Watch.stream() timeout occurred, no need for further checks
return False
except ProtocolError:
info('http connection error - ignored')
def create_if_absent(self, namespace: str):
'''Create a new namespace iff it does not already exist'''
not_empty(namespace)
existing_namespace = self.get_namespace(namespace)
if not existing_namespace:
self.create_namespace(namespace)
def create_tls_secret(
tls_config: TlsConfig,
tls_secret_name: str,
namespace: str,
):
"""Creates the configured TLS secret for the Concourse web-component in the K8s cluster"""
not_none(tls_config)
not_empty(tls_secret_name)
not_empty(namespace)
ctx = kube_ctx
namespace_helper = ctx.namespace_helper()
namespace_helper.create_if_absent(namespace)
secret_helper = ctx.secret_helper()
if not secret_helper.get_secret(tls_secret_name, namespace):
data = {
'tls.key': tls_config.private_key(),
'tls.crt': tls_config.certificate(),
}
secret_helper.put_secret(
name=tls_secret_name,
data=data,
namespace=namespace,
)
def create_image_pull_secret(
credentials: GcrCredentials,
image_pull_secret_name: str,
namespace: str,
):
"""Create an image pull secret in the K8s cluster to allow pods to download images from gcr"""
not_none(credentials)
not_empty(image_pull_secret_name)
not_empty(namespace)
ctx = kube_ctx
namespace_helper = ctx.namespace_helper()
namespace_helper.create_if_absent(namespace)
secret_helper = ctx.secret_helper()
if not secret_helper.get_secret(image_pull_secret_name, namespace):
secret_helper.create_gcr_secret(
namespace=namespace,
name=image_pull_secret_name,
password=credentials.passwd(),
user_name=credentials.username(),
email=credentials.email(),
server_url=credentials.host(),
)
service_account_helper = ctx.service_account_helper()
service_account_helper.patch_image_pull_secret_into_service_account(
name="default",
namespace=namespace,
image_pull_secret_name=image_pull_secret_name
)
def _cfg_element_names(self, cfg_type_name: str):
'''Returns cfg-elements of the given cfg_type
Parameters
----------
cfg_type_name: str
The cfg type name
Returns
-------
Iterable[str]
Contains the names of all cfg-elements of the given cfg_type known to this ConfigFactory.
Raises
------
ValueError
If the specified cfg_type is unknown.
'''
not_empty(cfg_type_name)
known_types = self._cfg_types()
if cfg_type_name not in known_types:
raise ValueError(
"Unknown config type '{c}'. Known types: {k}".format(
c=cfg_type_name,
k=', '.join(known_types.keys()),
))
if cfg_type_name in self.raw:
return set(self.raw[cfg_type_name].keys())
else:
return set()
def create_deployment(self, namespace: str, deployment: V1Deployment):
'''Create a deployment in a given namespace. Raises an `ApiException` if such a deployment
already exists.'''
not_empty(namespace)
not_none(deployment)
self.apps_api.create_namespaced_deployment(namespace=namespace, body=deployment)
def create_tls_secret(tls_config: TlsConfig,
tls_secret_name: str,
namespace: str,
basic_auth_cred: BasicAuthCred = None):
""" Creates a secret with the configured TLS certificates in the K8s cluster.
Optionally adds credentials for Basic Authentication"""
not_none(tls_config)
not_empty(tls_secret_name)
not_empty(namespace)
ctx = kube_ctx
namespace_helper = ctx.namespace_helper()
namespace_helper.create_if_absent(namespace)
secret_helper = ctx.secret_helper()
if not secret_helper.get_secret(tls_secret_name, namespace):
data = {
'tls.key': tls_config.private_key(),
'tls.crt': tls_config.certificate(),
}
if basic_auth_cred:
ht = HtpasswdFile()
ht.set_password(basic_auth_cred.user, basic_auth_cred.password)
data['auth'] = ht.to_string().decode('utf-8')
secret_helper.put_secret(
name=tls_secret_name,
data=data,
namespace=namespace,
)
def create_service(self, namespace: str, service: V1Service):
'''Create a service in a given namespace. Raises an `ApiException` if such a Service
already exists.
'''
not_empty(namespace)
not_none(service)
self.core_api.create_namespaced_service(namespace=namespace, body=service)
def create_or_update_config_map(self, namespace: str, name: str, data: dict):
not_empty(namespace)
not_empty(name)
not_none(data)
if self.read_config_map(namespace=namespace, name=name):
self.replace_config_map(namespace=namespace, name=name, data=data)
else:
self.create_config_map(namespace=namespace, name=name, data=data)
def tag_exists(
self,
tag_name: str,
):
util.not_empty(tag_name)
try:
self.repository.ref('tags/' + tag_name)
return True
except NotFoundError:
return False
def __init__(
self,
github_config: GithubConfig,
repo_owner: str,
repo_name: str,
branch: str,
):
self._github_config = util.not_none(github_config)
self._repo_owner = util.not_empty(repo_owner)
self._repo_name = util.not_empty(repo_name)
self._branch = util.not_empty(branch)
def __init__(
self,
githubrepobranch: GitHubRepoBranch,
github_helper: GitHubRepositoryHelper,
release_version: str,
repo_dir: str,
):
self.githubrepobranch = not_none(githubrepobranch)
self.github_helper = not_none(github_helper)
self.release_version = not_empty(release_version)
self.repo_dir = os.path.abspath(not_empty(repo_dir))
def __init__(
self,
slack_cfg_name: str,
slack_channel: str,
release_version: str,
githubrepobranch: GitHubRepoBranch,
):
self.slack_cfg_name = not_empty(slack_cfg_name)
self.slack_channel = not_empty(slack_channel)
self.release_version = not_empty(release_version)
self.githubrepobranch = not_none(githubrepobranch)
def update_release_notes(
self,
tag_name: str,
body: str,
) -> bool:
util.not_empty(tag_name)
release = self.repository.release_from_tag(tag_name)
if not release:
raise RuntimeError(f"No release with tag '{tag_name}' found "
f"in repository {self.repository}")
return release.edit(body=body)
def _cfg_elements(self, cfg_type_name: str):
'''Returns all container cfg elements of the given cfg_type
Raises
------
ValueError
If the specified cfg_type is unknown.
'''
not_empty(cfg_type_name)
for element_name in self._cfg_element_names(cfg_type_name):
yield self._cfg_element(cfg_type_name, element_name)
def read_config_map(self, namespace: str, name: str):
'''Return the `V1ConfigMap` with the given name in the given namespace, or `None` if
no such config map exists.'''
not_empty(namespace)
not_empty(name)
try:
config_map = self.core_api.read_namespaced_config_map(namespace=namespace, name=name)
except ApiException as ae:
if ae.status == 404:
return None
raise ae
return config_map
def create_config_map(self, namespace: str, name: str, data: dict):
not_empty(namespace)
not_empty(name)
not_none(data)
self.core_api.create_namespaced_config_map(
namespace = namespace,
body = V1ConfigMap(
data=data,
metadata=V1ObjectMeta(name=name, namespace=namespace),
),
)
def test_not_empty(self):
result = examinee.not_empty('foo')
self.assertEqual('foo', result)
forbidden = ['', None, [], ()]
for value in forbidden:
with capture_out() as (stdout, stderr):
with self.assertRaises(Failure):
examinee.not_empty(value)
self.assertIn('must not be empty', stdout.getvalue().strip())
self.assertTrue(len(stderr.getvalue()) == 0)
def validate(self):
semver.parse(self.release_version)
# if a tag with the given release version exists, we cannot create another release
# pointing to it
if self.github_helper.tag_exists(tag_name=self.release_version):
raise RuntimeError(
f"Cannot create tag '{self.release_version}' for release: Tag already exists"
)
if self.component_descriptor_file_path:
existing_file(self.component_descriptor_file_path)
with open(self.component_descriptor_file_path) as f:
# TODO: Proper validation
not_empty(f.read().strip())
def get_deployment(self, namespace: str, name: str) -> V1Deployment:
'''Return the `V1Deployment` with the given name in the given namespace, or `None` if
no such deployment exists.'''
not_empty(namespace)
not_empty(name)
try:
deployment = self.apps_api.read_namespaced_deployment(name=name, namespace=namespace)
except ApiException as ae:
if ae.status == 404:
return None
raise ae
return deployment
def get_service(self, namespace: str, name: str) -> V1Service:
'''Return the `V1Service` with the given name in the given namespace, or `None` if
no such service exists.
'''
not_empty(namespace)
not_empty(name)
try:
service = self.core_api.read_namespaced_service(name=name, namespace=namespace)
except ApiException as ae:
if ae.status == 404:
return None
raise ae
return service
def send_mail(
email_cfg_name: CliHint(
help="reference to an email cfg (see repo cc-config / secrets-server)"
),
recipients: CliHint(typehint=[str], help="Recipient email address"),
mail_template_file: CliHints.existing_file(),
subject: CliHint(help="email subject"),
cc_recipients: CliHint(typehint=[str],
help="Carbon copy email address") = [],
replace_token: CliHint(typehint=[str],
help="<key>=<value> (replace <key> in body)") = [],
):
'''
Sends an email using the specified email_cfg (retrieved from a cfg_factory) to the specified
recipients. The mail body is read from a file. A simple token-replacement is done if
(optional) replace-tokens are given.
@param recipients: mail recipients (email addresses)
@param mail_template_file: path to the mail template file. Must exist.
@param subject: email subject
@param cc_recipients: cc mail recipients
@param replace_token: format: <token>=<replace-value> - tokens in mail-body are replaced
'''
not_empty(email_cfg_name)
cfg_factory = ctx().cfg_factory()
email_cfg = cfg_factory.email(email_cfg_name)
with open(mail_template_file) as f:
mail_template = f.read()
# validate template-tokens
invalid_tokens = filter(lambda t: not isinstance(t, str) or '=' not in t,
replace_token)
if len(list(invalid_tokens)) > 0:
fail('all replace-tokens must be of form <key>=<value>: ' +
' '.join(invalid_tokens))
# parse replace-tokens
replace_tokens = dict(map(lambda t: t.split('=', 1), replace_token))
_send_mail(
email_cfg=email_cfg,
recipients=recipients,
mail_template=mail_template,
subject=subject,
cc_recipients=cc_recipients,
replace_tokens=replace_tokens,
)
def list_pods(self, namespace: str, label_selector: str='', field_selector: str=''):
'''Find all pods matching given labels and/or fields in the given namespace'''
not_empty(namespace)
try:
pods = self.core_api.list_namespaced_pod(
namespace,
label_selector=label_selector,
field_selector=field_selector,
)
except ApiException as ae:
if ae.status == 404:
return None
raise ae
return pods
def _send_mail(
email_cfg: EmailConfig,
recipients: typing.Iterable[str],
mail_template: str,
subject: str,
replace_tokens: dict = {},
cc_recipients: typing.Iterable[str] = [],
mimetype='text',
):
not_none(email_cfg)
not_empty(recipients)
not_none(mail_template)
not_empty(subject)
# create body from template
mail_body = mailer.create_body(
mail_template=mail_template,
replace_tokens=replace_tokens,
)
recipients = set(map(str.lower, recipients))
cc_recipients = set(map(str.lower, cc_recipients))
# create mail envelope
mail = mailer.create_mail(
subject=subject,
sender=email_cfg.sender_name(),
recipients=recipients,
cc_recipients=cc_recipients,
text=mail_body,
mimetype=mimetype,
)
if email_cfg.use_tls():
smtp_server = smtplib.SMTP_SSL(email_cfg.smtp_host())
else:
smtp_server = smtplib.SMTP(email_cfg.smtp_host())
credentials = email_cfg.credentials()
smtp_server.login(user=credentials.username(),
password=credentials.passwd())
recipients.update(cc_recipients)
mailer.send_mail(smtp_server=smtp_server,
msg=mail,
sender=credentials.username(),
recipients=recipients)
def __init__(
self,
github_helper: GitHubRepositoryHelper,
release_version: str,
):
self.github_helper = not_none(github_helper)
self.release_version = not_empty(release_version)
def replace_or_create_ingress(self, namespace: str, ingress: V1beta1Ingress):
'''Create an ingress in a given namespace. If the ingress already exists,
the previous version will be deleted beforehand.
'''
not_empty(namespace)
not_none(ingress)
ingress_name = ingress.metadata.name
existing_ingress = self.get_ingress(namespace=namespace, name=ingress_name)
if existing_ingress:
self.extensions_v1beta1_api.delete_namespaced_ingress(
namespace=namespace,
name=ingress_name,
body=kubernetes.client.V1DeleteOptions()
)
self.create_ingress(namespace=namespace, ingress=ingress)
def get_ingress(self, namespace: str, name: str) -> V1beta1Ingress:
'''Return the `V1beta1Ingress` with the given name in the given namespace, or `None` if
no such ingress exists.'''
not_empty(namespace)
not_empty(name)
try:
ingress = self.extensions_v1beta1_api.read_namespaced_ingress(
name=name,
namespace=namespace
)
except ApiException as ae:
if ae.status == 404:
return None
raise ae
return ingress
def delete_pod(self, name: str, namespace: str, grace_period_seconds: int=0):
'''Delete a pod in the given namespace.
grace_period_seconds: the duration in seconds before the object should be deleted.
Value must be non-negative integer. The value zero indicates delete immediately.
'''
not_empty(namespace)
not_empty(name)
body = kubernetes.client.V1DeleteOptions()
try:
self.core_api.delete_namespaced_pod(
name, namespace, body=body, grace_period_seconds=grace_period_seconds
)
except ApiException as ae:
if ae.status == 404:
return None
raise ae
def replace_or_create_deployment(self, namespace: str, deployment: V1Deployment):
'''Create a deployment in a given namespace. If the deployment already exists,
the previous version will be deleted beforehand.
'''
not_empty(namespace)
not_none(deployment)
deployment_name = deployment.metadata.name
existing_deployment = self.get_deployment(namespace=namespace, name=deployment_name)
if existing_deployment:
self.apps_api.delete_namespaced_deployment(
namespace=namespace,
name=deployment_name,
body=kubernetes.client.V1DeleteOptions()
)
self.create_deployment(namespace=namespace, deployment=deployment)
def benchmark_local_home_dir(self):
return not_empty(self._conf['benchmark_local_dir'], self._CURRENT_DIR)
def benchmark_remote_logs_dir(self):
remote_logs_dir = not_empty(self._conf['benchmark_remote_logs_dir'], self._CURRENT_DIR)
return path(self.benchmark_remote_home_dir, remote_logs_dir, self.db_profile)
def workload_java_parameters(self):
return not_empty(self.workload_parameters.get('java_properties'), [])