def reset_kubeadm():
"""Uninstall kubernetes on a node.
Runs `kubeadm reset` on the specified machine in order to remove the
kubernetes services and undo all configuration set by `kubeadm init`.
"""
# Get script from path.
script = os.path.join(os.path.dirname(__file__), 'mega-reset.sh')
ctx.download_resource(
os.path.join('scripts', 'mega-reset.sh'), script
)
# Get worker.
conn = MistConnectionClient()
machine = conn.get_machine(
cloud_id=ctx.instance.runtime_properties['cloud_id'],
machine_id=ctx.instance.runtime_properties['machine_id'],
)
ctx.logger.info('Running "kubeadm reset" on %s', machine)
_add_run_remove_script(
cloud_id=machine.cloud.id,
machine_id=machine.id,
script_path=os.path.abspath(script),
script_name='kubeadm_reset_%s' % random_string(length=4)
)
def configure_kubernetes_master():
"""Configure the kubernetes master.
Sets up the master node and stores the necessary settings inside the node
instance's runtime properties, which are required by worker nodes in order
to join the kubernetes cluster.
"""
ctx.logger.info('Setting up kubernetes master node')
prepare_kubernetes_script()
conn = MistConnectionClient()
machine = conn.get_machine(
cloud_id=ctx.instance.runtime_properties['cloud_id'],
machine_id=ctx.instance.runtime_properties['machine_id'],
)
# Token for secure master-worker communication.
token = '%s.%s' % (random_string(length=6), random_string(length=16))
ctx.instance.runtime_properties['master_token'] = token.lower()
# Store kubernetes dashboard credentials in runtime properties.
ctx.instance.runtime_properties.update({
'auth_user':
ctx.node.properties['auth_user'],
'auth_pass':
ctx.node.properties['auth_pass'] or random_string(10),
})
ctx.logger.info('Installing kubernetes on master node')
# Prepare script parameters.
params = "-u '%s' " % ctx.instance.runtime_properties['auth_user']
params += "-p '%s' " % ctx.instance.runtime_properties['auth_pass']
params += "-t '%s' " % ctx.instance.runtime_properties['master_token']
params += "-r 'master'"
# Run the script.
script = conn.client.run_script(
script_id=ctx.instance.runtime_properties['script_id'],
su=True,
machine_id=machine.id,
cloud_id=machine.cloud.id,
script_params=params,
)
ctx.instance.runtime_properties['job_id'] = script['job_id']
def wait_for_event(job_id, job_kwargs, timeout=1800):
"""Wait for an event to take place.
This method enters a loop, while waiting for a specific event to
take place.
This method polls the workflow's logs over the mist.io API in order
to decide whether the specified event has finally occurred.
Parameters:
job_id: the UUID of the job that includes the desired event
job_kwargs: a dict of key-value pairs that must match the event
timeout: seconds to wait for, before raising an exception
This method will either exit with an exit code 0, if the desired
event occurs within a given timeframe, otherwise it will raise a
non-recoverable error.
"""
ctx.logger.info('Waiting for event %s with kwargs=%s', job_id, job_kwargs)
# FIXME Imported here due to circular dependency issues.
from plugin.connection import MistConnectionClient
conn = MistConnectionClient()
# Mark the beginning of the polling period.
started_at = time.time()
timeout_at = started_at + timeout
# Wait for newly indexed events to become available/searchable.
for _ in range(30):
try:
conn.client.get_job(job_id)
except Exception as exc:
ctx.logger.debug('Failed to get logs of %s: %r', job_id, exc)
time.sleep(1)
else:
break
else:
raise
# Poll for the event with the specified key-value pairs.
while True:
for log in conn.client.get_job(job_id).get('logs', []):
if all([log.get(k) == v for k, v in job_kwargs.iteritems()]):
if log.get('error'):
msg = log.get('stdout', '') + log.get('extra_output', '')
msg = msg or log['error']
ctx.logger.error(msg)
raise NonRecoverableError('Error in event %s' % job_id)
return log
if time.time() > timeout_at:
raise NonRecoverableError('Time threshold exceeded!')
time.sleep(10)
def configure_kubernetes_master():
"""Configure the kubernetes master.
Sets up the master node and stores the necessary settings inside the node
instance's runtime properties, which are required by worker nodes in order
to join the kubernetes cluster.
"""
ctx.logger.info('Setting up kubernetes master node')
prepare_kubernetes_script()
conn = MistConnectionClient()
machine = conn.get_machine(
cloud_id=ctx.instance.runtime_properties['cloud_id'],
machine_id=ctx.instance.runtime_properties['machine_id'],
)
# Token for secure master-worker communication.
token = '%s.%s' % (random_string(length=6), random_string(length=16))
ctx.instance.runtime_properties['master_token'] = token.lower()
# Store kubernetes dashboard credentials in runtime properties.
ctx.instance.runtime_properties.update({
'auth_user': ctx.node.properties['auth_user'],
'auth_pass': ctx.node.properties['auth_pass'] or random_string(10),
})
ctx.logger.info('Installing kubernetes on master node')
# Prepare script parameters.
params = "-u '%s' " % ctx.instance.runtime_properties['auth_user']
params += "-p '%s' " % ctx.instance.runtime_properties['auth_pass']
params += "-t '%s' " % ctx.instance.runtime_properties['master_token']
params += "-r 'master'"
# Run the script.
script = conn.client.run_script(
script_id=ctx.instance.runtime_properties['script_id'], su=True,
machine_id=machine.id,
cloud_id=machine.cloud.id,
script_params=params,
)
ctx.instance.runtime_properties['job_id'] = script['job_id']
def configure_kubernetes_worker():
"""Configure a new kubernetes node.
Configures a new worker node and connects it to the kubernetes master.
"""
# Get master node from relationships schema.
master = ctx.instance.relationships[0]._target.instance
ctx.instance.runtime_properties.update({
'script_id':
master.runtime_properties.get('script_id', ''),
'master_ip':
master.runtime_properties.get('master_ip', ''),
'master_token':
master.runtime_properties.get('master_token', ''),
})
ctx.logger.info('Setting up kubernetes worker')
prepare_kubernetes_script()
conn = MistConnectionClient()
machine = conn.get_machine(
cloud_id=ctx.instance.runtime_properties['cloud_id'],
machine_id=ctx.instance.runtime_properties['machine_id'],
)
ctx.logger.info('Configuring kubernetes node')
# Prepare script parameters.
params = "-m '%s' " % ctx.instance.runtime_properties['master_ip']
params += "-t '%s' " % ctx.instance.runtime_properties['master_token']
params += "-r 'node'"
# Run the script.
script = conn.client.run_script(
script_id=ctx.instance.runtime_properties['script_id'],
su=True,
machine_id=machine.id,
cloud_id=machine.cloud.id,
script_params=params,
)
ctx.instance.runtime_properties['job_id'] = script['job_id']
def drain_and_remove():
"""Mark the node as unschedulable, evict all pods, and remove it.
Runs `kubectl drain` and `kubectl delete nodes` on the kubernetes
master in order to drain and afterwards remove the specified node
from the cluster.
"""
if ctx.node.properties['master']: # FIXME Is this necessary?
return
# Get master instance.
master = ctx.instance.relationships[0]._target.instance
# Render script.
script = os.path.join(os.path.dirname(__file__), 'drain-node.sh')
ctx.download_resource_and_render(
os.path.join('scripts', 'drain-node.sh'), script,
template_variables={
'server_ip': master.runtime_properties.get('server_ip',''),
'auth_user': master.runtime_properties['auth_user'],
'auth_pass': master.runtime_properties['auth_pass'],
'hostname': ctx.instance.runtime_properties.get('machine_name', '').lower()
},
)
conn = MistConnectionClient()
machine = conn.get_machine(
cloud_id=master.runtime_properties['cloud_id'],
machine_id=master.runtime_properties['machine_id'],
)
ctx.logger.info('Running "kubectl drain && kubectl delete" on %s', machine)
_add_run_remove_script(
cloud_id=machine.cloud.id,
machine_id=machine.id,
script_path=os.path.abspath(script),
script_name='kubectl_drain_%s' % random_string(length=4)
)
def configure_kubernetes_worker():
"""Configure a new kubernetes node.
Configures a new worker node and connects it to the kubernetes master.
"""
# Get master node from relationships schema.
master = ctx.instance.relationships[0]._target.instance
ctx.instance.runtime_properties.update({
'script_id': master.runtime_properties.get('script_id', ''),
'master_ip': master.runtime_properties.get('master_ip', ''),
'master_token': master.runtime_properties.get('master_token', ''),
})
ctx.logger.info('Setting up kubernetes worker')
prepare_kubernetes_script()
conn = MistConnectionClient()
machine = conn.get_machine(
cloud_id=ctx.instance.runtime_properties['cloud_id'],
machine_id=ctx.instance.runtime_properties['machine_id'],
)
ctx.logger.info('Configuring kubernetes node')
# Prepare script parameters.
params = "-m '%s' " % ctx.instance.runtime_properties['master_ip']
params += "-t '%s' " % ctx.instance.runtime_properties['master_token']
params += "-r 'node'"
# Run the script.
script = conn.client.run_script(
script_id=ctx.instance.runtime_properties['script_id'], su=True,
machine_id=machine.id,
cloud_id=machine.cloud.id,
script_params=params,
)
ctx.instance.runtime_properties['job_id'] = script['job_id']
def prepare_kubernetes_script():
"""Upload kubernetes installation script, if missing.
This method is executed at the very beginning, in a pre-configuration
phase, to make sure that the kubernetes installation script has been
uploaded to mist.io.
This method is meant to be invoked early on by:
configure_kubernetes_master()
configure_kubernetes_worker()
The script_id inside each instance's runtime properties is used later
on in order to configure kubernetes on the provisioned machines.
"""
if ctx.instance.runtime_properties.get('script_id'):
ctx.logger.info('Kubernetes installation script already exists')
else:
ctx.logger.info('Uploading fresh kubernetes installation script')
# If a script_id does not exist in the node instance's runtime
# properties, perhaps because this is the first node that is being
# configured, load the script from file, upload it to mist.io, and
# run it over ssh.
client = MistConnectionClient().client
script = os.path.join(os.path.dirname(__file__), 'mega-deploy.sh')
ctx.download_resource(
os.path.join('scripts', 'mega-deploy.sh'), script
)
with open(os.path.abspath(script)) as fobj:
script = fobj.read()
script = client.add_script(
name='install_kubernetes_%s' % random_string(length=4),
script=script, location_type='inline', exec_type='executable'
)
ctx.instance.runtime_properties['script_id'] = script['id']
def prepare_kubernetes_script():
"""Upload kubernetes installation script, if missing.
This method is executed at the very beginning, in a pre-configuration
phase, to make sure that the kubernetes installation script has been
uploaded to mist.io.
This method is meant to be invoked early on by:
configure_kubernetes_master()
configure_kubernetes_worker()
The script_id inside each instance's runtime properties is used later
on in order to configure kubernetes on the provisioned machines.
"""
if ctx.instance.runtime_properties.get('script_id'):
ctx.logger.info('Kubernetes installation script already exists')
else:
ctx.logger.info('Uploading fresh kubernetes installation script')
# If a script_id does not exist in the node instance's runtime
# properties, perhaps because this is the first node that is being
# configured, load the script from file, upload it to mist.io, and
# run it over ssh.
client = MistConnectionClient().client
script = os.path.join(os.path.dirname(__file__), 'mega-deploy.sh')
ctx.download_resource(os.path.join('scripts', 'mega-deploy.sh'),
script)
with open(os.path.abspath(script)) as fobj:
script = fobj.read()
script = client.add_script(name='install_kubernetes_%s' %
random_string(length=4),
script=script,
location_type='inline',
exec_type='executable')
ctx.instance.runtime_properties['script_id'] = script['id']
def remove_kubernetes_script():
"""Attempt to remove the kubernetes installation script.
This method tries to remove the already uploaded installation script after
each kubernetes node has been provisioned to prevent multiple scripts from
accumulating in the user's account.
If an error is raised, it's logged and the workflow execution is carried
on.
"""
# FIXME Perhaps, scrtipt should not be handled here or this way. The
# cloudify-mist plugin should define a `Script` node type to execute
# operations on scripts, such as uploading, deleting, etc.
script_id = ctx.instance.runtime_properties.pop('script_id', '')
if script_id:
try:
MistConnectionClient().client.remove_script(script_id)
except Exception as exc:
ctx.logger.warn('Failed to remove installation script: %r', exc)
def _add_run_remove_script(cloud_id, machine_id, script_path, script_name):
"""Helper method to add a script, run it, and, finally, remove it."""
conn = MistConnectionClient()
# Upload script.
with open(script_path) as fobj:
script = conn.client.add_script(
name=script_name, script=fobj.read(),
location_type='inline', exec_type='executable'
)
# Run the script.
job = conn.client.run_script(script_id=script['id'], machine_id=machine_id,
cloud_id=cloud_id)
# Wait for the script to exit. The script should exit fairly quickly,
# thus we only wait for a couple of minutes for the corresponding log
# entry.
try:
wait_for_event(
job_id=job['job_id'],
job_kwargs={
'action': 'script_finished',
'machine_id': machine_id,
},
timeout=180
)
except Exception:
ctx.logger.warn('Script %s finished with errors!', script_name)
else:
ctx.logger.info('Script %s finished successfully', script_name)
# Remove the script.
try:
conn.client.remove_script(script['id'])
except Exception as exc:
ctx.logger.warn('Failed to remove script %s: %r', script_name, exc)
params += "-t '%s' " % ctx.instance.runtime_properties['master_token']
params += "-r 'node'"
# Run the script.
script = conn.client.run_script(
script_id=ctx.instance.runtime_properties['script_id'], su=True,
machine_id=machine.id,
cloud_id=machine.cloud.id,
script_params=params,
)
ctx.instance.runtime_properties['job_id'] = script['job_id']
if __name__ == '__main__':
"""Setup kubernetes on the machines defined by the blueprint."""
conn = MistConnectionClient()
cloud = conn.get_cloud(ctx.instance.runtime_properties['cloud_id'])
if cloud.provider in constants.CLOUD_INIT_PROVIDERS:
wait_for_event(
job_id=ctx.instance.runtime_properties['job_id'],
job_kwargs={
'action': 'cloud_init_finished',
'machine_name': ctx.instance.runtime_properties['machine_name']
}
)
elif not ctx.node.properties['configured']:
if not ctx.node.properties['master']:
configure_kubernetes_worker()
else:
configure_kubernetes_master()
try:
if __name__ == '__main__':
"""Create the nodes on which to install kubernetes.
Besides creating the nodes, this method also decides the way kubernetes
will be configured on each of the nodes.
The legacy way is to upload the script and execute it over SSH. However,
if the cloud provider supports cloud-init, a cloud-config can be used as
a wrapper around the actual script. In this case, the `configure` lifecycle
operation of the blueprint is mostly skipped. More precisely, it just waits
to be signalled regarding cloud-init's result and exits immediately without
performing any additional actions.
"""
conn = MistConnectionClient()
ctx.instance.runtime_properties['job_id'] = conn.job_id
# Create a copy of the node's immutable properties in order to update them.
node_properties = ctx.node.properties.copy()
# Override the node's properties with parameters passed from workflows.
for key in params:
if key in constants.INSTANCE_REQUIRED_PROPERTIES + ('machine_id', ):
node_properties['parameters'][key] = params[key]
ctx.logger.info('Added %s=%s to node parameters', key, params[key])
# Generate a somewhat random machine name. NOTE that we need the name at
# this early point in order to be passed into cloud-init, if used, so that
# we may use it later on to match log entries.
name = generate_name(
# Run the script.
script = client.run_script(
script_id=ctx.instance.runtime_properties['script_id'],
su=True,
machine_id=machine.id,
cloud_id=machine.cloud.id,
script_params=script_params,
)
ctx.instance.runtime_properties['job_id'] = script['job_id']
if __name__ == '__main__':
"""Setup kubernetes on the machines defined by the blueprint."""
# FIXME Re-think this.
if MistConnectionClient().cloud.provider in constants.CLOUD_INIT_PROVIDERS:
wait_for_event(job_id=ctx.instance.runtime_properties['job_id'],
job_kwargs={
'action':
'cloud_init_finished',
'machine_name':
ctx.instance.runtime_properties['machine_name']
})
elif not ctx.node.properties['configured']:
if not ctx.node.properties['master']:
configure_kubernetes_worker()
else:
configure_kubernetes_master()
try:
wait_for_event(job_id=ctx.instance.runtime_properties['job_id'],
job_kwargs={
if __name__ == '__main__':
"""Create the nodes on which to install kubernetes.
Besides creating the nodes, this method also decides the way kubernetes
will be configured on each of the nodes.
The legacy way is to upload the script and execute it over SSH. However,
if the cloud provider supports cloud-init, a cloud-config can be used as
a wrapper around the actual script. In this case, the `configure` lifecycle
operation of the blueprint is mostly skipped. More precisely, it just waits
to be signalled regarding cloud-init's result and exits immediately without
performing any additional actions.
"""
# FIXME Re-think this.
conn = MistConnectionClient()
ctx.instance.runtime_properties['job_id'] = conn.client.job_id
# Create a copy of the node's immutable properties in order to update them.
node_properties = ctx.node.properties.copy()
# Override the node's properties with parameters passed from workflows.
for key in params:
if key in node_properties['parameters']:
node_properties['parameters'][key] = params[key]
ctx.logger.info('Added %s=%s to node properties', key, params[key])
# Generate a somewhat random machine name. NOTE that we need the name at
# this early point in order to be passed into cloud-init, if used, so that
# we may use it later on to match log entries.
name = generate_name(
params += "-r 'node'"
# Run the script.
script = conn.client.run_script(
script_id=ctx.instance.runtime_properties['script_id'],
su=True,
machine_id=machine.id,
cloud_id=machine.cloud.id,
script_params=params,
)
ctx.instance.runtime_properties['job_id'] = script['job_id']
if __name__ == '__main__':
"""Setup kubernetes on the machines defined by the blueprint."""
conn = MistConnectionClient()
cloud = conn.get_cloud(ctx.instance.runtime_properties['cloud_id'])
if cloud.provider in constants.CLOUD_INIT_PROVIDERS:
wait_for_event(job_id=ctx.instance.runtime_properties['job_id'],
job_kwargs={
'action':
'cloud_init_finished',
'machine_name':
ctx.instance.runtime_properties['machine_name']
})
elif not ctx.node.properties['configured']:
if not ctx.node.properties['master']:
configure_kubernetes_worker()
else:
configure_kubernetes_master()
try:
