def launch_appliance(name,
cloud_version_config,
credentials,
app_config,
user_data,
task_id=None):
"""Call the appropriate app handler and initiate the app launch process."""
try:
LOG.debug("Launching appliance %s", name)
handler = util.import_class(
cloud_version_config.application_version.backend_component_name)()
provider = domain_model.get_cloud_provider(cloud_version_config.cloud,
credentials)
cloud_config = util.serialize_cloud_config(cloud_version_config)
launch_result = handler.launch_app(provider, Task(launch_appliance),
name, cloud_config, app_config,
user_data)
# Schedule a task to migrate result one hour from now
migrate_launch_task.apply_async([launch_appliance.request.id],
countdown=3600)
return launch_result
except SoftTimeLimitExceeded:
launch_appliance.update_state(state="FAILURE",
meta={
"exc_message":
"Task time limit exceeded; "
"stopping the task."
})
raise Ignore # This keeps the custom state set above
def launch_app(self, task, name, cloud_version_config, credentials, app_config, user_data):
cloudlaunch_config = app_config.get("config_cloudlaunch", {})
provider = domain_model.get_cloud_provider(cloud_version_config.cloud, credentials)
img = provider.compute.images.get(cloud_version_config.image.image_id)
task.update_state(state='PROGRESSING', meta={'action': "Retrieving or creating a keypair"})
kp = self._get_or_create_kp(provider, cloudlaunch_config.get('keyPair') or 'cloudlaunch_key_pair')
task.update_state(state='PROGRESSING', meta={'action': "Applying firewall settings"})
sg = self.apply_app_firewall_settings(
provider, cloudlaunch_config)
cb_launch_config = self._get_cb_launch_config(provider, cloudlaunch_config)
inst_type = cloudlaunch_config.get(
'instanceType', cloud_version_config.default_instance_type)
placement_zone = cloudlaunch_config.get('placementZone')
ud = yaml.dump(user_data, default_flow_style=False, allow_unicode=False)
print("Launching with ud:\n%s" % (ud,))
task.update_state(state='PROGRESSING', meta={'action': "Launching an instance of type %s with keypair %s in zone %s" %
(inst_type, kp.name, placement_zone)})
inst = provider.compute.instances.create(name=name, image=img,
instance_type=inst_type, key_pair=kp, security_groups=[sg],
zone = placement_zone, user_data=ud, launch_config=cb_launch_config)
task.update_state(state='PROGRESSING', meta={'action': "Waiting for instance %s to be ready.." % (inst.id, )})
inst.wait_till_ready()
results = {}
results['keyPair'] = { 'id' : kp.id, 'name' : kp.name, 'material' : kp.material }
results['securityGroup'] = {'id' : sg.id, 'name' : sg.name }
results['publicIP'] = inst.public_ips[0]
task.update_state(state='PROGRESSING', meta={'action': "Launch successful. Public IP %s" % (inst.public_ips[0], )})
return {'cloudLaunch' : results }
def manage_appliance(self, action, deployment, credentials):
"""
Perform supplied action on this app.
@type action: ``str``
@param action: Accepted values are ``restart`` or ``delete``.
"""
LOG.debug("Performing %s on deployment %s", action, deployment.name)
handler = _get_app_handler(deployment)
dpl = _serialize_deployment(deployment)
provider = domain_model.get_cloud_provider(deployment.target_cloud,
credentials)
if action.lower() == 'restart':
result = handler.restart(provider, dpl)
elif action.lower() == 'delete':
result = handler.delete(provider, dpl)
else:
LOG.error(
"Unrecognized action: %s. Acceptable values are 'delete' "
"or 'restart'", action)
return None
# Schedule a task to migrate results right after task completion
# Do this as a separate task because until this task completes, we
# cannot obtain final status or traceback.
migrate_task_result.apply_async([self.request.id], countdown=1)
return result
def get_cloud_provider(view, cloud_id = None):
"""
Returns a cloud provider for the current user. The relevant
cloud is discovered from the view and the credentials are retrieved
from the request or user profile. Return ``None`` if no credentials were
retrieved.
"""
cloud_pk = cloud_id or view.kwargs.get("cloud_pk")
cloud = models.Cloud.objects.filter(
slug=cloud_pk).select_subclasses().first()
request_creds = get_credentials(cloud, view.request)
return domain_model.get_cloud_provider(cloud, request_creds)
def get_cloud_provider(view, cloud_id = None):
"""
Returns a cloud provider for the current user. The relevant
cloud is discovered from the view and the credentials are retrieved
from the request or user profile. Return ``None`` if no credentials were
retrieved.
"""
cloud_pk = cloud_id or view.kwargs.get("cloud_pk")
cloud = models.Cloud.objects.filter(
slug=cloud_pk).select_subclasses().first()
request_creds = get_credentials(cloud, view.request)
return domain_model.get_cloud_provider(cloud, request_creds)
def launch_app(self, task, name, cloud_version_config, credentials,
app_config, user_data):
"""
Handle the app launch process.
This will:
- Perform necessary checks and env setup, most notably create a new
key pair.
- Launch an instance and wait until ssh access can be established
- Run an Ansible playbook to configure the instance
"""
# Implicitly create a new KP for this instance
# Note that this relies on the baseVMApp implementation!
kp_name = "CL-" + "".join(
[c for c in name if c.isalpha() or c.isdigit()]).rstrip()
app_config['config_cloudlaunch']['keyPair'] = kp_name
# Launch an instance and check ssh connectivity
result = super(CloudMan2AppPlugin,
self).launch_app(task,
name,
cloud_version_config,
credentials,
app_config,
user_data=None)
provider = domain_model.get_cloud_provider(cloud_version_config.cloud,
credentials)
inst = provider.compute.instances.get(
result.get('cloudLaunch', {}).get('instance', {}).get('id'))
pk = result.get('cloudLaunch', {}).get('keyPair', {}).get('material')
timeout = 0
while (not self._check_ssh(inst.public_ips[0], pk=pk)
or timeout > 200):
log.info("Waiting for ssh on {0}...".format(inst.name))
time.sleep(5)
timeout += 5
# Configure the instance
task.update_state(
state='PROGRESSING',
meta={'action': 'Configuring container cluster manager.'})
self._run_playbook(inst.public_ips[0], pk)
result['cloudLaunch']['applicationURL'] = \
'http://{0}:8080/'.format(result['cloudLaunch']['publicIP'])
task.update_state(state='PROGRESSING',
meta={
'action':
"Waiting for CloudMan to become ready at %s" %
result['cloudLaunch']['applicationURL']
})
self.wait_for_http(result['cloudLaunch']['applicationURL'])
return result
def health_check(self, deployment, credentials):
"""
Check the health of the supplied deployment.
Conceptually, the health check can be as elaborate as the deployed
appliance supports via a custom implementation. At the minimum, and
by default, the health reflects the status of the cloud instance by
querying the cloud provider.
"""
LOG.debug("Checking health of deployment %s", deployment.name)
handler = _get_app_handler(deployment)
dpl = _serialize_deployment(deployment)
provider = domain_model.get_cloud_provider(deployment.target_cloud,
credentials)
result = handler.health_check(provider, dpl)
# We only keep the two most recent health check task results so delete
# any older ones
signals.health_check.send(sender=None, deployment=deployment)
# Schedule a task to migrate results right after task completion
# Do this as a separate task because until this task completes, we
# cannot obtain final status or traceback.
migrate_task_result.apply_async([self.request.id], countdown=1)
return result
def launch_app(self, task, name, cloud_version_config, credentials,
app_config, user_data):
"""Initiate the app launch process."""
cloudlaunch_config = app_config.get("config_cloudlaunch", {})
provider = domain_model.get_cloud_provider(cloud_version_config.cloud,
credentials)
custom_image_id = cloudlaunch_config.get("customImageID", None)
img = provider.compute.images.get(
custom_image_id or cloud_version_config.image.image_id)
task.update_state(state='PROGRESSING',
meta={'action': "Retrieving or creating a key pair"})
kp = self._get_or_create_kp(
provider,
cloudlaunch_config.get('keyPair') or 'cloudlaunch_key_pair')
task.update_state(state='PROGRESSING',
meta={'action': "Applying firewall settings"})
subnet_id, placement_zone, sgs = self.resolve_launch_properties(
provider, cloudlaunch_config)
cb_launch_config = self._get_cb_launch_config(provider, img,
cloudlaunch_config)
inst_type = cloudlaunch_config.get(
'instanceType', cloud_version_config.default_instance_type)
log.debug("Launching with subnet %s and SGs %s" % (subnet_id, sgs))
log.info("Launching base_vm with UD:\n%s" % user_data)
task.update_state(state='PROGRESSING',
meta={
'action':
"Launching an instance of type %s "
"with keypair %s in zone %s" %
(inst_type, kp.name, placement_zone)
})
inst = provider.compute.instances.create(
name=name,
image=img,
instance_type=inst_type,
subnet=subnet_id,
key_pair=kp,
security_groups=sgs,
zone=placement_zone,
user_data=user_data,
launch_config=cb_launch_config)
task.update_state(state='PROGRESSING',
meta={'action': "Waiting for instance %s" % inst.id})
log.debug("Waiting for instance {0} to be ready...".format(inst.id))
inst.wait_till_ready()
static_ip = cloudlaunch_config.get('staticIP')
if static_ip:
task.update_state(state='PROGRESSING',
meta={
'action':
"Assigning requested floating "
"IP: %s" % static_ip
})
inst.add_floating_ip(static_ip)
inst.refresh()
results = {}
results['keyPair'] = {
'id': kp.id,
'name': kp.name,
'material': kp.material
}
# FIXME: this does not account for multiple SGs and expects one
results['securityGroup'] = {'id': sgs[0].id, 'name': sgs[0].name}
results['instance'] = {'id': inst.id}
results['publicIP'] = self.attach_public_ip(provider, inst)
task.update_state(state='PROGRESSING',
meta={
"action":
"Instance created successfully. " +
"Public IP: %s" % results['publicIP']
if results['publicIP'] else ""
})
if self.base_app:
if results['publicIP']:
results['applicationURL'] = 'http://%s/' % results['publicIP']
task.update_state(
state='PROGRESSING',
meta={
'action':
"Waiting for application to become ready "
"at %s" % results['applicationURL']
})
self.wait_for_http(results['applicationURL'])
else:
results['applicationURL'] = 'N/A'
return {'cloudLaunch': results}
def process_app_config(name, cloud_version_config, credentials, app_config):
cloudman_config = get_required_val(
app_config, "config_cloudman", "CloudMan configuration data must be provided.")
user_data = {}
user_data['bucket_default'] = get_required_val(
cloudman_config, "defaultBucket", "default bucket is required.")
user_data['cluster_name'] = name
user_data['password'] = get_required_val(
cloudman_config, "clusterPassword", "cluster name is required.")
user_data['initial_cluster_type'] = get_required_val(
cloudman_config, "clusterType", "cluster type is required.")
user_data['cluster_storage_type'] = get_required_val(
cloudman_config, "storageType", "storage type is required.")
user_data['storage_size'] = cloudman_config.get("storageSize")
user_data['post_start_script_url'] = cloudman_config.get(
"masterPostStartScript")
user_data['worker_post_start_script_url'] = cloudman_config.get(
"workerPostStartScript")
if cloudman_config.get("clusterSharedString"):
user_data['share_string'] = cloudman_config.get("clusterSharedString")
user_data['cluster_templates'] = cloudman_config.get(
"cluster_templates")
cloud = cloud_version_config.cloud
if hasattr(cloud, 'aws'):
user_data['cloud_type'] = 'ec2'
user_data['region_name'] = cloud.aws.compute.ec2_region_name
user_data['region_endpoint'] = cloud.aws.compute.ec2_region_endpoint
user_data['ec2_port'] = cloud.aws.compute.ec2_port
user_data['ec2_conn_path'] = cloud.aws.compute.ec2_conn_path
user_data['is_secure'] = cloud.aws.compute.ec2_is_secure
user_data['s3_host'] = cloud.aws.object_store.s3_host
user_data['s3_port'] = cloud.aws.object_store.s3_port
user_data['s3_conn_path'] = cloud.aws.object_store.s3_conn_path
user_data['access_key'] = credentials.get('aws_access_key')
user_data['secret_key'] = credentials.get('aws_secret_key')
elif hasattr(cloud, 'openstack'):
user_data['cloud_type'] = 'openstack'
provider = domain_model.get_cloud_provider(cloud_version_config.cloud, credentials)
ec2_endpoints = provider.security.get_ec2_endpoints()
if not ec2_endpoints.get('ec2_endpoint'):
raise ValidationError({ "error":
"This version of CloudMan supports only EC2-compatible clouds. This OpenStack"
" cloud provider does not appear to have an ec2 endpoint"})
uri_comp = urlparse(ec2_endpoints.get('ec2_endpoint'))
user_data['region_name'] = cloud.openstack.region_name
user_data['region_endpoint'] = uri_comp.hostname
user_data['ec2_port'] = uri_comp.port
user_data['ec2_conn_path'] = uri_comp.path
user_data['is_secure'] = uri_comp.scheme == "https"
if ec2_endpoints.get('s3_endpoint'):
uri_comp = urlparse(ec2_endpoints.get('s3_endpoint'))
user_data['s3_host'] = uri_comp.hostname
user_data['s3_port'] = uri_comp.port
user_data['s3_conn_path'] = uri_comp.path
else:
user_data['use_object_store'] = False
ec2_creds = provider.security.get_ec2_credentials()
user_data['access_key'] = ec2_creds.access
user_data['secret_key'] = ec2_creds.secret
else:
raise ValidationError({ "error":
"This version of CloudMan supports only EC2-compatible clouds."})
return user_data
def process_app_config(name, cloud_version_config, credentials,
app_config):
cloudman_config = get_required_val(
app_config, "config_cloudman",
"CloudMan configuration data must be provided.")
user_data = {}
user_data['bucket_default'] = get_required_val(
cloudman_config, "defaultBucket", "default bucket is required.")
user_data['cluster_name'] = name
user_data['password'] = get_required_val(cloudman_config,
"clusterPassword",
"cluster name is required.")
user_data['initial_cluster_type'] = get_required_val(
cloudman_config, "clusterType", "cluster type is required.")
user_data['cluster_storage_type'] = get_required_val(
cloudman_config, "storageType", "storage type is required.")
user_data['storage_size'] = cloudman_config.get("storageSize")
user_data['post_start_script_url'] = cloudman_config.get(
"masterPostStartScript")
user_data['worker_post_start_script_url'] = cloudman_config.get(
"workerPostStartScript")
if cloudman_config.get("clusterSharedString"):
user_data['share_string'] = cloudman_config.get(
"clusterSharedString")
user_data['cluster_templates'] = cloudman_config.get(
"cluster_templates")
cloud = cloud_version_config.cloud
if hasattr(cloud, 'aws'):
user_data['cloud_type'] = 'ec2'
user_data['region_name'] = cloud.aws.compute.ec2_region_name
user_data[
'region_endpoint'] = cloud.aws.compute.ec2_region_endpoint
user_data['ec2_port'] = cloud.aws.compute.ec2_port
user_data['ec2_conn_path'] = cloud.aws.compute.ec2_conn_path
user_data['is_secure'] = cloud.aws.compute.ec2_is_secure
user_data['s3_host'] = cloud.aws.object_store.s3_host
user_data['s3_port'] = cloud.aws.object_store.s3_port
user_data['s3_conn_path'] = cloud.aws.object_store.s3_conn_path
user_data['access_key'] = credentials.get('aws_access_key')
user_data['secret_key'] = credentials.get('aws_secret_key')
elif hasattr(cloud, 'openstack'):
user_data['cloud_type'] = 'openstack'
provider = domain_model.get_cloud_provider(
cloud_version_config.cloud, credentials)
ec2_endpoints = provider.security.get_ec2_endpoints()
if not ec2_endpoints.get('ec2_endpoint'):
raise ValidationError({
"error":
"This version of CloudMan supports only EC2-compatible clouds. This OpenStack"
" cloud provider does not appear to have an ec2 endpoint"
})
uri_comp = urlparse(ec2_endpoints.get('ec2_endpoint'))
user_data['region_name'] = cloud.openstack.region_name
user_data['region_endpoint'] = uri_comp.hostname
user_data['ec2_port'] = uri_comp.port
user_data['ec2_conn_path'] = uri_comp.path
user_data['is_secure'] = uri_comp.scheme == "https"
if ec2_endpoints.get('s3_endpoint'):
uri_comp = urlparse(ec2_endpoints.get('s3_endpoint'))
user_data['s3_host'] = uri_comp.hostname
user_data['s3_port'] = uri_comp.port
user_data['s3_conn_path'] = uri_comp.path
else:
user_data['use_object_store'] = False
ec2_creds = provider.security.get_ec2_credentials()
user_data['access_key'] = ec2_creds.access
user_data['secret_key'] = ec2_creds.secret
else:
raise ValidationError({
"error":
"This version of CloudMan supports only EC2-compatible clouds."
})
return user_data
def process_app_config(name, cloud_version_config, credentials,
app_config):
cloudman_config = get_required_val(
app_config, "config_cloudman",
"CloudMan configuration data must be provided.")
user_data = {}
user_data['bucket_default'] = get_required_val(
cloudman_config, "defaultBucket", "default bucket is required.")
user_data['cluster_name'] = name
if cloudman_config.get('restartCluster') and cloudman_config[
'restartCluster'].get('cluster_name'):
user_data['cluster_name'] = cloudman_config['restartCluster'][
'cluster_name']
user_data['machine_image_id'] = cloudman_config[
'restartCluster'].get('persistent_data',
{}).get('machine_image_id')
user_data['placement'] = cloudman_config['restartCluster'][
'placement']['placement']
user_data['password'] = get_required_val(cloudman_config,
"clusterPassword",
"cluster name is required.")
user_data['initial_cluster_type'] = get_required_val(
cloudman_config, "clusterType", "cluster type is required.")
user_data['cluster_storage_type'] = get_required_val(
cloudman_config, "storageType", "storage type is required.")
user_data['storage_type'] = user_data['cluster_storage_type']
user_data['storage_size'] = cloudman_config.get("storageSize")
user_data['post_start_script_url'] = cloudman_config.get(
"masterPostStartScript")
user_data['worker_post_start_script_url'] = cloudman_config.get(
"workerPostStartScript")
if cloudman_config.get("clusterSharedString"):
user_data['share_string'] = cloudman_config.get(
"clusterSharedString")
user_data['cluster_templates'] = cloudman_config.get(
"cluster_templates", [])
# File system template default value for file system size overwrites
# the user-provided storage_size value so remove it if both exits
for ct in user_data['cluster_templates']:
for ft in ct.get('filesystem_templates', []):
if ft.get('size') and user_data['storage_size']:
del ft['size']
extra_user_data = cloudman_config.get("extraUserData")
if extra_user_data:
log.debug("Processing CloudMan extra user data: {0}".format(
extra_user_data))
for key, value in yaml.load(extra_user_data).items():
user_data[key] = value
cloud = cloud_version_config.cloud
if hasattr(cloud, 'aws'):
user_data['cloud_type'] = 'ec2'
user_data['region_name'] = cloud.aws.compute.ec2_region_name
user_data[
'region_endpoint'] = cloud.aws.compute.ec2_region_endpoint
user_data['ec2_port'] = cloud.aws.compute.ec2_port
user_data['ec2_conn_path'] = cloud.aws.compute.ec2_conn_path
user_data['is_secure'] = cloud.aws.compute.ec2_is_secure
user_data['s3_host'] = cloud.aws.object_store.s3_host
user_data['s3_port'] = cloud.aws.object_store.s3_port
user_data['s3_conn_path'] = cloud.aws.object_store.s3_conn_path
user_data['access_key'] = credentials.get('aws_access_key')
user_data['secret_key'] = credentials.get('aws_secret_key')
elif hasattr(cloud, 'openstack'):
user_data['cloud_type'] = 'openstack'
provider = domain_model.get_cloud_provider(
cloud_version_config.cloud, credentials)
ec2_endpoints = provider.security.get_ec2_endpoints()
if not ec2_endpoints.get('ec2_endpoint'):
raise ValidationError({
"error":
"This version of CloudMan supports only EC2-compatible clouds. This OpenStack"
" cloud provider does not appear to have an ec2 endpoint"
})
uri_comp = urlparse(ec2_endpoints.get('ec2_endpoint'))
user_data['region_name'] = cloud.openstack.region_name
user_data['region_endpoint'] = uri_comp.hostname
user_data['ec2_port'] = uri_comp.port
user_data['ec2_conn_path'] = uri_comp.path
user_data['is_secure'] = uri_comp.scheme == "https"
if ec2_endpoints.get('s3_endpoint'):
uri_comp = urlparse(ec2_endpoints.get('s3_endpoint'))
user_data['s3_host'] = uri_comp.hostname
user_data['s3_port'] = uri_comp.port
user_data['s3_conn_path'] = uri_comp.path
else:
user_data['use_object_store'] = False
ec2_creds = provider.security.get_or_create_ec2_credentials()
user_data['access_key'] = ec2_creds.access
user_data['secret_key'] = ec2_creds.secret
if hasattr(cloud, 'azure'):
user_data['cloud_type'] = 'azure'
user_data['region_name'] = cloud.azure.region_name
user_data['resource_group'] = cloud.azure.resource_group
user_data['storage_account'] = cloud.azure.storage_account
user_data[
'vm_default_user_name'] = cloud.azure.vm_default_user_name
user_data['subscription_id'] = credentials.get(
'azure_subscription_id')
user_data['client_id'] = credentials.get('azure_client_id')
user_data['secret'] = credentials.get('azure_secret')
user_data['tenant'] = credentials.get('azure_tenant')
else:
raise ValidationError({
"error":
"This version of CloudMan supports only EC2-compatible clouds."
})
return user_data
def launch_app(self, task, name, cloud_version_config, credentials,
app_config, user_data):
cloudlaunch_config = app_config.get("config_cloudlaunch", {})
provider = domain_model.get_cloud_provider(cloud_version_config.cloud,
credentials)
img = provider.compute.images.get(cloud_version_config.image.image_id)
task.update_state(state='PROGRESSING',
meta={'action': "Retrieving or creating a keypair"})
kp = self._get_or_create_kp(
provider,
cloudlaunch_config.get('keyPair') or 'cloudlaunch_key_pair')
task.update_state(state='PROGRESSING',
meta={'action': "Applying firewall settings"})
sg = self.apply_app_firewall_settings(provider, cloudlaunch_config)
cb_launch_config = self._get_cb_launch_config(provider,
cloudlaunch_config)
inst_type = cloudlaunch_config.get(
'instanceType', cloud_version_config.default_instance_type)
placement_zone = cloudlaunch_config.get('placementZone')
ud = yaml.dump(user_data,
default_flow_style=False,
allow_unicode=False)
print("Launching with ud:\n%s" % (ud, ))
task.update_state(
state='PROGRESSING',
meta={
'action':
"Launching an instance of type %s with keypair %s in zone %s" %
(inst_type, kp.name, placement_zone)
})
inst = provider.compute.instances.create(
name=name,
image=img,
instance_type=inst_type,
key_pair=kp,
security_groups=[sg],
zone=placement_zone,
user_data=ud,
launch_config=cb_launch_config)
task.update_state(state='PROGRESSING',
meta={
'action':
"Waiting for instance %s to be ready.." %
(inst.id, )
})
inst.wait_till_ready()
results = {}
results['keyPair'] = {
'id': kp.id,
'name': kp.name,
'material': kp.material
}
results['securityGroup'] = {'id': sg.id, 'name': sg.name}
results['publicIP'] = inst.public_ips[0]
task.update_state(state='PROGRESSING',
meta={
'action':
"Launch successful. Public IP %s" %
(inst.public_ips[0], )
})
return {'cloudLaunch': results}