def launch_appliance(name, cloud_version_config, credentials, app_config, user_data, task_id=None): """Call the appropriate app handler and initiate the app launch process.""" try: LOG.debug("Launching appliance %s", name) handler = util.import_class( cloud_version_config.application_version.backend_component_name)() provider = domain_model.get_cloud_provider(cloud_version_config.cloud, credentials) cloud_config = util.serialize_cloud_config(cloud_version_config) launch_result = handler.launch_app(provider, Task(launch_appliance), name, cloud_config, app_config, user_data) # Schedule a task to migrate result one hour from now migrate_launch_task.apply_async([launch_appliance.request.id], countdown=3600) return launch_result except SoftTimeLimitExceeded: launch_appliance.update_state(state="FAILURE", meta={ "exc_message": "Task time limit exceeded; " "stopping the task." }) raise Ignore # This keeps the custom state set above
def launch_app(self, task, name, cloud_version_config, credentials, app_config, user_data): cloudlaunch_config = app_config.get("config_cloudlaunch", {}) provider = domain_model.get_cloud_provider(cloud_version_config.cloud, credentials) img = provider.compute.images.get(cloud_version_config.image.image_id) task.update_state(state='PROGRESSING', meta={'action': "Retrieving or creating a keypair"}) kp = self._get_or_create_kp(provider, cloudlaunch_config.get('keyPair') or 'cloudlaunch_key_pair') task.update_state(state='PROGRESSING', meta={'action': "Applying firewall settings"}) sg = self.apply_app_firewall_settings( provider, cloudlaunch_config) cb_launch_config = self._get_cb_launch_config(provider, cloudlaunch_config) inst_type = cloudlaunch_config.get( 'instanceType', cloud_version_config.default_instance_type) placement_zone = cloudlaunch_config.get('placementZone') ud = yaml.dump(user_data, default_flow_style=False, allow_unicode=False) print("Launching with ud:\n%s" % (ud,)) task.update_state(state='PROGRESSING', meta={'action': "Launching an instance of type %s with keypair %s in zone %s" % (inst_type, kp.name, placement_zone)}) inst = provider.compute.instances.create(name=name, image=img, instance_type=inst_type, key_pair=kp, security_groups=[sg], zone = placement_zone, user_data=ud, launch_config=cb_launch_config) task.update_state(state='PROGRESSING', meta={'action': "Waiting for instance %s to be ready.." % (inst.id, )}) inst.wait_till_ready() results = {} results['keyPair'] = { 'id' : kp.id, 'name' : kp.name, 'material' : kp.material } results['securityGroup'] = {'id' : sg.id, 'name' : sg.name } results['publicIP'] = inst.public_ips[0] task.update_state(state='PROGRESSING', meta={'action': "Launch successful. Public IP %s" % (inst.public_ips[0], )}) return {'cloudLaunch' : results }
def manage_appliance(self, action, deployment, credentials): """ Perform supplied action on this app. @type action: ``str`` @param action: Accepted values are ``restart`` or ``delete``. """ LOG.debug("Performing %s on deployment %s", action, deployment.name) handler = _get_app_handler(deployment) dpl = _serialize_deployment(deployment) provider = domain_model.get_cloud_provider(deployment.target_cloud, credentials) if action.lower() == 'restart': result = handler.restart(provider, dpl) elif action.lower() == 'delete': result = handler.delete(provider, dpl) else: LOG.error( "Unrecognized action: %s. Acceptable values are 'delete' " "or 'restart'", action) return None # Schedule a task to migrate results right after task completion # Do this as a separate task because until this task completes, we # cannot obtain final status or traceback. migrate_task_result.apply_async([self.request.id], countdown=1) return result
def get_cloud_provider(view, cloud_id = None): """ Returns a cloud provider for the current user. The relevant cloud is discovered from the view and the credentials are retrieved from the request or user profile. Return ``None`` if no credentials were retrieved. """ cloud_pk = cloud_id or view.kwargs.get("cloud_pk") cloud = models.Cloud.objects.filter( slug=cloud_pk).select_subclasses().first() request_creds = get_credentials(cloud, view.request) return domain_model.get_cloud_provider(cloud, request_creds)
def get_cloud_provider(view, cloud_id = None): """ Returns a cloud provider for the current user. The relevant cloud is discovered from the view and the credentials are retrieved from the request or user profile. Return ``None`` if no credentials were retrieved. """ cloud_pk = cloud_id or view.kwargs.get("cloud_pk") cloud = models.Cloud.objects.filter( slug=cloud_pk).select_subclasses().first() request_creds = get_credentials(cloud, view.request) return domain_model.get_cloud_provider(cloud, request_creds)
def launch_app(self, task, name, cloud_version_config, credentials, app_config, user_data): """ Handle the app launch process. This will: - Perform necessary checks and env setup, most notably create a new key pair. - Launch an instance and wait until ssh access can be established - Run an Ansible playbook to configure the instance """ # Implicitly create a new KP for this instance # Note that this relies on the baseVMApp implementation! kp_name = "CL-" + "".join( [c for c in name if c.isalpha() or c.isdigit()]).rstrip() app_config['config_cloudlaunch']['keyPair'] = kp_name # Launch an instance and check ssh connectivity result = super(CloudMan2AppPlugin, self).launch_app(task, name, cloud_version_config, credentials, app_config, user_data=None) provider = domain_model.get_cloud_provider(cloud_version_config.cloud, credentials) inst = provider.compute.instances.get( result.get('cloudLaunch', {}).get('instance', {}).get('id')) pk = result.get('cloudLaunch', {}).get('keyPair', {}).get('material') timeout = 0 while (not self._check_ssh(inst.public_ips[0], pk=pk) or timeout > 200): log.info("Waiting for ssh on {0}...".format(inst.name)) time.sleep(5) timeout += 5 # Configure the instance task.update_state( state='PROGRESSING', meta={'action': 'Configuring container cluster manager.'}) self._run_playbook(inst.public_ips[0], pk) result['cloudLaunch']['applicationURL'] = \ 'http://{0}:8080/'.format(result['cloudLaunch']['publicIP']) task.update_state(state='PROGRESSING', meta={ 'action': "Waiting for CloudMan to become ready at %s" % result['cloudLaunch']['applicationURL'] }) self.wait_for_http(result['cloudLaunch']['applicationURL']) return result
def health_check(self, deployment, credentials): """ Check the health of the supplied deployment. Conceptually, the health check can be as elaborate as the deployed appliance supports via a custom implementation. At the minimum, and by default, the health reflects the status of the cloud instance by querying the cloud provider. """ LOG.debug("Checking health of deployment %s", deployment.name) handler = _get_app_handler(deployment) dpl = _serialize_deployment(deployment) provider = domain_model.get_cloud_provider(deployment.target_cloud, credentials) result = handler.health_check(provider, dpl) # We only keep the two most recent health check task results so delete # any older ones signals.health_check.send(sender=None, deployment=deployment) # Schedule a task to migrate results right after task completion # Do this as a separate task because until this task completes, we # cannot obtain final status or traceback. migrate_task_result.apply_async([self.request.id], countdown=1) return result
def launch_app(self, task, name, cloud_version_config, credentials, app_config, user_data): """Initiate the app launch process.""" cloudlaunch_config = app_config.get("config_cloudlaunch", {}) provider = domain_model.get_cloud_provider(cloud_version_config.cloud, credentials) custom_image_id = cloudlaunch_config.get("customImageID", None) img = provider.compute.images.get( custom_image_id or cloud_version_config.image.image_id) task.update_state(state='PROGRESSING', meta={'action': "Retrieving or creating a key pair"}) kp = self._get_or_create_kp( provider, cloudlaunch_config.get('keyPair') or 'cloudlaunch_key_pair') task.update_state(state='PROGRESSING', meta={'action': "Applying firewall settings"}) subnet_id, placement_zone, sgs = self.resolve_launch_properties( provider, cloudlaunch_config) cb_launch_config = self._get_cb_launch_config(provider, img, cloudlaunch_config) inst_type = cloudlaunch_config.get( 'instanceType', cloud_version_config.default_instance_type) log.debug("Launching with subnet %s and SGs %s" % (subnet_id, sgs)) log.info("Launching base_vm with UD:\n%s" % user_data) task.update_state(state='PROGRESSING', meta={ 'action': "Launching an instance of type %s " "with keypair %s in zone %s" % (inst_type, kp.name, placement_zone) }) inst = provider.compute.instances.create( name=name, image=img, instance_type=inst_type, subnet=subnet_id, key_pair=kp, security_groups=sgs, zone=placement_zone, user_data=user_data, launch_config=cb_launch_config) task.update_state(state='PROGRESSING', meta={'action': "Waiting for instance %s" % inst.id}) log.debug("Waiting for instance {0} to be ready...".format(inst.id)) inst.wait_till_ready() static_ip = cloudlaunch_config.get('staticIP') if static_ip: task.update_state(state='PROGRESSING', meta={ 'action': "Assigning requested floating " "IP: %s" % static_ip }) inst.add_floating_ip(static_ip) inst.refresh() results = {} results['keyPair'] = { 'id': kp.id, 'name': kp.name, 'material': kp.material } # FIXME: this does not account for multiple SGs and expects one results['securityGroup'] = {'id': sgs[0].id, 'name': sgs[0].name} results['instance'] = {'id': inst.id} results['publicIP'] = self.attach_public_ip(provider, inst) task.update_state(state='PROGRESSING', meta={ "action": "Instance created successfully. " + "Public IP: %s" % results['publicIP'] if results['publicIP'] else "" }) if self.base_app: if results['publicIP']: results['applicationURL'] = 'http://%s/' % results['publicIP'] task.update_state( state='PROGRESSING', meta={ 'action': "Waiting for application to become ready " "at %s" % results['applicationURL'] }) self.wait_for_http(results['applicationURL']) else: results['applicationURL'] = 'N/A' return {'cloudLaunch': results}
def process_app_config(name, cloud_version_config, credentials, app_config): cloudman_config = get_required_val( app_config, "config_cloudman", "CloudMan configuration data must be provided.") user_data = {} user_data['bucket_default'] = get_required_val( cloudman_config, "defaultBucket", "default bucket is required.") user_data['cluster_name'] = name user_data['password'] = get_required_val( cloudman_config, "clusterPassword", "cluster name is required.") user_data['initial_cluster_type'] = get_required_val( cloudman_config, "clusterType", "cluster type is required.") user_data['cluster_storage_type'] = get_required_val( cloudman_config, "storageType", "storage type is required.") user_data['storage_size'] = cloudman_config.get("storageSize") user_data['post_start_script_url'] = cloudman_config.get( "masterPostStartScript") user_data['worker_post_start_script_url'] = cloudman_config.get( "workerPostStartScript") if cloudman_config.get("clusterSharedString"): user_data['share_string'] = cloudman_config.get("clusterSharedString") user_data['cluster_templates'] = cloudman_config.get( "cluster_templates") cloud = cloud_version_config.cloud if hasattr(cloud, 'aws'): user_data['cloud_type'] = 'ec2' user_data['region_name'] = cloud.aws.compute.ec2_region_name user_data['region_endpoint'] = cloud.aws.compute.ec2_region_endpoint user_data['ec2_port'] = cloud.aws.compute.ec2_port user_data['ec2_conn_path'] = cloud.aws.compute.ec2_conn_path user_data['is_secure'] = cloud.aws.compute.ec2_is_secure user_data['s3_host'] = cloud.aws.object_store.s3_host user_data['s3_port'] = cloud.aws.object_store.s3_port user_data['s3_conn_path'] = cloud.aws.object_store.s3_conn_path user_data['access_key'] = credentials.get('aws_access_key') user_data['secret_key'] = credentials.get('aws_secret_key') elif hasattr(cloud, 'openstack'): user_data['cloud_type'] = 'openstack' provider = domain_model.get_cloud_provider(cloud_version_config.cloud, credentials) ec2_endpoints = provider.security.get_ec2_endpoints() if not ec2_endpoints.get('ec2_endpoint'): raise ValidationError({ "error": "This version of CloudMan supports only EC2-compatible clouds. This OpenStack" " cloud provider does not appear to have an ec2 endpoint"}) uri_comp = urlparse(ec2_endpoints.get('ec2_endpoint')) user_data['region_name'] = cloud.openstack.region_name user_data['region_endpoint'] = uri_comp.hostname user_data['ec2_port'] = uri_comp.port user_data['ec2_conn_path'] = uri_comp.path user_data['is_secure'] = uri_comp.scheme == "https" if ec2_endpoints.get('s3_endpoint'): uri_comp = urlparse(ec2_endpoints.get('s3_endpoint')) user_data['s3_host'] = uri_comp.hostname user_data['s3_port'] = uri_comp.port user_data['s3_conn_path'] = uri_comp.path else: user_data['use_object_store'] = False ec2_creds = provider.security.get_ec2_credentials() user_data['access_key'] = ec2_creds.access user_data['secret_key'] = ec2_creds.secret else: raise ValidationError({ "error": "This version of CloudMan supports only EC2-compatible clouds."}) return user_data
def process_app_config(name, cloud_version_config, credentials, app_config): cloudman_config = get_required_val( app_config, "config_cloudman", "CloudMan configuration data must be provided.") user_data = {} user_data['bucket_default'] = get_required_val( cloudman_config, "defaultBucket", "default bucket is required.") user_data['cluster_name'] = name user_data['password'] = get_required_val(cloudman_config, "clusterPassword", "cluster name is required.") user_data['initial_cluster_type'] = get_required_val( cloudman_config, "clusterType", "cluster type is required.") user_data['cluster_storage_type'] = get_required_val( cloudman_config, "storageType", "storage type is required.") user_data['storage_size'] = cloudman_config.get("storageSize") user_data['post_start_script_url'] = cloudman_config.get( "masterPostStartScript") user_data['worker_post_start_script_url'] = cloudman_config.get( "workerPostStartScript") if cloudman_config.get("clusterSharedString"): user_data['share_string'] = cloudman_config.get( "clusterSharedString") user_data['cluster_templates'] = cloudman_config.get( "cluster_templates") cloud = cloud_version_config.cloud if hasattr(cloud, 'aws'): user_data['cloud_type'] = 'ec2' user_data['region_name'] = cloud.aws.compute.ec2_region_name user_data[ 'region_endpoint'] = cloud.aws.compute.ec2_region_endpoint user_data['ec2_port'] = cloud.aws.compute.ec2_port user_data['ec2_conn_path'] = cloud.aws.compute.ec2_conn_path user_data['is_secure'] = cloud.aws.compute.ec2_is_secure user_data['s3_host'] = cloud.aws.object_store.s3_host user_data['s3_port'] = cloud.aws.object_store.s3_port user_data['s3_conn_path'] = cloud.aws.object_store.s3_conn_path user_data['access_key'] = credentials.get('aws_access_key') user_data['secret_key'] = credentials.get('aws_secret_key') elif hasattr(cloud, 'openstack'): user_data['cloud_type'] = 'openstack' provider = domain_model.get_cloud_provider( cloud_version_config.cloud, credentials) ec2_endpoints = provider.security.get_ec2_endpoints() if not ec2_endpoints.get('ec2_endpoint'): raise ValidationError({ "error": "This version of CloudMan supports only EC2-compatible clouds. This OpenStack" " cloud provider does not appear to have an ec2 endpoint" }) uri_comp = urlparse(ec2_endpoints.get('ec2_endpoint')) user_data['region_name'] = cloud.openstack.region_name user_data['region_endpoint'] = uri_comp.hostname user_data['ec2_port'] = uri_comp.port user_data['ec2_conn_path'] = uri_comp.path user_data['is_secure'] = uri_comp.scheme == "https" if ec2_endpoints.get('s3_endpoint'): uri_comp = urlparse(ec2_endpoints.get('s3_endpoint')) user_data['s3_host'] = uri_comp.hostname user_data['s3_port'] = uri_comp.port user_data['s3_conn_path'] = uri_comp.path else: user_data['use_object_store'] = False ec2_creds = provider.security.get_ec2_credentials() user_data['access_key'] = ec2_creds.access user_data['secret_key'] = ec2_creds.secret else: raise ValidationError({ "error": "This version of CloudMan supports only EC2-compatible clouds." }) return user_data
def process_app_config(name, cloud_version_config, credentials, app_config): cloudman_config = get_required_val( app_config, "config_cloudman", "CloudMan configuration data must be provided.") user_data = {} user_data['bucket_default'] = get_required_val( cloudman_config, "defaultBucket", "default bucket is required.") user_data['cluster_name'] = name if cloudman_config.get('restartCluster') and cloudman_config[ 'restartCluster'].get('cluster_name'): user_data['cluster_name'] = cloudman_config['restartCluster'][ 'cluster_name'] user_data['machine_image_id'] = cloudman_config[ 'restartCluster'].get('persistent_data', {}).get('machine_image_id') user_data['placement'] = cloudman_config['restartCluster'][ 'placement']['placement'] user_data['password'] = get_required_val(cloudman_config, "clusterPassword", "cluster name is required.") user_data['initial_cluster_type'] = get_required_val( cloudman_config, "clusterType", "cluster type is required.") user_data['cluster_storage_type'] = get_required_val( cloudman_config, "storageType", "storage type is required.") user_data['storage_type'] = user_data['cluster_storage_type'] user_data['storage_size'] = cloudman_config.get("storageSize") user_data['post_start_script_url'] = cloudman_config.get( "masterPostStartScript") user_data['worker_post_start_script_url'] = cloudman_config.get( "workerPostStartScript") if cloudman_config.get("clusterSharedString"): user_data['share_string'] = cloudman_config.get( "clusterSharedString") user_data['cluster_templates'] = cloudman_config.get( "cluster_templates", []) # File system template default value for file system size overwrites # the user-provided storage_size value so remove it if both exits for ct in user_data['cluster_templates']: for ft in ct.get('filesystem_templates', []): if ft.get('size') and user_data['storage_size']: del ft['size'] extra_user_data = cloudman_config.get("extraUserData") if extra_user_data: log.debug("Processing CloudMan extra user data: {0}".format( extra_user_data)) for key, value in yaml.load(extra_user_data).items(): user_data[key] = value cloud = cloud_version_config.cloud if hasattr(cloud, 'aws'): user_data['cloud_type'] = 'ec2' user_data['region_name'] = cloud.aws.compute.ec2_region_name user_data[ 'region_endpoint'] = cloud.aws.compute.ec2_region_endpoint user_data['ec2_port'] = cloud.aws.compute.ec2_port user_data['ec2_conn_path'] = cloud.aws.compute.ec2_conn_path user_data['is_secure'] = cloud.aws.compute.ec2_is_secure user_data['s3_host'] = cloud.aws.object_store.s3_host user_data['s3_port'] = cloud.aws.object_store.s3_port user_data['s3_conn_path'] = cloud.aws.object_store.s3_conn_path user_data['access_key'] = credentials.get('aws_access_key') user_data['secret_key'] = credentials.get('aws_secret_key') elif hasattr(cloud, 'openstack'): user_data['cloud_type'] = 'openstack' provider = domain_model.get_cloud_provider( cloud_version_config.cloud, credentials) ec2_endpoints = provider.security.get_ec2_endpoints() if not ec2_endpoints.get('ec2_endpoint'): raise ValidationError({ "error": "This version of CloudMan supports only EC2-compatible clouds. This OpenStack" " cloud provider does not appear to have an ec2 endpoint" }) uri_comp = urlparse(ec2_endpoints.get('ec2_endpoint')) user_data['region_name'] = cloud.openstack.region_name user_data['region_endpoint'] = uri_comp.hostname user_data['ec2_port'] = uri_comp.port user_data['ec2_conn_path'] = uri_comp.path user_data['is_secure'] = uri_comp.scheme == "https" if ec2_endpoints.get('s3_endpoint'): uri_comp = urlparse(ec2_endpoints.get('s3_endpoint')) user_data['s3_host'] = uri_comp.hostname user_data['s3_port'] = uri_comp.port user_data['s3_conn_path'] = uri_comp.path else: user_data['use_object_store'] = False ec2_creds = provider.security.get_or_create_ec2_credentials() user_data['access_key'] = ec2_creds.access user_data['secret_key'] = ec2_creds.secret if hasattr(cloud, 'azure'): user_data['cloud_type'] = 'azure' user_data['region_name'] = cloud.azure.region_name user_data['resource_group'] = cloud.azure.resource_group user_data['storage_account'] = cloud.azure.storage_account user_data[ 'vm_default_user_name'] = cloud.azure.vm_default_user_name user_data['subscription_id'] = credentials.get( 'azure_subscription_id') user_data['client_id'] = credentials.get('azure_client_id') user_data['secret'] = credentials.get('azure_secret') user_data['tenant'] = credentials.get('azure_tenant') else: raise ValidationError({ "error": "This version of CloudMan supports only EC2-compatible clouds." }) return user_data
def launch_app(self, task, name, cloud_version_config, credentials, app_config, user_data): cloudlaunch_config = app_config.get("config_cloudlaunch", {}) provider = domain_model.get_cloud_provider(cloud_version_config.cloud, credentials) img = provider.compute.images.get(cloud_version_config.image.image_id) task.update_state(state='PROGRESSING', meta={'action': "Retrieving or creating a keypair"}) kp = self._get_or_create_kp( provider, cloudlaunch_config.get('keyPair') or 'cloudlaunch_key_pair') task.update_state(state='PROGRESSING', meta={'action': "Applying firewall settings"}) sg = self.apply_app_firewall_settings(provider, cloudlaunch_config) cb_launch_config = self._get_cb_launch_config(provider, cloudlaunch_config) inst_type = cloudlaunch_config.get( 'instanceType', cloud_version_config.default_instance_type) placement_zone = cloudlaunch_config.get('placementZone') ud = yaml.dump(user_data, default_flow_style=False, allow_unicode=False) print("Launching with ud:\n%s" % (ud, )) task.update_state( state='PROGRESSING', meta={ 'action': "Launching an instance of type %s with keypair %s in zone %s" % (inst_type, kp.name, placement_zone) }) inst = provider.compute.instances.create( name=name, image=img, instance_type=inst_type, key_pair=kp, security_groups=[sg], zone=placement_zone, user_data=ud, launch_config=cb_launch_config) task.update_state(state='PROGRESSING', meta={ 'action': "Waiting for instance %s to be ready.." % (inst.id, ) }) inst.wait_till_ready() results = {} results['keyPair'] = { 'id': kp.id, 'name': kp.name, 'material': kp.material } results['securityGroup'] = {'id': sg.id, 'name': sg.name} results['publicIP'] = inst.public_ips[0] task.update_state(state='PROGRESSING', meta={ 'action': "Launch successful. Public IP %s" % (inst.public_ips[0], ) }) return {'cloudLaunch': results}