Esempio n. 1
0
    def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc,
                  vm_image, vm_mem, vm_cores, vm_storage, customization=None,
                  pre_customization=None, vm_keepalive=0, instance_type="", 
                  maximum_price=0, job_per_core=False, securitygroup=[],
                  key_name="",use_cloud_init=False, extra_userdata=[]):
        """Attempt to boot a new VM on the cluster."""

        use_cloud_init = use_cloud_init or config.use_cloud_init
        log.verbose("Trying to boot %s on %s" % (vm_type, self.network_address))
        if len(securitygroup) != 0:
            sec_group = []
            for group in securitygroup:
                if group in self.security_groups:
                    sec_group.append(group)
            if len(sec_group) == 0:
                log.debug("No matching security groups - trying default config")
                sec_group = self.security_groups
                #sec_group.append("default") - don't just append default use what is in cloud_resources.conf for this cloud
        else:
            sec_group = self.security_groups

        try:
            if self.name in vm_image.keys():
                vm_ami = vm_image[self.name]
            else:
                vm_ami = vm_image[self.network_address]
        except:
            log.debug("No AMI for %s, trying default" % self.network_address)
            #try:
            #    vm_ami = vm_image["default"]
            #except:
                #log.debug("No given default - trying global defaults")
            try:
                vm_default_ami = _attr_list_to_dict(config.default_VMAMI)
                if self.name in vm_default_ami.keys():
                    vm_ami = vm_default_ami[self.name]
                else:
                    vm_ami = vm_default_ami[self.network_address]
            except:
                try:
                    vm_ami = vm_default_ami["default"]
                except:
                    log.exception("Can't find a suitable AMI")
                    self.failed_image_set.add(vm_ami)
                    return

        try:
            if self.name in instance_type.keys():
                i_type = instance_type[self.name]
            elif self.network_address in instance_type.keys():
                i_type = instance_type[self.network_address]
            else:
                i_type = instance_type["default"]
        except:
            log.debug("No instance type for %s, trying default" % self.network_address)
            #try:
            #    i_type = instance_type["default"]
            #except:
            #    if isinstance(instance_type, str):
            #        i_type = instance_type
            #    else:
            try:
                if self.name in self.DEFAULT_INSTANCE_TYPE_LIST.keys():
                    i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.name]
                else:
                    i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.network_address]
            except:
                log.debug("No default instance type found for %s, trying single default" % self.network_address)
                i_type = self.DEFAULT_INSTANCE_TYPE
        instance_type = i_type

        if key_name == None:
            key_name = self.key_name
        if customization:
            if not use_cloud_init:
                user_data = nimbus_xml.ws_optional(customization)
            else:
                user_data = cloud_init_util.build_write_files_cloud_init(customization)
        else:
            user_data = ""

        if pre_customization:
            if not use_cloud_init:
                for item in pre_customization:
                    user_data = '\n'.join([item, user_data])
            else:
                user_data = cloud_init_util.inject_customizations(pre_customization, user_data)
        elif use_cloud_init:
            user_data = cloud_init_util.inject_customizations([], user_data)[0]
        if len(extra_userdata) > 0:
            # need to use the multi-mime type functions
            user_data = cloud_init_util.build_multi_mime_message([(user_data, 'cloud-config', 'cloud_conf.yaml')], extra_userdata)

        if "AmazonEC2" == self.cloud_type and vm_networkassoc != "public":
            log.debug("You requested '%s' networking, but EC2 only supports 'public'" % vm_networkassoc)
            addressing_type = "public"
        else:
            addressing_type = vm_networkassoc

        try:
            connection = self._get_connection()
            image = None

            if not "Eucalyptus" == self.cloud_type:
                image = connection.get_image(vm_ami)

            else:   #HACK: for some reason Eucalyptus won't respond properly to
                    #      get_image("whateverimg"). Use a linear search until
                    #      this is fixed
                    # This is Eucalyptus bug #495670
                    # https://bugs.launchpad.net/eucalyptus/+bug/495670
                images = connection.get_all_images()
                for potential_match in images:
                    if potential_match.id == vm_ami:
                        image = potential_match
                        break

            # Compress the user data to try and get under the limit
            user_data = utilities.gzip_userdata(user_data)

            if image:
                if maximum_price is 0 or self.cloud_type == "OpenStack": # don't request a spot instance
                    try:
                        reservation = image.run(1,1, key_name=key_name,
                                                addressing_type=addressing_type,
                                                user_data=user_data,
                                                placement=self.placement_zone,
                                                security_groups=sec_group,
                                                instance_type=instance_type)
                        instance_id = reservation.instances[0].id
                        log.debug("Booted VM %s" % instance_id)
                    except boto.exception.EC2ResponseError, e:
                        log.exception("There was a problem creating an EC2 instance: %s" % e)
                        return self.ERROR
                    except Exception, e:
                        log.exception("There was an unexpected problem creating an EC2 instance: %s" % e)
                        return self.ERROR
Esempio n. 2
0
    def vm_create(self, vm_name, vm_type, vm_user,
                  vm_image, vm_mem, vm_cores, vm_storage, customization=None,
                  vm_keepalive=0, instance_type={}, job_per_core=False,
                  pre_customization=None, extra_userdata=[]):
        """ Create a VM on Azure."""

        use_cloud_init = True

        if customization:
            user_data = cloud_init_util.build_write_files_cloud_init(customization)
        else:
            user_data = ""
        if pre_customization:
            user_data = cloud_init_util.inject_customizations(pre_customization, user_data)
        elif use_cloud_init:
            user_data = cloud_init_util.inject_customizations([], user_data)
        if len(extra_userdata) > 0:
            # need to use the multi-mime type functions
            user_data = cloud_init_util.build_multi_mime_message([(user_data, 'cloud-config', 'cloud_conf.yaml')],
                                                                 extra_userdata)
            if not user_data:
                log.error("Problem building cloud-config user data.")
                return self.ERROR

        # Compress the user data to try and get under the limit
        user_data = utilities.gzip_userdata(user_data)

        try:
            if self.name in vm_image.keys():
                image = vm_image[self.name]
            elif self.network_address in vm_image.keys():
                image = vm_image[self.network_address]
            else:
                image = vm_image['default']
        except:
            try:
                vm_default_ami = _attr_list_to_dict(config.default_VMAMI)
                if self.name in vm_default_ami.keys():
                    image = vm_default_ami[self.name]
                else:
                    image = vm_default_ami[self.network_address]
            except:
                try:
                    image = vm_default_ami["default"]
                except:
                    log.exception("Can't find a suitable AMI")
                    return

        try:
            if self.name in instance_type.keys():
                i_type = instance_type[self.name]
            elif self.network_address in instance_type.keys():
                i_type = instance_type[self.network_address]
            else:
                i_type = instance_type['default']
        except:
            log.debug("No instance type for %s, trying default" % self.network_address)
            try:
                if self.name in self.DEFAULT_INSTANCE_TYPE_LIST.keys():
                    i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.name]
                else:
                    i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.network_address]
            except:
                log.debug("No default instance type found for %s, trying single default" % self.network_address)
                i_type = self.DEFAULT_INSTANCE_TYPE

        name = self._generate_next_name()
        instance = None
        req = None
        if name:
            sms = self._get_service_connection()
            try:
                conf_set = azure.servicemanagement.LinuxConfigurationSet(host_name=name, user_name=self.username,
                                                                         user_password=self.password,
                                                                         disable_ssh_password_authentication=False,
                                                                         custom_data=user_data)
                net_set = azure.servicemanagement.ConfigurationSet()
                vm_ssh_port = 20000+self.count
                net_set.input_endpoints.input_endpoints.append(
                    azure.servicemanagement.ConfigurationSetInputEndpoint(name='SSH',
                                                                          protocol='TCP',
                                                                          port=vm_ssh_port,
                                                                          local_port=22))
                self.count += 1
                if self.count > 15000:
                    self.count = 0
                os_hd = azure.servicemanagement.OSVirtualHardDisk(image, self.blob_url + name)

                res = sms.check_hosted_service_name_availability(self.azure_service_name)
                if res.result:
                    req = sms.create_hosted_service(self.azure_service_name, self.azure_service_name, location=self.regions[0])
                    sms.wait_for_operation_status(req.request_id)
                if len(self.vms) == 0:
                    req = sms.create_virtual_machine_deployment(service_name=self.azure_service_name,
                                                                deployment_name=self.azure_service_name,
                                                                deployment_slot='production',
                                                                role_name=name, label=name,
                                                                system_config=conf_set, network_config=net_set,
                                                                os_virtual_hard_disk=os_hd, role_size=i_type)
                    try:
                        op_status = sms.wait_for_operation_status(req.request_id)
                    except Exception as e:
                        log.error("Problem creating VM on Azure: %s" % e.result.error.message)
                        return 1
                else:
                    req = sms.add_role(service_name=self.azure_service_name, deployment_name=self.azure_service_name,
                                       role_name=name, system_config=conf_set, network_config=net_set,
                                       os_virtual_hard_disk=os_hd, role_size=i_type)
                    try:
                        op_status = sms.wait_for_operation_status(req.request_id)
                    except Exception as e:
                        log.error("Problem creating VM on Azure: %s" % e.result.error.message)
                        return 1
            except Exception as e:
                log.error("Unhandled exception while creating vm on %s: %s" % (self.name, e))
                return self.ERROR
            if req:
                if not vm_keepalive and self.keep_alive:  # if job didn't set a keep_alive use the clouds default
                    vm_keepalive = self.keep_alive

                new_vm = cluster_tools.VM(name=vm_name, id=name, vmtype=vm_type, user=vm_user,
                                          clusteraddr=self.network_address,
                                          hostname=name,
                                          cloudtype=self.cloud_type, network=None,
                                          image=vm_image, flavor=i_type,
                                          memory=vm_mem, cpucores=vm_cores, storage=vm_storage,
                                          keep_alive=vm_keepalive, job_per_core=job_per_core, ssh_port=vm_ssh_port)

                try:
                    self.resource_checkout(new_vm)
                    log.info("Launching 1 VM: %s on %s" % (name, self.name))
                except:
                    log.error("Unexpected Error checking out resources when creating a VM. Programming error?")
                    self.vm_destroy(new_vm, reason="Failed Resource checkout", return_resources=False)
                    return self.ERROR

                self.vms.append(new_vm)
            else:
                log.debug("Failed to create instance on %s" % self.name)
                return self.ERROR
        else:
            log.debug("Unable to generate name for %s" % self.name)
            return self.ERROR

        return 0
Esempio n. 3
0
    def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc, vm_cpuarch,
                  vm_image, vm_mem, vm_cores, vm_storage, customization=None,
                  vm_keepalive=0, instance_type="", maximum_price=0,
                  job_per_core=False, securitygroup=[],key_name=""):
        """Attempt to boot a new VM on the cluster."""
        #print vm_image
        #print instance_type
        #print securitygroup
        log.verbose("Trying to boot %s on %s" % (vm_type, self.network_address))
        if len(securitygroup) != 0:
            sec_group = []
            for group in securitygroup:
                if group in self.security_groups:
                    sec_group.append(group)
            if len(sec_group) == 0:
                log.warning("No matching security groups - trying default config")
                sec_group = self.security_groups
                #sec_group.append("default") - don't just append default use what is in cloud_resources.conf for this cloud
        else:
            sec_group = self.security_groups

        try:
            if self.name in vm_image.keys():
                vm_ami = vm_image[self.name]
            else:
                vm_ami = vm_image[self.network_address]
        except:
            log.debug("No AMI for %s, trying default" % self.network_address)
            #try:
            #    vm_ami = vm_image["default"]
            #except:
                #log.debug("No given default - trying global defaults")
            try:
                vm_default_ami = _attr_list_to_dict(config.default_VMAMI)
                if self.name in vm_default_ami.keys():
                    vm_ami = vm_default_ami[self.name]
                else:
                    vm_ami = vm_default_ami[self.network_address]
            except:
                try:
                    vm_ami = vm_default_ami["default"]
                except:
                    log.exception("Can't find a suitable AMI")
                    return

        try:
            if self.name in instance_type.keys():
                i_type = instance_type[self.name]
            else:
                i_type = instance_type[self.network_address]
        except:
            log.debug("No instance type for %s, trying default" % self.network_address)
            #try:
            #    i_type = instance_type["default"]
            #except:
            #    if isinstance(instance_type, str):
            #        i_type = instance_type
            #    else:
            try:
                if self.name in self.DEFAULT_INSTANCE_TYPE_LIST.keys():
                    i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.name]
                else:
                    i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.network_address]
            except:
                log.debug("No default instance type found for %s, trying single default" % self.network_address)
                i_type = self.DEFAULT_INSTANCE_TYPE
        instance_type = i_type
        if key_name == None:
            key_name = self.key_name
        if customization:
            user_data = nimbus_xml.ws_optional(customization)
        else:
            user_data = ""

        if "AmazonEC2" == self.cloud_type and vm_networkassoc != "public":
            log.debug("You requested '%s' networking, but EC2 only supports 'public'" % vm_networkassoc)
            addressing_type = "public"
        else:
            addressing_type = vm_networkassoc

        try:
            connection = self._get_connection()
            image = None

            if not "Eucalyptus" == self.cloud_type:
                image = connection.get_image(vm_ami)

            else: #HACK: for some reason Eucalyptus won't respond properly to
                  #      get_image("whateverimg"). Use a linear search until
                  #      this is fixed
                  # This is Eucalyptus bug #495670
                  # https://bugs.launchpad.net/eucalyptus/+bug/495670
                images = connection.get_all_images()
                for potential_match in images:
                    if potential_match.id == vm_ami:
                        image = potential_match
                        break

            if image:
                if maximum_price is 0: # don't request a spot instance
                    try:
                        reservation = image.run(1,1, key_name=key_name,
                                                addressing_type=addressing_type,
                                                user_data=user_data,
                                                placement=self.placement_zone,
                                                security_groups=sec_group,
                                                instance_type=instance_type)
                        instance_id = reservation.instances[0].id
                        log.debug("Booted VM %s" % instance_id)
                    except:
                        log.exception("There was a problem creating an EC2 instance...")
                        return self.ERROR

                else: # get a spot instance of no more than maximum_price
                    try:
                        price_in_dollars = str(float(maximum_price) / 100)
                        reservation = connection.request_spot_instances(
                                                  price_in_dollars,
                                                  image.id,
                                                  key_name=key_name,
                                                  user_data=user_data,
                                                  placement=self.placement_zone,
                                                  addressing_type=addressing_type,
                                                  security_groups=self.sec_group,
                                                  instance_type=instance_type)
                        spot_id = str(reservation[0].id)
                        instance_id = ""
                        log.debug("Reserved instance %s at no more than %s" % (spot_id, price_in_dollars))
                    except AttributeError:
                        log.exception("Your version of boto doesn't seem to support "\
                                  "spot instances. You need at least 1.9")
                        return self.ERROR
                    except:
                        log.exception("Problem creating an EC2 spot instance...")
                        return self.ERROR


            else:
                log.error("Couldn't find image %s on %s" % (vm_image, self.name))
                return self.ERROR

        except:
            log.exception("Problem creating EC2 instance on on %s" % self.name)
            return self.ERROR

        vm_mementry = self.find_mementry(vm_mem)
        if (vm_mementry < 0):
            #TODO: this is kind of pointless with EC2...
            log.debug("Cluster memory list has no sufficient memory " +\
                      "entries (Not supposed to happen). Returning error.")
            return self.ERROR
        log.verbose("vm_create - Memory entry found in given cluster: %d" %
                                                                    vm_mementry)
        new_vm = cluster_tools.VM(name = vm_name, id = instance_id, vmtype = vm_type, user = vm_user,
                    clusteraddr = self.network_address,
                    cloudtype = self.cloud_type, network = vm_networkassoc,
                    cpuarch = vm_cpuarch, image= vm_image,
                    memory = vm_mem, mementry = vm_mementry,
                    cpucores = vm_cores, storage = vm_storage, 
                    keep_alive = vm_keepalive, job_per_core = job_per_core)

        try:
            new_vm.spot_id = spot_id
        except:
            log.verbose("No spot ID to add to VM %s" % instance_id)

        try:
            self.resource_checkout(new_vm)
        except:
            log.exception("Unexpected Error checking out resources when creating a VM. Programming error?")
            self.vm_destroy(new_vm, reason="Failed Resource checkout")
            return self.ERROR

        self.vms.append(new_vm)

        return 0
Esempio n. 4
0
    def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc,
                  vm_image, vm_mem, vm_cores, vm_storage, customization=None,
                  vm_keepalive=0, instance_type="", job_per_core=False, 
                  securitygroup=[],key_name="", pre_customization=None, use_cloud_init=False, extra_userdata=[]):
        """ Create a VM on OpenStack."""

        import novaclient.exceptions
        use_cloud_init = use_cloud_init or config.use_cloud_init
        nova = self._get_creds_nova()
        if len(securitygroup) != 0:
            sec_group = []
            for group in securitygroup:
                if group in self.security_groups:
                    sec_group.append(group)
            if len(sec_group) == 0:
                log.debug("No defined security groups for job - trying default value from cloud_resources.conf")
                sec_group = self.security_groups
        else:
            sec_group = self.security_groups
        log.debug("Using security group: %s" % str(sec_group))
        if key_name and len(key_name) > 0:
            if not nova.keypairs.findall(name=key_name):
                key_name = ""
        else:
            key_name = self.key_name if self.key_name else ""
        if customization:
            if not use_cloud_init:
                user_data = nimbus_xml.ws_optional(customization)
            else:
                user_data = cloud_init_util.build_write_files_cloud_init(customization)
        else:
            user_data = ""
        if pre_customization:
            if not use_cloud_init:
                for item in pre_customization:
                    user_data = '\n'.join([item, user_data])
            else:
                user_data = cloud_init_util.inject_customizations(pre_customization, user_data)
        elif use_cloud_init:
            user_data = cloud_init_util.inject_customizations([], user_data)
        if len(extra_userdata) > 0:
            # need to use the multi-mime type functions
            user_data = cloud_init_util.build_multi_mime_message([(user_data, 'cloud-config', 'cloud_conf.yaml')], extra_userdata)

        # Compress the user data to try and get under the limit
        user_data = utilities.gzip_userdata(user_data)
        
        try:
            if self.name in vm_image.keys():
                image = vm_image[self.name]
            elif self.network_address in vm_image.keys():
                image = vm_image[self.network_address]
            else:
                image = vm_image['default']
        except:
            try:
                vm_default_ami = _attr_list_to_dict(config.default_VMAMI)
                if self.name in vm_default_ami.keys():
                    image = vm_default_ami[self.name]
                else:
                    image = vm_default_ami[self.network_address]
            except:
                try:
                    image = vm_default_ami["default"]
                except:
                    log.exception("Can't find a suitable AMI")
                    return
        try:
            imageobj = nova.images.find(name=image)
        except novaclient.exceptions.EndpointNotFound:
            log.error("Endpoint not found, are your region settings correct for %s" % self.name)
            return -4
        except Exception as e:
            log.warning("Exception occurred while trying to fetch image via name: %s %s" % (image, e))
            try:
                imageobj = nova.images.get(image)
                log.debug("Got image via uuid: %s" % image)
            except novaclient.exceptions.EndpointNotFound:
                log.error("Endpoint not found, are your region settings correct for %s" % self.name)
                return -4
            except Exception as e:
                log.exception("Unable to fetch image via uuid: %s %s" % (image, e))
                self.failed_image_set.add(image)
                return

        try:
            if self.name in instance_type.keys():
                i_type = instance_type[self.name]
            elif self.network_address in instance_type.keys():
                i_type = instance_type[self.network_address]
            else:
                i_type = instance_type['default']
        except:
            log.debug("No instance type for %s, trying default" % self.network_address)
            try:
                if self.name in self.DEFAULT_INSTANCE_TYPE_LIST.keys():
                    i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.name]
                else:
                    i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.network_address]
            except:
                log.debug("No default instance type found for %s, trying single default" % self.network_address)
                i_type = self.DEFAULT_INSTANCE_TYPE
        try:   
            flavor = nova.flavors.find(name=i_type)
        except Exception as e:
            log.warning("Exception occurred while trying to get flavor by name: %s - will attempt to use name value as a uuid." % e)
            try:
                flavor = nova.flavors.get(i_type)
                log.debug("Got flavor via uuid: %s" % i_type)
            except Exception as ex:
                log.error("Exception occurred trying to get flavor by uuid: %s" % ex)
                return
        self.flavor_set.add(flavor)
        # find the network id to use if more than one network
        if vm_networkassoc:
            network = self._find_network(vm_networkassoc)
            if network:
                netid = [{'net-id': network.id}]
            else:
                log.debug("Unable to find network named: %s on %s" % (vm_networkassoc, self.name))
                netid = []
        elif self.network_pools and len(self.network_pools) > 0:
            network = self._find_network(self.network_pools[0])
            if network:
                netid = [{'net-id': network.id}]
            else:
                log.debug("Unable to find network named: %s on %s" % (self.network_pools[0], self.name))
                netid = []
        else:
            netid = []
        # Need to get the rotating hostname from the google code to use for here.  
        name = self._generate_next_name()
        instance = None

        if name:
            try:
                instance = nova.servers.create(name=name, image=imageobj, flavor=flavor, key_name=key_name, 
                                               availability_zone=self.placement_zone, nics =netid, userdata=user_data, security_groups=sec_group)
                #print instance.__dict__
            except novaclient.exceptions.OverLimit as e:
                log.info("Unable to create VM without exceeded quota on %s: %s" % (self.name, e.message))
            except Exception as e:
                #print e
                log.error("Unhandled exception while creating vm on %s: %s" %(self.name, e))
            if instance:
                instance_id = instance.id
                if not vm_keepalive and self.keep_alive: #if job didn't set a keep_alive use the clouds default
                    vm_keepalive = self.keep_alive

                new_vm = cluster_tools.VM(name = vm_name, id = instance_id, vmtype = vm_type, user = vm_user,
                            clusteraddr = self.network_address, hostname = ''.join([name, self.vm_domain_name]),
                            cloudtype = self.cloud_type, network = vm_networkassoc,
                            image= vm_image, flavor=flavor.name,
                            memory = vm_mem, cpucores = vm_cores, storage = vm_storage, 
                            keep_alive = vm_keepalive, job_per_core = job_per_core)
    
                try:
                    self.resource_checkout(new_vm)
                    log.info("Launching 1 VM: %s on %s under tenant: %s" % (instance_id, self.name, self.tenant_name))
                except:
                    log.error("Unexpected Error checking out resources when creating a VM. Programming error?")
                    self.vm_destroy(new_vm, reason="Failed Resource checkout")
                    return self.ERROR
        
                self.vms.append(new_vm)
            else:
                log.debug("Failed to create instance on %s" % self.name)
                return self.ERROR
        else:
            log.debug("Unable to generate name for %" % self.name)
            return self.ERROR

        return 0
Esempio n. 5
0
    def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc,
                  vm_image, vm_mem, vm_cores, vm_storage, customization=None,
                  pre_customization=None, vm_keepalive=0, instance_type="",
                  maximum_price=0, job_per_core=False, securitygroup=[],
                  key_name="",use_cloud_init=False, extra_userdata=[]):
        """Attempt to boot a new VM on the cluster."""

        use_cloud_init = use_cloud_init or config.use_cloud_init
        log.verbose("Trying to boot %s on %s" % (vm_type, self.network_address))
        if len(securitygroup) != 0:
            sec_group = []
            for group in securitygroup:
                if group in self.security_groups:
                    sec_group.append(group)
            if len(sec_group) == 0:
                log.debug("No matching security groups - trying default config")
                sec_group = self.security_groups
                #sec_group.append("default") - don't just append default use what is in cloud_resources.conf for this cloud
        else:
            sec_group = self.security_groups

        try:
            if self.name in vm_image.keys():
                vm_ami = vm_image[self.name]
            else:
                vm_ami = vm_image[self.network_address]
        except:
            log.debug("No AMI for %s, trying default" % self.network_address)
            #try:
            #    vm_ami = vm_image["default"]
            #except:
                #log.debug("No given default - trying global defaults")
            try:
                vm_default_ami = _attr_list_to_dict(config.default_VMAMI)
                if self.name in vm_default_ami.keys():
                    vm_ami = vm_default_ami[self.name]
                else:
                    vm_ami = vm_default_ami[self.network_address]
            except:
                try:
                    vm_ami = vm_default_ami["default"]
                except:
                    log.exception("Can't find a suitable AMI")
                    self.failed_image_set.add(vm_ami)
                    return

        try:
            if self.name in instance_type.keys():
                i_type = instance_type[self.name]
            elif self.network_address in instance_type.keys():
                i_type = instance_type[self.network_address]
            else:
                i_type = instance_type["default"]
        except:
            log.debug("No instance type for %s, trying default" % self.network_address)
            #try:
            #    i_type = instance_type["default"]
            #except:
            #    if isinstance(instance_type, str):
            #        i_type = instance_type
            #    else:
            try:
                if self.name in self.DEFAULT_INSTANCE_TYPE_LIST.keys():
                    i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.name]
                else:
                    i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.network_address]
            except:
                log.debug("No default instance type found for %s, trying single default" % self.network_address)
                i_type = self.DEFAULT_INSTANCE_TYPE
        instance_type = i_type

        if key_name == "" or key_name == None:
            key_name = self.key_name if self.key_name else ""

        if customization:
            if not use_cloud_init:
                user_data = nimbus_xml.ws_optional(customization)
            else:
                user_data = cloud_init_util.build_write_files_cloud_init(customization)
        else:
            user_data = ""

        if pre_customization:
            if not use_cloud_init:
                for item in pre_customization:
                    user_data = '\n'.join([item, user_data])
            else:
                user_data = cloud_init_util.inject_customizations(pre_customization, user_data)
        elif use_cloud_init:
            user_data = cloud_init_util.inject_customizations([], user_data)[0]
        if len(extra_userdata) > 0:
            # need to use the multi-mime type functions
            user_data = cloud_init_util.build_multi_mime_message([(user_data, 'cloud-config', 'cloud_conf.yaml')], extra_userdata)

        if "AmazonEC2" == self.cloud_type and vm_networkassoc != "public":
            log.debug("You requested '%s' networking, but EC2 only supports 'public'" % vm_networkassoc)
            addressing_type = "public"
        else:
            addressing_type = vm_networkassoc

        user_data = utilities.gzip_userdata(user_data)
        try:
            client = self._get_connection()
            #Uncomment for debugging boto calls
            #boto3.set_stream_logger('botocore')
            #boto_file_handler = logging.handlers.WatchedFileHandler('/tmp/csboto3.yaml', )
            #botolog = logging.getLogger('botocore')
            #botolog.addHandler(boto_file_handler)
            resp = client.run_instances(ImageId=vm_ami, MinCount=1, MaxCount=1, InstanceType=instance_type, UserData=user_data, KeyName=key_name, SecurityGroups=sec_group)
            # will need to figure out how PlacementGroups will work still will probably just be Placement={"AvailabilityZone':placement_zone}
        except Exception as e:
            log.error("Problem creating instance %s" % e.__dict__)
            return self.ERROR

        if not vm_keepalive and self.keep_alive: #if job didn't set a keep_alive use the clouds default
            vm_keepalive = self.keep_alive
        if 'Instances' in resp.keys():
            new_vm_id =resp['Instances'][0]['InstanceId']
        else:
            #print resp.keys()
            new_vm_id = "unable to get id"
            return self.ERROR
        new_vm = cluster_tools.VM(name = vm_name, id = new_vm_id, vmtype = vm_type, user = vm_user,
                    clusteraddr = self.network_address,
                    cloudtype = self.cloud_type, network = vm_networkassoc,
                    image= vm_ami, flavor=instance_type,
                    memory = vm_mem, cpucores = vm_cores, storage = vm_storage,
                    keep_alive = vm_keepalive, job_per_core = job_per_core)

        #try:
        #    new_vm.spot_id = spot_id
        #except:
        #    log.verbose("No spot ID to add to VM %s" % instance_id)

        try:
            self.resource_checkout(new_vm)
        except:
            log.exception("Unexpected Error checking out resources when creating a VM. Programming error?")
            self.vm_destroy(new_vm, reason="Failed Resource checkout", return_resources=False)
            return self.ERROR

        self.vms.append(new_vm)

        return 0