def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc, vm_cpuarch, vm_image, vm_mem, vm_cores, vm_storage, customization=None, vm_keepalive=0, instance_type="", maximum_price=0, job_per_core=False, securitygroup=[]): try: vm_ami = vm_image[self.network_address] except: log.debug("No AMI for %s, trying default" % self.network_address) try: vm_ami = vm_image["default"] except: log.exception("Can't find a suitable AMI") return # Construct URLs #if instance_type: # vm_instance_type = instance_type #else: # vm_instance_type = self.DEFAULT_MACHINE_TYPE try: if self.name in instance_type.keys(): i_type = instance_type[self.name] else: i_type = instance_type[self.network_address] except: log.debug("No instance type for %s, trying default" % self.network_address) try: if self.name in self.DEFAULT_INSTANCE_TYPE_LIST.keys(): i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.name] else: i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.network_address] except: log.debug("No default instance type found for %s, trying single default" % self.network_address) i_type = self.DEFAULT_MACHINE_TYPE instance_type = i_type if vm_image: vm_image_name = vm_ami else: vm_image_name = self.DEFAULT_IMAGE image_url = '%s%s/global/images/%s' % ( self.GCE_URL, self.project_id, vm_image_name) machine_type_url = '%s/zones/%s/machineTypes/%s' % ( self.project_url, self.DEFAULT_ZONE, vm_instance_type) #zone_url = '%s/zones/%s' % (self.project_url, self.DEFAULT_ZONE) network_url = '%s/global/networks/%s' % (self.project_url, self.DEFAULT_NETWORK) if customization: user_data = nimbus_xml.ws_optional(customization) else: user_data = "" next_instance_name = self.generate_next_instance_name() # Construct the request body instance = { 'name': next_instance_name, 'machineType': machine_type_url, 'image': image_url, 'networkInterfaces': [{ 'accessConfigs': [{ 'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT' }], 'network': network_url }], 'serviceAccounts': [{ 'email': self.DEFAULT_SERVICE_EMAIL, 'scopes': self.DEFAULT_SCOPES }], 'metadata': { 'items': [ { 'key': 'user-data', 'value': user_data, }, # { # 'key': 'startup-script', # 'value': user_script, # } ] } } # Create the instance response = None request = self.gce_service.instances().insert( project=self.project_id, body=instance, zone=self.DEFAULT_ZONE) try: response = request.execute(self.auth_http) response = self._blocking_call(self.gce_service, self.auth_http, response) except Exception, e: log.error("Error creating VM on gce: %s" % e) pass
def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc, vm_image, vm_mem, vm_cores, vm_storage, customization=None, pre_customization=None, vm_keepalive=0, instance_type="", maximum_price=0, job_per_core=False, securitygroup=[], key_name="",use_cloud_init=False, extra_userdata=[]): """Attempt to boot a new VM on the cluster.""" use_cloud_init = use_cloud_init or config.use_cloud_init log.verbose("Trying to boot %s on %s" % (vm_type, self.network_address)) if len(securitygroup) != 0: sec_group = [] for group in securitygroup: if group in self.security_groups: sec_group.append(group) if len(sec_group) == 0: log.debug("No matching security groups - trying default config") sec_group = self.security_groups #sec_group.append("default") - don't just append default use what is in cloud_resources.conf for this cloud else: sec_group = self.security_groups try: if self.name in vm_image.keys(): vm_ami = vm_image[self.name] else: vm_ami = vm_image[self.network_address] except: log.debug("No AMI for %s, trying default" % self.network_address) #try: # vm_ami = vm_image["default"] #except: #log.debug("No given default - trying global defaults") try: vm_default_ami = _attr_list_to_dict(config.default_VMAMI) if self.name in vm_default_ami.keys(): vm_ami = vm_default_ami[self.name] else: vm_ami = vm_default_ami[self.network_address] except: try: vm_ami = vm_default_ami["default"] except: log.exception("Can't find a suitable AMI") self.failed_image_set.add(vm_ami) return try: if self.name in instance_type.keys(): i_type = instance_type[self.name] elif self.network_address in instance_type.keys(): i_type = instance_type[self.network_address] else: i_type = instance_type["default"] except: log.debug("No instance type for %s, trying default" % self.network_address) #try: # i_type = instance_type["default"] #except: # if isinstance(instance_type, str): # i_type = instance_type # else: try: if self.name in self.DEFAULT_INSTANCE_TYPE_LIST.keys(): i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.name] else: i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.network_address] except: log.debug("No default instance type found for %s, trying single default" % self.network_address) i_type = self.DEFAULT_INSTANCE_TYPE instance_type = i_type if key_name == None: key_name = self.key_name if customization: if not use_cloud_init: user_data = nimbus_xml.ws_optional(customization) else: user_data = cloud_init_util.build_write_files_cloud_init(customization) else: user_data = "" if pre_customization: if not use_cloud_init: for item in pre_customization: user_data = '\n'.join([item, user_data]) else: user_data = cloud_init_util.inject_customizations(pre_customization, user_data) elif use_cloud_init: user_data = cloud_init_util.inject_customizations([], user_data)[0] if len(extra_userdata) > 0: # need to use the multi-mime type functions user_data = cloud_init_util.build_multi_mime_message([(user_data, 'cloud-config', 'cloud_conf.yaml')], extra_userdata) if "AmazonEC2" == self.cloud_type and vm_networkassoc != "public": log.debug("You requested '%s' networking, but EC2 only supports 'public'" % vm_networkassoc) addressing_type = "public" else: addressing_type = vm_networkassoc try: connection = self._get_connection() image = None if not "Eucalyptus" == self.cloud_type: image = connection.get_image(vm_ami) else: #HACK: for some reason Eucalyptus won't respond properly to # get_image("whateverimg"). Use a linear search until # this is fixed # This is Eucalyptus bug #495670 # https://bugs.launchpad.net/eucalyptus/+bug/495670 images = connection.get_all_images() for potential_match in images: if potential_match.id == vm_ami: image = potential_match break # Compress the user data to try and get under the limit user_data = utilities.gzip_userdata(user_data) if image: if maximum_price is 0 or self.cloud_type == "OpenStack": # don't request a spot instance try: reservation = image.run(1,1, key_name=key_name, addressing_type=addressing_type, user_data=user_data, placement=self.placement_zone, security_groups=sec_group, instance_type=instance_type) instance_id = reservation.instances[0].id log.debug("Booted VM %s" % instance_id) except boto.exception.EC2ResponseError, e: log.exception("There was a problem creating an EC2 instance: %s" % e) return self.ERROR except Exception, e: log.exception("There was an unexpected problem creating an EC2 instance: %s" % e) return self.ERROR
def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc, vm_cpuarch, vm_image, vm_mem, vm_cores, vm_storage, customization=None, vm_keepalive=0, instance_type="", maximum_price=0, job_per_core=False, securitygroup=[],key_name=""): """Attempt to boot a new VM on the cluster.""" #print vm_image #print instance_type #print securitygroup log.verbose("Trying to boot %s on %s" % (vm_type, self.network_address)) if len(securitygroup) != 0: sec_group = [] for group in securitygroup: if group in self.security_groups: sec_group.append(group) if len(sec_group) == 0: log.warning("No matching security groups - trying default config") sec_group = self.security_groups #sec_group.append("default") - don't just append default use what is in cloud_resources.conf for this cloud else: sec_group = self.security_groups try: if self.name in vm_image.keys(): vm_ami = vm_image[self.name] else: vm_ami = vm_image[self.network_address] except: log.debug("No AMI for %s, trying default" % self.network_address) #try: # vm_ami = vm_image["default"] #except: #log.debug("No given default - trying global defaults") try: vm_default_ami = _attr_list_to_dict(config.default_VMAMI) if self.name in vm_default_ami.keys(): vm_ami = vm_default_ami[self.name] else: vm_ami = vm_default_ami[self.network_address] except: try: vm_ami = vm_default_ami["default"] except: log.exception("Can't find a suitable AMI") return try: if self.name in instance_type.keys(): i_type = instance_type[self.name] else: i_type = instance_type[self.network_address] except: log.debug("No instance type for %s, trying default" % self.network_address) #try: # i_type = instance_type["default"] #except: # if isinstance(instance_type, str): # i_type = instance_type # else: try: if self.name in self.DEFAULT_INSTANCE_TYPE_LIST.keys(): i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.name] else: i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.network_address] except: log.debug("No default instance type found for %s, trying single default" % self.network_address) i_type = self.DEFAULT_INSTANCE_TYPE instance_type = i_type if key_name == None: key_name = self.key_name if customization: user_data = nimbus_xml.ws_optional(customization) else: user_data = "" if "AmazonEC2" == self.cloud_type and vm_networkassoc != "public": log.debug("You requested '%s' networking, but EC2 only supports 'public'" % vm_networkassoc) addressing_type = "public" else: addressing_type = vm_networkassoc try: connection = self._get_connection() image = None if not "Eucalyptus" == self.cloud_type: image = connection.get_image(vm_ami) else: #HACK: for some reason Eucalyptus won't respond properly to # get_image("whateverimg"). Use a linear search until # this is fixed # This is Eucalyptus bug #495670 # https://bugs.launchpad.net/eucalyptus/+bug/495670 images = connection.get_all_images() for potential_match in images: if potential_match.id == vm_ami: image = potential_match break if image: if maximum_price is 0: # don't request a spot instance try: reservation = image.run(1,1, key_name=key_name, addressing_type=addressing_type, user_data=user_data, placement=self.placement_zone, security_groups=sec_group, instance_type=instance_type) instance_id = reservation.instances[0].id log.debug("Booted VM %s" % instance_id) except: log.exception("There was a problem creating an EC2 instance...") return self.ERROR else: # get a spot instance of no more than maximum_price try: price_in_dollars = str(float(maximum_price) / 100) reservation = connection.request_spot_instances( price_in_dollars, image.id, key_name=key_name, user_data=user_data, placement=self.placement_zone, addressing_type=addressing_type, security_groups=self.sec_group, instance_type=instance_type) spot_id = str(reservation[0].id) instance_id = "" log.debug("Reserved instance %s at no more than %s" % (spot_id, price_in_dollars)) except AttributeError: log.exception("Your version of boto doesn't seem to support "\ "spot instances. You need at least 1.9") return self.ERROR except: log.exception("Problem creating an EC2 spot instance...") return self.ERROR else: log.error("Couldn't find image %s on %s" % (vm_image, self.name)) return self.ERROR except: log.exception("Problem creating EC2 instance on on %s" % self.name) return self.ERROR vm_mementry = self.find_mementry(vm_mem) if (vm_mementry < 0): #TODO: this is kind of pointless with EC2... log.debug("Cluster memory list has no sufficient memory " +\ "entries (Not supposed to happen). Returning error.") return self.ERROR log.verbose("vm_create - Memory entry found in given cluster: %d" % vm_mementry) new_vm = cluster_tools.VM(name = vm_name, id = instance_id, vmtype = vm_type, user = vm_user, clusteraddr = self.network_address, cloudtype = self.cloud_type, network = vm_networkassoc, cpuarch = vm_cpuarch, image= vm_image, memory = vm_mem, mementry = vm_mementry, cpucores = vm_cores, storage = vm_storage, keep_alive = vm_keepalive, job_per_core = job_per_core) try: new_vm.spot_id = spot_id except: log.verbose("No spot ID to add to VM %s" % instance_id) try: self.resource_checkout(new_vm) except: log.exception("Unexpected Error checking out resources when creating a VM. Programming error?") self.vm_destroy(new_vm, reason="Failed Resource checkout") return self.ERROR self.vms.append(new_vm) return 0
def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc, vm_image, vm_mem, vm_cores, vm_storage, customization=None, vm_keepalive=0, instance_type="", job_per_core=False, securitygroup=[],key_name="", pre_customization=None, use_cloud_init=False, extra_userdata=[]): """ Create a VM on OpenStack.""" import novaclient.exceptions use_cloud_init = use_cloud_init or config.use_cloud_init nova = self._get_creds_nova() if len(securitygroup) != 0: sec_group = [] for group in securitygroup: if group in self.security_groups: sec_group.append(group) if len(sec_group) == 0: log.debug("No defined security groups for job - trying default value from cloud_resources.conf") sec_group = self.security_groups else: sec_group = self.security_groups log.debug("Using security group: %s" % str(sec_group)) if key_name and len(key_name) > 0: if not nova.keypairs.findall(name=key_name): key_name = "" else: key_name = self.key_name if self.key_name else "" if customization: if not use_cloud_init: user_data = nimbus_xml.ws_optional(customization) else: user_data = cloud_init_util.build_write_files_cloud_init(customization) else: user_data = "" if pre_customization: if not use_cloud_init: for item in pre_customization: user_data = '\n'.join([item, user_data]) else: user_data = cloud_init_util.inject_customizations(pre_customization, user_data) elif use_cloud_init: user_data = cloud_init_util.inject_customizations([], user_data) if len(extra_userdata) > 0: # need to use the multi-mime type functions user_data = cloud_init_util.build_multi_mime_message([(user_data, 'cloud-config', 'cloud_conf.yaml')], extra_userdata) # Compress the user data to try and get under the limit user_data = utilities.gzip_userdata(user_data) try: if self.name in vm_image.keys(): image = vm_image[self.name] elif self.network_address in vm_image.keys(): image = vm_image[self.network_address] else: image = vm_image['default'] except: try: vm_default_ami = _attr_list_to_dict(config.default_VMAMI) if self.name in vm_default_ami.keys(): image = vm_default_ami[self.name] else: image = vm_default_ami[self.network_address] except: try: image = vm_default_ami["default"] except: log.exception("Can't find a suitable AMI") return try: imageobj = nova.images.find(name=image) except novaclient.exceptions.EndpointNotFound: log.error("Endpoint not found, are your region settings correct for %s" % self.name) return -4 except Exception as e: log.warning("Exception occurred while trying to fetch image via name: %s %s" % (image, e)) try: imageobj = nova.images.get(image) log.debug("Got image via uuid: %s" % image) except novaclient.exceptions.EndpointNotFound: log.error("Endpoint not found, are your region settings correct for %s" % self.name) return -4 except Exception as e: log.exception("Unable to fetch image via uuid: %s %s" % (image, e)) self.failed_image_set.add(image) return try: if self.name in instance_type.keys(): i_type = instance_type[self.name] elif self.network_address in instance_type.keys(): i_type = instance_type[self.network_address] else: i_type = instance_type['default'] except: log.debug("No instance type for %s, trying default" % self.network_address) try: if self.name in self.DEFAULT_INSTANCE_TYPE_LIST.keys(): i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.name] else: i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.network_address] except: log.debug("No default instance type found for %s, trying single default" % self.network_address) i_type = self.DEFAULT_INSTANCE_TYPE try: flavor = nova.flavors.find(name=i_type) except Exception as e: log.warning("Exception occurred while trying to get flavor by name: %s - will attempt to use name value as a uuid." % e) try: flavor = nova.flavors.get(i_type) log.debug("Got flavor via uuid: %s" % i_type) except Exception as ex: log.error("Exception occurred trying to get flavor by uuid: %s" % ex) return self.flavor_set.add(flavor) # find the network id to use if more than one network if vm_networkassoc: network = self._find_network(vm_networkassoc) if network: netid = [{'net-id': network.id}] else: log.debug("Unable to find network named: %s on %s" % (vm_networkassoc, self.name)) netid = [] elif self.network_pools and len(self.network_pools) > 0: network = self._find_network(self.network_pools[0]) if network: netid = [{'net-id': network.id}] else: log.debug("Unable to find network named: %s on %s" % (self.network_pools[0], self.name)) netid = [] else: netid = [] # Need to get the rotating hostname from the google code to use for here. name = self._generate_next_name() instance = None if name: try: instance = nova.servers.create(name=name, image=imageobj, flavor=flavor, key_name=key_name, availability_zone=self.placement_zone, nics =netid, userdata=user_data, security_groups=sec_group) #print instance.__dict__ except novaclient.exceptions.OverLimit as e: log.info("Unable to create VM without exceeded quota on %s: %s" % (self.name, e.message)) except Exception as e: #print e log.error("Unhandled exception while creating vm on %s: %s" %(self.name, e)) if instance: instance_id = instance.id if not vm_keepalive and self.keep_alive: #if job didn't set a keep_alive use the clouds default vm_keepalive = self.keep_alive new_vm = cluster_tools.VM(name = vm_name, id = instance_id, vmtype = vm_type, user = vm_user, clusteraddr = self.network_address, hostname = ''.join([name, self.vm_domain_name]), cloudtype = self.cloud_type, network = vm_networkassoc, image= vm_image, flavor=flavor.name, memory = vm_mem, cpucores = vm_cores, storage = vm_storage, keep_alive = vm_keepalive, job_per_core = job_per_core) try: self.resource_checkout(new_vm) log.info("Launching 1 VM: %s on %s under tenant: %s" % (instance_id, self.name, self.tenant_name)) except: log.error("Unexpected Error checking out resources when creating a VM. Programming error?") self.vm_destroy(new_vm, reason="Failed Resource checkout") return self.ERROR self.vms.append(new_vm) else: log.debug("Failed to create instance on %s" % self.name) return self.ERROR else: log.debug("Unable to generate name for %" % self.name) return self.ERROR return 0
def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc, vm_cpuarch, vm_image, vm_mem, vm_cores, vm_storage, customization=None, vm_keepalive=0, instance_type="", maximum_price=0, job_per_core=False, securitygroup=[]): """Attempt to boot a new VM on the cluster.""" log.verbose("Trying to boot %s on %s" % (vm_type, self.network_address)) if len(securitygroup) != 0: sec_group = [] for group in securitygroup: if group in self.security_groups: sec_group.append(group) if len(sec_group) == 0: log.warning("No matching security groups - trying default") sec_group.append("default") else: sec_group = self.security_groups try: vm_ami = vm_image[self.network_address] except: log.debug("No AMI for %s, trying default" % self.network_address) try: vm_ami = vm_image["default"] except: log.exception("Can't find a suitable AMI") return try: i_type = instance_type[self.network_address] except: log.debug("No instance type for %s, trying default" % self.network_address) try: i_type = instance_type["default"] except: if isinstance(instance_type, str): i_type = instance_type else: i_type = self.DEFAULT_INSTANCE_TYPE instance_type = i_type if customization: user_data = nimbus_xml.ws_optional(customization) else: user_data = "" if "AmazonEC2" == self.cloud_type and vm_networkassoc != "public": log.debug("You requested '%s' networking, but EC2 only supports 'public'" % vm_networkassoc) addressing_type = "public" else: addressing_type = vm_networkassoc try: connection = self._get_connection() image = None if not "Eucalyptus" == self.cloud_type: image = connection.get_image(vm_ami) else: #HACK: for some reason Eucalyptus won't respond properly to # get_image("whateverimg"). Use a linear search until # this is fixed # This is Eucalyptus bug #495670 # https://bugs.launchpad.net/eucalyptus/+bug/495670 images = connection.get_all_images() for potential_match in images: if potential_match.id == vm_ami: image = potential_match break if image: if maximum_price is 0: # don't request a spot instance try: reservation = image.run(1,1, key_name=self.key_name, addressing_type=addressing_type, user_data=user_data, placement=self.placement_zone, security_groups=sec_group, instance_type=instance_type) instance_id = reservation.instances[0].id log.debug("Booted VM %s" % instance_id) except: log.exception("There was a problem creating an EC2 instance...") return self.ERROR else: # get a spot instance of no more than maximum_price try: price_in_dollars = str(float(maximum_price) / 100) reservation = connection.request_spot_instances( price_in_dollars, image.id, key_name=self.key_name, user_data=user_data, placement=self.placement_zone, addressing_type=addressing_type, security_groups=self.sec_group, instance_type=instance_type) spot_id = str(reservation[0].id) instance_id = "" log.debug("Reserved instance %s at no more than %s" % (spot_id, price_in_dollars)) except AttributeError: log.exception("Your version of boto doesn't seem to support "\ "spot instances. You need at least 1.9") return self.ERROR except: log.exception("Problem creating an EC2 spot instance...") return self.ERROR else: log.error("Couldn't find image %s on %s" % (vm_image, self.name)) return self.ERROR except: log.exception("Problem creating EC2 instance on on %s" % self.name) return self.ERROR vm_mementry = self.find_mementry(vm_mem) if (vm_mementry < 0): #TODO: this is kind of pointless with EC2... log.debug("Cluster memory list has no sufficient memory " +\ "entries (Not supposed to happen). Returning error.") return self.ERROR log.verbose("vm_create - Memory entry found in given cluster: %d" % vm_mementry) new_vm = cluster_tools.VM(name = vm_name, id = instance_id, vmtype = vm_type, user = vm_user, clusteraddr = self.network_address, cloudtype = self.cloud_type, network = vm_networkassoc, cpuarch = vm_cpuarch, image= vm_image, memory = vm_mem, mementry = vm_mementry, cpucores = vm_cores, storage = vm_storage, keep_alive = vm_keepalive, job_per_core = job_per_core) try: new_vm.spot_id = spot_id except: log.verbose("No spot ID to add to VM %s" % instance_id) try: self.resource_checkout(new_vm) except: log.exception("Unexpected Error checking out resources when creating a VM. Programming error?") self.vm_destroy(new_vm, reason="Failed Resource checkout") return self.ERROR self.vms.append(new_vm) return 0
def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc, vm_cpuarch, vm_image, vm_mem, vm_cores, vm_storage, customization=None, vm_keepalive=0, instance_type="", maximum_price=0, job_per_core=False, securitygroup=[]): try: vm_ami = vm_image[self.network_address] except: log.debug("No AMI for %s, trying default" % self.network_address) try: vm_ami = vm_image["default"] except: log.exception("Can't find a suitable AMI") return # Construct URLs if instance_type: vm_instance_type = instance_type else: vm_instance_type = self.DEFAULT_MACHINE_TYPE if vm_image: vm_image_name = vm_ami else: vm_image_name = self.DEFAULT_IMAGE image_url = '%s%s/images/%s' % ( self.GCE_URL, self.project_id, vm_image_name) machine_type_url = '%s/machineTypes/%s' % ( self.project_url, vm_instance_type) zone_url = '%s/zones/%s' % (self.project_url, self.DEFAULT_ZONE) network_url = '%s/networks/%s' % (self.project_url, self.DEFAULT_NETWORK) if customization: user_data = nimbus_xml.ws_optional(customization) else: user_data = "" next_instance_name = self.generate_next_instance_name() # Construct the request body instance = { 'name': next_instance_name, 'machineType': machine_type_url, 'image': image_url, 'zone': zone_url, 'networkInterfaces': [{ 'accessConfigs': [{ 'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT' }], 'network': network_url }], 'serviceAccounts': [{ 'email': self.DEFAULT_SERVICE_EMAIL, 'scopes': self.DEFAULT_SCOPES }], 'metadata': { 'items': [{ 'key': 'userdata', 'value': user_data, }] } } # Create the instance request = self.gce_service.instances().insert( project=self.project_id, body=instance) response = request.execute(self.auth_http) response = self._blocking_call(self.gce_service, self.auth_http, response) if 'targetId' in response: target_id = response['targetId'] else: print 'targetID missing' print response return vm_mementry = self.find_mementry(vm_mem) if (vm_mementry < 0): #TODO: this is kind of pointless with EC2..., but the resource code depends on it log.debug("Cluster memory list has no sufficient memory " +\ "entries (Not supposed to happen). Returning error.") return self.ERROR new_vm = cluster_tools.VM(name = next_instance_name, vmtype = vm_type, user = vm_user, clusteraddr = self.network_address, id = target_id, cloudtype = self.cloud_type, network = vm_networkassoc, hostname = self.construct_hostname(next_instance_name), cpuarch = vm_cpuarch, image= vm_image, mementry = vm_mementry, memory = vm_mem, cpucores = vm_cores, storage = vm_storage, keep_alive = vm_keepalive, job_per_core = job_per_core) try: self.resource_checkout(new_vm) except: log.exception("Unexpected Error checking out resources when creating a VM. Programming error?") self.vm_destroy(new_vm, reason="Failed Resource checkout") return self.ERROR self.vms.append(new_vm) return 0
def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc, vm_cpuarch, vm_image, vm_mem, vm_cores, vm_storage, customization=None, vm_keepalive=0, instance_type="", maximum_price=0, job_per_core=False, securitygroup=[]): try: vm_ami = vm_image[self.network_address] except: log.debug("No AMI for %s, trying default" % self.network_address) try: vm_ami = vm_image["default"] except: log.exception("Can't find a suitable AMI") return # Construct URLs #if instance_type: # vm_instance_type = instance_type #else: # vm_instance_type = self.DEFAULT_MACHINE_TYPE try: if self.name in instance_type.keys(): i_type = instance_type[self.name] else: i_type = instance_type[self.network_address] except: log.debug("No instance type for %s, trying default" % self.network_address) try: if self.name in self.DEFAULT_INSTANCE_TYPE_LIST.keys(): i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.name] else: i_type = self.DEFAULT_INSTANCE_TYPE_LIST[ self.network_address] except: log.debug( "No default instance type found for %s, trying single default" % self.network_address) i_type = self.DEFAULT_MACHINE_TYPE instance_type = i_type if vm_image: vm_image_name = vm_ami else: vm_image_name = self.DEFAULT_IMAGE image_url = '%s%s/global/images/%s' % (self.GCE_URL, self.project_id, vm_image_name) machine_type_url = '%s/zones/%s/machineTypes/%s' % ( self.project_url, self.DEFAULT_ZONE, vm_instance_type) #zone_url = '%s/zones/%s' % (self.project_url, self.DEFAULT_ZONE) network_url = '%s/global/networks/%s' % (self.project_url, self.DEFAULT_NETWORK) if customization: user_data = nimbus_xml.ws_optional(customization) else: user_data = "" next_instance_name = self.generate_next_instance_name() # Construct the request body instance = { 'name': next_instance_name, 'machineType': machine_type_url, 'image': image_url, 'networkInterfaces': [{ 'accessConfigs': [{ 'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT' }], 'network': network_url }], 'serviceAccounts': [{ 'email': self.DEFAULT_SERVICE_EMAIL, 'scopes': self.DEFAULT_SCOPES }], 'metadata': { 'items': [ { 'key': 'user-data', 'value': user_data, }, # { # 'key': 'startup-script', # 'value': user_script, # } ] } } # Create the instance response = None request = self.gce_service.instances().insert(project=self.project_id, body=instance, zone=self.DEFAULT_ZONE) try: response = request.execute(self.auth_http) response = self._blocking_call(self.gce_service, self.auth_http, response) except Exception, e: log.error("Error creating VM on gce: %s" % e) pass
def vm_create(self, vm_name, vm_type, vm_networkassoc, vm_cpuarch, vm_image, vm_mem, vm_cores, vm_storage, customization=None, vm_keepalive=0, instance_type="", maximum_price=0, job_per_core=False): log.debug("Trying to boot %s on %s" % (vm_type, self.network_address)) if not instance_type: instance_type = self.DEFAULT_INSTANCE_TYPE if customization: user_data = nimbus_xml.ws_optional(customization) else: user_data = "" try: connection = self._get_connection() image = None if not "Eucalyptus" == self.cloud_type: image = connection.get_image(vm_image) else: #HACK: for some reason Eucalyptus won't respond properly to # get_image("whateverimg"). Use a linear search until # this is fixed # This is Eucalyptus bug #495670 # https://bugs.launchpad.net/eucalyptus/+bug/495670 images = connection.get_all_images() for potential_match in images: if potential_match.id == vm_image: image = potential_match break if image: if maximum_price is 0: # don't request a spot instance try: reservation = image.run(1,1, user_data=user_data, security_groups=self.security_groups, instance_type=instance_type) instance_id = reservation.instances[0].id log.debug("Booted VM %s" % instance_id) except: log.exception("There was a problem creating an EC2 instance...") return self.ERROR else: # get a spot instance of no more than maximum_price try: price_in_dollars = str(float(maximum_price) / 100) reservation = connection.request_spot_instances( price_in_dollars, image.id, user_data=user_data, security_groups=self.security_groups, instance_type=instance_type) spot_id = str(reservation[0].id) instance_id = "" log.debug("Reserved instance %s at no more than %s" % (spot_id, price_in_dollars)) except AttributeError: log.exception("Your version of boto doesn't seem to support "\ "spot instances. You need at least 1.9") return self.ERROR except: log.exception("Problem creating an EC2 spot instance...") return self.ERROR else: log.error("Couldn't find image %s on %s" % (vm_image, self.name)) return self.ERROR except boto.exception.EC2ResponseError, e: log.error("Couldn't boot VM because: %s" % e.error_message) return self.ERROR
def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc, vm_image, vm_mem, vm_cores, vm_storage, customization=None, vm_keepalive=0, instance_type="", maximum_price=0, job_per_core=False, securitygroup=[],pre_customization=None,use_cloud_init=False, extra_userdata=[]): try: if self.network_address in vm_image.keys(): vm_ami = vm_image[self.network_address] elif self.name in vm_image.keys(): vm_ami = vm_image[self.name] except: log.debug("No AMI for %s, trying default" % self.network_address) try: vm_ami = vm_image["default"] except: log.exception("Can't find a suitable AMI") return # Construct URLs try: if self.name in instance_type.keys(): i_type = instance_type[self.name] else: i_type = instance_type[self.network_address] except: log.debug("No instance type for %s, trying default" % self.network_address) try: if self.name in self.DEFAULT_INSTANCE_TYPE_LIST.keys(): i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.name] else: i_type = self.DEFAULT_INSTANCE_TYPE_LIST[self.network_address] except: log.debug("No default instance type found for %s, trying single default" % self.network_address) i_type = self.DEFAULT_MACHINE_TYPE vm_instance_type = i_type if vm_image: vm_image_name = vm_ami else: vm_image_name = self.DEFAULT_IMAGE #this should replace disk_url when cloud-init supports GCE in CERNVM3 image_url = '%s%s/global/images/%s' % ( self.GCE_URL, self.project_id, vm_image_name) #Ensures the VM's Root Disks are Unique self.DEFAULT_ROOT_PD_NAME = '%s-%s'%('hepgc-uvic-root-pd',self.generate_next_instance_name()) #temporary variable for disk_url #https://www.googleapis.com/compute/v1/projects/atlasgce/zones/us-central1-b/disks/hepgc-uvic-root-pd disk_url = '%s%s/zones/%s/disks/%s'%(self.GCE_URL,self.project_id,self.DEFAULT_ZONE,self.DEFAULT_ROOT_PD_NAME) machine_type_url = '%s/zones/%s/machineTypes/%s' % ( self.project_url, self.DEFAULT_ZONE, vm_instance_type) #zone_url = '%s/zones/%s' % (self.project_url, self.DEFAULT_ZONE) network_url = '%s/global/networks/%s' % (self.project_url, self.DEFAULT_NETWORK) # Construct the request body disk = { 'name': self.DEFAULT_ROOT_PD_NAME, 'sourceSnapshot':'https://www.googleapis.com/compute/v1/projects/atlasgce/global/snapshots/%s'%vm_image_name, 'sizeGb':vm_storage } # Create the root pd try: request = self.gce_service.disks().insert(project=self.project_id, body=disk, zone=self.DEFAULT_ZONE) response = request.execute(http=self.auth_http) response = self._blocking_call(self.gce_service, self.auth_http, response) except: log.exception('Error Trying to create disk, one already exists ... returning ') return use_cloud_init = use_cloud_init or config.use_cloud_init if customization: if not use_cloud_init: user_data = nimbus_xml.ws_optional(customization) else: user_data = cloud_init_util.build_write_files_cloud_init(customization) else: user_data = "" if pre_customization: if not use_cloud_init: for item in pre_customization: user_data = '\n'.join([item, user_data]) else: user_data = cloud_init_util.inject_customizations(pre_customization, user_data) elif use_cloud_init: user_data = cloud_init_util.inject_customizations([], user_data)[0] if len(extra_userdata) > 0: # need to use the multi-mime type functions user_data = cloud_init_util.build_multi_mime_message([(user_data, 'cloud-config')], extra_userdata) # Compress the user data to try and get under the limit user_data = utilities.gzip_userdata(user_data) next_instance_name = self.generate_next_instance_name() instance = { 'name': next_instance_name, 'machineType': machine_type_url, #'disks': [{ # 'autoDelete': 'true', # 'boot': 'true', # 'type': 'PERSISTENT', # 'initializeParams' : { # 'diskname': self.DEFAULT_ROOT_PD_NAME, # 'sourceImage': image_url # } # }], 'disks': [{ 'source':disk_url, 'boot': 'true', 'autoDelete':'true', 'type': 'PERSISTENT', }], #'image': image_url, 'networkInterfaces': [{ 'accessConfigs': [{ 'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT' }], 'network': network_url }], 'serviceAccounts': [{ 'email': self.DEFAULT_SERVICE_EMAIL, 'scopes': self.DEFAULT_SCOPES }], 'metadata': { 'items': [ { 'key': 'user-data', 'value': user_data , }, ] } } # Create the instance response = None request = self.gce_service.instances().insert( project=self.project_id, body=instance, zone=self.DEFAULT_ZONE) try: response = request.execute(http=self.auth_http) response = self._blocking_call(self.gce_service, self.auth_http, response) #log.info('Created VM ') except Exception, e: log.error("Error creating VM on gce: %s" % e) pass