def start_ec2_instances(self, image_id=None, key_name=None, group_name=None, num=1, instance_type=None): if key_name is None: key_name = self.config['key_name'] if group_name is None: group_name = self.config['group_name'] if num is None: num = 1 if instance_type is None: instance_type = self.config['default_instance_type'] try: img = self.conn.get_all_images(image_ids=[image_id])[0] except IndexError: raise ProviderException("Could not find image_id={0}".format(image_id)) if img.state != "available": if img.state != "pending": raise ProviderException("Image {0} is not available, it's state is {1}.".format(image_id, img.state)) while img.state == "pending": print "Image {0} has state {1}, waiting {2} seconds for it to become available.".format(image_id, img.state, self.PENDING_IMAGE_WAITTIME) time.sleep(self.PENDING_IMAGE_WAITTIME) img.update() print "Starting {0} EC2 instance(s). This will take a minute...".format(num) reservation = self.conn.run_instances(image_id, min_count=num, max_count=num, key_name=key_name, security_groups=[group_name], instance_type=instance_type) instances = reservation.instances num_instance = len(instances) num_running = 0 while num_running < num_instance: num_running = 0 for instance in instances: instance.update() if instance.state == 'running': num_running += 1 if num_running < num_instance: time.sleep(5) print "EC2 instances started." return sorted(instances, key=lambda vm: vm.id)
def get_instance(self, instance_id): #logging.debug("get_instance(instance_id={0})".format(instance_id)) try: reservations = self.conn.get_all_reservations(instance_ids=[instance_id]) except EC2ResponseError: raise ProviderException("instance not found {0}".format(instance_id)) #logging.debug("get_instance() reservations:{0}".format(reservations)) for reservation in reservations: #logging.debug("get_instance() reservation.instances:{0}".format(reservation.instances)) for instance in reservation.instances: if instance.id == instance_id: return instance raise ProviderException("instance not found {0}".format(instance_id))
def resume_instance(self, instances): self._connect() if isinstance(instances, list): ec2_instances = [] for instance in instances: ec2_instance = self.ec2.get_instance(instance.provider_instance_identifier) ec2_instances.append(ec2_instance) new_ec2_instances = self.ec2.resume_ec2_instances(ec2_instances) instances_to_update = list(instances) while len(instances_to_update) > 0: instance = instances_to_update.pop() success=False for ec2_inst in new_ec2_instances: if ec2_inst.id == instance.provider_instance_identifier: instance.ip_address = ec2_inst.public_dns_name logging.debug("instance.id={0} updated with ip={1}".format(instance.provider_instance_identifier, instance.ip_address)) success=True break if not success: raise ProviderException("Could not update the IP of id={0} after resume".format(instance.provider_instance_identifier)) else: ec2_instance = self.ec2.get_instance(instances.provider_instance_identifier) new_instance = self.ec2.resume_ec2_instances([ec2_instance]) instances.ip_address = new_instance[0].public_dns_name logging.debug("instance.id={0} updated with ip={1}".format(instances.provider_instance_identifier, instances.ip_address))
def __init__(self, config=None, connect=True): if config is not None: self.config = config if self.config['aws_access_key'] is None or self.config['aws_secret_key'] is None: raise ProviderException("AWS_SECRET_KEY or AWS_ACCESS_KEY not set") if connect: self.connect()
def _terminate_instances(self, instance_ids): self._connect() if not isinstance(instance_ids, list): instance_ids = [instance_ids] try: instances = [] for instance_id in instance_ids: instance = self.nova.servers.get(instance_id) instances.append(instance) instance.delete() inst_to_check = instances while len(inst_to_check) > 0: time.sleep(5) inst_still_stopping = [] for instance in inst_to_check: # Retrieve the instance again so the status field updates try: instance = self.nova.servers.get(instance.id) logging.debug("Terminating node, status '{0}' [{1}]".format(instance.status, instance.id)) if instance.status != 'SHUTOFF': inst_still_stopping.append(instance) except novaclient.exceptions.NotFound as e: pass inst_to_check = inst_still_stopping except Exception as e: logging.exception(e) raise ProviderException("Failed to terminate vm(s)\n{0}".format(e))
def create_molns_image(self): """ Create the molns image is created. """ self._connect() # start vm instances = self.ec2.start_ec2_instances(image_id=self.config["ubuntu_image_name"]) instance = instances[0] # get login ip ip = instance.public_dns_name # install software try: logging.debug("installing software on server (ip={0})".format(ip)) install_vm_instance = installSoftware.InstallSW(ip, config=self) install_vm_instance.run_with_logging() # create image logging.debug("shutting down instance") self.ec2.stop_ec2_instances([instance]) logging.debug("creating image") image_id = instance.create_image(name=self._get_image_name()) except Exception as e: logging.exception(e) raise ProviderException("Failed to create molns image: {0}".format(e)) finally: logging.debug("terminating {0}".format(instance)) self.ec2.terminate_ec2_instances([instance]) return image_id
def start_instance(self, num=1): """ Start worker group vms. """ try: self._connect() instances = self.eucalyptus.start_eucalyptus_instances( image_id=self.provider.config["molns_image_name"], num=int(num), instance_type=self.config["instance_type"]) ret = [] for instance in instances: ip = instance.public_dns_name i = self.datastore.get_instance( provider_instance_identifier=instance.id, ip_address=ip, provider_id=self.provider.id, controller_id=self.controller.id, worker_group_id=self.id) ret.append(i) if num == 1: return ret[0] else: return ret except Exception as e: logging.exception(e) raise ProviderException( "Failed to start molns instance: {0}".format(e))
def _attach_floating_ip(self, instance): # Try to attach a floating IP to the controller logging.info("Attaching floating ip to the server...") try: floating_ip = self.nova.floating_ips.create(self.config['floating_ip_pool']) instance.add_floating_ip(floating_ip) logging.debug("ip={0}".format(floating_ip.ip)) return floating_ip.ip except Exception as e: raise ProviderException("Failed to attach a floating IP to the controller.\n{0}".format(e))
def _delete_floating_ip(self, ip): try: floating_ips = self.nova.floating_ips.list() for fip in floating_ips: if fip.ip == ip: fip.delete() return except Exception as e: logging.exception(e) raise ProviderException("Could not delete floating ip '{0}'".format(ip))
def get_instance_status(self, instance): try: status = self.provider._get_instance_status(instance.provider_instance_identifier) except novaclient.exceptions.NotFound as e: return self.STATUS_TERMINATED if status == 'ACTIVE' or status == 'BUILD': return self.STATUS_RUNNING if status == 'SHUTOFF': return self.STATUS_STOPPED if status == 'DELETED': return self.STATUS_TERMINATED raise ProviderException("OpenStackController.get_instance_status() got unknown status '{0}'".format(status))
def create_security_group(self, group_name): security_group = None for sg in self.conn.get_all_security_groups(): if sg.name == group_name: security_group = sg break if security_group is None: print "Security group not found, creating one." security_group = self.conn.create_security_group(group_name, 'MOLNs Security Group') self.set_security_group_rules(security_group) elif not self.check_security_group_rules(security_group): raise ProviderException("Security group {0} exists, but has the wrong firewall rules. Please delete the group, or choose a different one.") return security_group
def create_vm_image(self, image_name=None, key_name=None): if key_name is None: key_name = self.config['key_name'] if image_name is None: image_name = "MOLNS_{0}_{1}".format(key_name,int(time.time())) (stopped_vms, running_vms) = self.get_vm_status(key_name) if len(running_vms) != 1: raise ProviderException("Expected only one running vm, {0} are running".format(len(running_vms))) self.stop_ec2_instances(running_vms) instance = running_vms[0] image_ami = instance.create_image(image_name) print "Image created id={0} name={0}".format(image_ami, image_name) self.terminate_ec2_instances(running_vms) return image_ami
def get_instance_status(self, instance): self._connect() try: status = self.ec2.get_instance_status(instance.provider_instance_identifier) except Exception as e: #logging.exception(e) return self.STATUS_TERMINATED if status == 'running' or status == 'pending': return self.STATUS_RUNNING if status == 'stopped' or status == 'stopping': return self.STATUS_STOPPED if status == 'terminated' or status == 'shutting-down': return self.STATUS_TERMINATED raise ProviderException("EC2Controller.get_instance_status() got unknown status '{0}'".format(status))
def create_molns_image(self): """ Create a molns image, save it on localhost and return DockerImage ID of created image. """ file_to_remove = None try: dockerfile, file_to_remove = self._create_dockerfile( installSoftware.InstallSW.get_command_list()) image_id = self.docker.build_image(dockerfile) return image_id except Exception as e: logging.exception(e) raise ProviderException( "Failed to create molns image: {0}".format(e)) finally: if file_to_remove is not None: os.remove(file_to_remove)
def create_ssh_key(self): """ Create the ssh key and write the file locally. """ ssh_key_dir = os.path.join(self.config_dir, self.name) logging.debug('ssh_key_dir={0}'.format(ssh_key_dir)) if not os.path.isdir(ssh_key_dir): logging.debug('making ssh_key_dir={0}'.format(ssh_key_dir)) os.makedirs(ssh_key_dir) ssh_key_file = os.path.join(ssh_key_dir,self.config['key_name']+self.SSH_KEY_EXTENSION) if os.path.isfile(ssh_key_file): raise ProviderException("ssh_key_file '{0}' already exists".format(self.config['key_name'])) self._connect() new_key = self.nova.keypairs.create(name=self.config['key_name']) #with open(ssh_key_file, 'w') as fd: with os.fdopen(os.open(ssh_key_file, os.O_WRONLY | os.O_CREAT, 0600), 'w') as fd: fd.write(new_key.private_key)
def create_molns_image(self): """ Create the molns image is created. """ self._connect() # clear the network-related persisent udev rules: #echo "" > /etc/udev/rules.d/70-persistent-net.rules #echo "" > /lib/udev/rules.d/75-persistent-net-generator.rules # # start vm instances = self.eucalyptus.start_eucalyptus_instances( image_id=self.config["ubuntu_image_name"]) instance = instances[0] # get login ip ip = instance.public_dns_name # install software try: logging.debug("installing software on server (ip={0})".format(ip)) install_vm_instance = installSoftware.InstallSW(ip, config=self) #install_vm_instance.run_with_logging() # create image logging.debug("Shutting down instance") self.eucalyptus.stop_eucalyptus_instances([instance]) logging.debug("Creating image") image_id = instance.create_image(name=self._get_image_name()) #logging.debug("Finding volume of instance") #vol = None #for v in self.eucalyptus.conn.get_all_volumes(): # if v.attach_data is not None and v.attach_data.instance_id == instance.id: # vol = v # break #if vol is None: # raise Exception("Can not find volume associated with instance. Base image must be an EBS backed image.") #snap = vol.create_snapshot() #logging.debug('Snapshot {0} of volume {1}'.format(snap.id, vol.id)) #image_id = self.eucalyptus.conn.register_image(name=self._get_image_name(), snapshot_id=snap.id, delete_root_volume_on_termination=True) ##deleteOnTermination #image_id = self.eucalyptus.conn.register_image(name=self._get_image_name(), snapshot_id=snap.id) logging.debug("Image created: {0}".format(image_id)) except Exception as e: logging.exception(e) raise ProviderException( "Failed to create molns image: {0}".format(e)) finally: logging.debug("terminating {0}".format(instance)) self.eucalyptus.terminate_eucalyptus_instances([instance]) return image_id
def _resume_instances(self, instance_ids): self._connect() for instance_id in instance_ids: try: instance = self.nova.servers.get(instance_id) instance.start() logging.debug("instance={0}".format(instance)) # wait for boot to complete while instance.status == 'BUILD': logging.debug("Launching node, status '{0}'".format(instance.status)) time.sleep(5) # Retrieve the instance again so the status field updates instance = self.nova.servers.get(instance.id) #instance.update() logging.debug("status: {0}".format(instance.status)) return instance except Exception as e: logging.exception(e) logging.debug("terminating instance {0}".format(instance)) instance.delete() raise ProviderException("Failed to boot vm\n{0}".format(e))
def create_molns_image(self): """ Create the molns image is created. """ # start vm instance = self._boot_ubuntu_vm() # get login ip ip = self._attach_floating_ip(instance) # install software try: logging.debug("installing software on server (ip={0})".format(ip)) install_vm_instance = installSoftware.InstallSW(ip, config=self) install_vm_instance.run_with_logging() # create image logging.debug("shutting down instance") self._stop_vm(instance) logging.debug("creating image") image_id = instance.create_image(image_name=self._get_image_name()) logging.debug("image creation started image_id={0}".format(image_id)) time.sleep(10) t1 = time.time() while time.time() < t1+self.MAX_IMAGE_CREATION_WAITTIME: logging.debug("waiting for image creation to complete") time.sleep(30) img_status = self._get_image_status(image_id) logging.debug("image status is {0}".format(img_status)) if img_status != 'SAVING': break except Exception as e: logging.exception(e) raise ProviderException("Failed to create molns image: {0}".format(e)) finally: logging.debug("terminating {0}".format(instance)) instance.delete() try: logging.debug("deleteing floating ip {0}".format(ip)) self._delete_floating_ip(ip) except ProviderException as e: logging.error("Error deleteing floating IP: {0}".format(e)) return image_id
def __boot_vm(self, image_name, instance_type, num=1): self._connect() instances = [] try: image = self.nova.images.get(image_name) #logging.debug("image={0}".format(image)) flavor = self.nova.flavors.find(name=instance_type) #logging.debug("flavor={0}".format(flavor)) for n in range(int(num)): if 'neutron_nic' in self.config and self.config['neutron_nic'] != '': inst = self.nova.servers.create(name="molns_vm_"+self.name, image=image, flavor=flavor, key_name=self.config["key_name"], security_groups=[self.config["group_name"]],nics=[{'net-id':self.config['neutron_nic']}]) else: inst = self.nova.servers.create(name="molns_vm_"+self.name, image=image, flavor=flavor, key_name=self.config["key_name"], security_groups=[self.config["group_name"]]) instances.append(inst) #logging.debug("instance={0}".format(inst)) # wait for boot to complete inst_to_check = instances while len(inst_to_check) > 0: time.sleep(5) inst_still_building = [] for instance in inst_to_check: # Retrieve the instance again so the status field updates instance = self.nova.servers.get(instance.id) logging.debug("Launching node, status '{0}' [{1}]".format(instance.status, instance.id)) if instance.status == 'BUILD': inst_still_building.append(instance) inst_to_check = inst_still_building if num == 1: return instances[0] else: return instances except Exception as e: logging.exception(e) for instance in instances: logging.debug("terminating instance {0}".format(instance)) instance.delete() raise ProviderException("Failed to boot vm\n{0}".format(e))
def start_vms(self, image_id=None, key_name=None, group_name=None, num=None, instance_type=None): if key_name is None: key_name = self.config['key_name'] if group_name is None: group_name = self.config['group_name'] if num is None: num = 1 if instance_type is None: instance_type = self.config['default_instance_type'] # Check the group self.create_security_group(group_name) #(stopped_vms, running_vms) = self.get_vm_status(key_name) #if len(running_vms) > 0: # msg = "Error: {0} VMs are already running with key_name={1}".format(len(running_vms), # key_name) # print msg # raise ProviderException(msg) if len(stopped_vms) > 0: return self.resume_ec2_instances(stopped_vms) if image_id is None: raise ProviderException("Base Ubuntu image not specified.") else: self.image_id = image_id # Check image try: img = self.conn.get_all_images(image_ids=[self.image_id])[0] except IndexError: raise ProviderException("Could not find image_id={0}".format(self.image_id)) if img.state != "available": if img.state != "pending": raise ProviderException("Image {0} is not available, it has state is {1}.".format(self.image_id, img.state)) while img.state == "pending": print "Image {0} has state {1}, waiting {2} seconds for it to become available.".format(self.image_id, img.state, self.PENDING_IMAGE_WAITTIME) time.sleep(self.PENDING_IMAGE_WAITTIME) img.update() self.key_name = key_name self.group_name = group_name group_list = [] for _ in range(num): group_list.append(group_name) print "Starting {0} EC2 instance(s). This will take a minute...".format(num) reservation = self.conn.run_instances(self.image_id, min_count=num, max_count=num, key_name=key_name, security_groups=group_list, instance_type=instance_type) instances = reservation.instances num_instance = len(instances) num_running = 0 while num_running < num_instance: num_running = 0 for instance in instances: instance.update() if instance.state == 'running': num_running += 1 if num_running < num_instance: time.sleep(5) print "EC2 instances started." return sorted(instances, key=lambda vm: vm.id)