def _ensure_instance_running(self, instance_name): """ If an instance is terminated but still exists (hasn't been deleted calling this will start the instance up again. Raises an error if the instance no longer exists. """ try: instance_info = self._compute.instances().get( project=self.project, zone=self.zone, instance=instance_name).execute() if instance_info['status'] == 'RUNNING': pass elif instance_info['status'] == 'TERMINATED': self._start_terminated_server(instance_name) else: msg = ("Instance {} is in state {}, " "please start it from the console").format( instance_name, instance_info['status']) raise Exception(msg) # if we've started a terminated server, re-save # the networking info, if we have except HttpError as e: if e.resp.status == 404: log_red("Instance {} does not exist".format(instance_name)) log_yellow("you might need to remove state file.") else: log_red("Unknown error querying for instance {}".format( instance_name)) raise e
def create_from_saved_state(cls, config, saved_state, timeout=600): parsed_config = EC2Configuration.create(config) state = EC2State.create(saved_state) connection = _connect_to_ec2( region=state.region, credentials=parsed_config.credentials ) instance = connection.start_instances( instance_ids=state.instance_id)[0] instance.update() while instance.state != "running" and timeout > 1: log_yellow("Instance state: %s" % instance.state) sleep(10) timeout = timeout - 10 instance.update() # and make sure we don't return until the instance is fully up wait_for_ssh(instance.ip_address) return cls( connection=connection, instance=instance, config=parsed_config, state=state )
def _create_server(self): log_green("Started...") log_yellow("...Creating GCE instance...") latest_image = self._get_latest_image(self.config.base_image_project, self.config.base_image_prefix) self.startup_instance(self.state.instance_name, latest_image["selfLink"], disk_name=None) self._set_instance_networking()
def create_ami(connection, region, instance_id, name, description, block_device_mapping=None, log=False): ami = connection.create_image(instance_id, name, description, block_device_mapping) image_status = connection.get_image(ami) while (image_status.state != "available" and image_status.state != "failed"): if log: log_yellow('creating ami...') sleep_for_one_minute() image_status = connection.get_image(ami) if image_status.state == "available": if log: log_green("ami %s %s" % (ami, image_status)) return(ami) else: if log: log_red("ami %s %s" % (ami, image_status)) return False
def _ensure_instance_running(self, instance_name): """ If an instance is terminated but still exists (hasn't been deleted calling this will start the instance up again. Raises an error if the instance no longer exists. """ try: instance_info = ( self._compute.instances().get(project=self.project, zone=self.zone, instance=instance_name).execute() ) if instance_info["status"] == "RUNNING": pass elif instance_info["status"] == "TERMINATED": self._start_terminated_server(instance_name) else: msg = ("Instance {} is in state {}, " "please start it from the console").format( instance_name, instance_info["status"] ) raise Exception(msg) # if we've started a terminated server, re-save # the networking info, if we have except HttpError as e: if e.resp.status == 404: log_red("Instance {} does not exist".format(instance_name)) log_yellow("you might need to remove state file.") else: log_red("Unknown error querying for instance {}".format(instance_name)) raise e
def list_images(self): images = self._nova.images.list() log_yellow("creation time\timage_name\timage_id") for image in sorted(images, key=lambda x: x.created): log_green("{}\t{:50}\t{}".format( image.created, image.human_id, image.id) )
def down(self): log_yellow("downing server: {}".format(self.state.instance_name)) self._wait_until_done( self._compute.instances() .stop(project=self.project, zone=self.zone, instance=self.state.instance_name) .execute() )
def list_images(self): images = self.connection.get_all_images(owners='self') log_yellow("creation time\timage_name\timage_id") for image in sorted(images, key=lambda x: x.creationDate): log_green("{}\t{:50}\t{}".format( image.creationDate, image.name, image.id) )
def down(self): instance = self.connection.stop_instances( instance_ids=self.state.instance_id)[0] while instance.state != "stopped": log_yellow("Instance state: %s" % instance.state) sleep(10) instance.update() log_green('Instance state: %s' % instance.state)
def wait_for_ssh(host, port=22, timeout=600): """ probes the ssh port and waits until it is available """ log_yellow('waiting for ssh...') for iteration in xrange(1, timeout): #noqa sleep(1) if is_ssh_available(host, port): return True else: log_yellow('waiting for ssh...')
def _create_server(self): log_green("Started...") log_yellow("...Creating GCE instance...") latest_image = self._get_latest_image(self.config.base_image_project, self.config.base_image_prefix) self.startup_instance(self.state.instance_name, latest_image['selfLink'], disk_name=None) self._set_instance_networking()
def delete_image(self, image_id): images = self.connection.get_all_images(owners='self') found = False for image in images: if image.id == image_id: log_yellow("Deleting image {}".format(image_id)) image.deregister(delete_snapshot=True) found = True break if not found: log_red("Could not find image {}".format(image_id))
def down_ec2(connection, instance_id, region, log=False): """ shutdown of an existing EC2 instance """ # get the instance_id from the state file, and stop the instance instance = connection.stop_instances(instance_ids=instance_id)[0] while instance.state != "stopped": if log: log_yellow("Instance state: %s" % instance.state) sleep(10) instance.update() if log: log_green('Instance state: %s' % instance.state)
def destroy(self): disk_name = self.state.instance_name self._destroy_instance() try: self._wait_until_done( self._compute.disks().delete(project=self.project, zone=self.zone, disk=disk_name).execute() ) except HttpError as e: if e.resp.status == 404: log_yellow("the disk {} was already destroyed".format(disk_name)) else: raise e
def destroy(self): server = self._nova.servers.find(name=self.state.instance_name) log_yellow('deleting rackspace instance ...') server.delete() try: while True: server = self._nova.servers.get(server.id) log_yellow('waiting for deletion ...') sleep(5) except NotFound: pass log_green('The server has been deleted')
def _destroy_instance(self): log_yellow("destroying server: {}".format(self.state.instance_name)) try: self._wait_until_done( self._compute.instances() .delete(project=self.project, zone=self.zone, instance=self.state.instance_name) .execute() ) except HttpError as e: if e.resp.status == 404: log_yellow("the instance {} is already down".format(self.state.instance_name)) else: raise e
def _destroy_instance(self): log_yellow("destroying server: {}".format(self.state.instance_name)) try: self._wait_until_done(self._compute.instances().delete( project=self.project, zone=self.zone, instance=self.state.instance_name).execute()) except HttpError as e: if e.resp.status == 404: log_yellow("the instance {} is already down".format( self.state.instance_name)) else: raise e
def destroy(self): self.down() volumes = self.connection.get_all_volumes( filters={'attachment.instance-id': self.state.instance_id}) instance = self.connection.terminate_instances( instance_ids=[self.state.instance_id])[0] log_yellow('destroying instance ...') while instance.state != "terminated": log_yellow("Instance state: %s" % instance.state) sleep(10) instance.update() for volume in volumes: self._destroy_ebs_volume(volume.id)
def destroy(self): self.down() volumes = self.connection.get_all_volumes( filters={'attachment.instance-id': self.state.instance_id} ) instance = self.connection.terminate_instances( instance_ids=[self.state.instance_id])[0] log_yellow('destroying instance ...') while instance.state != "terminated": log_yellow("Instance state: %s" % instance.state) sleep(10) instance.update() for volume in volumes: self._destroy_ebs_volume(volume.id)
def destroy(self): disk_name = self.state.instance_name self._destroy_instance() try: self._wait_until_done( self._compute.disks().delete(project=self.project, zone=self.zone, disk=disk_name).execute()) except HttpError as e: if e.resp.status == 404: log_yellow( "the disk {} was already destroyed".format(disk_name)) else: raise e
def startup_instance(self, instance_name, image, disk_name=None): """ For now, jclouds is broken for GCE and we will have static slaves in Jenkins. Use this to boot them. """ log_green("Started...") log_yellow("...Starting GCE Jenkins Slave Instance...") instance_config = self._get_instance_config(instance_name, image, disk_name) operation = ( self._compute.instances().insert(project=self.project, zone=self.zone, body=instance_config).execute() ) result = self._wait_until_done(operation) if not result: raise RuntimeError("Creation of VM timed out or returned no result") log_green("Instance has booted")
def destroy_rackspace(connection, region, instance_id): """ terminates the instance """ server = connection.servers.get(instance_id) log_yellow('deleting rackspace instance ...') server.delete() # wait for server to be deleted try: while True: server = connection.servers.get(server.id) log_yellow('waiting for deletion ...') sleep(5) except: pass log_green('The server has been deleted')
def destroy_ebs_volume(connection, region, volume_id, log=False): """ destroys an ebs volume """ if ebs_volume_exists(connection, region, volume_id): if log: log_yellow('destroying EBS volume ...') try: connection.delete_volume(volume_id) except: # our EBS volume may be gone, but AWS info tables are stale # wait a bit and ask again sleep(5) if not ebs_volume_exists(connection, region, volume_id): pass else: raise("Couldn't delete EBS volume")
def create_server_rackspace(connection, distribution, disk_name, disk_size, ami, region, key_pair, instance_type, instance_name, tags={}, security_groups=None): """ Creates Rackspace Instance and saves it state in a local json file """ log_yellow("Creating Rackspace instance...") flavor = connection.flavors.find(name=instance_type) image = connection.images.find(name=ami) server = connection.servers.create(name=instance_name, flavor=flavor.id, image=image.id, region=region, availability_zone=region, key_name=key_pair) while server.status == 'BUILD': log_yellow("Waiting for build to finish...") sleep(5) server = connection.servers.get(server.id) # check for errors if server.status != 'ACTIVE': log_red("Error creating rackspace instance") exit(1) # the server was assigned IPv4 and IPv6 addresses, locate the IPv4 address ip_address = server.accessIPv4 if ip_address is None: log_red('No IP address assigned') exit(1) wait_for_ssh(ip_address) log_green('New server with IP address {0}.'.format(ip_address)) return server
def startup_instance(self, instance_name, image, disk_name=None): """ For now, jclouds is broken for GCE and we will have static slaves in Jenkins. Use this to boot them. """ log_green("Started...") log_yellow("...Starting GCE Jenkins Slave Instance...") instance_config = self._get_instance_config(instance_name, image, disk_name) operation = self._compute.instances().insert( project=self.project, zone=self.zone, body=instance_config).execute() result = self._wait_until_done(operation) if not result: raise RuntimeError( "Creation of VM timed out or returned no result") log_green("Instance has booted")
def _wait_until_done(self, operation): """ Perform a GCE operation, blocking until the operation completes. This function will then poll the operation until it reaches state 'DONE' or times out, and then returns the final operation resource dict. :param operation: A dict representing a pending GCE operation resource. :returns dict: A dict representing the concluded GCE operation resource. """ operation_name = operation['name'] if 'zone' in operation: zone_url_parts = operation['zone'].split('/') project = zone_url_parts[-3] zone = zone_url_parts[-1] def get_zone_operation(): return self._compute.zoneOperations().get( project=project, zone=zone, operation=operation_name) update = get_zone_operation else: project = operation['selfLink'].split('/')[-4] def get_global_operation(): return self._compute.globalOperations().get( project=project, operation=operation_name) update = get_global_operation done = False latest_operation = None start = time() timeout = 5 * 60 # seconds while not done: latest_operation = update().execute() if (latest_operation['status'] == 'DONE' or time() - start > timeout): done = True else: sleep(10) log_yellow("waiting for operation") return latest_operation
def destroy_ec2(connection, region, instance_id, log=False): """ terminates the instance """ data = get_ec2_info(connection=connection, instance_id=instance_id, region=region) instance = connection.terminate_instances(instance_ids=[data['id']])[0] if log: log_yellow('destroying instance ...') while instance.state != "terminated": if log: log_yellow("Instance state: %s" % instance.state) sleep(10) instance.update() volume_id = data['volume'] if volume_id: destroy_ebs_volume(connection, region, volume_id)
def _wait_until_done(self, operation): """ Perform a GCE operation, blocking until the operation completes. This function will then poll the operation until it reaches state 'DONE' or times out, and then returns the final operation resource dict. :param operation: A dict representing a pending GCE operation resource. :returns dict: A dict representing the concluded GCE operation resource. """ operation_name = operation["name"] if "zone" in operation: zone_url_parts = operation["zone"].split("/") project = zone_url_parts[-3] zone = zone_url_parts[-1] def get_zone_operation(): return self._compute.zoneOperations().get(project=project, zone=zone, operation=operation_name) update = get_zone_operation else: project = operation["selfLink"].split("/")[-4] def get_global_operation(): return self._compute.globalOperations().get(project=project, operation=operation_name) update = get_global_operation done = False latest_operation = None start = time() timeout = 5 * 60 # seconds while not done: latest_operation = update().execute() if latest_operation["status"] == "DONE" or time() - start > timeout: done = True else: sleep(10) log_yellow("waiting for operation") return latest_operation
def create_image(self, image_name): ami = self.connection.create_image( self.state.instance_id, image_name, description=self.config.image_description, ) image_status = self.connection.get_image(ami) while (image_status.state != "available" and image_status.state != "failed"): log_yellow('creating ami...') sleep(60) image_status = self.connection.get_image(ami) if image_status.state == "available": log_green("ami %s %s" % (ami, image_status)) return(ami) else: log_red("ami %s %s" % (ami, image_status)) return False
def _create_server(self): log_yellow("Creating Rackspace instance...") flavor = self._nova.flavors.find(name=self.config.instance_type) image = self._nova.images.find(name=self.config.ami) server = self._nova.servers.create(name=self.state.instance_name, flavor=flavor.id, image=image.id, region=self.state.region, availability_zone=self.state.region, key_name=self.config.key_pair) while server.status == 'BUILD': log_yellow("Waiting for build to finish...") sleep(10) server = self._nova.servers.get(server.id) # check for errors if server.status != 'ACTIVE': log_red("Error creating rackspace instance") exit(1) self._set_instance_networking(server)
def create_image(self, image_name): ami = self.connection.create_image( self.state.instance_id, image_name, description=self.config.image_description, ) image_status = self.connection.get_image(ami) while (image_status.state != "available" and image_status.state != "failed"): log_yellow('creating ami...') sleep(60) image_status = self.connection.get_image(ami) if image_status.state == "available": log_green("ami %s %s" % (ami, image_status)) return (ami) else: log_red("ami %s %s" % (ami, image_status)) return False
def create_from_saved_state(cls, config, saved_state, timeout=600): parsed_config = EC2Configuration.create(config) state = EC2State.create(saved_state) connection = _connect_to_ec2(region=state.region, credentials=parsed_config.credentials) instance = connection.start_instances( instance_ids=state.instance_id)[0] instance.update() while instance.state != "running" and timeout > 1: log_yellow("Instance state: %s" % instance.state) sleep(10) timeout = timeout - 10 instance.update() # and make sure we don't return until the instance is fully up wait_for_ssh(instance.ip_address) return cls(connection=connection, instance=instance, config=parsed_config, state=state)
def _create_server(self): log_yellow("Creating Rackspace instance...") flavor = self._nova.flavors.find(name=self.config.instance_type) image = self._nova.images.find(name=self.config.ami) server = self._nova.servers.create( name=self.state.instance_name, flavor=flavor.id, image=image.id, region=self.state.region, availability_zone=self.state.region, key_name=self.config.key_pair ) while server.status == 'BUILD': log_yellow("Waiting for build to finish...") sleep(10) server = self._nova.servers.get(server.id) # check for errors if server.status != 'ACTIVE': log_red("Error creating rackspace instance") exit(1) self._set_instance_networking(server)
def up_ec2(connection, region, instance_id, wait_for_ssh_available=True, log=False, timeout=600): """ boots an existing ec2_instance """ # boot the ec2 instance instance = connection.start_instances(instance_ids=instance_id)[0] instance.update() while instance.state != "running" and timeout > 1: log_yellow("Instance state: %s" % instance.state) if log: log_yellow("Instance state: %s" % instance.state) sleep(10) timeout = timeout - 10 instance.update() # and make sure we don't return until the instance is fully up if wait_for_ssh_available: wait_for_ssh(instance.ip_address)
def _destroy_ebs_volume(self, volume_id): """ destroys an ebs volume """ if self._ebs_volume_exists(volume_id): log_yellow('destroying EBS volume ...') try: self.connection.delete_volume(volume_id) except Exception as e: # our EBS volume may be gone, but AWS info tables are stale # wait a bit and ask again log_yellow("exception raised when deleting volume") log_yellow("{} -- {}".format(type(e), str(e))) worked = False for i in range(6): sleep(5) if not self._ebs_volume_exists(volume_id): log_green("It worked that time") worked = True if not worked: raise Exception("Couldn't delete EBS volume")
def list_images(self): results = self._compute.images().list(project=self.project).execute() log_yellow("creation time\timage_name") for item in results['items']: log_green("{}\t{}".format(item['creationTimestamp'], item['name']))
def _create_server_ec2(connection, region, disk_name, disk_size, ami, key_pair, instance_type, tags={}, security_groups=None, delete_on_termination=True, log=False, wait_for_ssh_available=True): """ Creates EC2 Instance """ if log: log_green("Started...") log_yellow("...Creating EC2 instance...") ebs_volume = EBSBlockDeviceType() ebs_volume.size = disk_size bdm = BlockDeviceMapping() bdm[disk_name] = ebs_volume # get an ec2 ami image object with our choosen ami image = connection.get_all_images(ami)[0] # start a new instance reservation = image.run(1, 1, key_name=key_pair, security_groups=security_groups, block_device_map=bdm, instance_type=instance_type) # and get our instance_id instance = reservation.instances[0] # and loop and wait until ssh is available while instance.state == u'pending': if log: log_yellow("Instance state: %s" % instance.state) sleep(10) instance.update() if log: log_green("Instance state: %s" % instance.state) if wait_for_ssh_available: wait_for_ssh(instance.public_dns_name) # update the EBS volumes to be deleted on instance termination if delete_on_termination: for dev, bd in instance.block_device_mapping.items(): instance.modify_attribute('BlockDeviceMapping', ["%s=%d" % (dev, 1)]) # add a tag to our instance if tags: connection.create_tags([instance.id], tags) if log: log_green("Public dns: %s" % instance.public_dns_name) # returns our new instance return instance
def _start_terminated_server(self, instance_name): log_yellow("starting terminated instance {}".format(instance_name)) operation = ( self._compute.instances().start(project=self.project, zone=self.zone, instance=instance_name).execute() ) self._wait_until_done(operation)
def list_images(self): images = self.connection.get_all_images(owners='self') log_yellow("creation time\timage_name\timage_id") for image in sorted(images, key=lambda x: x.creationDate): log_green("{}\t{:50}\t{}".format(image.creationDate, image.name, image.id))
def list_images(self): images = self._nova.images.list() log_yellow("creation time\timage_name\timage_id") for image in sorted(images, key=lambda x: x.created): log_green("{}\t{:50}\t{}".format(image.created, image.human_id, image.id))
def delete_image(self, image_name): log_green("Deleting image {}".format(image_name)) result = self._wait_until_done(self._compute.images().delete(project=self.project, image=image_name).execute()) log_yellow("Delete image returned status {}".format(result["status"]))
def _start_terminated_server(self, instance_name): log_yellow("starting terminated instance {}".format(instance_name)) operation = self._compute.instances().start( project=self.project, zone=self.zone, instance=instance_name).execute() self._wait_until_done(operation)
def down(self): log_yellow("downing server: {}".format(self.state.instance_name)) self._wait_until_done(self._compute.instances().stop( project=self.project, zone=self.zone, instance=self.state.instance_name).execute())
def delete_image(self, image_name): log_green("Deleting image {}".format(image_name)) result = self._wait_until_done(self._compute.images().delete( project=self.project, image=image_name).execute()) log_yellow("Delete image returned status {}".format(result['status']))
def list_images(self): results = self._compute.images().list(project=self.project).execute() log_yellow("creation time\timage_name") for item in results["items"]: log_green("{}\t{}".format(item["creationTimestamp"], item["name"]))