def does_disk_exist(self, parameters, disk): """ Queries Google Compute Engine to see if the specified persistent disk exists for this user. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. disk: A str containing the name of the disk that we should check for existence. Returns: True if the named persistent disk exists, and False otherwise. """ gce_service, credentials = self.open_connection(parameters) try: http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.disks().get( project=parameters[self.PARAM_PROJECT], disk=disk, zone=parameters[self.PARAM_ZONE]) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) return True except errors.HttpError: return False
def create_scratch_disk(self, parameters): """ Creates a disk from a given machine image. GCE does not support scratch disks on API version v1 and higher. We create a persistent disk upon creation to act like one to keep the abstraction used in other infrastructures. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. Returns: A str, the url to the disk to use. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) disk_name = self.generate_disk_name(parameters) project_url = '{0}{1}'.format(self.GCE_URL, parameters[self.PARAM_PROJECT]) source_image_url = '{0}{1}/global/images/{2}'.format( self.GCE_URL, parameters[self.PARAM_PROJECT], parameters[self.PARAM_IMAGE_ID]) request = gce_service.disks().insert( project=parameters[self.PARAM_PROJECT], zone=parameters[self.PARAM_ZONE], body={'name': disk_name}, sourceImage=source_image_url) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT]) disk_url = "{0}/zones/{1}/disks/{2}".format( project_url, parameters[self.PARAM_ZONE], disk_name) return disk_url
def add_access_config(self, parameters, instance_id, static_ip): """ Instructs Google Compute Engine to use the given IP address as the public IP for the named instance. This assumes that there is no existing public IP address for the named instance. If this is not the case, callers should use delete_access_config first to remove it. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key mapping to a list of instance names that should be deleted. instance_id: A str naming the running instance that the new public IP address should be added to. static_ip: A str naming the already allocated static IP address that will be used for the named instance. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.instances().addAccessConfig( project=parameters[self.PARAM_PROJECT], instance=instance_id, networkInterface="nic0", zone=parameters[self.PARAM_ZONE], body={ "kind": "compute#accessConfig", "type" : "ONE_TO_ONE_NAT", "name" : "External NAT", "natIP" : static_ip } ) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
def create_network(self, parameters): """ Creates a new network in Google Compute Engine with the specified name. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key indicating the name of the network that we should create in GCE. Returns: The URL corresponding to the name of the network that was created, for use with binding this network to one or more firewalls. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.networks().insert( project=parameters[self.PARAM_PROJECT], body={ "name" : parameters[self.PARAM_GROUP], "description" : "Network used for AppScale instances", "IPv4Range" : "10.240.0.0/16" } ) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT]) return response['targetLink']
def terminate_instances(self, parameters): """ Deletes the instances specified in 'parameters' running in Google Compute Engine. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key mapping to a list of instance names that should be deleted. """ instance_ids = parameters[self.PARAM_INSTANCE_IDS] responses = [] for instance_id in instance_ids: gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.instances().delete( project=parameters[self.PARAM_PROJECT], zone=parameters[self.PARAM_ZONE], instance=instance_id ) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) responses.append(response) for response in responses: gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT])
def does_address_exist(self, parameters): """ Queries Google Compute Engine to see if the specified static IP address exists for this user. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key indicating the name of the static IP address that we should check for existence. Returns: True if the named address exists, and False otherwise. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.addresses().list( project=parameters[self.PARAM_PROJECT], filter="address eq {0}".format(parameters[self.PARAM_STATIC_IP]), region=parameters[self.PARAM_REGION] ) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) if 'items' in response: return True else: return False
def assert_credentials_are_valid(self, parameters): """Contacts GCE to see if the given credentials are valid. Args: parameters: A dict containing the credentials necessary to interact with GCE. Raises: AgentConfigurationException: If an error is encountered during authentication. """ gce_service, credentials = self.open_connection(parameters) try: http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.instances().list( project=parameters[self.PARAM_PROJECT], zone=parameters[self.PARAM_ZONE]) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) return True except errors.HttpError as e: error_message = json.loads(e.content)['error']['message'] raise AgentConfigurationException(error_message)
def get_optimal_spot_price(self, conn, instance_type, zone): """ Returns the spot price for an EC2 instance of the specified instance type. The returned value is computed by averaging all the spot price history values returned by the back-end EC2 APIs and incrementing the average by extra 10%. Args: conn: A boto.EC2Connection that can be used to communicate with AWS. instance_type: A str representing the instance type whose prices we should speculate for. zone: A str representing the availability zone that the instance will be placed in. Returns: The estimated spot price for the specified instance type, in the specified availability zone. """ end_time = datetime.datetime.now() start_time = end_time - datetime.timedelta(days=7) history = conn.get_spot_price_history(start_time=start_time.isoformat(), end_time=end_time.isoformat(), product_description='Linux/UNIX', instance_type=instance_type, availability_zone=zone) var_sum = 0.0 for entry in history: var_sum += entry.price average = var_sum / len(history) bid_price = average * 1.10 AppScaleLogger.log('The average spot instance price for a {0} machine is'\ ' {1}, and 10% more is {2}'.format(instance_type, average, bid_price)) return bid_price
def create_firewall(self, parameters, network_url): """ Creates a new firewall in Google Compute Engine with the specified name, bound to the specified network. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key indicating the name of the firewall that we should create. network_url: A str containing the URL of the network that this new firewall should be applied to. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.firewalls().insert( project=parameters[self.PARAM_PROJECT], body={ "name" : parameters[self.PARAM_GROUP], "description" : "Firewall used for AppScale instances", "network" : network_url, "sourceRanges" : ["0.0.0.0/0"], "allowed" : [ {"IPProtocol" : "tcp", "ports": ["1-65535"]}, {"IPProtocol" : "udp", "ports": ["1-65535"]}, {"IPProtocol" : "icmp"} ] } ) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT])
def create_security_group(self, parameters, group): """Creates a new security group in AWS with the given name. Args: parameters: A dict that contains the credentials necessary to authenticate with AWS. group: A str that names the group that should be created. Raises: AgentRuntimeException: If the security group could not be created. """ AppScaleLogger.log('Creating security group: {0}'.format(group)) conn = self.open_connection(parameters) retries_left = self.SECURITY_GROUP_RETRY_COUNT while retries_left: try: conn.create_security_group(group, 'AppScale security group') except EC2ResponseError: pass try: conn.get_all_security_groups(group) return except EC2ResponseError: pass time.sleep(self.SLEEP_TIME) retries_left -= 1 raise AgentRuntimeException("Couldn't create security group with " \ "name {0}".format(group))
def handle_failure(self, msg): """ Log the specified error message and raise an AgentRuntimeException Args: msg: An error message to be logged and included in the raised exception. Raises: AgentRuntimeException Contains the input error message. """ AppScaleLogger.log(msg) raise AgentRuntimeException(msg)
def does_ssh_key_exist(self, parameters): """ Queries Google Compute Engine to see if the specified SSH key exists. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. We don't have an additional key for the name of the SSH key, since we use the one in ~/.ssh. Returns: A tuple of two items. The first item is a bool that is True if our public key's contents are in GCE, and False otherwise, while the second item is the contents of all SSH keys stored in GCE. """ our_public_ssh_key = None public_ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + \ parameters[self.PARAM_KEYNAME] + ".pub" with open(public_ssh_key_location) as file_handle: system_user = os.getenv('LOGNAME', default=pwd.getpwuid(os.getuid())[0]) our_public_ssh_key = system_user + ":" + file_handle.read().rstrip( ) gce_service, credentials = self.open_connection(parameters) try: http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.projects().get( project=parameters[self.PARAM_PROJECT]) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) if not 'items' in response['commonInstanceMetadata']: return False, "" metadata = response['commonInstanceMetadata']['items'] if not metadata: return False, "" all_ssh_keys = "" for item in metadata: if item['key'] != 'sshKeys': continue # Now that we know there's one or more SSH keys, just make sure that # ours is in this list. all_ssh_keys = item['value'] if our_public_ssh_key in all_ssh_keys: return True, all_ssh_keys return False, all_ssh_keys except errors.HttpError: return False, ""
def does_ssh_key_exist(self, parameters): """ Queries Google Compute Engine to see if the specified SSH key exists. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. We don't have an additional key for the name of the SSH key, since we use the one in ~/.ssh. Returns: A tuple of two items. The first item is a bool that is True if our public key's contents are in GCE, and False otherwise, while the second item is the contents of all SSH keys stored in GCE. """ our_public_ssh_key = None public_ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + \ parameters[self.PARAM_KEYNAME] + ".pub" with open(public_ssh_key_location) as file_handle: system_user = os.getenv('LOGNAME', default=pwd.getpwuid(os.getuid())[0]) our_public_ssh_key = system_user + ":" + file_handle.read().rstrip() gce_service, credentials = self.open_connection(parameters) try: http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.projects().get( project=parameters[self.PARAM_PROJECT]) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) if not 'items' in response['commonInstanceMetadata']: return False, "" metadata = response['commonInstanceMetadata']['items'] if not metadata: return False, "" all_ssh_keys = "" for item in metadata: if item['key'] != 'sshKeys': continue # Now that we know there's one or more SSH keys, just make sure that # ours is in this list. all_ssh_keys = item['value'] if our_public_ssh_key in all_ssh_keys: return True, all_ssh_keys return False, all_ssh_keys except errors.HttpError: return False, ""
def does_image_exist(self, parameters): """ Queries Amazon EC2 to see if the specified image exists. Args: parameters: A dict that contains the machine ID to check for existence. Returns: True if the machine ID exists, False otherwise. """ try: conn = self.open_connection(parameters) image_id = parameters[self.PARAM_IMAGE_ID] conn.get_image(image_id) AppScaleLogger.log('Machine image {0} does exist'.format(image_id)) return True except boto.exception.EC2ResponseError: AppScaleLogger.log('Machine image {0} does not exist'.format(image_id)) return False
def configure_instance_security(self, parameters): """ Setup EC2 security keys and groups. Required input values are read from the parameters dictionary. More specifically, this method expects to find a 'keyname' parameter and a 'group' parameter in the parameters dictionary. Using these provided values, this method will create a new EC2 key-pair and a security group. Security group will be granted permission to access any port on the instantiated VMs. (Also see documentation for the BaseAgent class) Args: parameters: A dictionary of parameters. """ keyname = parameters[self.PARAM_KEYNAME] group = parameters[self.PARAM_GROUP] AppScaleLogger.log("Verifying that keyname {0}".format(keyname) + \ " is not already registered.") conn = self.open_connection(parameters) if conn.get_key_pair(keyname): self.handle_failure("SSH keyname {0} is already registered. Please " \ "change the 'keyname' specified in your AppScalefile to a different " \ "value, or erase it to have one automatically generated for you." \ .format(keyname)) security_groups = conn.get_all_security_groups() for security_group in security_groups: if security_group.name == group: self.handle_failure("Security group {0} is already registered. Please" \ " change the 'group' specified in your AppScalefile to a different " \ "value, or erase it to have one automatically generated for you." \ .format(group)) AppScaleLogger.log("Creating key pair: {0}".format(keyname)) key_pair = conn.create_key_pair(keyname) ssh_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH, keyname) LocalState.write_key_file(ssh_key, key_pair.material) self.create_security_group(parameters, group) self.authorize_security_group(parameters, group, from_port=1, to_port=65535, ip_protocol='udp', cidr_ip='0.0.0.0/0') self.authorize_security_group(parameters, group, from_port=1, to_port=65535, ip_protocol='tcp', cidr_ip='0.0.0.0/0') self.authorize_security_group(parameters, group, from_port=-1, to_port=-1, ip_protocol='icmp', cidr_ip='0.0.0.0/0') return True
def configure_instance_security(self, parameters): """ Creates a GCE network and firewall with the specified name, and opens the ports on that firewall as needed for AppScale. We expect both the network and the firewall to not exist before this point, to avoid accidentally placing AppScale instances from different deployments in the same network and firewall (thus enabling them to see each other's web traffic). Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key indicating the name of the network and firewall that we should create in GCE. Returns: True, if the named network and firewall was created successfully. Raises: AgentRuntimeException: If the named network or firewall already exist in GCE. """ AppScaleLogger.log("Verifying that SSH key exists locally") keyname = parameters[self.PARAM_KEYNAME] private_key = LocalState.LOCAL_APPSCALE_PATH + keyname public_key = private_key + ".pub" if os.path.exists(private_key) or os.path.exists(public_key): raise AgentRuntimeException( "SSH key already found locally - please " + "use a different keyname") LocalState.generate_rsa_key(keyname, parameters[self.PARAM_VERBOSE]) ssh_key_exists, all_ssh_keys = self.does_ssh_key_exist(parameters) if not ssh_key_exists: self.create_ssh_key(parameters, all_ssh_keys) if self.does_network_exist(parameters): raise AgentRuntimeException("Network already exists - please use a " + \ "different group name.") if self.does_firewall_exist(parameters): raise AgentRuntimeException("Firewall already exists - please use a " + \ "different group name.") network_url = self.create_network(parameters) self.create_firewall(parameters, network_url)
def does_zone_exist(self, parameters): """ Queries Amazon EC2 to see if the specified availability zone exists. Args: parameters: A dict that contains the availability zone to check for existence. Returns: True if the availability zone exists, and False otherwise. """ try: conn = self.open_connection(parameters) zone = parameters[self.PARAM_ZONE] conn.get_all_zones(zone) AppScaleLogger.log('Availability zone {0} does exist'.format(zone)) return True except boto.exception.EC2ResponseError: AppScaleLogger.log('Availability zone {0} does not exist'.format(zone)) return False
def does_disk_exist(self, parameters, disk_name): """ Queries Amazon EC2 to see if the specified EBS volume exists. Args: parameters: A dict that contains the credentials needed to authenticate with AWS. disk_name: A str naming the EBS volume to check for existence. Returns: True if the named EBS volume exists, and False otherwise. """ conn = self.open_connection(parameters) try: conn.get_all_volumes([disk_name]) AppScaleLogger.log('EBS volume {0} does exist'.format(disk_name)) return True except boto.exception.EC2ResponseError: AppScaleLogger.log('EBS volume {0} does not exist'.format(disk_name)) return False
def configure_instance_security(self, parameters): """ Creates a GCE network and firewall with the specified name, and opens the ports on that firewall as needed for AppScale. We expect both the network and the firewall to not exist before this point, to avoid accidentally placing AppScale instances from different deployments in the same network and firewall (thus enabling them to see each other's web traffic). Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key indicating the name of the network and firewall that we should create in GCE. Returns: True, if the named network and firewall was created successfully. Raises: AgentRuntimeException: If the named network or firewall already exist in GCE. """ AppScaleLogger.log("Verifying that SSH key exists locally") keyname = parameters[self.PARAM_KEYNAME] private_key = LocalState.LOCAL_APPSCALE_PATH + keyname public_key = private_key + ".pub" if os.path.exists(private_key) or os.path.exists(public_key): raise AgentRuntimeException("SSH key already found locally - please " + "use a different keyname") LocalState.generate_rsa_key(keyname, parameters[self.PARAM_VERBOSE]) ssh_key_exists, all_ssh_keys = self.does_ssh_key_exist(parameters) if not ssh_key_exists: self.create_ssh_key(parameters, all_ssh_keys) if self.does_network_exist(parameters): raise AgentRuntimeException("Network already exists - please use a " + \ "different group name.") if self.does_firewall_exist(parameters): raise AgentRuntimeException("Firewall already exists - please use a " + \ "different group name.") network_url = self.create_network(parameters) self.create_firewall(parameters, network_url)
def detach_disk(self, parameters, disk_name, instance_id): """ Detaches the EBS mount specified in disk_name from the named instance. Args: parameters: A dict with keys for each parameter needed to connect to AWS. disk_name: A str naming the EBS volume to detach. instance_id: A str naming the id of the instance that the disk should be detached from. Returns: True if the disk was detached, and False otherwise. """ conn = self.open_connection(parameters) try: conn.detach_volume(disk_name, instance_id, device='/dev/sdc') return True except boto.exception.EC2ResponseError: AppScaleLogger.log("Could not detach volume with name {0}".format( disk_name)) return False
def does_address_exist(self, parameters): """ Queries Amazon EC2 to see if the specified Elastic IP address has been allocated with the given credentials. Args: parameters: A dict that contains the Elastic IP to check for existence. Returns: True if the given Elastic IP has been allocated, and False otherwise. """ try: conn = self.open_connection(parameters) elastic_ip = parameters[self.PARAM_STATIC_IP] conn.get_all_addresses(elastic_ip) AppScaleLogger.log('Elastic IP {0} can be used for this AppScale ' \ 'deployment.'.format(elastic_ip)) return True except boto.exception.EC2ResponseError: AppScaleLogger.log('Elastic IP {0} does not exist.'.format(elastic_ip)) return False
def authorize_security_group(self, parameters, group, from_port, to_port, ip_protocol, cidr_ip): """Opens up traffic on the given port range for traffic of the named type. Args: parameters: A dict that contains the credentials necessary to authenticate with AWS. group: A str that names the group whose ports should be opened. from_port: An int that names the first port that access should be allowed on. to_port: An int that names the last port that access should be allowed on. ip_protocol: A str that indicates if TCP, UDP, or ICMP traffic should be allowed. cidr_ip: A str that names the IP range that traffic should be allowed from. Raises: AgentRuntimeException: If the ports could not be opened on the security group. """ AppScaleLogger.log('Authorizing security group {0} for {1} traffic from ' \ 'port {2} to port {3}'.format(group, ip_protocol, from_port, to_port)) conn = self.open_connection(parameters) retries_left = self.SECURITY_GROUP_RETRY_COUNT while retries_left: try: conn.authorize_security_group(group, from_port=from_port, to_port=to_port, ip_protocol=ip_protocol, cidr_ip=cidr_ip) except EC2ResponseError: pass try: group_info = conn.get_all_security_groups(group)[0] for rule in group_info.rules: if int(rule.from_port) == from_port and int(rule.to_port) == to_port \ and rule.ip_protocol == ip_protocol: return except EC2ResponseError: pass time.sleep(self.SLEEP_TIME) retries_left -= 1 raise AgentRuntimeException("Couldn't authorize {0} traffic from port " \ "{1} to port {2} on CIDR IP {3}".format(ip_protocol, from_port, to_port, cidr_ip))
def does_zone_exist(self, parameters): """ Queries Eucalyptus to see if the specified availability zone exists. Args: parameters: A dict that contains the zone to check for existence. Returns: True if the availability zone exists, False otherwise. """ # Note that we can't use does_zone_exist in EC2Agent. There, if the image # doesn't exist, it throws an EC2ResponseError, but in Eucalyptus, it # doesn't (and returns None instead). conn = self.open_connection(parameters) zone = parameters[self.PARAM_ZONE] if conn.get_all_zones(zone): AppScaleLogger.log('Availability zone {0} does exist'.format(zone)) return True else: AppScaleLogger.log('Availability zone {0} does not exist'.format(zone)) return False
def cleanup_state(self, parameters): """ Removes the keyname and security group created during this AppScale deployment. Args: parameters: A dict that contains the keyname and security group to delete. """ AppScaleLogger.log("Deleting keyname {0}".format( parameters[self.PARAM_KEYNAME])) conn = self.open_connection(parameters) conn.delete_key_pair(parameters[self.PARAM_KEYNAME]) AppScaleLogger.log("Deleting security group {0}".format( parameters[self.PARAM_GROUP])) while True: try: conn.delete_security_group(parameters[self.PARAM_GROUP]) return except EC2ResponseError: time.sleep(5)
def stop_instances(self, parameters): """ Stop one of more EC2 instances. The input instance IDs are fetched from the 'instance_ids' parameters in the input map. (Also see documentation for the BaseAgent class) Args: parameters: A dictionary of parameters. """ instance_ids = parameters[self.PARAM_INSTANCE_IDS] conn = self.open_connection(parameters) conn.stop_instances(instance_ids) AppScaleLogger.log('Stopping instances: '+' '.join(instance_ids)) if not self.wait_for_status_change(parameters, conn, 'stopped', max_wait_time=120): AppScaleLogger.log("re-stopping instances: "+' '.join(instance_ids)) conn.stop_instances(instance_ids) if not self.wait_for_status_change(parameters, conn, 'stopped', max_wait_time=120): self.handle_failure("ERROR: could not stop instances: " + \ ' '.join(instance_ids))
def delete_firewall(self, parameters): """ Deletes a firewall in Google Compute Engine with the specified name. Callers should not invoke this method until they are certain that no instances are using the specified firewall, or this method will fail. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key indicating the name of the firewall that we should create. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.firewalls().delete( project=parameters[self.PARAM_PROJECT], firewall=parameters[self.PARAM_GROUP]) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT])
def does_zone_exist(self, parameters): """ Queries Eucalyptus to see if the specified availability zone exists. Args: parameters: A dict that contains the zone to check for existence. Returns: True if the availability zone exists, False otherwise. """ # Note that we can't use does_zone_exist in EC2Agent. There, if the image # doesn't exist, it throws an EC2ResponseError, but in Eucalyptus, it # doesn't (and returns None instead). conn = self.open_connection(parameters) zone = parameters[self.PARAM_ZONE] if conn.get_all_zones(zone): AppScaleLogger.log('Availability zone {0} does exist'.format(zone)) return True else: AppScaleLogger.log( 'Availability zone {0} does not exist'.format(zone)) return False
def create_ssh_key(self, parameters, all_ssh_keys): """ Creates a new SSH key in Google Compute Engine with the contents of our newly generated public key. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. all_ssh_keys: A str that contains all of the SSH keys that are currently passed in to GCE instances. """ our_public_ssh_key = None public_ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + \ parameters[self.PARAM_KEYNAME] + ".pub" with open(public_ssh_key_location) as file_handle: system_user = os.getenv('LOGNAME', default=pwd.getpwuid(os.getuid())[0]) our_public_ssh_key = system_user + ":" + file_handle.read().rstrip( ) if all_ssh_keys: new_all_ssh_keys = our_public_ssh_key + "\n" + all_ssh_keys else: new_all_ssh_keys = our_public_ssh_key gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.projects().setCommonInstanceMetadata( project=parameters[self.PARAM_PROJECT], body={ "kind": "compute#metadata", "items": [{ "key": "sshKeys", "value": new_all_ssh_keys }] }) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT])
def terminate_instances(self, parameters): """ Terminate one of more EC2 instances. The input instance IDs are fetched from the 'instance_ids' parameters in the input map. (Also see documentation for the BaseAgent class) Args: parameters: A dictionary of parameters. """ instance_ids = parameters[self.PARAM_INSTANCE_IDS] conn = self.open_connection(parameters) conn.terminate_instances(instance_ids) AppScaleLogger.log('Terminating instances: ' + ' '.join(instance_ids)) if not self.wait_for_status_change(parameters, conn, 'terminated', max_wait_time=120): AppScaleLogger.log("re-terminating instances: " + ' '.join(instance_ids)) conn.terminate_instances(instance_ids) if not self.wait_for_status_change(parameters, conn, 'terminated', max_wait_time=120): self.handle_failure("ERROR: could not terminate instances: " + \ ' '.join(instance_ids)) # Sending a second terminate to a terminated instance to remove it # from the system (ie no more in describe-instances). This helps when # bringing deployments up and down frequently and instances are still # associated with keyname (although they are terminated). AppScaleLogger.log("Removing terminated instances: " + ' '.join(instance_ids)) conn.terminate_instances(instance_ids)
def describe_instances(self, parameters, pending=False): """ Queries Google Compute Engine to see which instances are currently running, and retrieve information about their public and private IPs. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. pending: Boolean if we should show pending instances. Returns: A tuple of the form (public_ips, private_ips, instance_ids), where each member is a list. Items correspond to each other across these lists, so a caller is guaranteed that item X in each list belongs to the same virtual machine. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.instances().list( project=parameters[self.PARAM_PROJECT], filter="name eq appscale-{0}-.*".format( parameters[self.PARAM_GROUP]), zone=parameters[self.PARAM_ZONE]) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) instance_ids = [] public_ips = [] private_ips = [] if response and 'items' in response: instances = response['items'] for instance in instances: if instance['status'] == "RUNNING": instance_ids.append(instance['name']) network_interface = instance['networkInterfaces'][0] public_ips.append( network_interface['accessConfigs'][0]['natIP']) private_ips.append(network_interface['networkIP']) return public_ips, private_ips, instance_ids
def does_image_exist(self, parameters): """ Queries Google Compute Engine to see if the specified image exists for this user. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key indicating the name of the image that we should check for existence. Returns: True if the named image exists, and False otherwise. """ gce_service, credentials = self.open_connection(parameters) try: http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.images().get(project=parameters[self.PARAM_PROJECT], image=parameters[self.PARAM_IMAGE_ID]) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) return True except errors.HttpError: return False
def delete_access_config(self, parameters, instance_id): """ Instructs Google Compute Engine to remove the public IP address from the named instance. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key mapping to a list of instance names that should be deleted. instance_id: A str naming the running instance that the new public IP address should be added to. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.instances().deleteAccessConfig( project=parameters[self.PARAM_PROJECT], accessConfig="External NAT", instance=instance_id, networkInterface="nic0", zone=parameters[self.PARAM_ZONE]) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
def describe_instances(self, parameters, pending=False): """ Queries Google Compute Engine to see which instances are currently running, and retrieve information about their public and private IPs. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. pending: Boolean if we should show pending instances. Returns: A tuple of the form (public_ips, private_ips, instance_ids), where each member is a list. Items correspond to each other across these lists, so a caller is guaranteed that item X in each list belongs to the same virtual machine. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.instances().list( project=parameters[self.PARAM_PROJECT], filter="name eq {group}-.*".format(group=parameters[self.PARAM_GROUP]), zone=parameters[self.PARAM_ZONE] ) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) instance_ids = [] public_ips = [] private_ips = [] if response and 'items' in response: instances = response['items'] for instance in instances: if instance['status'] == "RUNNING": instance_ids.append(instance['name']) network_interface = instance['networkInterfaces'][0] public_ips.append(network_interface['accessConfigs'][0]['natIP']) private_ips.append(network_interface['networkIP']) return public_ips, private_ips, instance_ids
def delete_firewall(self, parameters): """ Deletes a firewall in Google Compute Engine with the specified name. Callers should not invoke this method until they are certain that no instances are using the specified firewall, or this method will fail. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key indicating the name of the firewall that we should create. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.firewalls().delete( project=parameters[self.PARAM_PROJECT], firewall=parameters[self.PARAM_GROUP] ) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT])
def create_ssh_key(self, parameters, all_ssh_keys): """ Creates a new SSH key in Google Compute Engine with the contents of our newly generated public key. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. all_ssh_keys: A str that contains all of the SSH keys that are currently passed in to GCE instances. """ our_public_ssh_key = None public_ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + \ parameters[self.PARAM_KEYNAME] + ".pub" with open(public_ssh_key_location) as file_handle: system_user = os.getenv('LOGNAME', default=pwd.getpwuid(os.getuid())[0]) our_public_ssh_key = system_user + ":" + file_handle.read().rstrip() if all_ssh_keys: new_all_ssh_keys = our_public_ssh_key + "\n" + all_ssh_keys else: new_all_ssh_keys = our_public_ssh_key gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.projects().setCommonInstanceMetadata( project=parameters[self.PARAM_PROJECT], body={ "kind": "compute#metadata", "items": [{ "key": "sshKeys", "value": new_all_ssh_keys }] } ) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT])
def delete_access_config(self, parameters, instance_id): """ Instructs Google Compute Engine to remove the public IP address from the named instance. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine, and an additional key mapping to a list of instance names that should be deleted. instance_id: A str naming the running instance that the new public IP address should be added to. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.instances().deleteAccessConfig( project=parameters[self.PARAM_PROJECT], accessConfig="External NAT", instance=instance_id, networkInterface="nic0", zone=parameters[self.PARAM_ZONE] ) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
def does_disk_exist(self, parameters, disk): """ Queries Google Compute Engine to see if the specified persistent disk exists for this user. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. disk: A str containing the name of the disk that we should check for existence. Returns: True if the named persistent disk exists, and False otherwise. """ gce_service, credentials = self.open_connection(parameters) try: http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.disks().get(project=parameters[self.PARAM_PROJECT], disk=disk, zone=parameters[self.PARAM_ZONE]) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) return True except errors.HttpError: return False
def assert_credentials_are_valid(self, parameters): """Contacts GCE to see if the given credentials are valid. Args: parameters: A dict containing the credentials necessary to interact with GCE. Raises: AgentConfigurationException: If an error is encountered during authentication. """ gce_service, credentials = self.open_connection(parameters) try: http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.instances().list(project=parameters [self.PARAM_PROJECT], zone=parameters[self.PARAM_ZONE]) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) return True except errors.HttpError as e: error_message = json.loads(e.content)['error']['message'] raise AgentConfigurationException(error_message)
def create_scratch_disk(self, parameters): """ Creates a disk from a given machine image. GCE does not support scratch disks on API version v1 and higher. We create a persistent disk upon creation to act like one to keep the abstraction used in other infrastructures. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. Returns: A str, the url to the disk to use. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) disk_name = self.generate_disk_name(parameters) project_url = '{0}{1}'.format(self.GCE_URL, parameters[self.PARAM_PROJECT]) source_image_url = '{0}{1}/global/images/{2}'.format(self.GCE_URL, parameters[self.PARAM_PROJECT], parameters[self.PARAM_IMAGE_ID]) request = gce_service.disks().insert( project=parameters[self.PARAM_PROJECT], zone=parameters[self.PARAM_ZONE], body={ 'name':disk_name }, sourceImage=source_image_url ) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT]) disk_url = "{0}/zones/{1}/disks/{2}".format( project_url, parameters[self.PARAM_ZONE], disk_name) return disk_url
def detach_disk(self, parameters, disk_name, instance_id): """ Detaches the persistent disk specified in 'disk_name' from the named instance. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. disk_name: A str naming the persistent disk to detach. instance_id: A str naming the id of the instance that the disk should be detached from. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) project_id = parameters[self.PARAM_PROJECT] request = gce_service.instances().detachDisk( project=project_id, zone=parameters[self.PARAM_ZONE], instance=instance_id, deviceName='sdb') response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT])