def does_disk_exist(self, parameters, disk):
""" Queries Google Compute Engine to see if the specified persistent disk
exists for this user.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
disk: A str containing the name of the disk that we should check for
existence.
Returns:
True if the named persistent disk exists, and False otherwise.
"""
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.disks().get(
project=parameters[self.PARAM_PROJECT],
disk=disk,
zone=parameters[self.PARAM_ZONE])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response),
parameters[self.PARAM_VERBOSE])
return True
except errors.HttpError:
return False
def create_scratch_disk(self, parameters):
""" Creates a disk from a given machine image.
GCE does not support scratch disks on API version v1 and higher. We create
a persistent disk upon creation to act like one to keep the abstraction used
in other infrastructures.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
Returns:
A str, the url to the disk to use.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
disk_name = self.generate_disk_name(parameters)
project_url = '{0}{1}'.format(self.GCE_URL,
parameters[self.PARAM_PROJECT])
source_image_url = '{0}{1}/global/images/{2}'.format(
self.GCE_URL, parameters[self.PARAM_PROJECT],
parameters[self.PARAM_IMAGE_ID])
request = gce_service.disks().insert(
project=parameters[self.PARAM_PROJECT],
zone=parameters[self.PARAM_ZONE],
body={'name': disk_name},
sourceImage=source_image_url)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
disk_url = "{0}/zones/{1}/disks/{2}".format(
project_url, parameters[self.PARAM_ZONE], disk_name)
return disk_url
def add_access_config(self, parameters, instance_id, static_ip):
""" Instructs Google Compute Engine to use the given IP address as the
public IP for the named instance.
This assumes that there is no existing public IP address for the named
instance. If this is not the case, callers should use delete_access_config
first to remove it.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key mapping to a list of
instance names that should be deleted.
instance_id: A str naming the running instance that the new public IP
address should be added to.
static_ip: A str naming the already allocated static IP address that
will be used for the named instance.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().addAccessConfig(
project=parameters[self.PARAM_PROJECT],
instance=instance_id,
networkInterface="nic0",
zone=parameters[self.PARAM_ZONE],
body={
"kind": "compute#accessConfig",
"type" : "ONE_TO_ONE_NAT",
"name" : "External NAT",
"natIP" : static_ip
}
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
def create_network(self, parameters):
""" Creates a new network in Google Compute Engine with the specified name.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
network that we should create in GCE.
Returns:
The URL corresponding to the name of the network that was created, for use
with binding this network to one or more firewalls.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.networks().insert(
project=parameters[self.PARAM_PROJECT],
body={
"name" : parameters[self.PARAM_GROUP],
"description" : "Network used for AppScale instances",
"IPv4Range" : "10.240.0.0/16"
}
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
return response['targetLink']
def terminate_instances(self, parameters):
""" Deletes the instances specified in 'parameters' running in Google
Compute Engine.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key mapping to a list of
instance names that should be deleted.
"""
instance_ids = parameters[self.PARAM_INSTANCE_IDS]
responses = []
for instance_id in instance_ids:
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().delete(
project=parameters[self.PARAM_PROJECT],
zone=parameters[self.PARAM_ZONE],
instance=instance_id
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
responses.append(response)
for response in responses:
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
def does_address_exist(self, parameters):
""" Queries Google Compute Engine to see if the specified static IP address
exists for this user.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
static IP address that we should check for existence.
Returns:
True if the named address exists, and False otherwise.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.addresses().list(
project=parameters[self.PARAM_PROJECT],
filter="address eq {0}".format(parameters[self.PARAM_STATIC_IP]),
region=parameters[self.PARAM_REGION]
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
if 'items' in response:
return True
else:
return False
def assert_credentials_are_valid(self, parameters):
"""Contacts GCE to see if the given credentials are valid.
Args:
parameters: A dict containing the credentials necessary to interact with
GCE.
Raises:
AgentConfigurationException: If an error is encountered during
authentication.
"""
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().list(
project=parameters[self.PARAM_PROJECT],
zone=parameters[self.PARAM_ZONE])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response),
parameters[self.PARAM_VERBOSE])
return True
except errors.HttpError as e:
error_message = json.loads(e.content)['error']['message']
raise AgentConfigurationException(error_message)
def get_optimal_spot_price(self, conn, instance_type, zone):
"""
Returns the spot price for an EC2 instance of the specified instance type.
The returned value is computed by averaging all the spot price history
values returned by the back-end EC2 APIs and incrementing the average by
extra 10%.
Args:
conn: A boto.EC2Connection that can be used to communicate with AWS.
instance_type: A str representing the instance type whose prices we
should speculate for.
zone: A str representing the availability zone that the instance will
be placed in.
Returns:
The estimated spot price for the specified instance type, in the
specified availability zone.
"""
end_time = datetime.datetime.now()
start_time = end_time - datetime.timedelta(days=7)
history = conn.get_spot_price_history(start_time=start_time.isoformat(),
end_time=end_time.isoformat(), product_description='Linux/UNIX',
instance_type=instance_type, availability_zone=zone)
var_sum = 0.0
for entry in history:
var_sum += entry.price
average = var_sum / len(history)
bid_price = average * 1.10
AppScaleLogger.log('The average spot instance price for a {0} machine is'\
' {1}, and 10% more is {2}'.format(instance_type, average, bid_price))
return bid_price
def create_firewall(self, parameters, network_url):
""" Creates a new firewall in Google Compute Engine with the specified name,
bound to the specified network.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
firewall that we should create.
network_url: A str containing the URL of the network that this new
firewall should be applied to.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.firewalls().insert(
project=parameters[self.PARAM_PROJECT],
body={
"name" : parameters[self.PARAM_GROUP],
"description" : "Firewall used for AppScale instances",
"network" : network_url,
"sourceRanges" : ["0.0.0.0/0"],
"allowed" : [
{"IPProtocol" : "tcp", "ports": ["1-65535"]},
{"IPProtocol" : "udp", "ports": ["1-65535"]},
{"IPProtocol" : "icmp"}
]
}
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
def create_security_group(self, parameters, group):
"""Creates a new security group in AWS with the given name.
Args:
parameters: A dict that contains the credentials necessary to authenticate
with AWS.
group: A str that names the group that should be created.
Raises:
AgentRuntimeException: If the security group could not be created.
"""
AppScaleLogger.log('Creating security group: {0}'.format(group))
conn = self.open_connection(parameters)
retries_left = self.SECURITY_GROUP_RETRY_COUNT
while retries_left:
try:
conn.create_security_group(group, 'AppScale security group')
except EC2ResponseError:
pass
try:
conn.get_all_security_groups(group)
return
except EC2ResponseError:
pass
time.sleep(self.SLEEP_TIME)
retries_left -= 1
raise AgentRuntimeException("Couldn't create security group with " \
"name {0}".format(group))
def handle_failure(self, msg):
""" Log the specified error message and raise an AgentRuntimeException
Args:
msg: An error message to be logged and included in the raised exception.
Raises:
AgentRuntimeException Contains the input error message.
"""
AppScaleLogger.log(msg)
raise AgentRuntimeException(msg)
def does_ssh_key_exist(self, parameters):
""" Queries Google Compute Engine to see if the specified SSH key exists.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine. We don't have an additional key for the name of
the SSH key, since we use the one in ~/.ssh.
Returns:
A tuple of two items. The first item is a bool that is True if
our public key's contents are in GCE, and False otherwise, while
the second item is the contents of all SSH keys stored in GCE.
"""
our_public_ssh_key = None
public_ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + \
parameters[self.PARAM_KEYNAME] + ".pub"
with open(public_ssh_key_location) as file_handle:
system_user = os.getenv('LOGNAME',
default=pwd.getpwuid(os.getuid())[0])
our_public_ssh_key = system_user + ":" + file_handle.read().rstrip(
)
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.projects().get(
project=parameters[self.PARAM_PROJECT])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response),
parameters[self.PARAM_VERBOSE])
if not 'items' in response['commonInstanceMetadata']:
return False, ""
metadata = response['commonInstanceMetadata']['items']
if not metadata:
return False, ""
all_ssh_keys = ""
for item in metadata:
if item['key'] != 'sshKeys':
continue
# Now that we know there's one or more SSH keys, just make sure that
# ours is in this list.
all_ssh_keys = item['value']
if our_public_ssh_key in all_ssh_keys:
return True, all_ssh_keys
return False, all_ssh_keys
except errors.HttpError:
return False, ""
def does_ssh_key_exist(self, parameters):
""" Queries Google Compute Engine to see if the specified SSH key exists.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine. We don't have an additional key for the name of
the SSH key, since we use the one in ~/.ssh.
Returns:
A tuple of two items. The first item is a bool that is True if
our public key's contents are in GCE, and False otherwise, while
the second item is the contents of all SSH keys stored in GCE.
"""
our_public_ssh_key = None
public_ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + \
parameters[self.PARAM_KEYNAME] + ".pub"
with open(public_ssh_key_location) as file_handle:
system_user = os.getenv('LOGNAME', default=pwd.getpwuid(os.getuid())[0])
our_public_ssh_key = system_user + ":" + file_handle.read().rstrip()
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.projects().get(
project=parameters[self.PARAM_PROJECT])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
if not 'items' in response['commonInstanceMetadata']:
return False, ""
metadata = response['commonInstanceMetadata']['items']
if not metadata:
return False, ""
all_ssh_keys = ""
for item in metadata:
if item['key'] != 'sshKeys':
continue
# Now that we know there's one or more SSH keys, just make sure that
# ours is in this list.
all_ssh_keys = item['value']
if our_public_ssh_key in all_ssh_keys:
return True, all_ssh_keys
return False, all_ssh_keys
except errors.HttpError:
return False, ""
def does_image_exist(self, parameters):
""" Queries Amazon EC2 to see if the specified image exists.
Args:
parameters: A dict that contains the machine ID to check for existence.
Returns:
True if the machine ID exists, False otherwise.
"""
try:
conn = self.open_connection(parameters)
image_id = parameters[self.PARAM_IMAGE_ID]
conn.get_image(image_id)
AppScaleLogger.log('Machine image {0} does exist'.format(image_id))
return True
except boto.exception.EC2ResponseError:
AppScaleLogger.log('Machine image {0} does not exist'.format(image_id))
return False
def configure_instance_security(self, parameters):
"""
Setup EC2 security keys and groups. Required input values are read from
the parameters dictionary. More specifically, this method expects to
find a 'keyname' parameter and a 'group' parameter in the parameters
dictionary. Using these provided values, this method will create a new
EC2 key-pair and a security group. Security group will be granted permission
to access any port on the instantiated VMs. (Also see documentation for the
BaseAgent class)
Args:
parameters: A dictionary of parameters.
"""
keyname = parameters[self.PARAM_KEYNAME]
group = parameters[self.PARAM_GROUP]
AppScaleLogger.log("Verifying that keyname {0}".format(keyname) + \
" is not already registered.")
conn = self.open_connection(parameters)
if conn.get_key_pair(keyname):
self.handle_failure("SSH keyname {0} is already registered. Please " \
"change the 'keyname' specified in your AppScalefile to a different " \
"value, or erase it to have one automatically generated for you." \
.format(keyname))
security_groups = conn.get_all_security_groups()
for security_group in security_groups:
if security_group.name == group:
self.handle_failure("Security group {0} is already registered. Please" \
" change the 'group' specified in your AppScalefile to a different " \
"value, or erase it to have one automatically generated for you." \
.format(group))
AppScaleLogger.log("Creating key pair: {0}".format(keyname))
key_pair = conn.create_key_pair(keyname)
ssh_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH, keyname)
LocalState.write_key_file(ssh_key, key_pair.material)
self.create_security_group(parameters, group)
self.authorize_security_group(parameters, group, from_port=1, to_port=65535,
ip_protocol='udp', cidr_ip='0.0.0.0/0')
self.authorize_security_group(parameters, group, from_port=1, to_port=65535,
ip_protocol='tcp', cidr_ip='0.0.0.0/0')
self.authorize_security_group(parameters, group, from_port=-1, to_port=-1,
ip_protocol='icmp', cidr_ip='0.0.0.0/0')
return True
def configure_instance_security(self, parameters):
""" Creates a GCE network and firewall with the specified name, and opens
the ports on that firewall as needed for AppScale.
We expect both the network and the firewall to not exist before this point,
to avoid accidentally placing AppScale instances from different deployments
in the same network and firewall (thus enabling them to see each other's web
traffic).
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
network and firewall that we should create in GCE.
Returns:
True, if the named network and firewall was created successfully.
Raises:
AgentRuntimeException: If the named network or firewall already exist in
GCE.
"""
AppScaleLogger.log("Verifying that SSH key exists locally")
keyname = parameters[self.PARAM_KEYNAME]
private_key = LocalState.LOCAL_APPSCALE_PATH + keyname
public_key = private_key + ".pub"
if os.path.exists(private_key) or os.path.exists(public_key):
raise AgentRuntimeException(
"SSH key already found locally - please " +
"use a different keyname")
LocalState.generate_rsa_key(keyname, parameters[self.PARAM_VERBOSE])
ssh_key_exists, all_ssh_keys = self.does_ssh_key_exist(parameters)
if not ssh_key_exists:
self.create_ssh_key(parameters, all_ssh_keys)
if self.does_network_exist(parameters):
raise AgentRuntimeException("Network already exists - please use a " + \
"different group name.")
if self.does_firewall_exist(parameters):
raise AgentRuntimeException("Firewall already exists - please use a " + \
"different group name.")
network_url = self.create_network(parameters)
self.create_firewall(parameters, network_url)
def does_zone_exist(self, parameters):
""" Queries Amazon EC2 to see if the specified availability zone exists.
Args:
parameters: A dict that contains the availability zone to check for
existence.
Returns:
True if the availability zone exists, and False otherwise.
"""
try:
conn = self.open_connection(parameters)
zone = parameters[self.PARAM_ZONE]
conn.get_all_zones(zone)
AppScaleLogger.log('Availability zone {0} does exist'.format(zone))
return True
except boto.exception.EC2ResponseError:
AppScaleLogger.log('Availability zone {0} does not exist'.format(zone))
return False
def does_disk_exist(self, parameters, disk_name):
""" Queries Amazon EC2 to see if the specified EBS volume exists.
Args:
parameters: A dict that contains the credentials needed to authenticate
with AWS.
disk_name: A str naming the EBS volume to check for existence.
Returns:
True if the named EBS volume exists, and False otherwise.
"""
conn = self.open_connection(parameters)
try:
conn.get_all_volumes([disk_name])
AppScaleLogger.log('EBS volume {0} does exist'.format(disk_name))
return True
except boto.exception.EC2ResponseError:
AppScaleLogger.log('EBS volume {0} does not exist'.format(disk_name))
return False
def configure_instance_security(self, parameters):
""" Creates a GCE network and firewall with the specified name, and opens
the ports on that firewall as needed for AppScale.
We expect both the network and the firewall to not exist before this point,
to avoid accidentally placing AppScale instances from different deployments
in the same network and firewall (thus enabling them to see each other's web
traffic).
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
network and firewall that we should create in GCE.
Returns:
True, if the named network and firewall was created successfully.
Raises:
AgentRuntimeException: If the named network or firewall already exist in
GCE.
"""
AppScaleLogger.log("Verifying that SSH key exists locally")
keyname = parameters[self.PARAM_KEYNAME]
private_key = LocalState.LOCAL_APPSCALE_PATH + keyname
public_key = private_key + ".pub"
if os.path.exists(private_key) or os.path.exists(public_key):
raise AgentRuntimeException("SSH key already found locally - please " +
"use a different keyname")
LocalState.generate_rsa_key(keyname, parameters[self.PARAM_VERBOSE])
ssh_key_exists, all_ssh_keys = self.does_ssh_key_exist(parameters)
if not ssh_key_exists:
self.create_ssh_key(parameters, all_ssh_keys)
if self.does_network_exist(parameters):
raise AgentRuntimeException("Network already exists - please use a " + \
"different group name.")
if self.does_firewall_exist(parameters):
raise AgentRuntimeException("Firewall already exists - please use a " + \
"different group name.")
network_url = self.create_network(parameters)
self.create_firewall(parameters, network_url)
def detach_disk(self, parameters, disk_name, instance_id):
""" Detaches the EBS mount specified in disk_name from the named instance.
Args:
parameters: A dict with keys for each parameter needed to connect to AWS.
disk_name: A str naming the EBS volume to detach.
instance_id: A str naming the id of the instance that the disk should be
detached from.
Returns:
True if the disk was detached, and False otherwise.
"""
conn = self.open_connection(parameters)
try:
conn.detach_volume(disk_name, instance_id, device='/dev/sdc')
return True
except boto.exception.EC2ResponseError:
AppScaleLogger.log("Could not detach volume with name {0}".format(
disk_name))
return False
def does_address_exist(self, parameters):
""" Queries Amazon EC2 to see if the specified Elastic IP address has been
allocated with the given credentials.
Args:
parameters: A dict that contains the Elastic IP to check for existence.
Returns:
True if the given Elastic IP has been allocated, and False otherwise.
"""
try:
conn = self.open_connection(parameters)
elastic_ip = parameters[self.PARAM_STATIC_IP]
conn.get_all_addresses(elastic_ip)
AppScaleLogger.log('Elastic IP {0} can be used for this AppScale ' \
'deployment.'.format(elastic_ip))
return True
except boto.exception.EC2ResponseError:
AppScaleLogger.log('Elastic IP {0} does not exist.'.format(elastic_ip))
return False
def authorize_security_group(self, parameters, group, from_port, to_port,
ip_protocol, cidr_ip):
"""Opens up traffic on the given port range for traffic of the named type.
Args:
parameters: A dict that contains the credentials necessary to authenticate
with AWS.
group: A str that names the group whose ports should be opened.
from_port: An int that names the first port that access should be allowed
on.
to_port: An int that names the last port that access should be allowed on.
ip_protocol: A str that indicates if TCP, UDP, or ICMP traffic should be
allowed.
cidr_ip: A str that names the IP range that traffic should be allowed
from.
Raises:
AgentRuntimeException: If the ports could not be opened on the security
group.
"""
AppScaleLogger.log('Authorizing security group {0} for {1} traffic from ' \
'port {2} to port {3}'.format(group, ip_protocol, from_port, to_port))
conn = self.open_connection(parameters)
retries_left = self.SECURITY_GROUP_RETRY_COUNT
while retries_left:
try:
conn.authorize_security_group(group, from_port=from_port,
to_port=to_port, ip_protocol=ip_protocol, cidr_ip=cidr_ip)
except EC2ResponseError:
pass
try:
group_info = conn.get_all_security_groups(group)[0]
for rule in group_info.rules:
if int(rule.from_port) == from_port and int(rule.to_port) == to_port \
and rule.ip_protocol == ip_protocol:
return
except EC2ResponseError:
pass
time.sleep(self.SLEEP_TIME)
retries_left -= 1
raise AgentRuntimeException("Couldn't authorize {0} traffic from port " \
"{1} to port {2} on CIDR IP {3}".format(ip_protocol, from_port, to_port,
cidr_ip))
def does_zone_exist(self, parameters):
"""
Queries Eucalyptus to see if the specified availability zone exists.
Args:
parameters: A dict that contains the zone to check for existence.
Returns:
True if the availability zone exists, False otherwise.
"""
# Note that we can't use does_zone_exist in EC2Agent. There, if the image
# doesn't exist, it throws an EC2ResponseError, but in Eucalyptus, it
# doesn't (and returns None instead).
conn = self.open_connection(parameters)
zone = parameters[self.PARAM_ZONE]
if conn.get_all_zones(zone):
AppScaleLogger.log('Availability zone {0} does exist'.format(zone))
return True
else:
AppScaleLogger.log('Availability zone {0} does not exist'.format(zone))
return False
def cleanup_state(self, parameters):
""" Removes the keyname and security group created during this AppScale
deployment.
Args:
parameters: A dict that contains the keyname and security group to delete.
"""
AppScaleLogger.log("Deleting keyname {0}".format(
parameters[self.PARAM_KEYNAME]))
conn = self.open_connection(parameters)
conn.delete_key_pair(parameters[self.PARAM_KEYNAME])
AppScaleLogger.log("Deleting security group {0}".format(
parameters[self.PARAM_GROUP]))
while True:
try:
conn.delete_security_group(parameters[self.PARAM_GROUP])
return
except EC2ResponseError:
time.sleep(5)
def stop_instances(self, parameters):
"""
Stop one of more EC2 instances. The input instance IDs are
fetched from the 'instance_ids' parameters in the input map. (Also
see documentation for the BaseAgent class)
Args:
parameters: A dictionary of parameters.
"""
instance_ids = parameters[self.PARAM_INSTANCE_IDS]
conn = self.open_connection(parameters)
conn.stop_instances(instance_ids)
AppScaleLogger.log('Stopping instances: '+' '.join(instance_ids))
if not self.wait_for_status_change(parameters, conn, 'stopped',
max_wait_time=120):
AppScaleLogger.log("re-stopping instances: "+' '.join(instance_ids))
conn.stop_instances(instance_ids)
if not self.wait_for_status_change(parameters, conn, 'stopped',
max_wait_time=120):
self.handle_failure("ERROR: could not stop instances: " + \
' '.join(instance_ids))
def delete_firewall(self, parameters):
""" Deletes a firewall in Google Compute Engine with the specified name.
Callers should not invoke this method until they are certain that no
instances are using the specified firewall, or this method will fail.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
firewall that we should create.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.firewalls().delete(
project=parameters[self.PARAM_PROJECT],
firewall=parameters[self.PARAM_GROUP])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
def does_zone_exist(self, parameters):
"""
Queries Eucalyptus to see if the specified availability zone exists.
Args:
parameters: A dict that contains the zone to check for existence.
Returns:
True if the availability zone exists, False otherwise.
"""
# Note that we can't use does_zone_exist in EC2Agent. There, if the image
# doesn't exist, it throws an EC2ResponseError, but in Eucalyptus, it
# doesn't (and returns None instead).
conn = self.open_connection(parameters)
zone = parameters[self.PARAM_ZONE]
if conn.get_all_zones(zone):
AppScaleLogger.log('Availability zone {0} does exist'.format(zone))
return True
else:
AppScaleLogger.log(
'Availability zone {0} does not exist'.format(zone))
return False
def create_ssh_key(self, parameters, all_ssh_keys):
""" Creates a new SSH key in Google Compute Engine with the contents of
our newly generated public key.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
all_ssh_keys: A str that contains all of the SSH keys that are
currently passed in to GCE instances.
"""
our_public_ssh_key = None
public_ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + \
parameters[self.PARAM_KEYNAME] + ".pub"
with open(public_ssh_key_location) as file_handle:
system_user = os.getenv('LOGNAME',
default=pwd.getpwuid(os.getuid())[0])
our_public_ssh_key = system_user + ":" + file_handle.read().rstrip(
)
if all_ssh_keys:
new_all_ssh_keys = our_public_ssh_key + "\n" + all_ssh_keys
else:
new_all_ssh_keys = our_public_ssh_key
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.projects().setCommonInstanceMetadata(
project=parameters[self.PARAM_PROJECT],
body={
"kind": "compute#metadata",
"items": [{
"key": "sshKeys",
"value": new_all_ssh_keys
}]
})
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
def terminate_instances(self, parameters):
"""
Terminate one of more EC2 instances. The input instance IDs are
fetched from the 'instance_ids' parameters in the input map. (Also
see documentation for the BaseAgent class)
Args:
parameters: A dictionary of parameters.
"""
instance_ids = parameters[self.PARAM_INSTANCE_IDS]
conn = self.open_connection(parameters)
conn.terminate_instances(instance_ids)
AppScaleLogger.log('Terminating instances: ' + ' '.join(instance_ids))
if not self.wait_for_status_change(parameters, conn, 'terminated',
max_wait_time=120):
AppScaleLogger.log("re-terminating instances: " + ' '.join(instance_ids))
conn.terminate_instances(instance_ids)
if not self.wait_for_status_change(parameters, conn, 'terminated',
max_wait_time=120):
self.handle_failure("ERROR: could not terminate instances: " + \
' '.join(instance_ids))
# Sending a second terminate to a terminated instance to remove it
# from the system (ie no more in describe-instances). This helps when
# bringing deployments up and down frequently and instances are still
# associated with keyname (although they are terminated).
AppScaleLogger.log("Removing terminated instances: " + ' '.join(instance_ids))
conn.terminate_instances(instance_ids)
def describe_instances(self, parameters, pending=False):
""" Queries Google Compute Engine to see which instances are currently
running, and retrieve information about their public and private IPs.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
pending: Boolean if we should show pending instances.
Returns:
A tuple of the form (public_ips, private_ips, instance_ids), where each
member is a list. Items correspond to each other across these lists,
so a caller is guaranteed that item X in each list belongs to the same
virtual machine.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().list(
project=parameters[self.PARAM_PROJECT],
filter="name eq appscale-{0}-.*".format(
parameters[self.PARAM_GROUP]),
zone=parameters[self.PARAM_ZONE])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
instance_ids = []
public_ips = []
private_ips = []
if response and 'items' in response:
instances = response['items']
for instance in instances:
if instance['status'] == "RUNNING":
instance_ids.append(instance['name'])
network_interface = instance['networkInterfaces'][0]
public_ips.append(
network_interface['accessConfigs'][0]['natIP'])
private_ips.append(network_interface['networkIP'])
return public_ips, private_ips, instance_ids
def does_image_exist(self, parameters):
""" Queries Google Compute Engine to see if the specified image exists for
this user.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
image that we should check for existence.
Returns:
True if the named image exists, and False otherwise.
"""
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.images().get(project=parameters[self.PARAM_PROJECT],
image=parameters[self.PARAM_IMAGE_ID])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
return True
except errors.HttpError:
return False
def delete_access_config(self, parameters, instance_id):
""" Instructs Google Compute Engine to remove the public IP address from
the named instance.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key mapping to a list of
instance names that should be deleted.
instance_id: A str naming the running instance that the new public IP
address should be added to.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().deleteAccessConfig(
project=parameters[self.PARAM_PROJECT],
accessConfig="External NAT",
instance=instance_id,
networkInterface="nic0",
zone=parameters[self.PARAM_ZONE])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
def describe_instances(self, parameters, pending=False):
""" Queries Google Compute Engine to see which instances are currently
running, and retrieve information about their public and private IPs.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
pending: Boolean if we should show pending instances.
Returns:
A tuple of the form (public_ips, private_ips, instance_ids), where each
member is a list. Items correspond to each other across these lists,
so a caller is guaranteed that item X in each list belongs to the same
virtual machine.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().list(
project=parameters[self.PARAM_PROJECT],
filter="name eq {group}-.*".format(group=parameters[self.PARAM_GROUP]),
zone=parameters[self.PARAM_ZONE]
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
instance_ids = []
public_ips = []
private_ips = []
if response and 'items' in response:
instances = response['items']
for instance in instances:
if instance['status'] == "RUNNING":
instance_ids.append(instance['name'])
network_interface = instance['networkInterfaces'][0]
public_ips.append(network_interface['accessConfigs'][0]['natIP'])
private_ips.append(network_interface['networkIP'])
return public_ips, private_ips, instance_ids
def delete_firewall(self, parameters):
""" Deletes a firewall in Google Compute Engine with the specified name.
Callers should not invoke this method until they are certain that no
instances are using the specified firewall, or this method will fail.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
firewall that we should create.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.firewalls().delete(
project=parameters[self.PARAM_PROJECT],
firewall=parameters[self.PARAM_GROUP]
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
def create_ssh_key(self, parameters, all_ssh_keys):
""" Creates a new SSH key in Google Compute Engine with the contents of
our newly generated public key.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
all_ssh_keys: A str that contains all of the SSH keys that are
currently passed in to GCE instances.
"""
our_public_ssh_key = None
public_ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + \
parameters[self.PARAM_KEYNAME] + ".pub"
with open(public_ssh_key_location) as file_handle:
system_user = os.getenv('LOGNAME', default=pwd.getpwuid(os.getuid())[0])
our_public_ssh_key = system_user + ":" + file_handle.read().rstrip()
if all_ssh_keys:
new_all_ssh_keys = our_public_ssh_key + "\n" + all_ssh_keys
else:
new_all_ssh_keys = our_public_ssh_key
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.projects().setCommonInstanceMetadata(
project=parameters[self.PARAM_PROJECT],
body={
"kind": "compute#metadata",
"items": [{
"key": "sshKeys",
"value": new_all_ssh_keys
}]
}
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
def delete_access_config(self, parameters, instance_id):
""" Instructs Google Compute Engine to remove the public IP address from
the named instance.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key mapping to a list of
instance names that should be deleted.
instance_id: A str naming the running instance that the new public IP
address should be added to.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().deleteAccessConfig(
project=parameters[self.PARAM_PROJECT],
accessConfig="External NAT",
instance=instance_id,
networkInterface="nic0",
zone=parameters[self.PARAM_ZONE]
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
def does_disk_exist(self, parameters, disk):
""" Queries Google Compute Engine to see if the specified persistent disk
exists for this user.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
disk: A str containing the name of the disk that we should check for
existence.
Returns:
True if the named persistent disk exists, and False otherwise.
"""
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.disks().get(project=parameters[self.PARAM_PROJECT],
disk=disk, zone=parameters[self.PARAM_ZONE])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
return True
except errors.HttpError:
return False
def assert_credentials_are_valid(self, parameters):
"""Contacts GCE to see if the given credentials are valid.
Args:
parameters: A dict containing the credentials necessary to interact with
GCE.
Raises:
AgentConfigurationException: If an error is encountered during
authentication.
"""
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().list(project=parameters
[self.PARAM_PROJECT], zone=parameters[self.PARAM_ZONE])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
return True
except errors.HttpError as e:
error_message = json.loads(e.content)['error']['message']
raise AgentConfigurationException(error_message)
def create_scratch_disk(self, parameters):
""" Creates a disk from a given machine image.
GCE does not support scratch disks on API version v1 and higher. We create
a persistent disk upon creation to act like one to keep the abstraction used
in other infrastructures.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
Returns:
A str, the url to the disk to use.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
disk_name = self.generate_disk_name(parameters)
project_url = '{0}{1}'.format(self.GCE_URL,
parameters[self.PARAM_PROJECT])
source_image_url = '{0}{1}/global/images/{2}'.format(self.GCE_URL,
parameters[self.PARAM_PROJECT], parameters[self.PARAM_IMAGE_ID])
request = gce_service.disks().insert(
project=parameters[self.PARAM_PROJECT],
zone=parameters[self.PARAM_ZONE],
body={
'name':disk_name
},
sourceImage=source_image_url
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
disk_url = "{0}/zones/{1}/disks/{2}".format(
project_url, parameters[self.PARAM_ZONE], disk_name)
return disk_url
def detach_disk(self, parameters, disk_name, instance_id):
""" Detaches the persistent disk specified in 'disk_name' from the named
instance.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
disk_name: A str naming the persistent disk to detach.
instance_id: A str naming the id of the instance that the disk should be
detached from.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
project_id = parameters[self.PARAM_PROJECT]
request = gce_service.instances().detachDisk(
project=project_id,
zone=parameters[self.PARAM_ZONE],
instance=instance_id,
deviceName='sdb')
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
