def create_security_group(self, parameters, group):
"""Creates a new security group in AWS with the given name.
Args:
parameters: A dict that contains the credentials necessary to authenticate
with AWS.
group: A str that names the group that should be created.
Raises:
AgentRuntimeException: If the security group could not be created.
"""
AppScaleLogger.log('Creating security group: {0}'.format(group))
conn = self.open_connection(parameters)
retries_left = self.SECURITY_GROUP_RETRY_COUNT
while retries_left:
try:
conn.create_security_group(group, 'AppScale security group')
except EC2ResponseError:
pass
try:
conn.get_all_security_groups(group)
return
except EC2ResponseError:
pass
time.sleep(self.SLEEP_TIME)
retries_left -= 1
raise AgentRuntimeException("Couldn't create security group with " \
"name {0}".format(group))
def get_optimal_spot_price(self, conn, instance_type, zone):
"""
Returns the spot price for an EC2 instance of the specified instance type.
The returned value is computed by averaging all the spot price history
values returned by the back-end EC2 APIs and incrementing the average by
extra 10%.
Args:
conn: A boto.EC2Connection that can be used to communicate with AWS.
instance_type: A str representing the instance type whose prices we
should speculate for.
zone: A str representing the availability zone that the instance will
be placed in.
Returns:
The estimated spot price for the specified instance type, in the
specified availability zone.
"""
end_time = datetime.datetime.now()
start_time = end_time - datetime.timedelta(days=7)
history = conn.get_spot_price_history(start_time=start_time.isoformat(),
end_time=end_time.isoformat(), product_description='Linux/UNIX',
instance_type=instance_type, availability_zone=zone)
var_sum = 0.0
for entry in history:
var_sum += entry.price
average = var_sum / len(history)
bid_price = average * 1.10
AppScaleLogger.log('The average spot instance price for a {0} machine is'\
' {1}, and 10% more is {2}'.format(instance_type, average, bid_price))
return bid_price
def handle_failure(self, msg):
""" Log the specified error message and raise an AgentRuntimeException
Args:
msg: An error message to be logged and included in the raised exception.
Raises:
AgentRuntimeException Contains the input error message.
"""
AppScaleLogger.log(msg)
raise AgentRuntimeException(msg)
def configure_instance_security(self, parameters):
"""
Setup EC2 security keys and groups. Required input values are read from
the parameters dictionary. More specifically, this method expects to
find a 'keyname' parameter and a 'group' parameter in the parameters
dictionary. Using these provided values, this method will create a new
EC2 key-pair and a security group. Security group will be granted permission
to access any port on the instantiated VMs. (Also see documentation for the
BaseAgent class)
Args:
parameters: A dictionary of parameters.
"""
keyname = parameters[self.PARAM_KEYNAME]
group = parameters[self.PARAM_GROUP]
AppScaleLogger.log("Verifying that keyname {0}".format(keyname) + \
" is not already registered.")
conn = self.open_connection(parameters)
if conn.get_key_pair(keyname):
self.handle_failure("SSH keyname {0} is already registered. Please " \
"change the 'keyname' specified in your AppScalefile to a different " \
"value, or erase it to have one automatically generated for you." \
.format(keyname))
security_groups = conn.get_all_security_groups()
for security_group in security_groups:
if security_group.name == group:
self.handle_failure("Security group {0} is already registered. Please" \
" change the 'group' specified in your AppScalefile to a different " \
"value, or erase it to have one automatically generated for you." \
.format(group))
AppScaleLogger.log("Creating key pair: {0}".format(keyname))
key_pair = conn.create_key_pair(keyname)
ssh_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH, keyname)
LocalState.write_key_file(ssh_key, key_pair.material)
self.create_security_group(parameters, group)
self.authorize_security_group(parameters, group, from_port=1, to_port=65535,
ip_protocol='udp', cidr_ip='0.0.0.0/0')
self.authorize_security_group(parameters, group, from_port=1, to_port=65535,
ip_protocol='tcp', cidr_ip='0.0.0.0/0')
self.authorize_security_group(parameters, group, from_port=-1, to_port=-1,
ip_protocol='icmp', cidr_ip='0.0.0.0/0')
return True
def does_image_exist(self, parameters):
""" Queries Amazon EC2 to see if the specified image exists.
Args:
parameters: A dict that contains the machine ID to check for existence.
Returns:
True if the machine ID exists, False otherwise.
"""
try:
conn = self.open_connection(parameters)
image_id = parameters[self.PARAM_IMAGE_ID]
conn.get_image(image_id)
AppScaleLogger.log('Machine image {0} does exist'.format(image_id))
return True
except boto.exception.EC2ResponseError:
AppScaleLogger.log('Machine image {0} does not exist'.format(image_id))
return False
def configure_instance_security(self, parameters):
""" Creates a GCE network and firewall with the specified name, and opens
the ports on that firewall as needed for AppScale.
We expect both the network and the firewall to not exist before this point,
to avoid accidentally placing AppScale instances from different deployments
in the same network and firewall (thus enabling them to see each other's web
traffic).
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
network and firewall that we should create in GCE.
Returns:
True, if the named network and firewall was created successfully.
Raises:
AgentRuntimeException: If the named network or firewall already exist in
GCE.
"""
AppScaleLogger.log("Verifying that SSH key exists locally")
keyname = parameters[self.PARAM_KEYNAME]
private_key = LocalState.LOCAL_APPSCALE_PATH + keyname
public_key = private_key + ".pub"
if os.path.exists(private_key) or os.path.exists(public_key):
raise AgentRuntimeException(
"SSH key already found locally - please " +
"use a different keyname")
LocalState.generate_rsa_key(keyname, parameters[self.PARAM_VERBOSE])
ssh_key_exists, all_ssh_keys = self.does_ssh_key_exist(parameters)
if not ssh_key_exists:
self.create_ssh_key(parameters, all_ssh_keys)
if self.does_network_exist(parameters):
raise AgentRuntimeException("Network already exists - please use a " + \
"different group name.")
if self.does_firewall_exist(parameters):
raise AgentRuntimeException("Firewall already exists - please use a " + \
"different group name.")
network_url = self.create_network(parameters)
self.create_firewall(parameters, network_url)
def does_zone_exist(self, parameters):
""" Queries Amazon EC2 to see if the specified availability zone exists.
Args:
parameters: A dict that contains the availability zone to check for
existence.
Returns:
True if the availability zone exists, and False otherwise.
"""
try:
conn = self.open_connection(parameters)
zone = parameters[self.PARAM_ZONE]
conn.get_all_zones(zone)
AppScaleLogger.log('Availability zone {0} does exist'.format(zone))
return True
except boto.exception.EC2ResponseError:
AppScaleLogger.log('Availability zone {0} does not exist'.format(zone))
return False
def does_disk_exist(self, parameters, disk_name):
""" Queries Amazon EC2 to see if the specified EBS volume exists.
Args:
parameters: A dict that contains the credentials needed to authenticate
with AWS.
disk_name: A str naming the EBS volume to check for existence.
Returns:
True if the named EBS volume exists, and False otherwise.
"""
conn = self.open_connection(parameters)
try:
conn.get_all_volumes([disk_name])
AppScaleLogger.log('EBS volume {0} does exist'.format(disk_name))
return True
except boto.exception.EC2ResponseError:
AppScaleLogger.log('EBS volume {0} does not exist'.format(disk_name))
return False
def configure_instance_security(self, parameters):
""" Creates a GCE network and firewall with the specified name, and opens
the ports on that firewall as needed for AppScale.
We expect both the network and the firewall to not exist before this point,
to avoid accidentally placing AppScale instances from different deployments
in the same network and firewall (thus enabling them to see each other's web
traffic).
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
network and firewall that we should create in GCE.
Returns:
True, if the named network and firewall was created successfully.
Raises:
AgentRuntimeException: If the named network or firewall already exist in
GCE.
"""
AppScaleLogger.log("Verifying that SSH key exists locally")
keyname = parameters[self.PARAM_KEYNAME]
private_key = LocalState.LOCAL_APPSCALE_PATH + keyname
public_key = private_key + ".pub"
if os.path.exists(private_key) or os.path.exists(public_key):
raise AgentRuntimeException("SSH key already found locally - please " +
"use a different keyname")
LocalState.generate_rsa_key(keyname, parameters[self.PARAM_VERBOSE])
ssh_key_exists, all_ssh_keys = self.does_ssh_key_exist(parameters)
if not ssh_key_exists:
self.create_ssh_key(parameters, all_ssh_keys)
if self.does_network_exist(parameters):
raise AgentRuntimeException("Network already exists - please use a " + \
"different group name.")
if self.does_firewall_exist(parameters):
raise AgentRuntimeException("Firewall already exists - please use a " + \
"different group name.")
network_url = self.create_network(parameters)
self.create_firewall(parameters, network_url)
def detach_disk(self, parameters, disk_name, instance_id):
""" Detaches the EBS mount specified in disk_name from the named instance.
Args:
parameters: A dict with keys for each parameter needed to connect to AWS.
disk_name: A str naming the EBS volume to detach.
instance_id: A str naming the id of the instance that the disk should be
detached from.
Returns:
True if the disk was detached, and False otherwise.
"""
conn = self.open_connection(parameters)
try:
conn.detach_volume(disk_name, instance_id, device='/dev/sdc')
return True
except boto.exception.EC2ResponseError:
AppScaleLogger.log("Could not detach volume with name {0}".format(
disk_name))
return False
def does_address_exist(self, parameters):
""" Queries Amazon EC2 to see if the specified Elastic IP address has been
allocated with the given credentials.
Args:
parameters: A dict that contains the Elastic IP to check for existence.
Returns:
True if the given Elastic IP has been allocated, and False otherwise.
"""
try:
conn = self.open_connection(parameters)
elastic_ip = parameters[self.PARAM_STATIC_IP]
conn.get_all_addresses(elastic_ip)
AppScaleLogger.log('Elastic IP {0} can be used for this AppScale ' \
'deployment.'.format(elastic_ip))
return True
except boto.exception.EC2ResponseError:
AppScaleLogger.log('Elastic IP {0} does not exist.'.format(elastic_ip))
return False
def authorize_security_group(self, parameters, group, from_port, to_port,
ip_protocol, cidr_ip):
"""Opens up traffic on the given port range for traffic of the named type.
Args:
parameters: A dict that contains the credentials necessary to authenticate
with AWS.
group: A str that names the group whose ports should be opened.
from_port: An int that names the first port that access should be allowed
on.
to_port: An int that names the last port that access should be allowed on.
ip_protocol: A str that indicates if TCP, UDP, or ICMP traffic should be
allowed.
cidr_ip: A str that names the IP range that traffic should be allowed
from.
Raises:
AgentRuntimeException: If the ports could not be opened on the security
group.
"""
AppScaleLogger.log('Authorizing security group {0} for {1} traffic from ' \
'port {2} to port {3}'.format(group, ip_protocol, from_port, to_port))
conn = self.open_connection(parameters)
retries_left = self.SECURITY_GROUP_RETRY_COUNT
while retries_left:
try:
conn.authorize_security_group(group, from_port=from_port,
to_port=to_port, ip_protocol=ip_protocol, cidr_ip=cidr_ip)
except EC2ResponseError:
pass
try:
group_info = conn.get_all_security_groups(group)[0]
for rule in group_info.rules:
if int(rule.from_port) == from_port and int(rule.to_port) == to_port \
and rule.ip_protocol == ip_protocol:
return
except EC2ResponseError:
pass
time.sleep(self.SLEEP_TIME)
retries_left -= 1
raise AgentRuntimeException("Couldn't authorize {0} traffic from port " \
"{1} to port {2} on CIDR IP {3}".format(ip_protocol, from_port, to_port,
cidr_ip))
def cleanup_state(self, parameters):
""" Removes the keyname and security group created during this AppScale
deployment.
Args:
parameters: A dict that contains the keyname and security group to delete.
"""
AppScaleLogger.log("Deleting keyname {0}".format(
parameters[self.PARAM_KEYNAME]))
conn = self.open_connection(parameters)
conn.delete_key_pair(parameters[self.PARAM_KEYNAME])
AppScaleLogger.log("Deleting security group {0}".format(
parameters[self.PARAM_GROUP]))
while True:
try:
conn.delete_security_group(parameters[self.PARAM_GROUP])
return
except EC2ResponseError:
time.sleep(5)
def does_zone_exist(self, parameters):
"""
Queries Eucalyptus to see if the specified availability zone exists.
Args:
parameters: A dict that contains the zone to check for existence.
Returns:
True if the availability zone exists, False otherwise.
"""
# Note that we can't use does_zone_exist in EC2Agent. There, if the image
# doesn't exist, it throws an EC2ResponseError, but in Eucalyptus, it
# doesn't (and returns None instead).
conn = self.open_connection(parameters)
zone = parameters[self.PARAM_ZONE]
if conn.get_all_zones(zone):
AppScaleLogger.log('Availability zone {0} does exist'.format(zone))
return True
else:
AppScaleLogger.log('Availability zone {0} does not exist'.format(zone))
return False
def stop_instances(self, parameters):
"""
Stop one of more EC2 instances. The input instance IDs are
fetched from the 'instance_ids' parameters in the input map. (Also
see documentation for the BaseAgent class)
Args:
parameters: A dictionary of parameters.
"""
instance_ids = parameters[self.PARAM_INSTANCE_IDS]
conn = self.open_connection(parameters)
conn.stop_instances(instance_ids)
AppScaleLogger.log('Stopping instances: '+' '.join(instance_ids))
if not self.wait_for_status_change(parameters, conn, 'stopped',
max_wait_time=120):
AppScaleLogger.log("re-stopping instances: "+' '.join(instance_ids))
conn.stop_instances(instance_ids)
if not self.wait_for_status_change(parameters, conn, 'stopped',
max_wait_time=120):
self.handle_failure("ERROR: could not stop instances: " + \
' '.join(instance_ids))
def does_zone_exist(self, parameters):
"""
Queries Eucalyptus to see if the specified availability zone exists.
Args:
parameters: A dict that contains the zone to check for existence.
Returns:
True if the availability zone exists, False otherwise.
"""
# Note that we can't use does_zone_exist in EC2Agent. There, if the image
# doesn't exist, it throws an EC2ResponseError, but in Eucalyptus, it
# doesn't (and returns None instead).
conn = self.open_connection(parameters)
zone = parameters[self.PARAM_ZONE]
if conn.get_all_zones(zone):
AppScaleLogger.log('Availability zone {0} does exist'.format(zone))
return True
else:
AppScaleLogger.log(
'Availability zone {0} does not exist'.format(zone))
return False
def terminate_instances(self, parameters):
"""
Terminate one of more EC2 instances. The input instance IDs are
fetched from the 'instance_ids' parameters in the input map. (Also
see documentation for the BaseAgent class)
Args:
parameters: A dictionary of parameters.
"""
instance_ids = parameters[self.PARAM_INSTANCE_IDS]
conn = self.open_connection(parameters)
conn.terminate_instances(instance_ids)
AppScaleLogger.log('Terminating instances: ' + ' '.join(instance_ids))
if not self.wait_for_status_change(parameters, conn, 'terminated',
max_wait_time=120):
AppScaleLogger.log("re-terminating instances: " + ' '.join(instance_ids))
conn.terminate_instances(instance_ids)
if not self.wait_for_status_change(parameters, conn, 'terminated',
max_wait_time=120):
self.handle_failure("ERROR: could not terminate instances: " + \
' '.join(instance_ids))
# Sending a second terminate to a terminated instance to remove it
# from the system (ie no more in describe-instances). This helps when
# bringing deployments up and down frequently and instances are still
# associated with keyname (although they are terminated).
AppScaleLogger.log("Removing terminated instances: " + ' '.join(instance_ids))
conn.terminate_instances(instance_ids)
def run_instances(self, count, parameters, security_configured):
""" Starts 'count' instances in Google Compute Engine, and returns once they
have been started.
Callers should create a network and attach a firewall to it before using
this method, or the newly created instances will not have a network and
firewall to attach to (and thus this method will fail).
Args:
count: An int that specifies how many virtual machines should be started.
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
security_configured: Unused, as we assume that the network and firewall
has already been set up.
"""
project_id = parameters[self.PARAM_PROJECT]
image_id = parameters[self.PARAM_IMAGE_ID]
instance_type = parameters[self.PARAM_INSTANCE_TYPE]
keyname = parameters[self.PARAM_KEYNAME]
group = parameters[self.PARAM_GROUP]
zone = parameters[self.PARAM_ZONE]
AppScaleLogger.log("Starting {0} machines with machine id {1}, with " \
"instance type {2}, keyname {3}, in security group {4}, in zone {5}" \
.format(count, image_id, instance_type, keyname, group, zone))
# First, see how many instances are running and what their info is.
start_time = datetime.datetime.now()
active_public_ips, active_private_ips, active_instances = \
self.describe_instances(parameters)
# Construct URLs
image_url = '{0}{1}/global/images/{2}'.format(self.GCE_URL, project_id,
image_id)
project_url = '{0}{1}'.format(self.GCE_URL, project_id)
machine_type_url = '{0}/zones/{1}/machineTypes/{2}'.format(project_url,
zone, instance_type)
network_url = '{0}/global/networks/{1}'.format(project_url, group)
# Construct the request body
for index in range(count):
disk_url = self.create_scratch_disk(parameters)
instances = {
# Truncate the name down to the first 62 characters, since GCE doesn't
# let us use arbitrarily long instance names.
'name': '{group}-{uuid}'.format(group=group, uuid=uuid.uuid4())[:62],
'machineType': machine_type_url,
'disks':[{
'source': disk_url,
'boot': 'true',
'type': 'PERSISTENT'
}],
'image': image_url,
'networkInterfaces': [{
'accessConfigs': [{
'type': 'ONE_TO_ONE_NAT',
'name': 'External NAT'
}],
'network': network_url
}],
'serviceAccounts': [{
'email': self.DEFAULT_SERVICE_EMAIL,
'scopes': [self.GCE_SCOPE]
}]
}
# Create the instance
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().insert(
project=project_id, body=instances, zone=zone)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
instance_ids = []
public_ips = []
private_ips = []
end_time = datetime.datetime.now() + datetime.timedelta(0,
self.MAX_VM_CREATION_TIME)
now = datetime.datetime.now()
while now < end_time:
AppScaleLogger.log("Waiting for your instances to start...")
instance_info = self.describe_instances(parameters)
public_ips = instance_info[0]
private_ips = instance_info[1]
instance_ids = instance_info[2]
public_ips = self.diff(public_ips, active_public_ips)
private_ips = self.diff(private_ips, active_private_ips)
instance_ids = self.diff(instance_ids, active_instances)
if count == len(public_ips):
break
time.sleep(self.SLEEP_TIME)
now = datetime.datetime.now()
if not public_ips:
self.handle_failure('No public IPs were able to be procured '
'within the time limit')
if len(public_ips) != count:
for index in range(0, len(public_ips)):
if public_ips[index] == '0.0.0.0':
instance_to_term = instance_ids[index]
AppScaleLogger.log('Instance {0} failed to get a public IP address'\
'and is being terminated'.format(instance_to_term))
self.terminate_instances([instance_to_term])
end_time = datetime.datetime.now()
total_time = end_time - start_time
AppScaleLogger.log("Started {0} on-demand instances in {1} seconds" \
.format(count, total_time.seconds))
return instance_ids, public_ips, private_ips
def run_instances(self, count, parameters, security_configured):
"""
Spawns the specified number of EC2 instances using the parameters
provided. This method is blocking in that it waits until the
requested VMs are properly booted up. However if the requested
VMs cannot be procured within 1800 seconds, this method will treat
it as an error and return. (Also see documentation for the BaseAgent
class)
Args:
count: Number of VMs to spawned.
parameters: A dictionary of parameters. This must contain
'keyname', 'group', 'image_id' and 'instance_type' parameters.
security_configured: Uses this boolean value as an heuristic to
detect brand new AppScale deployments.
Returns:
A tuple of the form (instances, public_ips, private_ips)
"""
image_id = parameters[self.PARAM_IMAGE_ID]
instance_type = parameters[self.PARAM_INSTANCE_TYPE]
keyname = parameters[self.PARAM_KEYNAME]
group = parameters[self.PARAM_GROUP]
spot = parameters[self.PARAM_SPOT]
zone = parameters[self.PARAM_ZONE]
AppScaleLogger.log("Starting {0} machines with machine id {1}, with " \
"instance type {2}, keyname {3}, in security group {4}, in availability" \
" zone {5}".format(count, image_id, instance_type, keyname, group, zone))
if spot:
AppScaleLogger.log("Using spot instances")
else:
AppScaleLogger.log("Using on-demand instances")
start_time = datetime.datetime.now()
active_public_ips = []
active_private_ips = []
active_instances = []
# Make sure we do not have terminated instances using the same keyname.
instances = self.__describe_instances(parameters)
term_instance_info = self.__get_instance_info(instances,
'terminated', keyname)
if len(term_instance_info[2]):
self.handle_failure('SSH keyname {0} is already registered to a '\
'terminated instance. Please change the "keyname" '\
'you specified in your AppScalefile to a different '\
'value. If the keyname was autogenerated, erase it '\
'to have a new one generated for you.'.format(keyname))
try:
attempts = 1
while True:
instance_info = self.describe_instances(parameters)
active_public_ips = instance_info[0]
active_private_ips = instance_info[1]
active_instances = instance_info[2]
# If security has been configured on this agent just now,
# that's an indication that this is a fresh cloud deployment.
# As such it's not expected to have any running VMs.
if len(active_instances) > 0 or security_configured:
break
elif attempts == self.DESCRIBE_INSTANCES_RETRY_COUNT:
self.handle_failure('Failed to invoke describe_instances')
attempts += 1
conn = self.open_connection(parameters)
if spot:
price = parameters[self.PARAM_SPOT_PRICE] or \
self.get_optimal_spot_price(conn, instance_type, zone)
conn.request_spot_instances(str(price), image_id, key_name=keyname,
security_groups=[group], instance_type=instance_type, count=count,
placement=zone)
else:
conn.run_instances(image_id, count, count, key_name=keyname,
security_groups=[group], instance_type=instance_type, placement=zone)
instance_ids = []
public_ips = []
private_ips = []
end_time = datetime.datetime.now() + datetime.timedelta(0,
self.MAX_VM_CREATION_TIME)
now = datetime.datetime.now()
while now < end_time:
AppScaleLogger.log("Waiting for your instances to start...")
public_ips, private_ips, instance_ids = self.describe_instances(
parameters)
public_ips = self.diff(public_ips, active_public_ips)
private_ips = self.diff(private_ips, active_private_ips)
instance_ids = self.diff(instance_ids, active_instances)
if count == len(public_ips):
break
time.sleep(self.SLEEP_TIME)
now = datetime.datetime.now()
if not public_ips:
self.handle_failure('No public IPs were able to be procured '
'within the time limit')
if len(public_ips) != count:
for index in range(0, len(public_ips)):
if public_ips[index] == '0.0.0.0':
instance_to_term = instance_ids[index]
AppScaleLogger.log('Instance {0} failed to get a public IP address'\
'and is being terminated'.format(instance_to_term))
conn.terminate_instances([instance_to_term])
end_time = datetime.datetime.now()
total_time = end_time - start_time
if spot:
AppScaleLogger.log("Started {0} spot instances in {1} seconds" \
.format(count, total_time.seconds))
else:
AppScaleLogger.log("Started {0} on-demand instances in {1} seconds" \
.format(count, total_time.seconds))
return instance_ids, public_ips, private_ips
except EC2ResponseError as exception:
self.handle_failure('EC2 response error while starting VMs: ' +
exception.error_message)
