def add_access_config(self, parameters, instance_id, static_ip):
""" Instructs Google Compute Engine to use the given IP address as the
public IP for the named instance.
This assumes that there is no existing public IP address for the named
instance. If this is not the case, callers should use delete_access_config
first to remove it.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key mapping to a list of
instance names that should be deleted.
instance_id: A str naming the running instance that the new public IP
address should be added to.
static_ip: A str naming the already allocated static IP address that
will be used for the named instance.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().addAccessConfig(
project=parameters[self.PARAM_PROJECT],
instance=instance_id,
networkInterface="nic0",
zone=parameters[self.PARAM_ZONE],
body={
"kind": "compute#accessConfig",
"type" : "ONE_TO_ONE_NAT",
"name" : "External NAT",
"natIP" : static_ip
}
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
def create_virtual_network(self, network_client, parameters, network_name,
subnet_name):
""" Creates the network resources, such as Virtual network and Subnet.
Args:
network_client: A NetworkManagementClient instance.
parameters: A dict, containing all the parameters necessary to
authenticate this user with Azure.
network_name: The name to use for the Virtual Network resource.
subnet_name: The name to use for the Subnet resource.
Returns:
A Subnet instance from the Virtual Network created.
"""
group_name = parameters[self.PARAM_RESOURCE_GROUP]
region = parameters[self.PARAM_ZONE]
verbose = parameters[self.PARAM_VERBOSE]
AppScaleLogger.verbose("Creating/Updating the Virtual Network '{}'".
format(network_name), verbose)
address_space = AddressSpace(address_prefixes=['10.1.0.0/16'])
subnet1 = Subnet(name=subnet_name, address_prefix='10.1.0.0/24')
result = network_client.virtual_networks.create_or_update(group_name, network_name,
VirtualNetwork(location=region, address_space=address_space,
subnets=[subnet1]))
self.sleep_until_update_operation_done(result, network_name, verbose)
subnet = network_client.subnets.get(group_name, network_name, subnet_name)
return subnet
def update_dispatch(cls, source_location, keyname, project_id):
""" Updates an application's dispatch routing rules from the configuration
file.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if cls.TAR_GZ_REGEX.search(source_location):
fetch_function = utils.config_from_tar_gz
version = Version.from_tar_gz(source_location)
elif cls.ZIP_REGEX.search(source_location):
fetch_function = utils.config_from_zip
version = Version.from_zip(source_location)
elif os.path.isdir(source_location):
fetch_function = utils.config_from_dir
version = Version.from_directory(source_location)
elif source_location.endswith('.yaml'):
fetch_function = utils.config_from_dir
version = Version.from_yaml_file(source_location)
source_location = os.path.dirname(source_location)
else:
raise BadConfigurationException(
'{} must be a directory, tar.gz, or zip'.format(source_location))
if project_id:
version.project_id = project_id
dispatch_rules = utils.dispatch_from_yaml(source_location, fetch_function)
if dispatch_rules is None:
return
AppScaleLogger.log('Updating dispatch for {}'.format(version.project_id))
load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer')
secret_key = LocalState.get_secret_key(keyname)
admin_client = AdminClient(load_balancer_ip, secret_key)
operation_id = admin_client.update_dispatch(version.project_id, dispatch_rules)
# Check on the operation.
AppScaleLogger.log("Please wait for your dispatch to be updated.")
deadline = time.time() + cls.MAX_OPERATION_TIME
while True:
if time.time() > deadline:
raise AppScaleException('The operation took too long.')
operation = admin_client.get_operation(version.project_id, operation_id)
if not operation['done']:
time.sleep(1)
continue
if 'error' in operation:
raise AppScaleException(operation['error']['message'])
dispatch_rules = operation['response']['dispatchRules']
break
AppScaleLogger.verbose(
"The following dispatchRules have been applied to your application's "
"configuration : {}".format(dispatch_rules))
AppScaleLogger.success('Dispatch has been updated for {}'.format(
version.project_id))
def create_network(self, parameters):
""" Creates a new network in Google Compute Engine with the specified name.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
network that we should create in GCE.
Returns:
The URL corresponding to the name of the network that was created, for use
with binding this network to one or more firewalls.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.networks().insert(
project=parameters[self.PARAM_PROJECT],
body={
"name" : parameters[self.PARAM_GROUP],
"description" : "Network used for AppScale instances",
"IPv4Range" : "10.240.0.0/16"
}
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
return response['targetLink']
def terminate_instances(self, parameters):
""" Deletes the instances specified in 'parameters' running in Google
Compute Engine.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key mapping to a list of
instance names that should be deleted.
"""
instance_ids = parameters[self.PARAM_INSTANCE_IDS]
responses = []
for instance_id in instance_ids:
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().delete(
project=parameters[self.PARAM_PROJECT],
zone=parameters[self.PARAM_ZONE],
instance=instance_id
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
responses.append(response)
for response in responses:
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
def create_firewall(self, parameters, network_url):
""" Creates a new firewall in Google Compute Engine with the specified name,
bound to the specified network.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
firewall that we should create.
network_url: A str containing the URL of the network that this new
firewall should be applied to.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.firewalls().insert(
project=parameters[self.PARAM_PROJECT],
body={
"name" : parameters[self.PARAM_GROUP],
"description" : "Firewall used for AppScale instances",
"network" : network_url,
"sourceRanges" : ["0.0.0.0/0"],
"allowed" : [
{"IPProtocol" : "tcp", "ports": ["1-65535"]},
{"IPProtocol" : "udp", "ports": ["1-65535"]},
{"IPProtocol" : "icmp"}
]
}
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
def create_virtual_network(self, network_client, parameters, network_name,
subnet_name):
""" Creates the network resources, such as Virtual network and Subnet.
Args:
network_client: A NetworkManagementClient instance.
parameters: A dict, containing all the parameters necessary to
authenticate this user with Azure.
network_name: The name to use for the Virtual Network resource.
subnet_name: The name to use for the Subnet resource.
Returns:
A Subnet instance from the Virtual Network created.
"""
group_name = parameters[self.PARAM_RESOURCE_GROUP]
region = parameters[self.PARAM_ZONE]
verbose = parameters[self.PARAM_VERBOSE]
AppScaleLogger.verbose(
"Creating/Updating the Virtual Network '{}'".format(network_name),
verbose)
address_space = AddressSpace(address_prefixes=['10.1.0.0/16'])
subnet1 = Subnet(name=subnet_name, address_prefix='10.1.0.0/24')
result = network_client.virtual_networks.create_or_update(
group_name, network_name,
VirtualNetwork(location=region,
address_space=address_space,
subnets=[subnet1]))
self.sleep_until_update_operation_done(result, network_name, verbose)
subnet = network_client.subnets.get(group_name, network_name,
subnet_name)
return subnet
def does_address_exist(self, parameters):
""" Queries Google Compute Engine to see if the specified static IP address
exists for this user.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
static IP address that we should check for existence.
Returns:
True if the named address exists, and False otherwise.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.addresses().list(
project=parameters[self.PARAM_PROJECT],
filter="address eq {0}".format(parameters[self.PARAM_STATIC_IP]),
region=parameters[self.PARAM_REGION]
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
if 'items' in response:
return True
else:
return False
def sleep_until_delete_operation_done(self, result, resource_name,
max_sleep, verbose):
""" Sleeps until the delete operation for the resource is completed
successfully.
Args:
result: An instance, of the AzureOperationPoller to poll for the status
of the operation being performed.
resource_name: The name of the resource being deleted.
max_sleep: The maximum number of seconds to sleep for the resources to
be deleted.
verbose: A boolean indicating whether or not in verbose mode.
"""
time_start = time.time()
while not result.done():
AppScaleLogger.verbose(
"Waiting {0} second(s) for {1} to be deleted.".format(
self.SLEEP_TIME, resource_name), verbose)
time.sleep(self.SLEEP_TIME)
total_sleep_time = time.time() - time_start
if total_sleep_time > max_sleep:
AppScaleLogger.log(
"Waited {0} second(s) for {1} to be deleted. "
"Operation has timed out.".format(total_sleep_time,
resource_name))
break
def terminate_instances(self, parameters):
""" Deletes the instances specified in 'parameters' running in Azure.
Args:
parameters: A dict, containing all the parameters necessary to
authenticate this user with Azure.
"""
credentials = self.open_connection(parameters)
resource_group = parameters[self.PARAM_RESOURCE_GROUP]
subscription_id = parameters[self.PARAM_SUBSCRIBER_ID]
verbose = parameters[self.PARAM_VERBOSE]
public_ips, private_ips, instance_ids = self.describe_instances(
parameters)
AppScaleLogger.verbose(
"Terminating the vm instance/s '{}'".format(instance_ids), verbose)
compute_client = ComputeManagementClient(credentials, subscription_id)
threads = []
for vm_name in instance_ids:
thread = threading.Thread(target=self.delete_virtual_machine,
args=(compute_client, resource_group,
verbose, vm_name))
thread.start()
threads.append(thread)
for x in threads:
x.join()
def create_network_interface(self, network_client, interface_name, ip_name,
subnet, parameters):
""" Creates the Public IP Address resource and uses that to create the
Network Interface.
Args:
network_client: A NetworkManagementClient instance.
interface_name: The name to use for the Network Interface.
ip_name: The name to use for the Public IP Address.
subnet: The Subnet resource from the Virtual Network created.
parameters: A dict, containing all the parameters necessary to
authenticate this user with Azure.
"""
group_name = parameters[self.PARAM_RESOURCE_GROUP]
region = parameters[self.PARAM_ZONE]
verbose = parameters[self.PARAM_VERBOSE]
AppScaleLogger.verbose("Creating/Updating the Public IP Address '{}'".
format(ip_name), verbose)
ip_address = PublicIPAddress(
location=region, public_ip_allocation_method=IPAllocationMethod.dynamic,
idle_timeout_in_minutes=4)
result = network_client.public_ip_addresses.create_or_update(
group_name, ip_name, ip_address)
self.sleep_until_update_operation_done(result, ip_name, verbose)
public_ip_address = network_client.public_ip_addresses.get(group_name, ip_name)
AppScaleLogger.verbose("Creating/Updating the Network Interface '{}'".
format(interface_name), verbose)
network_interface_ip_conf = NetworkInterfaceIPConfiguration(
name=interface_name, private_ip_allocation_method=IPAllocationMethod.dynamic,
subnet=subnet, public_ip_address=PublicIPAddress(id=(public_ip_address.id)))
result = network_client.network_interfaces.create_or_update(group_name,
interface_name, NetworkInterface(location=region,
ip_configurations=[network_interface_ip_conf]))
self.sleep_until_update_operation_done(result, interface_name, verbose)
def does_ssh_key_exist(self, parameters):
""" Queries Google Compute Engine to see if the specified SSH key exists.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine. We don't have an additional key for the name of
the SSH key, since we use the one in ~/.ssh.
Returns:
A tuple of two items. The first item is a bool that is True if
our public key's contents are in GCE, and False otherwise, while
the second item is the contents of all SSH keys stored in GCE.
"""
our_public_ssh_key = None
public_ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + \
parameters[self.PARAM_KEYNAME] + ".pub"
with open(public_ssh_key_location) as file_handle:
system_user = os.getenv('LOGNAME', default=pwd.getpwuid(os.getuid())[0])
our_public_ssh_key = system_user + ":" + file_handle.read().rstrip()
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.projects().get(
project=parameters[self.PARAM_PROJECT])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
if not 'items' in response['commonInstanceMetadata']:
return False, ""
metadata = response['commonInstanceMetadata']['items']
if not metadata:
return False, ""
all_ssh_keys = ""
for item in metadata:
if item['key'] != 'sshKeys':
continue
# Now that we know there's one or more SSH keys, just make sure that
# ours is in this list.
all_ssh_keys = item['value']
if our_public_ssh_key in all_ssh_keys:
return True, all_ssh_keys
return False, all_ssh_keys
except errors.HttpError:
return False, ""
def sleep_until_update_operation_done(self, result, resource_name, verbose):
""" Sleeps until the create/update operation for the resource is completed
successfully.
Args:
result: An instance, of the AzureOperationPoller to poll for the status
of the operation being performed.
resource_name: The name of the resource being updated.
"""
time_start = time.time()
while not result.done():
AppScaleLogger.verbose("Waiting {0} second(s) for {1} to be created/updated.".
format(self.SLEEP_TIME, resource_name), verbose)
time.sleep(self.SLEEP_TIME)
total_sleep_time = time.time() - time_start
if total_sleep_time > self.MAX_SLEEP_TIME:
AppScaleLogger.log("Waited {0} second(s) for {1} to be created/updated. "
"Operation has timed out.".format(total_sleep_time, resource_name))
break
def create_network_interface(self, network_client, interface_name, ip_name,
subnet, parameters):
""" Creates the Public IP Address resource and uses that to create the
Network Interface.
Args:
network_client: A NetworkManagementClient instance.
interface_name: The name to use for the Network Interface.
ip_name: The name to use for the Public IP Address.
subnet: The Subnet resource from the Virtual Network created.
parameters: A dict, containing all the parameters necessary to
authenticate this user with Azure.
"""
group_name = parameters[self.PARAM_RESOURCE_GROUP]
region = parameters[self.PARAM_ZONE]
verbose = parameters[self.PARAM_VERBOSE]
AppScaleLogger.verbose(
"Creating/Updating the Public IP Address '{}'".format(ip_name),
verbose)
ip_address = PublicIPAddress(
location=region,
public_ip_allocation_method=IPAllocationMethod.dynamic,
idle_timeout_in_minutes=4)
result = network_client.public_ip_addresses.create_or_update(
group_name, ip_name, ip_address)
self.sleep_until_update_operation_done(result, ip_name, verbose)
public_ip_address = network_client.public_ip_addresses.get(
group_name, ip_name)
AppScaleLogger.verbose(
"Creating/Updating the Network Interface '{}'".format(
interface_name), verbose)
network_interface_ip_conf = NetworkInterfaceIPConfiguration(
name=interface_name,
private_ip_allocation_method=IPAllocationMethod.dynamic,
subnet=subnet,
public_ip_address=PublicIPAddress(id=(public_ip_address.id)))
result = network_client.network_interfaces.create_or_update(
group_name, interface_name,
NetworkInterface(location=region,
ip_configurations=[network_interface_ip_conf]))
self.sleep_until_update_operation_done(result, interface_name, verbose)
def create_ssh_key(self, parameters, all_ssh_keys):
""" Creates a new SSH key in Google Compute Engine with the contents of
our newly generated public key.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
all_ssh_keys: A str that contains all of the SSH keys that are
currently passed in to GCE instances.
"""
our_public_ssh_key = None
public_ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + \
parameters[self.PARAM_KEYNAME] + ".pub"
with open(public_ssh_key_location) as file_handle:
system_user = os.getenv('LOGNAME', default=pwd.getpwuid(os.getuid())[0])
public_ssh_key = file_handle.read().rstrip()
our_public_ssh_keys = "{system_user}:{key}\nroot:{key}".format(
system_user=system_user, key=public_ssh_key)
if all_ssh_keys:
new_all_ssh_keys = our_public_ssh_keys + "\n" + all_ssh_keys
else:
new_all_ssh_keys = our_public_ssh_key
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.projects().setCommonInstanceMetadata(
project=parameters[self.PARAM_PROJECT],
body={
"kind": "compute#metadata",
"items": [{
"key": "sshKeys",
"value": new_all_ssh_keys
}]
}
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
def does_image_exist(self, parameters):
""" Queries Google Compute Engine to see if the specified image exists for
this user.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
image that we should check for existence.
Returns:
True if the named image exists, and False otherwise.
"""
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.images().get(project=parameters[self.PARAM_PROJECT],
image=parameters[self.PARAM_IMAGE_ID])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
return True
except errors.HttpError:
return False
def sleep_until_update_operation_done(self, result, resource_name,
verbose):
""" Sleeps until the create/update operation for the resource is completed
successfully.
Args:
result: An instance, of the AzureOperationPoller to poll for the status
of the operation being performed.
resource_name: The name of the resource being updated.
"""
time_start = time.time()
while not result.done():
AppScaleLogger.verbose(
"Waiting {0} second(s) for {1} to be created/updated.".format(
self.SLEEP_TIME, resource_name), verbose)
time.sleep(self.SLEEP_TIME)
total_sleep_time = time.time() - time_start
if total_sleep_time > self.MAX_SLEEP_TIME:
AppScaleLogger.log(
"Waited {0} second(s) for {1} to be created/updated. "
"Operation has timed out.".format(total_sleep_time,
resource_name))
break
def sleep_until_delete_operation_done(self, result, resource_name,
max_sleep, verbose):
""" Sleeps until the delete operation for the resource is completed
successfully.
Args:
result: An instance, of the AzureOperationPoller to poll for the status
of the operation being performed.
resource_name: The name of the resource being deleted.
max_sleep: The maximum number of seconds to sleep for the resources to
be deleted.
verbose: A boolean indicating whether or not in verbose mode.
"""
time_start = time.time()
while not result.done():
AppScaleLogger.verbose("Waiting {0} second(s) for {1} to be deleted.".
format(self.SLEEP_TIME, resource_name), verbose)
time.sleep(self.SLEEP_TIME)
total_sleep_time = time.time() - time_start
if total_sleep_time > max_sleep:
AppScaleLogger.log("Waited {0} second(s) for {1} to be deleted. "
"Operation has timed out.".format(total_sleep_time, resource_name))
break
def describe_instances(self, parameters, pending=False):
""" Queries Google Compute Engine to see which instances are currently
running, and retrieve information about their public and private IPs.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
pending: Boolean if we should show pending instances.
Returns:
A tuple of the form (public_ips, private_ips, instance_ids), where each
member is a list. Items correspond to each other across these lists,
so a caller is guaranteed that item X in each list belongs to the same
virtual machine.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().list(
project=parameters[self.PARAM_PROJECT],
filter="name eq {group}-.*".format(group=parameters[self.PARAM_GROUP]),
zone=parameters[self.PARAM_ZONE]
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
instance_ids = []
public_ips = []
private_ips = []
if response and 'items' in response:
instances = response['items']
for instance in instances:
if instance['status'] == "RUNNING":
instance_ids.append(instance['name'])
network_interface = instance['networkInterfaces'][0]
public_ips.append(network_interface['accessConfigs'][0]['natIP'])
private_ips.append(network_interface['networkIP'])
return public_ips, private_ips, instance_ids
def delete_firewall(self, parameters):
""" Deletes a firewall in Google Compute Engine with the specified name.
Callers should not invoke this method until they are certain that no
instances are using the specified firewall, or this method will fail.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
firewall that we should create.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.firewalls().delete(
project=parameters[self.PARAM_PROJECT],
firewall=parameters[self.PARAM_GROUP]
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
def assert_credentials_are_valid(self, parameters):
"""Contacts GCE to see if the given credentials are valid.
Args:
parameters: A dict containing the credentials necessary to interact with
GCE.
Raises:
AgentConfigurationException: If an error is encountered during
authentication.
"""
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().list(project=parameters
[self.PARAM_PROJECT], zone=parameters[self.PARAM_ZONE])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
return True
except errors.HttpError as e:
error_message = json.loads(e.content)['error']['message']
raise AgentConfigurationException(error_message)
def does_disk_exist(self, parameters, disk):
""" Queries Google Compute Engine to see if the specified persistent disk
exists for this user.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
disk: A str containing the name of the disk that we should check for
existence.
Returns:
True if the named persistent disk exists, and False otherwise.
"""
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.disks().get(project=parameters[self.PARAM_PROJECT],
disk=disk, zone=parameters[self.PARAM_ZONE])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
return True
except errors.HttpError:
return False
def delete_access_config(self, parameters, instance_id):
""" Instructs Google Compute Engine to remove the public IP address from
the named instance.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key mapping to a list of
instance names that should be deleted.
instance_id: A str naming the running instance that the new public IP
address should be added to.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().deleteAccessConfig(
project=parameters[self.PARAM_PROJECT],
accessConfig="External NAT",
instance=instance_id,
networkInterface="nic0",
zone=parameters[self.PARAM_ZONE]
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
def create_scratch_disk(self, parameters):
""" Creates a disk from a given machine image.
GCE does not support scratch disks on API version v1 and higher. We create
a persistent disk upon creation to act like one to keep the abstraction used
in other infrastructures.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
Returns:
A str, the url to the disk to use.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
disk_name = self.generate_disk_name(parameters)
project_url = '{0}{1}'.format(self.GCE_URL,
parameters[self.PARAM_PROJECT])
source_image_url = '{0}{1}/global/images/{2}'.format(self.GCE_URL,
parameters[self.PARAM_PROJECT], parameters[self.PARAM_IMAGE_ID])
request = gce_service.disks().insert(
project=parameters[self.PARAM_PROJECT],
zone=parameters[self.PARAM_ZONE],
body={
'name':disk_name
},
sourceImage=source_image_url
)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
disk_url = "{0}/zones/{1}/disks/{2}".format(
project_url, parameters[self.PARAM_ZONE], disk_name)
return disk_url
def terminate_instances(self, parameters):
""" Deletes the instances specified in 'parameters' running in Azure.
Args:
parameters: A dict, containing all the parameters necessary to
authenticate this user with Azure.
"""
credentials = self.open_connection(parameters)
resource_group = parameters[self.PARAM_RESOURCE_GROUP]
subscription_id = parameters[self.PARAM_SUBSCRIBER_ID]
verbose = parameters[self.PARAM_VERBOSE]
public_ips, private_ips, instance_ids = self.describe_instances(parameters)
AppScaleLogger.verbose("Terminating the vm instance/s '{}'".
format(instance_ids), verbose)
compute_client = ComputeManagementClient(credentials, subscription_id)
threads = []
for vm_name in instance_ids:
thread = threading.Thread(target=self.delete_virtual_machine,
args=(compute_client, resource_group, verbose,
vm_name))
thread.start()
threads.append(thread)
for x in threads:
x.join()
def detach_disk(self, parameters, disk_name, instance_id):
""" Detaches the persistent disk specified in 'disk_name' from the named
instance.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
disk_name: A str naming the persistent disk to detach.
instance_id: A str naming the id of the instance that the disk should be
detached from.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
project_id = parameters[self.PARAM_PROJECT]
request = gce_service.instances().detachDisk(
project=project_id,
zone=parameters[self.PARAM_ZONE],
instance=instance_id,
deviceName='sdb')
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
def create_virtual_machine(self, credentials, network_client, network_id,
parameters, vm_network_name):
""" Creates an Azure virtual machine using the network interface created.
Args:
credentials: A ServicePrincipalCredentials instance, that can be used to
access or create any resources.
network_client: A NetworkManagementClient instance.
network_id: The network id of the network interface created.
parameters: A dict, containing all the parameters necessary to
authenticate this user with Azure.
vm_network_name: The name of the virtual machine to use.
"""
resource_group = parameters[self.PARAM_RESOURCE_GROUP]
storage_account = parameters[self.PARAM_STORAGE_ACCOUNT]
zone = parameters[self.PARAM_ZONE]
verbose = parameters[self.PARAM_VERBOSE]
AppScaleLogger.verbose(
"Creating a Virtual Machine '{}'".format(vm_network_name), verbose)
subscription_id = parameters[self.PARAM_SUBSCRIBER_ID]
azure_instance_type = parameters[self.PARAM_INSTANCE_TYPE]
compute_client = ComputeManagementClient(credentials, subscription_id)
keyname = parameters[self.PARAM_KEYNAME]
private_key_path = LocalState.LOCAL_APPSCALE_PATH + keyname
public_key_path = private_key_path + ".pub"
with open(public_key_path, 'r') as pub_ssh_key_fd:
pub_ssh_key = pub_ssh_key_fd.read()
key_path = "/home/{}/.ssh/authorized_keys".format(self.ADMIN_USERNAME)
public_keys = [SshPublicKey(path=key_path, key_data=pub_ssh_key)]
ssh_config = SshConfiguration(public_keys=public_keys)
linux_config = LinuxConfiguration(disable_password_authentication=True,
ssh=ssh_config)
os_profile = OSProfile(admin_username=self.ADMIN_USERNAME,
computer_name=vm_network_name,
linux_configuration=linux_config)
hardware_profile = HardwareProfile(vm_size=azure_instance_type)
network_profile = NetworkProfile(
network_interfaces=[NetworkInterfaceReference(id=network_id)])
virtual_hd = VirtualHardDisk(
uri='https://{0}.blob.core.windows.net/vhds/{1}.vhd'.format(
storage_account, vm_network_name))
image_hd = VirtualHardDisk(uri=parameters[self.PARAM_IMAGE_ID])
os_type = OperatingSystemTypes.linux
os_disk = OSDisk(os_type=os_type,
caching=CachingTypes.read_write,
create_option=DiskCreateOptionTypes.from_image,
name=vm_network_name,
vhd=virtual_hd,
image=image_hd)
compute_client.virtual_machines.create_or_update(
resource_group, vm_network_name,
VirtualMachine(location=zone,
os_profile=os_profile,
hardware_profile=hardware_profile,
network_profile=network_profile,
storage_profile=StorageProfile(os_disk=os_disk)))
# Sleep until an IP address gets associated with the VM.
while True:
public_ip_address = network_client.public_ip_addresses.get(
resource_group, vm_network_name)
if public_ip_address.ip_address:
AppScaleLogger.log('Azure VM is available at {}'.format(
public_ip_address.ip_address))
break
AppScaleLogger.verbose(
"Waiting {} second(s) for IP address to be "
"available".format(self.SLEEP_TIME), verbose)
time.sleep(self.SLEEP_TIME)
def create_virtual_machine(self, credentials, network_client, network_id,
parameters, vm_network_name):
""" Creates an Azure virtual machine using the network interface created.
Args:
credentials: A ServicePrincipalCredentials instance, that can be used to
access or create any resources.
network_client: A NetworkManagementClient instance.
network_id: The network id of the network interface created.
parameters: A dict, containing all the parameters necessary to
authenticate this user with Azure.
vm_network_name: The name of the virtual machine to use.
"""
resource_group = parameters[self.PARAM_RESOURCE_GROUP]
storage_account = parameters[self.PARAM_STORAGE_ACCOUNT]
zone = parameters[self.PARAM_ZONE]
verbose = parameters[self.PARAM_VERBOSE]
AppScaleLogger.verbose("Creating a Virtual Machine '{}'".
format(vm_network_name), verbose)
subscription_id = parameters[self.PARAM_SUBSCRIBER_ID]
azure_instance_type = parameters[self.PARAM_INSTANCE_TYPE]
compute_client = ComputeManagementClient(credentials, subscription_id)
keyname = parameters[self.PARAM_KEYNAME]
private_key_path = LocalState.LOCAL_APPSCALE_PATH + keyname
public_key_path = private_key_path + ".pub"
with open(public_key_path, 'r') as pub_ssh_key_fd:
pub_ssh_key = pub_ssh_key_fd.read()
key_path = "/home/{}/.ssh/authorized_keys".format(self.ADMIN_USERNAME)
public_keys = [SshPublicKey(path=key_path, key_data=pub_ssh_key)]
ssh_config = SshConfiguration(public_keys=public_keys)
linux_config = LinuxConfiguration(disable_password_authentication=True,
ssh=ssh_config)
os_profile = OSProfile(admin_username=self.ADMIN_USERNAME,
computer_name=vm_network_name,
linux_configuration=linux_config)
hardware_profile = HardwareProfile(vm_size=azure_instance_type)
network_profile = NetworkProfile(
network_interfaces=[NetworkInterfaceReference(id=network_id)])
virtual_hd = VirtualHardDisk(
uri='https://{0}.blob.core.windows.net/vhds/{1}.vhd'.
format(storage_account, vm_network_name))
image_hd = VirtualHardDisk(uri=parameters[self.PARAM_IMAGE_ID])
os_type = OperatingSystemTypes.linux
os_disk = OSDisk(os_type=os_type, caching=CachingTypes.read_write,
create_option=DiskCreateOptionTypes.from_image,
name=vm_network_name, vhd=virtual_hd, image=image_hd)
compute_client.virtual_machines.create_or_update(
resource_group, vm_network_name, VirtualMachine(location=zone,
os_profile=os_profile,
hardware_profile=hardware_profile,
network_profile=network_profile,
storage_profile=StorageProfile(
os_disk=os_disk)))
# Sleep until an IP address gets associated with the VM.
while True:
public_ip_address = network_client.public_ip_addresses.get(resource_group,
vm_network_name)
if public_ip_address.ip_address:
AppScaleLogger.log('Azure VM is available at {}'.
format(public_ip_address.ip_address))
break
AppScaleLogger.verbose("Waiting {} second(s) for IP address to be "
"available".format(self.SLEEP_TIME), verbose)
time.sleep(self.SLEEP_TIME)
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
location = os.path.abspath(options.location)
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {}, already exists.".format(location))
load_balancer_ip = LocalState.get_host_with_role(
options.keyname, 'load_balancer')
secret = LocalState.get_secret_key(options.keyname)
acc = AppControllerClient(load_balancer_ip, secret)
try:
all_ips = acc.get_all_public_ips()
except socket.error: # Occurs when the AppController has failed.
AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \
"machines in this AppScale deployment. Using our locally cached " + \
"info instead.")
all_ips = LocalState.get_all_public_ips(options.keyname)
# Get information about roles and public IPs
# for creating navigation symlinks in gathered logs
try:
nodes_info = acc.get_role_info()
except socket.error: # Occurs when the AppController has failed.
AppScaleLogger.warn("Couldn't get an up-to-date nodes info. "
"Using our locally cached info instead.")
nodes_info = LocalState.get_local_nodes_info(options.keyname)
nodes_dict = {node['public_ip']: node for node in nodes_info}
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(location)
# make dir for private IP navigation links
private_ips_dir = os.path.join(location, 'symlinks', 'private-ips')
utils.mkdir(private_ips_dir)
# The log paths that we collect logs from.
log_paths = [
{'remote': '/opt/cassandra/cassandra/logs/*', 'local': 'cassandra'},
{'remote': '/var/log/appscale'},
{'remote': '/var/log/haproxy.log*'},
{'remote': '/var/log/kern.log*'},
{'remote': '/var/log/nginx'},
{'remote': '/var/log/rabbitmq/*', 'local': 'rabbitmq'},
{'remote': '/var/log/syslog*'},
{'remote': '/var/log/zookeeper'}
]
failures = False
for public_ip in all_ips:
# Get the logs from each node, and store them in our local directory
local_dir = os.path.join(location, public_ip)
utils.mkdir(local_dir)
local_link = os.path.join('..', '..', public_ip)
# Create symlinks for easier navigation in gathered logs
node_info = nodes_dict.get(public_ip)
if node_info:
private_ip_dir = os.path.join(private_ips_dir, node_info["private_ip"])
os.symlink(local_link, private_ip_dir)
for role in node_info['roles']:
role_dir = os.path.join(location, 'symlinks', role)
utils.mkdir(role_dir)
os.symlink(local_link, os.path.join(role_dir, public_ip))
for log_path in log_paths:
sub_dir = local_dir
if 'local' in log_path:
sub_dir = os.path.join(local_dir, log_path['local'])
utils.mkdir(sub_dir)
try:
RemoteHelper.scp_remote_to_local(
public_ip, options.keyname, log_path['remote'], sub_dir
)
except ShellException as shell_exception:
failures = True
AppScaleLogger.warn('Unable to collect logs from {} for host {}'.
format(log_path['remote'], public_ip))
AppScaleLogger.verbose(
'Encountered exception: {}'.format(str(shell_exception)))
if failures:
AppScaleLogger.log("Done copying to {}. There were failures while "
"collecting AppScale logs.".format(location))
else:
AppScaleLogger.success("Successfully collected all AppScale logs into "
"{}".format(location))
def run_instances(self, count, parameters, security_configured):
""" Starts 'count' instances in Google Compute Engine, and returns once they
have been started.
Callers should create a network and attach a firewall to it before using
this method, or the newly created instances will not have a network and
firewall to attach to (and thus this method will fail).
Args:
count: An int that specifies how many virtual machines should be started.
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
security_configured: Unused, as we assume that the network and firewall
has already been set up.
"""
project_id = parameters[self.PARAM_PROJECT]
image_id = parameters[self.PARAM_IMAGE_ID]
instance_type = parameters[self.PARAM_INSTANCE_TYPE]
keyname = parameters[self.PARAM_KEYNAME]
group = parameters[self.PARAM_GROUP]
zone = parameters[self.PARAM_ZONE]
AppScaleLogger.log("Starting {0} machines with machine id {1}, with " \
"instance type {2}, keyname {3}, in security group {4}, in zone {5}" \
.format(count, image_id, instance_type, keyname, group, zone))
# First, see how many instances are running and what their info is.
start_time = datetime.datetime.now()
active_public_ips, active_private_ips, active_instances = \
self.describe_instances(parameters)
# Construct URLs
image_url = '{0}{1}/global/images/{2}'.format(self.GCE_URL, project_id,
image_id)
project_url = '{0}{1}'.format(self.GCE_URL, project_id)
machine_type_url = '{0}/zones/{1}/machineTypes/{2}'.format(project_url,
zone, instance_type)
network_url = '{0}/global/networks/{1}'.format(project_url, group)
# Construct the request body
for index in range(count):
disk_url = self.create_scratch_disk(parameters)
instances = {
# Truncate the name down to the first 62 characters, since GCE doesn't
# let us use arbitrarily long instance names.
'name': '{group}-{uuid}'.format(group=group, uuid=uuid.uuid4())[:62],
'machineType': machine_type_url,
'disks':[{
'source': disk_url,
'boot': 'true',
'type': 'PERSISTENT'
}],
'image': image_url,
'networkInterfaces': [{
'accessConfigs': [{
'type': 'ONE_TO_ONE_NAT',
'name': 'External NAT'
}],
'network': network_url
}],
'serviceAccounts': [{
'email': self.DEFAULT_SERVICE_EMAIL,
'scopes': [self.GCE_SCOPE]
}]
}
# Create the instance
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().insert(
project=project_id, body=instances, zone=zone)
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
instance_ids = []
public_ips = []
private_ips = []
end_time = datetime.datetime.now() + datetime.timedelta(0,
self.MAX_VM_CREATION_TIME)
now = datetime.datetime.now()
while now < end_time:
AppScaleLogger.log("Waiting for your instances to start...")
instance_info = self.describe_instances(parameters)
public_ips = instance_info[0]
private_ips = instance_info[1]
instance_ids = instance_info[2]
public_ips = self.diff(public_ips, active_public_ips)
private_ips = self.diff(private_ips, active_private_ips)
instance_ids = self.diff(instance_ids, active_instances)
if count == len(public_ips):
break
time.sleep(self.SLEEP_TIME)
now = datetime.datetime.now()
if not public_ips:
self.handle_failure('No public IPs were able to be procured '
'within the time limit')
if len(public_ips) != count:
for index in range(0, len(public_ips)):
if public_ips[index] == '0.0.0.0':
instance_to_term = instance_ids[index]
AppScaleLogger.log('Instance {0} failed to get a public IP address'\
'and is being terminated'.format(instance_to_term))
self.terminate_instances([instance_to_term])
end_time = datetime.datetime.now()
total_time = end_time - start_time
AppScaleLogger.log("Started {0} on-demand instances in {1} seconds" \
.format(count, total_time.seconds))
return instance_ids, public_ips, private_ips
def run_instances(cls, options):
"""Starts a new AppScale deployment with the parameters given.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Raises:
AppControllerException: If the AppController on the head node crashes.
When this occurs, the message in the exception contains the reason why
the AppController crashed.
BadConfigurationException: If the user passes in options that are not
sufficient to start an AppScale deployment (e.g., running on EC2 but
not specifying the AMI to use), or if the user provides us
contradictory options (e.g., running on EC2 but not specifying EC2
credentials).
"""
LocalState.make_appscale_directory()
LocalState.ensure_appscale_isnt_running(options.keyname, options.force)
node_layout = NodeLayout(options)
if options.infrastructure:
if (not options.test and not options.force and
not (options.disks or node_layout.are_disks_used())):
LocalState.ensure_user_wants_to_run_without_disks()
reduced_version = '.'.join(x for x in APPSCALE_VERSION.split('.')[:2])
AppScaleLogger.log("Starting AppScale " + reduced_version)
my_id = str(uuid.uuid4())
AppScaleLogger.remote_log_tools_state(options, my_id, "started",
APPSCALE_VERSION)
head_node = node_layout.head_node()
# Start VMs in cloud via cloud agent.
if options.infrastructure:
node_layout = RemoteHelper.start_all_nodes(options, node_layout)
# Enables root logins and SSH access on the head node.
RemoteHelper.enable_root_ssh(options, head_node.public_ip)
AppScaleLogger.verbose("Node Layout: {}".format(node_layout.to_list()))
# Ensure all nodes are compatible.
RemoteHelper.ensure_machine_is_compatible(
head_node.public_ip, options.keyname)
# Use rsync to move custom code into the deployment.
if options.rsync_source:
AppScaleLogger.log("Copying over local copy of AppScale from {0}".
format(options.rsync_source))
RemoteHelper.rsync_files(head_node.public_ip, options.keyname,
options.rsync_source)
# Start services on head node.
RemoteHelper.start_head_node(options, my_id, node_layout)
# Write deployment metadata to disk (facilitates SSH operations, etc.)
db_master = node_layout.db_master().private_ip
head_node = node_layout.head_node().public_ip
LocalState.update_local_metadata(options, db_master, head_node)
# Copy the locations.json to the head node
RemoteHelper.copy_local_metadata(node_layout.head_node().public_ip,
options.keyname)
# Wait for services on head node to start.
secret_key = LocalState.get_secret_key(options.keyname)
acc = AppControllerClient(head_node, secret_key)
try:
while not acc.is_initialized():
AppScaleLogger.log('Waiting for head node to initialize...')
# This can take some time in particular the first time around, since
# we will have to initialize the database.
time.sleep(cls.SLEEP_TIME*3)
except socket.error as socket_error:
AppScaleLogger.warn('Unable to initialize AppController: {}'.
format(socket_error.message))
message = RemoteHelper.collect_appcontroller_crashlog(
head_node, options.keyname)
raise AppControllerException(message)
# Set up admin account.
try:
# We don't need to have any exception information here: we do expect
# some anyway while the UserAppServer is coming up.
acc.does_user_exist("non-existent-user", True)
except Exception:
AppScaleLogger.log('UserAppServer not ready yet. Retrying ...')
time.sleep(cls.SLEEP_TIME)
if options.admin_user and options.admin_pass:
AppScaleLogger.log("Using the provided admin username/password")
username, password = options.admin_user, options.admin_pass
elif options.test:
AppScaleLogger.log("Using default admin username/password")
username, password = LocalState.DEFAULT_USER, LocalState.DEFAULT_PASSWORD
else:
username, password = LocalState.get_credentials()
RemoteHelper.create_user_accounts(username, password, head_node,
options.keyname)
acc.set_admin_role(username, 'true', cls.ADMIN_CAPABILITIES)
# Wait for machines to finish loading and AppScale Dashboard to be deployed.
RemoteHelper.wait_for_machines_to_finish_loading(head_node, options.keyname)
try:
login_host = acc.get_property('login')['login']
except KeyError:
raise AppControllerException('login property not found')
RemoteHelper.sleep_until_port_is_open(
login_host, RemoteHelper.APP_DASHBOARD_PORT)
AppScaleLogger.success("AppScale successfully started!")
AppScaleLogger.success(
'View status information about your AppScale deployment at '
'http://{}:{}'.format(login_host, RemoteHelper.APP_DASHBOARD_PORT))
AppScaleLogger.remote_log_tools_state(options, my_id,
"finished", APPSCALE_VERSION)
def terminate_instances(cls, options):
"""Stops all services running in an AppScale deployment, and in cloud
deployments, also powers off the instances previously spawned.
Raises:
AppScaleException: If AppScale is not running, and thus can't be
terminated.
"""
try:
infrastructure = LocalState.get_infrastructure(options.keyname)
except IOError:
raise AppScaleException("Cannot find AppScale's configuration for keyname {0}".
format(options.keyname))
if infrastructure == "xen" and options.terminate:
raise AppScaleException("Terminate option is invalid for cluster mode.")
if infrastructure == "xen" or not options.terminate:
# We are in cluster mode: let's check if AppScale is running.
if not os.path.exists(LocalState.get_secret_key_location(options.keyname)):
raise AppScaleException("AppScale is not running with the keyname {0}".
format(options.keyname))
# Stop gracefully the AppScale deployment.
try:
RemoteHelper.terminate_virtualized_cluster(options.keyname,
options.clean)
except (IOError, AppScaleException, AppControllerException,
BadConfigurationException) as e:
if not (infrastructure in InfrastructureAgentFactory.VALID_AGENTS and
options.terminate):
raise
if options.test:
AppScaleLogger.warn(e)
else:
AppScaleLogger.verbose(e)
if isinstance(e, AppControllerException):
response = raw_input(
'AppScale may not have shut down properly, are you sure you want '
'to continue terminating? (y/N) ')
else:
response = raw_input(
'AppScale could not find the configuration files for this '
'deployment, are you sure you want to continue terminating? '
'(y/N) ')
if response.lower() not in ['y', 'yes']:
raise AppScaleException("Cancelled cloud termination.")
# And if we are on a cloud infrastructure, terminate instances if
# asked.
if (infrastructure in InfrastructureAgentFactory.VALID_AGENTS and
options.terminate):
RemoteHelper.terminate_cloud_infrastructure(options.keyname)
elif infrastructure in InfrastructureAgentFactory.VALID_AGENTS and not \
options.terminate:
AppScaleLogger.log("AppScale did not terminate any of your cloud "
"instances, to terminate them run 'appscale "
"down --terminate'")
if options.clean:
LocalState.clean_local_metadata(keyname=options.keyname)
