def assert_required_parameters(self, parameters, _):
""" Checks the given parameters to make sure that they can be used to
interact with Google Compute Engine.
Args:
parameters: A dict that maps the name of each credential to be used in GCE
with the value we should use.
operation: A BaseAgent.OPERATION that indicates if we wish to add or
delete instances. Unused here, as all operations require the same
credentials.
Raises:
AgentConfigurationException: If any of the required credentials are not
present, or if the client_secrets parameter refers to a file that is not
present on the local filesystem.
"""
# Make sure the user has set each parameter.
for param in self.REQUIRED_CREDENTIALS:
if not self.has_parameter(param, parameters):
raise AgentConfigurationException('The required parameter, {0}, was' \
' not specified.'.format(param))
# Next, make sure that either the client_secrets file or the oauth2
# credentials file exists.
credentials_file = parameters.get(
self.PARAM_SECRETS) or parameters.get(self.PARAM_STORAGE)
if not os.path.exists(os.path.expanduser(credentials_file)):
raise AgentConfigurationException('Could not find your credentials ' \
'file at {0}'.format(credentials_file))
def assert_credentials_are_valid(self, parameters):
""" Contacts Azure with the given credentials to ensure that they are
valid. Gets an access token and a Credentials instance in order to be
able to access any resources.
Args:
parameters: A dict, containing all the parameters necessary to
authenticate this user with Azure.
Returns:
True, if the credentials were valid.
A list, of resource group names under the subscription.
Raises:
AgentConfigurationException: If an error is encountered during
authentication.
"""
credentials = self.open_connection(parameters)
subscription_id = parameters[self.PARAM_SUBSCRIBER_ID]
try:
resource_client = ResourceManagementClient(credentials,
subscription_id)
resource_groups = resource_client.resource_groups.list()
rg_names = []
for rg in resource_groups:
rg_names.append(rg.name)
return True, rg_names
except CloudError as error:
raise AgentConfigurationException(
"Unable to authenticate using the "
"credentials provided. Reason: {}".format(error.message))
def create_storage_account(self, parameters, storage_client):
""" Creates a Storage Account under the Resource Group, if it does not
already exist. In the case where no resource group is specified, a default
storage account is created.
Args:
parameters: A dict, containing all the parameters necessary to authenticate
this user with Azure.
credentials: A ServicePrincipalCredentials instance, that can be used to access or
create any resources.
Raises:
AgentConfigurationException: If there was a problem creating or accessing
a storage account with the given subscription.
"""
storage_account = parameters[self.PARAM_STORAGE_ACCOUNT]
rg_name = parameters[self.PARAM_RESOURCE_GROUP]
try:
AppScaleLogger.log(
"Creating a new storage account '{0}' under the "
"resource group '{1}'.".format(storage_account, rg_name))
result = storage_client.storage_accounts.create(
rg_name, storage_account,
StorageAccountCreateParameters(
sku=Sku(SkuName.standard_lrs),
kind=Kind.storage,
location=parameters[self.PARAM_ZONE]))
# Result is a msrestazure.azure_operation.AzureOperationPoller instance.
# wait() insures polling the underlying async operation until it's done.
result.wait()
except CloudError as error:
raise AgentConfigurationException(
"Unable to create a storage account "
"using the credentials provided: {}".format(error.message))
def get_params_from_yaml(self, keyname):
"""Searches through the locations.yaml file to build a dict containing the
parameters necessary to interact with Amazon EC2.
Args:
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
"""
params = {
self.PARAM_CREDENTIALS : {},
self.PARAM_GROUP : LocalState.get_group(keyname),
self.PARAM_KEYNAME : keyname
}
zone = LocalState.get_zone(keyname)
if zone:
params[self.PARAM_REGION] = zone[:-1]
else:
params[self.PARAM_REGION] = self.DEFAULT_REGION
for credential in self.REQUIRED_CREDENTIALS:
if os.environ.get(credential):
params[self.PARAM_CREDENTIALS][credential] = os.environ[credential]
else:
raise AgentConfigurationException("no " + credential)
return params
def get_cloud_params(self, keyname):
"""Searches through the locations.json file with key
'infrastructure_info' to build a dict containing the
parameters necessary to interact with Amazon EC2.
Args:
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
"""
params = {
self.PARAM_CREDENTIALS: {},
self.PARAM_GROUP: LocalState.get_group(keyname),
self.PARAM_KEYNAME: keyname
}
zone = LocalState.get_zone(keyname)
if zone:
params[self.PARAM_REGION] = zone[:-1]
else:
params[self.PARAM_REGION] = self.DEFAULT_REGION
for credential in self.REQUIRED_CREDENTIALS:
cred = LocalState.get_infrastructure_option(tag=credential,
keyname=keyname)
if not cred:
raise AgentConfigurationException("no " + credential)
params[self.PARAM_CREDENTIALS][credential] = cred
return params
def assert_credentials_are_valid(self, parameters):
"""Contacts GCE to see if the given credentials are valid.
Args:
parameters: A dict containing the credentials necessary to interact with
GCE.
Raises:
AgentConfigurationException: If an error is encountered during
authentication.
"""
gce_service, credentials = self.open_connection(parameters)
try:
http = httplib2.Http()
auth_http = credentials.authorize(http)
request = gce_service.instances().list(
project=parameters[self.PARAM_PROJECT],
zone=parameters[self.PARAM_ZONE])
response = request.execute(http=auth_http)
AppScaleLogger.verbose(str(response),
parameters[self.PARAM_VERBOSE])
return True
except errors.HttpError as e:
error_message = json.loads(e.content)['error']['message']
raise AgentConfigurationException(error_message)
def create_resource_group(self, parameters, credentials):
""" Creates a Resource Group for the application using the Service Principal
Credentials, if it does not already exist. In the case where no resource
group is specified, a default group is created.
Args:
parameters: A dict, containing all the parameters necessary to
authenticate this user with Azure.
credentials: A ServicePrincipalCredentials instance, that can be used to
access or create any resources.
Raises:
AgentConfigurationException: If there was a problem creating or accessing
a resource group with the given subscription.
"""
subscription_id = parameters[self.PARAM_SUBSCRIBER_ID]
resource_client = ResourceManagementClient(credentials,
subscription_id)
rg_name = parameters[self.PARAM_RESOURCE_GROUP]
tag_name = 'default-tag'
if parameters[self.PARAM_TAG]:
tag_name = parameters[self.PARAM_TAG]
storage_client = StorageManagementClient(credentials, subscription_id)
resource_client.providers.register(self.MICROSOFT_STORAGE_RESOURCE)
try:
# If the resource group does not already exist, create a new one with the
# specified storage account.
if not parameters[self.PARAM_EXISTING_RG]:
AppScaleLogger.log(
"Creating a new resource group '{0}' with the tag "
"'{1}'.".format(rg_name, tag_name))
resource_client.resource_groups.create_or_update(
rg_name,
ResourceGroup(location=parameters[self.PARAM_ZONE],
tags={'tag': tag_name}))
self.create_storage_account(parameters, storage_client)
else:
# If it already exists, check if the specified storage account exists
# under it and if not, create a new account.
storage_accounts = storage_client.storage_accounts.\
list_by_resource_group(rg_name)
acct_names = []
for account in storage_accounts:
acct_names.append(account.name)
if parameters[self.PARAM_STORAGE_ACCOUNT] in acct_names:
AppScaleLogger.log(
"Storage account '{0}' under '{1}' resource group "
"already exists. So not creating it again.".format(
parameters[self.PARAM_STORAGE_ACCOUNT], rg_name))
else:
self.create_storage_account(parameters, storage_client)
except CloudError as error:
raise AgentConfigurationException(
"Unable to create a resource group "
"using the credentials provided: {}".format(error.message))
def get_params_from_args(self, args):
"""
Searches through args to build a dict containing the parameters
necessary to interact with Amazon EC2.
Args:
args: A Namespace containing the arguments that the user has
invoked an AppScale Tool with.
"""
# need to convert this to a dict if it is not already
if not isinstance(args, dict):
args = vars(args)
params = {
self.PARAM_CREDENTIALS: {},
self.PARAM_GROUP: args['group'],
self.PARAM_IMAGE_ID: args['machine'],
self.PARAM_INSTANCE_TYPE: args['instance_type'],
self.PARAM_KEYNAME: args['keyname'],
self.PARAM_STATIC_IP: args.get(self.PARAM_STATIC_IP),
self.PARAM_ZONE: args.get('zone'),
'IS_VERBOSE': args.get('verbose', False),
'autoscale_agent': False
}
if params[self.PARAM_ZONE]:
params[self.PARAM_REGION] = params[self.PARAM_ZONE][:-1]
else:
params[self.PARAM_REGION] = self.DEFAULT_REGION
for credential in self.REQUIRED_CREDENTIALS:
if os.environ.get(credential):
params[self.
PARAM_CREDENTIALS][credential] = os.environ[credential]
else:
raise AgentConfigurationException("Couldn't find {0} in your " \
"environment. Please set it and run AppScale again."
.format(credential))
self.assert_credentials_are_valid(params)
if args.get('use_spot_instances') == True:
params[self.PARAM_SPOT] = True
else:
params[self.PARAM_SPOT] = False
if params[self.PARAM_SPOT]:
if args.get('max_spot_price'):
params[self.PARAM_SPOT_PRICE] = args['max_spot_price']
else:
params[self.PARAM_SPOT_PRICE] = self.get_optimal_spot_price(
self.open_connection(params),
params[self.PARAM_INSTANCE_TYPE], params[self.PARAM_ZONE])
return params
def assert_required_parameters(self, parameters, _):
""" Checks the given parameters to make sure that they can be used to
interact with Google Compute Engine.
Args:
parameters: A dict that maps the name of each credential to be used in GCE
with the value we should use.
operation: A BaseAgent.OPERATION that indicates if we wish to add or
delete instances. Unused here, as all operations require the same
credentials.
Raises:
AgentConfigurationException: If any of the required credentials are not
present, or if the client_secrets parameter refers to a file that is not
present on the local filesystem.
"""
# Make sure the user has set each parameter.
for param in self.REQUIRED_CREDENTIALS:
if not self.has_parameter(param, parameters):
raise AgentConfigurationException('The required parameter, {0}, was' \
' not specified.'.format(param))
# Next, make sure that either the client_secrets file or the oauth2
# credentials file exists.
credentials_file = parameters.get(self.PARAM_SECRETS) or parameters.get(
self.PARAM_STORAGE)
if not os.path.exists(os.path.expanduser(credentials_file)):
raise AgentConfigurationException('Could not find your credentials ' \
'file at {0}'.format(credentials_file))
# TODO: Remove this warning once service accounts have been fully tested.
secrets_location = os.path.expanduser(parameters[self.PARAM_SECRETS])
secrets_type = GCEAgent.get_secrets_type(secrets_location)
if (secrets_type == CredentialTypes.SERVICE and
not parameters[self.PARAM_TEST]):
response = raw_input('It looks like you are using service account '
'credentials, which are not currently supported for cloud '
'autoscaling.\nWould you like to continue? (y/N)')
if response.lower() not in ['y', 'yes']:
raise AgentConfigurationException('User cancelled starting AppScale.')
def assert_required_parameters(self, parameters, operation):
required_params = ()
if operation == BaseAgent.OPERATION_PREPARE:
required_params = self.REQUIRED_FLEX_PREPARE_INSTANCES_PARAMS
elif operation == BaseAgent.OPERATION_DEREGISTER:
required_params = self.REQUIRED_FLEX_DEREGISTER_INSTANCES_PARAMS
logging.debug('required_params: {0}'.format(required_params))
for param in required_params:
logging.debug('param: {0}'.format(param))
if not utils.has_parameter(param, parameters):
raise AgentConfigurationException('no ' + param)
def get_params_from_args(self, args):
""" Constructs a dict with only the parameters necessary to interact with
Microsoft Azure.
Args:
args: A Namespace or dict, that maps all of the arguments the user has
invoked an AppScale command with their associated value.
Returns:
A dict, that maps each argument given to the value that was associated with
it.
Raises:
AgentConfigurationException: If unable to authenticate using the credentials
provided in the AppScalefile.
"""
if not isinstance(args, dict):
args = vars(args)
params = {
self.PARAM_APP_ID: args[self.PARAM_APP_ID],
self.PARAM_APP_SECRET: args[self.PARAM_APP_SECRET],
self.PARAM_IMAGE_ID: args['machine'],
self.PARAM_INSTANCE_TYPE: args[self.PARAM_INSTANCE_TYPE],
self.PARAM_GROUP: args[self.PARAM_GROUP],
self.PARAM_KEYNAME: args[self.PARAM_KEYNAME],
self.PARAM_RESOURCE_GROUP: args[self.PARAM_RESOURCE_GROUP],
self.PARAM_STORAGE_ACCOUNT: args[self.PARAM_STORAGE_ACCOUNT],
self.PARAM_SUBSCRIBER_ID: args[self.PARAM_SUBSCRIBER_ID],
self.PARAM_TAG: args[self.PARAM_TAG],
self.PARAM_TENANT_ID: args[self.PARAM_TENANT_ID],
self.PARAM_TEST: args[self.PARAM_TEST],
self.PARAM_VERBOSE: args.get('verbose', False),
self.PARAM_ZONE: args[self.PARAM_ZONE]
}
is_valid, rg_names = self.assert_credentials_are_valid(params)
if not is_valid:
raise AgentConfigurationException(
"Unable to authenticate using the "
"credentials provided.")
# Check if the resource group passed in exists already, if it does, then
# pass an existing group flag so that it is not created again.
# In case no resource group is passed, pass a default group.
params[self.PARAM_EXISTING_RG] = False
if not args[self.PARAM_RESOURCE_GROUP]:
params[self.PARAM_RESOURCE_GROUP] = self.DEFAULT_RESOURCE_GROUP
if args[self.PARAM_RESOURCE_GROUP] in rg_names:
params[self.PARAM_EXISTING_RG] = True
if not args[self.PARAM_STORAGE_ACCOUNT]:
params[self.PARAM_STORAGE_ACCOUNT] = self.DEFAULT_STORAGE_ACCT
return params
def assert_required_parameters(self, parameters, operation):
"""
Assert that all the parameters required for the EC2 agent are in place.
(Also see documentation for the BaseAgent class)
Args:
parameters: A dictionary of parameters.
operation: Operations to be invoked using the above parameters.
"""
required_params = ()
if operation == BaseAgent.OPERATION_RUN:
required_params = self.REQUIRED_EC2_RUN_INSTANCES_PARAMS
elif operation == BaseAgent.OPERATION_TERMINATE:
required_params = self.REQUIRED_EC2_TERMINATE_INSTANCES_PARAMS
# make sure the user set something for each parameter
for param in required_params:
if not self.has_parameter(param, parameters):
raise AgentConfigurationException('no ' + param)
# next, make sure the user actually put in their credentials
for credential in self.REQUIRED_EC2_CREDENTIALS:
if not self.has_parameter(credential, parameters['credentials']):
raise AgentConfigurationException('no ' + credential)
def assert_credentials_are_valid(self, parameters):
"""Contacts AWS to see if the given access key and secret key represent a
valid set of credentials.
Args:
parameters: A dict containing the user's AWS access key and secret key.
Raises:
AgentConfigurationException: If the given AWS access key and secret key
cannot be used to make requests to AWS.
"""
conn = self.open_connection(parameters)
try:
conn.get_all_instances()
except EC2ResponseError:
raise AgentConfigurationException("We couldn't validate your EC2 " + \
"access key and EC2 secret key. Are your credentials valid?")
def assert_required_parameters(self, parameters, operation):
""" Check whether all the parameters required to interact with Azure are
present in the provided dict.
Args:
parameters: A dict containing values necessary to authenticate with the
Azure.
operation: A str representing the operation for which the parameters
should be checked.
Raises:
AgentConfigurationException: If a required parameter is absent.
"""
# Make sure that the user has set each parameter.
for param in self.REQUIRED_CREDENTIALS:
if not self.has_parameter(param, parameters):
raise AgentConfigurationException(
'The required parameter, {0}, was not'
' specified.'.format(param))
def assert_required_parameters(self, parameters, operation):
"""
Assert that all the parameters required for the EC2 agent are in place.
(Also see documentation for the BaseAgent class)
Args:
parameters A dictionary of parameters
operation Operations to be invoked using the above parameters
"""
required_params = ()
if operation == BaseAgent.OPERATION_PREPARE:
required_params = self.REQUIRED_EC2_PREPARE_INSTANCES_PARAMS
elif operation == BaseAgent.OPERATION_DEREGISTER:
required_params = self.REQUIRED_EC2_DEREGISTER_INSTANCES_PARAMS
for param in required_params:
if not utils.has_parameter(param, parameters):
raise AgentConfigurationException('no ' + param)
def get_params_from_args(self, args):
""" Constructs a dict with only the parameters necessary to interact with
Google Compute Engine (here, the client_secrets file and the image name).
Args:
args: A Namespace or dict that maps all of the arguments the user has
invoked an AppScale command with their associated value.
Returns:
A dict containing the location of the client_secrets file and that name
of the image to use in GCE.
Raises:
AgentConfigurationException: If the caller fails to specify a
client_secrets file, or if it doesn't exist on the local filesystem.
"""
if not isinstance(args, dict):
args = vars(args)
if not args.get('client_secrets') and not args.get('oauth2_storage'):
raise AgentConfigurationException("Please specify a client_secrets " + \
"file or a oauth2_storage file in your AppScalefile when running " + \
"over Google Compute Engine.")
credentials_file = args.get('client_secrets') or args.get('oauth2_storage')
full_credentials = os.path.expanduser(credentials_file)
if not os.path.exists(full_credentials):
raise AgentConfigurationException("Couldn't find your credentials " + \
"at {0}".format(full_credentials))
if args.get('client_secrets'):
destination = LocalState.get_client_secrets_location(args['keyname'])
# Make sure the destination's parent directory exists.
destination_par = os.path.abspath(os.path.join(destination, os.pardir))
if not os.path.exists(destination_par):
os.makedirs(destination_par)
shutil.copy(full_credentials, destination)
elif args.get('oauth2_storage'):
destination = LocalState.get_oauth2_storage_location(args['keyname'])
# Make sure the destination's parent directory exists.
destination_par = os.path.abspath(os.path.join(destination, os.pardir))
if not os.path.exists(destination_par):
os.makedirs(destination_par)
shutil.copy(full_credentials, destination)
params = {
self.PARAM_GROUP : args['group'],
self.PARAM_IMAGE_ID : args['machine'],
self.PARAM_INSTANCE_TYPE : args[self.PARAM_INSTANCE_TYPE],
self.PARAM_KEYNAME : args['keyname'],
self.PARAM_PROJECT : args['project'],
self.PARAM_STATIC_IP : args.get(self.PARAM_STATIC_IP),
self.PARAM_ZONE : args['zone'],
self.PARAM_TEST: args['test'],
}
# A zone in GCE looks like 'us-central2-a', which is in the region
# 'us-central2'. Therefore, strip off the last two characters from the zone
# to get the region name.
if params[self.PARAM_ZONE]:
params[self.PARAM_REGION] = params[self.PARAM_ZONE][:-2]
else:
params[self.PARAM_REGION] = self.DEFAULT_REGION
if args.get(self.PARAM_SECRETS):
params[self.PARAM_SECRETS] = args.get(self.PARAM_SECRETS)
elif args.get(self.PARAM_STORAGE):
params[self.PARAM_STORAGE] = args.get(self.PARAM_STORAGE)
params[self.PARAM_VERBOSE] = args.get('verbose', False)
self.assert_credentials_are_valid(params)
return params
def get_params_from_args(self, args):
"""
Searches through args to build a dict containing the parameters
necessary to interact with Amazon EC2.
Args:
args: A Namespace containing the arguments that the user has
invoked an AppScale Tool with.
"""
# need to convert this to a dict if it is not already
if not isinstance(args, dict):
args = vars(args)
params = {
self.PARAM_CREDENTIALS: {},
self.PARAM_GROUP: args['group'],
self.PARAM_IMAGE_ID: args['machine'],
self.PARAM_INSTANCE_TYPE: args['instance_type'],
self.PARAM_KEYNAME: args['keyname'],
self.PARAM_STATIC_IP: args.get(self.PARAM_STATIC_IP),
self.PARAM_ZONE: args.get('zone'),
self.PARAM_VERBOSE: args.get('verbose', False),
self.PARAM_AUTOSCALE_AGENT: False
}
if params[self.PARAM_ZONE]:
params[self.PARAM_REGION] = params[self.PARAM_ZONE][:-1]
else:
params[self.PARAM_REGION] = self.DEFAULT_REGION
for credential in self.REQUIRED_CREDENTIALS:
if args.get(credential):
params[self.PARAM_CREDENTIALS][credential] = args[credential]
else:
raise AgentConfigurationException("Couldn't find {0} in your " \
"environment. Please set it and run AppScale again."
.format(credential))
self.assert_credentials_are_valid(params)
if args.get('use_spot_instances') == True:
params[self.PARAM_SPOT] = True
else:
params[self.PARAM_SPOT] = False
if params[self.PARAM_SPOT]:
if args.get('max_spot_price'):
params[self.PARAM_SPOT_PRICE] = args['max_spot_price']
else:
params[self.PARAM_SPOT_PRICE] = self.get_optimal_spot_price(
self.open_connection(params),
params[self.PARAM_INSTANCE_TYPE], params[self.PARAM_ZONE])
# If VPC id and Subnet id are not set assume classic networking should be
# used.
vpc_id = args.get(self.PARAM_VPC_ID)
subnet_id = args.get(self.PARAM_SUBNET_ID)
if not vpc_id and not subnet_id:
AppScaleLogger.log(
'Using Classic Networking since subnet and vpc were '
'not specified.')
# All further checks are for VPC Networking.
elif (vpc_id or subnet_id) and not (vpc_id and subnet_id):
raise AgentConfigurationException(
'Both VPC id and Subnet id must be '
'specified to use VPC Networking.')
else:
# VPC must exist.
vpc_conn = self.open_vpc_connection(params)
params[self.PARAM_VPC_ID] = args[self.PARAM_VPC_ID]
try:
vpc_conn.get_all_vpcs(params[self.PARAM_VPC_ID])
except EC2ResponseError as e:
raise AgentConfigurationException(
'Error looking for vpc: {}'.format(e.message))
# Subnet must exist.
all_subnets = vpc_conn.get_all_subnets(
filters={'vpcId': params[self.PARAM_VPC_ID]})
params[self.PARAM_SUBNET_ID] = args[self.PARAM_SUBNET_ID]
if not any(subnet.id == params[self.PARAM_SUBNET_ID]
for subnet in all_subnets):
raise AgentConfigurationException(
'Specified subnet {} does not exist '
'in vpc {}!'.format(params[self.PARAM_SUBNET_ID],
params[self.PARAM_VPC_ID]))
return params