def get_params_from_yaml(self, keyname):
"""Searches through the locations.yaml file to build a dict containing the
parameters necessary to interact with Amazon EC2.
Args:
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
"""
params = {
self.PARAM_CREDENTIALS : {},
self.PARAM_GROUP : LocalState.get_group(keyname),
self.PARAM_KEYNAME : keyname
}
zone = LocalState.get_zone(keyname)
if zone:
params[self.PARAM_REGION] = zone[:-1]
else:
params[self.PARAM_REGION] = self.DEFAULT_REGION
for credential in self.REQUIRED_CREDENTIALS:
if os.environ.get(credential):
params[self.PARAM_CREDENTIALS][credential] = os.environ[credential]
else:
raise AgentConfigurationException("no " + credential)
return params
def open_connection(self, parameters):
""" Connects to Google Compute Engine with the given credentials.
Args:
parameters: A dict that contains all the parameters necessary to
authenticate this user with Google Compute Engine. We assume that the
user has already authorized this account for use with GCE.
Returns:
An apiclient.discovery.Resource that is a connection valid for requests
to Google Compute Engine for the given user, and a Credentials object that
can be used to sign requests performed with that connection.
"""
# Perform OAuth 2.0 authorization.
flow = None
if self.PARAM_SECRETS in parameters:
flow = oauth2client.client.flow_from_clientsecrets(
os.path.expanduser(parameters[self.PARAM_SECRETS]),
scope=self.GCE_SCOPE)
storage = oauth2client.file.Storage(LocalState.get_oauth2_storage_location(
parameters[self.PARAM_KEYNAME]))
credentials = storage.get()
if credentials is None or credentials.invalid:
credentials = oauth2client.tools.run(flow, storage)
# Build the service
return discovery.build('compute', self.API_VERSION), credentials
def open_connection(self, parameters):
""" Connects to Google Compute Engine with the given credentials.
Args:
parameters: A dict that contains all the parameters necessary to
authenticate this user with Google Compute Engine. We assume that the
user has already authorized this account for use with GCE.
Returns:
An apiclient.discovery.Resource that is a connection valid for requests
to Google Compute Engine for the given user, and a Credentials object that
can be used to sign requests performed with that connection.
"""
# Perform OAuth 2.0 authorization.
flow = None
if self.PARAM_SECRETS in parameters:
flow = oauth2client.client.flow_from_clientsecrets(
os.path.expanduser(parameters[self.PARAM_SECRETS]),
scope=self.GCE_SCOPE)
storage = oauth2client.file.Storage(
LocalState.get_oauth2_storage_location(
parameters[self.PARAM_KEYNAME]))
credentials = storage.get()
if credentials is None or credentials.invalid:
flags = oauth2client.tools.argparser.parse_args(args=[])
credentials = oauth2client.tools.run_flow(flow, storage, flags)
# Build the service
return discovery.build('compute', self.API_VERSION), credentials
def configure_instance_security(self, parameters):
"""
Setup EC2 security keys and groups. Required input values are read from
the parameters dictionary. More specifically, this method expects to
find a 'keyname' parameter and a 'group' parameter in the parameters
dictionary. Using these provided values, this method will create a new
EC2 key-pair and a security group. Security group will be granted permission
to access any port on the instantiated VMs. (Also see documentation for the
BaseAgent class)
Args:
parameters: A dictionary of parameters.
"""
keyname = parameters[self.PARAM_KEYNAME]
group = parameters[self.PARAM_GROUP]
AppScaleLogger.log("Verifying that keyname {0}".format(keyname) + \
" is not already registered.")
conn = self.open_connection(parameters)
if conn.get_key_pair(keyname):
self.handle_failure("SSH keyname {0} is already registered. Please " \
"change the 'keyname' specified in your AppScalefile to a different " \
"value, or erase it to have one automatically generated for you." \
.format(keyname))
security_groups = conn.get_all_security_groups()
for security_group in security_groups:
if security_group.name == group:
self.handle_failure("Security group {0} is already registered. Please" \
" change the 'group' specified in your AppScalefile to a different " \
"value, or erase it to have one automatically generated for you." \
.format(group))
AppScaleLogger.log("Creating key pair: {0}".format(keyname))
key_pair = conn.create_key_pair(keyname)
ssh_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH, keyname)
LocalState.write_key_file(ssh_key, key_pair.material)
self.create_security_group(parameters, group)
self.authorize_security_group(parameters, group, from_port=1, to_port=65535,
ip_protocol='udp', cidr_ip='0.0.0.0/0')
self.authorize_security_group(parameters, group, from_port=1, to_port=65535,
ip_protocol='tcp', cidr_ip='0.0.0.0/0')
self.authorize_security_group(parameters, group, from_port=-1, to_port=-1,
ip_protocol='icmp', cidr_ip='0.0.0.0/0')
return True
def configure_instance_security(self, parameters):
""" Creates a GCE network and firewall with the specified name, and opens
the ports on that firewall as needed for AppScale.
We expect both the network and the firewall to not exist before this point,
to avoid accidentally placing AppScale instances from different deployments
in the same network and firewall (thus enabling them to see each other's web
traffic).
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
network and firewall that we should create in GCE.
Returns:
True, if the named network and firewall was created successfully.
Raises:
AgentRuntimeException: If the named network or firewall already exist in
GCE.
"""
AppScaleLogger.log("Verifying that SSH key exists locally")
keyname = parameters[self.PARAM_KEYNAME]
private_key = LocalState.LOCAL_APPSCALE_PATH + keyname
public_key = private_key + ".pub"
if os.path.exists(private_key) or os.path.exists(public_key):
raise AgentRuntimeException(
"SSH key already found locally - please " +
"use a different keyname")
LocalState.generate_rsa_key(keyname, parameters[self.PARAM_VERBOSE])
ssh_key_exists, all_ssh_keys = self.does_ssh_key_exist(parameters)
if not ssh_key_exists:
self.create_ssh_key(parameters, all_ssh_keys)
if self.does_network_exist(parameters):
raise AgentRuntimeException("Network already exists - please use a " + \
"different group name.")
if self.does_firewall_exist(parameters):
raise AgentRuntimeException("Firewall already exists - please use a " + \
"different group name.")
network_url = self.create_network(parameters)
self.create_firewall(parameters, network_url)
def configure_instance_security(self, parameters):
""" Creates a GCE network and firewall with the specified name, and opens
the ports on that firewall as needed for AppScale.
We expect both the network and the firewall to not exist before this point,
to avoid accidentally placing AppScale instances from different deployments
in the same network and firewall (thus enabling them to see each other's web
traffic).
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
network and firewall that we should create in GCE.
Returns:
True, if the named network and firewall was created successfully.
Raises:
AgentRuntimeException: If the named network or firewall already exist in
GCE.
"""
AppScaleLogger.log("Verifying that SSH key exists locally")
keyname = parameters[self.PARAM_KEYNAME]
private_key = LocalState.LOCAL_APPSCALE_PATH + keyname
public_key = private_key + ".pub"
if os.path.exists(private_key) or os.path.exists(public_key):
raise AgentRuntimeException("SSH key already found locally - please " +
"use a different keyname")
LocalState.generate_rsa_key(keyname, parameters[self.PARAM_VERBOSE])
ssh_key_exists, all_ssh_keys = self.does_ssh_key_exist(parameters)
if not ssh_key_exists:
self.create_ssh_key(parameters, all_ssh_keys)
if self.does_network_exist(parameters):
raise AgentRuntimeException("Network already exists - please use a " + \
"different group name.")
if self.does_firewall_exist(parameters):
raise AgentRuntimeException("Firewall already exists - please use a " + \
"different group name.")
network_url = self.create_network(parameters)
self.create_firewall(parameters, network_url)
def get_params_from_yaml(self, keyname):
""" Searches through the locations.yaml file to build a dict containing the
parameters necessary to interact with Google Compute Engine.
Args:
keyname: A str that uniquely identifies this AppScale deployment.
Returns:
A dict containing all of the credentials necessary to interact with
Google Compute Engine.
"""
params = {
self.PARAM_GROUP : LocalState.get_group(keyname),
self.PARAM_KEYNAME : keyname,
self.PARAM_PROJECT : LocalState.get_project(keyname),
self.PARAM_VERBOSE : False, # TODO(cgb): Don't put False in here.
self.PARAM_ZONE : LocalState.get_zone(keyname)
}
if os.path.exists(LocalState.get_client_secrets_location(keyname)):
params[self.PARAM_SECRETS] = \
LocalState.get_client_secrets_location(keyname)
else:
params[self.PARAM_STORAGE] = \
LocalState.get_oauth2_storage_location(keyname)
return params
def open_connection(self, parameters):
""" Connects to Google Compute Engine with the given credentials.
Args:
parameters: A dict that contains all the parameters necessary to
authenticate this user with Google Compute Engine. We assume that the
user has already authorized this account for use with GCE.
Returns:
An apiclient.discovery.Resource that is a connection valid for requests
to Google Compute Engine for the given user, and a Credentials object that
can be used to sign requests performed with that connection.
Raises:
AppScaleException if the user wants to abort.
"""
# Perform OAuth 2.0 authorization.
flow = None
if self.PARAM_SECRETS in parameters:
secrets_location = os.path.expanduser(parameters[self.PARAM_SECRETS])
secrets_type = GCEAgent.get_secrets_type(secrets_location)
if secrets_type == CredentialTypes.SERVICE:
scopes = [GCPScopes.COMPUTE]
credentials = ServiceAccountCredentials\
.from_json_keyfile_name(secrets_location, scopes=scopes)
return discovery.build('compute', self.API_VERSION), credentials
else:
flow = oauth2client.client.flow_from_clientsecrets(secrets_location,
scope=self.GCE_SCOPE)
storage = oauth2client.file.Storage(LocalState.get_oauth2_storage_location(
parameters[self.PARAM_KEYNAME]))
credentials = storage.get()
if credentials is None or credentials.invalid:
flags = oauth2client.tools.argparser.parse_args(args=[])
credentials = oauth2client.tools.run_flow(flow, storage, flags)
# Build the service
return discovery.build('compute', self.API_VERSION), credentials
def get_params_from_args(self, args):
""" Constructs a dict with only the parameters necessary to interact with
Google Compute Engine (here, the client_secrets file and the image name).
Args:
args: A Namespace or dict that maps all of the arguments the user has
invoked an AppScale command with their associated value.
Returns:
A dict containing the location of the client_secrets file and that name
of the image to use in GCE.
Raises:
AgentConfigurationException: If the caller fails to specify a
client_secrets file, or if it doesn't exist on the local filesystem.
"""
if not isinstance(args, dict):
args = vars(args)
if not args.get('client_secrets') and not args.get('oauth2_storage'):
raise AgentConfigurationException("Please specify a client_secrets " + \
"file or a oauth2_storage file in your AppScalefile when running " + \
"over Google Compute Engine.")
credentials_file = args.get('client_secrets') or args.get('oauth2_storage')
full_credentials = os.path.expanduser(credentials_file)
if not os.path.exists(full_credentials):
raise AgentConfigurationException("Couldn't find your credentials " + \
"at {0}".format(full_credentials))
if args.get('client_secrets'):
destination = LocalState.get_client_secrets_location(args['keyname'])
# Make sure the destination's parent directory exists.
destination_par = os.path.abspath(os.path.join(destination, os.pardir))
if not os.path.exists(destination_par):
os.makedirs(destination_par)
shutil.copy(full_credentials, destination)
elif args.get('oauth2_storage'):
destination = LocalState.get_oauth2_storage_location(args['keyname'])
# Make sure the destination's parent directory exists.
destination_par = os.path.abspath(os.path.join(destination, os.pardir))
if not os.path.exists(destination_par):
os.makedirs(destination_par)
shutil.copy(full_credentials, destination)
params = {
self.PARAM_GROUP : args['group'],
self.PARAM_IMAGE_ID : args['machine'],
self.PARAM_INSTANCE_TYPE : args['gce_instance_type'],
self.PARAM_KEYNAME : args['keyname'],
self.PARAM_PROJECT : args['project'],
self.PARAM_STATIC_IP : args.get(self.PARAM_STATIC_IP),
self.PARAM_ZONE : args['zone'],
self.PARAM_TEST: args['test'],
}
# A zone in GCE looks like 'us-central2-a', which is in the region
# 'us-central2'. Therefore, strip off the last two characters from the zone
# to get the region name.
if params[self.PARAM_ZONE]:
params[self.PARAM_REGION] = params[self.PARAM_ZONE][:-2]
else:
params[self.PARAM_REGION] = self.DEFAULT_REGION
if args.get(self.PARAM_SECRETS):
params[self.PARAM_SECRETS] = args.get(self.PARAM_SECRETS)
elif args.get(self.PARAM_STORAGE):
params[self.PARAM_STORAGE] = args.get(self.PARAM_STORAGE)
params[self.PARAM_VERBOSE] = args.get('verbose', False)
self.assert_credentials_are_valid(params)
return params