def test_spot_price_estimation(self):
(flexmock(EC2Connection)
.should_receive('get_spot_price_history')
.and_return([self.spot_price(0.1, '100'),self.spot_price(0.1, '100'),
self.spot_price(0.2, '100'), self.spot_price(0.3, '100'),
self.spot_price(0.2, '100'),self.spot_price(0.3, '100')]))
factory = InfrastructureAgentFactory()
agent = factory.create_agent('ec2')
self.assertEqual('{0:.2f}'.format(0.2 * 1.2),
'{0:.2f}'.format(agent.get_optimal_spot_price(EC2Connection('',''), 'm1.large')))
def test_create_agent(self):
factory = InfrastructureAgentFactory()
agent = factory.create_agent("ec2")
self.assertEquals(type(agent), type(EC2Agent()))
agent = factory.create_agent("euca")
self.assertEquals(type(agent), type(EucalyptusAgent()))
try:
factory.create_agent("bogus")
self.fail("No exception thrown for invalid infrastructure")
except NameError:
pass
except Exception:
self.fail("Unexpected exception thrown for invalid infrastructure")
def spawn_node_in_cloud(cls, options):
"""Starts a single virtual machine in a cloud infrastructure.
This method also prepares the virual machine for use by the AppScale Tools.
Specifically, it enables root logins on the machine, enables SSH access,
and copies the user's SSH key to that machine.
Args:
options: A Namespace that specifies the cloud infrastructure to use, as
well as how to interact with that cloud.
Returns:
The instance ID, public IP address, and private IP address of the machine
that was started.
"""
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
agent.configure_instance_security(params)
instance_ids, public_ips, private_ips = agent.run_instances(count=1,
parameters=params, security_configured=True)
AppScaleLogger.log("Please wait for your instance to boot up.")
cls.sleep_until_port_is_open(public_ips[0], cls.SSH_PORT, options.verbose)
cls.enable_root_login(public_ips[0], options.keyname,
options.infrastructure, options.verbose)
cls.copy_ssh_keys_to_node(public_ips[0], options.keyname, options.verbose)
return instance_ids[0], public_ips[0], private_ips[0]
def terminate_cloud_infrastructure(cls, keyname, is_verbose):
"""Powers off all machines in the currently running AppScale deployment.
Args:
keyname: The name of the SSH keypair used for this AppScale deployment.
is_verbose: A bool that indicates if we should print the commands executed
to stdout.
"""
AppScaleLogger.log("About to terminate instances spawned with keyname {0}"
.format(keyname))
# This sleep is here to allow a moment for user to Ctrl-C
time.sleep(2)
# get all the instance IDs for machines in our deployment
agent = InfrastructureAgentFactory.create_agent(
LocalState.get_infrastructure(keyname))
params = agent.get_params_from_yaml(keyname)
params['IS_VERBOSE'] = is_verbose
_, _, instance_ids = agent.describe_instances(params)
# terminate all the machines
params[agent.PARAM_INSTANCE_IDS] = instance_ids
agent.terminate_instances(params)
# delete the keyname and group
agent.cleanup_state(params)
def validate_machine_image(self):
"""Checks with the given cloud (if running in a cloud) to ensure that the
user-specified ami/emi exists, aborting if it does not.
Raises:
BadConfigurationException: If the given machine image does not exist.
"""
if not self.args.infrastructure:
return
cloud_agent = InfrastructureAgentFactory.create_agent(
self.args.infrastructure)
params = cloud_agent.get_params_from_args(self.args)
if not cloud_agent.does_image_exist(params):
raise BadConfigurationException("Couldn't find the given machine image.")
if not cloud_agent.does_zone_exist(params):
raise BadConfigurationException("Couldn't find the given zone.")
if not self.args.disks:
return
for disk in set(self.args.disks.values()):
if not cloud_agent.does_disk_exist(params, disk):
raise BadConfigurationException("Couldn't find disk {0}".format(disk))
def __init__(self, params=None, blocking=False):
"""
Create a new InfrastructureManager instance. This constructor
accepts an optional boolean parameter which decides whether the
InfrastructureManager instance should operate in blocking mode
or not. A blocking InfrastructureManager does not return until
each requested run/terminate operation is complete. This mode
is useful for testing and verification purposes. In a real-world
deployment it's advisable to instantiate the InfrastructureManager
in the non-blocking mode as run/terminate operations could take
a rather long time to complete. By default InfrastructureManager
instances are created in the non-blocking mode.
Args
params A dictionary of parameters. Optional parameter. If
specified it must at least include the 'store_type' parameter.
blocking Whether to operate in blocking mode or not. Optional
and defaults to false.
"""
self.blocking = blocking
self.secret = utils.get_secret()
self.agent_factory = InfrastructureAgentFactory()
if params is not None:
store_factory = PersistentStoreFactory()
store = store_factory.create_store(params)
self.reservations = PersistentDictionary(store)
else:
self.reservations = PersistentDictionary()
def validate_credentials(self):
if not self.args.infrastructure:
return
cloud_agent = InfrastructureAgentFactory.create_agent(
self.args.infrastructure)
params = cloud_agent.get_params_from_args(self.args)
cloud_agent.assert_required_parameters(params, BaseAgent.OPERATION_RUN)
def validate_credentials(self):
"""If running over a cloud infrastructure, makes sure that all of the
necessary credentials have been specified.
"""
if not self.args.infrastructure:
return
cloud_agent = InfrastructureAgentFactory.create_agent(self.args.infrastructure)
params = cloud_agent.get_params_from_args(self.args)
cloud_agent.assert_required_parameters(params, BaseAgent.OPERATION_RUN)
def terminate_cloud_instance(cls, instance_id, options):
"""Powers off a single instance in the currently AppScale deployment and
cleans up secret key from the local filesystem.
Args:
instance_id: str containing the instance id.
options: namespace containing the run parameters.
"""
AppScaleLogger.log("About to terminate instance {0}".format(instance_id))
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
params["IS_VERBOSE"] = options.verbose
params[agent.PARAM_INSTANCE_IDS] = [instance_id]
agent.terminate_instances(params)
agent.cleanup_state(params)
os.remove(LocalState.get_secret_key_location(options.keyname))
def terminate_cloud_infrastructure(cls, keyname, is_verbose):
"""Powers off all machines in the currently running AppScale deployment.
Args:
keyname: The name of the SSH keypair used for this AppScale deployment.
is_verbose: A bool that indicates if we should print the commands executed
to stdout.
"""
AppScaleLogger.log("About to terminate deployment and instances with "
"keyname {0}. Press Ctrl-C to stop.".format(keyname))
# This sleep is here to allow a moment for user to Ctrl-C
time.sleep(2)
# get all the instance IDs for machines in our deployment
agent = InfrastructureAgentFactory.create_agent(
LocalState.get_infrastructure(keyname))
params = agent.get_cloud_params(keyname)
params['IS_VERBOSE'] = is_verbose
params['autoscale_agent'] = False
# We want to terminate also the pending instances.
pending = True
_, _, instance_ids = agent.describe_instances(params, pending=pending)
# If using persistent disks, unmount them and detach them before we blow
# away the instances.
nodes = LocalState.get_local_nodes_info(keyname)
for node in nodes:
if node.get('disk'):
AppScaleLogger.log("Unmounting persistent disk at {0}".
format(node['public_ip']))
cls.unmount_persistent_disk(node['public_ip'], keyname, is_verbose)
agent.detach_disk(params, node['disk'], node['instance_id'])
# terminate all the machines
AppScaleLogger.log("Terminating instances spawned with keyname {0}"
.format(keyname))
params[agent.PARAM_INSTANCE_IDS] = instance_ids
agent.terminate_instances(params)
# Delete the network configuration created for the cloud.
agent.cleanup_state(params)
# Cleanup the keyname files created on the local filesystem.
# For GCE and Azure, the keypairs are created on the filesystem,
# rather than the cloud. So we have to clean up afterwards.
LocalState.cleanup_keyname(keyname)
def __init__(self, blocking=False):
"""
Create a new InfrastructureManager instance. This constructor
accepts an optional boolean parameter which decides whether the
InfrastructureManager instance should operate in blocking mode
or not. A blocking InfrastructureManager does not return until
each requested run/terminate operation is complete. This mode
is useful for testing and verification purposes. In a real-world
deployment it's advisable to instantiate the InfrastructureManager
in the non-blocking mode as run/terminate operations could take
a rather long time to complete. By default InfrastructureManager
instances are created in the non-blocking mode.
"""
self.blocking = blocking
self.secret = utils.get_secret()
self.reservations = {}
self.agent_factory = InfrastructureAgentFactory()
def spawn_nodes_in_cloud(cls, options, count=1):
"""Starts a single virtual machine in a cloud infrastructure.
This method also prepares the virual machine for use by the AppScale Tools.
Args:
options: A Namespace that specifies the cloud infrastructure to use, as
well as how to interact with that cloud.
count: A int, the number of instances to start.
Returns:
The instance ID, public IP address, and private IP address of the machine
that was started.
"""
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
# If we have running instances under the current keyname, we try to
# re-attach to them. If we have issue finding the locations file or the
# IP of the head node, we throw an exception.
login_ip = None
public_ips, private_ips, instance_ids = agent.describe_instances(params)
if public_ips:
try:
login_ip = LocalState.get_login_host(options.keyname)
except (IOError, BadConfigurationException):
raise AppScaleException(
"Couldn't get login ip for running deployment with keyname"
" {}.".format(options.keyname))
if login_ip not in public_ips:
raise AppScaleException(
"Couldn't recognize running instances for deployment with"
" keyname {}.".format(options.keyname))
if login_ip in public_ips:
AppScaleLogger.log("Reusing already running instances.")
return instance_ids, public_ips, private_ips
agent.configure_instance_security(params)
instance_ids, public_ips, private_ips = agent.run_instances(count=count,
parameters=params, security_configured=True)
if options.static_ip:
agent.associate_static_ip(params, instance_ids[0], options.static_ip)
public_ips[0] = options.static_ip
AppScaleLogger.log("Static IP associated with head node.")
return instance_ids, public_ips, private_ips
def terminate_cloud_infrastructure(cls, keyname, is_verbose):
"""Powers off all machines in the currently running AppScale deployment.
Args:
keyname: The name of the SSH keypair used for this AppScale deployment.
is_verbose: A bool that indicates if we should print the commands executed
to stdout.
"""
AppScaleLogger.log("About to terminate instances spawned with keyname {0}"
.format(keyname))
# This sleep is here to allow a moment for user to Ctrl-C
time.sleep(2)
# get all the instance IDs for machines in our deployment
agent = InfrastructureAgentFactory.create_agent(
LocalState.get_infrastructure(keyname))
params = agent.get_params_from_yaml(keyname)
params['IS_VERBOSE'] = is_verbose
# We want to terminate also the pending instances.
pending = True
_, _, instance_ids = agent.describe_instances(params, pending=pending)
# If using persistent disks, unmount them and detach them before we blow
# away the instances.
cls.terminate_virtualized_cluster(keyname, is_verbose)
nodes = LocalState.get_local_nodes_info(keyname)
for node in nodes:
if node.get('disk'):
AppScaleLogger.log("Unmounting persistent disk at {0}".format(
node['public_ip']))
cls.unmount_persistent_disk(node['public_ip'], keyname, is_verbose)
agent.detach_disk(params, node['disk'], node['instance_id'])
# terminate all the machines
params[agent.PARAM_INSTANCE_IDS] = instance_ids
agent.terminate_instances(params)
# delete the keyname and group
agent.cleanup_state(params)
def spawn_node_in_cloud(cls, options):
"""Starts a single virtual machine in a cloud infrastructure.
This method also prepares the virual machine for use by the AppScale Tools.
Specifically, it enables root logins on the machine, enables SSH access,
and copies the user's SSH key to that machine.
Args:
options: A Namespace that specifies the cloud infrastructure to use, as
well as how to interact with that cloud.
Returns:
The instance ID, public IP address, and private IP address of the machine
that was started.
"""
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
agent.configure_instance_security(params)
instance_ids, public_ips, private_ips = agent.run_instances(count=1,
parameters=params, security_configured=True)
if options.static_ip:
agent.associate_static_ip(params, instance_ids[0], options.static_ip)
public_ips[0] = options.static_ip
AppScaleLogger.log("Please wait for your instance to boot up.")
cls.sleep_until_port_is_open(public_ips[0], cls.SSH_PORT, options.verbose)
# Since GCE v1beta15, SSH keys don't immediately get injected to newly
# spawned VMs. It takes around 30 seconds, so sleep a bit longer to be
# sure.
if options.infrastructure == 'gce':
AppScaleLogger.log("Waiting for SSH keys to get injected to your "
"machine.")
time.sleep(60)
cls.enable_root_login(public_ips[0], options.keyname,
options.infrastructure, options.verbose)
cls.copy_ssh_keys_to_node(public_ips[0], options.keyname, options.verbose)
return instance_ids[0], public_ips[0], private_ips[0]
class InfrastructureManager(object):
"""
InfrastructureManager class is the main entry point to the AppScale
Infrastructure Manager implementation. An instance of this class can
be used to start new virtual machines in a specified cloud environment
and terminate virtual machines when they are no longer required. Instances
of this class also keep track of the virtual machines spawned by them
and hence each InfrastructureManager instance can be queried to obtain
information about any virtual machines spawned by each of them in the
past.
This implementation is completely cloud infrastructure agnostic
and hence can be used to spawn/terminate instances on a wide range of
cloud (IaaS) environments. All the cloud environment specific operations
are delegated to a separate cloud agent and the InfrastructureManager
initializes cloud agents on demand by looking at the 'infrastructure'
parameter passed into the methods of this class.
"""
# URLs
BACKEND_QUEUE_URL = '/backend/queue'
PREPARE_VMS_OP = 'prepare_vms'
# Default reasons which might be returned by this module
REASON_BAD_SECRET = 'bad secret'
REASON_BAD_VM_COUNT = 'bad vm count'
REASON_BAD_ARGUMENTS = 'bad arguments'
REASON_RESERVATION_NOT_FOUND = 'reservation_id not found'
REASON_NONE = 'none'
# Parameters required by InfrastructureManager
PARAM_RESERVATION_ID = 'reservation_id'
PARAM_INFRASTRUCTURE = 'infrastructure'
PARAM_VMS = 'vms'
PARAM_KEYNAME = 'keyname'
# States a particular VM deployment could be in
STATE_PENDING = 'pending'
STATE_RUNNING = 'running'
STATE_FAILED = 'failed'
# A list of parameters required to query the InfrastructureManager about
# the state of a prepare_instances request.
DESCRIBE_INSTANCES_REQUIRED_PARAMS = ( )
# A list of parameters required to initiate a VM deployment process
PREPARE_INSTANCES_REQUIRED_PARAMS = (
PARAM_INFRASTRUCTURE,
)
# A list of parameters required to initiate a VM termination process
TERMINATE_INSTANCES_REQUIRED_PARAMS = (PARAM_INFRASTRUCTURE, )
def __init__(self, params=None, blocking=False):
"""
Create a new InfrastructureManager instance. This constructor
accepts an optional boolean parameter which decides whether the
InfrastructureManager instance should operate in blocking mode
or not. A blocking InfrastructureManager does not return until
each requested run/terminate operation is complete. This mode
is useful for testing and verification purposes. In a real-world
deployment it's advisable to instantiate the InfrastructureManager
in the non-blocking mode as run/terminate operations could take
a rather long time to complete. By default InfrastructureManager
instances are created in the non-blocking mode.
Args
params A dictionary of parameters. Optional parameter. If
specified it must at least include the 'store_type' parameter.
blocking Whether to operate in blocking mode or not. Optional
and defaults to false.
"""
self.blocking = blocking
# self.secret = utils.get_secret()
self.agent_factory = InfrastructureAgentFactory()
if params is not None:
store_factory = PersistentStoreFactory()
store = store_factory.create_store(params)
self.reservations = PersistentDictionary(store)
else:
self.reservations = PersistentDictionary()
def describe_instances(self, parameters, secret, prefix=''):
"""
Query the InfrastructureManager instance for details regarding
a set of virtual machines spawned in the past. This method accepts
a dictionary of parameters and a secret for authentication purposes.
The dictionary of parameters must include a 'reservation_id' parameter
which is used to reference past virtual machine deployments.
Args:
parameters A dictionary of parameters which contains a valid
'reservation_id' parameter. A valid 'reservation_id'
is an ID issued by the prepare_instances method of the
same InfrastructureManager object. Alternatively one
may provide a valid JSON string instead of a dictionary
object.
secret A previously established secret
Returns:
If the provided secret key is valid and the parameters map contains
a valid 'reservation_id' parameter, this method will return a
dictionary containing information regarding the requested past
virtual machine deployment. This returned map contains several
keys including 'success', 'state', 'reason' and 'vm_info'. The value
of 'success' could be True of False depending on the outcome of the
virtual machine deployment process. If the value of 'success' happens
to be False, the 'reason' key would contain more details as to what
caused the deployment to fail. The 'state' key could contain a 'pending'
value or a 'running' value depending on the current state of the
virtual machine deployment. And finally the 'vm_info' key would point
to a another dictionary containing the IP addresses of the spawned virtual
machines. If the virtual machine deployment had failed or still in the
'pending' state, this key would contain the value None.
If this method receives an invalid key or an invalid 'reservation_id'
parameter, it will return a dictionary containing the keys 'success'
and 'reason' where 'success' would be set to False, and 'reason' is
set to a simple error message describing the cause of the error.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
# parameters, secret = self.__validate_args(parameters, secret)
# if self.secret != secret:
# return self.__generate_response(False, self.REASON_BAD_SECRET)
for param in self.DESCRIBE_INSTANCES_REQUIRED_PARAMS:
if not utils.has_parameter(param, parameters):
return self.__generate_response(False, 'no ' + param)
result = []
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
return agent.describe_instances(parameters, prefix)
# reservation_id = parameters[self.PARAM_RESERVATION_ID]
# if self.reservations.has_key(reservation_id):
# return self.reservations.get(reservation_id)
# else:
# return self.__generate_response(False, self.REASON_RESERVATION_NOT_FOUND)
def prepare_instances(self, parameters):
"""
Prepare and setup a new virtual machine deployment using the provided parameters. The
input parameter set must include an 'infrastructure' parameter which indicates
the exact cloud environment to use. Value of this parameter will be used to
instantiate a cloud environment specific agent which knows how to interact
with the specified cloud platform. The parameters map must also contain a
'num_vms' parameter which indicates the number of virtual machines that should
be spawned. In addition to that any parameters required to spawn VMs in the
specified cloud environment must be included in the parameters map.
If this InfrastructureManager instance has been created in the blocking mode,
this method will not return until the VM deployment is complete. Otherwise
this method will simply kick off the VM deployment process and return
immediately.
Args:
parameters A parameter map containing the keys 'infrastructure',
'num_vms' and any other cloud platform specific
parameters. Alternatively one may provide a valid
JSON string instead of a dictionary object.
Returns:
If the secret is valid and all the required parameters are available in
the input parameter map, this method will return a dictionary containing
a special 'reservation_id' key. If the secret is invalid or a required
parameter is missing, this method will return a different map with the
key 'success' set to False and 'reason' set to a simple error message.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
reservation_id = parameters['reservation_id']
logging.info('Received a request to prepare instances.')
logging.info('Requested reservation_id = {0}'.format(reservation_id))
logging.info('Request parameters are {0}'.format(parameters))
for param in self.PREPARE_INSTANCES_REQUIRED_PARAMS:
if not utils.has_parameter(param, parameters):
logging.error('no {0}'.format(param))
return self.__generate_response(False, 'no ' + param)
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
logging.info('infrastructure = {0}'.format(infrastructure))
agent = self.agent_factory.create_agent(infrastructure)
try:
agent.assert_required_parameters(parameters, BaseAgent.OPERATION_PREPARE)
except AgentConfigurationException as exception:
logging.error(str(exception))
return self.__generate_response(False, exception.message)
keyname = parameters[self.PARAM_KEYNAME]
if keyname is None:
logging.info('Invalid keyname: ' + keyname)
return self.__generate_response(False, self.REASON_BAD_ARGUMENTS)
if self.blocking:
logging.info('Running __prepare_vms in blocking mode')
result = self.__prepare_vms(agent, len(parameters['vms']), parameters, reservation_id)
# NOTE: We will only be able to return an IP for the started instances when run in blocking
# mode, but this is needed to update the queue head IP in celeryconfig.py.
return result
else:
logging.info('Running prepare_vms in non-blocking mode...')
# send the spawn vms task to backend server
from_fields = {
'op': InfrastructureManager.PREPARE_VMS_OP,
'infra': pickle.dumps(self),
'agent': pickle.dumps(agent),
'parameters': pickle.dumps(parameters),
'reservation_id': pickle.dumps(reservation_id)
}
taskqueue.add(url=InfrastructureManager.BACKEND_QUEUE_URL, params=from_fields, method='GET')
logging.info('Successfully sent request to backend server, reservation_id: {0}.'.format(reservation_id))
return self.__generate_response(True, 'Succeeded in sending request to backend server.')
def synchronize_db(self, params, force=False):
logging.debug('synchronize_db(force={0}) param={1}'.format(force, params))
last_time = None
set_gap_large = False
try:
e = db.GqlQuery("SELECT * FROM VMStateSyn").get()
if e:
last_time = e.last_syn
else:
last_time = datetime.datetime.now() - datetime.timedelta(1)
except Exception as e:
logging.error('Error: have errors in opening db_syn file. {0}'.format(e))
return
if last_time is None:
raise Exception('Error: cannot read last synchronization information of db!')
else:
now = datetime.datetime.now()
delta = now - last_time
gap = delta.total_seconds()
logging.info('Time now: {0}'.format(now))
logging.info('Time last synchronization: {0}'.format(last_time))
logging.info('Time in between: {0}'.format(gap))
infrastructure = params[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
if force:
VMStateModel.synchronize(agent = agent, parameters = params)
if gap < backend_handler.SynchronizeDB.PAUSE + 1:
logging.info('Less than {0} seconds to synchronize db.'.format(backend_handler.SynchronizeDB.PAUSE))
return
logging.info('Start synchronize db every {0} seconds.'.format(backend_handler.SynchronizeDB.PAUSE))
from_fields = {
'op': 'start_db_syn',
'agent': pickle.dumps(agent),
'parameters': pickle.dumps(params),
}
logging.info('\n\nAdding db syn task for agent = {}'.format(agent.AGENT_NAME))
taskqueue.add(url=InfrastructureManager.BACKEND_QUEUE_URL, params=from_fields, method='GET')
def deregister_instances(self, parameters, terminate):
"""
Deregister a group of virtual machines and possibly terminate them using
the provided parameters. The input parameter map must contain an 'infrastructure'
parameter which will be used to instantiate a suitable cloud agent. Any additional
environment specific parameters should also be available in the same
map.
If this InfrastructureManager instance has been created in the blocking mode,
this method will not return until the VM deployment is complete. Otherwise
this method simply starts the VM termination process and returns immediately.
Args:
parameters A dictionary of parameters containing the required
'infrastructure' parameter and any other platform
dependent required parameters. Alternatively one
may provide a valid JSON string instead of a dictionary
object.
terminate A boolean flag to indicate whether to terminate instances or not
Returns:
If the secret is valid and all the parameters required to successfully
start a termination process are present in the parameters dictionary,
this method will return a dictionary with the key 'success' set to
True. Otherwise it returns a dictionary with 'success' set to False
and 'reason' set to a simple error message.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
if self.blocking:
self.__deregister_vms(agent, parameters, terminate)
else:
thread.start_new_thread(self.__deregister_vms, (agent, parameters, terminate))
return self.__generate_response(True, self.REASON_NONE)
def __prepare_vms(self, agent, num_vms, parameters, reservation_id):
"""
Private method for preparing a set of VMs
Args:
agent Infrastructure agent in charge of current operation
num_vms No. of VMs to be spawned
parameters A dictionary of parameters
reservation_id Reservation ID of the current run request
"""
status_info = self.reservations.get(reservation_id)
try:
security_configured = agent.configure_instance_security(parameters)
instance_info = agent.prepare_instances(count=num_vms, parameters=parameters,
security_configured=security_configured)
ids = instance_info[0]
public_ips = instance_info[1]
private_ips = instance_info[2]
status_info['state'] = self.STATE_RUNNING
status_info['vm_info'] = {
'public_ips': public_ips,
'private_ips': private_ips,
'instance_ids': ids
}
logging.info('Successfully finished request {0}.'.format(reservation_id))
except AgentRuntimeException as exception:
status_info['state'] = self.STATE_FAILED
status_info['reason'] = exception.message
self.reservations.put(reservation_id, status_info)
return status_info
def __deregister_vms(self, agent, parameters, terminate):
"""
Private method for deregistering a set of VMs
Args:
agent Infrastructure agent in charge of current operation
parameters A dictionary of parameters
terminate Boolean Flag
"""
agent.deregister_instances(parameters=parameters, terminate=terminate)
def __generate_response(self, status, msg, extra=None):
"""
Generate an infrastructure manager service response
Args:
status A boolean value indicating the status
msg A reason message (useful if this a failed operation)
extra Any extra fields to be included in the response (Optional)
Returns:
A dictionary containing the operation response
"""
response = {'success': status, 'reason': msg}
if extra is not None:
for key, value in extra.items():
response[key] = value
return response
def __validate_args(self, parameters, secret):
"""
Validate the arguments provided by user.
Args:
parameters A dictionary (or a JSON string) provided by the client
secret Secret sent by the client
Returns:
Processed user arguments
Raises
TypeError If at least one user argument is not of the current type
"""
if type(parameters) != type('') and type(parameters) != type({}):
raise TypeError('Invalid data type for parameters. Must be a '
'JSON string or a dictionary.')
elif type(secret) != type(''):
raise TypeError('Invalid data type for secret. Must be a string.')
if type(parameters) == type(''):
parameters = json.loads(parameters)
return parameters, secret
def validate_credentials(self, parameters):
"""
Validate if the creds work with IAAS
Returns:
True if they work or else False is returned
"""
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
return agent.validate_credentials(parameters['credentials'])
def start_head_node(cls, options, my_id, node_layout):
"""Starts the first node in an AppScale deployment and instructs it to start
API services on its own node, as well as the other nodes in the deployment.
This includes spawning the first node in the deployment, copying over all
deployment-specific files to it, and starting its AppController service.
Args:
options: A Namespace that includes parameters passed in by the user that
define non-placement-strategy-related deployment options (e.g., keypair
names, security group names).
my_id: A str that is used to uniquely identify this AppScale deployment
with the remote start application.
node_layout: A NodeLayout that describes the placement strategy that
should be used for this AppScale deployment.
Returns:
The public IP and instance ID (a dummy value in non-cloud deployments)
corresponding to the node that was started.
Raises:
AppControllerException: If the AppController on the head node crashes.
The message in this exception indicates why the crash occurred.
"""
secret_key = LocalState.generate_secret_key(options.keyname)
AppScaleLogger.verbose("Secret key is {0}".format(secret_key),
options.verbose)
head_node = node_layout.head_node().public_ip
AppScaleLogger.log(
"Log in to your head node: ssh -i {0} root@{1}".format(
LocalState.get_key_path_from_name(options.keyname), head_node))
additional_params = {}
if options.infrastructure:
agent = InfrastructureAgentFactory.create_agent(
options.infrastructure)
params = agent.get_params_from_args(options)
additional_params = {}
if agent.PARAM_CREDENTIALS in params:
additional_params = params[agent.PARAM_CREDENTIALS]
if options.use_spot_instances:
additional_params[agent.PARAM_SPOT_PRICE] = \
str(params[agent.PARAM_SPOT_PRICE])
if agent.PARAM_REGION in params:
additional_params[agent.PARAM_REGION] = params[
agent.PARAM_REGION]
time.sleep(10) # gives machines in cloud extra time to boot up
cls.copy_deployment_credentials(head_node, options)
cls.run_user_commands(head_node, options.user_commands,
options.keyname, options.verbose)
cls.start_remote_appcontroller(head_node, options.keyname,
options.verbose)
AppScaleLogger.log(
"Head node successfully initialized at {0}.".format(head_node))
AppScaleLogger.remote_log_tools_state(options, my_id,
"started head node",
APPSCALE_VERSION)
# Construct serverside compatible parameters.
deployment_params = LocalState.generate_deployment_params(
options, node_layout, additional_params)
AppScaleLogger.verbose(str(LocalState.obscure_dict(deployment_params)),
options.verbose)
acc = AppControllerClient(head_node, secret_key)
try:
acc.set_parameters(node_layout.to_list(), deployment_params)
except Exception as exception:
AppScaleLogger.warn(
'Saw Exception while setting AC parameters: {0}'.format(
str(exception)))
message = RemoteHelper.collect_appcontroller_crashlog(
head_node, options.keyname, options.verbose)
raise AppControllerException(message)
def start_head_node(cls, options, my_id, node_layout):
"""Starts the first node in an AppScale deployment and instructs it to start
API services on its own node, as well as the other nodes in the deployment.
This includes spawning the first node in the deployment, copying over all
deployment-specific files to it, and starting its AppController service.
Args:
options: A Namespace that includes parameters passed in by the user that
define non-placement-strategy-related deployment options (e.g., keypair
names, security group names).
my_id: A str that is used to uniquely identify this AppScale deployment
with the remote start application.
node_layout: A NodeLayout that describes the placement strategy that
should be used for this AppScale deployment.
Returns:
The public IP and instance ID (a dummy value in non-cloud deployments)
corresponding to the node that was started.
"""
secret_key = LocalState.generate_secret_key(options.keyname)
AppScaleLogger.verbose("Secret key is {0}".format(secret_key),
options.verbose)
if options.infrastructure:
instance_id, public_ip, private_ip = cls.spawn_node_in_cloud(options)
else:
instance_id = cls.DUMMY_INSTANCE_ID
public_ip = node_layout.head_node().public_ip
private_ip = node_layout.head_node().private_ip
AppScaleLogger.log("Log in to your head node: ssh -i {0} root@{1}".format(
LocalState.get_key_path_from_name(options.keyname), public_ip))
try:
cls.ensure_machine_is_compatible(public_ip, options.keyname,
options.table, options.verbose)
except AppScaleException as ase:
# On failure shutdown the cloud instances, cleanup the keys, but only
# if --test is not set.
if options.infrastructure:
if not options.test:
try:
cls.terminate_cloud_instance(instance_id, options)
except Exception as tcie:
AppScaleLogger.log("Error terminating instances: {0}"
.format(str(tcie)))
raise AppScaleException("{0} Please ensure that the "\
"image {1} has AppScale {2} installed on it."
.format(str(ase),options.machine,APPSCALE_VERSION))
else:
raise AppScaleException("{0} Please login to that machine and ensure "\
"that AppScale {1} is installed on it."
.format(str(ase),APPSCALE_VERSION))
if options.scp:
AppScaleLogger.log("Copying over local copy of AppScale from {0}".format(
options.scp))
cls.rsync_files(public_ip, options.keyname, options.scp, options.verbose)
if options.infrastructure:
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
additional_params = params[agent.PARAM_CREDENTIALS]
if options.use_spot_instances:
additional_params[agent.PARAM_SPOT_PRICE] = str(params[agent.PARAM_SPOT_PRICE])
else:
additional_params = {}
deployment_params = LocalState.generate_deployment_params(options,
node_layout, public_ip, additional_params)
AppScaleLogger.verbose(str(LocalState.obscure_dict(deployment_params)),
options.verbose)
AppScaleLogger.log("Head node successfully initialized at {0}. It is now "\
"starting up {1}.".format(public_ip, options.table))
AppScaleLogger.remote_log_tools_state(options, my_id, "started head node",
APPSCALE_VERSION)
time.sleep(10) # gives machines in cloud extra time to boot up
cls.copy_deployment_credentials(public_ip, options)
cls.start_remote_appcontroller(public_ip, options.keyname, options.verbose)
acc = AppControllerClient(public_ip, secret_key)
locations = ["{0}:{1}:{2}:{3}:cloud1".format(public_ip, private_ip,
":".join(node_layout.head_node().roles), instance_id)]
acc.set_parameters(locations, LocalState.map_to_array(deployment_params))
return public_ip, instance_id
def start_head_node(cls, options, my_id, node_layout):
"""Starts the first node in an AppScale deployment and instructs it to start
API services on its own node, as well as the other nodes in the deployment.
This includes spawning the first node in the deployment, copying over all
deployment-specific files to it, and starting its AppController service.
Args:
options: A Namespace that includes parameters passed in by the user that
define non-placement-strategy-related deployment options (e.g., keypair
names, security group names).
my_id: A str that is used to uniquely identify this AppScale deployment
with the remote start application.
node_layout: A NodeLayout that describes the placement strategy that
should be used for this AppScale deployment.
Returns:
The public IP and instance ID (a dummy value in non-cloud deployments)
corresponding to the node that was started.
Raises:
AppControllerException: If the AppController on the head node crashes.
The message in this exception indicates why the crash occurred.
"""
secret_key = LocalState.generate_secret_key(options.keyname)
AppScaleLogger.verbose("Secret key is {0}".format(secret_key),
options.verbose)
if options.infrastructure:
instance_id, public_ip, private_ip = cls.spawn_node_in_cloud(
options)
else:
instance_id = cls.DUMMY_INSTANCE_ID
public_ip = node_layout.head_node().public_ip
private_ip = node_layout.head_node().private_ip
AppScaleLogger.log(
"Log in to your head node: ssh -i {0} root@{1}".format(
LocalState.get_key_path_from_name(options.keyname), public_ip))
try:
cls.ensure_machine_is_compatible(public_ip, options.keyname,
options.table, options.verbose)
except AppScaleException as ase:
# On failure shutdown the cloud instances, cleanup the keys, but only
# if --test is not set.
if options.infrastructure:
if not options.test:
try:
cls.terminate_cloud_instance(instance_id, options)
except Exception as tcie:
AppScaleLogger.log(
"Error terminating instances: {0}".format(
str(tcie)))
raise AppScaleException("{0} Please ensure that the "\
"image {1} has AppScale {2} installed on it."
.format(str(ase), options.machine, APPSCALE_VERSION))
else:
raise AppScaleException("{0} Please login to that machine and ensure "\
"that AppScale {1} is installed on it."
.format(str(ase), APPSCALE_VERSION))
if options.scp:
AppScaleLogger.log(
"Copying over local copy of AppScale from {0}".format(
options.scp))
cls.rsync_files(public_ip, options.keyname, options.scp,
options.verbose)
# On Euca, we've seen issues where attaching the EBS volume right after
# the instance starts doesn't work. This sleep lets the instance fully
# start up and get volumes attached to it correctly.
if options.infrastructure and options.infrastructure == 'euca' and \
options.disks:
time.sleep(30)
if options.infrastructure:
agent = InfrastructureAgentFactory.create_agent(
options.infrastructure)
params = agent.get_params_from_args(options)
additional_params = {}
if agent.PARAM_CREDENTIALS in params:
additional_params = params[agent.PARAM_CREDENTIALS]
if options.use_spot_instances:
additional_params[agent.PARAM_SPOT_PRICE] = str(
params[agent.PARAM_SPOT_PRICE])
if agent.PARAM_REGION in params:
additional_params[agent.PARAM_REGION] = params[
agent.PARAM_REGION]
else:
additional_params = {}
deployment_params = LocalState.generate_deployment_params(
options, node_layout, public_ip, additional_params)
AppScaleLogger.verbose(str(LocalState.obscure_dict(deployment_params)),
options.verbose)
AppScaleLogger.log("Head node successfully initialized at {0}. It is now "\
"starting up {1}.".format(public_ip, options.table))
AppScaleLogger.remote_log_tools_state(options, my_id,
"started head node",
APPSCALE_VERSION)
time.sleep(10) # gives machines in cloud extra time to boot up
cls.copy_deployment_credentials(public_ip, options)
cls.run_user_commands(public_ip, options.user_commands,
options.keyname, options.verbose)
cls.start_remote_appcontroller(public_ip, options.keyname,
options.verbose)
acc = AppControllerClient(public_ip, secret_key)
locations = [{
'public_ip': public_ip,
'private_ip': private_ip,
'jobs': node_layout.head_node().roles,
'instance_id': instance_id,
'disk': node_layout.head_node().disk
}]
try:
acc.set_parameters(locations,
LocalState.map_to_array(deployment_params))
except Exception:
message = RemoteHelper.collect_appcontroller_crashlog(
public_ip, options.keyname, options.verbose)
raise AppControllerException(message)
return public_ip, instance_id
def start_all_nodes(cls, options, node_layout):
""" Starts all nodes in the designated public cloud.
Args:
options: A Namespace that includes parameters passed in by the user that
define non-placement-strategy-related deployment options (e.g., keypair
names, security group names).
node_layout: The node layout of the system including roles.
Returns:
The node layout (dummy values in non-cloud deployments)
corresponding to the nodes that were started.
"""
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
# If we have running instances under the current keyname, we try to
# re-attach to them. If we have issue finding the locations file or the
# IP of the head node, we throw an exception.
login_ip = None
public_ips, private_ips, instance_ids = agent.describe_instances(params)
if public_ips:
try:
login_ip = LocalState.get_login_host(options.keyname)
except (IOError, BadConfigurationException):
raise AppScaleException(
"Couldn't get login ip for running deployment with keyname"
" {}.".format(options.keyname))
if login_ip not in public_ips:
raise AppScaleException(
"Couldn't recognize running instances for deployment with"
" keyname {}.".format(options.keyname))
if login_ip in public_ips:
AppScaleLogger.log("Reusing already running instances.")
# Get the node_info from the locations JSON.
node_info = LocalState.get_local_nodes_info(keyname=options.keyname)
previous_node_list = node_layout.from_locations_json_list(node_info)
# If this is None, the AppScalefile has been changed or the nodes could
# not be matched up by roles/jobs.
if previous_node_list is None:
raise BadConfigurationException("AppScale does not currently support "
"changes to AppScalefile or locations "
"JSON between a down and an up. If "
"you would like to "
"change the node layout use "
"down --terminate before an up.")
node_layout.nodes = previous_node_list
for node_index, node in enumerate(node_layout.nodes):
try:
index = instance_ids.index(node.instance_id)
except ValueError:
raise BadConfigurationException("Previous instance_id {} does not "
"currently exist."
.format(node.instance_id))
node_layout.nodes[node_index].public_ip = public_ips[index]
node_layout.nodes[node_index].private_ip = private_ips[index]
node_layout.nodes[node_index].instance_id = instance_ids[index]
return node_layout
agent.configure_instance_security(params)
load_balancer_roles = {}
instance_type_roles = {}
for node in node_layout.get_nodes('load_balancer', True):
load_balancer_roles.setdefault(node.instance_type, []).append(node)
for node in node_layout.get_nodes('load_balancer', False):
instance_type = instance_type_roles
instance_type.setdefault(node.instance_type, []).append(node)
spawned_instance_ids = []
for instance_type, load_balancer_nodes in load_balancer_roles.items():
# Copy parameters so we can modify the instance type.
instance_type_params = params.copy()
instance_type_params['instance_type'] = instance_type
try:
instance_ids, public_ips, private_ips = cls.spawn_nodes_in_cloud(
agent, instance_type_params, count=len(load_balancer_nodes),
load_balancer=True)
except (AgentRuntimeException, BotoServerError):
AppScaleLogger.warn("AppScale was unable to start the requested number "
"of instances, attempting to terminate those that "
"were started.")
if len(spawned_instance_ids) > 0:
AppScaleLogger.warn("Attempting to terminate those that were started.")
cls.terminate_spawned_instances(spawned_instance_ids, agent, params)
# Cleanup the keyname since it failed.
LocalState.cleanup_keyname(options.keyname)
# Re-raise the original exception.
raise
# Keep track of instances we have started.
spawned_instance_ids.extend(instance_ids)
for node_index, node in enumerate(load_balancer_nodes):
index = node_layout.nodes.index(node)
node_layout.nodes[index].public_ip = public_ips[node_index]
node_layout.nodes[index].private_ip = private_ips[node_index]
node_layout.nodes[index].instance_id = instance_ids[node_index]
if options.static_ip:
node = node_layout.head_node()
agent.associate_static_ip(params, node.instance_id,
options.static_ip)
node.public_ip = options.static_ip
AppScaleLogger.log("Static IP associated with head node.")
AppScaleLogger.log("\nPlease wait for AppScale to prepare your machines "
"for use. This can take few minutes.")
for instance_type, nodes in instance_type_roles.items():
# Copy parameters so we can modify the instance type.
instance_type_params = params.copy()
instance_type_params['instance_type'] = instance_type
try:
_instance_ids, _public_ips, _private_ips = cls.spawn_nodes_in_cloud(
agent, instance_type_params, count=len(nodes))
except (AgentRuntimeException, BotoServerError):
AppScaleLogger.warn("AppScale was unable to start the requested number "
"of instances, attempting to terminate those that "
"were started.")
if len(spawned_instance_ids) > 0:
AppScaleLogger.warn("Attempting to terminate those that were started.")
cls.terminate_spawned_instances(spawned_instance_ids, agent, params)
# Cleanup the keyname since it failed.
LocalState.cleanup_keyname(options.keyname)
# Re-raise the original exception.
raise
# Keep track of instances we have started.
spawned_instance_ids.extend(_instance_ids)
for node_index, node in enumerate(nodes):
index = node_layout.nodes.index(node)
node_layout.nodes[index].public_ip = _public_ips[node_index]
node_layout.nodes[index].private_ip = _private_ips[node_index]
node_layout.nodes[index].instance_id = _instance_ids[node_index]
return node_layout
def start_head_node(cls, options, my_id, node_layout):
"""Starts the first node in an AppScale deployment and instructs it to start
API services on its own node, as well as the other nodes in the deployment.
This includes spawning the first node in the deployment, copying over all
deployment-specific files to it, and starting its AppController service.
Args:
options: A Namespace that includes parameters passed in by the user that
define non-placement-strategy-related deployment options (e.g., keypair
names, security group names).
my_id: A str that is used to uniquely identify this AppScale deployment
with the remote start application.
node_layout: A NodeLayout that describes the placement strategy that
should be used for this AppScale deployment.
Returns:
The public IP and instance ID (a dummy value in non-cloud deployments)
corresponding to the node that was started.
Raises:
AppControllerException: If the AppController on the head node crashes.
The message in this exception indicates why the crash occurred.
"""
secret_key = LocalState.generate_secret_key(options.keyname)
AppScaleLogger.verbose("Secret key is {0}".format(secret_key),
options.verbose)
if options.infrastructure:
instance_id, public_ip, private_ip = cls.spawn_node_in_cloud(options)
else:
instance_id = cls.DUMMY_INSTANCE_ID
public_ip = node_layout.head_node().public_ip
private_ip = node_layout.head_node().private_ip
AppScaleLogger.log("Log in to your head node: ssh -i {0} root@{1}".format(
LocalState.get_key_path_from_name(options.keyname), public_ip))
try:
cls.ensure_machine_is_compatible(public_ip, options.keyname,
options.table, options.verbose)
except AppScaleException as ase:
# On failure shutdown the cloud instances, cleanup the keys, but only
# if --test is not set.
if options.infrastructure:
if not options.test:
try:
cls.terminate_cloud_instance(instance_id, options)
except Exception as tcie:
AppScaleLogger.log("Error terminating instances: {0}"
.format(str(tcie)))
raise AppScaleException("{0} Please ensure that the "\
"image {1} has AppScale {2} installed on it."
.format(str(ase), options.machine, APPSCALE_VERSION))
else:
raise AppScaleException("{0} Please login to that machine and ensure "\
"that AppScale {1} is installed on it."
.format(str(ase), APPSCALE_VERSION))
if options.scp:
AppScaleLogger.log("Copying over local copy of AppScale from {0}".format(
options.scp))
cls.rsync_files(public_ip, options.keyname, options.scp, options.verbose)
# On Euca, we've seen issues where attaching the EBS volume right after
# the instance starts doesn't work. This sleep lets the instance fully
# start up and get volumes attached to it correctly.
if options.infrastructure and options.infrastructure == 'euca' and \
options.disks:
time.sleep(30)
if options.infrastructure:
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
additional_params = {}
if agent.PARAM_CREDENTIALS in params:
additional_params = params[agent.PARAM_CREDENTIALS]
if options.use_spot_instances:
additional_params[agent.PARAM_SPOT_PRICE] = \
str(params[agent.PARAM_SPOT_PRICE])
if agent.PARAM_REGION in params:
additional_params[agent.PARAM_REGION] = params[agent.PARAM_REGION]
else:
additional_params = {}
deployment_params = LocalState.generate_deployment_params(options,
node_layout, public_ip, additional_params)
AppScaleLogger.verbose(str(LocalState.obscure_dict(deployment_params)),
options.verbose)
AppScaleLogger.log("Head node successfully initialized at {0}.".format(public_ip))
AppScaleLogger.remote_log_tools_state(options, my_id, "started head node",
APPSCALE_VERSION)
time.sleep(10) # gives machines in cloud extra time to boot up
cls.copy_deployment_credentials(public_ip, options)
cls.run_user_commands(public_ip, options.user_commands, options.keyname,
options.verbose)
cls.start_remote_appcontroller(public_ip, options.keyname, options.verbose)
acc = AppControllerClient(public_ip, secret_key)
locations = [{
'public_ip' : public_ip,
'private_ip' : private_ip,
'jobs' : node_layout.head_node().roles,
'instance_id' : instance_id,
'disk' : node_layout.head_node().disk
}]
try:
acc.set_parameters(locations, LocalState.map_to_array(deployment_params))
except Exception as exception:
AppScaleLogger.warn('Saw Exception while setting AC parameters: {0}' \
.format(str(exception)))
message = RemoteHelper.collect_appcontroller_crashlog(public_ip,
options.keyname, options.verbose)
raise AppControllerException(message)
return public_ip, instance_id
def start_head_node(cls, options, node_layout):
"""Starts the first node in an AppScale deployment and instructs it to start
API services on its own node, as well as the other nodes in the deployment.
This includes spawning the first node in the deployment, copying over all
deployment-specific files to it, and starting its AppController service.
Args:
options: A Namespace that includes parameters passed in by the user that
define non-placement-strategy-related deployment options (e.g., keypair
names, security group names).
node_layout: A NodeLayout that describes the placement strategy that
should be used for this AppScale deployment.
Returns:
The public IP and instance ID (a dummy value in non-cloud deployments)
corresponding to the node that was started.
"""
secret_key = LocalState.generate_secret_key(options.keyname)
AppScaleLogger.verbose("Secret key is {0}".format(secret_key),
options.verbose)
if options.infrastructure:
instance_id, public_ip, private_ip = cls.spawn_node_in_cloud(options)
else:
instance_id = cls.DUMMY_INSTANCE_ID
public_ip = node_layout.head_node().id
private_ip = node_layout.head_node().id
AppScaleLogger.log("Log in to your head node: ssh -i {0} root@{1}".format(
LocalState.get_key_path_from_name(options.keyname), public_ip))
cls.ensure_machine_is_compatible(public_ip, options.keyname, options.table,
options.verbose)
if options.scp:
AppScaleLogger.log("Copying over local copy of AppScale from {0}".format(
options.scp))
cls.rsync_files(public_ip, options.keyname, options.scp, options.verbose)
if options.infrastructure:
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
additional_params = params[agent.PARAM_CREDENTIALS]
else:
additional_params = {}
deployment_params = LocalState.generate_deployment_params(options,
node_layout, public_ip, additional_params)
AppScaleLogger.verbose(str(LocalState.obscure_dict(deployment_params)),
options.verbose)
AppScaleLogger.log("Head node successfully initialized at {0}. It is now starting up {1}.".format(public_ip, options.table))
AppScaleLogger.remote_log_tools_state(options, "started head node")
time.sleep(10) # gives machines in cloud extra time to boot up
cls.copy_deployment_credentials(public_ip, options)
cls.start_remote_appcontroller(public_ip, options.keyname, options.verbose)
acc = AppControllerClient(public_ip, secret_key)
locations = ["{0}:{1}:{2}:{3}:cloud1".format(public_ip, private_ip,
":".join(node_layout.head_node().roles), instance_id)]
acc.set_parameters(locations, LocalState.map_to_array(deployment_params))
return public_ip, instance_id
class InfrastructureManager:
"""
InfrastructureManager class is the main entry point to the AppScale
Infrastructure Manager implementation. An instance of this class can
be used to start new virtual machines in a specified cloud environment
and terminate virtual machines when they are no longer required. Instances
of this class also keep track of the virtual machines spawned by them
and hence each InfrastructureManager instance can be queried to obtain
information about any virtual machines spawned by each of them in the
past.
This implementation is completely cloud infrastructure agnostic
and hence can be used to spawn/terminate instances on a wide range of
cloud (IaaS) environments. All the cloud environment specific operations
are delegated to a separate cloud agent and the InfrastructureManager
initializes cloud agents on demand by looking at the 'infrastructure'
parameter passed into the methods of this class.
"""
# Default reasons which might be returned by this module
REASON_BAD_SECRET = 'bad secret'
REASON_BAD_VM_COUNT = 'bad vm count'
REASON_BAD_ARGUMENTS = 'bad arguments'
REASON_RESERVATION_NOT_FOUND = 'reservation_id not found'
REASON_NONE = 'none'
# Parameters required by InfrastructureManager
PARAM_RESERVATION_ID = 'reservation_id'
PARAM_INFRASTRUCTURE = 'infrastructure'
PARAM_NUM_VMS = 'num_vms'
# States a particular VM deployment could be in
STATE_PENDING = 'pending'
STATE_RUNNING = 'running'
STATE_FAILED = 'failed'
# A list of parameters required to query the InfrastructureManager about
# the state of a run_instances request.
DESCRIBE_INSTANCES_REQUIRED_PARAMS = (PARAM_RESERVATION_ID, )
# A list of parameters required to initiate a VM deployment process
RUN_INSTANCES_REQUIRED_PARAMS = (PARAM_INFRASTRUCTURE, PARAM_NUM_VMS)
# A list of parameters required to initiate a VM termination process
TERMINATE_INSTANCES_REQUIRED_PARAMS = (PARAM_INFRASTRUCTURE, )
def __init__(self, params=None, blocking=False):
"""
Create a new InfrastructureManager instance. This constructor
accepts an optional boolean parameter which decides whether the
InfrastructureManager instance should operate in blocking mode
or not. A blocking InfrastructureManager does not return until
each requested run/terminate operation is complete. This mode
is useful for testing and verification purposes. In a real-world
deployment it's advisable to instantiate the InfrastructureManager
in the non-blocking mode as run/terminate operations could take
a rather long time to complete. By default InfrastructureManager
instances are created in the non-blocking mode.
Args
params A dictionary of parameters. Optional parameter. If
specified it must at least include the 'store_type' parameter.
blocking Whether to operate in blocking mode or not. Optional
and defaults to false.
"""
self.blocking = blocking
self.secret = utils.get_secret()
self.agent_factory = InfrastructureAgentFactory()
if params is not None:
store_factory = PersistentStoreFactory()
store = store_factory.create_store(params)
self.reservations = PersistentDictionary(store)
else:
self.reservations = PersistentDictionary()
def describe_instances(self, parameters, secret):
"""
Query the InfrastructureManager instance for details regarding
a set of virtual machines spawned in the past. This method accepts
a dictionary of parameters and a secret for authentication purposes.
The dictionary of parameters must include a 'reservation_id' parameter
which is used to reference past virtual machine deployments.
Args:
parameters A dictionary of parameters which contains a valid
'reservation_id' parameter. A valid 'reservation_id'
is an ID issued by the run_instances method of the
same InfrastructureManager object. Alternatively one
may provide a valid JSON string instead of a dictionary
object.
secret A previously established secret
Returns:
If the provided secret key is valid and the parameters map contains
a valid 'reservation_id' parameter, this method will return a
dictionary containing information regarding the requested past
virtual machine deployment. This returned map contains several
keys including 'success', 'state', 'reason' and 'vm_info'. The value
of 'success' could be True of False depending on the outcome of the
virtual machine deployment process. If the value of 'success' happens
to be False, the 'reason' key would contain more details as to what
caused the deployment to fail. The 'state' key could contain a 'pending'
value or a 'running' value depending on the current state of the
virtual machine deployment. And finally the 'vm_info' key would point
to a another dictionary containing the IP addresses of the spawned virtual
machines. If the virtual machine deployment had failed or still in the
'pending' state, this key would contain the value None.
If this method receives an invalid key or an invalid 'reservation_id'
parameter, it will return a dictionary containing the keys 'success'
and 'reason' where 'success' would be set to False, and 'reason' is
set to a simple error message describing the cause of the error.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
parameters, secret = self.__validate_args(parameters, secret)
if self.secret != secret:
return self.__generate_response(False, self.REASON_BAD_SECRET)
for param in self.DESCRIBE_INSTANCES_REQUIRED_PARAMS:
if not utils.has_parameter(param, parameters):
return self.__generate_response(False, 'no ' + param)
reservation_id = parameters[self.PARAM_RESERVATION_ID]
if self.reservations.has_key(reservation_id):
return self.reservations.get(reservation_id)
else:
return self.__generate_response(False,
self.REASON_RESERVATION_NOT_FOUND)
def run_instances(self, parameters, secret):
"""
Start a new virtual machine deployment using the provided parameters. The
input parameter set must include an 'infrastructure' parameter which indicates
the exact cloud environment to use. Value of this parameter will be used to
instantiate a cloud environment specific agent which knows how to interact
with the specified cloud platform. The parameters map must also contain a
'num_vms' parameter which indicates the number of virtual machines that should
be spawned. In addition to that any parameters required to spawn VMs in the
specified cloud environment must be included in the parameters map.
If this InfrastructureManager instance has been created in the blocking mode,
this method will not return until the VM deployment is complete. Otherwise
this method will simply kick off the VM deployment process and return
immediately.
Args:
parameters A parameter map containing the keys 'infrastructure',
'num_vms' and any other cloud platform specific
parameters. Alternatively one may provide a valid
JSON string instead of a dictionary object.
secret A previously established secret
Returns:
If the secret is valid and all the required parameters are available in
the input parameter map, this method will return a dictionary containing
a special 'reservation_id' key. If the secret is invalid or a required
parameter is missing, this method will return a different map with the
key 'success' set to False and 'reason' set to a simple error message.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
parameters, secret = self.__validate_args(parameters, secret)
utils.log('Received a request to run instances.')
if self.secret != secret:
utils.log('Incoming secret {0} does not match the current secret {1} - '\
'Rejecting request.'.format(secret, self.secret))
return self.__generate_response(False, self.REASON_BAD_SECRET)
utils.log('Request parameters are {0}'.format(parameters))
for param in self.RUN_INSTANCES_REQUIRED_PARAMS:
if not utils.has_parameter(param, parameters):
return self.__generate_response(False, 'no ' + param)
num_vms = int(parameters[self.PARAM_NUM_VMS])
if num_vms <= 0:
utils.log('Invalid VM count: {0}'.format(num_vms))
return self.__generate_response(False, self.REASON_BAD_VM_COUNT)
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
try:
agent.assert_required_parameters(parameters,
BaseAgent.OPERATION_RUN)
except AgentConfigurationException as exception:
return self.__generate_response(False, str(exception))
reservation_id = utils.get_random_alphanumeric()
status_info = {
'success': True,
'reason': 'received run request',
'state': self.STATE_PENDING,
'vm_info': None
}
self.reservations.put(reservation_id, status_info)
utils.log('Generated reservation id {0} for this request.'.format(
reservation_id))
try:
if self.blocking:
self.__spawn_vms(agent, num_vms, parameters, reservation_id)
else:
thread.start_new_thread(
self.__spawn_vms,
(agent, num_vms, parameters, reservation_id))
except AgentConfigurationException as exception:
status_info = {
'success': False,
'reason': str(exception),
'state': self.STATE_FAILED,
'vm_info': None
}
self.reservations.put(reservation_id, status_info)
utils.log('Updated reservation id {0} with failed status because: {1}' \
.format(reservation_id, str(exception)))
utils.log('Successfully started request {0}.'.format(reservation_id))
return self.__generate_response(True, self.REASON_NONE,
{'reservation_id': reservation_id})
def terminate_instances(self, parameters, secret):
"""
Terminate a group of virtual machines using the provided parameters.
The input parameter map must contain an 'infrastructure' parameter which
will be used to instantiate a suitable cloud agent. Any additional
environment specific parameters should also be available in the same
map.
If this InfrastructureManager instance has been created in the blocking mode,
this method will not return until the VM deployment is complete. Otherwise
this method simply starts the VM termination process and returns immediately.
Args:
parameters A dictionary of parameters containing the required
'infrastructure' parameter and any other platform
dependent required parameters. Alternatively one
may provide a valid JSON string instead of a dictionary
object.
secret A previously established secret
Returns:
If the secret is valid and all the parameters required to successfully
start a termination process are present in the parameters dictionary,
this method will return a dictionary with the key 'success' set to
True. Otherwise it returns a dictionary with 'success' set to False
and 'reason' set to a simple error message.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
parameters, secret = self.__validate_args(parameters, secret)
if self.secret != secret:
return self.__generate_response(False, self.REASON_BAD_SECRET)
for param in self.TERMINATE_INSTANCES_REQUIRED_PARAMS:
if not utils.has_parameter(param, parameters):
return self.__generate_response(False, 'no ' + param)
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
try:
agent.assert_required_parameters(parameters,
BaseAgent.OPERATION_TERMINATE)
except AgentConfigurationException as exception:
return self.__generate_response(False, str(exception))
if self.blocking:
self.__kill_vms(agent, parameters)
else:
thread.start_new_thread(self.__kill_vms, (agent, parameters))
return self.__generate_response(True, self.REASON_NONE)
def attach_disk(self, parameters, disk_name, instance_id, secret):
""" Contacts the infrastructure named in 'parameters' and tells it to
attach a persistent disk to this machine.
Args:
parameters: A dict containing the credentials necessary to send requests
to the underlying cloud infrastructure.
disk_name: A str corresponding to the name of the persistent disk that
should be attached to this machine.
instance_id: A str naming the instance id that the disk should be attached
to (typically this machine).
secret: A str that authenticates the caller.
"""
parameters, secret = self.__validate_args(parameters, secret)
if self.secret != secret:
return self.__generate_response(False, self.REASON_BAD_SECRET)
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
disk_location = agent.attach_disk(parameters, disk_name, instance_id)
return self.__generate_response(True, self.REASON_NONE,
{'location': disk_location})
def __spawn_vms(self, agent, num_vms, parameters, reservation_id):
"""
Private method for starting a set of VMs
Args:
agent Infrastructure agent in charge of current operation
num_vms No. of VMs to be spawned
parameters A dictionary of parameters
reservation_id Reservation ID of the current run request
"""
status_info = self.reservations.get(reservation_id)
try:
security_configured = agent.configure_instance_security(parameters)
instance_info = agent.run_instances(num_vms, parameters,
security_configured)
ids = instance_info[0]
public_ips = instance_info[1]
private_ips = instance_info[2]
status_info['state'] = self.STATE_RUNNING
status_info['vm_info'] = {
'public_ips': public_ips,
'private_ips': private_ips,
'instance_ids': ids
}
utils.log(
'Successfully finished request {0}.'.format(reservation_id))
except AgentRuntimeException as exception:
status_info['state'] = self.STATE_FAILED
status_info['reason'] = str(exception)
self.reservations.put(reservation_id, status_info)
def __kill_vms(self, agent, parameters):
"""
Private method for stopping a set of VMs
Args:
agent Infrastructure agent in charge of current operation
parameters A dictionary of parameters
"""
agent.terminate_instances(parameters)
def __generate_response(self, status, msg, extra=None):
"""
Generate an infrastructure manager service response
Args:
status A boolean value indicating the status
msg A reason message (useful if this a failed operation)
extra Any extra fields to be included in the response (Optional)
Returns:
A dictionary containing the operation response
"""
utils.log("Sending success = {0}, reason = {1}".format(status, msg))
response = {'success': status, 'reason': msg}
if extra is not None:
for key, value in extra.items():
response[key] = value
return response
def __validate_args(self, parameters, secret):
"""
Validate the arguments provided by user.
Args:
parameters A dictionary (or a JSON string) provided by the client
secret Secret sent by the client
Returns:
Processed user arguments
Raises
TypeError If at least one user argument is not of the current type
"""
if type(parameters) != type('') and type(parameters) != type({}):
raise TypeError('Invalid data type for parameters. Must be a '
'JSON string or a dictionary.')
elif type(secret) != type(''):
raise TypeError('Invalid data type for secret. Must be a string.')
if type(parameters) == type(''):
parameters = json.loads(parameters)
return parameters, secret
class InfrastructureManager:
"""
InfrastructureManager class is the main entry point to the AppScale
Infrastructure Manager implementation. An instance of this class can
be used to start new virtual machines in a specified cloud environment
and terminate virtual machines when they are no longer required. Instances
of this class also keep track of the virtual machines spawned by them
and hence each InfrastructureManager instance can be queried to obtain
information about any virtual machines spawned by each of them in the
past.
This implementation is completely cloud infrastructure agnostic
and hence can be used to spawn/terminate instances on a wide range of
cloud (IaaS) environments. All the cloud environment specific operations
are delegated to a separate cloud agent and the InfrastructureManager
initializes cloud agents on demand by looking at the 'infrastructure'
parameter passed into the methods of this class.
"""
# Default reasons which might be returned by this module
REASON_BAD_SECRET = 'bad secret'
REASON_BAD_VM_COUNT = 'bad vm count'
REASON_BAD_ARGUMENTS = 'bad arguments'
REASON_RESERVATION_NOT_FOUND = 'reservation_id not found'
REASON_NONE = 'none'
# Parameters required by InfrastructureManager
PARAM_RESERVATION_ID = 'reservation_id'
PARAM_INFRASTRUCTURE = 'infrastructure'
PARAM_NUM_VMS = 'num_vms'
# States a particular VM deployment could be in
STATE_PENDING = 'pending'
STATE_RUNNING = 'running'
STATE_FAILED = 'failed'
# A list of parameters required to query the InfrastructureManager about
# the state of a run_instances request.
DESCRIBE_INSTANCES_REQUIRED_PARAMS = ( PARAM_RESERVATION_ID, )
# A list of parameters required to initiate a VM deployment process
RUN_INSTANCES_REQUIRED_PARAMS = (
PARAM_INFRASTRUCTURE,
PARAM_NUM_VMS
)
# A list of parameters required to initiate a VM termination process
TERMINATE_INSTANCES_REQUIRED_PARAMS = ( PARAM_INFRASTRUCTURE, )
def __init__(self, params=None, blocking=False):
"""
Create a new InfrastructureManager instance. This constructor
accepts an optional boolean parameter which decides whether the
InfrastructureManager instance should operate in blocking mode
or not. A blocking InfrastructureManager does not return until
each requested run/terminate operation is complete. This mode
is useful for testing and verification purposes. In a real-world
deployment it's advisable to instantiate the InfrastructureManager
in the non-blocking mode as run/terminate operations could take
a rather long time to complete. By default InfrastructureManager
instances are created in the non-blocking mode.
Args
params A dictionary of parameters. Optional parameter. If
specified it must at least include the 'store_type' parameter.
blocking Whether to operate in blocking mode or not. Optional
and defaults to false.
"""
self.blocking = blocking
self.secret = utils.get_secret()
self.agent_factory = InfrastructureAgentFactory()
if params is not None:
store_factory = PersistentStoreFactory()
store = store_factory.create_store(params)
self.reservations = PersistentDictionary(store)
else:
self.reservations = PersistentDictionary()
def describe_instances(self, parameters, secret):
"""
Query the InfrastructureManager instance for details regarding
a set of virtual machines spawned in the past. This method accepts
a dictionary of parameters and a secret for authentication purposes.
The dictionary of parameters must include a 'reservation_id' parameter
which is used to reference past virtual machine deployments.
Args:
parameters A dictionary of parameters which contains a valid
'reservation_id' parameter. A valid 'reservation_id'
is an ID issued by the run_instances method of the
same InfrastructureManager object. Alternatively one
may provide a valid JSON string instead of a dictionary
object.
secret A previously established secret
Returns:
If the provided secret key is valid and the parameters map contains
a valid 'reservation_id' parameter, this method will return a
dictionary containing information regarding the requested past
virtual machine deployment. This returned map contains several
keys including 'success', 'state', 'reason' and 'vm_info'. The value
of 'success' could be True of False depending on the outcome of the
virtual machine deployment process. If the value of 'success' happens
to be False, the 'reason' key would contain more details as to what
caused the deployment to fail. The 'state' key could contain a 'pending'
value or a 'running' value depending on the current state of the
virtual machine deployment. And finally the 'vm_info' key would point
to a another dictionary containing the IP addresses of the spawned virtual
machines. If the virtual machine deployment had failed or still in the
'pending' state, this key would contain the value None.
If this method receives an invalid key or an invalid 'reservation_id'
parameter, it will return a dictionary containing the keys 'success'
and 'reason' where 'success' would be set to False, and 'reason' is
set to a simple error message describing the cause of the error.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
parameters, secret = self.__validate_args(parameters, secret)
if self.secret != secret:
return self.__generate_response(False, self.REASON_BAD_SECRET)
for param in self.DESCRIBE_INSTANCES_REQUIRED_PARAMS:
if not utils.has_parameter(param, parameters):
return self.__generate_response(False, 'no ' + param)
reservation_id = parameters[self.PARAM_RESERVATION_ID]
if self.reservations.has_key(reservation_id):
return self.reservations.get(reservation_id)
else:
return self.__generate_response(False, self.REASON_RESERVATION_NOT_FOUND)
def run_instances(self, parameters, secret):
"""
Start a new virtual machine deployment using the provided parameters. The
input parameter set must include an 'infrastructure' parameter which indicates
the exact cloud environment to use. Value of this parameter will be used to
instantiate a cloud environment specific agent which knows how to interact
with the specified cloud platform. The parameters map must also contain a
'num_vms' parameter which indicates the number of virtual machines that should
be spawned. In addition to that any parameters required to spawn VMs in the
specified cloud environment must be included in the parameters map.
If this InfrastructureManager instance has been created in the blocking mode,
this method will not return until the VM deployment is complete. Otherwise
this method will simply kick off the VM deployment process and return
immediately.
Args:
parameters A parameter map containing the keys 'infrastructure',
'num_vms' and any other cloud platform specific
parameters. Alternatively one may provide a valid
JSON string instead of a dictionary object.
secret A previously established secret
Returns:
If the secret is valid and all the required parameters are available in
the input parameter map, this method will return a dictionary containing
a special 'reservation_id' key. If the secret is invalid or a required
parameter is missing, this method will return a different map with the
key 'success' set to False and 'reason' set to a simple error message.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
parameters, secret = self.__validate_args(parameters, secret)
utils.log('Received a request to run instances.')
if self.secret != secret:
utils.log('Incoming secret {0} does not match the current secret {1} - '\
'Rejecting request.'.format(secret, self.secret))
return self.__generate_response(False, self.REASON_BAD_SECRET)
for param in self.RUN_INSTANCES_REQUIRED_PARAMS:
if not utils.has_parameter(param, parameters):
return self.__generate_response(False, 'no ' + param)
num_vms = int(parameters[self.PARAM_NUM_VMS])
if num_vms <= 0:
utils.log('Invalid VM count: {0}'.format(num_vms))
return self.__generate_response(False, self.REASON_BAD_VM_COUNT)
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
try:
agent.assert_required_parameters(parameters, BaseAgent.OPERATION_RUN)
except AgentConfigurationException as exception:
return self.__generate_response(False, str(exception))
reservation_id = utils.get_random_alphanumeric()
status_info = {
'success': True,
'reason': 'received run request',
'state': self.STATE_PENDING,
'vm_info': None
}
self.reservations.put(reservation_id, status_info)
utils.log('Generated reservation id {0} for this request.'.format(
reservation_id))
try:
if self.blocking:
self.__spawn_vms(agent, num_vms, parameters, reservation_id)
else:
thread.start_new_thread(self.__spawn_vms,
(agent, num_vms, parameters, reservation_id))
except AgentConfigurationException as exception:
status_info = {
'success' : False,
'reason' : str(exception),
'state' : self.STATE_FAILED,
'vm_info' : None
}
self.reservations.put(reservation_id, status_info)
utils.log('Updated reservation id {0} with failed status because: {1}' \
.format(reservation_id, str(exception)))
utils.log('Successfully started request {0}.'.format(reservation_id))
return self.__generate_response(True,
self.REASON_NONE, {'reservation_id': reservation_id})
def terminate_instances(self, parameters, secret):
"""
Terminate a group of virtual machines using the provided parameters.
The input parameter map must contain an 'infrastructure' parameter which
will be used to instantiate a suitable cloud agent. Any additional
environment specific parameters should also be available in the same
map.
If this InfrastructureManager instance has been created in the blocking mode,
this method will not return until the VM deployment is complete. Otherwise
this method simply starts the VM termination process and returns immediately.
Args:
parameters A dictionary of parameters containing the required
'infrastructure' parameter and any other platform
dependent required parameters. Alternatively one
may provide a valid JSON string instead of a dictionary
object.
secret A previously established secret
Returns:
If the secret is valid and all the parameters required to successfully
start a termination process are present in the parameters dictionary,
this method will return a dictionary with the key 'success' set to
True. Otherwise it returns a dictionary with 'success' set to False
and 'reason' set to a simple error message.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
parameters, secret = self.__validate_args(parameters, secret)
if self.secret != secret:
return self.__generate_response(False, self.REASON_BAD_SECRET)
for param in self.TERMINATE_INSTANCES_REQUIRED_PARAMS:
if not utils.has_parameter(param, parameters):
return self.__generate_response(False, 'no ' + param)
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
try:
agent.assert_required_parameters(parameters,
BaseAgent.OPERATION_TERMINATE)
except AgentConfigurationException as exception:
return self.__generate_response(False, str(exception))
if self.blocking:
self.__kill_vms(agent, parameters)
else:
thread.start_new_thread(self.__kill_vms, (agent, parameters))
return self.__generate_response(True, self.REASON_NONE)
def attach_disk(self, parameters, disk_name, instance_id, secret):
""" Contacts the infrastructure named in 'parameters' and tells it to
attach a persistent disk to this machine.
Args:
parameters: A dict containing the credentials necessary to send requests
to the underlying cloud infrastructure.
disk_name: A str corresponding to the name of the persistent disk that
should be attached to this machine.
instance_id: A str naming the instance id that the disk should be attached
to (typically this machine).
secret: A str that authenticates the caller.
"""
parameters, secret = self.__validate_args(parameters, secret)
if self.secret != secret:
return self.__generate_response(False, self.REASON_BAD_SECRET)
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
disk_location = agent.attach_disk(parameters, disk_name, instance_id)
return self.__generate_response(True, self.REASON_NONE,
{'location' : disk_location})
def __spawn_vms(self, agent, num_vms, parameters, reservation_id):
"""
Private method for starting a set of VMs
Args:
agent Infrastructure agent in charge of current operation
num_vms No. of VMs to be spawned
parameters A dictionary of parameters
reservation_id Reservation ID of the current run request
"""
status_info = self.reservations.get(reservation_id)
try:
security_configured = agent.configure_instance_security(parameters)
instance_info = agent.run_instances(num_vms, parameters,
security_configured)
ids = instance_info[0]
public_ips = instance_info[1]
private_ips = instance_info[2]
status_info['state'] = self.STATE_RUNNING
status_info['vm_info'] = {
'public_ips': public_ips,
'private_ips': private_ips,
'instance_ids': ids
}
utils.log('Successfully finished request {0}.'.format(reservation_id))
except AgentRuntimeException as exception:
status_info['state'] = self.STATE_FAILED
status_info['reason'] = str(exception)
self.reservations.put(reservation_id, status_info)
def __kill_vms(self, agent, parameters):
"""
Private method for stopping a set of VMs
Args:
agent Infrastructure agent in charge of current operation
parameters A dictionary of parameters
"""
agent.terminate_instances(parameters)
def __generate_response(self, status, msg, extra=None):
"""
Generate an infrastructure manager service response
Args:
status A boolean value indicating the status
msg A reason message (useful if this a failed operation)
extra Any extra fields to be included in the response (Optional)
Returns:
A dictionary containing the operation response
"""
utils.log("Sending success = {0}, reason = {1}".format(status, msg))
response = {'success': status, 'reason': msg}
if extra is not None:
for key, value in extra.items():
response[key] = value
return response
def __validate_args(self, parameters, secret):
"""
Validate the arguments provided by user.
Args:
parameters A dictionary (or a JSON string) provided by the client
secret Secret sent by the client
Returns:
Processed user arguments
Raises
TypeError If at least one user argument is not of the current type
"""
if type(parameters) != type('') and type(parameters) != type({}):
raise TypeError('Invalid data type for parameters. Must be a '
'JSON string or a dictionary.')
elif type(secret) != type(''):
raise TypeError('Invalid data type for secret. Must be a string.')
if type(parameters) == type(''):
parameters = json.loads(parameters)
return parameters, secret
class InfrastructureManager(object):
"""
InfrastructureManager class is the main entry point to the AppScale
Infrastructure Manager implementation. An instance of this class can
be used to start new virtual machines in a specified cloud environment
and terminate virtual machines when they are no longer required. Instances
of this class also keep track of the virtual machines spawned by them
and hence each InfrastructureManager instance can be queried to obtain
information about any virtual machines spawned by each of them in the
past.
This implementation is completely cloud infrastructure agnostic
and hence can be used to spawn/terminate instances on a wide range of
cloud (IaaS) environments. All the cloud environment specific operations
are delegated to a separate cloud agent and the InfrastructureManager
initializes cloud agents on demand by looking at the 'infrastructure'
parameter passed into the methods of this class.
"""
# URLs
BACKEND_QUEUE_URL = '/backend/queue'
PREPARE_VMS_OP = 'prepare_vms'
# Default reasons which might be returned by this module
REASON_BAD_SECRET = 'bad secret'
REASON_BAD_VM_COUNT = 'bad vm count'
REASON_BAD_ARGUMENTS = 'bad arguments'
REASON_RESERVATION_NOT_FOUND = 'reservation_id not found'
REASON_NONE = 'none'
# Parameters required by InfrastructureManager
PARAM_RESERVATION_ID = 'reservation_id'
PARAM_INFRASTRUCTURE = 'infrastructure'
PARAM_VMS = 'vms'
PARAM_KEYNAME = 'keyname'
# States a particular VM deployment could be in
STATE_PENDING = 'pending'
STATE_RUNNING = 'running'
STATE_FAILED = 'failed'
# A list of parameters required to query the InfrastructureManager about
# the state of a prepare_instances request.
DESCRIBE_INSTANCES_REQUIRED_PARAMS = ()
# A list of parameters required to initiate a VM deployment process
PREPARE_INSTANCES_REQUIRED_PARAMS = (PARAM_INFRASTRUCTURE, )
# A list of parameters required to initiate a VM termination process
TERMINATE_INSTANCES_REQUIRED_PARAMS = (PARAM_INFRASTRUCTURE, )
def __init__(self, params=None, blocking=False):
"""
Create a new InfrastructureManager instance. This constructor
accepts an optional boolean parameter which decides whether the
InfrastructureManager instance should operate in blocking mode
or not. A blocking InfrastructureManager does not return until
each requested run/terminate operation is complete. This mode
is useful for testing and verification purposes. In a real-world
deployment it's advisable to instantiate the InfrastructureManager
in the non-blocking mode as run/terminate operations could take
a rather long time to complete. By default InfrastructureManager
instances are created in the non-blocking mode.
Args
params A dictionary of parameters. Optional parameter. If
specified it must at least include the 'store_type' parameter.
blocking Whether to operate in blocking mode or not. Optional
and defaults to false.
"""
self.blocking = blocking
# self.secret = utils.get_secret()
self.agent_factory = InfrastructureAgentFactory()
if params is not None:
store_factory = PersistentStoreFactory()
store = store_factory.create_store(params)
self.reservations = PersistentDictionary(store)
else:
self.reservations = PersistentDictionary()
def describe_instances(self, parameters, secret, prefix=''):
"""
Query the InfrastructureManager instance for details regarding
a set of virtual machines spawned in the past. This method accepts
a dictionary of parameters and a secret for authentication purposes.
The dictionary of parameters must include a 'reservation_id' parameter
which is used to reference past virtual machine deployments.
Args:
parameters A dictionary of parameters which contains a valid
'reservation_id' parameter. A valid 'reservation_id'
is an ID issued by the prepare_instances method of the
same InfrastructureManager object. Alternatively one
may provide a valid JSON string instead of a dictionary
object.
secret A previously established secret
Returns:
If the provided secret key is valid and the parameters map contains
a valid 'reservation_id' parameter, this method will return a
dictionary containing information regarding the requested past
virtual machine deployment. This returned map contains several
keys including 'success', 'state', 'reason' and 'vm_info'. The value
of 'success' could be True of False depending on the outcome of the
virtual machine deployment process. If the value of 'success' happens
to be False, the 'reason' key would contain more details as to what
caused the deployment to fail. The 'state' key could contain a 'pending'
value or a 'running' value depending on the current state of the
virtual machine deployment. And finally the 'vm_info' key would point
to a another dictionary containing the IP addresses of the spawned virtual
machines. If the virtual machine deployment had failed or still in the
'pending' state, this key would contain the value None.
If this method receives an invalid key or an invalid 'reservation_id'
parameter, it will return a dictionary containing the keys 'success'
and 'reason' where 'success' would be set to False, and 'reason' is
set to a simple error message describing the cause of the error.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
# parameters, secret = self.__validate_args(parameters, secret)
# if self.secret != secret:
# return self.__generate_response(False, self.REASON_BAD_SECRET)
for param in self.DESCRIBE_INSTANCES_REQUIRED_PARAMS:
if not utils.has_parameter(param, parameters):
return self.__generate_response(False, 'no ' + param)
result = []
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
return agent.describe_instances(parameters, prefix)
# reservation_id = parameters[self.PARAM_RESERVATION_ID]
# if self.reservations.has_key(reservation_id):
# return self.reservations.get(reservation_id)
# else:
# return self.__generate_response(False, self.REASON_RESERVATION_NOT_FOUND)
def prepare_instances(self, parameters):
"""
Prepare and setup a new virtual machine deployment using the provided parameters. The
input parameter set must include an 'infrastructure' parameter which indicates
the exact cloud environment to use. Value of this parameter will be used to
instantiate a cloud environment specific agent which knows how to interact
with the specified cloud platform. The parameters map must also contain a
'num_vms' parameter which indicates the number of virtual machines that should
be spawned. In addition to that any parameters required to spawn VMs in the
specified cloud environment must be included in the parameters map.
If this InfrastructureManager instance has been created in the blocking mode,
this method will not return until the VM deployment is complete. Otherwise
this method will simply kick off the VM deployment process and return
immediately.
Args:
parameters A parameter map containing the keys 'infrastructure',
'num_vms' and any other cloud platform specific
parameters. Alternatively one may provide a valid
JSON string instead of a dictionary object.
Returns:
If the secret is valid and all the required parameters are available in
the input parameter map, this method will return a dictionary containing
a special 'reservation_id' key. If the secret is invalid or a required
parameter is missing, this method will return a different map with the
key 'success' set to False and 'reason' set to a simple error message.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
reservation_id = parameters['reservation_id']
logging.info('Received a request to prepare instances.')
logging.info('Requested reservation_id = {0}'.format(reservation_id))
logging.info('Request parameters are {0}'.format(parameters))
for param in self.PREPARE_INSTANCES_REQUIRED_PARAMS:
if not utils.has_parameter(param, parameters):
logging.error('no {0}'.format(param))
return self.__generate_response(False, 'no ' + param)
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
logging.info('infrastructure = {0}'.format(infrastructure))
agent = self.agent_factory.create_agent(infrastructure)
try:
agent.assert_required_parameters(parameters,
BaseAgent.OPERATION_PREPARE)
except AgentConfigurationException as exception:
logging.error(str(exception))
return self.__generate_response(False, exception.message)
keyname = parameters[self.PARAM_KEYNAME]
if keyname is None:
logging.info('Invalid keyname: ' + keyname)
return self.__generate_response(False, self.REASON_BAD_ARGUMENTS)
if self.blocking:
logging.info('Running __prepare_vms in blocking mode')
result = self.__prepare_vms(agent, len(parameters['vms']),
parameters, reservation_id)
# NOTE: We will only be able to return an IP for the started instances when run in blocking
# mode, but this is needed to update the queue head IP in celeryconfig.py.
return result
else:
logging.info('Running prepare_vms in non-blocking mode...')
# send the spawn vms task to backend server
from_fields = {
'op': InfrastructureManager.PREPARE_VMS_OP,
'infra': pickle.dumps(self),
'agent': pickle.dumps(agent),
'parameters': pickle.dumps(parameters),
'reservation_id': pickle.dumps(reservation_id)
}
taskqueue.add(url=InfrastructureManager.BACKEND_QUEUE_URL,
params=from_fields,
method='GET')
logging.info(
'Successfully sent request to backend server, reservation_id: {0}.'
.format(reservation_id))
return self.__generate_response(
True, 'Succeeded in sending request to backend server.')
def synchronize_db(self, params, force=False):
logging.debug('synchronize_db(force={0}) param={1}'.format(
force, params))
last_time = None
set_gap_large = False
try:
e = db.GqlQuery("SELECT * FROM VMStateSyn").get()
if e:
last_time = e.last_syn
else:
last_time = datetime.datetime.now() - datetime.timedelta(1)
except Exception as e:
logging.error(
'Error: have errors in opening db_syn file. {0}'.format(e))
return
if last_time is None:
raise Exception(
'Error: cannot read last synchronization information of db!')
else:
now = datetime.datetime.now()
delta = now - last_time
gap = delta.total_seconds()
logging.info('Time now: {0}'.format(now))
logging.info('Time last synchronization: {0}'.format(last_time))
logging.info('Time in between: {0}'.format(gap))
infrastructure = params[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
if force:
VMStateModel.synchronize(agent=agent, parameters=params)
if gap < backend_handler.SynchronizeDB.PAUSE + 1:
logging.info('Less than {0} seconds to synchronize db.'.format(
backend_handler.SynchronizeDB.PAUSE))
return
logging.info('Start synchronize db every {0} seconds.'.format(
backend_handler.SynchronizeDB.PAUSE))
from_fields = {
'op': 'start_db_syn',
'agent': pickle.dumps(agent),
'parameters': pickle.dumps(params),
}
logging.info('\n\nAdding db syn task for agent = {}'.format(
agent.AGENT_NAME))
taskqueue.add(url=InfrastructureManager.BACKEND_QUEUE_URL,
params=from_fields,
method='GET')
def deregister_instances(self, parameters, terminate):
"""
Deregister a group of virtual machines and possibly terminate them using
the provided parameters. The input parameter map must contain an 'infrastructure'
parameter which will be used to instantiate a suitable cloud agent. Any additional
environment specific parameters should also be available in the same
map.
If this InfrastructureManager instance has been created in the blocking mode,
this method will not return until the VM deployment is complete. Otherwise
this method simply starts the VM termination process and returns immediately.
Args:
parameters A dictionary of parameters containing the required
'infrastructure' parameter and any other platform
dependent required parameters. Alternatively one
may provide a valid JSON string instead of a dictionary
object.
terminate A boolean flag to indicate whether to terminate instances or not
Returns:
If the secret is valid and all the parameters required to successfully
start a termination process are present in the parameters dictionary,
this method will return a dictionary with the key 'success' set to
True. Otherwise it returns a dictionary with 'success' set to False
and 'reason' set to a simple error message.
Raises:
TypeError If the inputs are not of the expected types
ValueError If the input JSON string (parameters) cannot be parsed properly
"""
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
if self.blocking:
self.__deregister_vms(agent, parameters, terminate)
else:
thread.start_new_thread(self.__deregister_vms,
(agent, parameters, terminate))
return self.__generate_response(True, self.REASON_NONE)
def __prepare_vms(self, agent, num_vms, parameters, reservation_id):
"""
Private method for preparing a set of VMs
Args:
agent Infrastructure agent in charge of current operation
num_vms No. of VMs to be spawned
parameters A dictionary of parameters
reservation_id Reservation ID of the current run request
"""
status_info = self.reservations.get(reservation_id)
try:
security_configured = agent.configure_instance_security(parameters)
instance_info = agent.prepare_instances(
count=num_vms,
parameters=parameters,
security_configured=security_configured)
ids = instance_info[0]
public_ips = instance_info[1]
private_ips = instance_info[2]
status_info['state'] = self.STATE_RUNNING
status_info['vm_info'] = {
'public_ips': public_ips,
'private_ips': private_ips,
'instance_ids': ids
}
logging.info(
'Successfully finished request {0}.'.format(reservation_id))
except AgentRuntimeException as exception:
status_info['state'] = self.STATE_FAILED
status_info['reason'] = exception.message
self.reservations.put(reservation_id, status_info)
return status_info
def __deregister_vms(self, agent, parameters, terminate):
"""
Private method for deregistering a set of VMs
Args:
agent Infrastructure agent in charge of current operation
parameters A dictionary of parameters
terminate Boolean Flag
"""
agent.deregister_instances(parameters=parameters, terminate=terminate)
def __generate_response(self, status, msg, extra=None):
"""
Generate an infrastructure manager service response
Args:
status A boolean value indicating the status
msg A reason message (useful if this a failed operation)
extra Any extra fields to be included in the response (Optional)
Returns:
A dictionary containing the operation response
"""
response = {'success': status, 'reason': msg}
if extra is not None:
for key, value in extra.items():
response[key] = value
return response
def __validate_args(self, parameters, secret):
"""
Validate the arguments provided by user.
Args:
parameters A dictionary (or a JSON string) provided by the client
secret Secret sent by the client
Returns:
Processed user arguments
Raises
TypeError If at least one user argument is not of the current type
"""
if type(parameters) != type('') and type(parameters) != type({}):
raise TypeError('Invalid data type for parameters. Must be a '
'JSON string or a dictionary.')
elif type(secret) != type(''):
raise TypeError('Invalid data type for secret. Must be a string.')
if type(parameters) == type(''):
parameters = json.loads(parameters)
return parameters, secret
def validate_credentials(self, parameters):
"""
Validate if the creds work with IAAS
Returns:
True if they work or else False is returned
"""
infrastructure = parameters[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
return agent.validate_credentials(parameters['credentials'])
def start_head_node(cls, options, my_id, node_layout):
"""Starts the first node in an AppScale deployment and instructs it to start
API services on its own node, as well as the other nodes in the deployment.
This includes spawning the first node in the deployment, copying over all
deployment-specific files to it, and starting its AppController service.
Args:
options: A Namespace that includes parameters passed in by the user that
define non-placement-strategy-related deployment options (e.g., keypair
names, security group names).
my_id: A str that is used to uniquely identify this AppScale deployment
with the remote start application.
node_layout: A NodeLayout that describes the placement strategy that
should be used for this AppScale deployment.
Returns:
The public IP and instance ID (a dummy value in non-cloud deployments)
corresponding to the node that was started.
Raises:
AppControllerException: If the AppController on the head node crashes.
The message in this exception indicates why the crash occurred.
"""
secret_key = LocalState.generate_secret_key(options.keyname)
AppScaleLogger.verbose("Secret key is {0}".
format(secret_key), options.verbose)
head_node = node_layout.head_node().public_ip
AppScaleLogger.log("Log in to your head node: ssh -i {0} root@{1}".format(
LocalState.get_key_path_from_name(options.keyname), head_node))
additional_params = {}
if options.infrastructure:
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
additional_params = {}
if agent.PARAM_CREDENTIALS in params:
additional_params = params[agent.PARAM_CREDENTIALS]
if options.use_spot_instances:
additional_params[agent.PARAM_SPOT_PRICE] = \
str(params[agent.PARAM_SPOT_PRICE])
if agent.PARAM_REGION in params:
additional_params[agent.PARAM_REGION] = params[agent.PARAM_REGION]
time.sleep(10) # gives machines in cloud extra time to boot up
cls.copy_deployment_credentials(head_node, options)
cls.run_user_commands(head_node, options.user_commands, options.keyname,
options.verbose)
cls.start_remote_appcontroller(head_node, options.keyname, options.verbose)
AppScaleLogger.log("Head node successfully initialized at {0}.".
format(head_node))
AppScaleLogger.remote_log_tools_state(
options, my_id, "started head node", APPSCALE_VERSION)
# Construct serverside compatible parameters.
deployment_params = LocalState.generate_deployment_params(
options, node_layout, additional_params)
AppScaleLogger.verbose(str(LocalState.obscure_dict(deployment_params)),
options.verbose)
acc = AppControllerClient(head_node, secret_key)
try:
acc.set_parameters(node_layout.to_list(), deployment_params)
except Exception as exception:
AppScaleLogger.warn(u'Saw Exception while setting AC parameters: {0}'.
format(str(exception)))
message = RemoteHelper.collect_appcontroller_crashlog(
head_node, options.keyname, options.verbose)
raise AppControllerException(message)