def add_instances(cls, options):
"""Adds additional machines to an AppScale deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if 'master' in options.ips.keys():
raise BadConfigurationException("Cannot add master nodes to an " + \
"already running AppScale deployment.")
# Skip checking for -n (replication) because we don't allow the user
# to specify it here (only allowed in run-instances).
additional_nodes_layout = NodeLayout(options)
# In virtualized cluster deployments, we need to make sure that the user
# has already set up SSH keys.
if LocalState.get_from_yaml(options.keyname, 'infrastructure') == "xen":
for ip in options.ips.values():
# throws a ShellException if the SSH key doesn't work
RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)
# Finally, find an AppController and send it a message to add
# the given nodes with the new roles.
AppScaleLogger.log("Sending request to add instances")
login_ip = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_ip, LocalState.get_secret_key(
options.keyname))
acc.start_roles_on_nodes(json.dumps(options.ips))
# TODO(cgb): Should we wait for the new instances to come up and get
# initialized?
AppScaleLogger.success("Successfully sent request to add instances " + \
"to this AppScale deployment.")
def terminate_cloud_infrastructure(cls, keyname, is_verbose):
"""Powers off all machines in the currently running AppScale deployment.
Args:
keyname: The name of the SSH keypair used for this AppScale deployment.
is_verbose: A bool that indicates if we should print the commands executed
to stdout.
"""
AppScaleLogger.log("About to terminate instances spawned with keyname {0}"
.format(keyname))
# This sleep is here to allow a moment for user to Ctrl-C
time.sleep(2)
# get all the instance IDs for machines in our deployment
agent = InfrastructureAgentFactory.create_agent(
LocalState.get_infrastructure(keyname))
params = agent.get_params_from_yaml(keyname)
params['IS_VERBOSE'] = is_verbose
_, _, instance_ids = agent.describe_instances(params)
# terminate all the machines
params[agent.PARAM_INSTANCE_IDS] = instance_ids
agent.terminate_instances(params)
# delete the keyname and group
agent.cleanup_state(params)
def get_application_info(cls, owner, app_language, app_dir):
name = AppEngineHelper.get_app_id_from_app_config(app_dir)
version = AppEngineHelper.get_app_version_from_app_config(app_dir)
app = Application(name, version, owner)
dependencies_path = app_dir + os.sep + 'dependencies.yaml'
if app_language == 'java':
dependencies_path = app_dir + os.sep + 'war' + os.sep + 'WEB-INF' + os.sep + 'dependencies.yaml'
api_specs_dir = app_dir + os.sep + 'war' + os.sep + 'WEB-INF' + os.sep + 'specs'
if os.path.exists(api_specs_dir):
for f in os.listdir(api_specs_dir):
if f.endswith('.json'):
api = API(api_specs_dir + os.sep + f)
AppScaleLogger.log('Detected API: {0}-v{1}'.format(api.name, api.version))
app.api_list.append(api)
if os.path.exists(dependencies_path):
dependencies_file = open(dependencies_path, 'r')
dependencies = yaml.load(dependencies_file)
dependencies_file.close()
if dependencies:
EagerHelper.validate_dependencies(dependencies)
app.dependencies = dependencies['dependencies']
return app
def get_serving_info(self, app_id, keyname):
"""Finds out what host and port are used to host the named application.
Args:
app_id: The application that we should find a serving URL for.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
"""
total_wait_time = 0
# first, wait for the app to start serving
sleep_time = self.STARTING_SLEEP_TIME
while True:
if self.does_app_exist(app_id):
break
else:
AppScaleLogger.log("Waiting {0} second(s) to check on application...".\
format(sleep_time))
time.sleep(sleep_time)
sleep_time = min(sleep_time * 2, self.MAX_SLEEP_TIME)
total_wait_time += sleep_time
if total_wait_time > self.MAX_WAIT_TIME:
raise AppScaleException("App took too long to upload")
# next, get the serving host and port
app_data = self.server.get_app_data(app_id, self.secret)
host = LocalState.get_login_host(keyname)
port = int(re.search(".*\sports: (\d+)[\s|:]", app_data).group(1))
return host, port
def remove_app(cls, options):
"""Instructs AppScale to no longer host the named application.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if not options.confirm:
response = raw_input("Are you sure you want to remove this " + \
"application? (Y/N) ")
if response not in ['y', 'yes', 'Y', 'YES']:
raise AppScaleException("Cancelled application removal.")
login_host = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_host, LocalState.get_secret_key(
options.keyname))
userappserver_host = acc.get_uaserver_host(options.verbose)
userappclient = UserAppClient(userappserver_host, LocalState.get_secret_key(
options.keyname))
if not userappclient.does_app_exist(options.appname):
raise AppScaleException("The given application is not currently running.")
acc.stop_app(options.appname)
AppScaleLogger.log("Please wait for your app to shut down.")
while True:
if acc.is_app_running(options.appname):
time.sleep(cls.SLEEP_TIME)
else:
break
AppScaleLogger.success("Done shutting down {0}".format(options.appname))
def start_remote_appcontroller(cls, host, keyname, is_verbose):
"""Starts the AppController daemon on the specified host.
Args:
host: A str representing the host to start the AppController on.
keyname: A str representing the name of the SSH keypair that can log into
the specified host.
is_verbose: A bool that indicates if we should print the commands needed
to start the AppController to stdout.
"""
AppScaleLogger.log("Starting AppController at {0}".format(host))
# remove any possible appcontroller state that may not have been
# properly removed in virtualized clusters
cls.ssh(host, keyname, 'rm -rf /etc/appscale/appcontroller-state.json',
is_verbose)
# start up god, who will start up the appcontroller once we give it the
# right config file
cls.ssh(host, keyname, 'god &', is_verbose)
time.sleep(1)
# scp over that config file
cls.scp(host, keyname, cls.TEMPLATE_GOD_CONFIG_FILE,
'/tmp/appcontroller.god', is_verbose)
# finally, tell god to start the appcontroller and then wait for it to start
cls.ssh(host, keyname, 'god load /tmp/appcontroller.god', is_verbose)
AppScaleLogger.log("Please wait for the AppController to finish " + \
"pre-processing tasks.")
cls.sleep_until_port_is_open(host, AppControllerClient.PORT, is_verbose)
def copy_app_to_host(cls, app_location, keyname, is_verbose):
"""Copies the given application to a machine running the Login service
within an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "{0}/appscale-app-{1}-{2}.tar.gz".format(tempfile.gettempdir(),
app_id, rand)
LocalState.shell("cd '{0}' && COPYFILE_DISABLE=1 tar -czhf {1} --exclude='*.pyc' *".format(
app_location, local_tarred_app), is_verbose)
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(cls.REMOTE_APP_DIR, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def enable_root_login(cls, host, keyname, infrastructure, is_verbose):
"""Logs into the named host and alters its ssh configuration to enable the
root user to directly log in.
Args:
host: A str representing the host to enable root logins on.
keyname: A str representing the name of the SSH keypair to login with.
infrastructure: A str representing the name of the cloud infrastructure
we're running on.
is_verbose: A bool indicating if we should print the command we execute to
enable root login to stdout.
"""
# First, see if we need to enable root login at all (some VMs have it
# already enabled).
try:
output = cls.ssh(host, keyname, 'ls', is_verbose, user='******')
except ShellException as exception:
# Google Compute Engine creates a user with the same name as the currently
# logged-in user, so log in as that user to enable root login.
if infrastructure == "gce":
cls.merge_authorized_keys(host, keyname, getpass.getuser(),
is_verbose)
return
else:
raise exception
# Amazon EC2 rejects a root login request and tells the user to log in as
# the ubuntu user, so do that to enable root login.
if re.search(cls.LOGIN_AS_UBUNTU_USER, output):
cls.merge_authorized_keys(host, keyname, 'ubuntu', is_verbose)
else:
AppScaleLogger.log("Root login already enabled - not re-enabling it.")
def ensure_appscale_isnt_running(cls, keyname, force):
"""Checks the secret key file to see if AppScale is running, and
aborts if it is.
Args:
keyname: The keypair name that is used to identify AppScale deployments.
force: A bool that is used to run AppScale even if the secret key file
is present.
Raises:
BadConfigurationException: If AppScale is already running.
"""
if force:
return
if os.path.exists(cls.get_secret_key_location(keyname)):
try:
login_host = cls.get_login_host(keyname)
secret_key = cls.get_secret_key(keyname)
except (IOError, AppScaleException, BadConfigurationException):
# If we don't have the locations files, we are not running.
return
acc = AppControllerClient(login_host, secret_key)
try:
acc.get_all_public_ips()
except AppControllerException:
# AC is not running, so we assume appscale is not up and running.
AppScaleLogger.log("AppController not running on login node.")
else:
raise BadConfigurationException("AppScale is already running. Terminate" +
" it, set 'force: True' in your AppScalefile, or use the --force flag" +
" to run anyways.")
def shut_down_appscale_if_running(cls, options):
""" Checks if AppScale is running and shuts it down as this is an offline upgrade.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if os.path.exists(LocalState.get_secret_key_location(options.keyname)):
AppScaleLogger.warn(
"AppScale needs to be down for this upgrade. "
"Upgrade process could take a while and it is not reversible."
)
if not options.test:
response = raw_input(
"Are you sure you want to proceed with shutting down AppScale to " "continue the upgrade? (y/N) "
)
if response.lower() not in ["y", "yes"]:
raise AppScaleException("Cancelled AppScale upgrade.")
AppScaleLogger.log("Shutting down AppScale...")
cls.terminate_instances(options)
else:
AppScaleLogger.warn("Upgrade process could take a while and it is not reversible.")
if options.test:
return
response = raw_input("Are you sure you want to proceed with the upgrade? (y/N) ")
if response.lower() not in ["y", "yes"]:
raise AppScaleException("Cancelled AppScale upgrade.")
def copy_app_to_host(cls, app_location, app_id, keyname, is_verbose,
extras=None, custom_service_yaml=None):
"""Copies the given application to a machine running the Login service
within an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
app_id: The project to use for this application.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
extras: A dictionary containing a list of files to include in the upload.
custom_service_yaml: A string specifying the location of the service
yaml being deployed.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "{0}/appscale-app-{1}-{2}.tar.gz".\
format(tempfile.gettempdir(), app_id, rand)
# Collect list of files that should be included in the tarball.
app_files = {}
for root, _, filenames in os.walk(app_location, followlinks=True):
relative_dir = os.path.relpath(root, app_location)
for filename in filenames:
# Ignore compiled Python files.
if filename.endswith('.pyc'):
continue
relative_path = os.path.join(relative_dir, filename)
app_files[relative_path] = os.path.join(root, filename)
if extras is not None:
app_files.update(extras)
with tarfile.open(local_tarred_app, 'w:gz') as app_tar:
for tarball_path, local_path in app_files.items():
# Replace app.yaml with the service yaml being deployed.
if custom_service_yaml and os.path.normpath(tarball_path) == 'app.yaml':
continue
app_tar.add(local_path, tarball_path)
if custom_service_yaml:
app_tar.add(custom_service_yaml, 'app.yaml')
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(cls.REMOTE_APP_DIR, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
AppScaleLogger.verbose("Removing local copy of tarred application",
is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def start_remote_appcontroller(cls, host, keyname, is_verbose):
"""Starts the AppController daemon on the specified host.
Args:
host: A str representing the host to start the AppController on.
keyname: A str representing the name of the SSH keypair that can log into
the specified host.
is_verbose: A bool that indicates if we should print the commands needed
to start the AppController to stdout.
"""
AppScaleLogger.log("Starting AppController at {0}".format(host))
# Remove any previous state. TODO: Don't do this with the tools.
cls.ssh(host, keyname,
'rm -rf {}/appcontroller-state.json'.format(cls.CONFIG_DIR), is_verbose)
# Remove any monit configuration files from previous AppScale deployments.
cls.ssh(host, keyname, 'rm -rf /etc/monit/conf.d/appscale-*.cfg', is_verbose)
cls.ssh(host, keyname, 'service monit start', is_verbose)
# Start the AppController.
cls.ssh(host, keyname, 'service appscale-controller start', is_verbose)
AppScaleLogger.log("Please wait for the AppController to finish " + \
"pre-processing tasks.")
cls.sleep_until_port_is_open(host, AppControllerClient.PORT, is_verbose)
def merge_authorized_keys(cls, host, keyname, user, is_verbose):
""" Adds the contents of the user's authorized_keys file to the root's
authorized_keys file.
Args:
host: A str representing the host to enable root logins on.
keyname: A str representing the name of the SSH keypair to login with.
user: A str representing the name of the user to login as.
is_verbose: A bool indicating if we should print the command we execute to
enable root login to stdout.
"""
AppScaleLogger.log('Root login not enabled - enabling it now.')
create_root_keys = 'sudo touch /root/.ssh/authorized_keys'
cls.ssh(host, keyname, create_root_keys, is_verbose, user=user)
set_permissions = 'sudo chmod 600 /root/.ssh/authorized_keys'
cls.ssh(host, keyname, set_permissions, is_verbose, user=user)
temp_file = cls.ssh(host, keyname, 'mktemp', is_verbose, user=user)
merge_to_tempfile = 'sudo sort -u ~/.ssh/authorized_keys '\
'/root/.ssh/authorized_keys -o {}'.format(temp_file)
cls.ssh(host, keyname, merge_to_tempfile, is_verbose, user=user)
overwrite_root_keys = "sudo sed -n '/.*Please login/d; "\
"w/root/.ssh/authorized_keys' {}".format(temp_file)
cls.ssh(host, keyname, overwrite_root_keys, is_verbose, user=user)
remove_tempfile = 'rm -f {0}'.format(temp_file)
cls.ssh(host, keyname, remove_tempfile, is_verbose, user=user)
return
def get_optimal_spot_price(self, conn, instance_type, zone):
"""
Returns the spot price for an EC2 instance of the specified instance type.
The returned value is computed by averaging all the spot price history
values returned by the back-end EC2 APIs and incrementing the average by
extra 10%.
Args:
conn: A boto.EC2Connection that can be used to communicate with AWS.
instance_type: A str representing the instance type whose prices we
should speculate for.
zone: A str representing the availability zone that the instance will
be placed in.
Returns:
The estimated spot price for the specified instance type, in the
specified availability zone.
"""
end_time = datetime.datetime.now()
start_time = end_time - datetime.timedelta(days=7)
history = conn.get_spot_price_history(start_time=start_time.isoformat(),
end_time=end_time.isoformat(), product_description='Linux/UNIX',
instance_type=instance_type, availability_zone=zone)
var_sum = 0.0
for entry in history:
var_sum += entry.price
average = var_sum / len(history)
bid_price = average * 1.10
AppScaleLogger.log('The average spot instance price for a {0} machine is'\
' {1}, and 10% more is {2}'.format(instance_type, average, bid_price))
return bid_price
def create_user_accounts(cls, email, password, uaserver_host, keyname):
"""Registers two new user accounts with the UserAppServer.
One account is the standard account that users log in with (via their
e-mail address. The other is their XMPP account, so that they can log into
any jabber-compatible service and send XMPP messages to their application
(and receive them).
Args:
email: The e-mail address that should be registered for the user's
standard account.
password: The password that should be used for both the standard and XMPP
accounts.
uaserver_host: The location of a UserAppClient, that can create new user
accounts.
keyname: The name of the SSH keypair used for this AppScale deployment.
"""
uaserver = UserAppClient(uaserver_host, LocalState.get_secret_key(keyname))
# first, create the standard account
encrypted_pass = LocalState.encrypt_password(email, password)
uaserver.create_user(email, encrypted_pass)
# next, create the XMPP account. if the user's e-mail is [email protected], then that
# means their XMPP account name is a@login_ip
username_regex = re.compile('\A(.*)@')
username = username_regex.match(email).groups()[0]
xmpp_user = "******".format(username, LocalState.get_login_host(keyname))
xmpp_pass = LocalState.encrypt_password(xmpp_user, password)
uaserver.create_user(xmpp_user, xmpp_pass)
AppScaleLogger.log("Your XMPP username is {0}".format(xmpp_user))
def create_security_group(self, parameters, group):
"""Creates a new security group in AWS with the given name.
Args:
parameters: A dict that contains the credentials necessary to authenticate
with AWS.
group: A str that names the group that should be created.
Raises:
AgentRuntimeException: If the security group could not be created.
"""
AppScaleLogger.log('Creating security group: {0}'.format(group))
conn = self.open_connection(parameters)
retries_left = self.SECURITY_GROUP_RETRY_COUNT
while retries_left:
try:
conn.create_security_group(group, 'AppScale security group')
except EC2ResponseError:
pass
try:
conn.get_all_security_groups(group)
return
except EC2ResponseError:
pass
time.sleep(self.SLEEP_TIME)
retries_left -= 1
raise AgentRuntimeException("Couldn't create security group with " \
"name {0}".format(group))
def spawn_node_in_cloud(cls, options):
"""Starts a single virtual machine in a cloud infrastructure.
This method also prepares the virual machine for use by the AppScale Tools.
Specifically, it enables root logins on the machine, enables SSH access,
and copies the user's SSH key to that machine.
Args:
options: A Namespace that specifies the cloud infrastructure to use, as
well as how to interact with that cloud.
Returns:
The instance ID, public IP address, and private IP address of the machine
that was started.
"""
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
agent.configure_instance_security(params)
instance_ids, public_ips, private_ips = agent.run_instances(count=1,
parameters=params, security_configured=True)
AppScaleLogger.log("Please wait for your instance to boot up.")
cls.sleep_until_port_is_open(public_ips[0], cls.SSH_PORT, options.verbose)
cls.enable_root_login(public_ips[0], options.keyname,
options.infrastructure, options.verbose)
cls.copy_ssh_keys_to_node(public_ips[0], options.keyname, options.verbose)
return instance_ids[0], public_ips[0], private_ips[0]
def copy_deployment_credentials(cls, host, options):
"""Copies credentials needed to start the AppController and have it create
other instances (in cloud deployments).
Args:
host: A str representing the machine (reachable from this computer) to
copy our deployment credentials to.
options: A Namespace that indicates which SSH keypair to use, and whether
or not we are running in a cloud infrastructure.
"""
cls.scp(host, options.keyname, LocalState.get_secret_key_location(
options.keyname), '/etc/appscale/secret.key', options.verbose)
cls.scp(host, options.keyname, LocalState.get_key_path_from_name(
options.keyname), '/etc/appscale/ssh.key', options.verbose)
LocalState.generate_ssl_cert(options.keyname, options.verbose)
cls.scp(host, options.keyname, LocalState.get_certificate_location(
options.keyname), '/etc/appscale/certs/mycert.pem', options.verbose)
cls.scp(host, options.keyname, LocalState.get_private_key_location(
options.keyname), '/etc/appscale/certs/mykey.pem', options.verbose)
AppScaleLogger.log("Copying over deployment credentials")
cert = LocalState.get_certificate_location(options.keyname)
private_key = LocalState.get_private_key_location(options.keyname)
cls.ssh(host, options.keyname, 'mkdir -p /etc/appscale/keys/cloud1',
options.verbose)
cls.scp(host, options.keyname, cert, "/etc/appscale/keys/cloud1/mycert.pem",
options.verbose)
cls.scp(host, options.keyname, private_key,
"/etc/appscale/keys/cloud1/mykey.pem", options.verbose)
def run_with_timeout(self, timeout_time, default, num_retries, function, *args):
"""Runs the given function, aborting it if it runs too long.
Args:
timeout_time: The number of seconds that we should allow function to
execute for.
default: The value that should be returned if the timeout is exceeded.
num_retries: The number of times we should retry the SOAP call if we see
an unexpected exception.
function: The function that should be executed.
*args: The arguments that will be passed to function.
Returns:
Whatever function(*args) returns if it runs within the timeout window, and
default otherwise.
Raises:
AppControllerException: If the AppController we're trying to connect to is
not running at the given IP address, or if it rejects the SOAP request.
"""
def timeout_handler(_, __):
"""Raises a TimeoutException if the function we want to execute takes
too long to run.
Raises:
TimeoutException: If a SIGALRM is raised.
"""
raise TimeoutException()
signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(timeout_time) # trigger alarm in timeout_time seconds
try:
retval = function(*args)
except TimeoutException:
return default
except socket.error as exception:
signal.alarm(0) # turn off the alarm before we retry
if num_retries > 0:
AppScaleLogger.log(
"Saw exception {0} when communicating with the "
"AppController, retrying momentarily.".format(str(exception))
)
time.sleep(1)
return self.run_with_timeout(timeout_time, default, num_retries - 1, function, *args)
else:
raise exception
except ssl.SSLError:
# these are intermittent, so don't decrement our retry count for this
signal.alarm(0) # turn off the alarm before we retry
return self.run_with_timeout(timeout_time, default, num_retries, function, *args)
finally:
signal.alarm(0) # turn off the alarm
if retval == self.BAD_SECRET_MESSAGE:
raise AppControllerException(
"Could not authenticate successfully"
+ " to the AppController. You may need to change the keyname in use."
)
return retval
def generate_xmpp_username(cls, username, length=6, chars=ascii_lowercase + digits):
AppScaleLogger.log("Generating a new XMPP username...")
character_choices = []
while len(character_choices) < length:
character_choices.append(choice(chars))
generated_username = ''.join(character_choices)
return username + '_' + generated_username
def handle_failure(self, msg):
""" Log the specified error message and raise an AgentRuntimeException
Args:
msg: An error message to be logged and included in the raised exception.
Raises:
AgentRuntimeException Contains the input error message.
"""
AppScaleLogger.log(msg)
raise AgentRuntimeException(msg)
def set_admin_role(self, username):
"""Grants the given user the ability to perform any administrative action.
Args:
username: The e-mail address that should be given administrative
authorizations.
"""
AppScaleLogger.log('Granting admin privileges to %s' % username)
self.server.set_cloud_admin_status(username, 'true', self.secret)
self.server.set_capabilities(username, self.ADMIN_CAPABILITIES, self.secret)
def remove_app(cls, options):
"""Instructs AppScale to no longer host the named application.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if not options.confirm:
response = raw_input(
'Are you sure you want to remove this application? (y/N) ')
if response.lower() not in ['y', 'yes']:
raise AppScaleException("Cancelled application removal.")
login_host = LocalState.get_login_host(options.keyname)
secret = LocalState.get_secret_key(options.keyname)
acc = AppControllerClient(login_host, secret)
if not acc.is_app_running(options.appname):
raise AppScaleException("The given application is not currently running.")
# Makes a call to the AppController to get all the stats and looks
# through them for the http port the app can be reached on.
http_port = None
for _ in range(cls.MAX_RETRIES + 1):
result = acc.get_all_stats()
try:
json_result = json.loads(result)
apps_result = json_result['apps']
current_app = apps_result[options.appname]
http_port = current_app['http']
if http_port:
break
time.sleep(cls.SLEEP_TIME)
except (KeyError, ValueError):
AppScaleLogger.verbose("Got json error from get_all_data result.",
options.verbose)
time.sleep(cls.SLEEP_TIME)
if not http_port:
raise AppScaleException(
"Unable to get the serving port for the application.")
acc.stop_app(options.appname)
AppScaleLogger.log("Please wait for your app to shut down.")
for _ in range(cls.MAX_RETRIES + 1):
if RemoteHelper.is_port_open(login_host, http_port, options.verbose):
time.sleep(cls.SLEEP_TIME)
AppScaleLogger.log("Waiting for {0} to terminate...".format(
options.appname))
else:
AppScaleLogger.success("Done shutting down {0}.".format(
options.appname))
return
AppScaleLogger.warn("App {0} may still be running.".format(
options.appname))
def set_admin_role(self, username):
"""Grants the given user the ability to perform any administrative action.
Args:
username: The e-mail address that should be given administrative
authorizations.
"""
AppScaleLogger.log('Granting admin privileges to %s' % username)
self.server.set_cloud_admin_status(username, 'true', self.secret)
self.server.set_capabilities(username, self.ADMIN_CAPABILITIES,
self.secret)
def handle_failure(self, msg):
"""
Log the specified error message and raise an AgentRuntimeException
Args:
msg An error message to be logged and included in the raised exception
Raises:
AgentRuntimeException Contains the input error message
"""
AppScaleLogger.log(msg)
raise AgentRuntimeException(msg)
def set_admin_role(self, username, is_cloud_admin, capabilities):
""" Grants the given user the ability to perform any administrative action.
Args:
username: The e-mail address that should be given administrative
authorizations.
"""
AppScaleLogger.log('Granting admin privileges to %s' % username)
return self.run_with_timeout(self.DEFAULT_TIMEOUT,
'Set admin role request timed out.', self.DEFAULT_NUM_RETRIES,
self.server.set_admin_role, username, is_cloud_admin,
capabilities, self.secret)
def copy_deployment_credentials(cls, host, options):
"""Copies credentials needed to start the AppController and have it create
other instances (in cloud deployments).
Args:
host: A str representing the machine (reachable from this computer) to
copy our deployment credentials to.
options: A Namespace that indicates which SSH keypair to use, and whether
or not we are running in a cloud infrastructure.
"""
cls.scp(host, options.keyname, LocalState.get_secret_key_location(
options.keyname), '/etc/appscale/secret.key', options.verbose)
cls.scp(host, options.keyname, LocalState.get_key_path_from_name(
options.keyname), '/etc/appscale/ssh.key', options.verbose)
LocalState.generate_ssl_cert(options.keyname, options.verbose)
cls.scp(host, options.keyname, LocalState.get_certificate_location(
options.keyname), '/etc/appscale/certs/mycert.pem', options.verbose)
cls.scp(host, options.keyname, LocalState.get_private_key_location(
options.keyname), '/etc/appscale/certs/mykey.pem', options.verbose)
hash_id = subprocess.Popen(["openssl", "x509", "-hash", "-noout", "-in",
LocalState.get_certificate_location(options.keyname)],
stdout=subprocess.PIPE).communicate()[0]
cls.ssh(host, options.keyname,
'ln -fs /etc/appscale/certs/mycert.pem /etc/ssl/certs/{0}.0'.\
format(hash_id.rstrip()),
options.verbose)
AppScaleLogger.log("Copying over deployment credentials")
cert = LocalState.get_certificate_location(options.keyname)
private_key = LocalState.get_private_key_location(options.keyname)
cls.ssh(host, options.keyname, 'mkdir -p /etc/appscale/keys/cloud1',
options.verbose)
cls.scp(host, options.keyname, cert, "/etc/appscale/keys/cloud1/mycert.pem",
options.verbose)
cls.scp(host, options.keyname, private_key,
"/etc/appscale/keys/cloud1/mykey.pem", options.verbose)
# In Google Compute Engine, we also need to copy over our client_secrets
# file and the OAuth2 file that the user has approved for use with their
# credentials, otherwise the AppScale VMs won't be able to interact with
# GCE.
if options.infrastructure and options.infrastructure == 'gce':
if os.path.exists(LocalState.get_client_secrets_location( \
options.keyname)):
cls.scp(host, options.keyname, LocalState.get_client_secrets_location(
options.keyname), '/etc/appscale/client_secrets.json',
options.verbose)
cls.scp(host, options.keyname, LocalState.get_oauth2_storage_location(
options.keyname), '/etc/appscale/oauth2.dat', options.verbose)
def publish_api_list(cls, api_list, url, keyname):
eager = EagerClient(LocalState.get_login_host(keyname),
LocalState.get_secret_key(keyname))
temp_api_list = []
for api in api_list:
temp_api_list.append(api.to_dict())
result = eager.publish_api_list(temp_api_list, url)
if result['success']:
AppScaleLogger.log('{0} APIs published to API store.'.format(len(api_list)))
else:
AppScaleLogger.warn(result['reason'])
if result.get('detail'):
AppScaleLogger.warn(str(result['detail']))
def copy_deployment_credentials(cls, host, options):
"""Copies credentials needed to start the AppController and have it create
other instances (in cloud deployments).
Args:
host: A str representing the machine (reachable from this computer) to
copy our deployment credentials to.
options: A Namespace that indicates which SSH keypair to use, and whether
or not we are running in a cloud infrastructure.
"""
cls.scp(host, options.keyname,
LocalState.get_secret_key_location(options.keyname),
'/etc/appscale/secret.key', options.verbose)
cls.scp(host, options.keyname,
LocalState.get_key_path_from_name(options.keyname),
'/etc/appscale/ssh.key', options.verbose)
LocalState.generate_ssl_cert(options.keyname, options.verbose)
cls.scp(host, options.keyname,
LocalState.get_certificate_location(options.keyname),
'/etc/appscale/certs/mycert.pem', options.verbose)
cls.scp(host, options.keyname,
LocalState.get_private_key_location(options.keyname),
'/etc/appscale/certs/mykey.pem', options.verbose)
AppScaleLogger.log("Copying over deployment credentials")
cert = LocalState.get_certificate_location(options.keyname)
private_key = LocalState.get_private_key_location(options.keyname)
cls.ssh(host, options.keyname, 'mkdir -p /etc/appscale/keys/cloud1',
options.verbose)
cls.scp(host, options.keyname, cert,
"/etc/appscale/keys/cloud1/mycert.pem", options.verbose)
cls.scp(host, options.keyname, private_key,
"/etc/appscale/keys/cloud1/mykey.pem", options.verbose)
# In Google Compute Engine, we also need to copy over our client_secrets
# file and the OAuth2 file that the user has approved for use with their
# credentials, otherwise the AppScale VMs won't be able to interact with
# GCE.
if options.infrastructure and options.infrastructure == 'gce':
if os.path.exists(
LocalState.get_client_secrets_location(options.keyname)):
cls.scp(
host, options.keyname,
LocalState.get_client_secrets_location(options.keyname),
'/etc/appscale/client_secrets.json', options.verbose)
cls.scp(host, options.keyname,
LocalState.get_oauth2_storage_location(options.keyname),
'/etc/appscale/oauth2.dat', options.verbose)
def generate_crash_log(cls, exception, stacktrace):
"""Writes information to the local filesystem about an uncaught exception
that killed an AppScale Tool's execution, to aid in debugging at a later
time.
Args:
exception: The Exception that crashed executing an AppScale Tool, whose
information we want to log for debugging purposes.
stacktrace: A str that contains the newline-separated stacktrace
corresponding to the given exception.
Returns:
The location on the filesystem where the crash log was written to.
"""
crash_log_filename = '{0}log-{1}'.format(
LocalState.LOCAL_APPSCALE_PATH, uuid.uuid4())
try:
locale.setlocale(locale.LC_ALL, '')
this_locale = locale.getlocale()[0]
except locale.Error:
this_locale = "unknown"
log_info = {
# System-specific information
'platform' : platform.platform(),
'runtime' : platform.python_implementation(),
'locale' : this_locale,
# Crash-specific information
'exception' : exception.__class__.__name__,
'message' : str(exception),
'stacktrace' : stacktrace.rstrip(),
# AppScale Tools-specific information
'tools_version' : APPSCALE_VERSION
}
# If LOCAL_APPSCALE_PATH doesn't exist, create it so that we can write the
# crash log.
if not os.path.exists(LocalState.LOCAL_APPSCALE_PATH):
os.mkdir(LocalState.LOCAL_APPSCALE_PATH)
with open(crash_log_filename, 'w') as file_handle:
for key, value in log_info.iteritems():
file_handle.write("{0} : {1}\n\n".format(key, value))
AppScaleLogger.warn(str(exception))
AppScaleLogger.log("\nA log with more information is available " \
"at\n{0}.".format(crash_log_filename))
return crash_log_filename
def create_user(self, username, password, account_type='xmpp_user'):
"""Creates a new user account, with the given username and hashed password.
Args:
username: An e-mail address that should be set as the new username.
password: A sha1-hashed password that is bound to the given username.
account_type: A str that indicates if this account can be logged into by
XMPP users.
"""
AppScaleLogger.log("Creating new user account {0}".format(username))
result = self.server.commit_new_user(username, password, account_type,
self.secret)
if result != 'true':
raise Exception(result)
def terminate_instances(self, parameters):
"""
Stop one of more EC2 instances using. The input instance IDs are
fetched from the 'instance_ids' parameters in the input map. (Also
see documentation for the BaseAgent class)
Args:
parameters A dictionary of parameters
"""
instance_ids = parameters[self.PARAM_INSTANCE_IDS]
conn = self.open_connection(parameters)
terminated_instances = conn.terminate_instances(instance_ids)
for instance in terminated_instances:
AppScaleLogger.log('Instance {0} was terminated'.format(instance.id))
def generate_crash_log(cls, exception, stacktrace):
"""Writes information to the local filesystem about an uncaught exception
that killed an AppScale Tool's execution, to aid in debugging at a later
time.
Args:
exception: The Exception that crashed executing an AppScale Tool, whose
information we want to log for debugging purposes.
stacktrace: A str that contains the newline-separated stacktrace
corresponding to the given exception.
Returns:
The location on the filesystem where the crash log was written to.
"""
crash_log_filename = '{0}log-{1}'.format(
LocalState.LOCAL_APPSCALE_PATH, uuid.uuid4())
try:
locale.setlocale(locale.LC_ALL, '')
this_locale = locale.getlocale()[0]
except locale.Error:
this_locale = "unknown"
log_info = {
# System-specific information
'platform': platform.platform(),
'runtime': platform.python_implementation(),
'locale': this_locale,
# Crash-specific information
'exception': exception.__class__.__name__,
'message': str(exception),
'stacktrace': stacktrace.rstrip(),
# AppScale Tools-specific information
'tools_version': APPSCALE_VERSION
}
# If LOCAL_APPSCALE_PATH doesn't exist, create it so that we can write the
# crash log.
if not os.path.exists(LocalState.LOCAL_APPSCALE_PATH):
os.mkdir(LocalState.LOCAL_APPSCALE_PATH)
with open(crash_log_filename, 'w') as file_handle:
for key, value in log_info.iteritems():
file_handle.write("{0} : {1}\n\n".format(key, value))
AppScaleLogger.warn(str(exception))
AppScaleLogger.log("\nA log with more information is available " \
"at\n{0}.".format(crash_log_filename))
return crash_log_filename
def remove_app(cls, options):
"""Instructs AppScale to no longer host the named application.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if not options.confirm:
response = raw_input("Are you sure you want to remove this application? (y/N) ")
if response.lower() not in ["y", "yes"]:
raise AppScaleException("Cancelled application removal.")
login_host = LocalState.get_login_host(options.keyname)
secret = LocalState.get_secret_key(options.keyname)
acc = AppControllerClient(login_host, secret)
if not acc.is_app_running(options.appname):
raise AppScaleException("The given application is not currently running.")
# Makes a call to the AppController to get all the stats and looks
# through them for the http port the app can be reached on.
http_port = None
for _ in range(cls.MAX_RETRIES + 1):
result = acc.get_all_stats()
try:
json_result = json.loads(result)
apps_result = json_result["apps"]
current_app = apps_result[options.appname]
http_port = current_app["http"]
if http_port:
break
time.sleep(cls.SLEEP_TIME)
except (KeyError, ValueError):
AppScaleLogger.verbose("Got json error from get_all_data result.", options.verbose)
time.sleep(cls.SLEEP_TIME)
if not http_port:
raise AppScaleException("Unable to get the serving port for the application.")
acc.stop_app(options.appname)
AppScaleLogger.log("Please wait for your app to shut down.")
for _ in range(cls.MAX_RETRIES + 1):
if RemoteHelper.is_port_open(login_host, http_port, options.verbose):
time.sleep(cls.SLEEP_TIME)
AppScaleLogger.log("Waiting for {0} to terminate...".format(options.appname))
else:
AppScaleLogger.success("Done shutting down {0}.".format(options.appname))
return
AppScaleLogger.warn("App {0} may still be running.".format(options.appname))
def get_uaserver_host(self, is_verbose):
"""Queries the AppController to see which machine is hosting the
UserAppServer, and at what IP it can be reached.
Args:
is_verbose: A bool that indicates if we should print out the first
AppController's status when we query it.
Returns:
The IP address where a UserAppServer can be located (although it is not
guaranteed to be running).
Raises:
TimeoutException if MAX_RETRIES is attempted with no answer from
controller.
"""
last_known_state = None
retries = 0
while True:
try:
status = self.get_status()
AppScaleLogger.verbose('Received status from head node: ' + status,
is_verbose)
if status == self.BAD_SECRET_MESSAGE:
raise AppControllerException("Could not authenticate successfully" + \
" to the AppController. You may need to change the keyname in use.")
match = re.search(r'Database is at (.*)', status)
if match and match.group(1) != 'not-up-yet':
return match.group(1)
else:
match = re.search(r'Current State: (.*)', status)
if match:
if last_known_state != match.group(1):
last_known_state = match.group(1)
AppScaleLogger.log(last_known_state)
else:
AppScaleLogger.log('Waiting for AppScale nodes to complete '
'the initialization process')
except (AppControllerException, socket.error) as exception:
raise exception
except Exception as exception:
AppScaleLogger.warn('Saw {0}, waiting a few moments to try again' \
.format(str(exception)))
time.sleep(self.WAIT_TIME)
retries += 1
if retries >= self.MAX_RETRIES:
AppScaleLogger.warn("Too many retries to connect to UAServer.")
raise TimeoutException()
def spawn_nodes_in_cloud(cls, options, count=1):
"""Starts a single virtual machine in a cloud infrastructure.
This method also prepares the virual machine for use by the AppScale Tools.
Args:
options: A Namespace that specifies the cloud infrastructure to use, as
well as how to interact with that cloud.
count: A int, the number of instances to start.
Returns:
The instance ID, public IP address, and private IP address of the machine
that was started.
"""
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
# If we have running instances under the current keyname, we try to
# re-attach to them. If we have issue finding the locations file or the
# IP of the head node, we throw an exception.
login_ip = None
public_ips, private_ips, instance_ids = agent.describe_instances(
params)
if public_ips:
try:
login_ip = LocalState.get_login_host(options.keyname)
except (IOError, BadConfigurationException):
raise AppScaleException(
"Couldn't get login ip for running deployment with keyname"
" {}.".format(options.keyname))
if login_ip not in public_ips:
raise AppScaleException(
"Couldn't recognize running instances for deployment with"
" keyname {}.".format(options.keyname))
if login_ip in public_ips:
AppScaleLogger.log("Reusing already running instances.")
return instance_ids, public_ips, private_ips
agent.configure_instance_security(params)
instance_ids, public_ips, private_ips = agent.run_instances(
count=count, parameters=params, security_configured=True)
if options.static_ip:
agent.associate_static_ip(params, instance_ids[0],
options.static_ip)
public_ips[0] = options.static_ip
AppScaleLogger.log("Static IP associated with head node.")
return instance_ids, public_ips, private_ips
def enable_root_ssh(cls, options, public_ip):
"""Enables root logins and SSH access on the machine
and copies the user's SSH key to the head node. On the tools side this
should only be used for the "head node" since the server does this for all
other nodes.
Args:
options: A Namespace that specifies the cloud infrastructure to use, as
well as how to interact with that cloud.
public_ip: The IP address of the machine.
"""
AppScaleLogger.log("Enabling root ssh on {0}".format(public_ip))
cls.sleep_until_port_is_open(public_ip, cls.SSH_PORT, options.verbose)
cls.enable_root_login(public_ip, options.keyname,
options.infrastructure, options.verbose)
cls.copy_ssh_keys_to_node(public_ip, options.keyname, options.verbose)
def terminate_cloud_instance(cls, instance_id, options):
"""Powers off a single instance in the currently AppScale deployment and
cleans up secret key from the local filesystem.
Args:
instance_id: str containing the instance id.
options: namespace containing the run parameters.
"""
AppScaleLogger.log("About to terminate instance {0}"
.format(instance_id))
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
params['IS_VERBOSE'] = options.verbose
params[agent.PARAM_INSTANCE_IDS] = [instance_id]
agent.terminate_instances(params)
agent.cleanup_state(params)
os.remove(LocalState.get_secret_key_location(options.keyname))
def configure_instance_security(self, parameters):
""" Creates a GCE network and firewall with the specified name, and opens
the ports on that firewall as needed for AppScale.
We expect both the network and the firewall to not exist before this point,
to avoid accidentally placing AppScale instances from different deployments
in the same network and firewall (thus enabling them to see each other's web
traffic).
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine, and an additional key indicating the name of the
network and firewall that we should create in GCE.
Returns:
True, if the named network and firewall was created successfully.
Raises:
AgentRuntimeException: If the named network or firewall already exist in
GCE.
"""
AppScaleLogger.log("Verifying that SSH key exists locally")
keyname = parameters[self.PARAM_KEYNAME]
private_key = LocalState.LOCAL_APPSCALE_PATH + keyname
public_key = private_key + ".pub"
if os.path.exists(private_key) or os.path.exists(public_key):
raise AgentRuntimeException(
"SSH key already found locally - please " +
"use a different keyname")
LocalState.generate_rsa_key(keyname, parameters[self.PARAM_VERBOSE])
ssh_key_exists, all_ssh_keys = self.does_ssh_key_exist(parameters)
if not ssh_key_exists:
self.create_ssh_key(parameters, all_ssh_keys)
if self.does_network_exist(parameters):
raise AgentRuntimeException("Network already exists - please use a " + \
"different group name.")
if self.does_firewall_exist(parameters):
raise AgentRuntimeException("Firewall already exists - please use a " + \
"different group name.")
network_url = self.create_network(parameters)
self.create_firewall(parameters, network_url)
def does_image_exist(self, parameters):
"""
Queries Amazon EC2 to see if the specified image exists.
Args:
parameters A dict that contains the machine ID to check for existence.
Returns:
True if the machine ID exists, False otherwise.
"""
try:
conn = self.open_connection(parameters)
image_id = parameters[self.PARAM_IMAGE_ID]
conn.get_image(image_id)
AppScaleLogger.log('Machine image {0} does exist'.format(image_id))
return True
except boto.exception.EC2ResponseError:
AppScaleLogger.log('Machine image {0} does not exist'.format(image_id))
return False
def does_user_exist(self, username, silent=False):
"""Queries the UserAppServer to see if the given user exists.
Returns:
True if the given user exists, False otherwise.
"""
while 1:
try:
if self.server.does_user_exist(username, self.secret) == "true":
return True
else:
return False
except Exception, exception:
if not silent:
AppScaleLogger.log("Exception when checking if a user exists: {0}".\
format(exception))
AppScaleLogger.log("Backing off and trying again")
time.sleep(10)
def run_user_commands(cls, host, commands, keyname, is_verbose):
"""Runs any commands specified by the user before the AppController is
started.
Args:
host: A str representing the host to start the AppController on.
commands: A list of strs, where each str is a command that will be
executed on the remote machine.
keyname: A str representing the name of the SSH keypair that can log into
the specified host.
is_verbose: A bool that indicates if we should print the commands needed
to start the AppController to stdout.
"""
if commands:
AppScaleLogger.log(
"Running user-specified commands at {0}".format(host))
for command in commands:
cls.ssh(host, keyname, command, is_verbose)
def terminate_cloud_infrastructure(cls, keyname, is_verbose):
"""Powers off all machines in the currently running AppScale deployment.
Args:
keyname: The name of the SSH keypair used for this AppScale deployment.
is_verbose: A bool that indicates if we should print the commands executed
to stdout.
"""
AppScaleLogger.log(
"About to terminate instances spawned with keyname {0}".format(
keyname))
# This sleep is here to allow a moment for user to Ctrl-C
time.sleep(2)
# get all the instance IDs for machines in our deployment
agent = InfrastructureAgentFactory.create_agent(
LocalState.get_infrastructure(keyname))
params = agent.get_params_from_yaml(keyname)
params['IS_VERBOSE'] = is_verbose
# We want to terminate also the pending instances.
pending = True
_, _, instance_ids = agent.describe_instances(params, pending=pending)
# If using persistent disks, unmount them and detach them before we blow
# away the instances.
cls.terminate_virtualized_cluster(keyname, is_verbose)
nodes = LocalState.get_local_nodes_info(keyname)
for node in nodes:
if node.get('disk'):
AppScaleLogger.log("Unmounting persistent disk at {0}".format(
node['public_ip']))
cls.unmount_persistent_disk(node['public_ip'], keyname,
is_verbose)
agent.detach_disk(params, node['disk'], node['instance_id'])
# terminate all the machines
params[agent.PARAM_INSTANCE_IDS] = instance_ids
agent.terminate_instances(params)
# delete the keyname and group
agent.cleanup_state(params)
def spawn_node_in_cloud(cls, options):
"""Starts a single virtual machine in a cloud infrastructure.
This method also prepares the virual machine for use by the AppScale Tools.
Specifically, it enables root logins on the machine, enables SSH access,
and copies the user's SSH key to that machine.
Args:
options: A Namespace that specifies the cloud infrastructure to use, as
well as how to interact with that cloud.
Returns:
The instance ID, public IP address, and private IP address of the machine
that was started.
"""
agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
params = agent.get_params_from_args(options)
agent.configure_instance_security(params)
instance_ids, public_ips, private_ips = agent.run_instances(
count=1, parameters=params, security_configured=True)
if options.static_ip:
agent.associate_static_ip(params, instance_ids[0],
options.static_ip)
public_ips[0] = options.static_ip
AppScaleLogger.log("Please wait for your instance to boot up.")
cls.sleep_until_port_is_open(public_ips[0], cls.SSH_PORT,
options.verbose)
# Since GCE v1beta15, SSH keys don't immediately get injected to newly
# spawned VMs. It takes around 30 seconds, so sleep a bit longer to be
# sure.
if options.infrastructure == 'gce':
AppScaleLogger.log("Waiting for SSH keys to get injected to your "
"machine.")
time.sleep(60)
cls.enable_root_login(public_ips[0], options.keyname,
options.infrastructure, options.verbose)
cls.copy_ssh_keys_to_node(public_ips[0], options.keyname,
options.verbose)
return instance_ids[0], public_ips[0], private_ips[0]
def get_uaserver_host(self, is_verbose):
"""Queries the AppController to see which machine is hosting the
UserAppServer, and at what IP it can be reached.
Args:
is_verbose: A bool that indicates if we should print out the first
AppController's status when we query it.
Returns:
The IP address where a UserAppServer can be located (although it is not
guaranteed to be running).
"""
last_known_state = None
while True:
try:
status = self.get_status()
AppScaleLogger.verbose(
'Received status from head node: ' + status, is_verbose)
if status == self.BAD_SECRET_MESSAGE:
raise AppControllerException("Could not authenticate successfully" + \
" to the AppController. You may need to change the keyname in use.")
match = re.search(r'Database is at (.*)', status)
if match and match.group(1) != 'not-up-yet':
return match.group(1)
else:
match = re.search(r'Current State: (.*)', status)
if match:
if last_known_state != match.group(1):
last_known_state = match.group(1)
AppScaleLogger.log(last_known_state)
else:
AppScaleLogger.log(
'Waiting for AppScale nodes to complete '
'the initialization process')
except (AppControllerException, socket.error) as exception:
raise exception
except Exception as exception:
AppScaleLogger.warn('Saw {0}, waiting a few moments to try again' \
.format(str(exception)))
time.sleep(self.WAIT_TIME)
def does_zone_exist(self, parameters):
"""
Queries Eucalyptus to see if the specified availability zone exists.
Args:
parameters: A dict that contains the zone to check for existence.
Returns:
True if the availability zone exists, False otherwise.
"""
# Note that we can't use does_zone_exist in EC2Agent. There, if the image
# doesn't exist, it throws an EC2ResponseError, but in Eucalyptus, it
# doesn't (and returns None instead).
conn = self.open_connection(parameters)
zone = parameters[self.PARAM_ZONE]
if conn.get_all_zones(zone):
AppScaleLogger.log('Availability zone {0} does exist'.format(zone))
return True
else:
AppScaleLogger.log(
'Availability zone {0} does not exist'.format(zone))
return False
def cleanup_state(self, parameters):
"""
Removes the keyname and security group created during this AppScale
deployment.
Args:
parameters: A dict that contains the keyname and security group to delete.
"""
AppScaleLogger.log("Deleting keyname {0}".format(
parameters[self.PARAM_KEYNAME]))
conn = self.open_connection(parameters)
conn.delete_key_pair(parameters[self.PARAM_KEYNAME])
AppScaleLogger.log("Deleting security group {0}".format(
parameters[self.PARAM_GROUP]))
while True:
try:
conn.delete_security_group(parameters[self.PARAM_GROUP])
break
except EC2ResponseError:
time.sleep(5)
def reserve_app_id(self, username, app_id, app_language):
"""Tells the UserAppServer to reserve the given app_id for a particular
user.
Args:
username: A str representing the app administrator's e-mail address.
app_id: A str representing the application ID to reserve.
app_language: The runtime (Python 2.5/2.7, Java, or Go) that the app runs
over.
"""
result = self.server.commit_new_app(app_id, username, app_language,
self.secret)
if result == "true":
AppScaleLogger.log("We have reserved {0} for your app".format(app_id))
elif result == "Error: appname already exist":
AppScaleLogger.log("We are uploading a new version of your app.")
elif result == "Error: User not found":
raise AppScaleException("No information found about user {0}" \
.format(username))
else:
raise AppScaleException(result)
def does_image_exist(self, parameters):
"""
Queries Eucalyptus to see if the specified image exists.
Args:
parameters A dict that contains the machine ID to check for existence.
Returns:
True if the machine ID exists, False otherwise.
"""
# note that we can't use does_image_exist in EC2Agent. There, if the image
# doesn't exist, it throws an EC2ResponseError, but in Eucalyptus, it
# doesn't (and returns None instead).
conn = self.open_connection(parameters)
image_id = parameters[self.PARAM_IMAGE_ID]
if conn.get_image(image_id):
AppScaleLogger.log('Machine image {0} does exist'.format(image_id))
return True
else:
AppScaleLogger.log(
'Machine image {0} does not exist'.format(image_id))
return False
def terminate_instances(self, parameters):
"""
Terminate one of more EC2 instances. The input instance IDs are
fetched from the 'instance_ids' parameters in the input map. (Also
see documentation for the BaseAgent class)
Args:
parameters: A dictionary of parameters
"""
instance_ids = parameters[self.PARAM_INSTANCE_IDS]
conn = self.open_connection(parameters)
conn.terminate_instances(instance_ids)
AppScaleLogger.log('Terminating instances: '+' '.join(instance_ids))
if not self.wait_for_status_change(parameters, conn, 'terminated',\
max_wait_time=120):
AppScaleLogger.log("re-terminating instances: "+' '.join(instance_ids))
conn.terminate_instances(instance_ids)
if not self.wait_for_status_change(parameters, conn, 'terminated',\
max_wait_time=120):
self.handle_failure("ERROR: could not terminate instances: "+\
' '.join(instance_ids))
def reserve_application_name(self, username, application, language):
"""Tells the UserAppServer to reserve the given application ID and
register the given user as that application's administrator.
Args:
username: The e-mail address that should be set as the administrator for
the given application.
application: The application ID that should be reserved.
language: A str that indicates if the application is a Python 2.5 app,
a Java app, a Python 2.7 app, or a Go app.
Raises:
If the UserAppServer rejects the request to reserve the given
application ID.
"""
AppScaleLogger.log("Registering application name {0} (lang={1}) for " + \
"user {2}".format(application, language, username))
result = self.server.commit_new_app(application, username, language,
self.secret)
if result != 'true':
raise Exception(result)
def terminate_virtualized_cluster(cls, keyname, is_verbose):
"""Stops all API services running on all nodes in the currently running
AppScale deployment.
Args:
keyname: The name of the SSH keypair used for this AppScale deployment.
is_verbose: A bool that indicates if we should print the commands executed
to stdout.
"""
AppScaleLogger.log(
"Terminating appscale deployment with keyname {0}".format(keyname))
time.sleep(2)
shadow_host = LocalState.get_host_with_role(keyname, 'shadow')
try:
secret = LocalState.get_secret_key(keyname)
except IOError:
# We couldn't find the secret key: AppScale is most likely not
# running.
raise AppScaleException("Couldn't find AppScale secret key.")
acc = AppControllerClient(shadow_host, secret)
try:
all_ips = acc.get_all_public_ips()
except Exception as exception:
AppScaleLogger.warn(
'Saw Exception while getting deployments IPs {0}'.format(
str(exception)))
all_ips = LocalState.get_all_public_ips(keyname)
threads = []
for ip in all_ips:
thread = threading.Thread(target=cls.stop_remote_appcontroller,
args=(ip, keyname, is_verbose))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
boxes_shut_down = 0
is_running_regex = re.compile("appscale-controller stop")
for ip in all_ips:
AppScaleLogger.log(
"Shutting down AppScale API services at {0}".format(ip))
while True:
remote_output = cls.ssh(ip, keyname, 'ps x', is_verbose)
AppScaleLogger.verbose(remote_output, is_verbose)
if not is_running_regex.match(remote_output):
break
time.sleep(0.3)
boxes_shut_down += 1
if boxes_shut_down != len(all_ips):
raise AppScaleException(
"Couldn't terminate your AppScale deployment on"
" all machines - please do so manually.")
AppScaleLogger.log(
"Terminated AppScale on {0} machines.".format(boxes_shut_down))
def start_remote_appcontroller(cls, host, keyname, is_verbose):
"""Starts the AppController daemon on the specified host.
Args:
host: A str representing the host to start the AppController on.
keyname: A str representing the name of the SSH keypair that can log into
the specified host.
is_verbose: A bool that indicates if we should print the commands needed
to start the AppController to stdout.
"""
AppScaleLogger.log("Starting AppController at {0}".format(host))
# remove any possible appcontroller state that may not have been
# properly removed in virtualized clusters
cls.ssh(host, keyname, 'rm -rf /etc/appscale/appcontroller-state.json',
is_verbose)
# Remove any monit configuration files from previous AppScale deployments.
cls.ssh(host, keyname, 'rm -rf /etc/monit/conf.d/appscale-*.cfg',
is_verbose)
# Copy over the config file that indicates how the AppController should be
# started up.
cls.scp(host, keyname, cls.MONIT_APPCONTROLLER_CONFIG_FILE,
'/etc/monit/conf.d/appscale-controller-17443.cfg', is_verbose)
# Start up monit.
cls.ssh(host, keyname, 'monit quit; ', is_verbose)
cls.ssh(host, keyname, 'service monit start', is_verbose)
time.sleep(1)
# Start the AppController.
cls.ssh(host, keyname, 'monit start -g controller', is_verbose)
time.sleep(1)
AppScaleLogger.log("Please wait for the AppController to finish " + \
"pre-processing tasks.")
cls.sleep_until_port_is_open(host, AppControllerClient.PORT,
is_verbose)
def reserve_app_id(self, username, app_id, app_language):
""" Tells the AppController to reserve the given app_id for a particular
user.
Args:
username: A str representing the app administrator's e-mail address.
app_id: A str representing the application ID to reserve.
app_language: The runtime (Python 2.5/2.7, Java, or Go) that the app runs
over.
"""
result = self.run_with_timeout(self.DEFAULT_TIMEOUT,
'Reserve app id request timed out.',
self.DEFAULT_NUM_RETRIES,
self.server.reserve_app_id, username,
app_id, app_language, self.secret)
if result == "true":
AppScaleLogger.log(
"We have reserved {0} for your app".format(app_id))
elif result == "Error: appname already exists":
AppScaleLogger.log("We are uploading a new version of your app.")
elif result == "Error: User not found":
raise AppScaleException(
"No information found about user {0}".format(username))
else:
AppScaleLogger.log("Result {0}".format(result))
raise AppScaleException(result)
def copy_app_to_host(cls, app_location, keyname, is_verbose):
"""Copies the given application to a machine running the Login service within
an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
AppScaleLogger.log("Creating remote directory to copy app into")
app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
remote_app_dir = "/var/apps/{0}/app".format(app_id)
cls.ssh(LocalState.get_login_host(keyname), keyname,
'mkdir -p {0}'.format(remote_app_dir), is_verbose)
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "/tmp/appscale-app-{0}-{1}.tar.gz".format(app_id, rand)
LocalState.shell("cd {0} && tar -czf {1} *".format(app_location,
local_tarred_app), is_verbose)
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(remote_app_dir, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def enable_root_login(cls, host, keyname, infrastructure, is_verbose):
"""Logs into the named host and alters its ssh configuration to enable the
root user to directly log in.
Args:
host: A str representing the host to enable root logins on.
keyname: A str representing the name of the SSH keypair to login with.
infrastructure: A str representing the name of the cloud infrastructure
we're running on.
is_verbose: A bool indicating if we should print the command we execute to
enable root login to stdout.
"""
# First, see if we need to enable root login at all (some VMs have it
# already enabled).
output = cls.ssh(host, keyname, 'ls', is_verbose, user='******',
num_retries=1)
if re.search(cls.LOGIN_AS_UBUNTU_USER, output):
AppScaleLogger.log("Root login not enabled - enabling it now.")
cls.ssh(host, keyname, 'sudo cp ~/.ssh/authorized_keys /root/.ssh/',
is_verbose, user='******')
else:
AppScaleLogger.log("Root login already enabled - not re-enabling it.")
def add_instances(cls, options):
"""Adds additional machines to an AppScale deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if 'master' in options.ips.keys():
raise BadConfigurationException("Cannot add master nodes to an " + \
"already running AppScale deployment.")
# Skip checking for -n (replication) because we don't allow the user
# to specify it here (only allowed in run-instances).
additional_nodes_layout = NodeLayout(options)
# In virtualized cluster deployments, we need to make sure that the user
# has already set up SSH keys.
if LocalState.get_from_yaml(options.keyname,
'infrastructure') == "xen":
ips_to_check = []
for ip_group in options.ips.values():
ips_to_check.extend(ip_group)
for ip in ips_to_check:
# throws a ShellException if the SSH key doesn't work
RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)
# Finally, find an AppController and send it a message to add
# the given nodes with the new roles.
AppScaleLogger.log("Sending request to add instances")
login_ip = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_ip,
LocalState.get_secret_key(options.keyname))
acc.start_roles_on_nodes(json.dumps(options.ips))
# TODO(cgb): Should we wait for the new instances to come up and get
# initialized?
AppScaleLogger.success("Successfully sent request to add instances " + \
"to this AppScale deployment.")
def get_optimal_spot_price(self, conn, instance_type):
"""
Returns the spot price for an EC2 instance of the specified instance type.
The returned value is computed by averaging all the spot price history
values returned by the back-end EC2 APIs and incrementing the average by
extra 10%.
Args:
instance_type An EC2 instance type
Returns:
The estimated spot price for the specified instance type
"""
history = conn.get_spot_price_history(product_description='Linux/UNIX',
instance_type=instance_type)
var_sum = 0.0
for entry in history:
var_sum += entry.price
average = var_sum / len(history)
bid_price = average * 1.10
AppScaleLogger.log('The average spot instance price for a {0} machine is'\
' {1}, and 10% more is {2}'.format(instance_type, average, bid_price))
return bid_price
