def rsync_files(cls, host, keyname, local_appscale_dir, is_verbose):
"""Copies over an AppScale source directory from this machine to the
specified host.
Args:
host: A str representing a host that should be accessible from this
machine.
keyname: A str representing the name of the SSH keypair that can log into
the specified machine.
local_appscale_dir: A str representing the path on the local filesystem
where the AppScale source to copy over can be found.
is_verbose: A bool that indicates if we should print the rsync commands
we exec to stdout.
Raises:
BadConfigurationException: If local_appscale_dir does not exist locally,
or if any of the standard AppScale module folders do not exist.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
local_path = os.path.expanduser(local_appscale_dir)
if not os.path.exists(local_path):
raise BadConfigurationException("The location you specified to copy " \
"from, {0}, doesn't exist.".format(local_path))
LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv "
"--exclude='AppDB/logs/*' " \
"--exclude='AppDB/cassandra/cassandra/*' " \
"{2}/* root@{3}:/root/appscale/".format(ssh_key, cls.SSH_OPTIONS,
local_path, host), is_verbose)
def describe_instances(cls, options):
"""Queries each node in the currently running AppScale deployment and
reports on their status.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
login_host = LocalState.get_login_host(options.keyname)
login_acc = AppControllerClient(
login_host, LocalState.get_secret_key(options.keyname))
for ip in login_acc.get_all_public_ips():
acc = AppControllerClient(
ip, LocalState.get_secret_key(options.keyname))
AppScaleLogger.log("Status of node at {0}:".format(ip))
try:
AppScaleLogger.log(acc.get_status())
except Exception as exception:
AppScaleLogger.warn("Unable to contact machine: {0}\n".format(
str(exception)))
AppScaleLogger.success("View status information about your AppScale " + \
"deployment at http://{0}:{1}/status".format(login_host,
RemoteHelper.APP_DASHBOARD_PORT))
def terminate_virtualized_cluster(cls, keyname, is_verbose):
"""Stops all API services running on all nodes in the currently running
AppScale deployment.
Args:
keyname: The name of the SSH keypair used for this AppScale deployment.
is_verbose: A bool that indicates if we should print the commands executed
to stdout.
"""
AppScaleLogger.log(
"Terminating appscale deployment with keyname {0}".format(keyname))
time.sleep(2)
shadow_host = LocalState.get_host_with_role(keyname, 'shadow')
try:
secret = LocalState.get_secret_key(keyname)
except IOError:
# We couldn't find the secret key: AppScale is most likely not
# running.
raise AppScaleException("Couldn't find AppScale secret key.")
acc = AppControllerClient(shadow_host, secret)
try:
all_ips = acc.get_all_public_ips()
except Exception as exception:
AppScaleLogger.warn(
'Saw Exception while getting deployments IPs {0}'.format(
str(exception)))
all_ips = LocalState.get_all_public_ips(keyname)
threads = []
for ip in all_ips:
thread = threading.Thread(target=cls.stop_remote_appcontroller,
args=(ip, keyname, is_verbose))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
boxes_shut_down = 0
is_running_regex = re.compile("appscale-controller stop")
for ip in all_ips:
AppScaleLogger.log(
"Shutting down AppScale API services at {0}".format(ip))
while True:
remote_output = cls.ssh(ip, keyname, 'ps x', is_verbose)
AppScaleLogger.verbose(remote_output, is_verbose)
if not is_running_regex.match(remote_output):
break
time.sleep(0.3)
boxes_shut_down += 1
if boxes_shut_down != len(all_ips):
raise AppScaleException(
"Couldn't terminate your AppScale deployment on"
" all machines - please do so manually.")
AppScaleLogger.log(
"Terminated AppScale on {0} machines.".format(boxes_shut_down))
def remove_app(cls, options):
"""Instructs AppScale to no longer host the named application.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if not options.confirm:
response = raw_input("Are you sure you want to remove this " + \
"application? (Y/N) ")
if response not in ['y', 'yes', 'Y', 'YES']:
raise AppScaleException("Cancelled application removal.")
login_host = LocalState.get_login_host(options.keyname)
secret = LocalState.get_secret_key(options.keyname)
acc = AppControllerClient(login_host, secret)
if not acc.does_app_exist(options.appname):
raise AppScaleException(
"The given application is not currently running.")
acc.stop_app(options.appname)
AppScaleLogger.log("Please wait for your app to shut down.")
while True:
if acc.is_app_running(options.appname):
time.sleep(cls.SLEEP_TIME)
else:
break
AppScaleLogger.success("Done shutting down {0}".format(
options.appname))
def copy_app_to_host(cls, app_location, keyname, is_verbose):
"""Copies the given application to a machine running the Login service
within an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "{0}/appscale-app-{1}-{2}.tar.gz".format(
tempfile.gettempdir(), app_id, rand)
LocalState.shell(
"cd '{0}' && COPYFILE_DISABLE=1 tar -czhf {1} --exclude='*.pyc' *".
format(app_location, local_tarred_app), is_verbose)
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(cls.REMOTE_APP_DIR, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def terminate_instances(cls, options):
"""Stops all services running in an AppScale deployment, and in cloud
deployments, also powers off the instances previously spawned.
Raises:
AppScaleException: If AppScale is not running, and thus can't be
terminated.
"""
if not os.path.exists(
LocalState.get_locations_yaml_location(options.keyname)):
raise AppScaleException(
"AppScale is not running with the keyname {0}".format(
options.keyname))
if LocalState.get_infrastructure(options.keyname) in \
InfrastructureAgentFactory.VALID_AGENTS:
RemoteHelper.terminate_cloud_infrastructure(
options.keyname, options.verbose)
else:
RemoteHelper.terminate_virtualized_cluster(options.keyname,
options.verbose)
LocalState.cleanup_appscale_files(options.keyname)
AppScaleLogger.success(
"Successfully shut down your AppScale deployment.")
def rsync_files(cls, host, keyname, local_appscale_dir, is_verbose):
"""Copies over an AppScale source directory from this machine to the
specified host.
Args:
host: A str representing a host that should be accessible from this
machine.
keyname: A str representing the name of the SSH keypair that can log into
the specified machine.
local_appscale_dir: A str representing the path on the local filesystem
where the AppScale source to copy over can be found.
is_verbose: A bool that indicates if we should print the rsync commands
we exec to stdout.
Raises:
BadConfigurationException: If local_appscale_dir does not exist locally,
or if any of the standard AppScale module folders do not exist.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
local_path = os.path.expanduser(local_appscale_dir)
if not os.path.exists(local_path):
raise BadConfigurationException("The location you specified to copy " \
"from, {0}, doesn't exist.".format(local_path))
LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv "
"--exclude='AppDB/logs/*' " \
"--exclude='AppDB/cassandra/cassandra/*' " \
"{2}/* root@{3}:/root/appscale/".format(ssh_key, cls.SSH_OPTIONS,
local_path, host), is_verbose)
def scp_remote_to_local(cls,
host,
keyname,
source,
dest,
is_verbose,
user='******'):
"""Securely copies a file from a remote machine to this machine.
Args:
host: A str representing the machine that we should log into.
keyname: A str representing the name of the SSH keypair to log in with.
source: A str representing the path on the remote machine where the
file should be copied from.
dest: A str representing the path on the local machine where the file
should be copied to.
is_verbose: A bool that indicates if we should print the scp command to
stdout.
user: A str representing the user to log in as.
Returns:
A str representing the standard output of the secure copy and a str
representing the standard error of the secure copy.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
return LocalState.shell(
"scp -r -i {0} {1} {2}@{3}:{4} {5}".format(ssh_key,
cls.SSH_OPTIONS, user,
host, source, dest),
is_verbose)
def testCleanInClusterDeployment(self):
# calling 'appscale clean' in a cluster deployment should ssh into each of
# the boxes specified in the ips_layout and run the terminate script
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'ips_layout' : {
'controller': 'public1',
'servers': ['public2', 'public3']
},
'test' : True
}
yaml_dumped_contents = yaml.dump(contents)
flexmock(RemoteHelper)
RemoteHelper.should_receive('ssh') \
.with_args(re.compile('public[123]'), 'appscale', str, False)
flexmock(LocalState)
LocalState.should_receive('cleanup_appscale_files').with_args('appscale')
appscale = AppScale()
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
expected = ['public1', 'public2', 'public3']
self.assertEquals(expected, appscale.clean())
def test_appscale_in_two_node_virt_deployment(self):
# pretend that the place we're going to put logs into doesn't exist
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(
False)
# and mock out the mkdir operation
flexmock(os)
os.should_receive('mkdir').with_args('/tmp/foobaz').and_return()
# next, mock out finding the login ip address
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps([{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]))
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# and slip in a fake appcontroller to report on the two IP addrs
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args(
'the secret').and_return(json.dumps(['public1', 'public2']))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# fake the creation of the log directories locally
os.should_receive('mkdir').with_args(
'/tmp/foobaz/public1').and_return()
os.should_receive('mkdir').with_args(
'/tmp/foobaz/public2').and_return()
# finally, fake the copying of the log files
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"]
options = ParseArgs(argv, self.function).args
AppScaleTools.gather_logs(options)
def test_ensure_appscale_isnt_running_but_it_is_w_force(self):
# if there is a locations.yaml file and force is set,
# we shouldn't abort
os.path.should_receive('exists').with_args(self.locations_yaml) \
.and_return(True)
LocalState.ensure_appscale_isnt_running(self.keyname, True)
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {0}, already exists.".format(options.location))
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
for ip in acc.get_all_public_ips():
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
RemoteHelper.scp_remote_to_local(ip, options.keyname,
'/var/log/appscale', local_dir,
options.verbose)
AppScaleLogger.success("Successfully copied logs to {0}".format(
options.location))
def ssh(cls,
host,
keyname,
command,
is_verbose,
user='******',
num_retries=LocalState.DEFAULT_NUM_RETRIES):
"""Logs into the named host and executes the given command.
Args:
host: A str representing the machine that we should log into.
keyname: A str representing the name of the SSH keypair to log in with.
command: A str representing what to execute on the remote host.
is_verbose: A bool indicating if we should print the ssh command to
stdout.
user: A str representing the user to log in as.
Returns:
A str representing the standard output of the remote command and a str
representing the standard error of the remote command.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
return LocalState.shell(
"ssh -F /dev/null -i {0} {1} {2}@{3} bash".format(
ssh_key, cls.SSH_OPTIONS, user, host),
is_verbose,
num_retries,
stdin=command)
def terminate_instances(cls, options):
"""Stops all services running in an AppScale deployment, and in cloud
deployments, also powers off the instances previously spawned.
Raises:
AppScaleException: If AppScale is not running, and thus can't be
terminated.
"""
try:
infrastructure = LocalState.get_infrastructure(options.keyname)
except IOError:
raise AppScaleException("Cannot find AppScale's configuration for keyname {0}".format(options.keyname))
if infrastructure == "xen" and options.terminate:
raise AppScaleException("Terminate option is invalid for cluster mode.")
if infrastructure == "xen" or not options.terminate:
# We are in cluster mode: let's check if AppScale is running.
if not os.path.exists(LocalState.get_secret_key_location(options.keyname)):
raise AppScaleException("AppScale is not running with the keyname {0}".format(options.keyname))
# Stop gracefully the AppScale deployment.
try:
RemoteHelper.terminate_virtualized_cluster(options.keyname, options.verbose)
except (IOError, AppScaleException):
# Don't fail if we cannot find the configuration.
pass
# And if we are on a cloud infrastructure, terminate instances if
# asked.
if infrastructure in InfrastructureAgentFactory.VALID_AGENTS and options.terminate:
RemoteHelper.terminate_cloud_infrastructure(options.keyname, options.verbose)
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {0}, already exists.".format(options.location))
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
for ip in acc.get_all_public_ips():
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/appscale',
local_dir, options.verbose)
AppScaleLogger.success("Successfully copied logs to {0}".format(
options.location))
def add_instances(cls, options):
"""Adds additional machines to an AppScale deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if 'master' in options.ips.keys():
raise BadConfigurationException("Cannot add master nodes to an " + \
"already running AppScale deployment.")
# Skip checking for -n (replication) because we don't allow the user
# to specify it here (only allowed in run-instances).
additional_nodes_layout = NodeLayout(options)
# In virtualized cluster deployments, we need to make sure that the user
# has already set up SSH keys.
if LocalState.get_from_yaml(options.keyname, 'infrastructure') == "xen":
for ip in options.ips.values():
# throws a ShellException if the SSH key doesn't work
RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)
# Finally, find an AppController and send it a message to add
# the given nodes with the new roles.
AppScaleLogger.log("Sending request to add instances")
login_ip = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_ip, LocalState.get_secret_key(
options.keyname))
acc.start_roles_on_nodes(json.dumps(options.ips))
# TODO(cgb): Should we wait for the new instances to come up and get
# initialized?
AppScaleLogger.success("Successfully sent request to add instances " + \
"to this AppScale deployment.")
def remove_app(cls, options):
"""Instructs AppScale to no longer host the named application.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if not options.confirm:
response = raw_input("Are you sure you want to remove this " + \
"application? (Y/N) ")
if response not in ['y', 'yes', 'Y', 'YES']:
raise AppScaleException("Cancelled application removal.")
login_host = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_host, LocalState.get_secret_key(
options.keyname))
userappserver_host = acc.get_uaserver_host(options.verbose)
userappclient = UserAppClient(userappserver_host, LocalState.get_secret_key(
options.keyname))
if not userappclient.does_app_exist(options.appname):
raise AppScaleException("The given application is not currently running.")
acc.stop_app(options.appname)
AppScaleLogger.log("Please wait for your app to shut down.")
while True:
if acc.is_app_running(options.appname):
time.sleep(cls.SLEEP_TIME)
else:
break
AppScaleLogger.success("Done shutting down {0}".format(options.appname))
def relocate_app(cls, options):
"""Instructs AppScale to move the named application to a different port.
Args:
options: A Namespace that has fields for each parameter that can be passed
in via the command-line interface.
Raises:
AppScaleException: If the named application isn't running in this AppScale
cloud, if the destination port is in use by a different application, or
if the AppController rejects the request to relocate the application (in
which case it includes the reason why the rejection occurred).
"""
login_host = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_host, LocalState.get_secret_key(
options.keyname))
app_info_map = acc.get_app_info_map()
if options.appname not in app_info_map.keys():
raise AppScaleException("The given application, {0}, is not currently " \
"running in this AppScale cloud, so we can't move it to a different " \
"port.".format(options.appname))
relocate_result = acc.relocate_app(options.appname, options.http_port,
options.https_port)
if relocate_result == "OK":
AppScaleLogger.success("Successfully issued request to move {0} to " \
"ports {1} and {2}.".format(options.appname, options.http_port,
options.https_port))
AppScaleLogger.success("Your app serves unencrypted traffic at: " +
"http://{0}:{1}".format(login_host, options.http_port))
AppScaleLogger.success("Your app serves encrypted traffic at: " +
"https://{0}:{1}".format(login_host, options.https_port))
else:
raise AppScaleException(relocate_result)
def copy_app_to_host(cls, app_location, keyname, is_verbose):
"""Copies the given application to a machine running the Login service
within an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "{0}/appscale-app-{1}-{2}.tar.gz".format(tempfile.gettempdir(),
app_id, rand)
LocalState.shell("cd '{0}' && COPYFILE_DISABLE=1 tar -czhf {1} --exclude='*.pyc' *".format(
app_location, local_tarred_app), is_verbose)
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(cls.REMOTE_APP_DIR, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def copy_local_metadata(cls, host, keyname, is_verbose):
"""Copies the locations.yaml and locations.json files found locally (which
contain metadata about this AppScale deployment) to the specified host.
Args:
host: The machine that we should copy the metadata files to.
keyname: The name of the SSH keypair that we can use to log into the given
host.
is_verbose: A bool that indicates if we should print the SCP commands we
exec to stdout.
"""
# copy the metadata files for AppScale itself to use
cls.scp(host, keyname, LocalState.get_locations_yaml_location(keyname),
'/etc/appscale/locations-{0}.yaml'.format(keyname), is_verbose)
cls.scp(host, keyname, LocalState.get_locations_json_location(keyname),
'/etc/appscale/locations-{0}.json'.format(keyname), is_verbose)
# and copy the json file if the tools on that box wants to use it
cls.scp(host, keyname, LocalState.get_locations_json_location(keyname),
'/root/.appscale/locations-{0}.json'.format(keyname),
is_verbose)
# and copy the secret file if the tools on that box wants to use it
cls.scp(host, keyname, LocalState.get_secret_key_location(keyname),
'/root/.appscale/', is_verbose)
def get_params_from_yaml(self, keyname):
"""Searches through the locations.yaml file to build a dict containing the
parameters necessary to interact with Amazon EC2.
Args:
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
"""
params = {
self.PARAM_CREDENTIALS : {},
self.PARAM_GROUP : LocalState.get_group(keyname),
self.PARAM_KEYNAME : keyname
}
zone = LocalState.get_zone(keyname)
if zone:
params[self.PARAM_REGION] = zone[:-1]
else:
params[self.PARAM_REGION] = self.DEFAULT_REGION
for credential in self.REQUIRED_CREDENTIALS:
if os.environ.get(credential):
params[self.PARAM_CREDENTIALS][credential] = os.environ[credential]
else:
raise AgentConfigurationException("no " + credential)
return params
def test_ensure_appscale_isnt_running_but_it_is_w_force(self):
# if there is a locations.yaml file and force is set,
# we shouldn't abort
os.path.should_receive('exists').with_args(self.locations_yaml) \
.and_return(True)
LocalState.ensure_appscale_isnt_running(self.keyname, True)
def testCleanInClusterDeployment(self):
# calling 'appscale clean' in a cluster deployment should ssh into each of
# the boxes specified in the ips_layout and run the terminate script
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'ips_layout' : {
'controller': 'public1',
'servers': ['public2', 'public3']
},
'test' : True
}
yaml_dumped_contents = yaml.dump(contents)
flexmock(RemoteHelper)
RemoteHelper.should_receive('ssh') \
.with_args(re.compile('public[123]'), 'appscale', str, False)
flexmock(LocalState)
LocalState.should_receive('cleanup_appscale_files').with_args('appscale')
appscale = AppScale()
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
expected = ['public1', 'public2', 'public3']
self.assertEquals(expected, appscale.clean())
def rsync_files(cls, host, keyname, local_appscale_dir, is_verbose):
"""Copies over an AppScale source directory from this machine to the
specified host.
Args:
host: A str representing a host that should be accessible from this
machine.
keyname: A str representing the name of the SSH keypair that can log into
the specified machine.
local_appscale_dir: A str representing the path on the local filesystem
where the AppScale source to copy over can be found.
is_verbose: A bool that indicates if we should print the rsync commands
we exec to stdout.
Raises:
BadConfigurationException: If local_appscale_dir does not exist locally,
or if any of the standard AppScale module folders do not exist.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
appscale_dirs = ["lib", "AppController", "AppManager", "AppServer",
"AppLoadBalancer", "AppMonitoring", "Neptune", "InfrastructureManager"]
for dir_name in appscale_dirs:
local_path = os.path.expanduser(local_appscale_dir) + os.sep + dir_name
if not os.path.exists(local_path):
raise BadConfigurationException("The location you specified to copy " +
"from, {0}, doesn't contain a {1} folder.".format(local_appscale_dir,
local_path))
LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv {2}/* root@{3}:/root/appscale/{4}" \
.format(ssh_key, cls.SSH_OPTIONS, local_path, host, dir_name), is_verbose)
# Rsync AppDB separately, as it has a lot of paths we may need to exclude
# (e.g., built database binaries).
local_app_db = os.path.expanduser(local_appscale_dir) + os.sep + "AppDB/*"
LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv --exclude='logs/*' --exclude='hadoop-*' --exclude='hbase/hbase-*' --exclude='voldemort/voldemort/*' --exclude='cassandra/cassandra/*' {2} root@{3}:/root/appscale/AppDB".format(ssh_key, cls.SSH_OPTIONS, local_app_db, host), is_verbose)
def create_user_accounts(cls, email, password, uaserver_host, keyname):
"""Registers two new user accounts with the UserAppServer.
One account is the standard account that users log in with (via their
e-mail address. The other is their XMPP account, so that they can log into
any jabber-compatible service and send XMPP messages to their application
(and receive them).
Args:
email: The e-mail address that should be registered for the user's
standard account.
password: The password that should be used for both the standard and XMPP
accounts.
uaserver_host: The location of a UserAppClient, that can create new user
accounts.
keyname: The name of the SSH keypair used for this AppScale deployment.
"""
uaserver = UserAppClient(uaserver_host, LocalState.get_secret_key(keyname))
# first, create the standard account
encrypted_pass = LocalState.encrypt_password(email, password)
uaserver.create_user(email, encrypted_pass)
# next, create the XMPP account. if the user's e-mail is [email protected], then that
# means their XMPP account name is a@login_ip
username_regex = re.compile('\A(.*)@')
username = username_regex.match(email).groups()[0]
xmpp_user = "******".format(username, LocalState.get_login_host(keyname))
xmpp_pass = LocalState.encrypt_password(xmpp_user, password)
uaserver.create_user(xmpp_user, xmpp_pass)
AppScaleLogger.log("Your XMPP username is {0}".format(xmpp_user))
def add_instances(cls, options):
"""Adds additional machines to an AppScale deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if 'master' in options.ips.keys():
raise BadConfigurationException("Cannot add master nodes to an " + \
"already running AppScale deployment.")
# Skip checking for -n (replication) because we don't allow the user
# to specify it here (only allowed in run-instances).
additional_nodes_layout = NodeLayout(options)
# In virtualized cluster deployments, we need to make sure that the user
# has already set up SSH keys.
if LocalState.get_from_yaml(options.keyname,
'infrastructure') == "xen":
for ip in options.ips.values():
# throws a ShellException if the SSH key doesn't work
RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)
# Finally, find an AppController and send it a message to add
# the given nodes with the new roles.
AppScaleLogger.log("Sending request to add instances")
login_ip = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_ip,
LocalState.get_secret_key(options.keyname))
acc.start_roles_on_nodes(json.dumps(options.ips))
# TODO(cgb): Should we wait for the new instances to come up and get
# initialized?
AppScaleLogger.success("Successfully sent request to add instances " + \
"to this AppScale deployment.")
def copy_app_to_host(cls, app_location, keyname, is_verbose):
"""Copies the given application to a machine running the Login service within
an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
AppScaleLogger.log("Creating remote directory to copy app into")
app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
remote_app_dir = "/var/apps/{0}/app".format(app_id)
cls.ssh(LocalState.get_login_host(keyname), keyname,
'mkdir -p {0}'.format(remote_app_dir), is_verbose)
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "/tmp/appscale-app-{0}-{1}.tar.gz".format(app_id, rand)
LocalState.shell("cd {0} && tar -czf {1} *".format(app_location,
local_tarred_app), is_verbose)
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(remote_app_dir, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def test_appscale_in_two_node_virt_deployment(self):
# pretend that the place we're going to put logs into doesn't exist
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(False)
# and mock out the mkdir operation
flexmock(os)
os.should_receive('mkdir').with_args('/tmp/foobaz').and_return()
# next, mock out finding the login ip address
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(json.dumps([{
"public_ip" : "public1",
"private_ip" : "private1",
"jobs" : ["shadow", "login"]
}]))
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# and slip in a fake appcontroller to report on the two IP addrs
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args(
'the secret').and_return(json.dumps(['public1', 'public2']))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# fake the creation of the log directories locally
os.should_receive('mkdir').with_args('/tmp/foobaz/public1').and_return()
os.should_receive('mkdir').with_args('/tmp/foobaz/public2').and_return()
# finally, fake the copying of the log files
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
argv = [
"--keyname", self.keyname,
"--location", "/tmp/foobaz"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.gather_logs(options)
def terminate_virtualized_cluster(cls, keyname, is_verbose):
"""Stops all API services running on all nodes in the currently running
AppScale deployment.
Args:
keyname: The name of the SSH keypair used for this AppScale deployment.
is_verbose: A bool that indicates if we should print the commands executed
to stdout.
"""
AppScaleLogger.log("Terminating appscale deployment with keyname {0}"
.format(keyname))
time.sleep(2)
shadow_host = LocalState.get_host_with_role(keyname, 'shadow')
try:
secret = LocalState.get_secret_key(keyname)
except IOError:
# We couldn't find the secret key: AppScale is most likely not
# running.
raise AppScaleException("Couldn't find AppScale secret key.")
acc = AppControllerClient(shadow_host, secret)
try:
all_ips = acc.get_all_public_ips()
except Exception as exception:
AppScaleLogger.warn('Saw Exception while getting deployments IPs {0}'.
format(str(exception)))
all_ips = LocalState.get_all_public_ips(keyname)
threads = []
for ip in all_ips:
thread = threading.Thread(target=cls.stop_remote_appcontroller, args=(ip,
keyname, is_verbose))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
boxes_shut_down = 0
is_running_regex = re.compile("appscale-controller stop")
for ip in all_ips:
AppScaleLogger.log("Shutting down AppScale API services at {0}".
format(ip))
while True:
remote_output = cls.ssh(ip, keyname, 'ps x', is_verbose)
AppScaleLogger.verbose(remote_output, is_verbose)
if not is_running_regex.match(remote_output):
break
time.sleep(0.3)
boxes_shut_down += 1
if boxes_shut_down != len(all_ips):
raise AppScaleException("Couldn't terminate your AppScale deployment on"
" all machines - please do so manually.")
AppScaleLogger.log("Terminated AppScale on {0} machines.".
format(boxes_shut_down))
def test_describe_instances_with_two_nodes(self):
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1', 'public2']))
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at 1.2.3.4')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps([
{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
},
{
"public_ip": "public2",
"private_ip": "private2",
"jobs": ["appengine"]
},
]))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# assume that there are two machines running in our deployment
argv = ["--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
AppScaleTools.describe_instances(options)
def test_remove_app_and_app_is_running(self):
# mock out reading from stdin, and assume the user says 'YES'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('YES')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('stop_app').with_args('blargapp',
'the secret').and_return('OK')
fake_appcontroller.should_receive('is_app_running').with_args('blargapp',
'the secret').and_return(True).and_return(True).and_return(False)
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(json.dumps([{
"public_ip" : "public1",
"private_ip" : "private1",
"jobs" : ["shadow", "login"]
}]))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out calls to the UserAppServer and presume that the app does exist
fake_userappserver = flexmock(name='fake_uaserver')
fake_userappserver.should_receive('get_app_data').with_args(
'blargapp', 'the secret').and_return(json.dumps({
'hosts' : { '192.168.1.1' : { 'http' : '80', 'https' : '443' }}}))
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \
.and_return(fake_userappserver)
argv = [
"--appname", "blargapp",
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.remove_app(options)
def create_user_accounts(cls, email, password, public_ip, keyname):
"""Registers two new user accounts with the UserAppServer.
One account is the standard account that users log in with (via their
e-mail address. The other is their XMPP account, so that they can log into
any jabber-compatible service and send XMPP messages to their application
(and receive them).
Args:
email: The e-mail address that should be registered for the user's
standard account.
password: The password that should be used for both the standard and XMPP
accounts.
public_ip: The location where the AppController can be found.
keyname: The name of the SSH keypair used for this AppScale deployment.
"""
acc = AppControllerClient(public_ip, LocalState.get_secret_key(keyname))
is_new_user = False
# first, create the standard account
encrypted_pass = LocalState.encrypt_password(email, password)
if acc.does_user_exist(email):
AppScaleLogger.log("User {0} already exists, so not creating it again.".
format(email))
else:
acc.create_user(email, encrypted_pass)
is_new_user = True
# next, create the XMPP account. if the user's e-mail is [email protected], then that
# means their XMPP account name is a@login_ip
username_regex = re.compile('\A(.*)@')
username = username_regex.match(email).groups()[0]
try:
login_host = acc.get_property('login')['login']
except KeyError:
raise AppControllerException('login property not found')
xmpp_user = "******".format(username, login_host)
xmpp_pass = LocalState.encrypt_password(xmpp_user, password)
is_xmpp_user_exist = acc.does_user_exist(xmpp_user)
if is_xmpp_user_exist and is_new_user:
AppScaleLogger.log("XMPP User {0} conflict!".format(xmpp_user))
generated_xmpp_username = LocalState.generate_xmpp_username(username)
xmpp_user = "******".format(generated_xmpp_username, login_host)
xmpp_pass = LocalState.encrypt_password(xmpp_user, password)
acc.create_user(xmpp_user, xmpp_pass)
elif is_xmpp_user_exist and not is_new_user:
AppScaleLogger.log("XMPP User {0} already exists, so not creating it again.".format(xmpp_user))
else:
acc.create_user(xmpp_user, xmpp_pass)
AppScaleLogger.log("Your XMPP username is {0}".format(xmpp_user))
def test_update_local_metadata(self):
# mock out getting all the ips in the deployment from the head node
fake_soap = flexmock(name='fake_soap')
fake_soap.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1']))
role_info = [{
'public_ip' : 'public1',
'private_ip' : 'private1',
'jobs' : ['shadow', 'db_master']
}]
fake_soap.should_receive('get_role_info').with_args('the secret') \
.and_return(json.dumps(role_info))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_soap)
# mock out reading the secret key
fake_secret = flexmock(name='fake_secret')
fake_secret.should_receive('read').and_return('the secret')
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location('booscale'), 'r') \
.and_return(fake_secret)
# mock out writing the yaml file
fake_locations_yaml = flexmock(name='fake_locations_yaml')
fake_locations_yaml.should_receive('write').with_args(yaml.dump({
'load_balancer': 'public1', 'instance_id': 'i-ABCDEFG',
'secret': 'the secret', 'infrastructure': 'ec2',
'group': 'boogroup', 'ips': 'public1', 'table': 'cassandra',
'db_master': 'node-0', 'zone' : 'my-zone-1b'
})).and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_yaml_location('booscale'), 'w') \
.and_return(fake_locations_yaml)
# and mock out writing the json file
fake_locations_json = flexmock(name='fake_locations_json')
fake_locations_json.should_receive('write').with_args(json.dumps(
role_info)).and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location('booscale'), 'w') \
.and_return(fake_locations_json)
options = flexmock(name='options', table='cassandra', infrastructure='ec2',
keyname='booscale', group='boogroup', zone='my-zone-1b')
node_layout = NodeLayout(options={
'min' : 1,
'max' : 1,
'infrastructure' : 'ec2',
'table' : 'cassandra'
})
host = 'public1'
instance_id = 'i-ABCDEFG'
LocalState.update_local_metadata(options, node_layout, host, instance_id)
def test_remove_app_and_app_is_running(self):
# mock out reading from stdin, and assume the user says 'YES'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('YES')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('stop_app').with_args(
'blargapp', 'the secret').and_return('OK')
fake_appcontroller.should_receive('is_app_running').with_args(
'blargapp',
'the secret').and_return(True).and_return(True).and_return(False)
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps([{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out calls to the UserAppServer and presume that the app does exist
fake_userappserver = flexmock(name='fake_uaserver')
fake_userappserver.should_receive('get_app_data').with_args(
'blargapp', 'the secret').and_return('\nnum_ports:2\n')
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \
.and_return(fake_userappserver)
argv = ["--appname", "blargapp", "--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
AppScaleTools.remove_app(options)
def test_update_local_metadata(self):
# mock out getting all the ips in the deployment from the head node
fake_soap = flexmock(name='fake_soap')
fake_soap.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1']))
role_info = [{
'public_ip' : 'public1',
'private_ip' : 'private1',
'jobs' : ['shadow', 'db_master']
}]
fake_soap.should_receive('get_role_info').with_args('the secret') \
.and_return(json.dumps(role_info))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_soap)
# mock out reading the secret key
fake_secret = flexmock(name='fake_secret')
fake_secret.should_receive('read').and_return('the secret')
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location('booscale'), 'r') \
.and_return(fake_secret)
# mock out writing the yaml file
fake_locations_yaml = flexmock(name='fake_locations_yaml')
fake_locations_yaml.should_receive('write').with_args(yaml.dump({
'load_balancer': 'public1', 'instance_id': 'i-ABCDEFG',
'secret': 'the secret', 'infrastructure': 'ec2',
'group': 'boogroup', 'ips': 'public1', 'table': 'cassandra',
'db_master': 'node-0', 'zone' : 'my-zone-1b'
})).and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_yaml_location('booscale'), 'w') \
.and_return(fake_locations_yaml)
# and mock out writing the json file
fake_locations_json = flexmock(name='fake_locations_json')
fake_locations_json.should_receive('write').with_args(json.dumps(
role_info)).and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location('booscale'), 'w') \
.and_return(fake_locations_json)
options = flexmock(name='options', table='cassandra', infrastructure='ec2',
keyname='booscale', group='boogroup', zone='my-zone-1b')
node_layout = NodeLayout(options={
'min' : 1,
'max' : 1,
'infrastructure' : 'ec2',
'table' : 'cassandra'
})
host = 'public1'
instance_id = 'i-ABCDEFG'
LocalState.update_local_metadata(options, node_layout, host, instance_id)
def test_describe_instances_with_two_nodes(self):
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1', 'public2']))
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at 1.2.3.4')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(json.dumps([{
"public_ip" : "public1",
"private_ip" : "private1",
"jobs" : ["shadow", "login"]
},
{
"public_ip" : "public2",
"private_ip" : "private2",
"jobs" : ["appengine"]
},
]))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# assume that there are two machines running in our deployment
argv = [
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.describe_instances(options)
def test_reset_password_for_user_that_doesnt_exist(self):
# put in a mock for reading the secret file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the username and new password from the user
builtins.should_receive('raw_input').and_return('*****@*****.**')
flexmock(getpass)
getpass.should_receive('getpass').and_return('the password')
# mock out finding the login node's IP address from the json file
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_secret")
fake_nodes_json.should_receive('read').and_return(
json.dumps([{
'public_ip': 'public1',
'private_ip': 'private1',
'jobs': ['login', 'db_master']
}]))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out grabbing the userappserver ip from an appcontroller
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at public1')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@foo.goo', str,
'the secret').and_return('Error: user does not exist')
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \
.and_return(fake_userappserver)
argv = ["--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
self.assertRaises(SystemExit, AppScaleTools.reset_password, options)
def remove_app(cls, options):
"""Instructs AppScale to no longer host the named application.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if not options.confirm:
response = raw_input(
'Are you sure you want to remove this application? (y/N) ')
if response.lower() not in ['y', 'yes']:
raise AppScaleException("Cancelled application removal.")
login_host = LocalState.get_login_host(options.keyname)
secret = LocalState.get_secret_key(options.keyname)
acc = AppControllerClient(login_host, secret)
if not acc.is_app_running(options.appname):
raise AppScaleException("The given application is not currently running.")
# Makes a call to the AppController to get all the stats and looks
# through them for the http port the app can be reached on.
http_port = None
for _ in range(cls.MAX_RETRIES + 1):
result = acc.get_all_stats()
try:
json_result = json.loads(result)
apps_result = json_result['apps']
current_app = apps_result[options.appname]
http_port = current_app['http']
if http_port:
break
time.sleep(cls.SLEEP_TIME)
except (KeyError, ValueError):
AppScaleLogger.verbose("Got json error from get_all_data result.",
options.verbose)
time.sleep(cls.SLEEP_TIME)
if not http_port:
raise AppScaleException(
"Unable to get the serving port for the application.")
acc.stop_app(options.appname)
AppScaleLogger.log("Please wait for your app to shut down.")
for _ in range(cls.MAX_RETRIES + 1):
if RemoteHelper.is_port_open(login_host, http_port, options.verbose):
time.sleep(cls.SLEEP_TIME)
AppScaleLogger.log("Waiting for {0} to terminate...".format(
options.appname))
else:
AppScaleLogger.success("Done shutting down {0}.".format(
options.appname))
return
AppScaleLogger.warn("App {0} may still be running.".format(
options.appname))
def test_reset_password_for_user_that_exists(self):
# put in a mock for reading the secret file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the username and new password from the user
builtins.should_receive('raw_input').and_return('*****@*****.**')
flexmock(getpass)
getpass.should_receive('getpass').and_return('the password')
# mock out finding the login node's IP address from the json file
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_secret")
fake_nodes_json.should_receive('read').and_return(json.dumps([{
'public_ip' : 'public1',
'private_ip' : 'private1',
'jobs' : ['login', 'db_master']
}]))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out grabbing the userappserver ip from an appcontroller
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at public1')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@foo.goo', str, 'the secret').and_return('true')
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \
.and_return(fake_userappserver)
argv = [
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.reset_password(options)
def get_params_from_args(self, args):
""" Constructs a dict with only the parameters necessary to interact with
Google Compute Engine (here, the client_secrets file and the image name).
Args:
args: A Namespace or dict that maps all of the arguments the user has
invoked an AppScale command with their associated value.
Returns:
A dict containing the location of the client_secrets file and that name
of the image to use in GCE.
Raises:
AgentConfigurationException: If the caller fails to specify a
client_secrets file, or if it doesn't exist on the local filesystem.
"""
if not isinstance(args, dict):
args = vars(args)
if not args.get('client_secrets') and not args.get('oauth2_storage'):
raise AgentConfigurationException("Please specify a client_secrets " + \
"file or a oauth2_storage file in your AppScalefile when running " + \
"over Google Compute Engine.")
credentials_file = args.get('client_secrets') or args.get(
'oauth2_storage')
full_credentials = os.path.expanduser(credentials_file)
if not os.path.exists(full_credentials):
raise AgentConfigurationException("Couldn't find your credentials " + \
"at {0}".format(full_credentials))
if args.get('client_secrets'):
destination = LocalState.get_client_secrets_location(
args['keyname'])
elif args.get('oauth2_storage'):
destination = LocalState.get_oauth2_storage_location(
args['keyname'])
shutil.copy(full_credentials, destination)
params = {
self.PARAM_GROUP: args['group'],
self.PARAM_IMAGE_ID: args['machine'],
self.PARAM_INSTANCE_TYPE: args['gce_instance_type'],
self.PARAM_KEYNAME: args['keyname'],
self.PARAM_PROJECT: args['project'],
self.PARAM_ZONE: args['zone']
}
if args.get(self.PARAM_SECRETS):
params[self.PARAM_SECRETS] = args.get(self.PARAM_SECRETS)
elif args.get(self.PARAM_STORAGE):
params[self.PARAM_STORAGE] = args.get(self.PARAM_STORAGE)
params[self.PARAM_VERBOSE] = args.get('verbose', False)
return params
def configure_instance_security(self, parameters):
"""
Setup Euca security keys and groups. Required input values are read from
the parameters dictionary. More specifically, this method expects to
find a 'keyname' parameter and a 'group' parameter in the parameters
dictionary. Using these provided values, this method will create a new
Euca key-pair and a security group. Security group will be granted permissions
to access any port on the instantiated VMs. (Also see documentation for the
BaseAgent class)
Args:
parameters A dictionary of parameters
"""
keyname = parameters[self.PARAM_KEYNAME]
group = parameters[self.PARAM_GROUP]
AppScaleLogger.log("Verifying that keyname {0}".format(keyname) + \
" is not already registered.")
conn = self.open_connection(parameters)
try:
conn.get_key_pair(keyname)
self.handle_failure(
'SSH key found locally - please use a different keyname')
except IndexError: # in euca, this means the key doesn't exist
pass
security_groups = conn.get_all_security_groups()
group_exists = False
for security_group in security_groups:
if security_group.name == group:
self.handle_failure("Security group already exists - please use a " + \
"different group name")
AppScaleLogger.log('Creating key pair: ' + keyname)
key_pair = conn.create_key_pair(keyname)
ssh_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH, keyname)
LocalState.write_key_file(ssh_key, key_pair.material)
AppScaleLogger.log('Creating security group: {0}'.format(group))
conn.create_security_group(group, 'AppScale security group')
conn.authorize_security_group_deprecated(group,
from_port=1,
to_port=65535,
ip_protocol='udp',
cidr_ip='0.0.0.0/0')
conn.authorize_security_group_deprecated(group,
from_port=1,
to_port=65535,
ip_protocol='tcp',
cidr_ip='0.0.0.0/0')
conn.authorize_security_group_deprecated(group,
ip_protocol='icmp',
cidr_ip='0.0.0.0/0')
return True
def copy_deployment_credentials(cls, host, options):
"""Copies credentials needed to start the AppController and have it create
other instances (in cloud deployments).
Args:
host: A str representing the machine (reachable from this computer) to
copy our deployment credentials to.
options: A Namespace that indicates which SSH keypair to use, and whether
or not we are running in a cloud infrastructure.
"""
local_secret_key = LocalState.get_secret_key_location(options.keyname)
cls.scp(host, options.keyname, local_secret_key,
'{}/secret.key'.format(cls.CONFIG_DIR), options.verbose)
local_ssh_key = LocalState.get_key_path_from_name(options.keyname)
cls.scp(host, options.keyname, local_ssh_key,
'{}/ssh.key'.format(cls.CONFIG_DIR), options.verbose)
LocalState.generate_ssl_cert(options.keyname, options.verbose)
local_cert = LocalState.get_certificate_location(options.keyname)
cls.scp(host, options.keyname, local_cert,
'{}/certs/mycert.pem'.format(cls.CONFIG_DIR), options.verbose)
local_private_key = LocalState.get_private_key_location(
options.keyname)
cls.scp(host, options.keyname, local_private_key,
'{}/certs/mykey.pem'.format(cls.CONFIG_DIR), options.verbose)
hash_id = subprocess.Popen([
"openssl", "x509", "-hash", "-noout", "-in",
LocalState.get_certificate_location(options.keyname)
],
stdout=subprocess.PIPE).communicate()[0]
symlink_cert = 'ln -fs {}/certs/mycert.pem /etc/ssl/certs/{}.0'.\
format(cls.CONFIG_DIR, hash_id.rstrip())
cls.ssh(host, options.keyname, symlink_cert, options.verbose)
# In Google Compute Engine, we also need to copy over our client_secrets
# file and the OAuth2 file that the user has approved for use with their
# credentials, otherwise the AppScale VMs won't be able to interact with
# GCE.
if options.infrastructure and options.infrastructure == 'gce':
secrets_location = LocalState.get_client_secrets_location(
options.keyname)
if not os.path.exists(secrets_location):
raise AppScaleException(
'{} does not exist.'.format(secrets_location))
secrets_type = GCEAgent.get_secrets_type(secrets_location)
cls.scp(host, options.keyname, secrets_location,
'{}/client_secrets.json'.format(cls.CONFIG_DIR),
options.verbose)
if secrets_type == CredentialTypes.OAUTH:
local_oauth = LocalState.get_oauth2_storage_location(
options.keyname)
cls.scp(host, options.keyname, local_oauth,
'{}/oauth2.dat'.format(cls.CONFIG_DIR),
options.verbose)
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {0}, already exists.".format(options.location))
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
try:
all_ips = acc.get_all_public_ips()
except socket.error: # Occurs when the AppController has failed.
AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \
"machines in this AppScale deployment. Using our locally cached " + \
"info instead.")
all_ips = LocalState.get_all_public_ips(options.keyname)
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
for ip in all_ips:
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/appscale',
local_dir, options.verbose)
try:
RemoteHelper.scp_remote_to_local(ip, options.keyname,
'/var/log/cassandra', local_dir, options.verbose)
except ShellException:
pass
try:
RemoteHelper.scp_remote_to_local(ip, options.keyname,
'/var/log/zookeeper', local_dir, options.verbose)
except ShellException:
pass
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/kern.log',
local_dir, options.verbose)
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/syslog',
local_dir, options.verbose)
AppScaleLogger.success("Successfully copied logs to {0}".format(
options.location))
def configure_instance_security(self, parameters):
"""
Setup Euca security keys and groups. Required input values are read from
the parameters dictionary. More specifically, this method expects to
find a 'keyname' parameter and a 'group' parameter in the parameters
dictionary. Using these provided values, this method will create a new
Euca key-pair and a security group. Security group will be granted permissions
to access any port on the instantiated VMs. (Also see documentation for the
BaseAgent class)
Args:
parameters A dictionary of parameters
"""
keyname = parameters[self.PARAM_KEYNAME]
group = parameters[self.PARAM_GROUP]
AppScaleLogger.log("Verifying that keyname {0}".format(keyname) + \
" is not already registered.")
conn = self.open_connection(parameters)
try:
conn.get_key_pair(keyname)
self.handle_failure("SSH keyname {0} is already registered. Please " \
"change the 'keyname' specified in your AppScalefile to a different " \
"value, or erase it to have one automatically generated for you." \
.format(keyname))
except IndexError: # in euca, this means the key doesn't exist
pass
security_groups = conn.get_all_security_groups()
group_exists = False
for security_group in security_groups:
if security_group.name == group:
self.handle_failure("Security group {0} is already registered. Please" \
" change the 'group' specified in your AppScalefile to a different " \
"value, or erase it to have one automatically generated for you." \
.format(group))
AppScaleLogger.log('Creating key pair: ' + keyname)
key_pair = conn.create_key_pair(keyname)
ssh_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH, keyname)
LocalState.write_key_file(ssh_key, key_pair.material)
AppScaleLogger.log('Creating security group: {0}'.format(group))
conn.create_security_group(group, 'AppScale security group')
conn.authorize_security_group_deprecated(group, from_port=1,
to_port=65535, ip_protocol='udp', cidr_ip='0.0.0.0/0')
conn.authorize_security_group_deprecated(group, from_port=1,
to_port=65535, ip_protocol='tcp', cidr_ip='0.0.0.0/0')
conn.authorize_security_group_deprecated(group, from_port=-1,
to_port=-1, ip_protocol='icmp', cidr_ip='0.0.0.0/0')
return True
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Raises:
AppScaleException: If any of the machines named in the ips_layout are
not running, or do not have the SSH daemon running.
"""
LocalState.require_ssh_commands(options.auto, options.verbose)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
public_key = path + ".pub"
private_key = path
else:
public_key, private_key = LocalState.generate_rsa_key(options.keyname,
options.verbose)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were problems with your " + \
"placement strategy: " + str(node_layout.errors()))
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, make sure ssh is actually running on the host machine
if not RemoteHelper.is_port_open(ip, RemoteHelper.SSH_PORT,
options.verbose):
raise AppScaleException("SSH does not appear to be running at {0}. " \
"Is the machine at {0} up and running? Make sure your IPs are " \
"correct!".format(ip))
# next, set up passwordless ssh
AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password), options.verbose)
else:
LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip),
options.verbose)
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Raises:
AppScaleException: If any of the machines named in the ips_layout are
not running, or do not have the SSH daemon running.
"""
LocalState.require_ssh_commands(options.auto, options.verbose)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
public_key = path + ".pub"
private_key = path
else:
public_key, private_key = LocalState.generate_rsa_key(options.keyname,
options.verbose)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were problems with your " + \
"placement strategy: " + str(node_layout.errors()))
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, make sure ssh is actually running on the host machine
if not RemoteHelper.is_port_open(ip, RemoteHelper.SSH_PORT,
options.verbose):
raise AppScaleException("SSH does not appear to be running at {0}. " \
"Is the machine at {0} up and running? Make sure your IPs are " \
"correct!".format(ip))
# next, set up passwordless ssh
AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password), options.verbose)
else:
LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip),
options.verbose)
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
LocalState.require_ssh_commands(options.auto, options.verbose)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
public_key = path + ".pub"
private_key = path
else:
public_key, private_key = LocalState.generate_rsa_key(
options.keyname, options.verbose)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were problems with your " + \
"placement strategy: " + str(node_layout.errors()))
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, set up passwordless ssh
AppScaleLogger.log(
"Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell(
"{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password),
options.verbose)
else:
LocalState.shell(
"ssh-copy-id -i {0} root@{1}".format(private_key, ip),
options.verbose)
# next, copy over the ssh keypair we generate
RemoteHelper.scp(ip, options.keyname, public_key,
'/root/.ssh/id_rsa.pub', options.verbose)
RemoteHelper.scp(ip, options.keyname, private_key,
'/root/.ssh/id_rsa', options.verbose)
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def publish_api_list(cls, api_list, url, keyname):
eager = EagerClient(LocalState.get_login_host(keyname),
LocalState.get_secret_key(keyname))
temp_api_list = []
for api in api_list:
temp_api_list.append(api.to_dict())
result = eager.publish_api_list(temp_api_list, url)
if result['success']:
AppScaleLogger.log('{0} APIs published to API store.'.format(len(api_list)))
else:
AppScaleLogger.warn(result['reason'])
if result.get('detail'):
AppScaleLogger.warn(str(result['detail']))
def perform_eager_validation(cls, app, keyname):
eager = EagerClient(LocalState.get_login_host(keyname),
LocalState.get_secret_key(keyname))
AppScaleLogger.log('Running EAGER validations for application.')
result = eager.validate_application_for_deployment(app.to_dict())
if not result['success']:
AppScaleLogger.log('Validation errors encountered: {0}'.format(result['reason']))
if hasattr(result, 'detail'):
errors = result['detail'].split('|')
AppScaleLogger.log('Following error details are available:')
for e in errors:
AppScaleLogger.log(' * {0}'.format(e))
return result['success']
def test_copy_deployment_credentials_in_cloud(self):
# mock out the scp'ing to public1 and assume they succeed
subprocess.should_receive('Popen').with_args(re.compile('secret.key'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('ssh.key'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# mock out generating the private key
flexmock(M2Crypto.RSA)
fake_rsa_key = flexmock(name='fake_rsa_key')
fake_rsa_key.should_receive('save_key').with_args(
LocalState.get_private_key_location('bookey'), None)
M2Crypto.RSA.should_receive('gen_key').and_return(fake_rsa_key)
flexmock(M2Crypto.EVP)
fake_pkey = flexmock(name='fake_pkey')
fake_pkey.should_receive('assign_rsa').with_args(fake_rsa_key).and_return()
M2Crypto.EVP.should_receive('PKey').and_return(fake_pkey)
# and mock out generating the certificate
flexmock(M2Crypto.X509)
fake_cert = flexmock(name='fake_x509')
fake_cert.should_receive('set_pubkey').with_args(fake_pkey).and_return()
fake_cert.should_receive('set_subject')
fake_cert.should_receive('set_issuer_name')
fake_cert.should_receive('set_not_before')
fake_cert.should_receive('set_not_after')
fake_cert.should_receive('sign').with_args(fake_pkey, md="sha256")
fake_cert.should_receive('save_pem').with_args(
LocalState.get_certificate_location('bookey'))
M2Crypto.X509.should_receive('X509').and_return(fake_cert)
# next, mock out copying the private key and certificate
subprocess.should_receive('Popen').with_args(re.compile('mycert.pem'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('mykey.pem'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
options = flexmock(name='options', keyname='bookey', infrastructure='ec2',
verbose=True)
RemoteHelper.copy_deployment_credentials('public1', options)
def test_make_appscale_directory_creation(self):
# let's say that our ~/.appscale directory
# does not exist
os.path.should_receive('exists') \
.with_args(LocalState.LOCAL_APPSCALE_PATH) \
.and_return(False) \
.once()
# thus, mock out making the appscale dir
os.should_receive('mkdir') \
.with_args(LocalState.LOCAL_APPSCALE_PATH) \
.and_return()
LocalState.make_appscale_directory()
def test_make_appscale_directory_creation(self):
# let's say that our ~/.appscale directory
# does not exist
os.path.should_receive('exists') \
.with_args(LocalState.LOCAL_APPSCALE_PATH) \
.and_return(False) \
.once()
# thus, mock out making the appscale dir
os.should_receive('mkdir') \
.with_args(LocalState.LOCAL_APPSCALE_PATH) \
.and_return()
LocalState.make_appscale_directory()
def copy_deployment_credentials(cls, host, options):
"""Copies credentials needed to start the AppController and have it create
other instances (in cloud deployments).
Args:
host: A str representing the machine (reachable from this computer) to
copy our deployment credentials to.
options: A Namespace that indicates which SSH keypair to use, and whether
or not we are running in a cloud infrastructure.
"""
cls.scp(host, options.keyname, LocalState.get_secret_key_location(
options.keyname), '/etc/appscale/secret.key', options.verbose)
cls.scp(host, options.keyname, LocalState.get_key_path_from_name(
options.keyname), '/etc/appscale/ssh.key', options.verbose)
LocalState.generate_ssl_cert(options.keyname, options.verbose)
cls.scp(host, options.keyname, LocalState.get_certificate_location(
options.keyname), '/etc/appscale/certs/mycert.pem', options.verbose)
cls.scp(host, options.keyname, LocalState.get_private_key_location(
options.keyname), '/etc/appscale/certs/mykey.pem', options.verbose)
AppScaleLogger.log("Copying over deployment credentials")
cert = LocalState.get_certificate_location(options.keyname)
private_key = LocalState.get_private_key_location(options.keyname)
cls.ssh(host, options.keyname, 'mkdir -p /etc/appscale/keys/cloud1',
options.verbose)
cls.scp(host, options.keyname, cert, "/etc/appscale/keys/cloud1/mycert.pem",
options.verbose)
cls.scp(host, options.keyname, private_key,
"/etc/appscale/keys/cloud1/mykey.pem", options.verbose)
def test_extract_app_to_dir(self):
flexmock(os)
os.should_receive('mkdir').and_return()
flexmock(os.path)
os.path.should_receive('abspath').with_args('relative/app.tar.gz')\
.and_return('/tmp/relative/app.tar.gz')
flexmock(LocalState)
LocalState.should_receive('shell')\
.with_args(re.compile('tar zxvf /tmp/relative/app.tar.gz'),False)\
.and_return()
LocalState.extract_app_to_dir('relative/app.tar.gz',False)
def create_new_user(self,
email,
password,
response,
account_type='xmpp_user'):
""" Creates a new user account, by making both a standard login and an
XMPP login account.
Args:
email: A str containing the e-mail address of the new user.
password: A str containing the cleartext password for the new user.
response: A webapp2 response that the new user's logged in cookie
should be set in.
Returns:
True, if the user account was successfully created.
Raises:
AppHelperException: If the user account could not be created.
"""
try:
uaserver = self.get_uaserver()
# First, create the standard account.
encrypted_pass = LocalState.encrypt_password(email, password)
result = uaserver.commit_new_user(email, encrypted_pass,
account_type, GLOBAL_SECRET_KEY)
if result != 'true':
raise AppHelperException(result)
# Next, create the XMPP account. if the user's e-mail is [email protected], then that
# means their XMPP account name is a@login_ip.
username_regex = re.compile(self.USERNAME_FROM_EMAIL_REGEX)
username = username_regex.match(email).groups()[0]
xmpp_user = "******".format(username, self.get_login_host())
xmpp_pass = LocalState.encrypt_password(xmpp_user, password)
result = uaserver.commit_new_user(xmpp_user, xmpp_pass,
account_type, GLOBAL_SECRET_KEY)
if result != 'true':
raise AppHelperException(result)
# TODO: We may not even be using this token since the switch to
# full proxy nginx. Investigate this.
self.create_token(email, email)
self.set_appserver_cookie(email, self.get_user_app_list(email),
response)
except AppHelperException as err:
logging.exception(err)
raise AppHelperException(str(err))
except Exception as err:
logging.exception(err)
raise AppHelperException(str(err))
return True
def test_get_property(self):
# put in a mock for reading the secret file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out finding the shadow node's IP address from the json file
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_secret")
fake_nodes_json.should_receive('read').and_return(
json.dumps([{
'public_ip': 'public1',
'private_ip': 'private1',
'jobs': ['login', 'shadow']
}]))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out grabbing the userappserver ip from an appcontroller
property_name = "name"
property_value = "value"
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('set_property').with_args(
property_name, property_value, 'the secret').and_return('OK')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
argv = [
"--keyname", self.keyname, "--property_name", property_name,
"--property_value", property_value
]
options = ParseArgs(argv, self.function).args
result = AppScaleTools.set_property(options)
self.assertEqual(None, result)
def login_user(self, email, password, response):
""" Checks to see if the user has entered in a valid email and password,
logging the user in if they have.
Args:
email: A str containing the e-mail address of the user to login.
password: A str containing the cleartext password of the user to login.
response: A webapp2 response that the new user's logged in cookie
should be set in.
Return:
True if the user logged in successfully, and False otherwise.
"""
user_data = self.query_user_data(email)
server_re = re.search(self.USER_DATA_PASSWORD_REGEX, user_data)
if not server_re:
logging.error("Failed Login: {0} regex failed".format(email))
return False
server_pwd = server_re.group(1)
encrypted_pass = LocalState.encrypt_password(email, password)
if server_pwd != encrypted_pass:
logging.info("Failed Login: {0} password mismatch".format(email))
return False
self.create_token(email, email)
self.set_appserver_cookie(email, self.get_user_app_list(email),
response)
return True