def add_instances(cls, options):
"""Adds additional machines to an AppScale deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if 'master' in options.ips.keys():
raise BadConfigurationException("Cannot add master nodes to an " + \
"already running AppScale deployment.")
# Skip checking for -n (replication) because we don't allow the user
# to specify it here (only allowed in run-instances).
additional_nodes_layout = NodeLayout(options)
# In virtualized cluster deployments, we need to make sure that the user
# has already set up SSH keys.
if LocalState.get_from_yaml(options.keyname, 'infrastructure') == "xen":
for ip in options.ips.values():
# throws a ShellException if the SSH key doesn't work
RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)
# Finally, find an AppController and send it a message to add
# the given nodes with the new roles.
AppScaleLogger.log("Sending request to add instances")
login_ip = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_ip, LocalState.get_secret_key(
options.keyname))
acc.start_roles_on_nodes(json.dumps(options.ips))
# TODO(cgb): Should we wait for the new instances to come up and get
# initialized?
AppScaleLogger.success("Successfully sent request to add instances " + \
"to this AppScale deployment.")
def valid_ssh_key(self, config, run_instances_opts):
""" Checks if the tools can log into the head node with the current key.
Args:
config: A dictionary that includes the IPs layout (which itself is a dict
mapping role names to IPs) and, optionally, the keyname to use.
run_instances_opts: The arguments parsed from the appscale-run-instances
command.
Returns:
A bool indicating whether or not the specified keyname can be used to log
into the head node.
Raises:
BadConfigurationException: If the IPs layout was not a dictionary.
"""
keyname = config['keyname']
verbose = config.get('verbose', False)
if not isinstance(config['ips_layout'], dict):
raise BadConfigurationException(
'ips_layout should be a dictionary. Please fix it and try again.')
ssh_key_location = self.APPSCALE_DIRECTORY + keyname + ".key"
if not os.path.exists(ssh_key_location):
return False
all_ips = LocalState.get_all_public_ips(keyname)
# If a login node is defined, use that to communicate with other nodes.
node_layout = NodeLayout(run_instances_opts)
head_node = node_layout.head_node()
if head_node is not None:
remote_key = '{}/ssh.key'.format(RemoteHelper.CONFIG_DIR)
try:
RemoteHelper.scp(
head_node.public_ip, keyname, ssh_key_location, remote_key, verbose)
except ShellException:
return False
for ip in all_ips:
ssh_to_ip = 'ssh -i {key} -o StrictHostkeyChecking=no root@{ip} true'\
.format(key=remote_key, ip=ip)
try:
RemoteHelper.ssh(
head_node.public_ip, keyname, ssh_to_ip, verbose, user='******')
except ShellException:
return False
return True
for ip in all_ips:
if not self.can_ssh_to_ip(ip, keyname, verbose):
return False
return True
def run_bootstrap(cls, ip, options, error_ips):
try:
RemoteHelper.ssh(ip, options.keyname, cls.BOOTSTRAP_CMD, options.verbose)
AppScaleLogger.success("Successfully updated and built AppScale on {}".format(ip))
except ShellException:
error_ips.append(ip)
AppScaleLogger.warn(
"Unable to upgrade AppScale code on {}.\n"
"Please correct any errors listed in /var/log/appscale/bootstrap.log "
"on that machine and re-run appscale upgrade.".format(ip)
)
return error_ips
def run_bootstrap(cls, ip, options, error_ips):
try:
RemoteHelper.ssh(ip, options.keyname, cls.BOOTSTRAP_CMD,
options.verbose)
AppScaleLogger.success(
'Successfully updated and built AppScale on {}'.format(ip))
except ShellException:
error_ips.append(ip)
AppScaleLogger.warn(
'Unable to upgrade AppScale code on {}.\n'
'Please correct any errors listed in /var/log/appscale/bootstrap.log '
'on that machine and re-run appscale upgrade.'.format(ip))
return error_ips
def async_layout_upgrade(ip, keyname, script, error_bucket, verbose=False):
""" Run a command over SSH and place exceptions in a bucket.
Args:
ip: A string containing and IP address.
keyname: A string containing the deployment keyname.
script: A string to run as a command over SSH.
error_bucket: A thread-safe queue.
verbose: A boolean indicating whether or not to log verbosely.
"""
try:
RemoteHelper.ssh(ip, keyname, script, verbose)
except ShellException as ssh_error:
error_bucket.put(ssh_error)
def async_layout_upgrade(ip, keyname, script, error_bucket, verbose=False):
""" Run a command over SSH and place exceptions in a bucket.
Args:
ip: A string containing and IP address.
keyname: A string containing the deployment keyname.
script: A string to run as a command over SSH.
error_bucket: A thread-safe queue.
verbose: A boolean indicating whether or not to log verbosely.
"""
try:
RemoteHelper.ssh(ip, keyname, script, verbose)
except ShellException as ssh_error:
error_bucket.put(ssh_error)
def clean(self):
"""'clean' provides a mechanism that will forcefully shut down all AppScale-
related services on virtual machines in a cluster deployment.
Returns:
A list of the IP addresses where AppScale was shut down.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
BadConfigurationException: If this method is invoked and the AppScalefile
indicates that a cloud deployment is being used.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
if 'ips_layout' not in contents_as_yaml:
raise BadConfigurationException("Cannot use 'appscale clean' in a " \
"cloud deployment.")
if 'verbose' in contents_as_yaml and contents_as_yaml[
'verbose'] == True:
is_verbose = contents_as_yaml['verbose']
else:
is_verbose = False
if 'keyname' in contents_as_yaml:
keyname = contents_as_yaml['keyname']
else:
keyname = 'appscale'
all_ips = self.get_all_ips(contents_as_yaml["ips_layout"])
for ip in all_ips:
RemoteHelper.ssh(ip, keyname, self.TERMINATE, is_verbose)
try:
LocalState.cleanup_appscale_files(keyname)
except Exception:
pass
AppScaleLogger.success(
"Successfully shut down your AppScale deployment.")
return all_ips
def clean(self):
"""'clean' provides a mechanism that will forcefully shut down all AppScale-
related services on virtual machines in a cluster deployment.
Returns:
A list of the IP addresses where AppScale was shut down.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
BadConfigurationException: If this method is invoked and the AppScalefile
indicates that a cloud deployment is being used.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
if 'ips_layout' not in contents_as_yaml:
raise BadConfigurationException("Cannot use 'appscale clean' in a " \
"cloud deployment.")
if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True:
is_verbose = contents_as_yaml['verbose']
else:
is_verbose = False
if 'keyname' in contents_as_yaml:
keyname = contents_as_yaml['keyname']
else:
keyname = 'appscale'
all_ips = self.get_all_ips(contents_as_yaml["ips_layout"])
for ip in all_ips:
RemoteHelper.ssh(ip, keyname, self.TERMINATE, is_verbose)
try:
LocalState.cleanup_appscale_files(keyname)
except Exception:
pass
AppScaleLogger.success("Successfully shut down your AppScale deployment.")
return all_ips
def can_ssh_to_ip(self, ip, keyname, is_verbose):
"""Attempts to SSH into the machine located at the given IP address with the
given SSH key.
Args:
ip: The IP address to attempt to SSH into.
keyname: The name of the SSH key that uniquely identifies this AppScale
deployment.
is_verbose: A bool that indicates if we should print the SSH command we
execute to stdout.
Returns:
A bool that indicates whether or not the given SSH key can log in without
a password to the given machine.
"""
try:
RemoteHelper.ssh(ip, keyname, 'ls', is_verbose, user='******')
return True
except ShellException:
return False
def can_ssh_to_ip(self, ip, keyname, is_verbose):
""" Attempts to SSH into the machine located at the given IP address with the
given SSH key.
Args:
ip: The IP address to attempt to SSH into.
keyname: The name of the SSH key that uniquely identifies this AppScale
deployment.
is_verbose: A bool that indicates if we should print the SSH command we
execute to stdout.
Returns:
A bool that indicates whether or not the given SSH key can log in without
a password to the given machine.
"""
try:
RemoteHelper.ssh(ip, keyname, 'ls', is_verbose, user='******')
return True
except ShellException:
return False
def add_instances(cls, options):
"""Adds additional machines to an AppScale deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if 'master' in options.ips.keys():
raise BadConfigurationException("Cannot add master nodes to an " + \
"already running AppScale deployment.")
# Skip checking for -n (replication) because we don't allow the user
# to specify it here (only allowed in run-instances).
additional_nodes_layout = NodeLayout(options)
# In virtualized cluster deployments, we need to make sure that the user
# has already set up SSH keys.
if LocalState.get_from_yaml(options.keyname,
'infrastructure') == "xen":
ips_to_check = []
for ip_group in options.ips.values():
ips_to_check.extend(ip_group)
for ip in ips_to_check:
# throws a ShellException if the SSH key doesn't work
RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)
# Finally, find an AppController and send it a message to add
# the given nodes with the new roles.
AppScaleLogger.log("Sending request to add instances")
login_ip = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_ip,
LocalState.get_secret_key(options.keyname))
acc.start_roles_on_nodes(json.dumps(options.ips))
# TODO(cgb): Should we wait for the new instances to come up and get
# initialized?
AppScaleLogger.success("Successfully sent request to add instances " + \
"to this AppScale deployment.")
def down(self, clean=False, terminate=False):
""" 'down' provides a nicer experience for users than the
appscale-terminate-instances command, by using the configuration options
present in the AppScalefile found in the current working directory.
Args:
clean: A boolean to indicate if the deployment data and metadata
needs to be clean. This will clear the datastore.
terminate: A boolean to indicate if instances needs to be terminated
(valid only if we spawn instances at start).
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
# Construct a terminate-instances command from the file's contents
command = []
contents_as_yaml = yaml.safe_load(contents)
if 'verbose' in contents_as_yaml and contents_as_yaml[
'verbose'] == True:
is_verbose = contents_as_yaml['verbose']
command.append("--verbose")
else:
is_verbose = False
if 'keyname' in contents_as_yaml:
keyname = contents_as_yaml['keyname']
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
else:
keyname = 'appscale'
if "EC2_ACCESS_KEY" in contents_as_yaml:
os.environ["EC2_ACCESS_KEY"] = contents_as_yaml["EC2_ACCESS_KEY"]
if "EC2_SECRET_KEY" in contents_as_yaml:
os.environ["EC2_SECRET_KEY"] = contents_as_yaml["EC2_SECRET_KEY"]
if "EC2_URL" in contents_as_yaml:
os.environ["EC2_URL"] = contents_as_yaml["EC2_URL"]
if clean:
if 'test' not in contents_as_yaml or contents_as_yaml[
'test'] != True:
LocalState.confirm_or_abort(
"Clean will delete every data in the deployment.")
all_ips = LocalState.get_all_public_ips(keyname)
for ip in all_ips:
RemoteHelper.ssh(ip, keyname, self.TERMINATE, is_verbose)
AppScaleLogger.success(
"Successfully cleaned your AppScale deployment.")
if terminate:
infrastructure = LocalState.get_infrastructure(keyname)
if infrastructure != "xen" and not LocalState.are_disks_used(
keyname) and 'test' not in contents_as_yaml:
LocalState.confirm_or_abort(
"Terminate will delete instances and the data on them.")
command.append("--terminate")
if 'test' in contents_as_yaml and contents_as_yaml['test'] == True:
command.append("--test")
# Finally, exec the command. Don't worry about validating it -
# appscale-terminate-instances will do that for us.
options = ParseArgs(command, "appscale-terminate-instances").args
AppScaleTools.terminate_instances(options)
LocalState.cleanup_appscale_files(keyname, terminate)
AppScaleLogger.success(
"Successfully shut down your AppScale deployment.")
def run_upgrade_script(cls, options, node_layout):
""" Runs the upgrade script which checks for any upgrades needed to be performed.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
node_layout: A NodeLayout object for the deployment.
"""
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
db_ips = [node.private_ip for node in node_layout.nodes
if node.is_role('db_master') or node.is_role('db_slave')]
zk_ips = [node.private_ip for node in node_layout.nodes
if node.is_role('zookeeper')]
upgrade_script_command = '{script} --keyname {keyname} '\
'--log-postfix {timestamp} '\
'--db-master {db_master} '\
'--zookeeper {zk_ips} '\
'--database {db_ips} '\
'--replication {replication}'.format(
script=cls.UPGRADE_SCRIPT,
keyname=options.keyname,
timestamp=timestamp,
db_master=node_layout.db_master().private_ip,
zk_ips=' '.join(zk_ips),
db_ips=' '.join(db_ips),
replication=node_layout.replication
)
master_public_ip = node_layout.head_node().public_ip
AppScaleLogger.log("Running upgrade script to check if any other upgrade is needed.")
# Run the upgrade command as a background process.
error_bucket = Queue.Queue()
threading.Thread(
target=async_layout_upgrade,
args=(master_public_ip, options.keyname, upgrade_script_command,
error_bucket, options.verbose)
).start()
last_message = None
while True:
# Check if the SSH thread has crashed.
try:
ssh_error = error_bucket.get(block=False)
AppScaleLogger.warn('Error executing upgrade script')
LocalState.generate_crash_log(ssh_error, traceback.format_exc())
except Queue.Empty:
pass
upgrade_status_file = cls.UPGRADE_STATUS_FILE_LOC + timestamp + ".json"
command = 'cat' + " " + upgrade_status_file
upgrade_status = RemoteHelper.ssh(
master_public_ip, options.keyname, command, options.verbose)
json_status = json.loads(upgrade_status)
if 'status' not in json_status or 'message' not in json_status:
raise AppScaleException('Invalid status log format')
if json_status['status'] == 'complete':
AppScaleLogger.success(json_status['message'])
break
if json_status['status'] == 'inProgress':
if json_status['message'] != last_message:
AppScaleLogger.log(json_status['message'])
last_message = json_status['message']
time.sleep(cls.SLEEP_TIME)
continue
# Assume the message is an error.
AppScaleLogger.warn(json_status['message'])
raise AppScaleException(json_status['message'])
def down(self, clean=False, terminate=False):
""" 'down' provides a nicer experience for users than the
appscale-terminate-instances command, by using the configuration options
present in the AppScalefile found in the current working directory.
Args:
clean: A boolean to indicate if the deployment data and metadata
needs to be clean. This will clear the datastore.
terminate: A boolean to indicate if instances needs to be terminated
(valid only if we spawn instances at start).
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
# Construct a terminate-instances command from the file's contents
command = []
contents_as_yaml = yaml.safe_load(contents)
if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True:
is_verbose = contents_as_yaml['verbose']
command.append("--verbose")
else:
is_verbose = False
if 'keyname' in contents_as_yaml:
keyname = contents_as_yaml['keyname']
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
else:
keyname = 'appscale'
if "EC2_ACCESS_KEY" in contents_as_yaml:
os.environ["EC2_ACCESS_KEY"] = contents_as_yaml["EC2_ACCESS_KEY"]
if "EC2_SECRET_KEY" in contents_as_yaml:
os.environ["EC2_SECRET_KEY"] = contents_as_yaml["EC2_SECRET_KEY"]
if "EC2_URL" in contents_as_yaml:
os.environ["EC2_URL"] = contents_as_yaml["EC2_URL"]
if clean:
if 'test' not in contents_as_yaml or contents_as_yaml['test'] != True:
LocalState.confirm_or_abort("Clean will delete every data in the deployment.")
all_ips = LocalState.get_all_public_ips(keyname)
for ip in all_ips:
RemoteHelper.ssh(ip, keyname, self.TERMINATE, is_verbose)
AppScaleLogger.success("Successfully cleaned your AppScale deployment.")
if terminate:
infrastructure = LocalState.get_infrastructure(keyname)
if infrastructure != "xen" and not LocalState.are_disks_used(
keyname) and 'test' not in contents_as_yaml:
LocalState.confirm_or_abort("Terminate will delete instances and the data on them.")
command.append("--terminate")
if 'test' in contents_as_yaml and contents_as_yaml['test'] == True:
command.append("--test")
# Finally, exec the command. Don't worry about validating it -
# appscale-terminate-instances will do that for us.
options = ParseArgs(command, "appscale-terminate-instances").args
AppScaleTools.terminate_instances(options)
LocalState.cleanup_appscale_files(keyname, terminate)
AppScaleLogger.success("Successfully shut down your AppScale deployment.")
def run_upgrade_script(cls, options, node_layout):
""" Runs the upgrade script which checks for any upgrades needed to be performed.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
node_layout: A NodeLayout object for the deployment.
"""
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
db_ips = [
node.private_ip for node in node_layout.nodes if node.is_role("db_master") or node.is_role("db_slave")
]
zk_ips = [node.private_ip for node in node_layout.nodes if node.is_role("zookeeper")]
upgrade_script_command = (
"{script} --keyname {keyname} "
"--log-postfix {timestamp} "
"--db-master {db_master} "
"--zookeeper {zk_ips} "
"--database {db_ips} "
"--replication {replication}".format(
script=cls.UPGRADE_SCRIPT,
keyname=options.keyname,
timestamp=timestamp,
db_master=node_layout.db_master().private_ip,
zk_ips=" ".join(zk_ips),
db_ips=" ".join(db_ips),
replication=node_layout.replication,
)
)
master_public_ip = node_layout.head_node().public_ip
AppScaleLogger.log("Running upgrade script to check if any other upgrade is needed.")
# Run the upgrade command as a background process.
error_bucket = Queue.Queue()
threading.Thread(
target=async_layout_upgrade,
args=(master_public_ip, options.keyname, upgrade_script_command, error_bucket, options.verbose),
).start()
last_message = None
while True:
# Check if the SSH thread has crashed.
try:
ssh_error = error_bucket.get(block=False)
AppScaleLogger.warn("Error executing upgrade script")
LocalState.generate_crash_log(ssh_error, traceback.format_exc())
except Queue.Empty:
pass
upgrade_status_file = cls.UPGRADE_STATUS_FILE_LOC + timestamp + ".json"
command = "cat" + " " + upgrade_status_file
upgrade_status = RemoteHelper.ssh(master_public_ip, options.keyname, command, options.verbose)
json_status = json.loads(upgrade_status)
if "status" not in json_status or "message" not in json_status:
raise AppScaleException("Invalid status log format")
if json_status["status"] == "complete":
AppScaleLogger.success(json_status["message"])
break
if json_status["status"] == "inProgress":
if json_status["message"] != last_message:
AppScaleLogger.log(json_status["message"])
last_message = json_status["message"]
time.sleep(cls.SLEEP_TIME)
continue
# Assume the message is an error.
AppScaleLogger.warn(json_status["message"])
raise AppScaleException(json_status["message"])
