def ensure_appscale_isnt_running(cls, keyname, force):
"""Checks the secret key file to see if AppScale is running, and
aborts if it is.
Args:
keyname: The keypair name that is used to identify AppScale deployments.
force: A bool that is used to run AppScale even if the secret key file
is present.
Raises:
BadConfigurationException: If AppScale is already running.
"""
if force:
return
if os.path.exists(cls.get_secret_key_location(keyname)):
try:
login_host = cls.get_login_host(keyname)
secret_key = cls.get_secret_key(keyname)
except (IOError, AppScaleException, BadConfigurationException):
# If we don't have the locations files, we are not running.
return
acc = AppControllerClient(login_host, secret_key)
try:
acc.get_all_public_ips()
except AppControllerException:
# AC is not running, so we assume appscale is not up and running.
AppScaleLogger.log("AppController not running on login node.")
else:
raise BadConfigurationException("AppScale is already running. Terminate" +
" it, set 'force: True' in your AppScalefile, or use the --force flag" +
" to run anyways.")
def ensure_appscale_isnt_running(cls, keyname, force):
"""Checks the secret key file to see if AppScale is running, and
aborts if it is.
Args:
keyname: The keypair name that is used to identify AppScale deployments.
force: A bool that is used to run AppScale even if the secret key file
is present.
Raises:
BadConfigurationException: If AppScale is already running.
"""
if force:
return
if os.path.exists(cls.get_secret_key_location(keyname)):
try:
login_host = cls.get_login_host(keyname)
secret_key = cls.get_secret_key(keyname)
except (IOError, AppScaleException, BadConfigurationException):
# If we don't have the locations files, we are not running.
return
acc = AppControllerClient(login_host, secret_key)
try:
acc.get_all_public_ips()
except AppControllerException:
# AC is not running, so we assume appscale is not up and running.
AppScaleLogger.log("AppController not running on login node.")
else:
raise BadConfigurationException("AppScale is already running. Terminate" +
" it, set 'force: True' in your AppScalefile, or use the --force flag" +
" to run anyways.")
def describe_instances(cls, options):
"""Queries each node in the currently running AppScale deployment and
reports on their status.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
login_host = LocalState.get_login_host(options.keyname)
login_acc = AppControllerClient(
login_host, LocalState.get_secret_key(options.keyname))
for ip in login_acc.get_all_public_ips():
acc = AppControllerClient(
ip, LocalState.get_secret_key(options.keyname))
AppScaleLogger.log("Status of node at {0}:".format(ip))
try:
AppScaleLogger.log(acc.get_status())
except Exception as exception:
AppScaleLogger.warn("Unable to contact machine: {0}\n".format(
str(exception)))
AppScaleLogger.success("View status information about your AppScale " + \
"deployment at http://{0}:{1}/status".format(login_host,
RemoteHelper.APP_DASHBOARD_PORT))
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {0}, already exists.".format(options.location))
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
for ip in acc.get_all_public_ips():
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/appscale',
local_dir, options.verbose)
AppScaleLogger.success("Successfully copied logs to {0}".format(
options.location))
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {0}, already exists.".format(options.location))
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
for ip in acc.get_all_public_ips():
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
RemoteHelper.scp_remote_to_local(ip, options.keyname,
'/var/log/appscale', local_dir,
options.verbose)
AppScaleLogger.success("Successfully copied logs to {0}".format(
options.location))
def terminate_virtualized_cluster(cls, keyname, is_verbose):
"""Stops all API services running on all nodes in the currently running
AppScale deployment.
Args:
keyname: The name of the SSH keypair used for this AppScale deployment.
is_verbose: A bool that indicates if we should print the commands executed
to stdout.
"""
AppScaleLogger.log(
"Terminating appscale deployment with keyname {0}".format(keyname))
time.sleep(2)
shadow_host = LocalState.get_host_with_role(keyname, 'shadow')
try:
secret = LocalState.get_secret_key(keyname)
except IOError:
# We couldn't find the secret key: AppScale is most likely not
# running.
raise AppScaleException("Couldn't find AppScale secret key.")
acc = AppControllerClient(shadow_host, secret)
try:
all_ips = acc.get_all_public_ips()
except Exception as exception:
AppScaleLogger.warn(
'Saw Exception while getting deployments IPs {0}'.format(
str(exception)))
all_ips = LocalState.get_all_public_ips(keyname)
threads = []
for ip in all_ips:
thread = threading.Thread(target=cls.stop_remote_appcontroller,
args=(ip, keyname, is_verbose))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
boxes_shut_down = 0
is_running_regex = re.compile("appscale-controller stop")
for ip in all_ips:
AppScaleLogger.log(
"Shutting down AppScale API services at {0}".format(ip))
while True:
remote_output = cls.ssh(ip, keyname, 'ps x', is_verbose)
AppScaleLogger.verbose(remote_output, is_verbose)
if not is_running_regex.match(remote_output):
break
time.sleep(0.3)
boxes_shut_down += 1
if boxes_shut_down != len(all_ips):
raise AppScaleException(
"Couldn't terminate your AppScale deployment on"
" all machines - please do so manually.")
AppScaleLogger.log(
"Terminated AppScale on {0} machines.".format(boxes_shut_down))
def terminate_virtualized_cluster(cls, keyname, is_verbose):
"""Stops all API services running on all nodes in the currently running
AppScale deployment.
Args:
keyname: The name of the SSH keypair used for this AppScale deployment.
is_verbose: A bool that indicates if we should print the commands executed
to stdout.
"""
AppScaleLogger.log("Terminating appscale deployment with keyname {0}"
.format(keyname))
time.sleep(2)
shadow_host = LocalState.get_host_with_role(keyname, 'shadow')
try:
secret = LocalState.get_secret_key(keyname)
except IOError:
# We couldn't find the secret key: AppScale is most likely not
# running.
raise AppScaleException("Couldn't find AppScale secret key.")
acc = AppControllerClient(shadow_host, secret)
try:
all_ips = acc.get_all_public_ips()
except Exception as exception:
AppScaleLogger.warn('Saw Exception while getting deployments IPs {0}'.
format(str(exception)))
all_ips = LocalState.get_all_public_ips(keyname)
threads = []
for ip in all_ips:
thread = threading.Thread(target=cls.stop_remote_appcontroller, args=(ip,
keyname, is_verbose))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
boxes_shut_down = 0
is_running_regex = re.compile("appscale-controller stop")
for ip in all_ips:
AppScaleLogger.log("Shutting down AppScale API services at {0}".
format(ip))
while True:
remote_output = cls.ssh(ip, keyname, 'ps x', is_verbose)
AppScaleLogger.verbose(remote_output, is_verbose)
if not is_running_regex.match(remote_output):
break
time.sleep(0.3)
boxes_shut_down += 1
if boxes_shut_down != len(all_ips):
raise AppScaleException("Couldn't terminate your AppScale deployment on"
" all machines - please do so manually.")
AppScaleLogger.log("Terminated AppScale on {0} machines.".
format(boxes_shut_down))
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {0}, already exists.".format(options.location))
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
try:
all_ips = acc.get_all_public_ips()
except socket.error: # Occurs when the AppController has failed.
AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \
"machines in this AppScale deployment. Using our locally cached " + \
"info instead.")
all_ips = LocalState.get_all_public_ips(options.keyname)
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
for ip in all_ips:
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/appscale',
local_dir, options.verbose)
try:
RemoteHelper.scp_remote_to_local(ip, options.keyname,
'/var/log/cassandra', local_dir, options.verbose)
except ShellException:
pass
try:
RemoteHelper.scp_remote_to_local(ip, options.keyname,
'/var/log/zookeeper', local_dir, options.verbose)
except ShellException:
pass
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/kern.log',
local_dir, options.verbose)
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/syslog',
local_dir, options.verbose)
AppScaleLogger.success("Successfully copied logs to {0}".format(
options.location))
def update_local_metadata(cls, options, node_layout, host, instance_id):
"""Writes a locations.yaml and locations.json file to the local filesystem,
that the tools can use to locate machines in an AppScale deployment.
Args:
options: A Namespace that indicates deployment-specific parameters not
relating to the placement strategy in use.
node_layout: A NodeLayout that indicates the placement strategy in use
for this deployment.
host: A str representing the location we can reach an AppController at.
instance_id: The instance ID (if running in a cloud environment)
associated with the given host.
"""
# find out every machine's IP address and what they're doing
acc = AppControllerClient(host, cls.get_secret_key(options.keyname))
all_ips = [str(ip) for ip in acc.get_all_public_ips()]
role_info = acc.get_role_info()
infrastructure = options.infrastructure or 'xen'
# write our yaml metadata file
yaml_contents = {
'load_balancer': str(host),
'instance_id': str(instance_id),
'table': options.table,
'secret': cls.get_secret_key(options.keyname),
'db_master': node_layout.db_master().public_ip,
'ips': all_ips,
'infrastructure': infrastructure,
'group': options.group,
}
if infrastructure != "xen":
yaml_contents['zone'] = options.zone
if infrastructure == "gce":
yaml_contents['project'] = options.project
with open(cls.get_locations_yaml_location(options.keyname), 'w') \
as file_handle:
file_handle.write(
yaml.dump(yaml_contents, default_flow_style=False))
# and now we can write the json metadata file
with open(cls.get_locations_json_location(options.keyname), 'w') \
as file_handle:
file_handle.write(json.dumps(role_info))
def update_local_metadata(cls, options, node_layout, host, instance_id):
"""Writes a locations.yaml and locations.json file to the local filesystem,
that the tools can use to locate machines in an AppScale deployment.
Args:
options: A Namespace that indicates deployment-specific parameters not
relating to the placement strategy in use.
node_layout: A NodeLayout that indicates the placement strategy in use
for this deployment.
host: A str representing the location we can reach an AppController at.
instance_id: The instance ID (if running in a cloud environment)
associated with the given host.
"""
# find out every machine's IP address and what they're doing
acc = AppControllerClient(host, cls.get_secret_key(options.keyname))
all_ips = [str(ip) for ip in acc.get_all_public_ips()]
role_info = acc.get_role_info()
infrastructure = options.infrastructure or 'xen'
# write our yaml metadata file
yaml_contents = {
'load_balancer' : str(host),
'instance_id' : str(instance_id),
'table' : options.table,
'secret' : cls.get_secret_key(options.keyname),
'db_master' : node_layout.db_master().public_ip,
'ips' : all_ips,
'infrastructure' : infrastructure,
'group' : options.group,
}
if infrastructure != "xen":
yaml_contents['zone'] = options.zone
if infrastructure == "gce":
yaml_contents['project'] = options.project
with open(cls.get_locations_yaml_location(options.keyname), 'w') \
as file_handle:
file_handle.write(yaml.dump(yaml_contents, default_flow_style=False))
# and now we can write the json metadata file
with open(cls.get_locations_json_location(options.keyname), 'w') \
as file_handle:
file_handle.write(json.dumps(role_info))
def wait_for_machines_to_finish_loading(cls, host, keyname):
"""Queries all of the AppControllers in this AppScale deployment to see if
they have started all of the API services on their machine, and if not,
waits until they have.
Args:
host: The location where an AppController can be found, who will then have
the locations of all the other AppControllers in this AppScale
deployment.
keyname: The name of the SSH keypair used for this AppScale deployment.
"""
acc = AppControllerClient(host, LocalState.get_secret_key(keyname))
all_ips = acc.get_all_public_ips()
for ip in all_ips:
while True:
acc = AppControllerClient(ip, LocalState.get_secret_key(keyname))
if acc.is_initialized():
break
else:
time.sleep(cls.WAIT_TIME)
def describe_instances(cls, options):
"""Queries each node in the currently running AppScale deployment and
reports on their status.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
login_host = LocalState.get_login_host(options.keyname)
login_acc = AppControllerClient(login_host,
LocalState.get_secret_key(options.keyname))
for ip in login_acc.get_all_public_ips():
acc = AppControllerClient(ip, LocalState.get_secret_key(options.keyname))
AppScaleLogger.log("Status of node at {0}:".format(ip))
try:
AppScaleLogger.log(acc.get_status())
except Exception as e:
AppScaleLogger.warn("Unable to contact machine: {0}\n".format(str(e)))
AppScaleLogger.success("View status information about your AppScale " + \
"deployment at http://{0}/status".format(login_host))
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {0}, already exists.".format(options.location))
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
try:
all_ips = acc.get_all_public_ips()
except socket.error: # Occurs when the AppController has failed.
AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \
"machines in this AppScale deployment. Using our locally cached " + \
"info instead.")
all_ips = LocalState.get_all_public_ips(options.keyname)
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
# The log paths that we collect logs from.
log_paths = [
'/var/log/appscale', '/var/log/kern.log*', '/var/log/monit.log*',
'/var/log/nginx', '/var/log/syslog*', '/var/log/zookeeper'
]
failures = False
for ip in all_ips:
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
for log_path in log_paths:
try:
RemoteHelper.scp_remote_to_local(ip, options.keyname,
log_path, local_dir,
options.verbose)
except ShellException as shell_exception:
failures = True
AppScaleLogger.warn(
"Unable to collect logs from '{}' for host '{}'".
format(log_path, ip))
AppScaleLogger.verbose(
"Encountered exception: {}".format(
str(shell_exception)), options.verbose)
if failures:
AppScaleLogger.log(
"Done copying to {0}. There were "
"failures while collecting AppScale logs.".format(
options.location))
else:
AppScaleLogger.success(
"Successfully collected all AppScale logs into "
"{0}".format(options.location))
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException(
"Can't gather logs, as the location you " + "specified, {0}, already exists.".format(options.location)
)
acc = AppControllerClient(
LocalState.get_login_host(options.keyname), LocalState.get_secret_key(options.keyname)
)
try:
all_ips = acc.get_all_public_ips()
except socket.error: # Occurs when the AppController has failed.
AppScaleLogger.warn(
"Couldn't get an up-to-date listing of the "
+ "machines in this AppScale deployment. Using our locally cached "
+ "info instead."
)
all_ips = LocalState.get_all_public_ips(options.keyname)
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
# The log paths that we collect logs from.
log_paths = [
"/var/log/appscale",
"/var/log/kern.log*",
"/var/log/monit.log*",
"/var/log/nginx",
"/var/log/syslog*",
"/var/log/zookeeper",
]
failures = False
for ip in all_ips:
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
for log_path in log_paths:
try:
RemoteHelper.scp_remote_to_local(ip, options.keyname, log_path, local_dir, options.verbose)
except ShellException as shell_exception:
failures = True
AppScaleLogger.warn("Unable to collect logs from '{}' for host '{}'".format(log_path, ip))
AppScaleLogger.verbose("Encountered exception: {}".format(str(shell_exception)), options.verbose)
if failures:
AppScaleLogger.log(
"Done copying to {0}. There were " "failures while collecting AppScale logs.".format(options.location)
)
else:
AppScaleLogger.success("Successfully collected all AppScale logs into " "{0}".format(options.location))
def terminate_virtualized_cluster(cls, keyname, clean, is_verbose):
"""Stops all API services running on all nodes in the currently running
AppScale deployment.
Args:
keyname: The name of the SSH keypair used for this AppScale deployment.
is_verbose: A bool that indicates if we should print the commands executed
to stdout.
clean: A bool representing whether clean should be ran on the nodes.
"""
AppScaleLogger.log("Stopping appscale deployment with keyname {0}"
.format(keyname))
time.sleep(2)
shadow_host = LocalState.get_host_with_role(keyname, 'shadow')
try:
secret = LocalState.get_secret_key(keyname)
except IOError:
# We couldn't find the secret key: AppScale is most likely not
# running.
raise AppScaleException("Couldn't find AppScale secret key.")
acc = AppControllerClient(shadow_host, secret)
try:
machines = len(acc.get_all_public_ips()) - 1
acc.run_terminate(clean)
terminated_successfully = True
log_dump = u""
while not acc.is_appscale_terminated():
# For terminate receive_server_message will return a JSON string that
# is a list of dicts with keys: ip, status, output
try:
output_list = yaml.safe_load(acc.receive_server_message())
except Exception as e:
log_dump += e.message
continue
for node in output_list:
if node.get("status"):
machines -= 1
AppScaleLogger.success("Node at {node_ip}: {status}".format(
node_ip=node.get("ip"), status="Stopping AppScale finished"))
else:
AppScaleLogger.warn("Node at {node_ip}: {status}".format(
node_ip=node.get("ip"), status="Stopping AppScale failed"))
terminated_successfully = False
log_dump += u"Node at {node_ip}: {status}\nNode Output:"\
u"{output}".format(node_ip=node.get("ip"),
status="Stopping AppScale failed",
output=node.get("output"))
AppScaleLogger.verbose(u"Output of node at {node_ip}:\n"
u"{output}".format(node_ip=node.get("ip"),
output=node.get("output")),
is_verbose)
if not terminated_successfully or machines > 0:
LocalState.generate_crash_log(AppControllerException, log_dump)
raise AppScaleException("{0} node(s) failed stopping AppScale, "
"head node is still running AppScale services."
.format(machines))
cls.stop_remote_appcontroller(shadow_host, keyname, is_verbose, clean)
except socket.error as socket_error:
AppScaleLogger.warn(u'Unable to talk to AppController: {}'.
format(socket_error.message))
raise
except Exception as exception:
AppScaleLogger.verbose(u'Saw Exception while stopping AppScale {0}'.
format(str(exception)), is_verbose)
raise
