def paasta_emergency_restart(args):
"""Performs an emergency restart on a given service instance on a given cluster
Warning: This command is only intended to be used in an emergency.
It should not be needed in normal circumstances.
"""
service = figure_out_service_name(args)
print "Performing an emergency restart on %s...\n" % compose_job_id(service, args.instance)
execute_paasta_serviceinit_on_remote_master('restart', args.cluster, service, args.instance)
print "%s" % "\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:])
print "Run this to see the status:"
print "paasta status --service %s --clusters %s" % (service, args.cluster)
def paasta_emergency_restart(args):
"""Performs an emergency restart on a given service instance on a given cluster
Warning: This command is only intended to be used in an emergency.
It should not be needed in normal circumstances.
"""
service = figure_out_service_name(args)
print "Performing an emergency restart on %s...\n" % compose_job_id(
service, args.instance)
execute_paasta_serviceinit_on_remote_master('restart', args.cluster,
service, args.instance)
print "%s" % "\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:])
print "Run this to see the status:"
print "paasta status --service %s --clusters %s" % (service, args.cluster)
def test_execute_paasta_serviceinit_status_on_remote_master_happy_path(
mock_run_paasta_serviceinit,
mock_find_connectable_master,
mock_calculate_remote_masters,
):
cluster = 'fake_cluster_name'
service = 'fake_service'
instancename = 'fake_instance'
remote_masters = (
'fake_master1',
'fake_master2',
'fake_master3',
)
mock_calculate_remote_masters.return_value = (remote_masters, None)
mock_find_connectable_master.return_value = ('fake_connectable_master', None)
actual = utils.execute_paasta_serviceinit_on_remote_master('status', cluster, service, instancename)
mock_calculate_remote_masters.assert_called_once_with(cluster)
mock_find_connectable_master.assert_called_once_with(remote_masters)
mock_run_paasta_serviceinit.assert_called_once_with(
'status',
'fake_connectable_master',
service,
instancename,
cluster,
)
assert actual == mock_run_paasta_serviceinit.return_value
def test_execute_paasta_serviceinit_status_on_remote_master_happy_path(
mock_run_paasta_serviceinit,
mock_find_connectable_master,
mock_calculate_remote_masters,
system_paasta_config,
):
cluster = "fake_cluster_name"
service = "fake_service"
instancename = "fake_instance"
remote_masters = ["fake_master1", "fake_master2", "fake_master3"]
mock_run_paasta_serviceinit.return_value = (
mock.sentinel.paasta_serviceinit_return_code,
mock.sentinel.paasta_serviceinit_output,
)
mock_calculate_remote_masters.return_value = (remote_masters, None)
mock_find_connectable_master.return_value = ("fake_connectable_master",
None)
return_code, actual = utils.execute_paasta_serviceinit_on_remote_master(
"status", cluster, service, instancename, system_paasta_config)
mock_calculate_remote_masters.assert_called_once_with(
cluster, system_paasta_config)
mock_find_connectable_master.assert_called_once_with(remote_masters)
mock_run_paasta_serviceinit.assert_called_once_with(
"status", "fake_connectable_master", service, instancename, cluster,
False)
assert return_code == mock.sentinel.paasta_serviceinit_return_code
assert actual == mock.sentinel.paasta_serviceinit_output
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
Warning: This command is not magic and cannot actually get a service to start if it couldn't
run before. This includes configurations that prevent the service from running,
such as 'instances: 0' (for Marathon apps).
All it does for Marathon apps is ask Marathon to resume normal operation by scaling up to
the instance count defined in the service's config.
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency start on %s..." % compose_job_id(
service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand='start',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config)
print "%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:])
print "Output: %s" % PaastaColors.grey(output)
print "Run this command to see the status:"
print "paasta status --service %s --clusters %s" % (service, args.cluster)
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
Warning: This command is not magic and cannot actually get a service to start if it couldn't
run before. This includes configurations that prevent the service from running,
such as 'instances: 0' (for Marathon apps).
All it does for Marathon apps is ask Marathon to resume normal operation by scaling up to
the instance count defined in the service's config.
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency start on %s..." % compose_job_id(service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand="start",
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
print "%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:])
print "Output: %s" % PaastaColors.grey(output)
print "Run this command to see the status:"
print "paasta status --service %s --clusters %s" % (service, args.cluster)
def paasta_emergency_stop(args):
"""Performs an emergency stop on a given service instance on a given cluster
Warning: This command does not permanently stop the service. The next time the service is updated
(config change, deploy, bounce, etc.), those settings will override the emergency stop.
If you want this stop to be permanant, adjust the relevant config file to reflect that.
For example, this can be done for Marathon apps by setting 'instances: 0', or
for Chronos jobs by setting 'disabled: True'. Alternatively, remove the config yaml entirely.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency stop on %s..." % compose_job_id(service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand="stop",
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
app_id=args.appid,
)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
print "To start this service again asap, run:"
print "paasta emergency-start --service %s --instance %s --cluster %s" % (service, args.instance, args.cluster)
def report_status_for_cluster(service, cluster, deploy_pipeline, actual_deployments, instance_whitelist, verbose=0):
"""With a given service and cluster, prints the status of the instances
in that cluster"""
print
print "cluster: %s" % cluster
seen_instances = []
for namespace in deploy_pipeline:
cluster_in_pipeline, instance = namespace.split('.')
seen_instances.append(instance)
if cluster_in_pipeline != cluster:
continue
if instance_whitelist and instance not in instance_whitelist:
continue
# Case: service deployed to cluster.instance
if namespace in actual_deployments:
formatted_instance = PaastaColors.blue(instance)
version = actual_deployments[namespace][:8]
# TODO: Perform sanity checks once per cluster instead of for each namespace
status = execute_paasta_serviceinit_on_remote_master('status', cluster, service, instance, verbose=verbose)
# Case: service NOT deployed to cluster.instance
else:
formatted_instance = PaastaColors.red(instance)
version = 'None'
status = None
print ' instance: %s' % formatted_instance
print ' Git sha: %s' % version
if status is not None:
for line in status.rstrip().split('\n'):
print ' %s' % line
print report_invalid_whitelist_values(instance_whitelist, seen_instances, 'instance')
def paasta_emergency_stop(args):
"""Performs an emergency stop on a given service instance on a given cluster
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
paasta_print("Performing an emergency stop on %s..." %
compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand="stop",
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
_log_audit(
action="emergency-stop",
service=service,
cluster=args.cluster,
instance=args.instance,
)
paasta_print("Output: %s" % output)
paasta_print("%s" %
"\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:]))
paasta_print("To start this service again asap, run:")
paasta_print(
f"paasta emergency-start --service {service} --instance {args.instance} --cluster {args.cluster}"
)
return return_code
def test_execute_paasta_serviceinit_status_on_remote_master_happy_path(
mock_run_paasta_serviceinit,
mock_find_connectable_master,
mock_calculate_remote_masters,
):
cluster = 'fake_cluster_name'
service = 'fake_service'
instancename = 'fake_instance'
remote_masters = (
'fake_master1',
'fake_master2',
'fake_master3',
)
mock_run_paasta_serviceinit.return_value = (
mock.sentinel.paasta_serviceinit_return_code,
mock.sentinel.paasta_serviceinit_output)
mock_calculate_remote_masters.return_value = (remote_masters, None)
mock_find_connectable_master.return_value = ('fake_connectable_master',
None)
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
return_code, actual = utils.execute_paasta_serviceinit_on_remote_master(
'status', cluster, service, instancename, fake_system_paasta_config)
mock_calculate_remote_masters.assert_called_once_with(
cluster, fake_system_paasta_config)
mock_find_connectable_master.assert_called_once_with(remote_masters)
mock_run_paasta_serviceinit.assert_called_once_with(
'status', 'fake_connectable_master', service, instancename, cluster,
False)
assert return_code == mock.sentinel.paasta_serviceinit_return_code
assert actual == mock.sentinel.paasta_serviceinit_output
def test_execute_paasta_serviceinit_status_on_remote_master_happy_path(
mock_run_paasta_serviceinit,
mock_find_connectable_master,
mock_calculate_remote_masters,
):
cluster = 'fake_cluster_name'
service = 'fake_service'
instancename = 'fake_instance'
remote_masters = (
'fake_master1',
'fake_master2',
'fake_master3',
)
mock_run_paasta_serviceinit.return_value = (
mock.sentinel.paasta_serviceinit_return_code, mock.sentinel.paasta_serviceinit_output)
mock_calculate_remote_masters.return_value = (remote_masters, None)
mock_find_connectable_master.return_value = ('fake_connectable_master', None)
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
return_code, actual = utils.execute_paasta_serviceinit_on_remote_master('status', cluster, service, instancename,
fake_system_paasta_config)
mock_calculate_remote_masters.assert_called_once_with(cluster, fake_system_paasta_config)
mock_find_connectable_master.assert_called_once_with(remote_masters)
mock_run_paasta_serviceinit.assert_called_once_with(
'status',
'fake_connectable_master',
service,
instancename,
cluster,
False
)
assert return_code == mock.sentinel.paasta_serviceinit_return_code
assert actual == mock.sentinel.paasta_serviceinit_output
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
paasta_print("Performing an emergency start on %s..." % compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand='start',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
_log_audit(
action='emergency-start',
service=service,
cluster=args.cluster,
instance=args.instance,
)
paasta_print("%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:]))
paasta_print("Output: %s" % PaastaColors.grey(output))
paasta_print("Run this command to see the status:")
paasta_print(f"paasta status --service {service} --clusters {args.cluster}")
return return_code
def paasta_emergency_restart(args):
"""Performs an emergency restart on a given service instance on a given cluster
Warning: This command is only intended to be used in an emergency.
It should not be needed in normal circumstances.
"""
service = figure_out_service_name(args, args.soa_dir)
system_paasta_config = load_system_paasta_config()
paasta_print("Performing an emergency restart on %s...\n" %
compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand='restart',
cluster=args.cluster,
service=args.service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
paasta_print("Output: %s" % output)
paasta_print("%s" %
"\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:]))
paasta_print("Run this to see the status:")
paasta_print(
f"paasta status --service {service} --clusters {args.cluster}")
return return_code
def test_execute_paasta_serviceinit_status_on_remote_master_happy_path(
mock_run_paasta_serviceinit,
mock_find_connectable_master,
mock_calculate_remote_masters,
):
cluster = 'fake_cluster_name'
service = 'fake_service'
instancename = 'fake_instance'
remote_masters = (
'fake_master1',
'fake_master2',
'fake_master3',
)
mock_calculate_remote_masters.return_value = (remote_masters, None)
mock_find_connectable_master.return_value = ('fake_connectable_master',
None)
actual = utils.execute_paasta_serviceinit_on_remote_master(
'status', cluster, service, instancename)
mock_calculate_remote_masters.assert_called_once_with(cluster)
mock_find_connectable_master.assert_called_once_with(remote_masters)
mock_run_paasta_serviceinit.assert_called_once_with(
'status',
'fake_connectable_master',
service,
instancename,
cluster,
)
assert actual == mock_run_paasta_serviceinit.return_value
def paasta_emergency_stop(args):
"""Performs an emergency stop on a given service instance on a given cluster
Warning: This command does not permanently stop the service. The next time the service is updated
(config change, deploy, bounce, etc.), those settings will override the emergency stop.
If you want this stop to be permanant, adjust the relevant config file to reflect that.
For example, this can be done for Marathon apps by setting 'instances: 0', or
for Chronos jobs by setting 'disabled: True'. Alternatively, remove the config yaml entirely.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency stop on %s..." % compose_job_id(
service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand='stop',
cluster=args.cluster,
service=args.service,
instances=args.instance,
system_paasta_config=system_paasta_config,
app_id=args.appid)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
print "To start this service again asap, run:"
print "paasta emergency-start --service %s --instance %s --cluster %s" % (
service, args.instance, args.cluster)
def report_status_for_cluster(
service, cluster, deploy_pipeline, actual_deployments, instance_whitelist,
system_paasta_config, verbose=0, use_api_endpoint=False,
):
"""With a given service and cluster, prints the status of the instances
in that cluster"""
paasta_print()
paasta_print("service: %s" % service)
paasta_print("cluster: %s" % cluster)
seen_instances = []
deployed_instances = []
for namespace in deploy_pipeline:
cluster_in_pipeline, instance = namespace.split('.')
seen_instances.append(instance)
if cluster_in_pipeline != cluster:
continue
if instance_whitelist and instance not in instance_whitelist:
continue
# Case: service deployed to cluster.instance
if namespace in actual_deployments:
deployed_instances.append(instance)
# Case: service NOT deployed to cluster.instance
else:
paasta_print(' instance: %s' % PaastaColors.red(instance))
paasta_print(' Git sha: None (not deployed yet)')
return_code = 0
if len(deployed_instances) > 0:
if use_api_endpoint:
return_codes = [
paasta_status_on_api_endpoint(
cluster,
service,
deployed_instance,
system_paasta_config,
verbose=verbose,
)
for deployed_instance in deployed_instances
]
if any(return_code != 200 for return_code in return_codes):
return_code = 1
else:
return_code, status = execute_paasta_serviceinit_on_remote_master(
'status', cluster, service, ','.join(deployed_instances),
system_paasta_config, stream=True, verbose=verbose,
ignore_ssh_output=True,
)
# Status results are streamed. This print is for possible error messages.
if status is not None:
for line in status.rstrip().split('\n'):
paasta_print(' %s' % line)
paasta_print(report_invalid_whitelist_values(instance_whitelist, seen_instances, 'instance'))
return return_code
def report_status_for_cluster(service, cluster, deploy_pipeline, actual_deployments, instance_whitelist,
system_paasta_config, verbose=0, use_api_endpoint=False):
"""With a given service and cluster, prints the status of the instances
in that cluster"""
paasta_print()
paasta_print("cluster: %s" % cluster)
seen_instances = []
deployed_instances = []
for namespace in deploy_pipeline:
cluster_in_pipeline, instance = namespace.split('.')
seen_instances.append(instance)
if cluster_in_pipeline != cluster:
continue
if instance_whitelist and instance not in instance_whitelist:
continue
# Case: service deployed to cluster.instance
if namespace in actual_deployments:
deployed_instances.append(instance)
# Case: service NOT deployed to cluster.instance
else:
paasta_print(' instance: %s' % PaastaColors.red(instance))
paasta_print(' Git sha: None (not deployed yet)')
return_code = 0
if len(deployed_instances) > 0:
if use_api_endpoint:
return_codes = [
paasta_status_on_api_endpoint(
cluster,
service,
deployed_instance,
system_paasta_config,
verbose=verbose,
)
for deployed_instance in deployed_instances
]
if any(return_code != 200 for return_code in return_codes):
return_code = 1
else:
return_code, status = execute_paasta_serviceinit_on_remote_master(
'status', cluster, service, ','.join(deployed_instances),
system_paasta_config, stream=True, verbose=verbose,
ignore_ssh_output=True)
# Status results are streamed. This print is for possible error messages.
if status is not None:
for line in status.rstrip().split('\n'):
paasta_print(' %s' % line)
paasta_print(report_invalid_whitelist_values(instance_whitelist, seen_instances, 'instance'))
return return_code
def report_status_for_cluster(service,
cluster,
deploy_pipeline,
actual_deployments,
instance_whitelist,
system_paasta_config,
verbose=0):
"""With a given service and cluster, prints the status of the instances
in that cluster"""
print
print "cluster: %s" % cluster
seen_instances = []
deployed_instances = []
for namespace in deploy_pipeline:
cluster_in_pipeline, instance = namespace.split('.')
seen_instances.append(instance)
if cluster_in_pipeline != cluster:
continue
if instance_whitelist and instance not in instance_whitelist:
continue
# Case: service deployed to cluster.instance
if namespace in actual_deployments:
deployed_instances.append(instance)
# Case: service NOT deployed to cluster.instance
else:
print ' instance: %s' % PaastaColors.red(instance)
print ' Git sha: None (not deployed yet)'
if len(deployed_instances) > 0:
status = execute_paasta_serviceinit_on_remote_master(
'status',
cluster,
service,
','.join(deployed_instances),
system_paasta_config,
stream=True,
verbose=verbose,
ignore_ssh_output=True)
# Status results are streamed. This print is for possible error messages.
if status is not None:
for line in status.rstrip().split('\n'):
print ' %s' % line
print report_invalid_whitelist_values(instance_whitelist, seen_instances,
'instance')
def paasta_emergency_scale(args):
"""Performs an emergency scale on a given service instance on a given cluster
Warning: This command does not permanently scale the service. The next time the service is updated
(config change, deploy, bounce, etc.), those settings will override the emergency scale.
If you want this scale to be permanant, adjust the relevant config file to reflect that.
For example, this can be done for Marathon apps by setting 'instances: n'
"""
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
print "Performing an emergency scale on %s..." % compose_job_id(service, args.instance)
output = execute_paasta_serviceinit_on_remote_master('scale', args.cluster, service, args.instance,
app_id=args.appid, delta=args.delta)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_scale.__doc__.splitlines()[-7:])
def paasta_emergency_scale(args):
"""Performs an emergency scale on a given service instance on a given cluster
Warning: This command does not permanently scale the service. The next time the service is updated
(config change, deploy, bounce, etc.), those settings will override the emergency scale.
If you want this scale to be permanant, adjust the relevant config file to reflect that.
For example, this can be done for Marathon apps by setting 'instances: n'
"""
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
print "Performing an emergency scale on %s..." % compose_job_id(service, args.instance)
output = execute_paasta_serviceinit_on_remote_master('scale', args.cluster, service, args.instance,
app_id=args.appid, delta=args.delta)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_scale.__doc__.splitlines()[-7:])
def test_execute_paasta_serviceinit_on_remote_no_connectable_master(
mock_run_paasta_serviceinit,
mock_check_ssh_and_sudo_on_master,
mock_find_connectable_master,
mock_calculate_remote_masters,
):
cluster = 'fake_cluster_name'
service = 'fake_service'
instancename = 'fake_instance'
mock_find_connectable_master.return_value = (None, "fake_err_msg")
mock_calculate_remote_masters.return_value = (['fake_master'], None)
actual = utils.execute_paasta_serviceinit_on_remote_master('status', cluster, service, instancename)
assert mock_check_ssh_and_sudo_on_master.call_count == 0
assert 'ERROR: could not find connectable master in cluster %s' % cluster in actual
assert "fake_err_msg" in actual
def test_execute_paasta_serviceinit_on_remote_no_connectable_master(
mock_run_paasta_serviceinit,
mock_check_ssh_and_sudo_on_master,
mock_find_connectable_master,
mock_calculate_remote_masters,
):
cluster = 'fake_cluster_name'
service = 'fake_service'
instancename = 'fake_instance'
mock_find_connectable_master.return_value = (None, "fake_err_msg")
mock_calculate_remote_masters.return_value = (['fake_master'], None)
actual = utils.execute_paasta_serviceinit_on_remote_master('status', cluster, service, instancename)
assert mock_check_ssh_and_sudo_on_master.call_count == 0
assert 'ERROR: could not find connectable master in cluster %s' % cluster in actual
assert "fake_err_msg" in actual
def paasta_emergency_stop(args):
"""Performs an emergency stop on a given service instance on a given cluster
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency stop on %s..." % compose_job_id(
service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand='stop',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
print "To start this service again asap, run:"
print "paasta emergency-start --service %s --instance %s --cluster %s" % (
service, args.instance, args.cluster)
def test_execute_paasta_serviceinit_on_remote_no_connectable_master(
mock_run_paasta_serviceinit,
mock_check_ssh_on_master,
mock_find_connectable_master,
mock_calculate_remote_masters,
system_paasta_config,
):
cluster = "fake_cluster_name"
service = "fake_service"
instancename = "fake_instance"
mock_find_connectable_master.return_value = (None, "fake_err_msg")
mock_calculate_remote_masters.return_value = (["fake_master"], None)
return_code, actual = utils.execute_paasta_serviceinit_on_remote_master(
"status", cluster, service, instancename, system_paasta_config)
assert mock_check_ssh_on_master.call_count == 0
assert "ERROR: could not find connectable master in cluster %s" % cluster in actual
assert return_code == 255
assert "fake_err_msg" in actual
def paasta_emergency_stop(args):
"""Performs an emergency stop on a given service instance on a given cluster
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency stop on %s..." % compose_job_id(service, args.instance)
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand='stop',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
print "Output: %s" % output
print "%s" % "\n".join(paasta_emergency_stop.__doc__.splitlines()[-7:])
print "To start this service again asap, run:"
print "paasta emergency-start --service %s --instance %s --cluster %s" % (service, args.instance, args.cluster)
return return_code
def test_execute_paasta_serviceinit_on_remote_no_connectable_master(
mock_run_paasta_serviceinit,
mock_check_ssh_on_master,
mock_find_connectable_master,
mock_calculate_remote_masters,
):
cluster = 'fake_cluster_name'
service = 'fake_service'
instancename = 'fake_instance'
mock_find_connectable_master.return_value = (None, "fake_err_msg")
mock_calculate_remote_masters.return_value = (['fake_master'], None)
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
return_code, actual = utils.execute_paasta_serviceinit_on_remote_master(
'status', cluster, service, instancename, fake_system_paasta_config)
assert mock_check_ssh_on_master.call_count == 0
assert 'ERROR: could not find connectable master in cluster %s' % cluster in actual
assert return_code == 255
assert "fake_err_msg" in actual
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
paasta_print("Performing an emergency start on %s..." % compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand='start',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config
)
paasta_print("%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:]))
paasta_print("Output: %s" % PaastaColors.grey(output))
paasta_print("Run this command to see the status:")
paasta_print("paasta status --service %s --clusters %s" % (service, args.cluster))
return return_code
def paasta_emergency_restart(args):
"""Performs an emergency restart on a given service instance on a given cluster
Warning: This command is only intended to be used in an emergency.
It should not be needed in normal circumstances.
"""
service = figure_out_service_name(args, args.soa_dir)
system_paasta_config = load_system_paasta_config()
paasta_print("Performing an emergency restart on %s...\n" % compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand='restart',
cluster=args.cluster,
service=args.service,
instances=args.instance,
system_paasta_config=system_paasta_config
)
paasta_print("Output: %s" % output)
paasta_print("%s" % "\n".join(paasta_emergency_restart.__doc__.splitlines()[-7:]))
paasta_print("Run this to see the status:")
paasta_print("paasta status --service %s --clusters %s" % (service, args.cluster))
return return_code
def report_status_for_cluster(service, cluster, deploy_pipeline, actual_deployments, instance_whitelist,
system_paasta_config, verbose=0):
"""With a given service and cluster, prints the status of the instances
in that cluster"""
print
print "cluster: %s" % cluster
seen_instances = []
deployed_instances = []
for namespace in deploy_pipeline:
cluster_in_pipeline, instance = namespace.split('.')
seen_instances.append(instance)
if cluster_in_pipeline != cluster:
continue
if instance_whitelist and instance not in instance_whitelist:
continue
# Case: service deployed to cluster.instance
if namespace in actual_deployments:
deployed_instances.append(instance)
# Case: service NOT deployed to cluster.instance
else:
print ' instance: %s' % PaastaColors.red(instance)
print ' Git sha: None (not deployed yet)'
if len(deployed_instances) > 0:
status = execute_paasta_serviceinit_on_remote_master('status', cluster, service, ','.join(deployed_instances),
system_paasta_config, stream=True, verbose=verbose)
# Status results are streamed. This print is for possible error messages.
if status is not None:
for line in status.rstrip().split('\n'):
print ' %s' % line
print report_invalid_whitelist_values(instance_whitelist, seen_instances, 'instance')
def report_status_for_cluster(
service: str,
cluster: str,
deploy_pipeline: Sequence[str],
actual_deployments: Mapping[str, str],
instance_whitelist: Mapping[str, Type[InstanceConfig]],
system_paasta_config: SystemPaastaConfig,
verbose: int = 0,
use_api_endpoint: bool = False,
) -> Tuple[int, Sequence[str]]:
"""With a given service and cluster, prints the status of the instances
in that cluster"""
output = ["", "service: %s" % service, "cluster: %s" % cluster]
seen_instances = []
deployed_instances = []
instances = instance_whitelist.keys()
http_only_instances = [
instance
for instance, instance_config_class in instance_whitelist.items()
if instance_config_class in HTTP_ONLY_INSTANCE_CONFIG
]
ssh_only_instances = [
instance
for instance, instance_config_class in instance_whitelist.items()
if instance_config_class in SSH_ONLY_INSTANCE_CONFIG
]
tron_jobs = [
instance
for instance, instance_config_class in instance_whitelist.items()
if instance_config_class == TronActionConfig
]
for namespace in deploy_pipeline:
cluster_in_pipeline, instance = namespace.split(".")
seen_instances.append(instance)
if cluster_in_pipeline != cluster:
continue
if instances and instance not in instances:
continue
# Case: service deployed to cluster.instance
if namespace in actual_deployments:
deployed_instances.append(instance)
# Case: flink instances don't use `deployments.json`
elif instance_whitelist.get(instance) == FlinkDeploymentConfig:
deployed_instances.append(instance)
# Case: service NOT deployed to cluster.instance
else:
output.append(" instance: %s" % PaastaColors.red(instance))
output.append(" Git sha: None (not deployed yet)")
api_return_code = 0
ssh_return_code = 0
if len(deployed_instances) > 0:
http_only_deployed_instances = [
deployed_instance
for deployed_instance in deployed_instances
if (
deployed_instance in http_only_instances
or deployed_instance not in ssh_only_instances
and use_api_endpoint
)
]
if len(http_only_deployed_instances):
return_codes = [
paasta_status_on_api_endpoint(
cluster=cluster,
service=service,
instance=deployed_instance,
output=output,
system_paasta_config=system_paasta_config,
verbose=verbose,
)
for deployed_instance in http_only_deployed_instances
]
if any(return_codes):
api_return_code = 1
ssh_only_deployed_instances = [
deployed_instance
for deployed_instance in deployed_instances
if (
deployed_instance in ssh_only_instances
or deployed_instance not in http_only_instances
and not use_api_endpoint
)
]
if len(ssh_only_deployed_instances):
ssh_return_code, status = execute_paasta_serviceinit_on_remote_master(
"status",
cluster,
service,
",".join(
deployed_instance
for deployed_instance in ssh_only_deployed_instances
),
system_paasta_config,
stream=False,
verbose=verbose,
ignore_ssh_output=True,
)
# Status results are streamed. This print is for possible error messages.
if status is not None:
for line in status.rstrip().split("\n"):
output.append(" %s" % line)
if len(tron_jobs) > 0:
return_codes = [
paasta_status_on_api_endpoint(
cluster=cluster,
service=service,
instance=tron_job,
output=output,
system_paasta_config=system_paasta_config,
verbose=verbose,
)
for tron_job in tron_jobs
]
seen_instances.extend(tron_jobs)
output.append(
report_invalid_whitelist_values(instances, seen_instances, "instance")
)
if ssh_return_code:
return_code = ssh_return_code
elif api_return_code:
return_code = api_return_code
else:
return_code = 0
return return_code, output
def report_status_for_cluster(
service: str,
cluster: str,
deploy_pipeline: Sequence[str],
actual_deployments: Mapping[str, str],
instance_whitelist: Mapping[str, Type[InstanceConfig]],
system_paasta_config: SystemPaastaConfig,
verbose: int = 0,
use_api_endpoint: bool = False,
) -> Tuple[int, Sequence[str]]:
"""With a given service and cluster, prints the status of the instances
in that cluster"""
output = ['', 'service: %s' % service, 'cluster: %s' % cluster]
seen_instances = []
deployed_instances = []
instances = instance_whitelist.keys()
http_only_instances = [
instance
for instance, instance_config_class in instance_whitelist.items()
if instance_config_class in HTTP_ONLY_INSTANCE_CONFIG
]
for namespace in deploy_pipeline:
cluster_in_pipeline, instance = namespace.split('.')
seen_instances.append(instance)
if cluster_in_pipeline != cluster:
continue
if instances and instance not in instances:
continue
# Case: service deployed to cluster.instance
if namespace in actual_deployments:
deployed_instances.append(instance)
# Case: service NOT deployed to cluster.instance
else:
output.append(' instance: %s' % PaastaColors.red(instance))
output.append(' Git sha: None (not deployed yet)')
ssh_instances = [
i for i in deployed_instances if i not in http_only_instances
]
api_return_code = 0
ssh_return_code = 0
if len(deployed_instances) > 0:
if use_api_endpoint or http_only_instances:
return_codes = [
paasta_status_on_api_endpoint(
cluster=cluster,
service=service,
instance=deployed_instance,
output=output,
system_paasta_config=system_paasta_config,
verbose=verbose,
) for deployed_instance in deployed_instances if
(deployed_instance in http_only_instances or use_api_endpoint)
]
if any(return_codes):
api_return_code = 1
if not use_api_endpoint and ssh_instances or not http_only_instances:
ssh_return_code, status = execute_paasta_serviceinit_on_remote_master(
'status',
cluster,
service,
','.join(ssh_instances),
system_paasta_config,
stream=False,
verbose=verbose,
ignore_ssh_output=True,
)
# Status results are streamed. This print is for possible error messages.
if status is not None:
for line in status.rstrip().split('\n'):
output.append(' %s' % line)
output.append(
report_invalid_whitelist_values(instances, seen_instances, 'instance'))
if ssh_return_code:
return_code = ssh_return_code
elif api_return_code:
return_code = api_return_code
else:
return_code = 0
return return_code, output
