def status_mesos_tasks_verbose(job_id, get_short_task_id):
"""Returns detailed information about the mesos tasks for a service.
:param job_id: An id used for looking up Mesos tasks
:param get_short_task_id: A function which given a
task_id returns a short task_id suitable for
printing.
"""
output = []
running_and_active_tasks = get_running_tasks_from_active_frameworks(job_id)
output.append(" Running Tasks:")
rows_running = [[
"Mesos Task ID",
"Host deployed to",
"Ram",
"CPU",
"Deployed at what localtime"
]]
for task in running_and_active_tasks:
rows_running.append(format_running_mesos_task_row(task, get_short_task_id))
output.extend([" %s" % row for row in format_table(rows_running)])
non_running_tasks = reversed(get_non_running_tasks_from_active_frameworks(job_id)[-10:])
output.append(PaastaColors.grey(" Non-Running Tasks"))
rows_non_running = [[
PaastaColors.grey("Mesos Task ID"),
PaastaColors.grey("Host deployed to"),
PaastaColors.grey("Deployed at what localtime"),
PaastaColors.grey("Status"),
]]
for task in non_running_tasks:
rows_non_running.append(format_non_running_mesos_task_row(task, get_short_task_id))
output.extend([" %s" % row for row in format_table(rows_non_running)])
return "\n".join(output)
def paasta_status(args):
"""Print the status of a Yelp service running on PaaSTA.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
actual_deployments = get_actual_deployments(service, soa_dir)
system_paasta_config = load_system_paasta_config()
if args.clusters is not None:
cluster_whitelist = args.clusters.split(",")
else:
cluster_whitelist = []
if args.instances is not None:
instance_whitelist = args.instances.split(",")
else:
instance_whitelist = []
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(service, soa_dir))
try:
report_status(
service=service,
deploy_pipeline=deploy_pipeline,
actual_deployments=actual_deployments,
cluster_whitelist=cluster_whitelist,
instance_whitelist=instance_whitelist,
system_paasta_config=system_paasta_config,
verbose=args.verbose,
)
except CalledProcessError as e:
print PaastaColors.grey(PaastaColors.bold(e.cmd + " exited with non-zero return code."))
print PaastaColors.grey(e.output)
return e.returncode
else:
print missing_deployments_message(service)
def format_non_running_mesos_task_row(task, get_short_task_id):
"""Returns a pretty formatted string of a running mesos task attributes"""
return (
PaastaColors.grey(get_short_task_id(task['id'])),
PaastaColors.grey(get_short_hostname_from_task(task)),
PaastaColors.grey(get_first_status_timestamp(task)),
PaastaColors.grey(task['state']),
)
def _format_disabled_status(job):
status = PaastaColors.red("UNKNOWN")
if job.get("disabled", False):
status = PaastaColors.grey("Not scheduled")
else:
status = PaastaColors.green("Scheduled")
return status
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
paasta_print("Performing an emergency start on %s..." %
compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand="start",
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
_log_audit(
action="emergency-start",
service=service,
cluster=args.cluster,
instance=args.instance,
)
paasta_print("%s" %
"\n".join(paasta_emergency_start.__doc__.splitlines()[-8:]))
paasta_print("Output: %s" % PaastaColors.grey(output))
paasta_print("Run this command to see the status:")
paasta_print(
f"paasta status --service {service} --clusters {args.cluster}")
return return_code
def test_format_chronos_job_zero_mesos_tasks():
example_job = {}
desired_state = ''
running_tasks = []
verbose = False
actual = chronos_serviceinit.format_chronos_job_status(example_job, desired_state, running_tasks, verbose)
assert PaastaColors.grey('Not running') in actual
def test_format_chronos_job_zero_mesos_tasks(mock_status):
example_job = {'name': 'my_service my_instance', 'schedule': 'foo'}
running_tasks = []
verbose = False
mock_client = mock.Mock()
actual = chronos_serviceinit.format_chronos_job_status(mock_client, example_job, running_tasks, verbose)
assert PaastaColors.grey('Not running') in actual
def format_marathon_task_table(tasks):
rows = [("Mesos Task ID", "Host deployed to", "Deployed at what localtime",
"Health")]
for task in tasks:
local_deployed_datetime = datetime_from_utc_to_local(
datetime.fromtimestamp(task.deployed_timestamp))
if task.host is not None:
hostname = f"{task.host}:{task.port}"
else:
hostname = "Unknown"
if task.is_healthy is None:
health_check_status = PaastaColors.grey("N/A")
elif task.is_healthy:
health_check_status = PaastaColors.green("Healthy")
else:
health_check_status = PaastaColors.red("Unhealthy")
rows.append((
task.id,
hostname,
"{} ({})".format(
local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
humanize.naturaltime(local_deployed_datetime),
),
health_check_status,
))
return format_table(rows)
def create_mesos_non_running_tasks_table(non_running_tasks):
rows = []
table_header = [
"Mesos Task ID",
"Host deployed to",
"Deployed at what localtime",
"Status",
]
rows.append(table_header)
for task in non_running_tasks or []:
if task.deployed_timestamp is None:
deployed_at_string = "Unknown"
else:
deployed_at = datetime.fromtimestamp(task.deployed_timestamp)
deployed_at_string = "{} ({})".format(
deployed_at.strftime("%Y-%m-%dT%H:%M"),
humanize.naturaltime(deployed_at),
)
rows.append([task.id, task.hostname, deployed_at_string, task.state])
rows.extend(format_tail_lines_for_mesos_task(task.tail_lines, task.id))
table = format_table(rows)
return [PaastaColors.grey(formatted_row) for formatted_row in table]
def marathon_mesos_status_human(
error_message,
running_task_count,
expected_instance_count,
running_tasks,
non_running_tasks,
):
if error_message:
return [f"Mesos: {PaastaColors.red(error_message)}"]
output = []
output.append(
marathon_mesos_status_summary(running_task_count,
expected_instance_count))
if running_tasks or non_running_tasks:
output.append(" Running Tasks:")
running_tasks_table = create_mesos_running_tasks_table(running_tasks)
output.extend([f" {line}" for line in running_tasks_table])
output.append(PaastaColors.grey(" Non-running Tasks:"))
non_running_tasks_table = create_mesos_non_running_tasks_table(
non_running_tasks)
output.extend([f" {line}" for line in non_running_tasks_table])
return output
def _format_disabled_status(job):
status = PaastaColors.red("UNKNOWN")
if job.get("disabled", False):
status = PaastaColors.grey("Not scheduled")
else:
status = PaastaColors.green("Scheduled")
return status
def run_docker_container(
container_name,
volumes,
environment,
docker_img,
docker_cmd,
dry_run,
):
docker_run_args = dict(
container_name=container_name,
volumes=volumes,
env=environment,
docker_img=docker_img,
docker_cmd=docker_cmd,
)
docker_run_cmd = get_docker_run_cmd(**docker_run_args)
joined_docker_run_cmd = ' '.join(docker_run_cmd)
if dry_run:
paasta_print(json.dumps(docker_run_cmd))
return 0
else:
paasta_print('Running docker command:\n%s' %
PaastaColors.grey(joined_docker_run_cmd))
os.execlp('paasta_docker_wrapper', *docker_run_cmd)
return 0
def format_haproxy_backend_row(backend, is_correct_instance):
"""Pretty Prints the status of a given haproxy backend
Takes the fields described in the CSV format of haproxy:
http://www.haproxy.org/download/1.5/doc/configuration.txt
And tries to make a good guess about how to represent them in text
"""
backend_name = backend['svname']
backend_hostname = backend_name.split("_")[-1]
backend_port = backend_name.split("_")[0].split(":")[-1]
pretty_backend_name = "%s:%s" % (backend_hostname, backend_port)
if backend['status'] == "UP":
status = PaastaColors.default(backend['status'])
elif backend['status'] == 'DOWN' or backend['status'] == 'MAINT':
status = PaastaColors.red(backend['status'])
else:
status = PaastaColors.yellow(backend['status'])
lastcheck = "%s/%s in %sms" % (backend['check_status'],
backend['check_code'],
backend['check_duration'])
lastchange = humanize.naturaltime(
datetime.timedelta(seconds=int(backend['lastchg'])))
row = (
' %s' % pretty_backend_name,
lastcheck,
lastchange,
status,
)
if is_correct_instance:
return row
else:
return tuple(
PaastaColors.grey(remove_ansi_escape_sequences(col))
for col in row)
def format_kubernetes_pod_table(pods):
rows = [("Pod ID", "Host deployed to", "Deployed at what localtime",
"Health")]
for pod in pods:
local_deployed_datetime = datetime_from_utc_to_local(
datetime.fromtimestamp(pod.deployed_timestamp))
hostname = f"{pod.host}" if pod.host is not None else "Unknown"
if pod.phase is None or pod.phase == "Pending":
health_check_status = PaastaColors.grey("N/A")
elif pod.phase == "Running":
health_check_status = PaastaColors.green("Healthy")
else:
health_check_status = PaastaColors.red("Unhealthy")
rows.append((
pod.name,
hostname,
"{} ({})".format(
local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
humanize.naturaltime(local_deployed_datetime),
),
health_check_status,
))
rows.extend(format_tail_lines_for_mesos_task(pod.tail_lines, pod.name))
return format_table(rows)
def format_haproxy_backend_row(backend, is_correct_instance):
"""Pretty Prints the status of a given haproxy backend
Takes the fields described in the CSV format of haproxy:
http://www.haproxy.org/download/1.5/doc/configuration.txt
And tries to make a good guess about how to represent them in text
"""
backend_name = backend['svname']
backend_hostname = backend_name.split("_")[-1]
backend_port = backend_name.split("_")[0].split(":")[-1]
pretty_backend_name = "%s:%s" % (backend_hostname, backend_port)
if backend['status'] == "UP":
status = PaastaColors.default(backend['status'])
elif backend['status'] == 'DOWN' or backend['status'] == 'MAINT':
status = PaastaColors.red(backend['status'])
else:
status = PaastaColors.yellow(backend['status'])
lastcheck = "%s/%s in %sms" % (backend['check_status'], backend['check_code'], backend['check_duration'])
lastchange = humanize.naturaltime(datetime.timedelta(seconds=int(backend['lastchg'])))
row = (
' %s' % pretty_backend_name,
lastcheck,
lastchange,
status,
)
if is_correct_instance:
return row
else:
return tuple(PaastaColors.grey(remove_ansi_escape_sequences(col)) for col in row)
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
Warning: This command is not magic and cannot actually get a service to start if it couldn't
run before. This includes configurations that prevent the service from running,
such as 'instances: 0' (for Marathon apps).
All it does for Marathon apps is ask Marathon to resume normal operation by scaling up to
the instance count defined in the service's config.
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency start on %s..." % compose_job_id(service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand="start",
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config,
)
print "%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:])
print "Output: %s" % PaastaColors.grey(output)
print "Run this command to see the status:"
print "paasta status --service %s --clusters %s" % (service, args.cluster)
def _output_on_failure():
returncode = docker_client.inspect_container(container_id)['State']['ExitCode']
sys.stdout.write('Container exited: %d)' % returncode)
sys.stdout.write('Here is the stdout and stderr:\n')
sys.stdout.write(PaastaColors.grey(
docker_client.attach(container_id, stderr=True, stream=False, logs=True)
))
def test_format_chronos_job_zero_mesos_tasks(mock_status):
example_job = {"name": "my_service my_instance", "schedule": "foo"}
running_tasks = []
verbose = False
mock_client = mock.Mock()
actual = chronos_serviceinit.format_chronos_job_status(
mock_client, example_job, running_tasks, verbose)
assert PaastaColors.grey("Not running") in actual
def test_format_chronos_job_status_disabled():
example_job = {
'disabled': True,
}
running_tasks = []
verbose = False
actual = chronos_serviceinit.format_chronos_job_status(example_job, running_tasks, verbose)
assert PaastaColors.grey('Not scheduled') in actual
def _format_mesos_status(running_tasks):
mesos_status = PaastaColors.red("UNKNOWN")
num_tasks = len(running_tasks)
if num_tasks == 0:
mesos_status = PaastaColors.grey("Not running")
else:
mesos_status = PaastaColors.yellow("Running")
return mesos_status
def test_format_chronos_job_zero_mesos_tasks(mock_status):
example_job = {'name': 'my_service my_instance', 'schedule': 'foo'}
running_tasks = []
verbose = False
mock_client = mock.Mock()
actual = chronos_serviceinit.format_chronos_job_status(
mock_client, example_job, running_tasks, verbose)
assert PaastaColors.grey('Not running') in actual
def test_format_chronos_job_status_disabled():
example_job = {
'disabled': True,
}
running_tasks = []
verbose = False
actual = chronos_serviceinit.format_chronos_job_status(example_job, running_tasks, verbose)
assert PaastaColors.grey('Not scheduled') in actual
def build_and_push_docker_image(args):
"""
Build an image if the default Spark service image is not preferred.
The image needs to be pushed to a registry for the Spark executors
to pull.
"""
if not makefile_responds_to('cook-image'):
paasta_print(
"A local Makefile with a 'cook-image' target is required for --build",
file=sys.stderr,
)
return None
default_tag = '{}-{}'.format(DEFAULT_SPARK_DOCKER_IMAGE_PREFIX,
get_username())
docker_tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = docker_tag
cook_return = paasta_cook_image(
args=None,
service=args.service,
soa_dir=args.yelpsoa_config_root,
)
if cook_return is not 0:
return None
docker_url = f'{args.docker_registry}/{docker_tag}'
command = f'docker tag {docker_tag} {docker_url}'
paasta_print(PaastaColors.grey(command))
retcode, _ = _run(command, stream=True)
if retcode is not 0:
return None
if args.docker_registry != DEFAULT_SPARK_DOCKER_REGISTRY:
command = 'sudo -H docker push %s' % docker_url
else:
command = 'docker push %s' % docker_url
paasta_print(PaastaColors.grey(command))
retcode, output = _run(command, stream=True)
if retcode is not 0:
return None
return docker_url
def _format_mesos_status(job, running_tasks):
mesos_status = PaastaColors.red("UNKNOWN")
num_tasks = len(running_tasks)
if num_tasks == 0:
mesos_status = PaastaColors.grey("Not running")
elif num_tasks == 1:
mesos_status = PaastaColors.yellow("Running")
else:
mesos_status = PaastaColors.red("Critical - %d tasks running (expected 1)" % num_tasks)
return mesos_status
def _output_on_failure():
sys.stdout.write(
'Your service failed to start, here is the stdout and stderr\n'
)
sys.stdout.write(
PaastaColors.grey(
docker_client.attach(container_id,
stderr=True,
stream=False,
logs=True)))
def _format_mesos_status(job, running_tasks):
mesos_status = PaastaColors.red("UNKNOWN")
num_tasks = len(running_tasks)
if num_tasks == 0:
mesos_status = PaastaColors.grey("Not running")
elif num_tasks == 1:
mesos_status = PaastaColors.yellow("Running")
else:
mesos_status = PaastaColors.red("Critical - %d tasks running (expected 1)" % num_tasks)
return mesos_status
def _output_on_failure():
returncode = docker_client.inspect_container(
container_id)['State']['ExitCode']
sys.stdout.write('Container exited: %d)' % returncode)
sys.stdout.write('Here is the stdout and stderr:\n')
sys.stdout.write(
PaastaColors.grey(
docker_client.attach(container_id,
stderr=True,
stream=False,
logs=True)))
def get_verbose_status_of_marathon_app(marathon_client, app, service, instance,
cluster, soa_dir):
"""Takes a given marathon app object and returns the verbose details
about the tasks, times, hosts, etc"""
output = []
create_datetime = datetime_from_utc_to_local(
isodate.parse_datetime(app.version))
output.append(" Marathon app ID: %s" % PaastaColors.bold(app.id))
output.append(
" App created: %s (%s)" %
(str(create_datetime), humanize.naturaltime(create_datetime)))
autoscaling_info = get_autoscaling_info(marathon_client, service, instance,
cluster, soa_dir)
if autoscaling_info:
output.append(" Autoscaling Info:")
headers = [
field.replace("_", " ").capitalize()
for field in ServiceAutoscalingInfo._fields
]
table = [headers, autoscaling_info]
output.append('\n'.join(
[" %s" % line for line in format_table(table)]))
output.append(" Tasks:")
rows = [("Mesos Task ID", "Host deployed to", "Deployed at what localtime",
"Health")]
for task in app.tasks:
local_deployed_datetime = datetime_from_utc_to_local(task.staged_at)
if task.host is not None:
hostname = "%s:%s" % (task.host.split(".")[0], task.ports[0])
else:
hostname = "Unknown"
if not task.health_check_results:
health_check_status = PaastaColors.grey("N/A")
elif marathon_tools.is_task_healthy(task):
health_check_status = PaastaColors.green("Healthy")
else:
health_check_status = PaastaColors.red("Unhealthy")
rows.append((
get_short_task_id(task.id),
hostname,
'%s (%s)' % (
local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
humanize.naturaltime(local_deployed_datetime),
),
health_check_status,
))
output.append('\n'.join([" %s" % line
for line in format_table(rows)]))
if len(app.tasks) == 0:
output.append(" No tasks associated with this marathon app")
return app.tasks, "\n".join(output)
def test_format_chronos_job_chronos_status(mock_status):
example_job = {
"name": "my_service my_instance",
"disabled": True,
"schedule": "foo",
}
running_tasks = []
verbose = False
mock_client = mock.Mock()
actual = chronos_serviceinit.format_chronos_job_status(
mock_client, example_job, running_tasks, verbose)
assert PaastaColors.grey("fake_chronos_status") in actual
def test_format_chronos_job_chronos_status(mock_status):
example_job = {
'name': 'my_service my_instance',
'disabled': True,
'schedule': 'foo',
}
running_tasks = []
verbose = False
mock_client = mock.Mock()
actual = chronos_serviceinit.format_chronos_job_status(
mock_client, example_job, running_tasks, verbose)
assert PaastaColors.grey('fake_chronos_status') in actual
def paasta_status(args):
"""Print the status of a Yelp service running on PaaSTA.
:param args: argparse.Namespace obj created from sys.args by cli"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
actual_deployments = get_actual_deployments(service, soa_dir)
system_paasta_config = load_system_paasta_config()
if 'USE_API_ENDPOINT' in os.environ:
use_api_endpoint = strtobool(os.environ.get('USE_API_ENDPOINT'))
else:
use_api_endpoint = False
if args.clusters is not None:
cluster_whitelist = args.clusters.split(",")
else:
cluster_whitelist = []
if args.instances is not None:
instance_whitelist = args.instances.split(",")
else:
instance_whitelist = []
if actual_deployments:
deploy_pipeline = list(get_planned_deployments(service, soa_dir))
try:
report_status(service=service,
deploy_pipeline=deploy_pipeline,
actual_deployments=actual_deployments,
cluster_whitelist=cluster_whitelist,
instance_whitelist=instance_whitelist,
system_paasta_config=system_paasta_config,
verbose=args.verbose,
use_api_endpoint=use_api_endpoint)
except CalledProcessError as e:
print PaastaColors.grey(
PaastaColors.bold(e.cmd +
" exited with non-zero return code."))
print PaastaColors.grey(e.output)
return e.returncode
else:
print missing_deployments_message(service)
def format_chronos_job_status(client, job, running_tasks, verbose=0):
"""Given a job, returns a pretty-printed human readable output regarding
the status of the job.
:param job: dictionary of the job status
:param running_tasks: a list of Mesos tasks associated with ``job``, e.g. the
result of ``mesos_tools.get_running_tasks_from_active_frameworks()``.
:param verbose: int verbosity level
"""
job_name = _format_job_name(job)
is_temporary = chronos_tools.is_temporary_job(job) if 'name' in job else 'UNKNOWN'
job_name = modify_string_for_rerun_status(job_name, is_temporary)
disabled_state = _format_disabled_status(job)
service, instance = chronos_tools.decompose_job_id(job['name'])
chronos_state = chronos_tools.get_chronos_status_for_job(client, service, instance)
(last_result, formatted_time) = _format_last_result(job)
job_type = chronos_tools.get_job_type(job)
schedule_type = _get_schedule_field_for_job_type(job_type)
schedule_formatter = get_schedule_formatter(job_type, verbose)
schedule_value = schedule_formatter(job)
command = _format_command(job)
mesos_status = _format_mesos_status(job, running_tasks)
if verbose > 0:
tail_lines = calculate_tail_lines(verbose_level=verbose)
mesos_status_verbose = status_mesos_tasks_verbose(
job_id=job["name"],
get_short_task_id=get_short_task_id,
tail_lines=tail_lines,
)
mesos_status = "%s\n%s" % (mesos_status, mesos_status_verbose)
return (
"Job: %(job_name)s\n"
" Status: %(disabled_state)s (%(chronos_state)s)"
" Last: %(last_result)s (%(formatted_time)s)\n"
" %(schedule_type)s: %(schedule_value)s\n"
" Command: %(command)s\n"
" Mesos: %(mesos_status)s" % {
"job_name": job_name,
"is_temporary": is_temporary,
"schedule_type": schedule_type,
"chronos_state": PaastaColors.grey(chronos_state),
"disabled_state": disabled_state,
"last_result": last_result,
"formatted_time": formatted_time,
"schedule_value": schedule_value,
"command": command,
"mesos_status": mesos_status,
}
)
def format_chronos_job_status(client, job, running_tasks, verbose=0):
"""Given a job, returns a pretty-printed human readable output regarding
the status of the job.
:param job: dictionary of the job status
:param running_tasks: a list of Mesos tasks associated with ``job``, e.g. the
result of ``mesos_tools.get_running_tasks_from_frameworks()``.
:param verbose: int verbosity level
"""
job_name = _format_job_name(job)
is_temporary = chronos_tools.is_temporary_job(
job) if 'name' in job else 'UNKNOWN'
job_name = modify_string_for_rerun_status(job_name, is_temporary)
disabled_state = _format_disabled_status(job)
service, instance = chronos_tools.decompose_job_id(job['name'])
chronos_state = chronos_tools.get_chronos_status_for_job(
client, service, instance)
(last_result, formatted_time) = _format_last_result(job)
job_type = chronos_tools.get_job_type(job)
schedule_type = _get_schedule_field_for_job_type(job_type)
schedule_formatter = get_schedule_formatter(job_type, verbose)
schedule_value = schedule_formatter(job)
command = _format_command(job)
mesos_status = _format_mesos_status(running_tasks)
if verbose > 0:
tail_lines = calculate_tail_lines(verbose_level=verbose)
mesos_status_verbose = status_mesos_tasks_verbose(
job_id=job["name"],
get_short_task_id=get_short_task_id,
tail_lines=tail_lines,
)
mesos_status = "%s\n%s" % (mesos_status, mesos_status_verbose)
return ("Job: %(job_name)s\n"
" Status: %(disabled_state)s (%(chronos_state)s)"
" Last: %(last_result)s (%(formatted_time)s)\n"
" %(schedule_type)s: %(schedule_value)s\n"
" Command: %(command)s\n"
" Mesos: %(mesos_status)s" % {
"job_name": job_name,
"is_temporary": is_temporary,
"schedule_type": schedule_type,
"chronos_state": PaastaColors.grey(chronos_state),
"disabled_state": disabled_state,
"last_result": last_result,
"formatted_time": formatted_time,
"schedule_value": schedule_value,
"command": command,
"mesos_status": mesos_status,
})
def prettify_level(level, requested_levels):
"""Colorize level. 'event' is special and gets bolded; everything else gets
lightened.
requested_levels is an iterable of levels that will be displayed. If only
one level will be displayed, don't bother to print it (return empty string).
If multiple levels will be displayed, emit the (prettified) level so the
resulting log output is not ambiguous.
"""
pretty_level = ''
if len(requested_levels) > 1:
if level == 'event':
pretty_level = PaastaColors.bold('[%s]' % level)
else:
pretty_level = PaastaColors.grey('[%s]' % level)
return pretty_level
def prettify_level(level, requested_levels):
"""Colorize level. 'event' is special and gets bolded; everything else gets
lightened.
requested_levels is an iterable of levels that will be displayed. If only
one level will be displayed, don't bother to print it (return empty string).
If multiple levels will be displayed, emit the (prettified) level so the
resulting log output is not ambiguous.
"""
pretty_level = ''
if len(requested_levels) > 1:
if level == 'event':
pretty_level = PaastaColors.bold('[%s]' % level)
else:
pretty_level = PaastaColors.grey('[%s]' % level)
return pretty_level
def get_marathon_app_deploy_status_human(app, app_id, client):
status, delay = get_marathon_app_deploy_status(app, app_id, client)
status_string = MarathonDeployStatus.tostring(status)
if status == MarathonDeployStatus.Waiting:
deploy_status = "%s (new tasks are not launching due to lack of capacity)" % PaastaColors.red(status_string)
elif status == MarathonDeployStatus.Delayed:
deploy_status = "%s (next task won't launch for %s seconds due to previous failures)" % (
PaastaColors.red(status_string), delay)
elif status == MarathonDeployStatus.Deploying:
deploy_status = PaastaColors.yellow(status_string)
elif status == MarathonDeployStatus.Stopped:
deploy_status = PaastaColors.grey(status_string)
else:
deploy_status = PaastaColors.bold(status_string)
return deploy_status
def get_verbose_status_of_marathon_app(app):
"""Takes a given marathon app object and returns the verbose details
about the tasks, times, hosts, etc"""
output = []
create_datetime = datetime_from_utc_to_local(
isodate.parse_datetime(app.version))
output.append(" Marathon app ID: %s" % PaastaColors.bold(app.id))
output.append(
" App created: %s (%s)" %
(str(create_datetime), humanize.naturaltime(create_datetime)))
output.append(" Tasks:")
rows = [("Mesos Task ID", "Host deployed to", "Deployed at what localtime",
"Health")]
for task in app.tasks:
local_deployed_datetime = datetime_from_utc_to_local(task.staged_at)
if task.host is not None:
hostname = "%s:%s" % (task.host.split(".")[0], task.ports[0])
else:
hostname = "Unknown"
health_check_results = [
health_check.alive for health_check in task.health_check_results
if health_check.alive is not None
]
if not health_check_results:
health_check_status = PaastaColors.grey("N/A")
elif all(health_check_results):
health_check_status = PaastaColors.green("Healthy")
else:
health_check_status = PaastaColors.red("Unhealthy")
rows.append((
get_short_task_id(task.id),
hostname,
'%s (%s)' % (
local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
humanize.naturaltime(local_deployed_datetime),
),
health_check_status,
))
output.append('\n'.join([" %s" % line
for line in format_table(rows)]))
if len(app.tasks) == 0:
output.append(" No tasks associated with this marathon app")
return app.tasks, "\n".join(output)
def marathon_app_deploy_status_human(status, backoff_seconds=None):
status_string = marathon_tools.MarathonDeployStatus.tostring(status)
if status == marathon_tools.MarathonDeployStatus.Waiting:
deploy_status = "%s (new tasks waiting for capacity to become available)" % PaastaColors.red(status_string)
elif status == marathon_tools.MarathonDeployStatus.Delayed:
deploy_status = "%s (tasks are crashing, next won't launch for another %s seconds)" % (
PaastaColors.red(status_string), backoff_seconds)
elif status == marathon_tools.MarathonDeployStatus.Deploying:
deploy_status = PaastaColors.yellow(status_string)
elif status == marathon_tools.MarathonDeployStatus.Stopped:
deploy_status = PaastaColors.grey(status_string)
elif status == marathon_tools.MarathonDeployStatus.Running:
deploy_status = PaastaColors.bold(status_string)
else:
deploy_status = status_string
return deploy_status
def marathon_app_deploy_status_human(status, backoff_seconds=None):
status_string = marathon_tools.MarathonDeployStatus.tostring(status)
if status == marathon_tools.MarathonDeployStatus.Waiting:
deploy_status = "%s (new tasks waiting for capacity to become available)" % PaastaColors.red(status_string)
elif status == marathon_tools.MarathonDeployStatus.Delayed:
deploy_status = "%s (tasks are crashing, next won't launch for another %s seconds)" % (
PaastaColors.red(status_string), backoff_seconds)
elif status == marathon_tools.MarathonDeployStatus.Deploying:
deploy_status = PaastaColors.yellow(status_string)
elif status == marathon_tools.MarathonDeployStatus.Stopped:
deploy_status = PaastaColors.grey(status_string)
elif status == marathon_tools.MarathonDeployStatus.Running:
deploy_status = PaastaColors.bold(status_string)
else:
deploy_status = status_string
return deploy_status
def status_marathon_job(service, instance, app_id, normal_instance_count,
client):
name = PaastaColors.cyan(compose_job_id(service, instance))
if marathon_tools.is_app_id_running(app_id, client):
app = client.get_app(app_id)
running_instances = app.tasks_running
if len(app.deployments) == 0:
deploy_status = PaastaColors.bold("Running")
elif app.instances == 0 and app.tasks_running == 0:
deploy_status = PaastaColors.grey("Stopped")
else:
# App is currently deploying so we should check the launch queue for more info
is_overdue, backoff_seconds = marathon_tools.get_app_queue_status(
client, app_id)
if is_overdue:
deploy_status = "%s (new tasks are not launching due to lack of capacity)" % PaastaColors.red(
"Waiting")
elif backoff_seconds:
deploy_status = "%s (next task won't launch for %s seconds due to previous failures)" % (
PaastaColors.red("Delayed"), backoff_seconds)
else:
deploy_status = PaastaColors.yellow("Deploying")
if running_instances >= normal_instance_count:
status = PaastaColors.green("Healthy")
instance_count = PaastaColors.green(
"(%d/%d)" % (running_instances, normal_instance_count))
elif running_instances == 0:
status = PaastaColors.yellow("Critical")
instance_count = PaastaColors.red(
"(%d/%d)" % (running_instances, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
instance_count = PaastaColors.yellow(
"(%d/%d)" % (running_instances, normal_instance_count))
return "Marathon: %s - up with %s instances. Status: %s" % (
status, instance_count, deploy_status)
else:
red_not = PaastaColors.red("NOT")
status = PaastaColors.red("Critical")
return "Marathon: %s - %s (app %s) is %s running in Marathon." % (
status, name, app_id, red_not)
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
print "Performing an emergency start on %s..." % compose_job_id(
service, args.instance)
output = execute_paasta_serviceinit_on_remote_master(
subcommand='start',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config)
print "%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:])
print "Output: %s" % PaastaColors.grey(output)
print "Run this command to see the status:"
print "paasta status --service %s --clusters %s" % (service, args.cluster)
def marathon_app_deploy_status_human(status, backoff_seconds=None):
status_string = marathon_tools.MarathonDeployStatus.tostring(status)
if status == marathon_tools.MarathonDeployStatus.Waiting:
deploy_status = "%s (new tasks are not launching due to lack of capacity)" % PaastaColors.red(
status_string)
elif status == marathon_tools.MarathonDeployStatus.Delayed:
deploy_status = "%s (next task won't launch for %s seconds due to previous failures)" % (
PaastaColors.red(status_string), backoff_seconds)
elif status == marathon_tools.MarathonDeployStatus.Deploying:
deploy_status = PaastaColors.yellow(status_string)
elif status == marathon_tools.MarathonDeployStatus.Stopped:
deploy_status = PaastaColors.grey(status_string)
elif status == marathon_tools.MarathonDeployStatus.Running:
deploy_status = PaastaColors.bold(status_string)
else:
deploy_status = status_string
return deploy_status
def create_queue_entries_table(service_instances) -> List[str]:
if len(service_instances) == 0:
return [PaastaColors.grey("Empty")]
table_header = [
"Service Instance",
"Bounce by",
"Wait until",
"Enqueue time",
"Bounce Start Time",
"Processed Count",
"Failures",
"Watcher",
]
rows = [table_header]
for service_instance in service_instances:
now = time.time()
bounce_by = format_timestamp(service_instance.bounce_by)
if service_instance.bounce_by < now:
bounce_by = PaastaColors.red(bounce_by)
failures = str(service_instance.failures)
if service_instance.failures > 10:
failures = PaastaColors.red(failures)
processed_count = str(service_instance.processed_count)
if service_instance.processed_count > 50:
processed_count = PaastaColors.red(processed_count)
rows.append([
f"{service_instance.service}.{service_instance.instance}",
bounce_by,
format_timestamp(service_instance.wait_until),
format_timestamp(service_instance.enqueue_time),
format_timestamp(service_instance.bounce_start_time),
processed_count,
failures,
service_instance.watcher,
])
return format_table(rows)
def paasta_emergency_start(args):
"""Performs an emergency start on a given service instance on a given cluster
All it does for Chronos jobs is send the latest version of the job config to Chronos and run it immediately.
"""
system_paasta_config = load_system_paasta_config()
service = figure_out_service_name(args, soa_dir=args.soa_dir)
paasta_print("Performing an emergency start on %s..." % compose_job_id(service, args.instance))
return_code, output = execute_paasta_serviceinit_on_remote_master(
subcommand='start',
cluster=args.cluster,
service=service,
instances=args.instance,
system_paasta_config=system_paasta_config
)
paasta_print("%s" % "\n".join(paasta_emergency_start.__doc__.splitlines()[-8:]))
paasta_print("Output: %s" % PaastaColors.grey(output))
paasta_print("Run this command to see the status:")
paasta_print("paasta status --service %s --clusters %s" % (service, args.cluster))
return return_code
def get_verbose_status_of_marathon_app(app):
"""Takes a given marathon app object and returns the verbose details
about the tasks, times, hosts, etc"""
output = []
create_datetime = datetime_from_utc_to_local(isodate.parse_datetime(app.version))
output.append(" Marathon app ID: %s" % PaastaColors.bold(app.id))
output.append(" App created: %s (%s)" % (str(create_datetime), humanize.naturaltime(create_datetime)))
output.append(" Tasks:")
rows = [("Mesos Task ID", "Host deployed to", "Deployed at what localtime", "Health")]
for task in app.tasks:
local_deployed_datetime = datetime_from_utc_to_local(task.staged_at)
if task.host is not None:
hostname = "%s:%s" % (task.host.split(".")[0], task.ports[0])
else:
hostname = "Unknown"
health_check_results = [health_check.alive for health_check in task.health_check_results
if health_check.alive is not None]
if not health_check_results:
health_check_status = PaastaColors.grey("N/A")
elif all(health_check_results):
health_check_status = PaastaColors.green("Healthy")
else:
health_check_status = PaastaColors.red("Unhealthy")
rows.append((
get_short_task_id(task.id),
hostname,
'%s (%s)' % (
local_deployed_datetime.strftime("%Y-%m-%dT%H:%M"),
humanize.naturaltime(local_deployed_datetime),
),
health_check_status,
))
output.append('\n'.join([" %s" % line for line in format_table(rows)]))
if len(app.tasks) == 0:
output.append(" No tasks associated with this marathon app")
return app.tasks, "\n".join(output)
def status_marathon_job(service, instance, app_id, normal_instance_count, client):
name = PaastaColors.cyan(compose_job_id(service, instance))
if marathon_tools.is_app_id_running(app_id, client):
app = client.get_app(app_id)
running_instances = app.tasks_running
if len(app.deployments) == 0:
deploy_status = PaastaColors.bold("Running")
elif app.instances == 0 and app.tasks_running == 0:
deploy_status = PaastaColors.grey("Stopped")
else:
# App is currently deploying so we should check the launch queue for more info
is_overdue, backoff_seconds = marathon_tools.get_app_queue_status(client, app_id)
if is_overdue:
deploy_status = "%s (new tasks are not launching due to lack of capacity)" % PaastaColors.red("Waiting")
elif backoff_seconds:
deploy_status = "%s (next task won't launch for %s seconds due to previous failures)" % (
PaastaColors.red("Delayed"), backoff_seconds)
else:
deploy_status = PaastaColors.yellow("Deploying")
if running_instances >= normal_instance_count:
status = PaastaColors.green("Healthy")
instance_count = PaastaColors.green("(%d/%d)" % (running_instances, normal_instance_count))
elif running_instances == 0:
status = PaastaColors.yellow("Critical")
instance_count = PaastaColors.red("(%d/%d)" % (running_instances, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
instance_count = PaastaColors.yellow("(%d/%d)" % (running_instances, normal_instance_count))
return "Marathon: %s - up with %s instances. Status: %s" % (status, instance_count, deploy_status)
else:
red_not = PaastaColors.red("NOT")
status = PaastaColors.red("Critical")
return "Marathon: %s - %s (app %s) is %s running in Marathon." % (status, name, app_id, red_not)
def run_docker_container(
docker_client,
service,
instance,
docker_hash,
volumes,
interactive,
command,
healthcheck,
healthcheck_only,
user_port,
instance_config,
soa_dir=DEFAULT_SOA_DIR,
dry_run=False,
json_dict=False,
framework=None,
):
"""docker-py has issues running a container with a TTY attached, so for
consistency we execute 'docker run' directly in both interactive and
non-interactive modes.
In non-interactive mode when the run is complete, stop the container and
remove it (with docker-py).
"""
if user_port:
if check_if_port_free(user_port):
chosen_port = user_port
else:
paasta_print(
PaastaColors.red(
"The chosen port is already in use!\n"
"Try specifying another one, or omit (--port|-o) and paasta will find a free one for you"
),
file=sys.stderr,
)
sys.exit(1)
else:
chosen_port = pick_random_port()
environment = instance_config.get_env_dictionary()
local_run_environment = get_local_run_environment_vars(
instance_config=instance_config,
port0=chosen_port,
framework=framework,
)
environment.update(local_run_environment)
net = instance_config.get_net()
memory = instance_config.get_mem()
container_name = get_container_name()
docker_params = instance_config.format_docker_parameters()
try:
container_port = instance_config.get_container_port()
except AttributeError:
container_port = None
docker_run_args = dict(
memory=memory,
chosen_port=chosen_port,
container_port=container_port,
container_name=container_name,
volumes=volumes,
env=environment,
interactive=interactive,
docker_hash=docker_hash,
command=command,
net=net,
docker_params=docker_params,
)
docker_run_cmd = get_docker_run_cmd(**docker_run_args)
joined_docker_run_cmd = ' '.join(docker_run_cmd)
healthcheck_mode, healthcheck_data = get_healthcheck_for_instance(
service, instance, instance_config, chosen_port, soa_dir=soa_dir)
if dry_run:
if json_dict:
paasta_print(json.dumps(docker_run_args))
else:
paasta_print(json.dumps(docker_run_cmd))
return 0
else:
paasta_print('Running docker command:\n%s' % PaastaColors.grey(joined_docker_run_cmd))
if interactive:
# NOTE: This immediately replaces us with the docker run cmd. Docker
# run knows how to clean up the running container in this situation.
execlp('paasta_docker_wrapper', *docker_run_cmd)
# For testing, when execlp is patched out and doesn't replace us, we
# still want to bail out.
return 0
container_started = False
container_id = None
try:
(returncode, output) = _run(docker_run_cmd)
if returncode != 0:
paasta_print(
'Failure trying to start your container!'
'Returncode: %d'
'Output:'
'%s'
''
'Fix that problem and try again.'
'http://y/paasta-troubleshooting'
% (returncode, output),
sep='\n',
)
# Container failed to start so no need to cleanup; just bail.
sys.exit(1)
container_started = True
container_id = get_container_id(docker_client, container_name)
paasta_print('Found our container running with CID %s' % container_id)
# If the service has a healthcheck, simulate it
if healthcheck_mode is not None:
healthcheck_result = simulate_healthcheck_on_service(
instance_config=instance_config,
docker_client=docker_client,
container_id=container_id,
healthcheck_mode=healthcheck_mode,
healthcheck_data=healthcheck_data,
healthcheck_enabled=healthcheck,
)
def _output_stdout_and_exit_code():
returncode = docker_client.inspect_container(container_id)['State']['ExitCode']
paasta_print('Container exited: %d)' % returncode)
paasta_print('Here is the stdout and stderr:\n\n')
paasta_print(
docker_client.attach(container_id, stderr=True, stream=False, logs=True)
)
if healthcheck_only:
if container_started:
_output_stdout_and_exit_code()
_cleanup_container(docker_client, container_id)
if healthcheck_mode is None:
paasta_print('--healthcheck-only, but no healthcheck is defined for this instance!')
sys.exit(1)
elif healthcheck_result is True:
sys.exit(0)
else:
sys.exit(1)
running = docker_client.inspect_container(container_id)['State']['Running']
if running:
paasta_print('Your service is now running! Tailing stdout and stderr:')
for line in docker_client.attach(container_id, stderr=True, stream=True, logs=True):
paasta_print(line)
else:
_output_stdout_and_exit_code()
returncode = 3
except KeyboardInterrupt:
returncode = 3
# Cleanup if the container exits on its own or interrupted.
if container_started:
returncode = docker_client.inspect_container(container_id)['State']['ExitCode']
_cleanup_container(docker_client, container_id)
return returncode
def _output_on_failure():
sys.stdout.write('Your service failed to start, here is the stdout and stderr\n')
sys.stdout.write(PaastaColors.grey(
docker_client.attach(container_id, stderr=True, stream=False, logs=True)
))
def run_docker_container(
docker_client,
service,
instance,
docker_hash,
volumes,
interactive,
command,
hostname,
healthcheck,
healthcheck_only,
instance_config,
soa_dir=DEFAULT_SOA_DIR,
dry_run=False
):
"""docker-py has issues running a container with a TTY attached, so for
consistency we execute 'docker run' directly in both interactive and
non-interactive modes.
In non-interactive mode when the run is complete, stop the container and
remove it (with docker-py).
"""
if interactive:
sys.stderr.write(PaastaColors.yellow(
"Warning! You're running a container in interactive mode.\n"
"This is *NOT* how Mesos runs containers.\n"
"To run the container exactly as Mesos does, omit the -I flag.\n\n"
))
else:
sys.stderr.write(PaastaColors.yellow(
"You're running a container in non-interactive mode.\n"
"This is how Mesos runs containers.\n"
"Note that some programs behave differently when running with no\n"
"tty attached (as your program is about to run).\n\n"
))
environment = instance_config.get_env_dictionary()
net = instance_config.get_net()
memory = instance_config.get_mem()
random_port = pick_random_port()
container_name = get_container_name()
docker_params = instance_config.format_docker_parameters()
docker_run_cmd = get_docker_run_cmd(
memory=memory,
random_port=random_port,
container_name=container_name,
volumes=volumes,
env=environment,
interactive=interactive,
docker_hash=docker_hash,
command=command,
hostname=hostname,
net=net,
docker_params=docker_params,
)
joined_docker_run_cmd = ' '.join(docker_run_cmd)
healthcheck_mode, healthcheck_data = get_healthcheck_for_instance(
service, instance, instance_config, random_port, soa_dir=soa_dir)
if dry_run:
sys.stdout.write(json.dumps(docker_run_cmd) + '\n')
sys.exit(0)
else:
sys.stdout.write('Running docker command:\n%s\n' % PaastaColors.grey(joined_docker_run_cmd))
if interactive:
# NOTE: This immediately replaces us with the docker run cmd. Docker
# run knows how to clean up the running container in this situation.
execlp('docker', *docker_run_cmd)
# For testing, when execlp is patched out and doesn't replace us, we
# still want to bail out.
return
container_started = False
container_id = None
try:
(returncode, output) = _run(joined_docker_run_cmd)
if returncode != 0:
sys.stdout.write(
'Failure trying to start your container!\n'
'Returncode: %d\n'
'Output:\n'
'%s\n'
'\n'
'Fix that problem and try again.\n'
'http://y/paasta-troubleshooting\n'
% (returncode, output)
)
# Container failed to start so no need to cleanup; just bail.
sys.exit(1)
container_started = True
container_id = get_container_id(docker_client, container_name)
sys.stdout.write('Found our container running with CID %s\n' % container_id)
# If the service has a healthcheck, simulate it
if healthcheck_mode:
status, _ = simulate_healthcheck_on_service(
instance_config, docker_client, container_id, healthcheck_mode, healthcheck_data, healthcheck)
else:
status = True
def _output_on_failure():
sys.stdout.write('Your service failed to start, here is the stdout and stderr\n')
sys.stdout.write(PaastaColors.grey(
docker_client.attach(container_id, stderr=True, stream=False, logs=True)
))
if healthcheck_only:
sys.stdout.write('Detected --healthcheck-only flag, exiting now.\n')
if container_started:
_cleanup_container(docker_client, container_id)
if status:
sys.exit(0)
else:
_output_on_failure()
sys.exit(1)
running = docker_client.inspect_container(container_id)['State']['Running']
if running:
sys.stdout.write('Your service is now running! Tailing stdout and stderr:\n')
for line in docker_client.attach(container_id, stderr=True, stream=True, logs=True):
sys.stdout.write(PaastaColors.grey(line))
else:
_output_on_failure()
returncode = 3
except KeyboardInterrupt:
returncode = 3
# Cleanup if the container exits on its own or interrupted.
if container_started:
returncode = docker_client.inspect_container(container_id)['State']['ExitCode']
_cleanup_container(docker_client, container_id)
sys.exit(returncode)
def run_docker_container(
docker_client,
service,
instance,
docker_url,
volumes,
interactive,
command,
healthcheck,
healthcheck_only,
user_port,
instance_config,
secret_provider_name,
soa_dir=DEFAULT_SOA_DIR,
dry_run=False,
json_dict=False,
framework=None,
secret_provider_kwargs={},
skip_secrets=False,
):
"""docker-py has issues running a container with a TTY attached, so for
consistency we execute 'docker run' directly in both interactive and
non-interactive modes.
In non-interactive mode when the run is complete, stop the container and
remove it (with docker-py).
"""
if user_port:
if check_if_port_free(user_port):
chosen_port = user_port
else:
paasta_print(
PaastaColors.red(
"The chosen port is already in use!\n"
"Try specifying another one, or omit (--port|-o) and paasta will find a free one for you"
),
file=sys.stderr,
)
sys.exit(1)
else:
chosen_port = pick_random_port(service)
environment = instance_config.get_env_dictionary()
if not skip_secrets:
secret_environment = decrypt_secret_environment_variables(
secret_provider_name=secret_provider_name,
environment=environment,
soa_dir=soa_dir,
service_name=service,
cluster_name=instance_config.cluster,
secret_provider_kwargs=secret_provider_kwargs,
)
environment.update(secret_environment)
local_run_environment = get_local_run_environment_vars(
instance_config=instance_config, port0=chosen_port, framework=framework
)
environment.update(local_run_environment)
net = instance_config.get_net()
memory = instance_config.get_mem()
container_name = get_container_name()
docker_params = instance_config.format_docker_parameters()
healthcheck_mode, healthcheck_data = get_healthcheck_for_instance(
service, instance, instance_config, chosen_port, soa_dir=soa_dir
)
if healthcheck_mode is None:
container_port = None
interactive = True
elif not user_port and not healthcheck and not healthcheck_only:
container_port = None
else:
try:
container_port = instance_config.get_container_port()
except AttributeError:
container_port = None
simulate_healthcheck = (
healthcheck_only or healthcheck
) and healthcheck_mode is not None
docker_run_args = dict(
memory=memory,
chosen_port=chosen_port,
container_port=container_port,
container_name=container_name,
volumes=volumes,
env=environment,
interactive=interactive,
detach=simulate_healthcheck,
docker_hash=docker_url,
command=command,
net=net,
docker_params=docker_params,
)
docker_run_cmd = get_docker_run_cmd(**docker_run_args)
joined_docker_run_cmd = " ".join(docker_run_cmd)
if dry_run:
if json_dict:
paasta_print(json.dumps(docker_run_args))
else:
paasta_print(json.dumps(docker_run_cmd))
return 0
else:
paasta_print(
"Running docker command:\n%s" % PaastaColors.grey(joined_docker_run_cmd)
)
merged_env = {**os.environ, **environment}
if interactive or not simulate_healthcheck:
# NOTE: This immediately replaces us with the docker run cmd. Docker
# run knows how to clean up the running container in this situation.
wrapper_path = shutil.which("paasta_docker_wrapper")
# To properly simulate mesos, we pop the PATH, which is not available to
# The executor
merged_env.pop("PATH")
execlpe(wrapper_path, *docker_run_cmd, merged_env)
# For testing, when execlpe is patched out and doesn't replace us, we
# still want to bail out.
return 0
container_started = False
container_id = None
try:
(returncode, output) = _run(docker_run_cmd, env=merged_env)
if returncode != 0:
paasta_print(
"Failure trying to start your container!"
"Returncode: %d"
"Output:"
"%s"
""
"Fix that problem and try again."
"http://y/paasta-troubleshooting" % (returncode, output),
sep="\n",
)
# Container failed to start so no need to cleanup; just bail.
sys.exit(1)
container_started = True
container_id = get_container_id(docker_client, container_name)
paasta_print("Found our container running with CID %s" % container_id)
if simulate_healthcheck:
healthcheck_result = simulate_healthcheck_on_service(
instance_config=instance_config,
docker_client=docker_client,
container_id=container_id,
healthcheck_mode=healthcheck_mode,
healthcheck_data=healthcheck_data,
healthcheck_enabled=healthcheck,
)
def _output_exit_code():
returncode = docker_client.inspect_container(container_id)["State"][
"ExitCode"
]
paasta_print(f"Container exited: {returncode})")
if healthcheck_only:
if container_started:
_output_exit_code()
_cleanup_container(docker_client, container_id)
if healthcheck_mode is None:
paasta_print(
"--healthcheck-only, but no healthcheck is defined for this instance!"
)
sys.exit(1)
elif healthcheck_result is True:
sys.exit(0)
else:
sys.exit(1)
running = docker_client.inspect_container(container_id)["State"]["Running"]
if running:
paasta_print("Your service is now running! Tailing stdout and stderr:")
for line in docker_client.attach(
container_id, stderr=True, stream=True, logs=True
):
paasta_print(line)
else:
_output_exit_code()
returncode = 3
except KeyboardInterrupt:
returncode = 3
# Cleanup if the container exits on its own or interrupted.
if container_started:
returncode = docker_client.inspect_container(container_id)["State"]["ExitCode"]
_cleanup_container(docker_client, container_id)
return returncode
def run_docker_container(
docker_client,
service,
instance,
docker_hash,
volumes,
interactive,
command,
hostname,
healthcheck,
healthcheck_only,
instance_config,
soa_dir=DEFAULT_SOA_DIR,
dry_run=False,
json_dict=False
):
"""docker-py has issues running a container with a TTY attached, so for
consistency we execute 'docker run' directly in both interactive and
non-interactive modes.
In non-interactive mode when the run is complete, stop the container and
remove it (with docker-py).
"""
environment = instance_config.get_env_dictionary()
net = instance_config.get_net()
memory = instance_config.get_mem()
random_port = pick_random_port()
container_name = get_container_name()
docker_params = instance_config.format_docker_parameters()
docker_run_args = dict(
memory=memory,
random_port=random_port,
container_name=container_name,
volumes=volumes,
env=environment,
interactive=interactive,
docker_hash=docker_hash,
command=command,
hostname=hostname,
net=net,
docker_params=docker_params,
)
docker_run_cmd = get_docker_run_cmd(**docker_run_args)
joined_docker_run_cmd = ' '.join(docker_run_cmd)
healthcheck_mode, healthcheck_data = get_healthcheck_for_instance(
service, instance, instance_config, random_port, soa_dir=soa_dir)
if dry_run:
if json_dict:
paasta_print(json.dumps(docker_run_args))
else:
paasta_print(json.dumps(docker_run_cmd))
sys.exit(0)
else:
paasta_print('Running docker command:\n%s' % PaastaColors.grey(joined_docker_run_cmd))
if interactive:
# NOTE: This immediately replaces us with the docker run cmd. Docker
# run knows how to clean up the running container in this situation.
execlp('docker', *docker_run_cmd)
# For testing, when execlp is patched out and doesn't replace us, we
# still want to bail out.
return
container_started = False
container_id = None
try:
(returncode, output) = _run(joined_docker_run_cmd)
if returncode != 0:
paasta_print(
'Failure trying to start your container!'
'Returncode: %d'
'Output:'
'%s'
''
'Fix that problem and try again.'
'http://y/paasta-troubleshooting'
% (returncode, output),
sep='\n',
)
# Container failed to start so no need to cleanup; just bail.
sys.exit(1)
container_started = True
container_id = get_container_id(docker_client, container_name)
paasta_print('Found our container running with CID %s' % container_id)
# If the service has a healthcheck, simulate it
if healthcheck_mode is not None:
healthcheck_result = simulate_healthcheck_on_service(
instance_config=instance_config,
docker_client=docker_client,
container_id=container_id,
healthcheck_mode=healthcheck_mode,
healthcheck_data=healthcheck_data,
healthcheck_enabled=healthcheck,
)
def _output_on_failure():
returncode = docker_client.inspect_container(container_id)['State']['ExitCode']
paasta_print('Container exited: %d)' % returncode)
paasta_print('Here is the stdout and stderr:\n')
paasta_print(PaastaColors.grey(
docker_client.attach(container_id, stderr=True, stream=False, logs=True)
))
if healthcheck_only:
if container_started:
_cleanup_container(docker_client, container_id)
if healthcheck_mode is None:
paasta_print('--healthcheck-only, but no healthcheck is defined for this instance!')
sys.exit(1)
elif healthcheck_result:
sys.exit(0)
else:
_output_on_failure()
sys.exit(1)
running = docker_client.inspect_container(container_id)['State']['Running']
if running:
paasta_print('Your service is now running! Tailing stdout and stderr:')
for line in docker_client.attach(container_id, stderr=True, stream=True, logs=True):
paasta_print(PaastaColors.grey(line))
else:
_output_on_failure()
returncode = 3
except KeyboardInterrupt:
returncode = 3
# Cleanup if the container exits on its own or interrupted.
if container_started:
returncode = docker_client.inspect_container(container_id)['State']['ExitCode']
_cleanup_container(docker_client, container_id)
sys.exit(returncode)
def simulate_healthcheck_on_service(
instance_config,
docker_client,
container_id,
healthcheck_mode,
healthcheck_data,
healthcheck_enabled
):
"""Simulates Marathon-style healthcheck on given service if healthcheck is enabled
:param instance_config: service manifest
:param docker_client: Docker client object
:param container_id: Docker container id
:param healthcheck_data: tuple url to healthcheck
:param healthcheck_enabled: boolean
:returns: if healthcheck_enabled is true, then returns output of healthcheck, otherwise simply returns true
"""
healthcheck_link = PaastaColors.cyan(healthcheck_data)
if healthcheck_enabled:
grace_period = instance_config.get_healthcheck_grace_period_seconds()
timeout = instance_config.get_healthcheck_timeout_seconds()
interval = instance_config.get_healthcheck_interval_seconds()
max_failures = instance_config.get_healthcheck_max_consecutive_failures()
sys.stdout.write('\nStarting health check via %s (waiting %s seconds before '
'considering failures due to grace period):\n' % (healthcheck_link, grace_period))
# silenty start performing health checks until grace period ends or first check succeeds
graceperiod_end_time = time.time() + grace_period
while True:
healthcheck_succeeded = run_healthcheck_on_container(
docker_client, container_id, healthcheck_mode, healthcheck_data, timeout)
if healthcheck_succeeded or time.time() > graceperiod_end_time:
break
else:
sys.stdout.write("%s\n" % PaastaColors.grey("Healthcheck failed (disregarded due to grace period)"))
time.sleep(interval)
failure = False
for attempt in range(1, max_failures + 1):
healthcheck_succeeded = run_healthcheck_on_container(
docker_client, container_id, healthcheck_mode, healthcheck_data, timeout)
if healthcheck_succeeded:
sys.stdout.write("%s (via: %s)\n" %
(PaastaColors.green("Healthcheck succeeded!"), healthcheck_link))
failure = False
break
else:
sys.stdout.write("%s (via: %s)\n" %
(PaastaColors.red("Healthcheck failed! (Attempt %d of %d)" % (attempt, max_failures)),
healthcheck_link))
failure = True
time.sleep(interval)
if failure:
healthcheck_status = False
else:
healthcheck_status = True
else:
sys.stdout.write('\nMesos would have healthchecked your service via\n%s\n' % healthcheck_link)
healthcheck_status = True
return healthcheck_status
def colorize(x):
return(PaastaColors.grey(x))
def run_docker_container(
docker_client,
service,
instance,
docker_hash,
volumes,
interactive,
command,
healthcheck,
healthcheck_only,
instance_config,
soa_dir=service_configuration_lib.DEFAULT_SOA_DIR,
):
"""docker-py has issues running a container with a TTY attached, so for
consistency we execute 'docker run' directly in both interactive and
non-interactive modes.
In non-interactive mode when the run is complete, stop the container and
remove it (with docker-py).
"""
if interactive:
sys.stderr.write(
PaastaColors.yellow(
"Warning! You're running a container in interactive mode.\n"
"This is *NOT* how Mesos runs containers.\n"
"To run the container exactly as Mesos does, omit the -I flag.\n\n"
)
)
else:
sys.stderr.write(
PaastaColors.yellow(
"You're running a container in non-interactive mode.\n"
"This is how Mesos runs containers.\n"
"Note that some programs behave differently when running with no\n"
"tty attached (as your program is about to run).\n\n"
)
)
environment = instance_config.get_unformatted_env()
memory = instance_config.get_mem()
random_port = pick_random_port()
container_name = get_container_name()
docker_run_cmd = get_docker_run_cmd(
memory=memory,
random_port=random_port,
container_name=container_name,
volumes=volumes,
env=environment,
interactive=interactive,
docker_hash=docker_hash,
command=command,
)
# http://stackoverflow.com/questions/4748344/whats-the-reverse-of-shlex-split
joined_docker_run_cmd = " ".join(pipes.quote(word) for word in docker_run_cmd)
healthcheck_mode, healthcheck_data = get_healthcheck_for_instance(
service, instance, instance_config, random_port, soa_dir=soa_dir
)
sys.stdout.write("Running docker command:\n%s\n" % PaastaColors.grey(joined_docker_run_cmd))
if interactive:
# NOTE: This immediately replaces us with the docker run cmd. Docker
# run knows how to clean up the running container in this situation.
execlp("docker", *docker_run_cmd)
# For testing, when execlp is patched out and doesn't replace us, we
# still want to bail out.
return
container_started = False
container_id = None
try:
(returncode, output) = _run(joined_docker_run_cmd)
if returncode != 0:
sys.stdout.write(
"Failure trying to start your container!\n"
"Returncode: %d\n"
"Output:\n"
"%s\n"
"\n"
"Fix that problem and try again.\n"
"http://y/paasta-troubleshooting\n" % (returncode, output)
)
# Container failed to start so no need to cleanup; just bail.
sys.exit(1)
container_started = True
container_id = get_container_id(docker_client, container_name)
sys.stdout.write("Found our container running with CID %s\n" % container_id)
# If the service has a healthcheck, simulate it
if healthcheck_mode:
status = simulate_healthcheck_on_service(
instance_config, docker_client, container_id, healthcheck_mode, healthcheck_data, healthcheck
)
else:
status = True
if healthcheck_only:
sys.stdout.write("Detected --healthcheck-only flag, exiting now.\n")
if container_started:
_cleanup_container(docker_client, container_id)
if status:
sys.exit(0)
else:
sys.exit(1)
sys.stdout.write("Your service is now running! Tailing stdout and stderr:\n")
for line in docker_client.attach(container_id, stderr=True, stream=True, logs=True):
sys.stdout.write(PaastaColors.grey(line))
except KeyboardInterrupt:
returncode = 3
pass
# Cleanup if the container exits on its own or interrupted.
if container_started:
returncode = docker_client.inspect_container(container_id)["State"]["ExitCode"]
_cleanup_container(docker_client, container_id)
sys.exit(returncode)
def test_format_chronos_job_zero_mesos_tasks():
example_job = {'schedule': 'foo'}
running_tasks = []
verbose = False
actual = chronos_serviceinit.format_chronos_job_status(example_job, running_tasks, verbose)
assert PaastaColors.grey('Not running') in actual
