def perform_command(command, service, instance, cluster, verbose, soa_dir, app_id=None, delta=None):
"""Performs a start/stop/restart/status on an instance
:param command: String of start, stop, restart, status
:param service: service name
:param instance: instance name, like "main" or "canary"
:param cluster: cluster name
:param verbose: int verbosity level
:returns: A unix-style return code
"""
system_config = load_system_paasta_config()
marathon_config = marathon_tools.load_marathon_config()
job_config = marathon_tools.load_marathon_service_config(service, instance, cluster, soa_dir=soa_dir)
if not app_id:
try:
app_id = job_config.format_marathon_app_dict()['id']
except NoDockerImageError:
job_id = compose_job_id(service, instance)
print "Docker image for %s not in deployments.json. Exiting. Has Jenkins deployed it?" % job_id
return 1
normal_instance_count = job_config.get_instances()
normal_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace(service, instance, cluster)
proxy_port = marathon_tools.get_proxy_port_for_instance(service, instance, cluster, soa_dir=soa_dir)
client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(),
marathon_config.get_password())
if command == 'restart':
restart_marathon_job(service, instance, app_id, client, cluster)
elif command == 'status':
print status_desired_state(service, instance, client, job_config)
print status_marathon_job(service, instance, app_id, normal_instance_count, client)
tasks, out = status_marathon_job_verbose(service, instance, client)
if verbose > 0:
print out
print status_mesos_tasks(service, instance, normal_instance_count)
if verbose > 0:
tail_lines = calculate_tail_lines(verbose_level=verbose)
print status_mesos_tasks_verbose(
job_id=app_id,
get_short_task_id=get_short_task_id,
tail_lines=tail_lines,
)
if proxy_port is not None:
print status_smartstack_backends(
service=service,
instance=instance,
cluster=cluster,
job_config=job_config,
tasks=tasks,
expected_count=normal_smartstack_count,
soa_dir=soa_dir,
verbose=verbose > 0,
synapse_port=system_config.get_synapse_port(),
synapse_haproxy_url_format=system_config.get_synapse_haproxy_url_format(),
)
else:
# The command parser shouldn't have let us get this far...
raise NotImplementedError("Command %s is not implemented!" % command)
return 0
def perform_command(command, service, instance, cluster, verbose, soa_dir, app_id=None, delta=None):
"""Performs a start/stop/restart/status on an instance
:param command: String of start, stop, restart, status
:param service: service name
:param instance: instance name, like "main" or "canary"
:param cluster: cluster name
:param verbose: int verbosity level
:returns: A unix-style return code
"""
system_config = load_system_paasta_config()
marathon_config = marathon_tools.load_marathon_config()
job_config = marathon_tools.load_marathon_service_config(service, instance, cluster, soa_dir=soa_dir)
if not app_id:
try:
app_id = job_config.format_marathon_app_dict()['id']
except NoDockerImageError:
job_id = compose_job_id(service, instance)
print "Docker image for %s not in deployments.json. Exiting. Has Jenkins deployed it?" % job_id
return 1
normal_instance_count = job_config.get_instances()
normal_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace(service, instance, cluster)
proxy_port = marathon_tools.get_proxy_port_for_instance(service, instance, cluster, soa_dir=soa_dir)
client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(),
marathon_config.get_password())
if command == 'restart':
restart_marathon_job(service, instance, app_id, client, cluster)
elif command == 'status':
print status_desired_state(service, instance, client, job_config)
print status_marathon_job(service, instance, app_id, normal_instance_count, client)
tasks, out = status_marathon_job_verbose(service, instance, client)
if verbose > 0:
print out
print status_mesos_tasks(service, instance, normal_instance_count)
if verbose > 0:
tail_lines = calculate_tail_lines(verbose_level=verbose)
print status_mesos_tasks_verbose(
job_id=app_id,
get_short_task_id=get_short_task_id,
tail_lines=tail_lines,
)
if proxy_port is not None:
print status_smartstack_backends(
service=service,
instance=instance,
cluster=cluster,
job_config=job_config,
tasks=tasks,
expected_count=normal_smartstack_count,
soa_dir=soa_dir,
verbose=verbose > 0,
synapse_port=system_config.get_synapse_port(),
synapse_haproxy_url_format=system_config.get_synapse_haproxy_url_format(),
)
else:
# The command parser shouldn't have let us get this far...
raise NotImplementedError("Command %s is not implemented!" % command)
return 0
def perform_command(command, service, instance, cluster, verbose, soa_dir, app_id=None, delta=None):
"""Performs a start/stop/restart/status/scale on an instance
:param command: String of start, stop, restart, status or scale
:param service: service name
:param instance: instance name, like "main" or "canary"
:param cluster: cluster name
:param verbose: bool if the output should be verbose or not
:returns: A unix-style return code
"""
marathon_config = marathon_tools.load_marathon_config()
job_config = marathon_tools.load_marathon_service_config(service, instance, cluster, soa_dir=soa_dir)
if not app_id:
try:
app_id = marathon_tools.create_complete_config(service, instance, marathon_config, soa_dir=soa_dir)['id']
except NoDockerImageError:
job_id = compose_job_id(service, instance)
print "Docker image for %s not in deployments.json. Exiting. Has Jenkins deployed it?" % job_id
return 1
normal_instance_count = job_config.get_instances()
normal_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace(service, instance)
proxy_port = marathon_tools.get_proxy_port_for_instance(service, instance, soa_dir=soa_dir)
client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(),
marathon_config.get_password())
if command == 'start':
start_marathon_job(service, instance, app_id, normal_instance_count, client, cluster)
elif command == 'stop':
stop_marathon_job(service, instance, app_id, client, cluster)
elif command == 'restart':
restart_marathon_job(service, instance, app_id, normal_instance_count, client, cluster)
elif command == 'status':
# Setting up transparent cache for http API calls
requests_cache.install_cache('paasta_serviceinit', backend='memory')
print status_desired_state(service, instance, client, job_config)
print status_marathon_job(service, instance, app_id, normal_instance_count, client)
tasks, out = status_marathon_job_verbose(service, instance, client)
if verbose:
print out
print status_mesos_tasks(service, instance, normal_instance_count)
if verbose:
print status_mesos_tasks_verbose(app_id, get_short_task_id)
if proxy_port is not None:
print status_smartstack_backends(
service=service,
instance=instance,
cluster=cluster,
job_config=job_config,
tasks=tasks,
expected_count=normal_smartstack_count,
soa_dir=soa_dir,
verbose=verbose,
)
elif command == 'scale':
scale_marathon_job(service, instance, app_id, delta, client, cluster)
else:
# The command parser shouldn't have let us get this far...
raise NotImplementedError("Command %s is not implemented!" % command)
return 0
def test_status_mesos_tasks_verbose():
with contextlib.nested(
mock.patch("paasta_tools.mesos_tools.get_running_tasks_from_active_frameworks", autospec=True),
mock.patch("paasta_tools.mesos_tools.get_non_running_tasks_from_active_frameworks", autospec=True),
mock.patch("paasta_tools.mesos_tools.format_running_mesos_task_row", autospec=True),
mock.patch("paasta_tools.mesos_tools.format_non_running_mesos_task_row", autospec=True),
) as (
get_running_mesos_tasks_patch,
get_non_running_mesos_tasks_patch,
format_running_mesos_task_row_patch,
format_non_running_mesos_task_row_patch,
):
get_running_mesos_tasks_patch.return_value = ["doing a lap"]
get_non_running_mesos_tasks_patch.return_value = ["eating a burrito"]
format_running_mesos_task_row_patch.return_value = ["id", "host", "mem", "cpu", "time"]
format_non_running_mesos_task_row_patch.return_value = ["id", "host", "time", "state"]
job_id = (format_job_id("fake_service", "fake_instance"),)
def get_short_task_id(_):
return "short_task_id"
actual = mesos_tools.status_mesos_tasks_verbose(job_id, get_short_task_id)
assert "Running Tasks" in actual
assert "Non-Running Tasks" in actual
format_running_mesos_task_row_patch.assert_called_once_with("doing a lap", get_short_task_id)
format_non_running_mesos_task_row_patch.assert_called_once_with("eating a burrito", get_short_task_id)
def test_status_mesos_tasks_verbose():
with contextlib.nested(
mock.patch('paasta_tools.mesos_tools.get_running_tasks_from_active_frameworks', autospec=True,),
mock.patch('paasta_tools.mesos_tools.get_non_running_tasks_from_active_frameworks', autospec=True,),
mock.patch('paasta_tools.mesos_tools.format_running_mesos_task_row', autospec=True,),
mock.patch('paasta_tools.mesos_tools.format_non_running_mesos_task_row', autospec=True,),
) as (
get_running_mesos_tasks_patch,
get_non_running_mesos_tasks_patch,
format_running_mesos_task_row_patch,
format_non_running_mesos_task_row_patch,
):
get_running_mesos_tasks_patch.return_value = ['doing a lap']
get_non_running_mesos_tasks_patch.return_value = ['eating a burrito']
format_running_mesos_task_row_patch.return_value = ['id', 'host', 'mem', 'cpu', 'disk', 'time']
format_non_running_mesos_task_row_patch.return_value = ['id', 'host', 'time', 'state']
job_id = format_job_id('fake_service', 'fake_instance'),
def get_short_task_id(_):
return 'short_task_id'
actual = mesos_tools.status_mesos_tasks_verbose(job_id, get_short_task_id)
assert 'Running Tasks' in actual
assert 'Non-Running Tasks' in actual
format_running_mesos_task_row_patch.assert_called_once_with('doing a lap', get_short_task_id)
format_non_running_mesos_task_row_patch.assert_called_once_with('eating a burrito', get_short_task_id)
def format_chronos_job_status(job, running_tasks, verbose):
"""Given a job, returns a pretty-printed human readable output regarding
the status of the job.
:param job: dictionary of the job status
:param running_tasks: a list of Mesos tasks associated with ``job``, e.g. the
result of ``mesos_tools.get_running_tasks_from_active_frameworks()``.
"""
config_hash = _format_config_hash(job)
disabled_state = _format_disabled_status(job)
(last_result, formatted_time) = _format_last_result(job)
schedule = _format_schedule(job)
command = _format_command(job)
mesos_status = _format_mesos_status(job, running_tasks)
if verbose:
mesos_status_verbose = status_mesos_tasks_verbose(job["name"], get_short_task_id)
mesos_status = "%s\n%s" % (mesos_status, mesos_status_verbose)
return (
"Config: %(config_hash)s\n"
" Status: %(disabled_state)s\n"
" Last: %(last_result)s (%(formatted_time)s)\n"
" Schedule: %(schedule)s\n"
" Command: %(command)s\n"
" Mesos: %(mesos_status)s" % {
"config_hash": config_hash,
"disabled_state": disabled_state,
"last_result": last_result,
"formatted_time": formatted_time,
"schedule": schedule,
"command": command,
"mesos_status": mesos_status,
}
)
def format_chronos_job_status(client, job, running_tasks, verbose=0):
"""Given a job, returns a pretty-printed human readable output regarding
the status of the job.
:param job: dictionary of the job status
:param running_tasks: a list of Mesos tasks associated with ``job``, e.g. the
result of ``mesos_tools.get_running_tasks_from_frameworks()``.
:param verbose: int verbosity level
"""
job_name = _format_job_name(job)
is_temporary = chronos_tools.is_temporary_job(
job) if 'name' in job else 'UNKNOWN'
job_name = modify_string_for_rerun_status(job_name, is_temporary)
disabled_state = _format_disabled_status(job)
service, instance = chronos_tools.decompose_job_id(job['name'])
chronos_state = chronos_tools.get_chronos_status_for_job(
client, service, instance)
(last_result, formatted_time) = _format_last_result(job)
job_type = chronos_tools.get_job_type(job)
schedule_type = _get_schedule_field_for_job_type(job_type)
schedule_formatter = get_schedule_formatter(job_type, verbose)
schedule_value = schedule_formatter(job)
command = _format_command(job)
mesos_status = _format_mesos_status(running_tasks)
if verbose > 0:
tail_lines = calculate_tail_lines(verbose_level=verbose)
mesos_status_verbose = status_mesos_tasks_verbose(
job_id=job["name"],
get_short_task_id=get_short_task_id,
tail_lines=tail_lines,
)
mesos_status = "%s\n%s" % (mesos_status, mesos_status_verbose)
return ("Job: %(job_name)s\n"
" Status: %(disabled_state)s (%(chronos_state)s)"
" Last: %(last_result)s (%(formatted_time)s)\n"
" %(schedule_type)s: %(schedule_value)s\n"
" Command: %(command)s\n"
" Mesos: %(mesos_status)s" % {
"job_name": job_name,
"is_temporary": is_temporary,
"schedule_type": schedule_type,
"chronos_state": PaastaColors.grey(chronos_state),
"disabled_state": disabled_state,
"last_result": last_result,
"formatted_time": formatted_time,
"schedule_value": schedule_value,
"command": command,
"mesos_status": mesos_status,
})
def format_chronos_job_status(client, job, running_tasks, verbose=0):
"""Given a job, returns a pretty-printed human readable output regarding
the status of the job.
:param job: dictionary of the job status
:param running_tasks: a list of Mesos tasks associated with ``job``, e.g. the
result of ``mesos_tools.get_running_tasks_from_active_frameworks()``.
:param verbose: int verbosity level
"""
job_name = _format_job_name(job)
is_temporary = chronos_tools.is_temporary_job(job) if 'name' in job else 'UNKNOWN'
job_name = modify_string_for_rerun_status(job_name, is_temporary)
disabled_state = _format_disabled_status(job)
service, instance = chronos_tools.decompose_job_id(job['name'])
chronos_state = chronos_tools.get_chronos_status_for_job(client, service, instance)
(last_result, formatted_time) = _format_last_result(job)
job_type = chronos_tools.get_job_type(job)
schedule_type = _get_schedule_field_for_job_type(job_type)
schedule_formatter = get_schedule_formatter(job_type, verbose)
schedule_value = schedule_formatter(job)
command = _format_command(job)
mesos_status = _format_mesos_status(job, running_tasks)
if verbose > 0:
tail_lines = calculate_tail_lines(verbose_level=verbose)
mesos_status_verbose = status_mesos_tasks_verbose(
job_id=job["name"],
get_short_task_id=get_short_task_id,
tail_lines=tail_lines,
)
mesos_status = "%s\n%s" % (mesos_status, mesos_status_verbose)
return (
"Job: %(job_name)s\n"
" Status: %(disabled_state)s (%(chronos_state)s)"
" Last: %(last_result)s (%(formatted_time)s)\n"
" %(schedule_type)s: %(schedule_value)s\n"
" Command: %(command)s\n"
" Mesos: %(mesos_status)s" % {
"job_name": job_name,
"is_temporary": is_temporary,
"schedule_type": schedule_type,
"chronos_state": PaastaColors.grey(chronos_state),
"disabled_state": disabled_state,
"last_result": last_result,
"formatted_time": formatted_time,
"schedule_value": schedule_value,
"command": command,
"mesos_status": mesos_status,
}
)
def perform_command(command, service, instance, cluster, verbose, soa_dir):
tail_lines = calculate_tail_lines(verbose_level=verbose)
# We have to add a spacer at the end to make sure we only return
# things for service.main and not service.main_foo
task_id_prefix = "{}{}".format(compose_job_id(service, instance), MESOS_TASK_SPACER)
if command == 'status':
paasta_print(status_mesos_tasks_verbose(
job_id=task_id_prefix,
get_short_task_id=lambda x: x,
tail_lines=tail_lines,
))
def test_status_mesos_tasks_verbose(test_case):
tail_lines, expected_format_tail_call_count = test_case
with contextlib.nested(
mock.patch('paasta_tools.mesos_tools.get_running_tasks_from_active_frameworks', autospec=True,),
mock.patch('paasta_tools.mesos_tools.get_non_running_tasks_from_active_frameworks', autospec=True,),
mock.patch('paasta_tools.mesos_tools.format_running_mesos_task_row', autospec=True,),
mock.patch('paasta_tools.mesos_tools.format_non_running_mesos_task_row', autospec=True,),
mock.patch('paasta_tools.mesos_tools.format_stdstreams_tail_for_task', autospec=True,),
) as (
get_running_mesos_tasks_patch,
get_non_running_mesos_tasks_patch,
format_running_mesos_task_row_patch,
format_non_running_mesos_task_row_patch,
format_stdstreams_tail_for_task_patch,
):
get_running_mesos_tasks_patch.return_value = ['doing a lap']
template_task_return = {
'statuses': [{'timestamp': '##########'}],
'state': 'NOT_RUNNING',
}
non_running_mesos_tasks = []
for _ in xrange(15): # excercise the code that sorts/truncates the list of non running tasks
task_return = template_task_return.copy()
task_return['statuses'][0]['timestamp'] = str(1457109986 + random.randrange(-60 * 60 * 24, 60 * 60 * 24))
non_running_mesos_tasks.append(task_return)
get_non_running_mesos_tasks_patch.return_value = non_running_mesos_tasks
format_running_mesos_task_row_patch.return_value = ['id', 'host', 'mem', 'cpu', 'time']
format_non_running_mesos_task_row_patch.return_value = ['id', 'host', 'time', 'state']
format_stdstreams_tail_for_task_patch.return_value = ['tail']
job_id = format_job_id('fake_service', 'fake_instance'),
def get_short_task_id(_):
return 'short_task_id'
actual = mesos_tools.status_mesos_tasks_verbose(
job_id=job_id,
get_short_task_id=get_short_task_id,
tail_lines=tail_lines,
)
assert 'Running Tasks' in actual
assert 'Non-Running Tasks' in actual
format_running_mesos_task_row_patch.assert_called_once_with('doing a lap', get_short_task_id)
assert format_non_running_mesos_task_row_patch.call_count == 10 # maximum n of tasks we display
assert format_stdstreams_tail_for_task_patch.call_count == expected_format_tail_call_count
def test_status_mesos_tasks_verbose(test_case):
tail_lines, expected_format_tail_call_count = test_case
filter_string = format_job_id('fake_service', 'fake_instance')
with asynctest.patch(
'paasta_tools.mesos_tools.get_cached_list_of_running_tasks_from_frameworks', autospec=True,
return_value=[{'id': filter_string}],
), asynctest.patch(
'paasta_tools.mesos_tools.get_cached_list_of_not_running_tasks_from_frameworks', autospec=True,
) as get_cached_list_of_not_running_tasks_from_frameworks_patch, asynctest.patch(
'paasta_tools.mesos_tools.format_running_mesos_task_row', autospec=True,
) as format_running_mesos_task_row_patch, asynctest.patch(
'paasta_tools.mesos_tools.format_non_running_mesos_task_row', autospec=True,
) as format_non_running_mesos_task_row_patch, asynctest.patch(
'paasta_tools.mesos_tools.format_stdstreams_tail_for_task', autospec=True,
) as format_stdstreams_tail_for_task_patch:
template_task_return = {
'id': filter_string,
'statuses': [{'timestamp': '##########'}],
'state': 'NOT_RUNNING',
}
non_running_mesos_tasks = []
for _ in range(15): # excercise the code that sorts/truncates the list of non running tasks
task_return = template_task_return.copy()
task_return['statuses'][0]['timestamp'] = str(1457109986 + random.randrange(-60 * 60 * 24, 60 * 60 * 24))
non_running_mesos_tasks.append(task_return)
get_cached_list_of_not_running_tasks_from_frameworks_patch.return_value = non_running_mesos_tasks
format_running_mesos_task_row_patch.return_value = ['id', 'host', 'mem', 'cpu', 'time']
format_non_running_mesos_task_row_patch.return_value = ['id', 'host', 'time', 'state']
format_stdstreams_tail_for_task_patch.return_value = ['tail']
actual = mesos_tools.status_mesos_tasks_verbose(
filter_string=filter_string,
get_short_task_id=mock.sentinel.get_short_task_id,
tail_lines=tail_lines,
)
assert 'Running Tasks' in actual
assert 'Non-Running Tasks' in actual
format_running_mesos_task_row_patch.assert_called_once_with(
{'id': filter_string},
mock.sentinel.get_short_task_id,
)
assert format_non_running_mesos_task_row_patch.call_count == 10 # maximum n of tasks we display
assert format_stdstreams_tail_for_task_patch.call_count == expected_format_tail_call_count
def test_status_mesos_tasks_verbose():
with contextlib.nested(
mock.patch(
'paasta_tools.mesos_tools.get_running_tasks_from_active_frameworks',
autospec=True,
),
mock.patch(
'paasta_tools.mesos_tools.get_non_running_tasks_from_active_frameworks',
autospec=True,
),
mock.patch(
'paasta_tools.mesos_tools.format_running_mesos_task_row',
autospec=True,
),
mock.patch(
'paasta_tools.mesos_tools.format_non_running_mesos_task_row',
autospec=True,
),
) as (
get_running_mesos_tasks_patch,
get_non_running_mesos_tasks_patch,
format_running_mesos_task_row_patch,
format_non_running_mesos_task_row_patch,
):
get_running_mesos_tasks_patch.return_value = ['doing a lap']
get_non_running_mesos_tasks_patch.return_value = ['eating a burrito']
format_running_mesos_task_row_patch.return_value = [
'id', 'host', 'mem', 'cpu', 'time'
]
format_non_running_mesos_task_row_patch.return_value = [
'id', 'host', 'time', 'state'
]
job_id = format_job_id('fake_service', 'fake_instance'),
def get_short_task_id(_):
return 'short_task_id'
actual = mesos_tools.status_mesos_tasks_verbose(
job_id, get_short_task_id)
assert 'Running Tasks' in actual
assert 'Non-Running Tasks' in actual
format_running_mesos_task_row_patch.assert_called_once_with(
'doing a lap', get_short_task_id)
format_non_running_mesos_task_row_patch.assert_called_once_with(
'eating a burrito', get_short_task_id)
def status_mesos_tasks(
service: str,
instance: str,
normal_instance_count: int,
verbose: int,
) -> str:
job_id = marathon_tools.format_job_id(service, instance)
# We have to add a spacer at the end to make sure we only return
# things for service.main and not service.main_foo
filter_string = f"{job_id}{marathon_tools.MESOS_TASK_SPACER}"
try:
count = len(
select_tasks_by_id(
a_sync.block(get_cached_list_of_running_tasks_from_frameworks),
filter_string))
if count >= normal_instance_count:
status = PaastaColors.green("Healthy")
count_str = PaastaColors.green("(%d/%d)" %
(count, normal_instance_count))
elif count == 0:
status = PaastaColors.red("Critical")
count_str = PaastaColors.red("(%d/%d)" %
(count, normal_instance_count))
else:
status = PaastaColors.yellow("Warning")
count_str = PaastaColors.yellow("(%d/%d)" %
(count, normal_instance_count))
running_string = PaastaColors.bold('TASK_RUNNING')
output = f"Mesos: {status} - {count_str} tasks in the {running_string} state."
except ReadTimeout:
return "Error: talking to Mesos timed out. It may be overloaded."
if verbose > 0:
tail_lines = calculate_tail_lines(verbose_level=verbose)
output += '\n' + status_mesos_tasks_verbose(
filter_string=filter_string,
get_short_task_id=get_short_task_id,
tail_lines=tail_lines,
)
return output
def test_status_mesos_tasks_verbose(test_case):
tail_stdstreams, expected_format_tail_call_count = test_case
with contextlib.nested(
mock.patch("paasta_tools.mesos_tools.get_running_tasks_from_active_frameworks", autospec=True),
mock.patch("paasta_tools.mesos_tools.get_non_running_tasks_from_active_frameworks", autospec=True),
mock.patch("paasta_tools.mesos_tools.format_running_mesos_task_row", autospec=True),
mock.patch("paasta_tools.mesos_tools.format_non_running_mesos_task_row", autospec=True),
mock.patch("paasta_tools.mesos_tools.format_stdstreams_tail_for_task", autospec=True),
) as (
get_running_mesos_tasks_patch,
get_non_running_mesos_tasks_patch,
format_running_mesos_task_row_patch,
format_non_running_mesos_task_row_patch,
format_stdstreams_tail_for_task_patch,
):
get_running_mesos_tasks_patch.return_value = ["doing a lap"]
template_task_return = {"statuses": [{"timestamp": "##########"}], "state": "NOT_RUNNING"}
non_running_mesos_tasks = []
for _ in xrange(15): # excercise the code that sorts/truncates the list of non running tasks
task_return = template_task_return.copy()
task_return["statuses"][0]["timestamp"] = str(1457109986 + random.randrange(-60 * 60 * 24, 60 * 60 * 24))
non_running_mesos_tasks.append(task_return)
get_non_running_mesos_tasks_patch.return_value = non_running_mesos_tasks
format_running_mesos_task_row_patch.return_value = ["id", "host", "mem", "cpu", "time"]
format_non_running_mesos_task_row_patch.return_value = ["id", "host", "time", "state"]
format_stdstreams_tail_for_task_patch.return_value = ["tail"]
job_id = (format_job_id("fake_service", "fake_instance"),)
def get_short_task_id(_):
return "short_task_id"
actual = mesos_tools.status_mesos_tasks_verbose(job_id, get_short_task_id, tail_stdstreams)
assert "Running Tasks" in actual
assert "Non-Running Tasks" in actual
format_running_mesos_task_row_patch.assert_called_once_with("doing a lap", get_short_task_id)
assert format_non_running_mesos_task_row_patch.call_count == 10 # maximum n of tasks we display
assert format_stdstreams_tail_for_task_patch.call_count == expected_format_tail_call_count
def perform_command(command,
service,
instance,
cluster,
verbose,
soa_dir,
app_id=None,
delta=None):
"""Performs a start/stop/restart/status/scale on an instance
:param command: String of start, stop, restart, status or scale
:param service: service name
:param instance: instance name, like "main" or "canary"
:param cluster: cluster name
:param verbose: bool if the output should be verbose or not
:returns: A unix-style return code
"""
marathon_config = marathon_tools.load_marathon_config()
job_config = marathon_tools.load_marathon_service_config(service,
instance,
cluster,
soa_dir=soa_dir)
if not app_id:
try:
app_id = marathon_tools.create_complete_config(
service, instance, marathon_config, soa_dir=soa_dir)['id']
except NoDockerImageError:
job_id = compose_job_id(service, instance)
print "Docker image for %s not in deployments.json. Exiting. Has Jenkins deployed it?" % job_id
return 1
normal_instance_count = job_config.get_instances()
normal_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace(
service, instance)
proxy_port = marathon_tools.get_proxy_port_for_instance(service,
instance,
soa_dir=soa_dir)
client = marathon_tools.get_marathon_client(marathon_config.get_url(),
marathon_config.get_username(),
marathon_config.get_password())
if command == 'start':
start_marathon_job(service, instance, app_id, normal_instance_count,
client, cluster)
elif command == 'stop':
stop_marathon_job(service, instance, app_id, client, cluster)
elif command == 'restart':
restart_marathon_job(service, instance, app_id, normal_instance_count,
client, cluster)
elif command == 'status':
# Setting up transparent cache for http API calls
requests_cache.install_cache('paasta_serviceinit', backend='memory')
print status_desired_state(service, instance, client, job_config)
print status_marathon_job(service, instance, app_id,
normal_instance_count, client)
tasks, out = status_marathon_job_verbose(service, instance, client)
if verbose:
print out
print status_mesos_tasks(service, instance, normal_instance_count)
if verbose:
print status_mesos_tasks_verbose(app_id, get_short_task_id)
if proxy_port is not None:
print status_smartstack_backends(
service=service,
instance=instance,
cluster=cluster,
job_config=job_config,
tasks=tasks,
expected_count=normal_smartstack_count,
soa_dir=soa_dir,
verbose=verbose,
)
elif command == 'scale':
scale_marathon_job(service, instance, app_id, delta, client, cluster)
else:
# The command parser shouldn't have let us get this far...
raise NotImplementedError("Command %s is not implemented!" % command)
return 0
def test_status_mesos_tasks_verbose(test_case):
tail_lines, expected_format_tail_call_count = test_case
filter_string = format_job_id("fake_service", "fake_instance")
with asynctest.patch(
"paasta_tools.mesos_tools.get_cached_list_of_running_tasks_from_frameworks",
autospec=True,
return_value=[{
"id": filter_string
}],
), asynctest.patch(
"paasta_tools.mesos_tools.get_cached_list_of_not_running_tasks_from_frameworks",
autospec=True,
) as get_cached_list_of_not_running_tasks_from_frameworks_patch, asynctest.patch(
"paasta_tools.mesos_tools.format_running_mesos_task_row",
autospec=True
) as format_running_mesos_task_row_patch, asynctest.patch(
"paasta_tools.mesos_tools.format_non_running_mesos_task_row",
autospec=True
) as format_non_running_mesos_task_row_patch, asynctest.patch(
"paasta_tools.mesos_tools.format_stdstreams_tail_for_task",
autospec=True) as format_stdstreams_tail_for_task_patch:
template_task_return = {
"id": filter_string,
"statuses": [{
"timestamp": "##########"
}],
"state": "NOT_RUNNING",
}
non_running_mesos_tasks = []
for _ in range(
15
): # exercise the code that sorts/truncates the list of non running tasks
task_return = template_task_return.copy()
task_return["statuses"][0]["timestamp"] = str(
1457109986 + random.randrange(-60 * 60 * 24, 60 * 60 * 24))
non_running_mesos_tasks.append(task_return)
get_cached_list_of_not_running_tasks_from_frameworks_patch.return_value = (
non_running_mesos_tasks)
format_running_mesos_task_row_patch.return_value = [
"id",
"host",
"mem",
"cpu",
"time",
]
format_non_running_mesos_task_row_patch.return_value = [
"id",
"host",
"time",
"state",
]
format_stdstreams_tail_for_task_patch.return_value = ["tail"]
actual = mesos_tools.status_mesos_tasks_verbose(
filter_string=filter_string,
get_short_task_id=mock.sentinel.get_short_task_id,
tail_lines=tail_lines,
)
assert "Running Tasks" in actual
assert "Non-Running Tasks" in actual
format_running_mesos_task_row_patch.assert_called_once_with(
{"id": filter_string}, mock.sentinel.get_short_task_id)
assert (format_non_running_mesos_task_row_patch.call_count == 10
) # maximum n of tasks we display
assert (format_stdstreams_tail_for_task_patch.call_count ==
expected_format_tail_call_count)