def paasta_autoscale(args):
log.setLevel(logging.DEBUG)
service = figure_out_service_name(args)
api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
if not api:
paasta_print(
"Could not connect to paasta api. Maybe you misspelled the cluster?"
)
return 1
if args.set is None:
log.debug("Getting the current autoscaler count...")
res, http = api.autoscaler.get_autoscaler_count(
service=service, instance=args.instance).result()
else:
log.debug(f"Setting desired instances to {args.set}.")
body = {"desired_instances": int(args.set)}
res, http = api.autoscaler.update_autoscaler_count(
service=service, instance=args.instance, json_body=body).result()
_log_audit(
action="manual-scale",
action_details=body,
service=service,
instance=args.instance,
cluster=args.cluster,
)
log.debug(f"Res: {res} Http: {http}")
print(res["desired_instances"])
return 0
def paasta_metastatus_on_api_endpoint(
cluster: str,
system_paasta_config: SystemPaastaConfig,
groupings: Sequence[str],
verbose: int,
autoscaling_info: bool = False,
use_mesos_cache: bool = False,
) -> Tuple[int, str]:
client = get_paasta_api_client(cluster, system_paasta_config)
if not client:
paasta_print('Cannot get a paasta-api client')
exit(1)
try:
cmd_args, _ = get_paasta_metastatus_cmd_args(
groupings=groupings,
verbose=verbose,
autoscaling_info=autoscaling_info,
use_mesos_cache=use_mesos_cache,
)
res = client.metastatus.metastatus(
cmd_args=[str(arg) for arg in cmd_args]).result()
output, exit_code = res.output, res.exit_code
except HTTPError as exc:
output, exit_code = exc.response.text, exc.status_code
return exit_code, output
def paasta_autoscale(args):
log.setLevel(logging.DEBUG)
service = figure_out_service_name(args)
api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
if not api:
paasta_print(
'Could not connect to paasta api. Maybe you misspelled the cluster?'
)
return 1
if args.set is None:
log.debug("Getting the current autoscaler count...")
res, http = api.autoscaler.get_autoscaler_count(
service=service, instance=args.instance).result()
else:
log.debug("Setting desired instances to {}.".format(args.set))
body = {'desired_instances': int(args.set)}
res, http = api.autoscaler.update_autoscaler_count(
service=service,
instance=args.instance,
json_body=body,
).result()
log.debug("Res: {} Http: {}".format(res, http))
print(res["desired_instances"])
return 0
def paasta_status_on_api_endpoint(
cluster: str,
service: str,
instance: str,
output: List[str],
system_paasta_config: SystemPaastaConfig,
verbose: int,
) -> int:
client = get_paasta_api_client(cluster, system_paasta_config)
if not client:
paasta_print('Cannot get a paasta-api client')
exit(1)
try:
status = client.service.status_instance(service=service,
instance=instance).result()
except HTTPError as exc:
paasta_print(exc.response.text)
return exc.status_code
output.append(' instance: %s' % PaastaColors.blue(instance))
output.append(' Git sha: %s (desired)' % status.git_sha)
if status.marathon is not None:
return print_marathon_status(service, instance, output,
status.marathon)
elif status.kubernetes is not None:
return print_kubernetes_status(service, instance, output,
status.kubernetes)
else:
paasta_print(
"Not implemented: Looks like %s is not a Marathon or Kubernetes instance"
% instance)
return 0
def test_get_paasta_api_client(system_paasta_config):
with mock.patch("paasta_tools.api.client.load_system_paasta_config",
autospec=True) as mock_load_system_paasta_config:
mock_load_system_paasta_config.return_value = system_paasta_config
client = get_paasta_api_client()
assert client
def get_service_autoscale_pause_time(cluster):
api = client.get_paasta_api_client(cluster=cluster, http_res=True)
if not api:
paasta_print(
'Could not connect to paasta api. Maybe you misspelled the cluster?'
)
return 1
pause_time, http = api.service_autoscaler.get_service_autoscaler_pause(
).result()
if http.status_code == 500:
paasta_print('Could not connect to zookeeper server')
return 2
pause_time = float(pause_time)
if pause_time < time.time():
paasta_print('Service autoscaler is not paused')
else:
local_tz = get_localzone()
paused_readable = local_tz.localize(
datetime.fromtimestamp(pause_time)).strftime(
'%F %H:%M:%S %Z') # NOQA
paasta_print(
'Service autoscaler is paused until {}'.format(paused_readable))
return 0
def paasta_status_on_api_endpoint(cluster, service, instance,
system_paasta_config, verbose):
client = get_paasta_api_client(cluster, system_paasta_config)
if not client:
paasta_print('Cannot get a paasta-api client')
exit(1)
try:
status = client.service.status_instance(service=service,
instance=instance).result()
except HTTPError as exc:
paasta_print(exc.response.text)
return exc.status_code
paasta_print('instance: %s' % PaastaColors.blue(instance))
paasta_print('Git sha: %s (desired)' % status.git_sha)
marathon_status = status.marathon
if marathon_status is None:
paasta_print(
"Not implemented: Looks like %s is not a Marathon instance" %
instance)
return 0
elif marathon_status.error_message:
paasta_print(marathon_status.error_message)
return 1
bouncing_status = bouncing_status_human(
marathon_status.app_count,
marathon_status.bounce_method,
)
desired_state = desired_state_human(
marathon_status.desired_state,
marathon_status.expected_instance_count,
)
paasta_print("State: %s - Desired state: %s" %
(bouncing_status, desired_state))
status = MarathonDeployStatus.fromstring(marathon_status.deploy_status)
if status != MarathonDeployStatus.NotRunning:
if status == MarathonDeployStatus.Delayed:
deploy_status = marathon_app_deploy_status_human(
status, marathon_status.backoff_seconds)
else:
deploy_status = marathon_app_deploy_status_human(status)
else:
deploy_status = 'NotRunning'
paasta_print(
status_marathon_job_human(
service=service,
instance=instance,
deploy_status=deploy_status,
desired_app_id=marathon_status.app_id,
app_count=marathon_status.app_count,
running_instances=marathon_status.running_instance_count,
normal_instance_count=marathon_status.expected_instance_count,
), )
return 0
def get_status_for_instance(cluster, service, instance):
api = client.get_paasta_api_client(cluster=cluster)
if not api:
sys.exit(1)
status = api.service.status_instance(service=service, instance=instance).result()
if not status.marathon:
log.error("Not a marathon service, exiting")
sys.exit(1)
return status
def get_status_for_instance(cluster, service, instance):
api = client.get_paasta_api_client(cluster=cluster)
if not api:
sys.exit(1)
status = api.service.status_instance(service=service, instance=instance).result()
if not status.marathon:
log.error("Not a marathon service, exiting")
sys.exit(1)
return status
def get_task_from_instance(cluster,
service,
instance,
slave_hostname=None,
task_id=None,
verbose=True):
api = client.get_paasta_api_client(cluster=cluster)
if not api:
log.error(f"Could not get API client for cluster {cluster}")
raise PaastaTaskNotFound
if task_id:
log.warning("Specifying a task_id, so ignoring hostname if specified")
task = api.service.task_instance(
service=service,
instance=instance,
verbose=True,
task_id=task_id,
).result()
return task
try:
if task_id:
log.warning(
"Specifying a task_id, so ignoring hostname if specified")
task = api.service.task_instance(
service=service,
instance=instance,
verbose=True,
task_id=task_id,
).result()
return task
tasks = api.service.tasks_instance(
service=service,
instance=instance,
verbose=True,
slave_hostname=slave_hostname,
).result()
except HTTPNotFound:
log.error(
"Cannot find instance {}, for service {}, in cluster {}".format(
instance,
service,
cluster,
))
raise PaastaTaskNotFound
except HTTPError as e:
log.error("Problem with API call to find task details")
log.error(e.response.text)
raise PaastaTaskNotFound
if not tasks:
log.error(
"Cannot find any tasks on host: {} or with task_id: {}".format(
slave_hostname,
task_id,
))
raise PaastaTaskNotFound
return tasks[0]
def paasta_status_on_api_endpoint(
cluster: str,
service: str,
instance: str,
output: List[str],
system_paasta_config: SystemPaastaConfig,
verbose: int,
) -> int:
output.append(" instance: %s" % PaastaColors.blue(instance))
client = get_paasta_api_client(cluster, system_paasta_config)
if not client:
paasta_print("Cannot get a paasta-api client")
exit(1)
try:
status = client.service.status_instance(
service=service, instance=instance, verbose=verbose
).result()
except HTTPError as exc:
output.append(PaastaColors.red(exc.response.text))
return exc.status_code
except (BravadoConnectionError, BravadoTimeoutError) as exc:
output.append(
PaastaColors.red(f"Could not connect to API: {exc.__class__.__name__}")
)
return 1
except Exception:
tb = sys.exc_info()[2]
output.append(PaastaColors.red(f"Exception when talking to the API:"))
output.extend(line.strip() for line in traceback.format_tb(tb))
return 1
if status.git_sha != "":
output.append(" Git sha: %s (desired)" % status.git_sha)
if status.marathon is not None:
return print_marathon_status(service, instance, output, status.marathon)
elif status.kubernetes is not None:
return print_kubernetes_status(service, instance, output, status.kubernetes)
elif status.tron is not None:
return print_tron_status(service, instance, output, status.tron, verbose)
elif status.adhoc is not None:
return print_adhoc_status(
cluster, service, instance, output, status.adhoc, verbose
)
elif status.flink is not None:
return print_flink_status(
cluster, service, instance, output, status.flink, verbose
)
else:
paasta_print(
"Not implemented: Looks like %s is not a Marathon or Kubernetes instance"
% instance
)
return 0
def test_get_paasta_api_client():
with mock.patch('paasta_tools.api.client.load_system_paasta_config',
autospec=True) as mock_load_system_paasta_config:
mock_load_system_paasta_config.return_value = SystemPaastaConfig({
'api_endpoints': {
'fake_cluster': "http://fake_cluster:5054"
},
'cluster': 'fake_cluster'
}, 'fake_directory')
client = get_paasta_api_client()
assert client
def delete_service_autoscale_pause_time(cluster):
api = client.get_paasta_api_client(cluster=cluster, http_res=True)
if not api:
print("Could not connect to paasta api. Maybe you misspelled the cluster?")
return 1
res, http = api.service_autoscaler.delete_service_autoscaler_pause().result()
if http.status_code == 500:
print("Could not connect to zookeeper server")
return 2
print("Service autoscaler is unpaused")
return 0
def update_service_autoscale_pause_time(cluster, mins):
api = client.get_paasta_api_client(cluster=cluster, http_res=True)
if not api:
paasta_print('Could not connect to paasta api. Maybe you misspelled the cluster?')
return 1
body = {'minutes': mins}
res, http = api.service_autoscaler.update_service_autoscaler_pause(json_body=body).result()
if http.status_code == 500:
paasta_print('Could not connect to zookeeper server')
return 2
paasta_print(f'Service autoscaler is paused for {mins}')
return 0
def test_get_paasta_api_client():
with mock.patch('paasta_tools.api.client.load_system_paasta_config',
autospec=True) as mock_load_system_paasta_config:
mock_load_system_paasta_config.return_value = SystemPaastaConfig(
{
'api_endpoints': {
'fake_cluster': "http://fake_cluster:5054"
},
'cluster': 'fake_cluster'
}, 'fake_directory')
client = get_paasta_api_client()
assert client
def paasta_status_on_api_endpoint(cluster, service, instance, system_paasta_config, verbose):
client = get_paasta_api_client(cluster, system_paasta_config)
if not client:
paasta_print('Cannot get a paasta-api client')
exit(1)
try:
status = client.service.status_instance(service=service, instance=instance).result()
except HTTPError as exc:
paasta_print(exc.response.text)
return exc.status_code
paasta_print('instance: %s' % PaastaColors.blue(instance))
paasta_print('Git sha: %s (desired)' % status.git_sha)
marathon_status = status.marathon
if marathon_status is None:
paasta_print("Not implemented: Looks like %s is not a Marathon instance" % instance)
return 0
elif marathon_status.error_message:
paasta_print(marathon_status.error_message)
return 1
bouncing_status = bouncing_status_human(marathon_status.app_count,
marathon_status.bounce_method)
desired_state = desired_state_human(marathon_status.desired_state,
marathon_status.expected_instance_count)
paasta_print("State: %s - Desired state: %s" % (bouncing_status, desired_state))
status = MarathonDeployStatus.fromstring(marathon_status.deploy_status)
if status != MarathonDeployStatus.NotRunning:
if status == MarathonDeployStatus.Delayed:
deploy_status = marathon_app_deploy_status_human(status, marathon_status.backoff_seconds)
else:
deploy_status = marathon_app_deploy_status_human(status)
else:
deploy_status = 'NotRunning'
paasta_print(
status_marathon_job_human(
service,
instance,
deploy_status,
marathon_status.app_id,
marathon_status.running_instance_count,
marathon_status.expected_instance_count,
)
)
return 0
def paasta_autoscale(args):
log.setLevel(logging.DEBUG)
service = figure_out_service_name(args)
api = client.get_paasta_api_client(cluster=args.cluster, http_res=True)
if not api:
paasta_print(
"Could not connect to paasta api. Maybe you misspelled the cluster?"
)
return 1
try:
if args.set is None:
log.debug("Getting the current autoscaler count...")
res, http = api.autoscaler.get_autoscaler_count(
service=service, instance=args.instance
).result()
else:
log.debug(f"Setting desired instances to {args.set}.")
body = {"desired_instances": int(args.set)}
res, http = api.autoscaler.update_autoscaler_count(
service=service, instance=args.instance, json_body=body
).result()
_log_audit(
action="manual-scale",
action_details=body,
service=service,
instance=args.instance,
cluster=args.cluster,
)
except HTTPNotFound:
paasta_print(
PaastaColors.red(
f"ERROR: '{args.instance}' is not configured to autoscale, "
f"so paasta autoscale could not scale it up on demand. "
f"If you want to be able to boost this service, please configure autoscaling for the service "
f"in its config file by setting min and max instances. Example: \n"
f"{args.instance}:\n"
f" min_instances: 5\n"
f" max_instances: 50"
)
)
return 0
log.debug(f"Res: {res} Http: {http}")
print(res["desired_instances"])
return 0
def paasta_status_on_api_endpoint(
cluster: str,
service: str,
instance: str,
output: List[str],
system_paasta_config: SystemPaastaConfig,
verbose: int,
) -> int:
client = get_paasta_api_client(cluster, system_paasta_config)
if not client:
paasta_print("Cannot get a paasta-api client")
exit(1)
try:
status = client.service.status_instance(
service=service, instance=instance, verbose=verbose
).result()
except HTTPError as exc:
paasta_print(exc.response.text)
return exc.status_code
output.append(" instance: %s" % PaastaColors.blue(instance))
if status.git_sha != "":
output.append(" Git sha: %s (desired)" % status.git_sha)
if status.marathon is not None:
return print_marathon_status(service, instance, output, status.marathon)
elif status.kubernetes is not None:
return print_kubernetes_status(service, instance, output, status.kubernetes)
elif status.tron is not None:
return print_tron_status(service, instance, output, status.tron, verbose)
elif status.adhoc is not None:
return print_adhoc_status(
cluster, service, instance, output, status.adhoc, verbose
)
elif status.flink is not None:
return print_flink_status(
cluster, service, instance, output, status.flink.get("status"), verbose
)
elif status.chronos is not None:
return print_chronos_status(output, status.chronos.output)
else:
paasta_print(
"Not implemented: Looks like %s is not a Marathon or Kubernetes instance"
% instance
)
return 0
def list_deploy_queue(args, ) -> int:
cluster = args.cluster
all_clusters = list_clusters(soa_dir=args.soa_dir)
if cluster not in all_clusters:
paasta_print(
f"{cluster} does not appear to be a valid cluster. Run `paasta "
"list-clusters` to see available options.")
return 1
system_paasta_config = load_system_paasta_config()
client = get_paasta_api_client(cluster,
system_paasta_config,
http_res=True)
if not client:
paasta_print("Cannot get a paasta API client")
return 1
try:
deploy_queues, raw_response = client.deploy_queue.deploy_queue(
).result()
except HTTPError as exc:
paasta_print(PaastaColors.red(exc.response.text))
return exc.status_code
except (BravadoConnectionError, BravadoTimeoutError) as exc:
paasta_print(
PaastaColors.red(
f"Could not connect to API: {exc.__class__.__name__}"))
return 1
except Exception:
tb = sys.exc_info()[2]
paasta_print(PaastaColors.red(f"Exception when talking to the API:"))
paasta_print("".join(traceback.format_tb(tb)))
return 1
if args.json:
paasta_print(raw_response.text)
else:
formatted_deploy_queues = format_deploy_queues(deploy_queues, cluster)
paasta_print(formatted_deploy_queues)
return 0
def paasta_status_on_api_endpoint(cluster, service, instance, system_paasta_config, verbose):
client = get_paasta_api_client(cluster, system_paasta_config)
if not client:
print 'Cannot get a paasta-api client'
exit(1)
try:
status = client.service.status_instance(service=service, instance=instance).result()
except HTTPError as exc:
print exc.response.text
return
print 'instance: %s' % PaastaColors.blue(instance)
print 'Git sha: %s (desired)' % status.git_sha
marathon_status = status.marathon
if marathon_status.error_message:
print marathon_status.error_message
return
bouncing_status = bouncing_status_human(marathon_status.app_count,
marathon_status.bounce_method)
desired_state = desired_state_human(marathon_status.desired_state,
marathon_status.expected_instance_count)
print "State: %s - Desired state: %s" % (bouncing_status, desired_state)
status = MarathonDeployStatus.fromstring(marathon_status.deploy_status)
if status != MarathonDeployStatus.NotRunning:
if status == MarathonDeployStatus.Delayed:
deploy_status = marathon_app_deploy_status_human(status, marathon_status.backoff_seconds)
else:
deploy_status = marathon_app_deploy_status_human(status)
else:
deploy_status = 'NotRunning'
print status_marathon_job_human(service, instance, deploy_status,
marathon_status.app_id,
marathon_status.running_instance_count,
marathon_status.expected_instance_count)
def get_task_from_instance(cluster, service, instance, slave_hostname=None, task_id=None, verbose=True):
api = client.get_paasta_api_client(cluster=cluster)
if not api:
log.error("Could not get API client for cluster {0}".format(cluster))
raise PaastaTaskNotFound
if task_id:
log.warning("Specifying a task_id, so ignoring hostname if specified")
task = api.service.task_instance(service=service,
instance=instance,
verbose=True,
task_id=task_id).result()
return task
try:
if task_id:
log.warning("Specifying a task_id, so ignoring hostname if specified")
task = api.service.task_instance(service=service,
instance=instance,
verbose=True,
task_id=task_id).result()
return task
tasks = api.service.tasks_instance(service=service,
instance=instance,
verbose=True,
slave_hostname=slave_hostname).result()
except HTTPNotFound:
log.error("Cannot find instance {0}, for service {1}, in cluster {2}".format(instance,
service,
cluster))
raise PaastaTaskNotFound
except HTTPError as e:
log.error("Problem with API call to find task details")
log.error(e.response.text)
raise PaastaTaskNotFound
if not tasks:
log.error("Cannot find any tasks on host: {0} or with task_id: {1}".format(slave_hostname,
task_id))
raise PaastaTaskNotFound
return tasks[0]
def are_instances_deployed(cluster, service, instances, git_sha):
api = client.get_paasta_api_client(cluster=cluster)
if not api:
# Assume not deployed if we can't reach API
return False
statuses = []
for instance in instances:
try:
statuses.append(
api.service.status_instance(service=service,
instance=instance).result())
except HTTPError as e:
if e.response.status_code == 404:
log.warning(
"Can't get status for instance {0}, service {1} in cluster {2}. "
"This is normally because it is a new service that hasn't been "
"deployed by PaaSTA yet".format(instance, service,
cluster))
statuses.append(None)
results = []
for status in statuses:
if not status:
results.append(False)
# if it's a chronos service etc then skip waiting for it to deploy
elif not status.marathon:
results.append(True)
elif status.marathon.expected_instance_count == 0 or status.marathon.desired_state == 'stop':
results.append(True)
else:
results.append(
git_sha.startswith(status.git_sha)
and status.marathon.app_count == 1
and status.marathon.deploy_status == 'Running'
and status.marathon.expected_instance_count
== status.marathon.running_instance_count)
return all(results)
def instances_deployed(cluster, service, instances, git_sha):
api = client.get_paasta_api_client(cluster=cluster)
if not api:
log.warning(
"Couldn't reach the PaaSTA api for {}! Assuming it is not deployed there yet."
.format(cluster))
return False
statuses = []
for instance in instances:
log.info("Inspecting the deployment status of {}.{} on {}".format(
service, instance, cluster))
try:
status = api.service.status_instance(service=service,
instance=instance).result()
statuses.append(status)
except HTTPError as e:
if e.response.status_code == 404:
log.warning(
"Can't get status for instance {0}, service {1} in cluster {2}. "
"This is normally because it is a new service that hasn't been "
"deployed by PaaSTA yet".format(instance, service,
cluster))
else:
log.warning(
"Error getting service status from PaaSTA API: {0}: {1}".
format(e.response.status_code, e.response.text))
statuses.append(None)
except ConnectionError as e:
log.warning(
"Error getting service status from PaaSTA API for {0}: {1}".
format(cluster, e))
statuses.append(None)
results = []
for status in statuses:
if not status:
log.info(
"No status for an unknown instance in {}. Not deployed yet.".
format(cluster))
results.append(False)
elif not status.marathon:
log.info("{}.{} in {} is not a Marathon job. Marked as deployed.".
format(service, status.instance, cluster))
results.append(True)
elif status.marathon.expected_instance_count == 0 or status.marathon.desired_state == 'stop':
log.info(
"{}.{} in {} is marked as stopped. Marked as deployed.".format(
service, status.instance, cluster))
results.append(True)
else:
if status.marathon.app_count != 1:
log.info("{}.{} on {} is still bouncing, {} versions running".
format(service, status.instance, cluster,
status.marathon.app_count))
results.append(False)
continue
if not git_sha.startswith(status.git_sha):
log.info(
"{}.{} on {} doesn't have the right sha yet: {}".format(
service, status.instance, cluster, status.git_sha))
results.append(False)
continue
if status.marathon.deploy_status != 'Running':
log.info("{}.{} on {} isn't running yet: {}".format(
service, status.instance, cluster,
status.marathon.deploy_status))
results.append(False)
continue
if status.marathon.expected_instance_count != status.marathon.running_instance_count:
log.info(
"{}.{} on {} isn't scaled up yet, has {} out of {}".format(
service, status.instance, cluster,
status.marathon.running_instance_count,
status.marathon.expected_instance_count))
results.append(False)
continue
log.info(
"{}.{} on {} looks 100% deployed at {} instances on {}".format(
service, status.instance, cluster,
status.marathon.running_instance_count, status.git_sha))
results.append(True)
return sum(results)
def _run_instance_worker(cluster_data, instances_out, green_light):
"""Get instances from the instances_in queue and check them one by one.
If an instance isn't deployed, add it to the instances_out queue
to re-check it later.
:param cluster_data: an instance of ClusterData.
:param instances_out: See the docstring for instances_deployed().
:param green_light: See the docstring for _query_clusters().
"""
api = client.get_paasta_api_client(cluster=cluster_data.cluster)
if not api:
log.warning("Couldn't reach the PaaSTA api for {}! Assuming it is not "
"deployed there yet.".format(cluster_data.cluster))
while not cluster_data.instances_queue.empty():
try:
instance_config = cluster_data.instances_queue.get(block=False)
except Empty:
return
cluster_data.instances_queue.task_done()
instances_out.put(instance_config)
while not cluster_data.instances_queue.empty() and green_light.is_set():
try:
instance_config = cluster_data.instances_queue.get(block=False)
except Empty:
return
instance = instance_config.get_instance()
log.debug("Inspecting the deployment status of {}.{} on {}".format(
cluster_data.service, instance, cluster_data.cluster))
try:
status = None
status = api.service.status_instance(
service=cluster_data.service,
instance=instance,
).result()
except HTTPError as e:
if e.response.status_code == 404:
log.warning(
"Can't get status for instance {}, service {} in "
"cluster {}. This is normally because it is a new "
"service that hasn't been deployed by PaaSTA yet".format(
instance,
cluster_data.service,
cluster_data.cluster,
))
else:
log.warning(
"Error getting service status from PaaSTA API for {}: {}"
"{}".format(
cluster_data.cluster,
e.response.status_code,
e.response.text,
))
except ConnectionError as e:
log.warning("Error getting service status from PaaSTA API for {}:"
"{}".format(cluster_data.cluster, e))
if not status:
log.debug("No status for {}.{}, in {}. Not deployed yet.".format(
cluster_data.service,
instance,
cluster_data.cluster,
))
cluster_data.instances_queue.task_done()
instances_out.put(instance_config)
elif not status.marathon:
log.debug("{}.{} in {} is not a Marathon job. Marked as deployed.".
format(
cluster_data.service,
instance,
cluster_data.cluster,
))
elif (status.marathon.expected_instance_count == 0
or status.marathon.desired_state == 'stop'):
log.debug(
"{}.{} in {} is marked as stopped. Marked as deployed.".format(
cluster_data.service,
status.instance,
cluster_data.cluster,
))
else:
if status.marathon.app_count != 1:
paasta_print(" {}.{} on {} is still bouncing, {} versions "
"running".format(
cluster_data.service,
status.instance,
cluster_data.cluster,
status.marathon.app_count,
))
cluster_data.instances_queue.task_done()
instances_out.put(instance_config)
continue
if not cluster_data.git_sha.startswith(status.git_sha):
paasta_print(
" {}.{} on {} doesn't have the right sha yet: {}".format(
cluster_data.service,
instance,
cluster_data.cluster,
status.git_sha,
))
cluster_data.instances_queue.task_done()
instances_out.put(instance_config)
continue
if status.marathon.deploy_status not in [
'Running', 'Deploying', 'Waiting'
]:
paasta_print(" {}.{} on {} isn't running yet: {}".format(
cluster_data.service,
instance,
cluster_data.cluster,
status.marathon.deploy_status,
))
cluster_data.instances_queue.task_done()
instances_out.put(instance_config)
continue
# The bounce margin factor defines what proportion of instances we need to be "safe",
# so consider it scaled up "enough" if we have that proportion of instances ready.
required_instance_count = int(
math.ceil(
instance_config.get_bounce_margin_factor() *
status.marathon.expected_instance_count, ))
if required_instance_count > status.marathon.running_instance_count:
paasta_print(
" {}.{} on {} isn't scaled up yet, "
"has {} out of {} required instances (out of a total of {})"
.format(
cluster_data.service,
instance,
cluster_data.cluster,
status.marathon.running_instance_count,
required_instance_count,
status.marathon.expected_instance_count,
))
cluster_data.instances_queue.task_done()
instances_out.put(instance_config)
continue
paasta_print("Complete: {}.{} on {} looks 100% deployed at {} "
"instances on {}".format(
cluster_data.service,
instance,
cluster_data.cluster,
status.marathon.running_instance_count,
status.git_sha,
))
cluster_data.instances_queue.task_done()
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
soa_dir = args.soa_dir
pargs = apply_args_filters(args)
if len(pargs) == 0:
return 1
affected_services = {
s
for service_list in pargs.values() for s in service_list.keys()
}
if len(affected_services) > 1:
print(
PaastaColors.red(
"Warning: trying to start/stop/restart multiple services:"))
for cluster, services_instances in pargs.items():
print("Cluster %s:" % cluster)
for service, instances in services_instances.items():
print(" Service %s:" % service)
print(" Instances %s" % ",".join(instances.keys()))
if sys.stdin.isatty():
confirm = choice.Binary("Are you sure you want to continue?",
False).ask()
else:
confirm = False
if not confirm:
print()
print("exiting")
return 1
invalid_deploy_groups = []
marathon_message_printed = False
affected_flinks = []
if args.clusters is None or args.instances is None:
if confirm_to_continue(pargs.items(), desired_state) is False:
print()
print("exiting")
return 1
for cluster, services_instances in pargs.items():
for service, instances in services_instances.items():
for instance in instances.keys():
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
if isinstance(service_config, FlinkDeploymentConfig):
affected_flinks.append(service_config)
continue
try:
remote_refs = get_remote_refs(service, soa_dir)
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment.") % str(e)
print(msg)
return 1
deploy_group = service_config.get_deploy_group()
(deploy_tag,
_) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(
datetime.datetime.utcnow())
if (isinstance(service_config, MarathonServiceConfig)
and not marathon_message_printed):
print_marathon_message(desired_state)
marathon_message_printed = True
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
# TODO: Refactor to discover if set_state is available for given
# instance_type in API
if affected_flinks:
print_flink_message(desired_state)
csi = defaultdict(lambda: defaultdict(list))
for service_config in affected_flinks:
csi[service_config.cluster][service_config.service].append(
service_config.instance)
system_paasta_config = load_system_paasta_config()
for cluster, services_instances in csi.items():
client = get_paasta_api_client(cluster, system_paasta_config)
if not client:
print("Cannot get a paasta-api client")
exit(1)
for service, instances in services_instances.items():
for instance in instances:
try:
client.service.instance_set_state(
service=service,
instance=instance,
desired_state=desired_state,
).result()
except HTTPError as exc:
print(exc.response.text)
return exc.status_code
return_val = 0
if invalid_deploy_groups:
print(f"No deploy tags found for {', '.join(invalid_deploy_groups)}.")
print(f"Has {service} been deployed there yet?")
return_val = 1
return return_val
def instances_deployed(cluster, service, instances, git_sha):
api = client.get_paasta_api_client(cluster=cluster)
if not api:
log.warning("Couldn't reach the PaaSTA api for {}! Assuming it is not deployed there yet.".format(cluster))
return False
statuses = []
for instance in instances:
log.debug("Inspecting the deployment status of {}.{} on {}".format(service, instance, cluster))
try:
status = api.service.status_instance(service=service, instance=instance).result()
statuses.append(status)
except HTTPError as e:
if e.response.status_code == 404:
log.warning("Can't get status for instance {0}, service {1} in cluster {2}. "
"This is normally because it is a new service that hasn't been "
"deployed by PaaSTA yet".format(instance, service, cluster))
else:
log.warning("Error getting service status from PaaSTA API: {0}: {1}".format(e.response.status_code,
e.response.text))
statuses.append(None)
except ConnectionError as e:
log.warning("Error getting service status from PaaSTA API for {0}: {1}".format(cluster, e))
statuses.append(None)
results = []
for status in statuses:
if not status:
log.debug("No status for an unknown instance in {}. Not deployed yet.".format(cluster))
results.append(False)
elif not status.marathon:
log.debug("{}.{} in {} is not a Marathon job. Marked as deployed.".format(
service, status.instance, cluster))
results.append(True)
elif status.marathon.expected_instance_count == 0 or status.marathon.desired_state == 'stop':
log.debug("{}.{} in {} is marked as stopped. Marked as deployed.".format(
service, status.instance, cluster))
results.append(True)
else:
if status.marathon.app_count != 1:
paasta_print(" {}.{} on {} is still bouncing, {} versions running".format(
service, status.instance, cluster, status.marathon.app_count))
results.append(False)
continue
if not git_sha.startswith(status.git_sha):
paasta_print(" {}.{} on {} doesn't have the right sha yet: {}".format(
service, status.instance, cluster, status.git_sha))
results.append(False)
continue
if status.marathon.deploy_status != 'Running':
paasta_print(" {}.{} on {} isn't running yet: {}".format(
service, status.instance, cluster, status.marathon.deploy_status))
results.append(False)
continue
if status.marathon.expected_instance_count != status.marathon.running_instance_count:
paasta_print(" {}.{} on {} isn't scaled up yet, has {} out of {}".format(
service, status.instance, cluster, status.marathon.running_instance_count,
status.marathon.expected_instance_count))
results.append(False)
continue
paasta_print("Complete: {}.{} on {} looks 100% deployed at {} instances on {}".format(
service, status.instance, cluster, status.marathon.running_instance_count, status.git_sha))
results.append(True)
return sum(results)
def run_capacity_check():
options = parse_capacity_check_options()
system_paasta_config = load_system_paasta_config()
cluster = options.cluster if options.cluster is not None else system_paasta_config.get_cluster(
)
value_to_check = options.type
client = get_paasta_api_client(cluster=cluster)
if client is None:
paasta_print('UNKNOWN Failed to load paasta api client')
sys.exit(3)
overrides = read_overrides(options.overrides)
attributes = options.attributes.split(',')
try:
resource_use = client.resources.resources(
groupings=attributes).result()
except HTTPError as e:
paasta_print("UNKNOWN recieved exception from paasta api:\n\t%s" % e)
sys.exit(3)
default_check = {
'warn': {
'cpus': options.warn,
'mem': options.warn,
'disk': options.warn,
},
'crit': {
'cpus': options.crit,
'mem': options.crit,
'disk': options.crit,
},
}
failures = defaultdict(list)
for usage_value in resource_use:
check = get_check_from_overrides(overrides, default_check,
usage_value['groupings'])
usage_percent = calc_percent_usage(usage_value, value_to_check)
for c in ['crit', 'warn']:
if usage_percent > check[c][value_to_check]:
failures[c].append({
'attrs': [{
'attr': a,
'value': v
} for a, v in usage_value['groupings'].items()],
'maximum':
check[c][value_to_check],
'current':
usage_percent,
})
break
return_value = [0]
if len(failures['crit']) > 0:
result = error_message(failures['crit'], 'CRITICAL', cluster,
value_to_check)
paasta_print(result)
return_value.append(2)
if len(failures['warn']) > 0:
result = error_message(failures['warn'], 'WARNING', cluster,
value_to_check)
paasta_print(result)
return_value.append(1)
if max(return_value) == 0:
paasta_print("OK cluster %s is below critical capacity in %s" %
(cluster, value_to_check))
sys.exit(max(return_value))
def _run_instance_worker(cluster_data, instances_out, green_light):
"""Get instances from the instances_in queue and check them one by one.
If an instance isn't deployed, add it to the instances_out queue
to re-check it later.
:param cluster_data: an instance of ClusterData.
:param instances_out: See the docstring for instances_deployed().
:param green_light: See the docstring for _query_clusters().
"""
api = client.get_paasta_api_client(cluster=cluster_data.cluster)
if not api:
log.warning("Couldn't reach the PaaSTA api for {}! Assuming it is not "
"deployed there yet.".format(cluster_data.cluster))
while not cluster_data.instances_queue.empty():
instances_out.put(cluster_data.instances_queue.get())
cluster_data.instances_queue.task_done()
while not cluster_data.instances_queue.empty() and green_light.is_set():
instance = cluster_data.instances_queue.get()
log.debug("Inspecting the deployment status of {}.{} on {}".format(
cluster_data.service, instance, cluster_data.cluster))
try:
status = None
status = api.service.status_instance(service=cluster_data.service,
instance=instance).result()
except HTTPError as e:
if e.response.status_code == 404:
log.warning(
"Can't get status for instance {0}, service {1} in "
"cluster {2}. This is normally because it is a new "
"service that hasn't been deployed by PaaSTA yet".format(
instance, cluster_data.service, cluster_data.cluster))
else:
log.warning(
"Error getting service status from PaaSTA API: {0}:"
"{1}".format(e.response.status_code, e.response.text))
except ConnectionError as e:
log.warning("Error getting service status from PaaSTA API for {0}:"
"{1}".format(cluster_data.cluster, e))
if not status:
log.debug("No status for {}.{}, in {}. Not deployed yet.".format(
cluster_data.service, instance, cluster_data.cluster))
cluster_data.instances_queue.task_done()
instances_out.put(instance)
elif not status.marathon:
log.debug("{}.{} in {} is not a Marathon job. Marked as deployed.".
format(cluster_data.service, instance,
cluster_data.cluster))
elif (status.marathon.expected_instance_count == 0
or status.marathon.desired_state == 'stop'):
log.debug(
"{}.{} in {} is marked as stopped. Marked as deployed.".format(
cluster_data.service, status.instance,
cluster_data.cluster))
else:
if status.marathon.app_count != 1:
paasta_print(" {}.{} on {} is still bouncing, {} versions "
"running".format(cluster_data.service,
status.instance,
cluster_data.cluster,
status.marathon.app_count))
cluster_data.instances_queue.task_done()
instances_out.put(instance)
continue
if not cluster_data.git_sha.startswith(status.git_sha):
paasta_print(
" {}.{} on {} doesn't have the right sha yet: {}".format(
cluster_data.service, instance, cluster_data.cluster,
status.git_sha))
cluster_data.instances_queue.task_done()
instances_out.put(instance)
continue
if status.marathon.deploy_status != 'Running':
paasta_print(" {}.{} on {} isn't running yet: {}".format(
cluster_data.service, instance, cluster_data.cluster,
status.marathon.deploy_status))
cluster_data.instances_queue.task_done()
instances_out.put(instance)
continue
if (status.marathon.expected_instance_count !=
status.marathon.running_instance_count):
paasta_print(" {}.{} on {} isn't scaled up yet, "
"has {} out of {}".format(
cluster_data.service, instance,
cluster_data.cluster,
status.marathon.running_instance_count,
status.marathon.expected_instance_count))
cluster_data.instances_queue.task_done()
instances_out.put(instance)
continue
paasta_print("Complete: {}.{} on {} looks 100% deployed at {} "
"instances on {}".format(
cluster_data.service, instance,
cluster_data.cluster,
status.marathon.running_instance_count,
status.git_sha))
cluster_data.instances_queue.task_done()
def run_capacity_check():
options = parse_capacity_check_options()
system_paasta_config = load_system_paasta_config()
cluster = (
options.cluster
if options.cluster is not None
else system_paasta_config.get_cluster()
)
value_to_check = options.type
client = get_paasta_api_client(cluster=cluster)
if client is None:
print("UNKNOWN Failed to load paasta api client")
sys.exit(3)
overrides = read_overrides(options.overrides)
attributes = options.attributes.split(",")
try:
resource_use = client.resources.resources(groupings=attributes).result()
except HTTPError as e:
print("UNKNOWN received exception from paasta api:\n\t%s" % e)
sys.exit(3)
default_check = {
"warn": {"cpus": options.warn, "mem": options.warn, "disk": options.warn},
"crit": {"cpus": options.crit, "mem": options.crit, "disk": options.crit},
}
failures = defaultdict(list)
for usage_value in resource_use:
check = get_check_from_overrides(
overrides, default_check, usage_value["groupings"]
)
usage_percent = calc_percent_usage(usage_value, value_to_check)
for c in ["crit", "warn"]:
if usage_percent > check[c][value_to_check]:
failures[c].append(
{
"attrs": [
{"attr": a, "value": v}
for a, v in usage_value["groupings"].items()
],
"maximum": check[c][value_to_check],
"current": usage_percent,
}
)
break
return_value = [0]
if len(failures["crit"]) > 0:
result = error_message(failures["crit"], "CRITICAL", cluster, value_to_check)
print(result)
return_value.append(2)
if len(failures["warn"]) > 0:
result = error_message(failures["warn"], "WARNING", cluster, value_to_check)
print(result)
return_value.append(1)
if max(return_value) == 0:
print(f"OK cluster {cluster} is below critical capacity in {value_to_check}")
sys.exit(max(return_value))
