def paasta_metastatus_on_api_endpoint(
cluster: str,
system_paasta_config: SystemPaastaConfig,
groupings: Sequence[str],
verbose: int,
autoscaling_info: bool = False,
use_mesos_cache: bool = False,
) -> Tuple[int, str]:
client = get_paasta_oapi_client(cluster, system_paasta_config)
if not client:
print("Cannot get a paasta-api client")
exit(1)
try:
cmd_args, _ = get_paasta_metastatus_cmd_args(
groupings=groupings,
verbose=verbose,
autoscaling_info=autoscaling_info,
use_mesos_cache=use_mesos_cache,
)
res = client.default.metastatus(
cmd_args=[str(arg) for arg in cmd_args])
output, exit_code = res.output, res.exit_code
except paastaapi.ApiException as exc:
output, exit_code = exc.body, exc.status
return exit_code, output
def test_get_paasta_oapi_client(system_paasta_config):
with mock.patch("paasta_tools.api.client.load_system_paasta_config",
autospec=True) as mock_load_system_paasta_config:
mock_load_system_paasta_config.return_value = system_paasta_config
client = get_paasta_oapi_client()
assert client
def get_status_for_instance(cluster, service, instance):
api = client.get_paasta_oapi_client(cluster=cluster)
if not api:
sys.exit(1)
status = api.service.status_instance(service=service, instance=instance)
if not status.marathon:
log.error("Not a marathon service, exiting")
sys.exit(1)
return status
def paasta_mesh_status_on_api_endpoint(
cluster: str,
service: str,
instance: str,
system_paasta_config: SystemPaastaConfig,
) -> Tuple[int, List[str]]:
client = get_paasta_oapi_client(cluster, system_paasta_config)
if not client:
print("ERROR: Cannot get a paasta-api client")
exit(1)
try:
mesh_status = client.service.mesh_instance(
service=service,
instance=instance,
include_smartstack=False,
)
except client.api_error as exc:
# 405 (method not allowed) is returned for instances that are not configured
# for the mesh, or for which getting mesh status is not supported
return (
exc.status,
[PaastaColors.red(exc.body if exc.status == 405 else exc.reason)],
)
except (client.connection_error, client.timeout_error) as exc:
return (
1,
[
PaastaColors.red(
f"Could not connect to API: {exc.__class__.__name__}")
],
)
except Exception as e:
output = [PaastaColors.red(f"Exception when talking to the API:")]
output.extend(str(e).split("\n"))
return 1, output
output = []
if mesh_status.smartstack is not None:
smartstack_status_human = get_smartstack_status_human(
mesh_status.smartstack.registration,
mesh_status.smartstack.expected_backends_per_location,
mesh_status.smartstack.locations,
)
output.extend(smartstack_status_human)
if mesh_status.envoy is not None:
envoy_status_human = get_envoy_status_human(
mesh_status.envoy.registration,
mesh_status.envoy.expected_backends_per_location,
mesh_status.envoy.locations,
)
output.extend(envoy_status_human)
return 0, output
def paasta_autoscale(args):
log.setLevel(logging.DEBUG)
service = figure_out_service_name(args)
api = client.get_paasta_oapi_client(cluster=args.cluster, http_res=True)
if not api:
print(
"Could not connect to paasta api. Maybe you misspelled the cluster?"
)
return 1
try:
if args.set is None:
log.debug("Getting the current autoscaler count...")
res, status, _ = api.autoscaler.get_autoscaler_count(
service=service,
instance=args.instance,
_return_http_data_only=False)
else:
log.debug(f"Setting desired instances to {args.set}.")
msg = paastamodels.AutoscalerCountMsg(
desired_instances=int(args.set))
res, status, _ = api.autoscaler.update_autoscaler_count(
service=service,
instance=args.instance,
autoscaler_count_msg=msg,
_return_http_data_only=False,
)
_log_audit(
action="manual-scale",
action_details=str(msg),
service=service,
instance=args.instance,
cluster=args.cluster,
)
except api.api_error as exc:
status = exc.status
if not 200 <= status <= 299:
print(
PaastaColors.red(
f"ERROR: '{args.instance}' is not configured to autoscale, "
f"so paasta autoscale could not scale it up on demand. "
f"If you want to be able to boost this service, please configure autoscaling for the service "
f"in its config file by setting min and max instances. Example: \n"
f"{args.instance}:\n"
f" min_instances: 5\n"
f" max_instances: 50"))
return 0
log.debug(f"Res: {res} Http: {status}")
print(res.desired_instances)
return 0
def delete_service_autoscale_pause_time(cluster):
api = client.get_paasta_oapi_client(cluster=cluster, http_res=True)
if not api:
print(
"Could not connect to paasta api. Maybe you misspelled the cluster?"
)
return 1
res, status, _ = api.default.delete_service_autoscaler_pause(
_return_http_data_only=False)
if status == 500:
print("Could not connect to zookeeper server")
return 2
print("Service autoscaler is unpaused")
return 0
def update_service_autoscale_pause_time(cluster, mins):
api = client.get_paasta_oapi_client(cluster=cluster, http_res=True)
if not api:
print(
"Could not connect to paasta api. Maybe you misspelled the cluster?"
)
return 1
res, status, _ = api.default.update_service_autoscaler_pause(
paastamodels.InlineObject(minutes=int(mins)),
_return_http_data_only=False)
if status == 500:
print("Could not connect to zookeeper server")
return 2
print(f"Service autoscaler is paused for {mins}")
return 0
def get_service_autoscale_pause_time(cluster):
api = client.get_paasta_oapi_client(cluster=cluster, http_res=True)
if not api:
print(
"Could not connect to paasta api. Maybe you misspelled the cluster?"
)
return 1
pause_time, status, _ = api.default.get_service_autoscaler_pause(
_return_http_data_only=False)
if status == 500:
print("Could not connect to zookeeper server")
return 2
pause_time = float(pause_time)
if pause_time < time.time():
print("Service autoscaler is not paused")
else:
print_paused_message(pause_time)
return 0
def list_deploy_queue(args) -> int:
cluster = args.cluster
all_clusters = list_clusters(soa_dir=args.soa_dir)
if cluster not in all_clusters:
print(f"{cluster} does not appear to be a valid cluster. Run `paasta "
"list-clusters` to see available options.")
return 1
system_paasta_config = load_system_paasta_config()
client = get_paasta_oapi_client(cluster,
system_paasta_config,
http_res=True)
if not client:
print("Cannot get a paasta API client")
return 1
try:
deploy_queues = client.default.deploy_queue()
except client.api_error as exc:
print(PaastaColors.red(exc.reason))
return exc.status
except (client.connection_error, client.timeout_error) as exc:
print(
PaastaColors.red(
f"Could not connect to API: {exc.__class__.__name__}"))
return 1
except Exception as exc:
tb = sys.exc_info()[2]
print(PaastaColors.red(f"Exception when talking to the API: {exc}"))
print("".join(traceback.format_tb(tb)))
return 1
if args.json:
json.dump(deploy_queues.to_dict(), sys.stdout)
else:
formatted_deploy_queues = format_deploy_queues(deploy_queues, cluster)
print(formatted_deploy_queues)
return 0
def run_capacity_check():
options = parse_capacity_check_options()
system_paasta_config = load_system_paasta_config()
cluster = (options.cluster if options.cluster is not None else
system_paasta_config.get_cluster())
value_to_check = options.type
client = get_paasta_oapi_client(cluster=cluster)
if client is None:
print("UNKNOWN Failed to load paasta api client")
sys.exit(3)
overrides = read_overrides(options.overrides)
attributes = options.attributes.split(",")
try:
resource_use = client.resources.resources(groupings=attributes)
except client.api_error as e:
print(f"UNKNOWN received exception from paasta api:\n\t%s{e}")
sys.exit(3)
default_check = {
"warn": {
"cpus": options.warn,
"mem": options.warn,
"disk": options.warn
},
"crit": {
"cpus": options.crit,
"mem": options.crit,
"disk": options.crit
},
}
failures = defaultdict(list)
for usage_value in resource_use.value:
check = get_check_from_overrides(overrides, default_check,
usage_value["groupings"])
usage_percent = calc_percent_usage(usage_value, value_to_check)
for c in ["crit", "warn"]:
if usage_percent > check[c][value_to_check]:
failures[c].append({
"attrs": [{
"attr": a,
"value": v
} for a, v in usage_value["groupings"].items()],
"maximum":
check[c][value_to_check],
"current":
usage_percent,
})
break
return_value = [0]
if len(failures["crit"]) > 0:
result = error_message(failures["crit"], "CRITICAL", cluster,
value_to_check)
print(result)
return_value.append(2)
if len(failures["warn"]) > 0:
result = error_message(failures["warn"], "WARNING", cluster,
value_to_check)
print(result)
return_value.append(1)
if max(return_value) == 0:
print(
f"OK cluster {cluster} is below critical capacity in {value_to_check}"
)
sys.exit(max(return_value))
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
soa_dir = args.soa_dir
pargs = apply_args_filters(args)
if len(pargs) == 0:
return 1
affected_services = {
s
for service_list in pargs.values() for s in service_list.keys()
}
if len(affected_services) > 1:
print(
PaastaColors.red(
"Warning: trying to start/stop/restart multiple services:"))
for cluster, services_instances in pargs.items():
print("Cluster %s:" % cluster)
for service, instances in services_instances.items():
print(" Service %s:" % service)
print(" Instances %s" % ",".join(instances.keys()))
if sys.stdin.isatty():
confirm = choice.Binary("Are you sure you want to continue?",
False).ask()
else:
confirm = False
if not confirm:
print()
print("exiting")
return 1
if not all([
can_user_deploy_service(get_deploy_info(service, soa_dir), service)
for service in affected_services
]):
print(PaastaColors.red("Exiting due to missing deploy permissions"))
return 1
invalid_deploy_groups = []
marathon_message_printed = False
affected_flinks = []
if args.clusters is None or args.instances is None:
if confirm_to_continue(pargs.items(), desired_state) is False:
print()
print("exiting")
return 1
for cluster, services_instances in pargs.items():
for service, instances in services_instances.items():
for instance in instances.keys():
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
if isinstance(service_config, FlinkDeploymentConfig):
affected_flinks.append(service_config)
continue
try:
remote_refs = get_remote_refs(service, soa_dir)
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment.") % str(e)
print(msg)
return 1
deploy_group = service_config.get_deploy_group()
(deploy_tag, _,
_) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(
datetime.datetime.utcnow())
if (isinstance(service_config, MarathonServiceConfig)
and not marathon_message_printed):
print_marathon_message(desired_state)
marathon_message_printed = True
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
# TODO: Refactor to discover if set_state is available for given
# instance_type in API
if affected_flinks:
print_flink_message(desired_state)
csi = defaultdict(lambda: defaultdict(list))
for service_config in affected_flinks:
csi[service_config.cluster][service_config.service].append(
service_config.instance)
system_paasta_config = load_system_paasta_config()
for cluster, services_instances in csi.items():
client = get_paasta_oapi_client(cluster, system_paasta_config)
if not client:
print("Cannot get a paasta-api client")
exit(1)
for service, instances in services_instances.items():
for instance in instances:
try:
client.service.instance_set_state(
service=service,
instance=instance,
desired_state=desired_state,
)
except client.api_error as exc:
print(exc.reason)
return exc.status
return_val = 0
if invalid_deploy_groups:
print(f"No deploy tags found for {', '.join(invalid_deploy_groups)}.")
print(f"Has {service} been deployed there yet?")
return_val = 1
return return_val