def check_secrets_for_instance(instance_config_dict, soa_dir, service_path,
vault_env):
return_value = True
for env_value in instance_config_dict.get("env", {}).values():
if is_secret_ref(env_value):
secret_name = get_secret_name_from_ref(env_value)
if is_shared_secret(env_value):
secret_file_name = f"{soa_dir}/_shared/secrets/{secret_name}.json"
else:
secret_file_name = f"{service_path}/secrets/{secret_name}.json"
if os.path.isfile(secret_file_name):
secret_json = get_config_file_dict(secret_file_name)
if "ciphertext" not in secret_json["environments"].get(
vault_env, {}):
print(
failure(
f"Secret {secret_name} not defined for ecosystem {vault_env} on secret file {secret_file_name}",
"",
))
return_value = False
else:
print(
failure(f"Secret file {secret_file_name} not defined", ""))
return_value = False
return return_value
def check_service_path(service_path):
"""Check that the specified path exists and has yaml files
:param service_path: Path to directory that should contain yaml files
"""
if not service_path or not os.path.isdir(service_path):
print failure("%s is not a directory" % service_path,
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
return False
if not glob(os.path.join(service_path, "*.yaml")):
print failure("%s does not contain any .yaml files" % service_path,
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
return False
return True
def check_service_path(service_path):
"""Check that the specified path exists and has yaml files
:param service_path: Path to directory that should contain yaml files
"""
if not service_path or not os.path.isdir(service_path):
print failure("%s is not a directory" % service_path,
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html")
return False
if not glob(os.path.join(service_path, "*.yaml")):
print failure("%s does not contain any .yaml files" % service_path,
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html")
return False
return True
def validate_secrets(service_path):
soa_dir, service = path_to_soa_dir_service(service_path)
system_paasta_config = load_system_paasta_config()
vault_cluster_map = system_paasta_config.get_vault_cluster_config()
return_value = True
for cluster in list_clusters(service, soa_dir):
vault_env = vault_cluster_map.get(cluster)
if not vault_env:
print(failure(f"{cluster} not found on vault_cluster_map", ""))
return_value = False
continue
for instance in list_all_instances_for_service(service=service,
clusters=[cluster],
soa_dir=soa_dir):
instance_config = get_instance_config(
service=service,
instance=instance,
cluster=cluster,
load_deployments=False,
soa_dir=soa_dir,
)
if not check_secrets_for_instance(instance_config.config_dict,
soa_dir, service_path,
vault_env):
return_value = False
if return_value:
print(success("No orphan secrets found"))
return return_value
def validate_min_max_instances(service_path):
soa_dir, service = path_to_soa_dir_service(service_path)
returncode = True
for cluster in list_clusters(service, soa_dir):
for instance in list_all_instances_for_service(service=service,
clusters=[cluster],
soa_dir=soa_dir):
instance_config = get_instance_config(
service=service,
instance=instance,
cluster=cluster,
load_deployments=False,
soa_dir=soa_dir,
)
if instance_config.get_instance_type() != "tron":
min_instances = instance_config.get_min_instances()
max_instances = instance_config.get_max_instances()
if min_instances is not None and max_instances is not None:
if max_instances < min_instances:
returncode = False
print(
failure(
f"Instance {instance} on cluster {cluster} has a greater number of min_instances than max_instances."
+
f"The number of min_instances ({min_instances}) cannot be greater than the max_instances ({max_instances}).",
"",
))
return returncode
def validate_paasta_objects(service_path):
soa_dir, service = path_to_soa_dir_service(service_path)
returncode = True
messages = []
for cluster in list_clusters(service, soa_dir):
for instance in list_all_instances_for_service(service=service,
clusters=[cluster],
soa_dir=soa_dir):
instance_config = get_instance_config(
service=service,
instance=instance,
cluster=cluster,
load_deployments=False,
soa_dir=soa_dir,
)
messages.extend(instance_config.validate())
returncode = len(messages) == 0
if messages:
errors = "\n".join(messages)
paasta_print(
failure((f"There were failures validating {service}: {errors}"),
""))
else:
paasta_print(
success(f"All PaaSTA Instances for are valid for all clusters"))
return returncode
def duplicate_instance_names_message(service, cluster, instance_names):
instance_name_list = "\n\t".join(instance_names)
message = (
f"Service {service} uses the following duplicate instance names for "
f"cluster {cluster}:\n\t{instance_name_list}\n")
return failure(
message,
"https://paasta.readthedocs.io/en/latest/yelpsoa_configs.html")
def validate_service_name(service):
if len(sanitise_kubernetes_name(service)) > 63:
paasta_print(
failure(
f"Length of service name {service} should be no more than 63."
+ " Note _ is replaced with - due to Kubernetes restriction",
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html",
))
return False
return True
def get_service_path(service, soa_dir):
"""Determine the path of the directory containing the conf files
:param args: argparse.Namespace obj created from sys.args by cli
"""
if service:
service_path = os.path.join(soa_dir, service)
else:
if soa_dir == os.getcwd():
service_path = os.getcwd()
else:
print UNKNOWN_SERVICE
return None
if not os.path.isdir(service_path):
print failure("%s is not a directory" % service_path,
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
return None
if not glob(os.path.join(service_path, "*.yaml")):
print failure("%s does not contain any .yaml files" % service_path,
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
return None
return service_path
def get_service_path(service, soa_dir):
"""Determine the path of the directory containing the conf files
:param args: argparse.Namespace obj created from sys.args by cli
"""
if service:
service_path = os.path.join(soa_dir, service)
else:
if soa_dir == os.getcwd():
service_path = os.getcwd()
else:
print UNKNOWN_SERVICE
return None
if not os.path.isdir(service_path):
print failure(
"%s is not a directory" % service_path,
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
return None
if not glob(os.path.join(service_path, "*.yaml")):
print failure(
"%s does not contain any .yaml files" % service_path,
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
return None
return service_path
def validate_instance_names(config_file_object, file_path):
errors = []
for instance_name in config_file_object:
if (not instance_name.startswith("_")
and len(sanitise_kubernetes_name(instance_name)) > 63):
errors.append(instance_name)
if errors:
error_string = "\n".join(errors)
paasta_print(
failure(
f"Length of instance name \n{error_string}\n should be no more than 63."
+ " Note _ is replaced with -- due to Kubernetes restriction",
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html",
))
return len(errors) == 0
def invalid_tron_namespace(cluster, output, filename):
return failure(
'%s is invalid:\n %s\n '
'More info:' % (filename, output),
"http://tron.readthedocs.io/en/latest/jobs.html",
)
def invalid_chronos_instance(cluster, instance, output):
return failure(
'chronos-%s.yaml has an invalid instance: %s.\n %s\n '
'More info:' % (cluster, instance, output),
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html#chronos-clustername-yaml")
from paasta_tools.chronos_tools import load_chronos_job_config
from paasta_tools.cli.utils import failure
from paasta_tools.cli.utils import get_file_contents
from paasta_tools.cli.utils import lazy_choices_completer
from paasta_tools.cli.utils import list_services
from paasta_tools.cli.utils import PaastaColors
from paasta_tools.cli.utils import success
from paasta_tools.utils import list_all_instances_for_service
from paasta_tools.utils import list_clusters
SCHEMA_VALID = success("Successfully validated schema")
SCHEMA_INVALID = failure(
"Failed to validate schema. More info:",
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
SCHEMA_NOT_FOUND = failure(
"Failed to find schema to validate against. More info:",
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
FAILED_READING_FILE = failure(
"Failed to read file. More info:",
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
UNKNOWN_SERVICE = "Unable to determine service to validate.\n" \
"Please supply the %s name you wish to " \
"validate with the %s option." \
% (PaastaColors.cyan('SERVICE'), PaastaColors.cyan('-s'))
from paasta_tools.chronos_tools import TMP_JOB_IDENTIFIER
from paasta_tools.cli.utils import failure
from paasta_tools.cli.utils import get_file_contents
from paasta_tools.cli.utils import lazy_choices_completer
from paasta_tools.cli.utils import list_services
from paasta_tools.cli.utils import PaastaColors
from paasta_tools.cli.utils import success
from paasta_tools.utils import get_services_for_cluster
from paasta_tools.utils import list_all_instances_for_service
from paasta_tools.utils import list_clusters
from paasta_tools.utils import paasta_print
SCHEMA_VALID = success("Successfully validated schema")
SCHEMA_INVALID = failure(
"Failed to validate schema. More info:",
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html",
)
SCHEMA_NOT_FOUND = failure(
"Failed to find schema to validate against. More info:",
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html",
)
FAILED_READING_FILE = failure(
"Failed to read file. More info:",
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html",
)
UNKNOWN_SERVICE = "Unable to determine service to validate.\n" \
"Please supply the %s name you wish to " \
"validate with the %s option." \
def invalid_chronos_instance(cluster, instance, output):
return failure(
'chronos-%s.yaml has an invalid instance: %s.\n %s\n '
'More info:' % (cluster, instance, output),
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html#chronos-clustername-yaml",
)
def invalid_chronos_instances(output):
return failure(
f"{output}" "More info:",
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html#chronos-clustername-yaml",
)