def check_secrets_for_instance(instance_config_dict, soa_dir, service_path, vault_env): return_value = True for env_value in instance_config_dict.get("env", {}).values(): if is_secret_ref(env_value): secret_name = get_secret_name_from_ref(env_value) if is_shared_secret(env_value): secret_file_name = f"{soa_dir}/_shared/secrets/{secret_name}.json" else: secret_file_name = f"{service_path}/secrets/{secret_name}.json" if os.path.isfile(secret_file_name): secret_json = get_config_file_dict(secret_file_name) if "ciphertext" not in secret_json["environments"].get( vault_env, {}): print( failure( f"Secret {secret_name} not defined for ecosystem {vault_env} on secret file {secret_file_name}", "", )) return_value = False else: print( failure(f"Secret file {secret_file_name} not defined", "")) return_value = False return return_value
def check_service_path(service_path): """Check that the specified path exists and has yaml files :param service_path: Path to directory that should contain yaml files """ if not service_path or not os.path.isdir(service_path): print failure("%s is not a directory" % service_path, "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html") return False if not glob(os.path.join(service_path, "*.yaml")): print failure("%s does not contain any .yaml files" % service_path, "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html") return False return True
def check_service_path(service_path): """Check that the specified path exists and has yaml files :param service_path: Path to directory that should contain yaml files """ if not service_path or not os.path.isdir(service_path): print failure("%s is not a directory" % service_path, "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html") return False if not glob(os.path.join(service_path, "*.yaml")): print failure("%s does not contain any .yaml files" % service_path, "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html") return False return True
def validate_secrets(service_path): soa_dir, service = path_to_soa_dir_service(service_path) system_paasta_config = load_system_paasta_config() vault_cluster_map = system_paasta_config.get_vault_cluster_config() return_value = True for cluster in list_clusters(service, soa_dir): vault_env = vault_cluster_map.get(cluster) if not vault_env: print(failure(f"{cluster} not found on vault_cluster_map", "")) return_value = False continue for instance in list_all_instances_for_service(service=service, clusters=[cluster], soa_dir=soa_dir): instance_config = get_instance_config( service=service, instance=instance, cluster=cluster, load_deployments=False, soa_dir=soa_dir, ) if not check_secrets_for_instance(instance_config.config_dict, soa_dir, service_path, vault_env): return_value = False if return_value: print(success("No orphan secrets found")) return return_value
def validate_min_max_instances(service_path): soa_dir, service = path_to_soa_dir_service(service_path) returncode = True for cluster in list_clusters(service, soa_dir): for instance in list_all_instances_for_service(service=service, clusters=[cluster], soa_dir=soa_dir): instance_config = get_instance_config( service=service, instance=instance, cluster=cluster, load_deployments=False, soa_dir=soa_dir, ) if instance_config.get_instance_type() != "tron": min_instances = instance_config.get_min_instances() max_instances = instance_config.get_max_instances() if min_instances is not None and max_instances is not None: if max_instances < min_instances: returncode = False print( failure( f"Instance {instance} on cluster {cluster} has a greater number of min_instances than max_instances." + f"The number of min_instances ({min_instances}) cannot be greater than the max_instances ({max_instances}).", "", )) return returncode
def validate_paasta_objects(service_path): soa_dir, service = path_to_soa_dir_service(service_path) returncode = True messages = [] for cluster in list_clusters(service, soa_dir): for instance in list_all_instances_for_service(service=service, clusters=[cluster], soa_dir=soa_dir): instance_config = get_instance_config( service=service, instance=instance, cluster=cluster, load_deployments=False, soa_dir=soa_dir, ) messages.extend(instance_config.validate()) returncode = len(messages) == 0 if messages: errors = "\n".join(messages) paasta_print( failure((f"There were failures validating {service}: {errors}"), "")) else: paasta_print( success(f"All PaaSTA Instances for are valid for all clusters")) return returncode
def duplicate_instance_names_message(service, cluster, instance_names): instance_name_list = "\n\t".join(instance_names) message = ( f"Service {service} uses the following duplicate instance names for " f"cluster {cluster}:\n\t{instance_name_list}\n") return failure( message, "https://paasta.readthedocs.io/en/latest/yelpsoa_configs.html")
def validate_service_name(service): if len(sanitise_kubernetes_name(service)) > 63: paasta_print( failure( f"Length of service name {service} should be no more than 63." + " Note _ is replaced with - due to Kubernetes restriction", "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html", )) return False return True
def get_service_path(service, soa_dir): """Determine the path of the directory containing the conf files :param args: argparse.Namespace obj created from sys.args by cli """ if service: service_path = os.path.join(soa_dir, service) else: if soa_dir == os.getcwd(): service_path = os.getcwd() else: print UNKNOWN_SERVICE return None if not os.path.isdir(service_path): print failure("%s is not a directory" % service_path, "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html") return None if not glob(os.path.join(service_path, "*.yaml")): print failure("%s does not contain any .yaml files" % service_path, "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html") return None return service_path
def get_service_path(service, soa_dir): """Determine the path of the directory containing the conf files :param args: argparse.Namespace obj created from sys.args by cli """ if service: service_path = os.path.join(soa_dir, service) else: if soa_dir == os.getcwd(): service_path = os.getcwd() else: print UNKNOWN_SERVICE return None if not os.path.isdir(service_path): print failure( "%s is not a directory" % service_path, "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html") return None if not glob(os.path.join(service_path, "*.yaml")): print failure( "%s does not contain any .yaml files" % service_path, "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html") return None return service_path
def validate_instance_names(config_file_object, file_path): errors = [] for instance_name in config_file_object: if (not instance_name.startswith("_") and len(sanitise_kubernetes_name(instance_name)) > 63): errors.append(instance_name) if errors: error_string = "\n".join(errors) paasta_print( failure( f"Length of instance name \n{error_string}\n should be no more than 63." + " Note _ is replaced with -- due to Kubernetes restriction", "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html", )) return len(errors) == 0
def invalid_tron_namespace(cluster, output, filename): return failure( '%s is invalid:\n %s\n ' 'More info:' % (filename, output), "http://tron.readthedocs.io/en/latest/jobs.html", )
def invalid_chronos_instance(cluster, instance, output): return failure( 'chronos-%s.yaml has an invalid instance: %s.\n %s\n ' 'More info:' % (cluster, instance, output), "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html#chronos-clustername-yaml")
from paasta_tools.chronos_tools import load_chronos_job_config from paasta_tools.cli.utils import failure from paasta_tools.cli.utils import get_file_contents from paasta_tools.cli.utils import lazy_choices_completer from paasta_tools.cli.utils import list_services from paasta_tools.cli.utils import PaastaColors from paasta_tools.cli.utils import success from paasta_tools.utils import list_all_instances_for_service from paasta_tools.utils import list_clusters SCHEMA_VALID = success("Successfully validated schema") SCHEMA_INVALID = failure( "Failed to validate schema. More info:", "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html") SCHEMA_NOT_FOUND = failure( "Failed to find schema to validate against. More info:", "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html") FAILED_READING_FILE = failure( "Failed to read file. More info:", "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html") UNKNOWN_SERVICE = "Unable to determine service to validate.\n" \ "Please supply the %s name you wish to " \ "validate with the %s option." \ % (PaastaColors.cyan('SERVICE'), PaastaColors.cyan('-s'))
from paasta_tools.chronos_tools import TMP_JOB_IDENTIFIER from paasta_tools.cli.utils import failure from paasta_tools.cli.utils import get_file_contents from paasta_tools.cli.utils import lazy_choices_completer from paasta_tools.cli.utils import list_services from paasta_tools.cli.utils import PaastaColors from paasta_tools.cli.utils import success from paasta_tools.utils import get_services_for_cluster from paasta_tools.utils import list_all_instances_for_service from paasta_tools.utils import list_clusters from paasta_tools.utils import paasta_print SCHEMA_VALID = success("Successfully validated schema") SCHEMA_INVALID = failure( "Failed to validate schema. More info:", "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html", ) SCHEMA_NOT_FOUND = failure( "Failed to find schema to validate against. More info:", "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html", ) FAILED_READING_FILE = failure( "Failed to read file. More info:", "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html", ) UNKNOWN_SERVICE = "Unable to determine service to validate.\n" \ "Please supply the %s name you wish to " \ "validate with the %s option." \
def invalid_chronos_instance(cluster, instance, output): return failure( 'chronos-%s.yaml has an invalid instance: %s.\n %s\n ' 'More info:' % (cluster, instance, output), "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html#chronos-clustername-yaml", )
def invalid_chronos_instances(output): return failure( f"{output}" "More info:", "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html#chronos-clustername-yaml", )