def marathon_deployments_check(service):
"""Checks for consistency between deploy.yaml and the marathon yamls"""
the_return = True
pipeline_deployments = get_pipeline_config(service)
pipeline_steps = [step['instancename'] for step in pipeline_deployments]
pipeline_steps = [step for step in pipeline_steps if step not in DEPLOY_PIPELINE_NON_DEPLOY_STEPS]
marathon_steps = get_marathon_steps(service)
in_marathon_not_deploy = set(marathon_steps) - set(pipeline_steps)
if len(in_marathon_not_deploy) > 0:
print "%s There are some instance(s) you have asked to run in marathon that" % x_mark()
print " do not have a corresponding entry in deploy.yaml:"
print " %s" % PaastaColors.bold(", ".join(in_marathon_not_deploy))
print " You should probably add entries to deploy.yaml for them so they"
print " are deployed to those clusters."
the_return = False
in_deploy_not_marathon = set(pipeline_steps) - set(marathon_steps)
if len(in_deploy_not_marathon) > 0:
print "%s There are some instance(s) in deploy.yaml that are not referenced" % x_mark()
print " by any marathon instance:"
print " %s" % PaastaColors.bold((", ".join(in_deploy_not_marathon)))
print " You should probably delete these deploy.yaml entries if they are unused."
the_return = False
if the_return is True:
print success("All entries in deploy.yaml correspond to a marathon entry")
print success("All marathon instances have a corresponding deploy.yaml entry")
return the_return
def deployments_check(service, soa_dir):
"""Checks for consistency between deploy.yaml and the marathon/chronos yamls"""
the_return = True
pipeline_steps = [
step['step'] for step in get_pipeline_config(service, soa_dir)
]
pipeline_deploy_groups = [
step for step in pipeline_steps if is_deploy_step(step)
]
framework_deploy_groups = {}
in_deploy_not_frameworks = set(pipeline_deploy_groups)
for it in INSTANCE_TYPES:
framework_deploy_groups[it] = get_deploy_groups_used_by_framework(
it, service, soa_dir)
in_framework_not_deploy = set(
framework_deploy_groups[it]) - set(pipeline_deploy_groups)
in_deploy_not_frameworks -= set(framework_deploy_groups[it])
if len(in_framework_not_deploy) > 0:
paasta_print(
"%s There are some instance(s) you have asked to run in %s that"
% (x_mark(), it))
paasta_print(" do not have a corresponding entry in deploy.yaml:")
paasta_print(" %s" %
PaastaColors.bold(", ".join(in_framework_not_deploy)))
paasta_print(
" You should probably configure these to use a 'deploy_group' or"
)
paasta_print(
" add entries to deploy.yaml for them so they are deployed to those clusters."
)
the_return = False
if len(in_deploy_not_frameworks) > 0:
paasta_print(
"%s There are some instance(s) in deploy.yaml that are not referenced"
% x_mark())
paasta_print(" by any marathon, chronos or adhoc instance:")
paasta_print(" %s" % PaastaColors.bold(
(", ".join(in_deploy_not_frameworks))))
paasta_print(
" You should probably delete these deploy.yaml entries if they are unused."
)
the_return = False
if the_return is True:
paasta_print(
success(
"All entries in deploy.yaml correspond to a marathon, chronos or adhoc entry"
))
for it in INSTANCE_TYPES:
if len(framework_deploy_groups[it]) > 0:
paasta_print(
success(
"All %s instances have a corresponding deploy.yaml entry"
% it))
return the_return
def validate_paasta_objects(service_path):
soa_dir, service = path_to_soa_dir_service(service_path)
returncode = True
messages = []
for cluster in list_clusters(service, soa_dir):
for instance in list_all_instances_for_service(service=service,
clusters=[cluster],
soa_dir=soa_dir):
instance_config = get_instance_config(
service=service,
instance=instance,
cluster=cluster,
load_deployments=False,
soa_dir=soa_dir,
)
messages.extend(instance_config.validate())
returncode = len(messages) == 0
if messages:
errors = "\n".join(messages)
paasta_print(
failure((f"There were failures validating {service}: {errors}"),
""))
else:
paasta_print(
success(f"All PaaSTA Instances for are valid for all clusters"))
return returncode
def validate_secrets(service_path):
soa_dir, service = path_to_soa_dir_service(service_path)
system_paasta_config = load_system_paasta_config()
vault_cluster_map = system_paasta_config.get_vault_cluster_config()
return_value = True
for cluster in list_clusters(service, soa_dir):
vault_env = vault_cluster_map.get(cluster)
if not vault_env:
print(failure(f"{cluster} not found on vault_cluster_map", ""))
return_value = False
continue
for instance in list_all_instances_for_service(service=service,
clusters=[cluster],
soa_dir=soa_dir):
instance_config = get_instance_config(
service=service,
instance=instance,
cluster=cluster,
load_deployments=False,
soa_dir=soa_dir,
)
if not check_secrets_for_instance(instance_config.config_dict,
soa_dir, service_path,
vault_env):
return_value = False
if return_value:
print(success("No orphan secrets found"))
return return_value
def deployments_check(service, soa_dir):
"""Checks for consistency between deploy.yaml and the marathon/chronos yamls"""
the_return = True
pipeline_deployments = get_pipeline_config(service, soa_dir)
pipeline_steps = [step["step"] for step in pipeline_deployments]
pipeline_steps = [step for step in pipeline_steps if is_deploy_step(step)]
marathon_steps = get_marathon_steps(service, soa_dir)
chronos_steps = get_chronos_steps(service, soa_dir)
in_marathon_not_deploy = set(marathon_steps) - set(pipeline_steps)
in_chronos_not_deploy = set(chronos_steps) - set(pipeline_steps)
if len(in_marathon_not_deploy) > 0:
paasta_print("%s There are some instance(s) you have asked to run in marathon that" % x_mark())
paasta_print(" do not have a corresponding entry in deploy.yaml:")
paasta_print(" %s" % PaastaColors.bold(", ".join(in_marathon_not_deploy)))
paasta_print(" You should probably configure these to use a 'deploy_group' or")
paasta_print(" add entries to deploy.yaml for them so they are deployed to those clusters.")
the_return = False
if len(in_chronos_not_deploy) > 0:
paasta_print("%s There are some instance(s) you have asked to run in chronos that" % x_mark())
paasta_print(" do not have a corresponding entry in deploy.yaml:")
paasta_print(" %s" % PaastaColors.bold(", ".join(in_chronos_not_deploy)))
paasta_print(" You should probably configure these to use a 'deploy_group' or")
paasta_print(" add entries to deploy.yaml for them so they are deployed to those clusters.")
the_return = False
in_deploy_not_marathon_chronos = set(pipeline_steps) - set(marathon_steps) - set(chronos_steps)
if len(in_deploy_not_marathon_chronos) > 0:
paasta_print("%s There are some instance(s) in deploy.yaml that are not referenced" % x_mark())
paasta_print(" by any marathon or chronos instance:")
paasta_print(" %s" % PaastaColors.bold((", ".join(in_deploy_not_marathon_chronos))))
paasta_print(" You should probably delete these deploy.yaml entries if they are unused.")
the_return = False
if the_return is True:
paasta_print(success("All entries in deploy.yaml correspond to a marathon or chronos entry"))
if len(marathon_steps) > 0:
paasta_print(success("All marathon instances have a corresponding deploy.yaml entry"))
if len(chronos_steps) > 0:
paasta_print(success("All chronos instances have a corresponding deploy.yaml entry"))
return the_return
def deployments_check(service, soa_dir):
"""Checks for consistency between deploy.yaml and the marathon/chronos yamls"""
the_return = True
pipeline_deployments = get_pipeline_config(service, soa_dir)
pipeline_steps = [step['instancename'] for step in pipeline_deployments]
pipeline_steps = [
step for step in pipeline_steps
if step not in DEPLOY_PIPELINE_NON_DEPLOY_STEPS
]
marathon_steps = get_marathon_steps(service, soa_dir)
chronos_steps = get_chronos_steps(service, soa_dir)
in_marathon_not_deploy = set(marathon_steps) - set(pipeline_steps)
in_chronos_not_deploy = set(chronos_steps) - set(pipeline_steps)
if len(in_marathon_not_deploy) > 0:
print "%s There are some instance(s) you have asked to run in marathon that" % x_mark(
)
print " do not have a corresponding entry in deploy.yaml:"
print " %s" % PaastaColors.bold(", ".join(in_marathon_not_deploy))
print " You should probably add entries to deploy.yaml for them so they"
print " are deployed to those clusters."
the_return = False
if len(in_chronos_not_deploy) > 0:
print "%s There are some instance(s) you have asked to run in chronos that" % x_mark(
)
print " do not have a corresponding entry in deploy.yaml:"
print " %s" % PaastaColors.bold(", ".join(in_marathon_not_deploy))
print " You should probably add entries to deploy.yaml for them so they"
print " are deployed to those clusters."
the_return = False
in_deploy_not_marathon_chronos = set(pipeline_steps) - set(
marathon_steps) - set(chronos_steps)
if len(in_deploy_not_marathon_chronos) > 0:
print "%s There are some instance(s) in deploy.yaml that are not referenced" % x_mark(
)
print " by any marathon or chronos instance:"
print " %s" % PaastaColors.bold(
(", ".join(in_deploy_not_marathon_chronos)))
print " You should probably delete these deploy.yaml entries if they are unused."
the_return = False
if the_return is True:
print success(
"All entries in deploy.yaml correspond to a marathon or chronos entry"
)
if len(marathon_steps) > 0:
print success(
"All marathon instances have a corresponding deploy.yaml entry"
)
if len(chronos_steps) > 0:
print success(
"All chronos instances have a corresponding deploy.yaml entry")
return the_return
def valid_chronos_instance(cluster, instance):
return success(f'chronos-{cluster}.yaml has a valid instance: {instance}.')
import paasta_tools.chronos_tools
from paasta_tools.chronos_tools import check_parent_format
from paasta_tools.chronos_tools import load_chronos_job_config
from paasta_tools.chronos_tools import TMP_JOB_IDENTIFIER
from paasta_tools.cli.utils import failure
from paasta_tools.cli.utils import get_file_contents
from paasta_tools.cli.utils import lazy_choices_completer
from paasta_tools.cli.utils import list_services
from paasta_tools.cli.utils import PaastaColors
from paasta_tools.cli.utils import success
from paasta_tools.utils import get_services_for_cluster
from paasta_tools.utils import list_all_instances_for_service
from paasta_tools.utils import list_clusters
from paasta_tools.utils import paasta_print
SCHEMA_VALID = success("Successfully validated schema")
SCHEMA_INVALID = failure(
"Failed to validate schema. More info:",
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html",
)
SCHEMA_NOT_FOUND = failure(
"Failed to find schema to validate against. More info:",
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html",
)
FAILED_READING_FILE = failure(
"Failed to read file. More info:",
"http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html",
)
def valid_chronos_instance(cluster, instance):
return success('chronos-%s.yaml has a valid instance: %s.' %
(cluster, instance))
def valid_tron_namespace(cluster, filename):
return success(f'{filename} is valid.')
def no_duplicate_instance_names_message(service, cluster):
return success(
f"All {service}'s instance names in cluster {cluster} are unique")
def valid_chronos_instance(cluster, instance):
return success('chronos-%s.yaml has a valid instance: %s.' % (cluster, instance))
from jsonschema import Draft4Validator
from jsonschema import FormatChecker
from jsonschema import ValidationError
from paasta_tools.chronos_tools import load_chronos_job_config
from paasta_tools.cli.utils import failure
from paasta_tools.cli.utils import get_file_contents
from paasta_tools.cli.utils import lazy_choices_completer
from paasta_tools.cli.utils import list_services
from paasta_tools.cli.utils import PaastaColors
from paasta_tools.cli.utils import success
from paasta_tools.utils import list_all_instances_for_service
from paasta_tools.utils import list_clusters
SCHEMA_VALID = success("Successfully validated schema")
SCHEMA_INVALID = failure(
"Failed to validate schema. More info:",
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
SCHEMA_NOT_FOUND = failure(
"Failed to find schema to validate against. More info:",
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
FAILED_READING_FILE = failure(
"Failed to read file. More info:",
"http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
UNKNOWN_SERVICE = "Unable to determine service to validate.\n" \
"Please supply the %s name you wish to " \
def deployments_check(service, soa_dir):
"""Checks for consistency between deploy.yaml and the marathon/chronos yamls"""
the_return = True
pipeline_deployments = get_pipeline_config(service, soa_dir)
pipeline_steps = [step['step'] for step in pipeline_deployments]
pipeline_steps = [step for step in pipeline_steps if is_deploy_step(step)]
marathon_steps = get_marathon_steps(service, soa_dir)
chronos_steps = get_chronos_steps(service, soa_dir)
in_marathon_not_deploy = set(marathon_steps) - set(pipeline_steps)
in_chronos_not_deploy = set(chronos_steps) - set(pipeline_steps)
if len(in_marathon_not_deploy) > 0:
paasta_print(
"%s There are some instance(s) you have asked to run in marathon that"
% x_mark())
paasta_print(" do not have a corresponding entry in deploy.yaml:")
paasta_print(" %s" %
PaastaColors.bold(", ".join(in_marathon_not_deploy)))
paasta_print(
" You should probably configure these to use a 'deploy_group' or")
paasta_print(
" add entries to deploy.yaml for them so they are deployed to those clusters."
)
the_return = False
if len(in_chronos_not_deploy) > 0:
paasta_print(
"%s There are some instance(s) you have asked to run in chronos that"
% x_mark())
paasta_print(" do not have a corresponding entry in deploy.yaml:")
paasta_print(" %s" %
PaastaColors.bold(", ".join(in_chronos_not_deploy)))
paasta_print(
" You should probably configure these to use a 'deploy_group' or")
paasta_print(
" add entries to deploy.yaml for them so they are deployed to those clusters."
)
the_return = False
in_deploy_not_marathon_chronos = set(pipeline_steps) - set(
marathon_steps) - set(chronos_steps)
if len(in_deploy_not_marathon_chronos) > 0:
paasta_print(
"%s There are some instance(s) in deploy.yaml that are not referenced"
% x_mark())
paasta_print(" by any marathon or chronos instance:")
paasta_print(" %s" % PaastaColors.bold(
(", ".join(in_deploy_not_marathon_chronos))))
paasta_print(
" You should probably delete these deploy.yaml entries if they are unused."
)
the_return = False
if the_return is True:
paasta_print(
success(
"All entries in deploy.yaml correspond to a marathon or chronos entry"
))
if len(marathon_steps) > 0:
paasta_print(
success(
"All marathon instances have a corresponding deploy.yaml entry"
))
if len(chronos_steps) > 0:
paasta_print(
success(
"All chronos instances have a corresponding deploy.yaml entry"
))
return the_return
def valid_chronos_instances():
return success(f"chronos instances are valid.")