def list_instances(**kwargs):
"""Returns a sorted list of all possible instance names
for tab completion. We try to guess what service you might be
operating on, otherwise we just provide *all* of them
"""
all_instances: Set[str] = set()
service = guess_service_name()
try:
validate_service_name(service)
all_instances = set(list_all_instances_for_service(service))
except NoSuchService:
for service in list_services():
for instance in list_all_instances_for_service(service):
all_instances.add(instance)
return sorted(all_instances)
def list_instances():
"""Returns a sorted list of all possible instance names
for tab completion. We try to guess what service you might be
operating on, otherwise we just provide *all* of them
"""
all_instances = set()
service = guess_service_name()
try:
validate_service_name(service)
all_instances = set(list_all_instances_for_service(service))
except NoSuchService:
for service in list_services():
for instance in list_all_instances_for_service(service):
all_instances.add(instance)
return sorted(all_instances)
def validate_secrets(service_path):
soa_dir, service = path_to_soa_dir_service(service_path)
system_paasta_config = load_system_paasta_config()
vault_cluster_map = system_paasta_config.get_vault_cluster_config()
return_value = True
for cluster in list_clusters(service, soa_dir):
vault_env = vault_cluster_map.get(cluster)
if not vault_env:
print(failure(f"{cluster} not found on vault_cluster_map", ""))
return_value = False
continue
for instance in list_all_instances_for_service(service=service,
clusters=[cluster],
soa_dir=soa_dir):
instance_config = get_instance_config(
service=service,
instance=instance,
cluster=cluster,
load_deployments=False,
soa_dir=soa_dir,
)
if not check_secrets_for_instance(instance_config.config_dict,
soa_dir, service_path,
vault_env):
return_value = False
if return_value:
print(success("No orphan secrets found"))
return return_value
def bounce_service(self, service_name):
self.log.info("Checking if any instances for {} need bouncing".format(
service_name))
instances = list_all_instances_for_service(
service=service_name,
clusters=[self.filewatcher.cluster],
instance_type='marathon',
cache=False)
self.log.debug(instances)
service_instances = [(service_name, instance)
for instance in instances]
service_instances = get_service_instances_with_changed_id(
self.marathon_client, service_instances, self.filewatcher.cluster)
for service, instance in service_instances:
self.log.info(
"{}.{} has a new marathon app ID, and so needs bouncing".
format(service, instance))
service_instances = [
ServiceInstance(service=service,
instance=instance,
bounce_by=int(time.time()),
watcher=self.__class__.__name__,
bounce_timers=None)
for service, instance in service_instances
]
for service_instance in service_instances:
self.filewatcher.inbox_q.put(service_instance)
def validate_min_max_instances(service_path):
soa_dir, service = path_to_soa_dir_service(service_path)
returncode = True
for cluster in list_clusters(service, soa_dir):
for instance in list_all_instances_for_service(service=service,
clusters=[cluster],
soa_dir=soa_dir):
instance_config = get_instance_config(
service=service,
instance=instance,
cluster=cluster,
load_deployments=False,
soa_dir=soa_dir,
)
if instance_config.get_instance_type() != "tron":
min_instances = instance_config.get_min_instances()
max_instances = instance_config.get_max_instances()
if min_instances is not None and max_instances is not None:
if max_instances < min_instances:
returncode = False
print(
failure(
f"Instance {instance} on cluster {cluster} has a greater number of min_instances than max_instances."
+
f"The number of min_instances ({min_instances}) cannot be greater than the max_instances ({max_instances}).",
"",
))
return returncode
def validate_chronos(service_path):
soa_dir, service = path_to_soa_dir_service(service_path)
instance_type = 'chronos'
returncode = 0
for cluster in list_clusters(service, soa_dir, instance_type):
for instance in list_all_instances_for_service(
service=service,
clusters=[cluster],
instance_type=instance_type,
soa_dir=soa_dir):
cjc = load_chronos_job_config(service, instance, cluster, False,
soa_dir)
checks_passed, check_msgs = cjc.validate()
# Remove duplicate check_msgs
unique_check_msgs = list(set(check_msgs))
if not checks_passed:
print invalid_chronos_instance(cluster, instance,
"\n ".join(unique_check_msgs))
returncode = 1
else:
print valid_chronos_instance(cluster, instance)
return returncode
def bounce_service(self, service_name):
self.log.info(
f"Checking if any marathon instances of {service_name} need bouncing."
)
instances = list_all_instances_for_service(
service=service_name,
clusters=[self.filewatcher.cluster],
instance_type="marathon",
cache=False,
)
self.log.debug(instances)
service_instance_configs = get_service_instances_needing_update(
self.marathon_clients,
[(service_name, instance) for instance in instances],
self.filewatcher.cluster,
)
for service, instance, config in service_instance_configs:
self.log.info(
f"{service}.{instance} has a new marathon app ID. Enqueuing it to be bounced."
)
now = time.time()
self.filewatcher.instances_to_bounce.put(
ServiceInstance( # type: ignore
service=service,
instance=instance,
cluster=self.filewatcher.cluster,
bounce_by=now + config.get_bounce_start_deadline(),
wait_until=now,
watcher=type(self).__name__,
bounce_timers=None,
failures=0,
))
def guess_instance(service, cluster, args):
"""Returns instance from args if available, otherwise uses 'main' if it is a valid instance,
otherwise takes a good guess and returns the first instance available"""
if args.instance:
instance = args.instance
else:
try:
instances = list_all_instances_for_service(
service=service, clusters=[cluster], instance_type=None, soa_dir=args.yelpsoa_config_root
)
if "main" in instances:
instance = "main"
else:
instance = list(instances)[0]
except NoConfigurationForServiceError:
sys.stderr.write(
PaastaColors.red(
"Could not automatically detect instance to emulate. Please specify one with the --instance option.\n"
)
)
sys.exit(2)
sys.stderr.write(
PaastaColors.yellow(
"Guessing instance configuration for %s. To override, use the --instance option.\n" % instance
)
)
return instance
def process_default(self, event):
self.log.debug(event)
self.watch_new_folder(event)
event = self.filter_event(event)
if event:
self.log.info("Change of {} in {}".format(event.name, event.path))
service_name = event.path.split('/')[-1]
self.log.info("Checking if any instances for {} need bouncing".format(service_name))
instances = list_all_instances_for_service(service=service_name,
clusters=[self.filewatcher.cluster],
instance_type='marathon')
self.log.debug(instances)
service_instances = [(service_name, instance) for instance in instances]
service_instances = get_service_instances_with_changed_id(self.marathon_client,
service_instances,
self.filewatcher.cluster)
for service, instance in service_instances:
self.log.info("{}.{} has a new marathon app ID, and so needs bouncing".format(service, instance))
service_instances = [ServiceInstance(service=service,
instance=instance,
bounce_by=int(time.time()),
watcher=self.__class__.__name__,
bounce_timers=None)
for service, instance in service_instances]
for service_instance in service_instances:
self.filewatcher.inbox_q.put(service_instance)
def list_service_instances():
"""Returns a sorted list of service<SPACER>instance names"""
the_list = []
for service in list_services():
for instance in list_all_instances_for_service(service):
the_list.append(compose_job_id(service, instance))
return the_list
def list_service_instances():
"""Returns a sorted list of service<SPACER>instance names"""
the_list = []
for service in list_services():
for instance in list_all_instances_for_service(service):
the_list.append(compose_job_id(service, instance))
return the_list
def bounce_service(self, service_name):
self.log.info(f"Checking if any instances for {service_name} need bouncing")
instances = list_all_instances_for_service(
service=service_name,
clusters=[self.filewatcher.cluster],
instance_type='marathon',
cache=False,
)
self.log.debug(instances)
service_instances = [(service_name, instance) for instance in instances]
service_instances = get_service_instances_needing_update(
self.marathon_clients,
service_instances,
self.filewatcher.cluster,
)
for service, instance in service_instances:
self.log.info(f"{service}.{instance} has a new marathon app ID, and so needs bouncing")
service_instances_to_queue = [
# https://github.com/python/mypy/issues/2852
ServiceInstance( # type: ignore
service=service,
instance=instance,
cluster=self.filewatcher.cluster,
bounce_by=int(time.time()),
watcher=type(self).__name__,
bounce_timers=None,
failures=0,
)
for service, instance in service_instances
]
for service_instance in service_instances_to_queue:
self.filewatcher.inbox_q.put(service_instance)
def validate_paasta_objects(service_path):
soa_dir, service = path_to_soa_dir_service(service_path)
returncode = True
messages = []
for cluster in list_clusters(service, soa_dir):
for instance in list_all_instances_for_service(service=service,
clusters=[cluster],
soa_dir=soa_dir):
instance_config = get_instance_config(
service=service,
instance=instance,
cluster=cluster,
load_deployments=False,
soa_dir=soa_dir,
)
messages.extend(instance_config.validate())
returncode = len(messages) == 0
if messages:
errors = "\n".join(messages)
paasta_print(
failure((f"There were failures validating {service}: {errors}"),
""))
else:
paasta_print(
success(f"All PaaSTA Instances for are valid for all clusters"))
return returncode
def guess_instance(service, cluster, args):
"""Returns instance from args if available, otherwise uses 'main' if it is a valid instance,
otherwise takes a good guess and returns the first instance available"""
if args.instance:
instance = args.instance
else:
try:
instances = list_all_instances_for_service(
service=service,
clusters=[cluster],
instance_type=None,
soa_dir=args.yelpsoa_config_root)
if 'main' in instances:
instance = 'main'
else:
instance = list(instances)[0]
except NoConfigurationForServiceError:
sys.stderr.write(
PaastaColors.red(
'Could not automatically detect instance to emulate. Please specify one with the --instance option.\n'
))
sys.exit(2)
sys.stderr.write(
PaastaColors.yellow(
'Guessing instance configuration for %s. To override, use the --instance option.\n'
% instance))
return instance
def list_service_instances(soa_dir: str = DEFAULT_SOA_DIR):
"""Returns a sorted list of service<SPACER>instance names"""
the_list = []
for service in list_services(soa_dir):
for instance in list_all_instances_for_service(service=service,
soa_dir=soa_dir):
the_list.append(compose_job_id(service, instance))
return the_list
def list_paasta_services():
"""Returns a sorted list of services that happen to have at
least one service.instance (including Marathon and Chronos instances), which indicates it is on PaaSTA
"""
the_list = []
for service in list_services():
if list_all_instances_for_service(service):
the_list.append(service)
return the_list
def list_paasta_services(soa_dir: str = DEFAULT_SOA_DIR):
"""Returns a sorted list of services that happen to have at
least one service.instance, which indicates it is on PaaSTA
"""
the_list = []
for service in list_services(soa_dir):
if list_all_instances_for_service(service, soa_dir=soa_dir):
the_list.append(service)
return the_list
def list_paasta_services():
"""Returns a sorted list of services that happen to have at
least one service.instance (including Marathon and Chronos instances), which indicates it is on PaaSTA
"""
the_list = []
for service in list_services():
if list_all_instances_for_service(service):
the_list.append(service)
return the_list
def verify_instances(args_instances: str, service: str,
clusters: Sequence[str]) -> Sequence[str]:
"""Verify that a list of instances specified by user is correct for this service.
:param args_instances: a list of instances.
:param service: the service name
:param cluster: a list of clusters
:returns: a list of instances specified in args_instances without any exclusions.
"""
unverified_instances = args_instances.split(",")
service_instances: Set[str] = list_all_instances_for_service(
service, clusters=clusters)
misspelled_instances: Sequence[str] = [
i for i in unverified_instances if i not in service_instances
]
if len(misspelled_instances) == 0:
return misspelled_instances
# Check for instances with suffixes other than Tron instances (i.e. Flink instances)
instances_without_suffixes = [
x.split(".")[0] for x in unverified_instances
]
misspelled_instances = [
i for i in instances_without_suffixes if i not in service_instances
]
if misspelled_instances:
suggestions: List[str] = []
for instance in misspelled_instances:
matches = difflib.get_close_matches(instance,
service_instances,
n=5,
cutoff=0.5)
suggestions.extend(matches) # type: ignore
suggestions = list(set(suggestions))
if clusters:
message = "{} doesn't have any instances matching {} on {}.".format(
service,
", ".join(sorted(misspelled_instances)),
", ".join(sorted(clusters)),
)
else:
message = "{} doesn't have any instances matching {}.".format(
service, ", ".join(sorted(misspelled_instances)))
print(PaastaColors.red(message))
if suggestions:
print("Did you mean any of these?")
for instance in sorted(suggestions):
print(" %s" % instance)
return misspelled_instances
def validate_chronos(service_path):
"""Check that any chronos configurations are valid"""
soa_dir, service = path_to_soa_dir_service(service_path)
instance_type = 'chronos'
chronos_spacer = paasta_tools.chronos_tools.INTERNAL_SPACER
returncode = True
if service.startswith(TMP_JOB_IDENTIFIER):
paasta_print((
"Services using scheduled tasks cannot be named %s, as it clashes with the "
"identifier used for temporary jobs" % TMP_JOB_IDENTIFIER))
return False
for cluster in list_clusters(service, soa_dir, instance_type):
services_in_cluster = get_services_for_cluster(cluster=cluster,
instance_type='chronos',
soa_dir=soa_dir)
valid_services = {
f"{name}{chronos_spacer}{instance}"
for name, instance in services_in_cluster
}
for instance in list_all_instances_for_service(
service=service,
clusters=[cluster],
instance_type=instance_type,
soa_dir=soa_dir,
):
cjc = load_chronos_job_config(service, instance, cluster, False,
soa_dir)
parents = cjc.get_parents() or []
checks_passed, check_msgs = cjc.validate()
for parent in parents:
if not check_parent_format(parent):
continue
if f"{service}{chronos_spacer}{instance}" == parent:
checks_passed = False
check_msgs.append("Job %s cannot depend on itself" %
parent)
elif parent not in valid_services:
checks_passed = False
check_msgs.append("Parent job %s could not be found" %
parent)
# Remove duplicate check_msgs
unique_check_msgs = list(set(check_msgs))
if not checks_passed:
paasta_print(
invalid_chronos_instance(cluster, instance,
"\n ".join(unique_check_msgs)))
returncode = False
else:
paasta_print(valid_chronos_instance(cluster, instance))
return returncode
def test_list_all_instances_for_service():
service = "fake_service"
clusters = ["fake_cluster"]
mock_instances = [(service, "instance1"), (service, "instance2")]
expected = set(["instance1", "instance2"])
with contextlib.nested(
mock.patch("paasta_tools.utils.list_clusters", autospec=True),
mock.patch("paasta_tools.utils.get_service_instance_list", autospec=True),
) as (mock_list_clusters, mock_service_instance_list):
mock_list_clusters.return_value = clusters
mock_service_instance_list.return_value = mock_instances
actual = utils.list_all_instances_for_service(service)
assert actual == expected
mock_list_clusters.assert_called_once_with(service, soa_dir=mock.ANY)
mock_service_instance_list.assert_called_once_with(service, clusters[0], None, soa_dir=mock.ANY)
def verify_instances(
args_instances: str,
service: str,
clusters: Sequence[str],
) -> Sequence[str]:
"""Verify that a list of instances specified by user is correct for this service.
:param args_instances: a list of instances.
:param service: the service name
:param cluster: a list of clusters
:returns: a list of instances specified in args_instances without any exclusions.
"""
unverified_instances = args_instances.split(",")
service_instances: Set[str] = list_all_instances_for_service(
service, clusters=clusters)
misspelled_instances: Sequence[str] = [
i for i in unverified_instances if i not in service_instances
]
if misspelled_instances:
suggestions: List[str] = []
for instance in misspelled_instances:
suggestions.extend(
difflib.get_close_matches(instance,
service_instances,
n=5,
cutoff=0.5)) # type: ignore
suggestions = list(set(suggestions))
if clusters:
message = ("%s doesn't have any instances matching %s on %s." % (
service,
', '.join(sorted(misspelled_instances)),
', '.join(sorted(clusters)),
))
else:
message = ("%s doesn't have any instances matching %s." %
(service, ', '.join(sorted(misspelled_instances))))
paasta_print(PaastaColors.red(message))
if suggestions:
paasta_print("Did you mean any of these?")
for instance in sorted(suggestions):
paasta_print(" %s" % instance)
return unverified_instances
def validate_chronos(service_path):
"""Check that any chronos configurations are valid"""
soa_dir, service = path_to_soa_dir_service(service_path)
instance_type = 'chronos'
chronos_spacer = paasta_tools.chronos_tools.INTERNAL_SPACER
returncode = True
for cluster in list_clusters(service, soa_dir, instance_type):
services_in_cluster = get_services_for_cluster(cluster=cluster,
instance_type='chronos',
soa_dir=soa_dir)
valid_services = set([
"%s%s%s" % (name, chronos_spacer, instance)
for name, instance in services_in_cluster
])
for instance in list_all_instances_for_service(
service=service,
clusters=[cluster],
instance_type=instance_type,
soa_dir=soa_dir):
cjc = load_chronos_job_config(service, instance, cluster, False,
soa_dir)
parents = cjc.get_parents() or []
checks_passed, check_msgs = cjc.validate()
for parent in parents:
if not check_parent_format(parent):
continue
if "%s%s%s" % (service, chronos_spacer, instance) == parent:
checks_passed = False
check_msgs.append("Job %s cannot depend on itself" %
parent)
elif parent not in valid_services:
checks_passed = False
check_msgs.append("Parent job %s could not be found" %
parent)
# Remove duplicate check_msgs
unique_check_msgs = list(set(check_msgs))
if not checks_passed:
print invalid_chronos_instance(cluster, instance,
"\n ".join(unique_check_msgs))
returncode = False
else:
print valid_chronos_instance(cluster, instance)
return returncode
def test_list_all_instances_for_service():
service = 'fake_service'
clusters = ['fake_cluster']
mock_instances = [(service, 'instance1'), (service, 'instance2')]
expected = set(['instance1', 'instance2'])
with contextlib.nested(
mock.patch('paasta_tools.utils.list_clusters', autospec=True),
mock.patch('paasta_tools.utils.get_service_instance_list', autospec=True),
) as (
mock_list_clusters,
mock_service_instance_list,
):
mock_list_clusters.return_value = clusters
mock_service_instance_list.return_value = mock_instances
actual = utils.list_all_instances_for_service(service)
assert actual == expected
mock_list_clusters.assert_called_once_with(service, soa_dir=mock.ANY)
mock_service_instance_list.assert_called_once_with(service, clusters[0], None, soa_dir=mock.ANY)
def paasta_rollback(args):
"""Call mark_for_deployment with rollback parameters
:param args: contains all the arguments passed onto the script: service,
cluster, instance and sha. These arguments will be verified and passed onto
mark_for_deployment.
"""
service = figure_out_service_name(args)
cluster = args.cluster
git_url = get_git_url(service)
commit = args.commit
given_instances = args.instances.split(",")
if cluster in list_clusters(service):
service_instances = list_all_instances_for_service(service)
instances, invalid = validate_given_instances(service_instances,
given_instances)
if len(invalid) > 0:
print PaastaColors.yellow(
"These instances are not valid and will be skipped: %s.\n" %
(",").join(invalid))
if len(instances) is 0:
print PaastaColors.red(
"ERROR: No valid instances specified for %s.\n" % (service))
returncode = 1
for instance in instances:
returncode = mark_for_deployment(
git_url=git_url,
cluster=cluster,
instance=instance,
service=service,
commit=commit,
)
else:
print PaastaColors.red(
"ERROR: The service %s is not deployed into cluster %s.\n" %
(service, cluster))
returncode = 1
sys.exit(returncode)
def validate_chronos(service_path):
"""Check that any chronos configurations are valid"""
soa_dir, service = path_to_soa_dir_service(service_path)
instance_type = 'chronos'
chronos_spacer = paasta_tools.chronos_tools.INTERNAL_SPACER
returncode = True
if service.startswith(TMP_JOB_IDENTIFIER):
print ("Services using scheduled tasks cannot be named %s, as it clashes with the"
" identifier used for temporary jobs" % TMP_JOB_IDENTIFIER)
return False
for cluster in list_clusters(service, soa_dir, instance_type):
services_in_cluster = get_services_for_cluster(cluster=cluster, instance_type='chronos', soa_dir=soa_dir)
valid_services = set(["%s%s%s" % (name, chronos_spacer, instance) for name, instance in services_in_cluster])
for instance in list_all_instances_for_service(
service=service, clusters=[cluster], instance_type=instance_type,
soa_dir=soa_dir):
cjc = load_chronos_job_config(service, instance, cluster, False, soa_dir)
parents = cjc.get_parents() or []
checks_passed, check_msgs = cjc.validate()
for parent in parents:
if not check_parent_format(parent):
continue
if "%s%s%s" % (service, chronos_spacer, instance) == parent:
checks_passed = False
check_msgs.append("Job %s cannot depend on itself" % parent)
elif parent not in valid_services:
checks_passed = False
check_msgs.append("Parent job %s could not be found" % parent)
# Remove duplicate check_msgs
unique_check_msgs = list(set(check_msgs))
if not checks_passed:
print invalid_chronos_instance(cluster, instance, "\n ".join(unique_check_msgs))
returncode = False
else:
print valid_chronos_instance(cluster, instance)
return returncode
def validate_chronos(service_path):
soa_dir, service = path_to_soa_dir_service(service_path)
instance_type = 'chronos'
returncode = 0
for cluster in list_clusters(service, soa_dir, instance_type):
for instance in list_all_instances_for_service(
service=service, clusters=[cluster], instance_type=instance_type,
soa_dir=soa_dir):
cjc = load_chronos_job_config(service, instance, cluster, False, soa_dir)
checks_passed, check_msgs = cjc.validate()
# Remove duplicate check_msgs
unique_check_msgs = list(set(check_msgs))
if not checks_passed:
print invalid_chronos_instance(cluster, instance, "\n ".join(unique_check_msgs))
returncode = 1
else:
print valid_chronos_instance(cluster, instance)
return returncode
def paasta_rollback(args):
"""Call mark_for_deployment with rollback parameters
:param args: contains all the arguments passed onto the script: service,
cluster, instance and sha. These arguments will be verified and passed onto
mark_for_deployment.
"""
service = figure_out_service_name(args)
cluster = args.cluster
git_url = get_git_url(service)
commit = args.commit
given_instances = args.instances.split(",")
if cluster in list_clusters(service):
service_instances = list_all_instances_for_service(service)
instances, invalid = validate_given_instances(service_instances, given_instances)
if len(invalid) > 0:
print PaastaColors.yellow("These instances are not valid and will be skipped: %s.\n" % (",").join(invalid))
if len(instances) is 0:
print PaastaColors.red("ERROR: No valid instances specified for %s.\n" % (service))
returncode = 1
for instance in instances:
returncode = mark_for_deployment(
git_url=git_url,
cluster=cluster,
instance=instance,
service=service,
commit=commit,
)
else:
print PaastaColors.red("ERROR: The service %s is not deployed into cluster %s.\n" % (service, cluster))
returncode = 1
sys.exit(returncode)
def validate_chronos(service_path):
"""Check that any chronos configurations are valid"""
soa_dir, service = path_to_soa_dir_service(service_path)
instance_type = "chronos"
chronos_spacer = paasta_tools.chronos_tools.INTERNAL_SPACER
returncode = True
for cluster in list_clusters(service, soa_dir, instance_type):
services_in_cluster = get_services_for_cluster(cluster=cluster, instance_type="chronos", soa_dir=soa_dir)
valid_services = set(["%s%s%s" % (name, chronos_spacer, instance) for name, instance in services_in_cluster])
for instance in list_all_instances_for_service(
service=service, clusters=[cluster], instance_type=instance_type, soa_dir=soa_dir
):
cjc = load_chronos_job_config(service, instance, cluster, False, soa_dir)
parents = cjc.get_parents() or []
checks_passed, check_msgs = cjc.validate()
for parent in parents:
if not check_parent_format(parent):
continue
if "%s%s%s" % (service, chronos_spacer, instance) == parent:
checks_passed = False
check_msgs.append("Job %s cannot depend on itself" % parent)
elif parent not in valid_services:
checks_passed = False
check_msgs.append("Parent job %s could not be found" % parent)
# Remove duplicate check_msgs
unique_check_msgs = list(set(check_msgs))
if not checks_passed:
print invalid_chronos_instance(cluster, instance, "\n ".join(unique_check_msgs))
returncode = False
else:
print valid_chronos_instance(cluster, instance)
return returncode
def list_instances(request):
service = request.swagger_data.get('service')
instances = list_all_instances_for_service(service, clusters=[settings.cluster])
return {'instances': list(instances)}
def list_instances(request):
service = request.matchdict['service']
instances = list_all_instances_for_service(service, clusters=[settings.cluster])
return {'instances': list(instances)}
def list_instances(request):
service = request.swagger_data.get('service')
instances = list_all_instances_for_service(service, clusters=[settings.cluster])
return {'instances': list(instances)}
