def validate_paasta_objects(service_path): soa_dir, service = path_to_soa_dir_service(service_path) returncode = True messages = [] for cluster in list_clusters(service, soa_dir): for instance in list_all_instances_for_service(service=service, clusters=[cluster], soa_dir=soa_dir): instance_config = get_instance_config( service=service, instance=instance, cluster=cluster, load_deployments=False, soa_dir=soa_dir, ) messages.extend(instance_config.validate()) returncode = len(messages) == 0 if messages: errors = "\n".join(messages) paasta_print( failure((f"There were failures validating {service}: {errors}"), "")) else: paasta_print( success(f"All PaaSTA Instances for are valid for all clusters")) return returncode
def paasta_metastatus(args): """Print the status of a PaaSTA clusters""" soa_dir = args.soa_dir system_paasta_config = load_system_paasta_config() all_clusters = list_clusters(soa_dir=soa_dir) clusters_to_inspect = figure_out_clusters_to_inspect(args, all_clusters) return_codes = [] for cluster in clusters_to_inspect: if cluster in all_clusters: return_codes.append( print_cluster_status( cluster=cluster, system_paasta_config=system_paasta_config, humanize=args.humanize, groupings=args.groupings, verbose=args.verbose, autoscaling_info=args.autoscaling_info, use_mesos_cache=args.use_mesos_cache, ), ) else: paasta_print("Cluster %s doesn't look like a valid cluster?" % args.clusters) paasta_print( "Try using tab completion to help complete the cluster name") return 0 if all([return_code == 0 for return_code in return_codes]) else 1
def validate_unique_instance_names(service_path): """Check that the service does not use the same instance name more than once""" soa_dir, service = path_to_soa_dir_service(service_path) check_passed = True for cluster in list_clusters(service, soa_dir): service_instances = get_service_instance_list(service=service, cluster=cluster, soa_dir=soa_dir) instance_names = [ service_instance[1] for service_instance in service_instances ] instance_name_to_count = Counter(instance_names) duplicate_instance_names = [ instance_name for instance_name, count in instance_name_to_count.items() if count > 1 ] if duplicate_instance_names: check_passed = False paasta_print( duplicate_instance_names_message( service, cluster, duplicate_instance_names, )) else: paasta_print(no_duplicate_instance_names_message(service, cluster)) return check_passed
def _get_secret_provider_for_service(service_name, cluster_names=None): if not is_service_folder(os.getcwd(), service_name): print( "{} not found.\n" "You must run this tool from the root of your local yelpsoa checkout\n" "The tool modifies files in yelpsoa-configs that you must then commit\n" "and push back to git.".format(os.path.join(service_name, "service.yaml")) ) sys.exit(1) system_paasta_config = load_system_paasta_config() secret_provider_kwargs = { "vault_cluster_config": system_paasta_config.get_vault_cluster_config() } clusters = ( cluster_names.split(",") if cluster_names else list_clusters(service=service_name, soa_dir=os.getcwd()) ) return get_secret_provider( secret_provider_name=system_paasta_config.get_secret_provider_name(), soa_dir=os.getcwd(), service_name=service_name, cluster_names=clusters, secret_provider_kwargs=secret_provider_kwargs, )
def validate_secrets(service_path): soa_dir, service = path_to_soa_dir_service(service_path) system_paasta_config = load_system_paasta_config() vault_cluster_map = system_paasta_config.get_vault_cluster_config() return_value = True for cluster in list_clusters(service, soa_dir): vault_env = vault_cluster_map.get(cluster) if not vault_env: print(failure(f"{cluster} not found on vault_cluster_map", "")) return_value = False continue for instance in list_all_instances_for_service(service=service, clusters=[cluster], soa_dir=soa_dir): instance_config = get_instance_config( service=service, instance=instance, cluster=cluster, load_deployments=False, soa_dir=soa_dir, ) if not check_secrets_for_instance(instance_config.config_dict, soa_dir, service_path, vault_env): return_value = False if return_value: print(success("No orphan secrets found")) return return_value
def validate_min_max_instances(service_path): soa_dir, service = path_to_soa_dir_service(service_path) returncode = True for cluster in list_clusters(service, soa_dir): for instance in list_all_instances_for_service(service=service, clusters=[cluster], soa_dir=soa_dir): instance_config = get_instance_config( service=service, instance=instance, cluster=cluster, load_deployments=False, soa_dir=soa_dir, ) if instance_config.get_instance_type() != "tron": min_instances = instance_config.get_min_instances() max_instances = instance_config.get_max_instances() if min_instances is not None and max_instances is not None: if max_instances < min_instances: returncode = False print( failure( f"Instance {instance} on cluster {cluster} has a greater number of min_instances than max_instances." + f"The number of min_instances ({min_instances}) cannot be greater than the max_instances ({max_instances}).", "", )) return returncode
def get_deploy_groups_used_by_framework(instance_type, service, soa_dir): """This is a kind of funny function that gets all the instances for specified service and framework, and massages it into a form that matches up with what deploy.yaml's steps look like. This is only so we can compare it 1-1 with what deploy.yaml has for linting. :param instance_type: one of 'marathon', 'chronos', 'adhoc' :param service: the service name :param soa_dir: The SOA configuration directory to read from :returns: a list of deploy group names used by the service. """ deploy_groups = [] for cluster in list_clusters(service, soa_dir): for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type=instance_type, soa_dir=soa_dir, ): try: config = get_instance_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, instance_type=instance_type, ) deploy_groups.append(config.get_deploy_group()) except NotImplementedError: pass return deploy_groups
def paasta_logs(args): """Print the logs for as Paasta service. :param args: argparse.Namespace obj created from sys.args by cli""" soa_dir = args.soa_dir service = figure_out_service_name(args, soa_dir) if args.clusters is None: clusters = list_clusters(service, soa_dir=soa_dir) else: clusters = args.clusters.split(",") if args.components is not None: components = args.components.split(",") else: components = DEFAULT_COMPONENTS if args.verbose: log.setLevel(logging.DEBUG) levels = [DEFAULT_LOGLEVEL, 'debug'] else: log.setLevel(logging.WARNING) levels = [DEFAULT_LOGLEVEL] log.info("Going to get logs for %s on clusters %s" % (service, clusters)) log_reader = get_log_reader() if args.tail: log_reader.tail_logs(service, levels, components, clusters, raw_mode=args.raw_mode) else: print "Non-tailing actions are not yet supported"
def paasta_logs(args): """Print the logs for as Paasta service. :param args: argparse.Namespace obj created from sys.args by cli""" service = figure_out_service_name(args) if args.clusters is None: clusters = list_clusters(service) else: clusters = args.clusters.split(",") if args.components is not None: components = args.components.split(",") else: components = DEFAULT_COMPONENTS if args.verbose: log.setLevel(logging.DEBUG) levels = [DEFAULT_LOGLEVEL, 'debug'] else: log.setLevel(logging.WARNING) levels = [DEFAULT_LOGLEVEL] log.info("Going to get logs for %s on clusters %s" % (service, clusters)) log_reader = get_log_reader() if args.tail: log_reader.tail_logs(service, levels, components, clusters, raw_mode=args.raw_mode) else: print "Non-tailing actions are not yet supported"
def paasta_metastatus(args, ) -> int: """Print the status of a PaaSTA clusters""" soa_dir = args.soa_dir system_paasta_config = load_system_paasta_config() if 'USE_API_ENDPOINT' in os.environ: use_api_endpoint = strtobool(os.environ['USE_API_ENDPOINT']) else: use_api_endpoint = True all_clusters = list_clusters(soa_dir=soa_dir) clusters_to_inspect = figure_out_clusters_to_inspect(args, all_clusters) return_codes = [] for cluster in clusters_to_inspect: if cluster in all_clusters: return_codes.append( print_cluster_status( cluster=cluster, system_paasta_config=system_paasta_config, groupings=args.groupings, verbose=args.verbose, autoscaling_info=args.autoscaling_info, use_mesos_cache=args.use_mesos_cache, use_api_endpoint=use_api_endpoint, ), ) else: paasta_print("Cluster %s doesn't look like a valid cluster?" % args.clusters) paasta_print( "Try using tab completion to help complete the cluster name") return 0 if all([return_code == 0 for return_code in return_codes]) else 1
def validate_chronos(service_path): soa_dir, service = path_to_soa_dir_service(service_path) instance_type = 'chronos' returncode = 0 for cluster in list_clusters(service, soa_dir, instance_type): for instance in list_all_instances_for_service( service=service, clusters=[cluster], instance_type=instance_type, soa_dir=soa_dir): cjc = load_chronos_job_config(service, instance, cluster, False, soa_dir) checks_passed, check_msgs = cjc.validate() # Remove duplicate check_msgs unique_check_msgs = list(set(check_msgs)) if not checks_passed: print invalid_chronos_instance(cluster, instance, "\n ".join(unique_check_msgs)) returncode = 1 else: print valid_chronos_instance(cluster, instance) return returncode
def paasta_logs(args): """Print the logs for as Paasta service. :param args: argparse.Namespace obj created from sys.args by cli""" if scribereader is None: sys.exit( "Unfortunately, `paasta logs` is unavailable without Scribe." " We're working to support alternative logging backends in PaaSTA:" " follow https://github.com/Yelp/paasta/issues/64 for updates." ) service = figure_out_service_name(args) if args.clusters is None: clusters = list_clusters(service) else: clusters = args.clusters.split(",") if args.components is not None: components = args.components.split(",") else: components = DEFAULT_COMPONENTS if args.debug: log.setLevel(logging.DEBUG) levels = [DEFAULT_LOGLEVEL, 'debug'] else: log.setLevel(logging.WARNING) levels = [DEFAULT_LOGLEVEL] log.info("Going to get logs for %s on clusters %s" % (service, clusters)) if args.tail: tail_paasta_logs(service, levels, components, clusters, raw_mode=args.raw_mode) else: print "Non-tailing actions are not yet supported"
def get_instance_config_for_service(soa_dir, service): for cluster in list_clusters( service=service, soa_dir=soa_dir, ): for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='marathon', ): yield load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, ) for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='chronos', ): yield load_chronos_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, )
def paasta_boost(args): soa_dir = args.soa_dir system_paasta_config = load_system_paasta_config() all_clusters = list_clusters(soa_dir=soa_dir) clusters = args.cluster.split(',') for cluster in clusters: if cluster not in all_clusters: paasta_print( "Error: {} doesn't look like a valid cluster. ".format(cluster) + "Here is a list of valid paasta clusters:\n" + "\n".join(all_clusters), ) return 1 return_code, output = execute_paasta_cluster_boost_on_remote_master( clusters=clusters, system_paasta_config=system_paasta_config, action=args.action, pool=args.pool, duration=args.duration if args.action == 'set' else None, override=args.override if args.action == 'set' else None, boost=args.boost if args.action == 'set' else None, verbose=args.verbose, ) paasta_print(output) return return_code
def clusters(self) -> Iterable[str]: """Returns an iterator that yields cluster names for the service. :returns: iterator that yields cluster names. """ if self._clusters is None: self._clusters = list_clusters(service=self._service, soa_dir=self._soa_dir) for cluster in self._clusters: yield cluster
def apply_args_filters( args, ) -> Mapping[str, Mapping[str, Mapping[str, Type[InstanceConfig]]]]: """ Take an args object and returns the dict of cluster:service:instances Currently, will filter by clusters, instances, services, and deploy_groups If no instances are found, will print a message and try to find matching instances for each service :param args: args object containing attributes to filter by :returns: Dict of dicts, in format {cluster_name: {service_name: {instance1, instance2}}} """ clusters_services_instances: DefaultDict[str, DefaultDict[str, Dict[ str, Type[InstanceConfig]]]] = defaultdict(lambda: defaultdict(dict)) if args.service is None and args.owner is None: args.service = figure_out_service_name(args, soa_dir=args.soa_dir) filters = get_filters(args) all_services = list_services(soa_dir=args.soa_dir) if args.service and args.service not in all_services: paasta_print( PaastaColors.red(f'The service "{args.service}" does not exist.')) suggestions = difflib.get_close_matches(args.service, all_services, n=5, cutoff=0.5) if suggestions: paasta_print(PaastaColors.red(f'Did you mean any of these?')) for suggestion in suggestions: paasta_print(PaastaColors.red(f' {suggestion}')) return clusters_services_instances i_count = 0 for service in all_services: if args.service and service != args.service: continue for instance_conf in get_instance_configs_for_service( service, soa_dir=args.soa_dir): if all([f(instance_conf) for f in filters]): cluster_service = clusters_services_instances[ instance_conf.get_cluster()][service] cluster_service[ instance_conf.get_instance()] = instance_conf.__class__ i_count += 1 if i_count == 0 and args.service and args.instances: if args.clusters: clusters = args.clusters.split(',') else: clusters = list_clusters() for service in args.service.split(','): verify_instances(args.instances, service, clusters) return clusters_services_instances
def paasta_metastatus(args): """Print the status of a PaaSTA clusters""" all_clusters = list_clusters() clusters_to_inspect = figure_out_clusters_to_inspect(args, all_clusters) for cluster in clusters_to_inspect: if cluster in all_clusters: print_cluster_status(cluster, args.verbose) else: print "Cluster %s doesn't look like a valid cluster?" % args.clusters print "Try using tab completion to help complete the cluster name"
def get_marathon_steps(service): """This is a kind of funny function that gets all the marathon instances for a service and massages it into a form that matches up with what deploy.yaml's steps look like. This is only so we can compare it 1-1 with what deploy.yaml has for linting.""" steps = [] for cluster in list_clusters(service): for instance in get_service_instance_list(service, cluster=cluster, instance_type='marathon'): steps.append("%s.%s" % (cluster, instance[1])) return steps
def validate_chronos(service_path): """Check that any chronos configurations are valid""" soa_dir, service = path_to_soa_dir_service(service_path) instance_type = 'chronos' chronos_spacer = paasta_tools.chronos_tools.INTERNAL_SPACER returncode = True if service.startswith(TMP_JOB_IDENTIFIER): paasta_print(( "Services using scheduled tasks cannot be named %s, as it clashes with the " "identifier used for temporary jobs" % TMP_JOB_IDENTIFIER)) return False for cluster in list_clusters(service, soa_dir, instance_type): services_in_cluster = get_services_for_cluster(cluster=cluster, instance_type='chronos', soa_dir=soa_dir) valid_services = { f"{name}{chronos_spacer}{instance}" for name, instance in services_in_cluster } for instance in list_all_instances_for_service( service=service, clusters=[cluster], instance_type=instance_type, soa_dir=soa_dir, ): cjc = load_chronos_job_config(service, instance, cluster, False, soa_dir) parents = cjc.get_parents() or [] checks_passed, check_msgs = cjc.validate() for parent in parents: if not check_parent_format(parent): continue if f"{service}{chronos_spacer}{instance}" == parent: checks_passed = False check_msgs.append("Job %s cannot depend on itself" % parent) elif parent not in valid_services: checks_passed = False check_msgs.append("Parent job %s could not be found" % parent) # Remove duplicate check_msgs unique_check_msgs = list(set(check_msgs)) if not checks_passed: paasta_print( invalid_chronos_instance(cluster, instance, "\n ".join(unique_check_msgs))) returncode = False else: paasta_print(valid_chronos_instance(cluster, instance)) return returncode
def paasta_start_or_stop(args, desired_state): """Requests a change of state to start or stop given branches of a service.""" instance = args.instance clusters = args.clusters soa_dir = args.soa_dir service = figure_out_service_name(args=args, soa_dir=soa_dir) if args.clusters is not None: clusters = args.clusters.split(",") else: clusters = list_clusters(service) try: remote_refs = remote_git.list_remote_refs( utils.get_git_url(service, soa_dir)) except remote_git.LSRemoteException as e: msg = ( "Error talking to the git server: %s\n" "This PaaSTA command requires access to the git server to operate.\n" "The git server may be down or not reachable from here.\n" "Try again from somewhere where the git server can be reached, " "like your developer environment.") % str(e) print msg return 1 invalid_deploy_groups = [] for cluster in clusters: service_config = get_instance_config( service=service, cluster=cluster, instance=instance, soa_dir=soa_dir, load_deployments=False, ) deploy_group = service_config.get_deploy_group() (deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group) if deploy_tag not in remote_refs: invalid_deploy_groups.append(deploy_group) else: force_bounce = utils.format_timestamp(datetime.datetime.utcnow()) issue_state_change_for_service( service_config=service_config, force_bounce=force_bounce, desired_state=desired_state, ) return_val = 0 if invalid_deploy_groups: print "No branches found for %s in %s." % \ (", ".join(invalid_deploy_groups), remote_refs) print "Has %s been deployed there yet?" % service return_val = 1 return return_val
def paasta_start_or_stop(args, desired_state): """Requests a change of state to start or stop given branches of a service.""" instance = args.instance clusters = args.clusters soa_dir = args.soa_dir service = figure_out_service_name(args=args, soa_dir=soa_dir) if args.clusters is not None: clusters = args.clusters.split(",") else: clusters = list_clusters(service) try: remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir)) except remote_git.LSRemoteException as e: msg = ( "Error talking to the git server: %s\n" "This PaaSTA command requires access to the git server to operate.\n" "The git server may be down or not reachable from here.\n" "Try again from somewhere where the git server can be reached, " "like your developer environment." ) % str(e) print msg return 1 invalid_deploy_groups = [] for cluster in clusters: service_config = get_instance_config( service=service, cluster=cluster, instance=instance, soa_dir=soa_dir, load_deployments=False, ) deploy_group = service_config.get_deploy_group() (deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group) if deploy_tag not in remote_refs: invalid_deploy_groups.append(deploy_group) else: force_bounce = utils.format_timestamp(datetime.datetime.utcnow()) issue_state_change_for_service( service_config=service_config, force_bounce=force_bounce, desired_state=desired_state, ) return_val = 0 if invalid_deploy_groups: print "No branches found for %s in %s." % \ (", ".join(invalid_deploy_groups), remote_refs) print "Has %s been deployed there yet?" % service return_val = 1 return return_val
def get_instance_configs_for_service(service, soa_dir, type_filter=None): for cluster in list_clusters( service=service, soa_dir=soa_dir, ): if type_filter is None: type_filter = ['marathon', 'chronos', 'adhoc'] if 'marathon' in type_filter: for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='marathon', soa_dir=soa_dir, ): yield load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) if 'chronos' in type_filter: for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='chronos', soa_dir=soa_dir, ): yield load_chronos_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) if 'adhoc' in type_filter: for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='adhoc', soa_dir=soa_dir, ): yield load_adhoc_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, )
def get_instance_configs_for_service( service: str, soa_dir: str, type_filter: Optional[Iterable[str]] = None, clusters: Optional[Sequence[str]] = None, instances: Optional[Sequence[str]] = None, ) -> Iterable[InstanceConfig]: if not clusters: clusters = list_clusters(service=service, soa_dir=soa_dir) if type_filter is None: type_filter = INSTANCE_TYPE_HANDLERS.keys() for cluster in list_clusters(service=service, soa_dir=soa_dir): for instance_type, instance_handlers in INSTANCE_TYPE_HANDLERS.items(): if instance_type not in type_filter: continue instance_lister, instance_loader = instance_handlers for _, instance in instance_lister( service=service, cluster=cluster, soa_dir=soa_dir, instance_type=instance_type, ): if instances and instance not in instances: continue yield instance_loader( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, )
def test_list_clusters_no_service_given_lists_all_of_them(): fake_soa_dir = "/nail/etc/services" fake_cluster_configs = [ "/nail/etc/services/service1/marathon-cluster1.yaml", "/nail/etc/services/service2/chronos-cluster2.yaml", ] expected = ["cluster1", "cluster2"] with contextlib.nested( mock.patch("os.path.join", autospec=True, return_value="%s/*" % fake_soa_dir), mock.patch("glob.glob", autospec=True, return_value=fake_cluster_configs), ) as (mock_join_path, mock_glob): actual = utils.list_clusters(soa_dir=fake_soa_dir) assert actual == expected mock_join_path.assert_called_once_with(fake_soa_dir, "*") mock_glob.assert_called_once_with("%s/*/*.yaml" % fake_soa_dir)
def get_marathon_steps(service, soa_dir): """This is a kind of funny function that gets all the marathon instances for a service and massages it into a form that matches up with what deploy.yaml's steps look like. This is only so we can compare it 1-1 with what deploy.yaml has for linting.""" steps = [] for cluster in list_clusters(service, soa_dir): for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type="marathon", soa_dir=soa_dir ): config = load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False ) steps.append(config.get_deploy_group()) return steps
def test_list_clusters_no_service_given_lists_all_of_them(): fake_soa_dir = '/nail/etc/services' fake_cluster_configs = ['/nail/etc/services/service1/marathon-cluster1.yaml', '/nail/etc/services/service2/chronos-cluster2.yaml'] expected = ['cluster1', 'cluster2'] with contextlib.nested( mock.patch('os.path.join', autospec=True, return_value='%s/*' % fake_soa_dir), mock.patch('glob.glob', autospec=True, return_value=fake_cluster_configs), ) as ( mock_join_path, mock_glob, ): actual = utils.list_clusters(soa_dir=fake_soa_dir) assert actual == expected mock_join_path.assert_called_once_with(fake_soa_dir, '*') mock_glob.assert_called_once_with('%s/*/*.yaml' % fake_soa_dir)
def test_list_clusters_ignores_bogus_clusters(): fake_soa_dir = "/nail/etc/services" fake_service = "fake_service" fake_cluster_configs = [ "/nail/etc/services/service1/marathon-cluster1.yaml", "/nail/etc/services/service1/marathon-PROD.yaml", "/nail/etc/services/service1/chronos-cluster2.yaml", "/nail/etc/services/service1/chronos-SHARED.yaml", ] expected = ["cluster1", "cluster2"] with contextlib.nested( mock.patch("os.path.join", autospec=True, return_value="%s/%s" % (fake_soa_dir, fake_service)), mock.patch("glob.glob", autospec=True, return_value=fake_cluster_configs), ) as (mock_join_path, mock_glob): actual = utils.list_clusters(service=fake_service) assert actual == expected
def validate_chronos(service_path): """Check that any chronos configurations are valid""" soa_dir, service = path_to_soa_dir_service(service_path) instance_type = 'chronos' chronos_spacer = paasta_tools.chronos_tools.INTERNAL_SPACER returncode = True for cluster in list_clusters(service, soa_dir, instance_type): services_in_cluster = get_services_for_cluster(cluster=cluster, instance_type='chronos', soa_dir=soa_dir) valid_services = set([ "%s%s%s" % (name, chronos_spacer, instance) for name, instance in services_in_cluster ]) for instance in list_all_instances_for_service( service=service, clusters=[cluster], instance_type=instance_type, soa_dir=soa_dir): cjc = load_chronos_job_config(service, instance, cluster, False, soa_dir) parents = cjc.get_parents() or [] checks_passed, check_msgs = cjc.validate() for parent in parents: if not check_parent_format(parent): continue if "%s%s%s" % (service, chronos_spacer, instance) == parent: checks_passed = False check_msgs.append("Job %s cannot depend on itself" % parent) elif parent not in valid_services: checks_passed = False check_msgs.append("Parent job %s could not be found" % parent) # Remove duplicate check_msgs unique_check_msgs = list(set(check_msgs)) if not checks_passed: print invalid_chronos_instance(cluster, instance, "\n ".join(unique_check_msgs)) returncode = False else: print valid_chronos_instance(cluster, instance) return returncode
def test_list_clusters_ignores_bogus_clusters(): fake_soa_dir = '/nail/etc/services' fake_service = 'fake_service' fake_cluster_configs = ['/nail/etc/services/service1/marathon-cluster1.yaml', '/nail/etc/services/service1/marathon-PROD.yaml', '/nail/etc/services/service1/chronos-cluster2.yaml', '/nail/etc/services/service1/chronos-SHARED.yaml'] expected = ['cluster1', 'cluster2'] with contextlib.nested( mock.patch('os.path.join', autospec=True, return_value='%s/%s' % (fake_soa_dir, fake_service)), mock.patch('glob.glob', autospec=True, return_value=fake_cluster_configs), ) as ( mock_join_path, mock_glob, ): actual = utils.list_clusters(service=fake_service) assert actual == expected
def paasta_metastatus(args): """Print the status of a PaaSTA clusters""" soa_dir = args.soa_dir system_paasta_config = load_system_paasta_config() all_clusters = list_clusters(soa_dir=soa_dir) clusters_to_inspect = figure_out_clusters_to_inspect(args, all_clusters) for cluster in clusters_to_inspect: if cluster in all_clusters: print_cluster_status( cluster=cluster, system_paasta_config=system_paasta_config, humanize=args.humanize, groupings=args.groupings, verbose=args.verbose ) else: print "Cluster %s doesn't look like a valid cluster?" % args.clusters print "Try using tab completion to help complete the cluster name"
def get_instance_configs_for_service(service, soa_dir): for cluster in list_clusters( service=service, soa_dir=soa_dir, ): for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='marathon', soa_dir=soa_dir, ): yield load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='chronos', soa_dir=soa_dir, ): yield load_chronos_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='adhoc', soa_dir=soa_dir, ): yield load_adhoc_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, )
def paasta_rollback(args): """Call mark_for_deployment with rollback parameters :param args: contains all the arguments passed onto the script: service, cluster, instance and sha. These arguments will be verified and passed onto mark_for_deployment. """ service = figure_out_service_name(args) cluster = args.cluster git_url = get_git_url(service) commit = args.commit given_instances = args.instances.split(",") if cluster in list_clusters(service): service_instances = list_all_instances_for_service(service) instances, invalid = validate_given_instances(service_instances, given_instances) if len(invalid) > 0: print PaastaColors.yellow( "These instances are not valid and will be skipped: %s.\n" % (",").join(invalid)) if len(instances) is 0: print PaastaColors.red( "ERROR: No valid instances specified for %s.\n" % (service)) returncode = 1 for instance in instances: returncode = mark_for_deployment( git_url=git_url, cluster=cluster, instance=instance, service=service, commit=commit, ) else: print PaastaColors.red( "ERROR: The service %s is not deployed into cluster %s.\n" % (service, cluster)) returncode = 1 sys.exit(returncode)
def get_marathon_steps(service, soa_dir): """This is a kind of funny function that gets all the marathon instances for a service and massages it into a form that matches up with what deploy.yaml's steps look like. This is only so we can compare it 1-1 with what deploy.yaml has for linting.""" steps = [] for cluster in list_clusters(service, soa_dir): for _, instance in get_service_instance_list(service=service, cluster=cluster, instance_type='marathon', soa_dir=soa_dir): config = load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) steps.append(config.get_deploy_group()) return steps
def list_deploy_queue(args, ) -> int: cluster = args.cluster all_clusters = list_clusters(soa_dir=args.soa_dir) if cluster not in all_clusters: paasta_print( f"{cluster} does not appear to be a valid cluster. Run `paasta " "list-clusters` to see available options.") return 1 system_paasta_config = load_system_paasta_config() client = get_paasta_api_client(cluster, system_paasta_config, http_res=True) if not client: paasta_print("Cannot get a paasta API client") return 1 try: deploy_queues, raw_response = client.deploy_queue.deploy_queue( ).result() except HTTPError as exc: paasta_print(PaastaColors.red(exc.response.text)) return exc.status_code except (BravadoConnectionError, BravadoTimeoutError) as exc: paasta_print( PaastaColors.red( f"Could not connect to API: {exc.__class__.__name__}")) return 1 except Exception: tb = sys.exc_info()[2] paasta_print(PaastaColors.red(f"Exception when talking to the API:")) paasta_print("".join(traceback.format_tb(tb))) return 1 if args.json: paasta_print(raw_response.text) else: formatted_deploy_queues = format_deploy_queues(deploy_queues, cluster) paasta_print(formatted_deploy_queues) return 0
def validate_chronos(service_path): """Check that any chronos configurations are valid""" soa_dir, service = path_to_soa_dir_service(service_path) instance_type = 'chronos' chronos_spacer = paasta_tools.chronos_tools.INTERNAL_SPACER returncode = True if service.startswith(TMP_JOB_IDENTIFIER): print ("Services using scheduled tasks cannot be named %s, as it clashes with the" " identifier used for temporary jobs" % TMP_JOB_IDENTIFIER) return False for cluster in list_clusters(service, soa_dir, instance_type): services_in_cluster = get_services_for_cluster(cluster=cluster, instance_type='chronos', soa_dir=soa_dir) valid_services = set(["%s%s%s" % (name, chronos_spacer, instance) for name, instance in services_in_cluster]) for instance in list_all_instances_for_service( service=service, clusters=[cluster], instance_type=instance_type, soa_dir=soa_dir): cjc = load_chronos_job_config(service, instance, cluster, False, soa_dir) parents = cjc.get_parents() or [] checks_passed, check_msgs = cjc.validate() for parent in parents: if not check_parent_format(parent): continue if "%s%s%s" % (service, chronos_spacer, instance) == parent: checks_passed = False check_msgs.append("Job %s cannot depend on itself" % parent) elif parent not in valid_services: checks_passed = False check_msgs.append("Parent job %s could not be found" % parent) # Remove duplicate check_msgs unique_check_msgs = list(set(check_msgs)) if not checks_passed: print invalid_chronos_instance(cluster, instance, "\n ".join(unique_check_msgs)) returncode = False else: print valid_chronos_instance(cluster, instance) return returncode
def apply_args_filters(args): """ Take an args object and returns the dict of cluster:service:instances Currently, will filter by clusters, instances, services, and deploy_groups If no instances are found, will print a message and try to find matching instances for each service :param args: args object containing attributes to filter by :returns: Dict of dicts, in format {cluster_name: {service_name: {instance1, instance2}}} """ clusters_services_instances = defaultdict(lambda: defaultdict(set)) if args.service is None and args.owner is None: args.service = figure_out_service_name(args, soa_dir=args.soa_dir) filters = get_filters(args) i_count = 0 for service in list_services(soa_dir=args.soa_dir): if args.service and service != args.service: continue for instance_conf in get_instance_configs_for_service( service, soa_dir=args.soa_dir): if all([f(instance_conf) for f in filters]): clusters_services_instances[ instance_conf.get_cluster()][service].add( instance_conf.get_instance()) i_count += 1 if i_count == 0 and args.service and args.instances: if args.clusters: clusters = args.clusters.split(',') else: clusters = list_clusters() for service in args.service.split(','): verify_instances(args.instances, service, clusters) return clusters_services_instances
def list_deploy_queue(args) -> int: cluster = args.cluster all_clusters = list_clusters(soa_dir=args.soa_dir) if cluster not in all_clusters: print(f"{cluster} does not appear to be a valid cluster. Run `paasta " "list-clusters` to see available options.") return 1 system_paasta_config = load_system_paasta_config() client = get_paasta_oapi_client(cluster, system_paasta_config, http_res=True) if not client: print("Cannot get a paasta API client") return 1 try: deploy_queues = client.default.deploy_queue() except client.api_error as exc: print(PaastaColors.red(exc.reason)) return exc.status except (client.connection_error, client.timeout_error) as exc: print( PaastaColors.red( f"Could not connect to API: {exc.__class__.__name__}")) return 1 except Exception as exc: tb = sys.exc_info()[2] print(PaastaColors.red(f"Exception when talking to the API: {exc}")) print("".join(traceback.format_tb(tb))) return 1 if args.json: json.dump(deploy_queues.to_dict(), sys.stdout) else: formatted_deploy_queues = format_deploy_queues(deploy_queues, cluster) print(formatted_deploy_queues) return 0
def paasta_rollback(args): """Call mark_for_deployment with rollback parameters :param args: contains all the arguments passed onto the script: service, cluster, instance and sha. These arguments will be verified and passed onto mark_for_deployment. """ service = figure_out_service_name(args) cluster = args.cluster git_url = get_git_url(service) commit = args.commit given_instances = args.instances.split(",") if cluster in list_clusters(service): service_instances = list_all_instances_for_service(service) instances, invalid = validate_given_instances(service_instances, given_instances) if len(invalid) > 0: print PaastaColors.yellow("These instances are not valid and will be skipped: %s.\n" % (",").join(invalid)) if len(instances) is 0: print PaastaColors.red("ERROR: No valid instances specified for %s.\n" % (service)) returncode = 1 for instance in instances: returncode = mark_for_deployment( git_url=git_url, cluster=cluster, instance=instance, service=service, commit=commit, ) else: print PaastaColors.red("ERROR: The service %s is not deployed into cluster %s.\n" % (service, cluster)) returncode = 1 sys.exit(returncode)
def validate_chronos(service_path): """Check that any chronos configurations are valid""" soa_dir, service = path_to_soa_dir_service(service_path) instance_type = "chronos" chronos_spacer = paasta_tools.chronos_tools.INTERNAL_SPACER returncode = True for cluster in list_clusters(service, soa_dir, instance_type): services_in_cluster = get_services_for_cluster(cluster=cluster, instance_type="chronos", soa_dir=soa_dir) valid_services = set(["%s%s%s" % (name, chronos_spacer, instance) for name, instance in services_in_cluster]) for instance in list_all_instances_for_service( service=service, clusters=[cluster], instance_type=instance_type, soa_dir=soa_dir ): cjc = load_chronos_job_config(service, instance, cluster, False, soa_dir) parents = cjc.get_parents() or [] checks_passed, check_msgs = cjc.validate() for parent in parents: if not check_parent_format(parent): continue if "%s%s%s" % (service, chronos_spacer, instance) == parent: checks_passed = False check_msgs.append("Job %s cannot depend on itself" % parent) elif parent not in valid_services: checks_passed = False check_msgs.append("Parent job %s could not be found" % parent) # Remove duplicate check_msgs unique_check_msgs = list(set(check_msgs)) if not checks_passed: print invalid_chronos_instance(cluster, instance, "\n ".join(unique_check_msgs)) returncode = False else: print valid_chronos_instance(cluster, instance) return returncode
def paasta_list_clusters(args, **kwargs): for cluster in list_clusters(soa_dir=args.soa_dir): paasta_print(cluster)
def paasta_logs(args): """Print the logs for as Paasta service. :param args: argparse.Namespace obj created from sys.args by cli""" soa_dir = args.soa_dir service = figure_out_service_name(args, soa_dir) if args.clusters is None: clusters = list_clusters(service, soa_dir=soa_dir) else: clusters = args.clusters.split(",") if args.instances is None: instances = None else: instances = args.instances.split(",") if args.components is not None: components = args.components.split(",") else: components = DEFAULT_COMPONENTS components = set(components) if "app_output" in components: components.remove("app_output") components.add("stdout") components.add("stderr") if args.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.WARNING) levels = [DEFAULT_LOGLEVEL, "debug"] log.info("Going to get logs for %s on clusters %s" % (service, clusters)) log_reader = get_log_reader() if not validate_filtering_args(args, log_reader): return 1 # They haven't specified what kind of filtering they want, decide for them if args.line_count is None and args.time_from is None and not args.tail: return pick_default_log_mode(args, log_reader, service, levels, components, clusters, instances) if args.tail: paasta_print(PaastaColors.cyan("Tailing logs"), file=sys.stderr) log_reader.tail_logs( service=service, levels=levels, components=components, clusters=clusters, instances=instances, raw_mode=args.raw_mode, ) return 0 # If the logger doesn't support offsetting the number of lines by a particular line number # there is no point in distinguishing between a positive/negative number of lines since it # can only get the last N lines if not log_reader.SUPPORTS_LINE_OFFSET and args.line_count is not None: args.line_count = abs(args.line_count) # Handle line based filtering if args.line_count is not None and args.line_offset is None: log_reader.print_last_n_logs( service=service, line_count=args.line_count, levels=levels, components=components, clusters=clusters, instances=instances, raw_mode=args.raw_mode, ) return 0 elif args.line_count is not None and args.line_offset is not None: log_reader.print_logs_by_offset( service=service, line_count=args.line_count, line_offset=args.line_offset, levels=levels, components=components, cluters=clusters, instances=instances, raw_mode=args.raw_mode, ) return 0 # Handle time based filtering try: start_time, end_time = generate_start_end_time(args.time_from, args.time_to) except ValueError as e: paasta_print(PaastaColors.red(e.message), file=sys.stderr) return 1 log_reader.print_logs_by_time( service=service, start_time=start_time, end_time=end_time, levels=levels, components=components, clusters=clusters, instances=instances, raw_mode=args.raw_mode, )
def paasta_start_or_stop(args, desired_state): """Requests a change of state to start or stop given branches of a service.""" soa_dir = args.soa_dir service = figure_out_service_name(args=args, soa_dir=soa_dir) if args.clusters is not None: clusters = args.clusters.split(",") else: clusters = list_clusters(service) if args.instances is not None: instances = args.instances.split(",") else: instances = None try: remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir)) except remote_git.LSRemoteException as e: msg = ( "Error talking to the git server: %s\n" "This PaaSTA command requires access to the git server to operate.\n" "The git server may be down or not reachable from here.\n" "Try again from somewhere where the git server can be reached, " "like your developer environment." ) % str(e) print msg return 1 invalid_deploy_groups = [] marathon_message_printed, chronos_message_printed = False, False for cluster in clusters: # If they haven't specified what instances to act on, do it for all of them. # If they have specified what instances, only iterate over them if they're # actually within this cluster. if instances is None: cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir) else: all_cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir) cluster_instances = all_cluster_instances.intersection(set(instances)) for instance in cluster_instances: service_config = get_instance_config( service=service, cluster=cluster, instance=instance, soa_dir=soa_dir, load_deployments=False, ) deploy_group = service_config.get_deploy_group() (deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group) if deploy_tag not in remote_refs: invalid_deploy_groups.append(deploy_group) else: force_bounce = utils.format_timestamp(datetime.datetime.utcnow()) if isinstance(service_config, MarathonServiceConfig) and not marathon_message_printed: print_marathon_message(desired_state) marathon_message_printed = True elif isinstance(service_config, ChronosJobConfig) and not chronos_message_printed: print_chronos_message(desired_state) chronos_message_printed = True issue_state_change_for_service( service_config=service_config, force_bounce=force_bounce, desired_state=desired_state, ) return_val = 0 if invalid_deploy_groups: print "No branches found for %s in %s." % \ (", ".join(invalid_deploy_groups), remote_refs) print "Has %s been deployed there yet?" % service return_val = 1 return return_val
def paasta_rerun(args): """Reruns a Chronos job. :param args: argparse.Namespace obj created from sys.args by cli""" soa_dir = args.soa_dir service = figure_out_service_name(args, soa_dir) # exit with an error if the service doesn't exist if args.execution_date: execution_date = args.execution_date else: execution_date = None all_clusters = list_clusters(soa_dir=soa_dir) actual_deployments = get_actual_deployments(service, soa_dir) # cluster.instance: sha if actual_deployments: deploy_pipeline = list(get_planned_deployments(service, soa_dir)) # cluster.instance deployed_clusters = list_deployed_clusters(deploy_pipeline, actual_deployments) deployed_cluster_instance = _get_cluster_instance(actual_deployments.keys()) if args.clusters is not None: clusters = args.clusters.split(",") else: clusters = deployed_clusters for cluster in clusters: print "cluster: %s" % cluster if cluster not in all_clusters: print " Warning: \"%s\" does not look like a valid cluster." % cluster continue if cluster not in deployed_clusters: print " Warning: service \"%s\" has not been deployed to \"%s\" yet." % (service, cluster) continue if not deployed_cluster_instance[cluster].get(args.instance, False): print (" Warning: instance \"%s\" is either invalid " "or has not been deployed to \"%s\" yet." % (args.instance, cluster)) continue try: chronos_job_config = chronos_tools.load_chronos_job_config( service, args.instance, cluster, load_deployments=False, soa_dir=soa_dir) if chronos_tools.uses_time_variables(chronos_job_config) and execution_date is None: print (" Warning: \"%s\" uses time variables interpolation, " "please supply a `--execution_date` argument." % args.instance) continue except chronos_tools.UnknownChronosJobError as e: print " Warning: %s" % e.message continue if execution_date is None: execution_date = _get_default_execution_date() rc, output = execute_chronos_rerun_on_remote_master( service=service, instancename=args.instance, cluster=cluster, verbose=args.verbose, execution_date=execution_date.strftime(chronos_tools.EXECUTION_DATE_FORMAT) ) if rc == 0: print PaastaColors.green(' successfully created job') else: print PaastaColors.red(' error') print output
def paasta_start_or_stop(args, desired_state): """Requests a change of state to start or stop given branches of a service.""" soa_dir = args.soa_dir service = figure_out_service_name(args=args, soa_dir=soa_dir) instances = args.instances.split(",") if args.instances else None # assert that each of the clusters that the user specifies are 'valid' # for the instance list provided; that is, assert that at least one of the instances # provided in the -i argument is deployed there. # if there are no instances defined in the args, then assert that the service # is deployed to that cluster. # If args.clusters is falsey, then default to *all* clusters that a service is deployed to, # and we figure out which ones are needed for each service later. if instances: instance_clusters = [list_clusters(service, soa_dir, instance) for instance in args.instances] valid_clusters = sorted(list(set([cluster for cluster_list in instance_clusters for cluster in cluster_list]))) else: valid_clusters = list_clusters(service, soa_dir) if args.clusters: clusters = args.clusters.split(",") invalid_clusters = [cluster for cluster in clusters if cluster not in valid_clusters] if invalid_clusters: print ("Invalid cluster name(s) specified: %s." "Valid options: %s") % ( " ".join(invalid_clusters), " ".join(valid_clusters), ) return 1 else: clusters = valid_clusters try: remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir)) except remote_git.LSRemoteException as e: msg = ( "Error talking to the git server: %s\n" "This PaaSTA command requires access to the git server to operate.\n" "The git server may be down or not reachable from here.\n" "Try again from somewhere where the git server can be reached, " "like your developer environment." ) % str(e) print msg return 1 invalid_deploy_groups = [] marathon_message_printed, chronos_message_printed = False, False for cluster in clusters: # If they haven't specified what instances to act on, do it for all of them. # If they have specified what instances, only iterate over them if they're # actually within this cluster. if instances is None: cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir) print ("no instances specified; restarting all instances for service") else: all_cluster_instances = list_all_instances_for_service(service, clusters=[cluster], soa_dir=soa_dir) cluster_instances = all_cluster_instances.intersection(set(instances)) for instance in cluster_instances: service_config = get_instance_config( service=service, cluster=cluster, instance=instance, soa_dir=soa_dir, load_deployments=False ) deploy_group = service_config.get_deploy_group() (deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group) if deploy_tag not in remote_refs: invalid_deploy_groups.append(deploy_group) else: force_bounce = utils.format_timestamp(datetime.datetime.utcnow()) if isinstance(service_config, MarathonServiceConfig) and not marathon_message_printed: print_marathon_message(desired_state) marathon_message_printed = True elif isinstance(service_config, ChronosJobConfig) and not chronos_message_printed: print_chronos_message(desired_state) chronos_message_printed = True issue_state_change_for_service( service_config=service_config, force_bounce=force_bounce, desired_state=desired_state ) return_val = 0 if invalid_deploy_groups: print "No branches found for %s in %s." % (", ".join(invalid_deploy_groups), remote_refs) print "Has %s been deployed there yet?" % service return_val = 1 return return_val
def paasta_list_clusters(args, **kwargs): for cluster in list_clusters(): print cluster
def completer_clusters(prefix, parsed_args, **kwargs): service = parsed_args.service or guess_service_name() if service in list_services(): return list_clusters(service) else: return list_clusters()