def get_cluster_instance_map_for_service(soa_dir, service, deploy_group=None, type_filter=None): if deploy_group: instances = [ config for config in get_instance_configs_for_service( soa_dir=soa_dir, service=service, type_filter=type_filter, ) if config.get_deploy_group() == deploy_group ] else: instances = get_instance_configs_for_service(soa_dir=soa_dir, service=service, type_filter=type_filter) cluster_map = defaultdict(lambda: defaultdict(list)) for instance_config in instances: cluster_map[instance_config.get_cluster()]['instances'].append(instance_config.get_instance()) return cluster_map
def get_cluster_instance_map_for_service(soa_dir, service, deploy_group=None): if deploy_group: instances = [config for config in get_instance_configs_for_service(soa_dir=soa_dir, service=service) if config.get_deploy_group() == deploy_group] else: instances = get_instance_configs_for_service(soa_dir=soa_dir, service=service) cluster_map = {} for instance_config in instances: try: cluster_map[instance_config.get_cluster()]['instances'].append(instance_config.get_instance()) except KeyError: cluster_map[instance_config.get_cluster()] = {'instances': []} cluster_map[instance_config.get_cluster()]['instances'].append(instance_config.get_instance()) return cluster_map
def apply_args_filters( args, ) -> Mapping[str, Mapping[str, Mapping[str, Type[InstanceConfig]]]]: """ Take an args object and returns the dict of cluster:service:instances Currently, will filter by clusters, instances, services, and deploy_groups If no instances are found, will print a message and try to find matching instances for each service :param args: args object containing attributes to filter by :returns: Dict of dicts, in format {cluster_name: {service_name: {instance1, instance2}}} """ clusters_services_instances: DefaultDict[str, DefaultDict[str, Dict[ str, Type[InstanceConfig]]]] = defaultdict(lambda: defaultdict(dict)) if args.service is None and args.owner is None: args.service = figure_out_service_name(args, soa_dir=args.soa_dir) filters = get_filters(args) all_services = list_services(soa_dir=args.soa_dir) if args.service and args.service not in all_services: paasta_print( PaastaColors.red(f'The service "{args.service}" does not exist.')) suggestions = difflib.get_close_matches(args.service, all_services, n=5, cutoff=0.5) if suggestions: paasta_print(PaastaColors.red(f'Did you mean any of these?')) for suggestion in suggestions: paasta_print(PaastaColors.red(f' {suggestion}')) return clusters_services_instances i_count = 0 for service in all_services: if args.service and service != args.service: continue for instance_conf in get_instance_configs_for_service( service, soa_dir=args.soa_dir): if all([f(instance_conf) for f in filters]): cluster_service = clusters_services_instances[ instance_conf.get_cluster()][service] cluster_service[ instance_conf.get_instance()] = instance_conf.__class__ i_count += 1 if i_count == 0 and args.service and args.instances: if args.clusters: clusters = args.clusters.split(',') else: clusters = list_clusters() for service in args.service.split(','): verify_instances(args.instances, service, clusters) return clusters_services_instances
def apply_args_filters(args): """ Take an args object and returns the dict of cluster:service:instances Currently, will filter by clusters, instances, services, and deploy_groups If no instances are found, will print a message and try to find matching instances for each service :param args: args object containing attributes to filter by :returns: Dict of dicts, in format {cluster_name: {service_name: {instance1, instance2}}} """ clusters_services_instances = defaultdict(lambda: defaultdict(set)) if args.service is None and args.owner is None: args.service = figure_out_service_name(args, soa_dir=args.soa_dir) filters = get_filters(args) i_count = 0 for service in list_services(soa_dir=args.soa_dir): if args.service and service != args.service: continue for instance_conf in get_instance_configs_for_service( service, soa_dir=args.soa_dir): if all([f(instance_conf) for f in filters]): clusters_services_instances[ instance_conf.get_cluster()][service].add( instance_conf.get_instance()) i_count += 1 if i_count == 0 and args.service and args.instances: if args.clusters: clusters = args.clusters.split(',') else: clusters = list_clusters() for service in args.service.split(','): verify_instances(args.instances, service, clusters) return clusters_services_instances
def get_deploy_group_mappings(soa_dir, service, old_mappings): """Gets mappings from service:deploy_group to services-service:paasta-hash, where hash is the current SHA at the HEAD of branch_name. This is done for all services in soa_dir. :param soa_dir: The SOA configuration directory to read from :param old_mappings: A dictionary like the return dictionary. Used for fallback if there is a problem with a new mapping. :returns: A dictionary mapping service:deploy_group to a dictionary containing: - 'docker_image': something like "services-service:paasta-hash". This is relative to the paasta docker registry. - 'desired_state': either 'start' or 'stop'. Says whether this branch should be running. - 'force_bounce': An arbitrary value, which may be None. A change in this value should trigger a bounce, even if the other properties of this app have not changed. """ mappings = {} v2_mappings = { 'deployments': {}, 'controls': {}, } service_configs = get_instance_configs_for_service( soa_dir=soa_dir, service=service, ) deploy_group_branch_mappings = {config.get_branch(): config.get_deploy_group() for config in service_configs} if not deploy_group_branch_mappings: log.info('Service %s has no valid deploy groups. Skipping.', service) return {} git_url = get_git_url( service=service, soa_dir=soa_dir, ) remote_refs = remote_git.list_remote_refs(git_url) for control_branch, deploy_group in deploy_group_branch_mappings.items(): (deploy_ref_name, _) = get_latest_deployment_tag(remote_refs, deploy_group) if deploy_ref_name in remote_refs: commit_sha = remote_refs[deploy_ref_name] control_branch_alias = '%s:paasta-%s' % (service, control_branch) control_branch_alias_v2 = '%s:%s' % (service, control_branch) docker_image = build_docker_image_name(service, commit_sha) log.info('Mapping %s to docker image %s', control_branch, docker_image) mapping = mappings.setdefault(control_branch_alias, {}) mapping['docker_image'] = docker_image v2_mappings['deployments'].setdefault(deploy_group, {})['docker_image'] = docker_image v2_mappings['deployments'][deploy_group]['git_sha'] = commit_sha desired_state, force_bounce = get_desired_state( branch=control_branch, remote_refs=remote_refs, deploy_group=deploy_group, ) mapping['desired_state'] = desired_state mapping['force_bounce'] = force_bounce v2_mappings['controls'].setdefault(control_branch_alias_v2, {})['desired_state'] = desired_state v2_mappings['controls'][control_branch_alias_v2]['force_bounce'] = force_bounce return mappings, v2_mappings
def get_deploy_group_mappings(soa_dir, service, old_mappings): """Gets mappings from service:deploy_group to services-service:paasta-hash, where hash is the current SHA at the HEAD of branch_name. This is done for all services in soa_dir. :param soa_dir: The SOA configuration directory to read from :param old_mappings: A dictionary like the return dictionary. Used for fallback if there is a problem with a new mapping. :returns: A dictionary mapping service:deploy_group to a dictionary containing: - 'docker_image': something like "services-service:paasta-hash". This is relative to the paasta docker registry. - 'desired_state': either 'start' or 'stop'. Says whether this branch should be running. - 'force_bounce': An arbitrary value, which may be None. A change in this value should trigger a bounce, even if the other properties of this app have not changed. """ mappings = {} v2_mappings = { 'deployments': {}, 'controls': {}, } service_configs = get_instance_configs_for_service( soa_dir=soa_dir, service=service, ) deploy_group_branch_mappings = dict((config.get_branch(), config.get_deploy_group()) for config in service_configs) if not deploy_group_branch_mappings: log.info('Service %s has no valid deploy groups. Skipping.', service) return {} git_url = get_git_url( service=service, soa_dir=soa_dir, ) remote_refs = remote_git.list_remote_refs(git_url) for control_branch, deploy_group in deploy_group_branch_mappings.items(): (deploy_ref_name, _) = get_latest_deployment_tag(remote_refs, deploy_group) if deploy_ref_name in remote_refs: commit_sha = remote_refs[deploy_ref_name] control_branch_alias = '%s:paasta-%s' % (service, control_branch) control_branch_alias_v2 = '%s:%s' % (service, control_branch) docker_image = build_docker_image_name(service, commit_sha) log.info('Mapping %s to docker image %s', control_branch, docker_image) mapping = mappings.setdefault(control_branch_alias, {}) mapping['docker_image'] = docker_image v2_mappings['deployments'].setdefault(deploy_group, {})['docker_image'] = docker_image v2_mappings['deployments'][deploy_group]['git_sha'] = commit_sha desired_state, force_bounce = get_desired_state( branch=control_branch, remote_refs=remote_refs, deploy_group=deploy_group, ) mapping['desired_state'] = desired_state mapping['force_bounce'] = force_bounce v2_mappings['controls'].setdefault(control_branch_alias_v2, {})['desired_state'] = desired_state v2_mappings['controls'][control_branch_alias_v2]['force_bounce'] = force_bounce return mappings, v2_mappings
def get_deploy_group_mappings( soa_dir: str, service: str, ) -> Tuple[Dict[str, V1_Mapping], V2_Mappings]: """Gets mappings from service:deploy_group to services-service:paasta-hash, where hash is the current SHA at the HEAD of branch_name. This is done for all services in soa_dir. :param soa_dir: The SOA configuration directory to read from :returns: A dictionary mapping service:deploy_group to a dictionary containing: - 'docker_image': something like "services-service:paasta-hash". This is relative to the paasta docker registry. - 'desired_state': either 'start' or 'stop'. Says whether this branch should be running. - 'force_bounce': An arbitrary value, which may be None. A change in this value should trigger a bounce, even if the other properties of this app have not changed. """ mappings: Dict[str, V1_Mapping] = {} v2_mappings: V2_Mappings = {'deployments': {}, 'controls': {}} service_configs = get_instance_configs_for_service( soa_dir=soa_dir, service=service, ) deploy_group_branch_mappings = { config.get_branch(): config.get_deploy_group() for config in service_configs } if not deploy_group_branch_mappings: log.info('Service %s has no valid deploy groups. Skipping.', service) return mappings, v2_mappings git_url = get_git_url( service=service, soa_dir=soa_dir, ) remote_refs = remote_git.list_remote_refs(git_url) for control_branch, deploy_group in deploy_group_branch_mappings.items(): (deploy_ref_name, _) = get_latest_deployment_tag(remote_refs, deploy_group) if deploy_ref_name in remote_refs: commit_sha = remote_refs[deploy_ref_name] control_branch_alias = f'{service}:paasta-{control_branch}' control_branch_alias_v2 = f'{service}:{control_branch}' docker_image = build_docker_image_name(service, commit_sha) desired_state, force_bounce = get_desired_state( branch=control_branch, remote_refs=remote_refs, deploy_group=deploy_group, ) log.info('Mapping %s to docker image %s', control_branch, docker_image) v2_mappings['deployments'][deploy_group] = { 'docker_image': docker_image, 'git_sha': commit_sha, } mappings[control_branch_alias] = { 'docker_image': docker_image, 'desired_state': desired_state, 'force_bounce': force_bounce, } v2_mappings['controls'][control_branch_alias_v2] = { 'desired_state': desired_state, 'force_bounce': force_bounce, } return mappings, v2_mappings
def get_deploy_group_mappings( soa_dir: str, service: str ) -> Tuple[Dict[str, V1_Mapping], V2_Mappings]: """Gets mappings from service:deploy_group to services-service:paasta-hash, where hash is the current SHA at the HEAD of branch_name. This is done for all services in soa_dir. :param soa_dir: The SOA configuration directory to read from :returns: A dictionary mapping service:deploy_group to a dictionary containing: - 'docker_image': something like "services-service:paasta-hash". This is relative to the paasta docker registry. - 'desired_state': either 'start' or 'stop'. Says whether this branch should be running. - 'force_bounce': An arbitrary value, which may be None. A change in this value should trigger a bounce, even if the other properties of this app have not changed. """ mappings: Dict[str, V1_Mapping] = {} v2_mappings: V2_Mappings = {"deployments": {}, "controls": {}} git_url = get_git_url(service=service, soa_dir=soa_dir) # Most of the time of this function is in two parts: # 1. getting remote refs from git. (Mostly IO, just waiting for git to get back to us.) # 2. loading instance configs. (Mostly CPU, copy.deepcopying yaml over and over again) # Let's do these two things in parallel. executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) remote_refs_future = executor.submit(remote_git.list_remote_refs, git_url) service_configs = get_instance_configs_for_service(soa_dir=soa_dir, service=service) deploy_group_branch_mappings = { config.get_branch(): config.get_deploy_group() for config in service_configs } if not deploy_group_branch_mappings: log.info("Service %s has no valid deploy groups. Skipping.", service) return mappings, v2_mappings remote_refs = remote_refs_future.result() tag_by_deploy_group = { dg: get_latest_deployment_tag(remote_refs, dg) for dg in set(deploy_group_branch_mappings.values()) } state_by_branch_and_sha = get_desired_state_by_branch_and_sha(remote_refs) for control_branch, deploy_group in deploy_group_branch_mappings.items(): (deploy_ref_name, deploy_ref_sha) = tag_by_deploy_group[deploy_group] if deploy_ref_name in remote_refs: commit_sha = remote_refs[deploy_ref_name] control_branch_alias = f"{service}:paasta-{control_branch}" control_branch_alias_v2 = f"{service}:{control_branch}" docker_image = build_docker_image_name(service, commit_sha) desired_state, force_bounce = state_by_branch_and_sha.get( (control_branch, deploy_ref_sha), ("start", None) ) log.info("Mapping %s to docker image %s", control_branch, docker_image) v2_mappings["deployments"][deploy_group] = { "docker_image": docker_image, "git_sha": commit_sha, } mappings[control_branch_alias] = { "docker_image": docker_image, "desired_state": desired_state, "force_bounce": force_bounce, } v2_mappings["controls"][control_branch_alias_v2] = { "desired_state": desired_state, "force_bounce": force_bounce, } return mappings, v2_mappings