def get_cluster_instance_map_for_service(soa_dir, service, deploy_group=None, type_filter=None):
if deploy_group:
instances = [
config for config in get_instance_configs_for_service(
soa_dir=soa_dir, service=service, type_filter=type_filter,
)
if config.get_deploy_group() == deploy_group
]
else:
instances = get_instance_configs_for_service(soa_dir=soa_dir, service=service, type_filter=type_filter)
cluster_map = defaultdict(lambda: defaultdict(list))
for instance_config in instances:
cluster_map[instance_config.get_cluster()]['instances'].append(instance_config.get_instance())
return cluster_map
def get_cluster_instance_map_for_service(soa_dir, service, deploy_group=None):
if deploy_group:
instances = [config for config in get_instance_configs_for_service(soa_dir=soa_dir, service=service)
if config.get_deploy_group() == deploy_group]
else:
instances = get_instance_configs_for_service(soa_dir=soa_dir, service=service)
cluster_map = {}
for instance_config in instances:
try:
cluster_map[instance_config.get_cluster()]['instances'].append(instance_config.get_instance())
except KeyError:
cluster_map[instance_config.get_cluster()] = {'instances': []}
cluster_map[instance_config.get_cluster()]['instances'].append(instance_config.get_instance())
return cluster_map
def get_cluster_instance_map_for_service(soa_dir, service, deploy_group=None):
if deploy_group:
instances = [config for config in get_instance_configs_for_service(soa_dir=soa_dir, service=service)
if config.get_deploy_group() == deploy_group]
else:
instances = get_instance_configs_for_service(soa_dir=soa_dir, service=service)
cluster_map = {}
for instance_config in instances:
try:
cluster_map[instance_config.get_cluster()]['instances'].append(instance_config.get_instance())
except KeyError:
cluster_map[instance_config.get_cluster()] = {'instances': []}
cluster_map[instance_config.get_cluster()]['instances'].append(instance_config.get_instance())
return cluster_map
def apply_args_filters(
args, ) -> Mapping[str, Mapping[str, Mapping[str, Type[InstanceConfig]]]]:
"""
Take an args object and returns the dict of cluster:service:instances
Currently, will filter by clusters, instances, services, and deploy_groups
If no instances are found, will print a message and try to find matching instances
for each service
:param args: args object containing attributes to filter by
:returns: Dict of dicts, in format {cluster_name: {service_name: {instance1, instance2}}}
"""
clusters_services_instances: DefaultDict[str, DefaultDict[str, Dict[
str, Type[InstanceConfig]]]] = defaultdict(lambda: defaultdict(dict))
if args.service is None and args.owner is None:
args.service = figure_out_service_name(args, soa_dir=args.soa_dir)
filters = get_filters(args)
all_services = list_services(soa_dir=args.soa_dir)
if args.service and args.service not in all_services:
paasta_print(
PaastaColors.red(f'The service "{args.service}" does not exist.'))
suggestions = difflib.get_close_matches(args.service,
all_services,
n=5,
cutoff=0.5)
if suggestions:
paasta_print(PaastaColors.red(f'Did you mean any of these?'))
for suggestion in suggestions:
paasta_print(PaastaColors.red(f' {suggestion}'))
return clusters_services_instances
i_count = 0
for service in all_services:
if args.service and service != args.service:
continue
for instance_conf in get_instance_configs_for_service(
service, soa_dir=args.soa_dir):
if all([f(instance_conf) for f in filters]):
cluster_service = clusters_services_instances[
instance_conf.get_cluster()][service]
cluster_service[
instance_conf.get_instance()] = instance_conf.__class__
i_count += 1
if i_count == 0 and args.service and args.instances:
if args.clusters:
clusters = args.clusters.split(',')
else:
clusters = list_clusters()
for service in args.service.split(','):
verify_instances(args.instances, service, clusters)
return clusters_services_instances
def apply_args_filters(args):
"""
Take an args object and returns the dict of cluster:service:instances
Currently, will filter by clusters, instances, services, and deploy_groups
If no instances are found, will print a message and try to find matching instances
for each service
:param args: args object containing attributes to filter by
:returns: Dict of dicts, in format {cluster_name: {service_name: {instance1, instance2}}}
"""
clusters_services_instances = defaultdict(lambda: defaultdict(set))
if args.service is None and args.owner is None:
args.service = figure_out_service_name(args, soa_dir=args.soa_dir)
filters = get_filters(args)
i_count = 0
for service in list_services(soa_dir=args.soa_dir):
if args.service and service != args.service:
continue
for instance_conf in get_instance_configs_for_service(
service, soa_dir=args.soa_dir):
if all([f(instance_conf) for f in filters]):
clusters_services_instances[
instance_conf.get_cluster()][service].add(
instance_conf.get_instance())
i_count += 1
if i_count == 0 and args.service and args.instances:
if args.clusters:
clusters = args.clusters.split(',')
else:
clusters = list_clusters()
for service in args.service.split(','):
verify_instances(args.instances, service, clusters)
return clusters_services_instances
def get_deploy_group_mappings(soa_dir, service, old_mappings):
"""Gets mappings from service:deploy_group to services-service:paasta-hash,
where hash is the current SHA at the HEAD of branch_name.
This is done for all services in soa_dir.
:param soa_dir: The SOA configuration directory to read from
:param old_mappings: A dictionary like the return dictionary. Used for fallback if there is a problem with a new
mapping.
:returns: A dictionary mapping service:deploy_group to a dictionary containing:
- 'docker_image': something like "services-service:paasta-hash". This is relative to the paasta docker
registry.
- 'desired_state': either 'start' or 'stop'. Says whether this branch should be running.
- 'force_bounce': An arbitrary value, which may be None. A change in this value should trigger a bounce, even if
the other properties of this app have not changed.
"""
mappings = {}
v2_mappings = {
'deployments': {},
'controls': {},
}
service_configs = get_instance_configs_for_service(
soa_dir=soa_dir,
service=service,
)
deploy_group_branch_mappings = {config.get_branch(): config.get_deploy_group() for config in service_configs}
if not deploy_group_branch_mappings:
log.info('Service %s has no valid deploy groups. Skipping.', service)
return {}
git_url = get_git_url(
service=service,
soa_dir=soa_dir,
)
remote_refs = remote_git.list_remote_refs(git_url)
for control_branch, deploy_group in deploy_group_branch_mappings.items():
(deploy_ref_name, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_ref_name in remote_refs:
commit_sha = remote_refs[deploy_ref_name]
control_branch_alias = '%s:paasta-%s' % (service, control_branch)
control_branch_alias_v2 = '%s:%s' % (service, control_branch)
docker_image = build_docker_image_name(service, commit_sha)
log.info('Mapping %s to docker image %s', control_branch, docker_image)
mapping = mappings.setdefault(control_branch_alias, {})
mapping['docker_image'] = docker_image
v2_mappings['deployments'].setdefault(deploy_group, {})['docker_image'] = docker_image
v2_mappings['deployments'][deploy_group]['git_sha'] = commit_sha
desired_state, force_bounce = get_desired_state(
branch=control_branch,
remote_refs=remote_refs,
deploy_group=deploy_group,
)
mapping['desired_state'] = desired_state
mapping['force_bounce'] = force_bounce
v2_mappings['controls'].setdefault(control_branch_alias_v2, {})['desired_state'] = desired_state
v2_mappings['controls'][control_branch_alias_v2]['force_bounce'] = force_bounce
return mappings, v2_mappings
def get_deploy_group_mappings(soa_dir, service, old_mappings):
"""Gets mappings from service:deploy_group to services-service:paasta-hash,
where hash is the current SHA at the HEAD of branch_name.
This is done for all services in soa_dir.
:param soa_dir: The SOA configuration directory to read from
:param old_mappings: A dictionary like the return dictionary. Used for fallback if there is a problem with a new
mapping.
:returns: A dictionary mapping service:deploy_group to a dictionary containing:
- 'docker_image': something like "services-service:paasta-hash". This is relative to the paasta docker
registry.
- 'desired_state': either 'start' or 'stop'. Says whether this branch should be running.
- 'force_bounce': An arbitrary value, which may be None. A change in this value should trigger a bounce, even if
the other properties of this app have not changed.
"""
mappings = {}
v2_mappings = {
'deployments': {},
'controls': {},
}
service_configs = get_instance_configs_for_service(
soa_dir=soa_dir,
service=service,
)
deploy_group_branch_mappings = dict((config.get_branch(), config.get_deploy_group()) for config in service_configs)
if not deploy_group_branch_mappings:
log.info('Service %s has no valid deploy groups. Skipping.', service)
return {}
git_url = get_git_url(
service=service,
soa_dir=soa_dir,
)
remote_refs = remote_git.list_remote_refs(git_url)
for control_branch, deploy_group in deploy_group_branch_mappings.items():
(deploy_ref_name, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_ref_name in remote_refs:
commit_sha = remote_refs[deploy_ref_name]
control_branch_alias = '%s:paasta-%s' % (service, control_branch)
control_branch_alias_v2 = '%s:%s' % (service, control_branch)
docker_image = build_docker_image_name(service, commit_sha)
log.info('Mapping %s to docker image %s', control_branch, docker_image)
mapping = mappings.setdefault(control_branch_alias, {})
mapping['docker_image'] = docker_image
v2_mappings['deployments'].setdefault(deploy_group, {})['docker_image'] = docker_image
v2_mappings['deployments'][deploy_group]['git_sha'] = commit_sha
desired_state, force_bounce = get_desired_state(
branch=control_branch,
remote_refs=remote_refs,
deploy_group=deploy_group,
)
mapping['desired_state'] = desired_state
mapping['force_bounce'] = force_bounce
v2_mappings['controls'].setdefault(control_branch_alias_v2, {})['desired_state'] = desired_state
v2_mappings['controls'][control_branch_alias_v2]['force_bounce'] = force_bounce
return mappings, v2_mappings
def get_deploy_group_mappings(
soa_dir: str,
service: str,
) -> Tuple[Dict[str, V1_Mapping], V2_Mappings]:
"""Gets mappings from service:deploy_group to services-service:paasta-hash,
where hash is the current SHA at the HEAD of branch_name.
This is done for all services in soa_dir.
:param soa_dir: The SOA configuration directory to read from
:returns: A dictionary mapping service:deploy_group to a dictionary
containing:
- 'docker_image': something like "services-service:paasta-hash". This is
relative to the paasta docker registry.
- 'desired_state': either 'start' or 'stop'. Says whether this branch
should be running.
- 'force_bounce': An arbitrary value, which may be None. A change in this
value should trigger a bounce, even if the other properties of this app
have not changed.
"""
mappings: Dict[str, V1_Mapping] = {}
v2_mappings: V2_Mappings = {'deployments': {}, 'controls': {}}
service_configs = get_instance_configs_for_service(
soa_dir=soa_dir,
service=service,
)
deploy_group_branch_mappings = {
config.get_branch(): config.get_deploy_group()
for config in service_configs
}
if not deploy_group_branch_mappings:
log.info('Service %s has no valid deploy groups. Skipping.', service)
return mappings, v2_mappings
git_url = get_git_url(
service=service,
soa_dir=soa_dir,
)
remote_refs = remote_git.list_remote_refs(git_url)
for control_branch, deploy_group in deploy_group_branch_mappings.items():
(deploy_ref_name,
_) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_ref_name in remote_refs:
commit_sha = remote_refs[deploy_ref_name]
control_branch_alias = f'{service}:paasta-{control_branch}'
control_branch_alias_v2 = f'{service}:{control_branch}'
docker_image = build_docker_image_name(service, commit_sha)
desired_state, force_bounce = get_desired_state(
branch=control_branch,
remote_refs=remote_refs,
deploy_group=deploy_group,
)
log.info('Mapping %s to docker image %s', control_branch,
docker_image)
v2_mappings['deployments'][deploy_group] = {
'docker_image': docker_image,
'git_sha': commit_sha,
}
mappings[control_branch_alias] = {
'docker_image': docker_image,
'desired_state': desired_state,
'force_bounce': force_bounce,
}
v2_mappings['controls'][control_branch_alias_v2] = {
'desired_state': desired_state,
'force_bounce': force_bounce,
}
return mappings, v2_mappings
def get_deploy_group_mappings(
soa_dir: str, service: str
) -> Tuple[Dict[str, V1_Mapping], V2_Mappings]:
"""Gets mappings from service:deploy_group to services-service:paasta-hash,
where hash is the current SHA at the HEAD of branch_name.
This is done for all services in soa_dir.
:param soa_dir: The SOA configuration directory to read from
:returns: A dictionary mapping service:deploy_group to a dictionary
containing:
- 'docker_image': something like "services-service:paasta-hash". This is
relative to the paasta docker registry.
- 'desired_state': either 'start' or 'stop'. Says whether this branch
should be running.
- 'force_bounce': An arbitrary value, which may be None. A change in this
value should trigger a bounce, even if the other properties of this app
have not changed.
"""
mappings: Dict[str, V1_Mapping] = {}
v2_mappings: V2_Mappings = {"deployments": {}, "controls": {}}
git_url = get_git_url(service=service, soa_dir=soa_dir)
# Most of the time of this function is in two parts:
# 1. getting remote refs from git. (Mostly IO, just waiting for git to get back to us.)
# 2. loading instance configs. (Mostly CPU, copy.deepcopying yaml over and over again)
# Let's do these two things in parallel.
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
remote_refs_future = executor.submit(remote_git.list_remote_refs, git_url)
service_configs = get_instance_configs_for_service(soa_dir=soa_dir, service=service)
deploy_group_branch_mappings = {
config.get_branch(): config.get_deploy_group() for config in service_configs
}
if not deploy_group_branch_mappings:
log.info("Service %s has no valid deploy groups. Skipping.", service)
return mappings, v2_mappings
remote_refs = remote_refs_future.result()
tag_by_deploy_group = {
dg: get_latest_deployment_tag(remote_refs, dg)
for dg in set(deploy_group_branch_mappings.values())
}
state_by_branch_and_sha = get_desired_state_by_branch_and_sha(remote_refs)
for control_branch, deploy_group in deploy_group_branch_mappings.items():
(deploy_ref_name, deploy_ref_sha) = tag_by_deploy_group[deploy_group]
if deploy_ref_name in remote_refs:
commit_sha = remote_refs[deploy_ref_name]
control_branch_alias = f"{service}:paasta-{control_branch}"
control_branch_alias_v2 = f"{service}:{control_branch}"
docker_image = build_docker_image_name(service, commit_sha)
desired_state, force_bounce = state_by_branch_and_sha.get(
(control_branch, deploy_ref_sha), ("start", None)
)
log.info("Mapping %s to docker image %s", control_branch, docker_image)
v2_mappings["deployments"][deploy_group] = {
"docker_image": docker_image,
"git_sha": commit_sha,
}
mappings[control_branch_alias] = {
"docker_image": docker_image,
"desired_state": desired_state,
"force_bounce": force_bounce,
}
v2_mappings["controls"][control_branch_alias_v2] = {
"desired_state": desired_state,
"force_bounce": force_bounce,
}
return mappings, v2_mappings
