def load_adhoc_job_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR):
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir
)
adhoc_conf_file = "adhoc-%s" % cluster
log.info("Reading adhoc configuration file: %s.yaml", adhoc_conf_file)
instance_configs = service_configuration_lib.read_extra_service_information(
service_name=service,
extra_info=adhoc_conf_file,
soa_dir=soa_dir
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, adhoc_conf_file)
)
general_config = deep_merge_dictionaries(overrides=instance_configs[instance], defaults=general_config)
branch_dict = {}
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
branch = general_config.get('branch', get_paasta_branch(cluster, instance))
deploy_group = general_config.get('deploy_group', branch)
branch_dict = deployments_json.get_branch_dict_v2(service, branch, deploy_group)
return AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
)
def get_currently_deployed_sha(service, deploy_group, soa_dir=DEFAULT_SOA_DIR):
"""Tries to determine the currently deployed sha for a service and deploy_group,
returns None if there isn't one ready yet"""
try:
deployments = load_v2_deployments_json(service=service, soa_dir=soa_dir)
return deployments.get_git_sha_for_deploy_group(deploy_group=deploy_group)
except NoDeploymentsAvailable:
return None
def load_paasta_native_job_config(
service: str,
instance: str,
cluster: str,
load_deployments: bool = True,
soa_dir: str = DEFAULT_SOA_DIR,
instance_type: str = "paasta_native",
config_overrides: Optional[NativeServiceConfigDict] = None,
) -> NativeServiceConfig:
instance_config_dict = cast(
NativeServiceConfigDict,
load_service_instance_config(
service=service,
instance=instance,
instance_type=instance_type,
cluster=cluster,
soa_dir=soa_dir,
),
)
branch_dict: Optional[BranchDictV2] = None
instance_config_dict.update(config_overrides or {})
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = NativeServiceConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=instance_config_dict,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
service_config = NativeServiceConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=instance_config_dict,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
service_namespace_config = load_service_namespace_config(
service=service,
namespace=service_config.get_nerve_namespace(),
soa_dir=soa_dir)
service_config.service_namespace_config = service_namespace_config
return service_config
def get_currently_deployed_version(
service,
deploy_group,
soa_dir=DEFAULT_SOA_DIR) -> Optional[DeploymentVersion]:
"""Tries to determine the currently deployed version for a service and deploy_group,
returns None if there isn't one ready yet"""
try:
deployments = load_v2_deployments_json(service=service,
soa_dir=soa_dir)
return deployments.get_deployment_version_for_deploy_group(
deploy_group=deploy_group)
except NoDeploymentsAvailable:
return None
def load_paasta_native_job_config(
service,
instance,
cluster,
load_deployments=True,
soa_dir=DEFAULT_SOA_DIR,
instance_type='paasta_native',
config_overrides=None,
) -> NativeServiceConfig:
service_paasta_native_jobs = read_service_config(
service=service,
instance=instance,
instance_type=instance_type,
cluster=cluster,
soa_dir=soa_dir,
)
branch_dict: Optional[BranchDictV2] = None
instance_config_dict = service_paasta_native_jobs[instance].copy()
instance_config_dict.update(config_overrides or {})
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = NativeServiceConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=instance_config_dict,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
service_config = NativeServiceConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=instance_config_dict,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
service_namespace_config = load_service_namespace_config(
service=service,
namespace=service_config.get_nerve_namespace(),
soa_dir=soa_dir,
)
service_config.service_namespace_config = service_namespace_config
return service_config
def get_default_interactive_config(
service: str,
cluster: str,
soa_dir: str,
load_deployments: bool = False) -> AdhocJobConfig:
default_job_config = {"cpus": 4, "mem": 10240, "disk": 1024}
try:
job_config = load_adhoc_job_config(service=service,
instance="interactive",
cluster=cluster,
soa_dir=soa_dir)
except NoConfigurationForServiceError:
job_config = AdhocJobConfig(
service=service,
instance="interactive",
cluster=cluster,
config_dict={},
branch_dict=None,
soa_dir=soa_dir,
)
except NoDeploymentsAvailable:
job_config = load_adhoc_job_config(
service=service,
instance="interactive",
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
if not job_config.branch_dict and load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
deploy_group = prompt_pick_one(deployments_json.get_deploy_groups(),
choosing="deploy group")
job_config.config_dict["deploy_group"] = deploy_group
job_config.branch_dict = {
"docker_image":
deployments_json.get_docker_image_for_deploy_group(deploy_group),
"git_sha":
deployments_json.get_git_sha_for_deploy_group(deploy_group),
"force_bounce":
None,
"desired_state":
"start",
}
for key, value in default_job_config.items():
job_config.config_dict.setdefault(key, value)
return job_config
def load_chronos_job_config(
service: str,
instance: str,
cluster: str,
load_deployments: bool = True,
soa_dir: str = DEFAULT_SOA_DIR,
) -> 'ChronosJobConfig':
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir,
)
if instance.startswith('_'):
raise InvalidJobNameError(
"Unable to load chronos job config for %s.%s as instance name starts with '_'"
% (service, instance), )
service_chronos_jobs = read_chronos_jobs_for_service(service,
cluster,
soa_dir=soa_dir)
if instance not in service_chronos_jobs:
raise NoConfigurationForServiceError(
'No job named "%s" in config file chronos-%s.yaml' %
(instance, cluster))
branch_dict = None
general_config = deep_merge_dictionaries(
overrides=service_chronos_jobs[instance], defaults=general_config)
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = ChronosJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
return ChronosJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def get_default_interactive_config(service,
cluster,
soa_dir,
load_deployments=False):
default_job_config = {
'cpus': 4,
'mem': 10240,
'disk': 1024,
}
try:
job_config = load_adhoc_job_config(service=service,
instance='interactive',
cluster=cluster,
soa_dir=soa_dir)
except NoConfigurationForServiceError:
job_config = AdhocJobConfig(
service=service,
instance='interactive',
cluster=cluster,
config_dict={},
branch_dict={},
soa_dir=soa_dir,
)
except NoDeploymentsAvailable:
job_config = load_adhoc_job_config(
service=service,
instance='interactive',
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
if not job_config.branch_dict and load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
deploy_group = prompt_pick_one(
(deployment
for deployment in deployments_json['deployments'].keys()),
choosing='deploy group',
)
job_config.config_dict['deploy_group'] = deploy_group
job_config.branch_dict[
'docker_image'] = deployments_json.get_docker_image_for_deploy_group(
deploy_group)
for key, value in default_job_config.items():
job_config.config_dict.setdefault(key, value)
return job_config
def load_adhoc_job_config(service,
instance,
cluster,
load_deployments=True,
soa_dir=DEFAULT_SOA_DIR):
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir,
)
adhoc_conf_file = "adhoc-%s" % cluster
instance_configs = service_configuration_lib.read_extra_service_information(
service_name=service,
extra_info=adhoc_conf_file,
soa_dir=soa_dir,
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." %
(instance, soa_dir, service, adhoc_conf_file), )
general_config = deep_merge_dictionaries(
overrides=instance_configs[instance], defaults=general_config)
branch_dict = None
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
return AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def _get_action_config(self, action_dict, default_paasta_cluster):
action_service = action_dict.setdefault('service', self.get_service())
action_deploy_group = action_dict.setdefault('deploy_group',
self.get_deploy_group())
if action_service and action_deploy_group and self.load_deployments:
try:
deployments_json = load_v2_deployments_json(
service=action_service, soa_dir=self.soa_dir)
branch_dict = {
'docker_image':
deployments_json.get_docker_image_for_deploy_group(
action_deploy_group),
'git_sha':
deployments_json.get_git_sha_for_deploy_group(
action_deploy_group),
# TODO: add Tron instances when generating deployments json
'desired_state':
'start',
'force_bounce':
None,
}
except NoDeploymentsAvailable:
raise InvalidTronConfig(
'No deployment found for action {action} in job {job}, looking for {deploy_group} '
'in service {service}'.format(
action=action_dict.get('name'),
job=self.get_name(),
deploy_group=action_deploy_group,
service=action_service,
), )
else:
branch_dict = None
cluster = action_dict.get(
'cluster') or self.get_cluster() or default_paasta_cluster
return TronActionConfig(
service=action_service,
instance=compose_instance(self.get_name(),
action_dict.get('name')),
cluster=cluster,
config_dict=action_dict,
branch_dict=branch_dict,
soa_dir=self.soa_dir,
)
def _get_action_config(self, action_name, action_dict):
action_service = action_dict.setdefault("service", self.get_service())
action_deploy_group = action_dict.setdefault(
"deploy_group", self.get_deploy_group()
)
if action_service and action_deploy_group and self.load_deployments:
try:
deployments_json = load_v2_deployments_json(
service=action_service, soa_dir=self.soa_dir
)
branch_dict = {
"docker_image": deployments_json.get_docker_image_for_deploy_group(
action_deploy_group
),
"git_sha": deployments_json.get_git_sha_for_deploy_group(
action_deploy_group
),
"image_version": deployments_json.get_image_version_for_deploy_group(
action_deploy_group
),
# TODO: add Tron instances when generating deployments json
"desired_state": "start",
"force_bounce": None,
}
except NoDeploymentsAvailable:
log.warning(
f'Docker image unavailable for {action_service}.{self.get_name()}.{action_dict.get("name")}'
" is it deployed yet?"
)
branch_dict = None
else:
branch_dict = None
action_dict["monitoring"] = self.get_monitoring()
return TronActionConfig(
service=action_service,
instance=compose_instance(self.get_name(), action_name),
cluster=self.get_cluster(),
config_dict=action_dict,
branch_dict=branch_dict,
soa_dir=self.soa_dir,
for_validation=self.for_validation,
)
def load_vitess_instance_config(
service: str,
instance: str,
cluster: str,
load_deployments: bool = True,
soa_dir: str = DEFAULT_SOA_DIR,
) -> VitessDeploymentConfig:
general_config = service_configuration_lib.read_service_configuration(
service, soa_dir=soa_dir)
instance_config = load_service_instance_config(service,
instance,
" vitesscluster",
cluster,
soa_dir=soa_dir)
general_config = deep_merge_dictionaries(overrides=instance_config,
defaults=general_config)
branch_dict: Optional[BranchDictV2] = None
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = VitessDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
return VitessDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def load_adhoc_job_config(
service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR
):
general_config = service_configuration_lib.read_service_configuration(
service, soa_dir=soa_dir
)
instance_config = load_service_instance_config(
service=service,
instance=instance,
instance_type="adhoc",
cluster=cluster,
soa_dir=soa_dir,
)
general_config = deep_merge_dictionaries(
overrides=instance_config, defaults=general_config
)
branch_dict = None
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group)
return AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def get_default_interactive_config(service, cluster, soa_dir, load_deployments=False):
default_job_config = {
'cpus': 4,
'mem': 10240,
'disk': 1024
}
try:
job_config = load_adhoc_job_config(service=service, instance='interactive', cluster=cluster, soa_dir=soa_dir)
except NoConfigurationForServiceError:
job_config = AdhocJobConfig(
service=service,
instance='interactive',
cluster=cluster,
config_dict={},
branch_dict={},
)
except NoDeploymentsAvailable:
job_config = load_adhoc_job_config(
service=service, instance='interactive', cluster=cluster, soa_dir=soa_dir, load_deployments=False)
if not job_config.branch_dict and load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
deploy_group = prompt_pick_one(
(
deployment.encode('utf-8')
for deployment in deployments_json['deployments'].keys()
),
choosing='deploy group',
)
job_config.config_dict['deploy_group'] = deploy_group
job_config.branch_dict['docker_image'] = deployments_json.get_docker_image_for_deploy_group(deploy_group)
for key, value in default_job_config.items():
job_config.config_dict.setdefault(key, value)
return job_config
def load_adhoc_job_config(service,
instance,
cluster,
load_deployments=True,
soa_dir=DEFAULT_SOA_DIR):
general_config = service_configuration_lib.read_service_configuration(
service, soa_dir=soa_dir)
adhoc_conf_file = "adhoc-%s" % cluster
log.info("Reading adhoc configuration file: %s.yaml", adhoc_conf_file)
instance_configs = service_configuration_lib.read_extra_service_information(
service_name=service, extra_info=adhoc_conf_file, soa_dir=soa_dir)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
"%s not found in config file %s/%s/%s.yaml." %
(instance, soa_dir, service, adhoc_conf_file))
general_config = deep_merge_dictionaries(
overrides=instance_configs[instance], defaults=general_config)
branch_dict = {}
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
branch = general_config.get('branch',
get_paasta_branch(cluster, instance))
deploy_group = general_config.get('deploy_group', branch)
branch_dict = deployments_json.get_branch_dict_v2(
service, branch, deploy_group)
return AdhocJobConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
def get_default_interactive_config(service, cluster, soa_dir, load_deployments=False):
default_job_config = {"cpus": 4, "mem": 10240, "disk": 1024}
try:
job_config = load_adhoc_job_config(service=service, instance="interactive", cluster=cluster, soa_dir=soa_dir)
except NoConfigurationForServiceError:
job_config = AdhocJobConfig(
service=service, instance="interactive", cluster=cluster, config_dict={}, branch_dict={}
)
except NoDeploymentsAvailable:
job_config = load_adhoc_job_config(
service=service, instance="interactive", cluster=cluster, soa_dir=soa_dir, load_deployments=False
)
if not job_config.branch_dict and load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
deploy_group = prompt_pick_one(deployments_json["deployments"].keys(), choosing="deploy group")
job_config.config_dict["deploy_group"] = deploy_group
job_config.branch_dict["docker_image"] = deployments_json.get_docker_image_for_deploy_group(deploy_group)
for key, value in default_job_config.items():
job_config.config_dict.setdefault(key, value)
return job_config
def load_kubernetes_service_config_no_cache(
service: str,
instance: str,
cluster: str,
load_deployments: bool = True,
soa_dir: str = DEFAULT_SOA_DIR,
) -> "KubernetesDeploymentConfig":
"""Read a service instance's configuration for kubernetes.
If a branch isn't specified for a config, the 'branch' key defaults to
paasta-${cluster}.${instance}.
:param name: The service name
:param instance: The instance of the service to retrieve
:param cluster: The cluster to read the configuration for
:param load_deployments: A boolean indicating if the corresponding deployments.json for this service
should also be loaded
:param soa_dir: The SOA configuration directory to read from
:returns: A dictionary of whatever was in the config for the service instance"""
general_config = service_configuration_lib.read_service_configuration(
service,
soa_dir=soa_dir,
)
kubernetes_conf_file = "kubernetes-%s" % cluster
instance_configs = service_configuration_lib.read_extra_service_information(
service,
kubernetes_conf_file,
soa_dir=soa_dir,
)
if instance.startswith('_'):
raise InvalidJobNameError(
f"Unable to load kubernetes job config for {service}.{instance} as instance name starts with '_'",
)
if instance not in instance_configs:
raise NoConfigurationForServiceError(
f"{instance} not found in config file {soa_dir}/{service}/{kubernetes_conf_file}.yaml.",
)
general_config = deep_merge_dictionaries(
overrides=instance_configs[instance], defaults=general_config)
branch_dict: Optional[BranchDictV2] = None
if load_deployments:
deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
temp_instance_config = KubernetesDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=None,
soa_dir=soa_dir,
)
branch = temp_instance_config.get_branch()
deploy_group = temp_instance_config.get_deploy_group()
branch_dict = deployments_json.get_branch_dict(service, branch,
deploy_group)
return KubernetesDeploymentConfig(
service=service,
cluster=cluster,
instance=instance,
config_dict=general_config,
branch_dict=branch_dict,
soa_dir=soa_dir,
)
