示例#1
0
def load_adhoc_job_config(service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR):
    general_config = service_configuration_lib.read_service_configuration(
        service,
        soa_dir=soa_dir
    )
    adhoc_conf_file = "adhoc-%s" % cluster
    log.info("Reading adhoc configuration file: %s.yaml", adhoc_conf_file)
    instance_configs = service_configuration_lib.read_extra_service_information(
        service_name=service,
        extra_info=adhoc_conf_file,
        soa_dir=soa_dir
    )

    if instance not in instance_configs:
        raise NoConfigurationForServiceError(
            "%s not found in config file %s/%s/%s.yaml." % (instance, soa_dir, service, adhoc_conf_file)
        )

    general_config = deep_merge_dictionaries(overrides=instance_configs[instance], defaults=general_config)

    branch_dict = {}
    if load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        branch = general_config.get('branch', get_paasta_branch(cluster, instance))
        deploy_group = general_config.get('deploy_group', branch)
        branch_dict = deployments_json.get_branch_dict_v2(service, branch, deploy_group)

    return AdhocJobConfig(
        service=service,
        cluster=cluster,
        instance=instance,
        config_dict=general_config,
        branch_dict=branch_dict,
    )
示例#2
0
def get_currently_deployed_sha(service, deploy_group, soa_dir=DEFAULT_SOA_DIR):
    """Tries to determine the currently deployed sha for a service and deploy_group,
    returns None if there isn't one ready yet"""
    try:
        deployments = load_v2_deployments_json(service=service, soa_dir=soa_dir)
        return deployments.get_git_sha_for_deploy_group(deploy_group=deploy_group)
    except NoDeploymentsAvailable:
        return None
def load_paasta_native_job_config(
    service: str,
    instance: str,
    cluster: str,
    load_deployments: bool = True,
    soa_dir: str = DEFAULT_SOA_DIR,
    instance_type: str = "paasta_native",
    config_overrides: Optional[NativeServiceConfigDict] = None,
) -> NativeServiceConfig:
    instance_config_dict = cast(
        NativeServiceConfigDict,
        load_service_instance_config(
            service=service,
            instance=instance,
            instance_type=instance_type,
            cluster=cluster,
            soa_dir=soa_dir,
        ),
    )
    branch_dict: Optional[BranchDictV2] = None
    instance_config_dict.update(config_overrides or {})
    if load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        temp_instance_config = NativeServiceConfig(
            service=service,
            cluster=cluster,
            instance=instance,
            config_dict=instance_config_dict,
            branch_dict=None,
            soa_dir=soa_dir,
        )
        branch = temp_instance_config.get_branch()
        deploy_group = temp_instance_config.get_deploy_group()
        branch_dict = deployments_json.get_branch_dict(service, branch,
                                                       deploy_group)

    service_config = NativeServiceConfig(
        service=service,
        cluster=cluster,
        instance=instance,
        config_dict=instance_config_dict,
        branch_dict=branch_dict,
        soa_dir=soa_dir,
    )

    service_namespace_config = load_service_namespace_config(
        service=service,
        namespace=service_config.get_nerve_namespace(),
        soa_dir=soa_dir)
    service_config.service_namespace_config = service_namespace_config

    return service_config
示例#4
0
def get_currently_deployed_version(
        service,
        deploy_group,
        soa_dir=DEFAULT_SOA_DIR) -> Optional[DeploymentVersion]:
    """Tries to determine the currently deployed version for a service and deploy_group,
    returns None if there isn't one ready yet"""
    try:
        deployments = load_v2_deployments_json(service=service,
                                               soa_dir=soa_dir)
        return deployments.get_deployment_version_for_deploy_group(
            deploy_group=deploy_group)
    except NoDeploymentsAvailable:
        return None
示例#5
0
def load_paasta_native_job_config(
    service,
    instance,
    cluster,
    load_deployments=True,
    soa_dir=DEFAULT_SOA_DIR,
    instance_type='paasta_native',
    config_overrides=None,
) -> NativeServiceConfig:
    service_paasta_native_jobs = read_service_config(
        service=service,
        instance=instance,
        instance_type=instance_type,
        cluster=cluster,
        soa_dir=soa_dir,
    )
    branch_dict: Optional[BranchDictV2] = None
    instance_config_dict = service_paasta_native_jobs[instance].copy()
    instance_config_dict.update(config_overrides or {})
    if load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        temp_instance_config = NativeServiceConfig(
            service=service,
            cluster=cluster,
            instance=instance,
            config_dict=instance_config_dict,
            branch_dict=None,
            soa_dir=soa_dir,
        )
        branch = temp_instance_config.get_branch()
        deploy_group = temp_instance_config.get_deploy_group()
        branch_dict = deployments_json.get_branch_dict(service, branch,
                                                       deploy_group)

    service_config = NativeServiceConfig(
        service=service,
        cluster=cluster,
        instance=instance,
        config_dict=instance_config_dict,
        branch_dict=branch_dict,
        soa_dir=soa_dir,
    )

    service_namespace_config = load_service_namespace_config(
        service=service,
        namespace=service_config.get_nerve_namespace(),
        soa_dir=soa_dir,
    )
    service_config.service_namespace_config = service_namespace_config

    return service_config
def get_default_interactive_config(
        service: str,
        cluster: str,
        soa_dir: str,
        load_deployments: bool = False) -> AdhocJobConfig:
    default_job_config = {"cpus": 4, "mem": 10240, "disk": 1024}

    try:
        job_config = load_adhoc_job_config(service=service,
                                           instance="interactive",
                                           cluster=cluster,
                                           soa_dir=soa_dir)
    except NoConfigurationForServiceError:
        job_config = AdhocJobConfig(
            service=service,
            instance="interactive",
            cluster=cluster,
            config_dict={},
            branch_dict=None,
            soa_dir=soa_dir,
        )
    except NoDeploymentsAvailable:
        job_config = load_adhoc_job_config(
            service=service,
            instance="interactive",
            cluster=cluster,
            soa_dir=soa_dir,
            load_deployments=False,
        )

    if not job_config.branch_dict and load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        deploy_group = prompt_pick_one(deployments_json.get_deploy_groups(),
                                       choosing="deploy group")
        job_config.config_dict["deploy_group"] = deploy_group
        job_config.branch_dict = {
            "docker_image":
            deployments_json.get_docker_image_for_deploy_group(deploy_group),
            "git_sha":
            deployments_json.get_git_sha_for_deploy_group(deploy_group),
            "force_bounce":
            None,
            "desired_state":
            "start",
        }

    for key, value in default_job_config.items():
        job_config.config_dict.setdefault(key, value)

    return job_config
示例#7
0
def load_chronos_job_config(
    service: str,
    instance: str,
    cluster: str,
    load_deployments: bool = True,
    soa_dir: str = DEFAULT_SOA_DIR,
) -> 'ChronosJobConfig':
    general_config = service_configuration_lib.read_service_configuration(
        service,
        soa_dir=soa_dir,
    )

    if instance.startswith('_'):
        raise InvalidJobNameError(
            "Unable to load chronos job config for %s.%s as instance name starts with '_'"
            % (service, instance), )
    service_chronos_jobs = read_chronos_jobs_for_service(service,
                                                         cluster,
                                                         soa_dir=soa_dir)
    if instance not in service_chronos_jobs:
        raise NoConfigurationForServiceError(
            'No job named "%s" in config file chronos-%s.yaml' %
            (instance, cluster))
    branch_dict = None
    general_config = deep_merge_dictionaries(
        overrides=service_chronos_jobs[instance], defaults=general_config)

    if load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        temp_instance_config = ChronosJobConfig(
            service=service,
            cluster=cluster,
            instance=instance,
            config_dict=general_config,
            branch_dict=None,
            soa_dir=soa_dir,
        )
        branch = temp_instance_config.get_branch()
        deploy_group = temp_instance_config.get_deploy_group()
        branch_dict = deployments_json.get_branch_dict(service, branch,
                                                       deploy_group)

    return ChronosJobConfig(
        service=service,
        cluster=cluster,
        instance=instance,
        config_dict=general_config,
        branch_dict=branch_dict,
        soa_dir=soa_dir,
    )
示例#8
0
def get_default_interactive_config(service,
                                   cluster,
                                   soa_dir,
                                   load_deployments=False):
    default_job_config = {
        'cpus': 4,
        'mem': 10240,
        'disk': 1024,
    }

    try:
        job_config = load_adhoc_job_config(service=service,
                                           instance='interactive',
                                           cluster=cluster,
                                           soa_dir=soa_dir)
    except NoConfigurationForServiceError:
        job_config = AdhocJobConfig(
            service=service,
            instance='interactive',
            cluster=cluster,
            config_dict={},
            branch_dict={},
            soa_dir=soa_dir,
        )
    except NoDeploymentsAvailable:
        job_config = load_adhoc_job_config(
            service=service,
            instance='interactive',
            cluster=cluster,
            soa_dir=soa_dir,
            load_deployments=False,
        )

    if not job_config.branch_dict and load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        deploy_group = prompt_pick_one(
            (deployment
             for deployment in deployments_json['deployments'].keys()),
            choosing='deploy group',
        )
        job_config.config_dict['deploy_group'] = deploy_group
        job_config.branch_dict[
            'docker_image'] = deployments_json.get_docker_image_for_deploy_group(
                deploy_group)

    for key, value in default_job_config.items():
        job_config.config_dict.setdefault(key, value)

    return job_config
示例#9
0
def load_adhoc_job_config(service,
                          instance,
                          cluster,
                          load_deployments=True,
                          soa_dir=DEFAULT_SOA_DIR):
    general_config = service_configuration_lib.read_service_configuration(
        service,
        soa_dir=soa_dir,
    )
    adhoc_conf_file = "adhoc-%s" % cluster
    instance_configs = service_configuration_lib.read_extra_service_information(
        service_name=service,
        extra_info=adhoc_conf_file,
        soa_dir=soa_dir,
    )

    if instance not in instance_configs:
        raise NoConfigurationForServiceError(
            "%s not found in config file %s/%s/%s.yaml." %
            (instance, soa_dir, service, adhoc_conf_file), )

    general_config = deep_merge_dictionaries(
        overrides=instance_configs[instance], defaults=general_config)

    branch_dict = None
    if load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        temp_instance_config = AdhocJobConfig(
            service=service,
            cluster=cluster,
            instance=instance,
            config_dict=general_config,
            branch_dict=None,
            soa_dir=soa_dir,
        )
        branch = temp_instance_config.get_branch()
        deploy_group = temp_instance_config.get_deploy_group()
        branch_dict = deployments_json.get_branch_dict(service, branch,
                                                       deploy_group)

    return AdhocJobConfig(
        service=service,
        cluster=cluster,
        instance=instance,
        config_dict=general_config,
        branch_dict=branch_dict,
        soa_dir=soa_dir,
    )
示例#10
0
    def _get_action_config(self, action_dict, default_paasta_cluster):
        action_service = action_dict.setdefault('service', self.get_service())
        action_deploy_group = action_dict.setdefault('deploy_group',
                                                     self.get_deploy_group())
        if action_service and action_deploy_group and self.load_deployments:
            try:
                deployments_json = load_v2_deployments_json(
                    service=action_service, soa_dir=self.soa_dir)
                branch_dict = {
                    'docker_image':
                    deployments_json.get_docker_image_for_deploy_group(
                        action_deploy_group),
                    'git_sha':
                    deployments_json.get_git_sha_for_deploy_group(
                        action_deploy_group),
                    # TODO: add Tron instances when generating deployments json
                    'desired_state':
                    'start',
                    'force_bounce':
                    None,
                }
            except NoDeploymentsAvailable:
                raise InvalidTronConfig(
                    'No deployment found for action {action} in job {job}, looking for {deploy_group} '
                    'in service {service}'.format(
                        action=action_dict.get('name'),
                        job=self.get_name(),
                        deploy_group=action_deploy_group,
                        service=action_service,
                    ), )
        else:
            branch_dict = None

        cluster = action_dict.get(
            'cluster') or self.get_cluster() or default_paasta_cluster

        return TronActionConfig(
            service=action_service,
            instance=compose_instance(self.get_name(),
                                      action_dict.get('name')),
            cluster=cluster,
            config_dict=action_dict,
            branch_dict=branch_dict,
            soa_dir=self.soa_dir,
        )
示例#11
0
文件: tron_tools.py 项目: Yelp/paasta
    def _get_action_config(self, action_name, action_dict):
        action_service = action_dict.setdefault("service", self.get_service())
        action_deploy_group = action_dict.setdefault(
            "deploy_group", self.get_deploy_group()
        )
        if action_service and action_deploy_group and self.load_deployments:
            try:
                deployments_json = load_v2_deployments_json(
                    service=action_service, soa_dir=self.soa_dir
                )
                branch_dict = {
                    "docker_image": deployments_json.get_docker_image_for_deploy_group(
                        action_deploy_group
                    ),
                    "git_sha": deployments_json.get_git_sha_for_deploy_group(
                        action_deploy_group
                    ),
                    "image_version": deployments_json.get_image_version_for_deploy_group(
                        action_deploy_group
                    ),
                    # TODO: add Tron instances when generating deployments json
                    "desired_state": "start",
                    "force_bounce": None,
                }
            except NoDeploymentsAvailable:
                log.warning(
                    f'Docker image unavailable for {action_service}.{self.get_name()}.{action_dict.get("name")}'
                    " is it deployed yet?"
                )
                branch_dict = None
        else:
            branch_dict = None
        action_dict["monitoring"] = self.get_monitoring()

        return TronActionConfig(
            service=action_service,
            instance=compose_instance(self.get_name(), action_name),
            cluster=self.get_cluster(),
            config_dict=action_dict,
            branch_dict=branch_dict,
            soa_dir=self.soa_dir,
            for_validation=self.for_validation,
        )
示例#12
0
def load_vitess_instance_config(
    service: str,
    instance: str,
    cluster: str,
    load_deployments: bool = True,
    soa_dir: str = DEFAULT_SOA_DIR,
) -> VitessDeploymentConfig:
    general_config = service_configuration_lib.read_service_configuration(
        service, soa_dir=soa_dir)
    instance_config = load_service_instance_config(service,
                                                   instance,
                                                   " vitesscluster",
                                                   cluster,
                                                   soa_dir=soa_dir)
    general_config = deep_merge_dictionaries(overrides=instance_config,
                                             defaults=general_config)

    branch_dict: Optional[BranchDictV2] = None
    if load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        temp_instance_config = VitessDeploymentConfig(
            service=service,
            cluster=cluster,
            instance=instance,
            config_dict=general_config,
            branch_dict=None,
            soa_dir=soa_dir,
        )
        branch = temp_instance_config.get_branch()
        deploy_group = temp_instance_config.get_deploy_group()
        branch_dict = deployments_json.get_branch_dict(service, branch,
                                                       deploy_group)

    return VitessDeploymentConfig(
        service=service,
        cluster=cluster,
        instance=instance,
        config_dict=general_config,
        branch_dict=branch_dict,
        soa_dir=soa_dir,
    )
示例#13
0
def load_adhoc_job_config(
    service, instance, cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR
):
    general_config = service_configuration_lib.read_service_configuration(
        service, soa_dir=soa_dir
    )
    instance_config = load_service_instance_config(
        service=service,
        instance=instance,
        instance_type="adhoc",
        cluster=cluster,
        soa_dir=soa_dir,
    )
    general_config = deep_merge_dictionaries(
        overrides=instance_config, defaults=general_config
    )

    branch_dict = None
    if load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        temp_instance_config = AdhocJobConfig(
            service=service,
            cluster=cluster,
            instance=instance,
            config_dict=general_config,
            branch_dict=None,
            soa_dir=soa_dir,
        )
        branch = temp_instance_config.get_branch()
        deploy_group = temp_instance_config.get_deploy_group()
        branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group)

    return AdhocJobConfig(
        service=service,
        cluster=cluster,
        instance=instance,
        config_dict=general_config,
        branch_dict=branch_dict,
        soa_dir=soa_dir,
    )
示例#14
0
def get_default_interactive_config(service, cluster, soa_dir, load_deployments=False):
    default_job_config = {
        'cpus': 4,
        'mem': 10240,
        'disk': 1024
    }

    try:
        job_config = load_adhoc_job_config(service=service, instance='interactive', cluster=cluster, soa_dir=soa_dir)
    except NoConfigurationForServiceError:
        job_config = AdhocJobConfig(
            service=service,
            instance='interactive',
            cluster=cluster,
            config_dict={},
            branch_dict={},
        )
    except NoDeploymentsAvailable:
        job_config = load_adhoc_job_config(
            service=service, instance='interactive', cluster=cluster, soa_dir=soa_dir, load_deployments=False)

    if not job_config.branch_dict and load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        deploy_group = prompt_pick_one(
            (
                deployment.encode('utf-8')
                for deployment in deployments_json['deployments'].keys()
            ),
            choosing='deploy group',
        )
        job_config.config_dict['deploy_group'] = deploy_group
        job_config.branch_dict['docker_image'] = deployments_json.get_docker_image_for_deploy_group(deploy_group)

    for key, value in default_job_config.items():
        job_config.config_dict.setdefault(key, value)

    return job_config
示例#15
0
def load_adhoc_job_config(service,
                          instance,
                          cluster,
                          load_deployments=True,
                          soa_dir=DEFAULT_SOA_DIR):
    general_config = service_configuration_lib.read_service_configuration(
        service, soa_dir=soa_dir)
    adhoc_conf_file = "adhoc-%s" % cluster
    log.info("Reading adhoc configuration file: %s.yaml", adhoc_conf_file)
    instance_configs = service_configuration_lib.read_extra_service_information(
        service_name=service, extra_info=adhoc_conf_file, soa_dir=soa_dir)

    if instance not in instance_configs:
        raise NoConfigurationForServiceError(
            "%s not found in config file %s/%s/%s.yaml." %
            (instance, soa_dir, service, adhoc_conf_file))

    general_config = deep_merge_dictionaries(
        overrides=instance_configs[instance], defaults=general_config)

    branch_dict = {}
    if load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        branch = general_config.get('branch',
                                    get_paasta_branch(cluster, instance))
        deploy_group = general_config.get('deploy_group', branch)
        branch_dict = deployments_json.get_branch_dict_v2(
            service, branch, deploy_group)

    return AdhocJobConfig(
        service=service,
        cluster=cluster,
        instance=instance,
        config_dict=general_config,
        branch_dict=branch_dict,
        soa_dir=soa_dir,
    )
示例#16
0
def get_default_interactive_config(service, cluster, soa_dir, load_deployments=False):
    default_job_config = {"cpus": 4, "mem": 10240, "disk": 1024}

    try:
        job_config = load_adhoc_job_config(service=service, instance="interactive", cluster=cluster, soa_dir=soa_dir)
    except NoConfigurationForServiceError:
        job_config = AdhocJobConfig(
            service=service, instance="interactive", cluster=cluster, config_dict={}, branch_dict={}
        )
    except NoDeploymentsAvailable:
        job_config = load_adhoc_job_config(
            service=service, instance="interactive", cluster=cluster, soa_dir=soa_dir, load_deployments=False
        )

    if not job_config.branch_dict and load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        deploy_group = prompt_pick_one(deployments_json["deployments"].keys(), choosing="deploy group")
        job_config.config_dict["deploy_group"] = deploy_group
        job_config.branch_dict["docker_image"] = deployments_json.get_docker_image_for_deploy_group(deploy_group)

    for key, value in default_job_config.items():
        job_config.config_dict.setdefault(key, value)

    return job_config
示例#17
0
def load_kubernetes_service_config_no_cache(
    service: str,
    instance: str,
    cluster: str,
    load_deployments: bool = True,
    soa_dir: str = DEFAULT_SOA_DIR,
) -> "KubernetesDeploymentConfig":
    """Read a service instance's configuration for kubernetes.

    If a branch isn't specified for a config, the 'branch' key defaults to
    paasta-${cluster}.${instance}.

    :param name: The service name
    :param instance: The instance of the service to retrieve
    :param cluster: The cluster to read the configuration for
    :param load_deployments: A boolean indicating if the corresponding deployments.json for this service
                             should also be loaded
    :param soa_dir: The SOA configuration directory to read from
    :returns: A dictionary of whatever was in the config for the service instance"""
    general_config = service_configuration_lib.read_service_configuration(
        service,
        soa_dir=soa_dir,
    )
    kubernetes_conf_file = "kubernetes-%s" % cluster
    instance_configs = service_configuration_lib.read_extra_service_information(
        service,
        kubernetes_conf_file,
        soa_dir=soa_dir,
    )

    if instance.startswith('_'):
        raise InvalidJobNameError(
            f"Unable to load kubernetes job config for {service}.{instance} as instance name starts with '_'",
        )
    if instance not in instance_configs:
        raise NoConfigurationForServiceError(
            f"{instance} not found in config file {soa_dir}/{service}/{kubernetes_conf_file}.yaml.",
        )

    general_config = deep_merge_dictionaries(
        overrides=instance_configs[instance], defaults=general_config)

    branch_dict: Optional[BranchDictV2] = None
    if load_deployments:
        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
        temp_instance_config = KubernetesDeploymentConfig(
            service=service,
            cluster=cluster,
            instance=instance,
            config_dict=general_config,
            branch_dict=None,
            soa_dir=soa_dir,
        )
        branch = temp_instance_config.get_branch()
        deploy_group = temp_instance_config.get_deploy_group()
        branch_dict = deployments_json.get_branch_dict(service, branch,
                                                       deploy_group)

    return KubernetesDeploymentConfig(
        service=service,
        cluster=cluster,
        instance=instance,
        config_dict=general_config,
        branch_dict=branch_dict,
        soa_dir=soa_dir,
    )