Ejemplo n.º 1
0
    def get_jinja_vars(self):
        # order for per-project variables (each overrides the previous):
        # 1. /etc/kolla/globals.yml and passwords.yml
        # 2. config/all.yml
        # 3. config/<project>/defaults/main.yml
        with open(file_utils.find_config_file('passwords.yml'), 'r') as gf:
            global_vars = yaml.load(gf)
        with open(file_utils.find_config_file('globals.yml'), 'r') as gf:
            global_vars.update(yaml.load(gf))

        all_yml_name = os.path.join(self.config_dir, 'all.yml')
        jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, global_vars))
        jvars.update(global_vars)

        for proj in self.get_projects():
            proj_yml_name = os.path.join(self.config_dir, proj,
                                         'defaults', 'main.yml')
            if os.path.exists(proj_yml_name):
                proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name,
                                                               jvars))

                jvars.update(proj_vars)
            else:
                LOG.warn('path missing %s' % proj_yml_name)

        # override node_config_directory to empty
        jvars.update({'node_config_directory': ''})
        return jvars
Ejemplo n.º 2
0
    def _write_common_config_to_zookeeper(self, zk, jinja_vars):
        # 1. At first write global tools to ZK. FIXME: Make it a common profile
        conf_path = os.path.join(self.config_dir, 'common',
                                 'common_config.yml.j2')
        common_cfg = yaml.load(jinja_utils.jinja_render(conf_path, jinja_vars))
        common_node = os.path.join('kolla', 'common')
        for script in common_cfg:
            script_node = os.path.join(common_node, script)
            zk.ensure_path(script_node)
            source_path = common_cfg[script]['source']

            src_file = source_path
            if not source_path.startswith('/'):
                src_file = file_utils.find_file(source_path)
            with open(src_file) as fp:
                content = fp.read()
            zk.set(script_node, content)

        # 2. Add startup config
        start_conf = os.path.join(self.config_dir,
                                  'common/kolla-start-config.json')
        # override container_config_directory
        cont_conf_dir = 'zk://%s' % (CONF.zookeeper.host)
        jinja_vars['container_config_directory'] = cont_conf_dir
        jinja_vars['deployment_id'] = self.deployment_id
        kolla_config = jinja_utils.jinja_render(start_conf, jinja_vars)
        kolla_config = kolla_config.replace('"', '\\"').replace('\n', '')

        return kolla_config
Ejemplo n.º 3
0
def _load_variables_from_file(service_dir, project_name):
    config_dir = os.path.join(service_dir, '..', 'config')
    with open(file_utils.find_config_file('passwords.yml'), 'r') as gf:
        global_vars = yaml.load(gf)
    with open(file_utils.find_config_file('globals.yml'), 'r') as gf:
        global_vars.update(yaml.load(gf))
    # all.yml file uses some its variables to template itself by jinja2,
    # so its raw content is used to template the file
    all_yml_name = os.path.join(config_dir, 'all.yml')
    with open(all_yml_name) as af:
        raw_vars = yaml.load(af)
    raw_vars.update(global_vars)
    jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, raw_vars))
    jvars.update(global_vars)

    proj_yml_name = os.path.join(config_dir, project_name,
                                 'defaults', 'main.yml')
    if os.path.exists(proj_yml_name):
        proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name,
                                                       jvars))
        jvars.update(proj_vars)
    else:
        LOG.warning('Path missing %s' % proj_yml_name)
    # Add deployment_id
    jvars.update({'deployment_id': CONF.kolla.deployment_id})
    # override node_config_directory to empty
    jvars.update({'node_config_directory': ''})
    # Add timestamp
    jvars.update({'timestamp': str(time.time())})
    config.apply_deployment_vars(jvars)
    config.get_marathon_framework(jvars)
    return jvars
Ejemplo n.º 4
0
    def generate_deployment_files(self, kolla_config, jinja_vars,
                                  temp_dir=None):
        _, proj, service = self._conf['name'].split('/')
        values = {
            'service_name': self._conf['name'],
            'chronos_service_id': self._conf['name'].replace('/', '-'),
            'kolla_config': kolla_config,
            'zookeeper_hosts': CONF.zookeeper.host,
            'private_interface': CONF.network.private_interface,
            'public_interface': CONF.network.public_interface,
        }

        app_file = os.path.join(self.base_dir, 'services',
                                'default.%s.j2' % self.type_name)
        content = jinja_utils.jinja_render(app_file, jinja_vars,
                                           extra=values)
        self.app_def = yaml.load(content)
        self._apply_service_def(self.app_def)
        if temp_dir is not None:
            self.app_file = os.path.join(temp_dir, proj,
                                         '%s.%s' % (service, self.type_name))
            file_utils.mkdir_p(os.path.dirname(self.app_file))
            LOG.info(self.app_file)
            with open(self.app_file, 'w') as f:
                f.write(json.dumps(self.app_def, indent=2))
Ejemplo n.º 5
0
 def write_openrc(self):
     # write an openrc to the base_dir for convience.
     openrc_file = os.path.join(self.base_dir, 'config', 'openrc.j2')
     content = jinja_utils.jinja_render(openrc_file, self.required_vars)
     with open('openrc', 'w') as f:
         f.write(content)
     LOG.info('Written OpenStack env to "openrc"')
Ejemplo n.º 6
0
    def generate_deployment_files(self, kolla_config, jinja_vars, temp_dir):
        if not self._enabled:
            return
        _, proj, service = self._conf['name'].split('/')
        values = {
            'role': service,
            'group': proj,
            'service_name': self._get_service_name(),
            'kolla_config': kolla_config,
            'zookeeper_hosts': CONF.zookeeper.host,
            'private_interface': CONF.network.private_interface,
            'public_interface': CONF.network.public_interface,
        }

        app_file = os.path.join(self.base_dir, 'services',
                                'default.%s.j2' % self.type_name)
        content = jinja_utils.jinja_render(app_file, jinja_vars,
                                           extra=values)
        app_def = yaml.load(content)
        self._apply_service_def(app_def)
        dest_file = os.path.join(temp_dir, proj,
                                 '%s.%s' % (service, self.type_name))
        file_utils.mkdir_p(os.path.dirname(dest_file))
        LOG.info(dest_file)
        with open(dest_file, 'w') as f:
            f.write(json.dumps(app_def, indent=2))
Ejemplo n.º 7
0
def get_start_config(config_dir, jinja_vars):
    start_conf = os.path.join(config_dir,
                              'common/kolla-start-config.json')
    # override container_config_directory
    cont_conf_dir = 'zk://%s' % (CONF.zookeeper.host)
    jinja_vars['container_config_directory'] = cont_conf_dir
    jinja_vars['deployment_id'] = CONF.kolla.deployment_id
    kolla_config = jinja_utils.jinja_render(start_conf, jinja_vars)
    kolla_config = kolla_config.replace('"', '\\"').replace('\n', '')
    return kolla_config
Ejemplo n.º 8
0
 def process_service_config(self, zk, proj, conf_path,
                            jinja_vars, kolla_config):
     conf = yaml.load(jinja_utils.jinja_render(conf_path, jinja_vars))
     if 'service' in conf:
         runner = MarathonApp(conf)
     else:
         runner = ChronosTask(conf)
     base_node = os.path.join('kolla', self.deployment_id)
     runner.write_to_zookeeper(zk, base_node)
     runner.generate_deployment_files(kolla_config, jinja_vars,
                                      self.temp_dir)
Ejemplo n.º 9
0
def get_tasks(deploy_id):
    """Get list of tasks

    Reads through all the kolla mesos services config files and
    parses the requirements and resister options.

    Returns a dictionary of all the values registered by tasks
    {
    taskpath1: {
      'requires': [require1, require2...]
      'register': register_path
      }
    }

    taskpath examples -
      'keystone/keystone/db_sync',
      'keystone/keystone_ansible_tasks/create_database',
    """

    def get_task_from_cmd(role, cmd, cmd_info):
        reg = '/kolla/%s/status/%s/%s/.done' % (deploy_id, role, cmd)
        task = {'register': reg, 'requires': []}
        for dep in cmd_info.get('dependencies', []):
            task['requires'].append(
                '/kolla/%s/status/%s/.done' % (deploy_id, dep))
        return task

    tasks = {}
    config_dir = os.path.join(file_utils.find_base_dir(), 'services')
    for root, _, files in os.walk(config_dir):
        for name in files:
            if 'default.' in name:
                continue
            fpath = os.path.join(root, name)
            mini_vars = {'cinder_volume_driver': 'lvm',
                         'deployment_id': deploy_id}
            cfg = yaml.load(jinja_utils.jinja_render(fpath, mini_vars))

            def get_commands():
                for cmd in cfg.get('commands', {}):
                    yield cmd, cfg['commands'][cmd]
                if 'service' in cfg:
                    yield 'daemon', cfg['service']['daemon']

            _, group, role = cfg['name'].split('/')
            for cmd, cmd_info in get_commands():
                task_name = '/%s/%s/%s' % (group, role, cmd)
                tasks[task_name] = get_task_from_cmd(role, cmd,
                                                     cmd_info)
    return tasks
Ejemplo n.º 10
0
def validate(filename, deps):
    mini_vars = {'cinder_volume_driver': 'lvm',
                 'deployment_id': 'test'}
    cnf = yaml.load(jinja_utils.jinja_render(filename, mini_vars))

    def get_commands():
        for cmd in cnf.get('commands', {}):
            yield cmd, cnf['commands'][cmd]
        if 'service' in cnf:
            yield 'daemon', cnf['service']['daemon']

    _, group, role = cnf['name'].split('/')

    for cmd, cmd_info in get_commands():
        validate_command(filename, cmd, cmd_info, deps, role)
Ejemplo n.º 11
0
    def get_jinja_vars(self):
        # order for per-project variables (each overrides the previous):
        # 1. /etc/kolla/globals.yml and passwords.yml
        # 2. config/all.yml
        # 3. config/<project>/defaults/main.yml
        with open(file_utils.find_config_file('passwords.yml'), 'r') as gf:
            global_vars = yaml.load(gf)
        with open(file_utils.find_config_file('globals.yml'), 'r') as gf:
            global_vars.update(yaml.load(gf))

        # all.yml file uses some its variables to template itself by jinja2,
        # so its raw content is used to template the file
        all_yml_name = os.path.join(self.config_dir, 'all.yml')
        with open(all_yml_name) as af:
            raw_vars = yaml.load(af)
        raw_vars.update(global_vars)
        jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, raw_vars))

        jvars.update(global_vars)

        for proj in self.get_projects():
            proj_yml_name = os.path.join(self.config_dir, proj,
                                         'defaults', 'main.yml')
            if os.path.exists(proj_yml_name):
                proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name,
                                                               jvars))

                jvars.update(proj_vars)
            else:
                LOG.warning('Path missing %s' % proj_yml_name)

        # Add deployment_id
        jvars.update({'deployment_id': self.deployment_id})
        # override node_config_directory to empty
        jvars.update({'node_config_directory': ''})
        return jvars
Ejemplo n.º 12
0
def write_common_config_to_zookeeper(config_dir, zk, jinja_vars,
                                     overwrite=True):
    # 1. At first write global tools to ZK. FIXME: Make it a common profile
    conf_path = os.path.join(config_dir, 'common',
                             'common_config.yml.j2')
    common_cfg = yaml.load(jinja_utils.jinja_render(conf_path, jinja_vars))
    common_node = os.path.join('kolla', 'common')
    for script in common_cfg:
        script_node = os.path.join(common_node, script)
        if not overwrite and zk.exists(script_node):
            LOG.debug('NOT Updating "%s" node in zookeeper(overwrite=False).',
                      script_node)
            continue

        zk.ensure_path(script_node)
        source_path = common_cfg[script]['source']
        src_file = source_path
        if not source_path.startswith('/'):
            src_file = file_utils.find_file(source_path)
        with open(src_file) as fp:
            content = fp.read()
        zk.set(script_node, content.encode('utf-8'))
Ejemplo n.º 13
0
def _get_config_tasks(config_path, tasks):
    controller_nodes, compute_nodes, storage_nodes, all_nodes = \
        mesos_utils.get_number_of_nodes()
    mini_vars = {'cinder_volume_driver': 'lvm',
                 'deployment_id': CONF.kolla.deployment_id,
                 'controller_nodes': str(controller_nodes),
                 'compute_nodes': str(compute_nodes),
                 'storage_nodes': str(storage_nodes),
                 'all_nodes': str(all_nodes)}
    config = yaml.load(jinja_utils.jinja_render(config_path, mini_vars))

    def get_commands():
        for cmd in config.get('commands', {}):
            yield cmd, config['commands'][cmd]
        if 'service' in config:
            yield 'daemon', config['service']['daemon']

    _, _, role = config['name'].split('/')
    for cmd, cmd_info in get_commands():
        task_name = '%s/%s' % (role, cmd)
        tasks[task_name] = _get_task_from_cmd(role, cmd, cmd_info)

    return tasks
Ejemplo n.º 14
0
def validate(service_name, service_dir, variables=None, deps=None):
    if variables is None:
        variables = {}
    if deps is None:
        deps = {}

    filename = find_service_file(service_name, service_dir)
    try:
        cnf = yaml.load(jinja_utils.jinja_render(filename, variables))
    except jinja2.exceptions.TemplateNotFound:
        raise exception.KollaNotFoundException(filename,
                                               entity='service definition')

    def get_commands():
        for cmd in cnf.get('commands', {}):
            yield cmd, cnf['commands'][cmd]
        if 'service' in cnf:
            yield 'daemon', cnf['service']['daemon']

    LOG.debug('%s: file found at %s' % (cnf['name'], filename))
    for cmd, cmd_info in get_commands():
        _validate_command(filename, cmd, cmd_info, deps,
                          cnf['name'], service_dir)
    return deps
Ejemplo n.º 15
0
    def write_config_to_zookeeper(self, zk):
        jinja_vars = self.get_jinja_vars()
        self.required_vars = jinja_vars

        for var in jinja_vars:
            if not jinja_vars[var]:
                LOG.info('empty %s=%s' % (var, jinja_vars[var]))
            if 'image' in var:
                LOG.info('%s=%s' % (var, jinja_vars[var]))

        for proj in self.get_projects():
            proj_dir = os.path.join(self.config_dir, proj)
            if not os.path.exists(proj_dir):
                continue

            conf_path = os.path.join(self.config_dir, proj,
                                     '%s_config.yml.j2' % proj)
            extra = yaml.load(jinja_utils.jinja_render(conf_path, jinja_vars))

            dest_node = os.path.join('kolla', 'config',
                                     proj, proj)  # TODO() should be service
            zk.ensure_path(dest_node)
            zk.set(dest_node, json.dumps(extra))

            for service in extra['config'][proj]:
                # write the config files
                for name, item in extra['config'][proj][service].iteritems():
                    dest_node = os.path.join('kolla', 'config',
                                             proj, service, name)
                    zk.ensure_path(dest_node)

                    if isinstance(item['source'], list):
                        content = self.merge_ini_files(item['source'])
                    else:
                        src_file = item['source']
                        if not src_file.startswith('/'):
                            src_file = file_utils.find_file(src_file)
                        with open(src_file) as fp:
                            content = fp.read()
                    zk.set(dest_node, content)

                # write the commands
                for name, item in extra['commands'][proj][service].iteritems():
                    dest_node = os.path.join('kolla', 'commands',
                                             proj, service, name)
                    zk.ensure_path(dest_node)
                    try:
                        zk.set(dest_node, json.dumps(item))
                    except Exception as te:
                        LOG.error('%s=%s -> %s' % (dest_node,
                                                   item, te))

                # 3. do the service's config.json (now KOLLA_CONFIG)
                kc_name = os.path.join(self.config_dir, 'config.json')
                # override container_config_directory
                cont_conf_dir = 'zk://%s' % (
                    CONF.zookeeper.host)
                jinja_vars['container_config_directory'] = cont_conf_dir
                kolla_config = jinja_utils.jinja_render(kc_name, jinja_vars)
                kolla_config = kolla_config.replace('"', '\\"').replace(
                    '\n', '')

                # 4. parse the marathon app file and add the KOLLA_CONFIG
                values = {
                    'kolla_config': kolla_config,
                    'zookeeper_hosts': CONF.zookeeper.host
                }
                for app_type in ['marathon', 'chronos']:
                    app_file = os.path.join(self.base_dir,
                                            'deployment_files',
                                            proj,
                                            '%s.%s.j2' % (service, app_type))
                    if not os.path.exists(app_file):
                        LOG.debug('potentially missing file %s' % app_file)
                        continue
                    content = jinja_utils.jinja_render(app_file, jinja_vars,
                                                       extra=values)
                    dest_file = os.path.join(self.temp_dir, proj,
                                             '%s.%s' % (service, app_type))
                    file_utils.mkdir_p(os.path.dirname(dest_file))
                    with open(dest_file, 'w') as f:
                        f.write(content)
Ejemplo n.º 16
0
 def load_from_file(cls, service_file, variables):
     return Runner(yaml.load(
                   jinja_utils.jinja_render(service_file, variables)))