def get_jinja_vars(self): # order for per-project variables (each overrides the previous): # 1. /etc/kolla/globals.yml and passwords.yml # 2. config/all.yml # 3. config/<project>/defaults/main.yml with open(file_utils.find_config_file('passwords.yml'), 'r') as gf: global_vars = yaml.load(gf) with open(file_utils.find_config_file('globals.yml'), 'r') as gf: global_vars.update(yaml.load(gf)) all_yml_name = os.path.join(self.config_dir, 'all.yml') jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, global_vars)) jvars.update(global_vars) for proj in self.get_projects(): proj_yml_name = os.path.join(self.config_dir, proj, 'defaults', 'main.yml') if os.path.exists(proj_yml_name): proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name, jvars)) jvars.update(proj_vars) else: LOG.warn('path missing %s' % proj_yml_name) # override node_config_directory to empty jvars.update({'node_config_directory': ''}) return jvars
def _write_common_config_to_zookeeper(self, zk, jinja_vars): # 1. At first write global tools to ZK. FIXME: Make it a common profile conf_path = os.path.join(self.config_dir, 'common', 'common_config.yml.j2') common_cfg = yaml.load(jinja_utils.jinja_render(conf_path, jinja_vars)) common_node = os.path.join('kolla', 'common') for script in common_cfg: script_node = os.path.join(common_node, script) zk.ensure_path(script_node) source_path = common_cfg[script]['source'] src_file = source_path if not source_path.startswith('/'): src_file = file_utils.find_file(source_path) with open(src_file) as fp: content = fp.read() zk.set(script_node, content) # 2. Add startup config start_conf = os.path.join(self.config_dir, 'common/kolla-start-config.json') # override container_config_directory cont_conf_dir = 'zk://%s' % (CONF.zookeeper.host) jinja_vars['container_config_directory'] = cont_conf_dir jinja_vars['deployment_id'] = self.deployment_id kolla_config = jinja_utils.jinja_render(start_conf, jinja_vars) kolla_config = kolla_config.replace('"', '\\"').replace('\n', '') return kolla_config
def _load_variables_from_file(service_dir, project_name): config_dir = os.path.join(service_dir, '..', 'config') with open(file_utils.find_config_file('passwords.yml'), 'r') as gf: global_vars = yaml.load(gf) with open(file_utils.find_config_file('globals.yml'), 'r') as gf: global_vars.update(yaml.load(gf)) # all.yml file uses some its variables to template itself by jinja2, # so its raw content is used to template the file all_yml_name = os.path.join(config_dir, 'all.yml') with open(all_yml_name) as af: raw_vars = yaml.load(af) raw_vars.update(global_vars) jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, raw_vars)) jvars.update(global_vars) proj_yml_name = os.path.join(config_dir, project_name, 'defaults', 'main.yml') if os.path.exists(proj_yml_name): proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name, jvars)) jvars.update(proj_vars) else: LOG.warning('Path missing %s' % proj_yml_name) # Add deployment_id jvars.update({'deployment_id': CONF.kolla.deployment_id}) # override node_config_directory to empty jvars.update({'node_config_directory': ''}) # Add timestamp jvars.update({'timestamp': str(time.time())}) config.apply_deployment_vars(jvars) config.get_marathon_framework(jvars) return jvars
def generate_deployment_files(self, kolla_config, jinja_vars, temp_dir=None): _, proj, service = self._conf['name'].split('/') values = { 'service_name': self._conf['name'], 'chronos_service_id': self._conf['name'].replace('/', '-'), 'kolla_config': kolla_config, 'zookeeper_hosts': CONF.zookeeper.host, 'private_interface': CONF.network.private_interface, 'public_interface': CONF.network.public_interface, } app_file = os.path.join(self.base_dir, 'services', 'default.%s.j2' % self.type_name) content = jinja_utils.jinja_render(app_file, jinja_vars, extra=values) self.app_def = yaml.load(content) self._apply_service_def(self.app_def) if temp_dir is not None: self.app_file = os.path.join(temp_dir, proj, '%s.%s' % (service, self.type_name)) file_utils.mkdir_p(os.path.dirname(self.app_file)) LOG.info(self.app_file) with open(self.app_file, 'w') as f: f.write(json.dumps(self.app_def, indent=2))
def write_openrc(self): # write an openrc to the base_dir for convience. openrc_file = os.path.join(self.base_dir, 'config', 'openrc.j2') content = jinja_utils.jinja_render(openrc_file, self.required_vars) with open('openrc', 'w') as f: f.write(content) LOG.info('Written OpenStack env to "openrc"')
def generate_deployment_files(self, kolla_config, jinja_vars, temp_dir): if not self._enabled: return _, proj, service = self._conf['name'].split('/') values = { 'role': service, 'group': proj, 'service_name': self._get_service_name(), 'kolla_config': kolla_config, 'zookeeper_hosts': CONF.zookeeper.host, 'private_interface': CONF.network.private_interface, 'public_interface': CONF.network.public_interface, } app_file = os.path.join(self.base_dir, 'services', 'default.%s.j2' % self.type_name) content = jinja_utils.jinja_render(app_file, jinja_vars, extra=values) app_def = yaml.load(content) self._apply_service_def(app_def) dest_file = os.path.join(temp_dir, proj, '%s.%s' % (service, self.type_name)) file_utils.mkdir_p(os.path.dirname(dest_file)) LOG.info(dest_file) with open(dest_file, 'w') as f: f.write(json.dumps(app_def, indent=2))
def get_start_config(config_dir, jinja_vars): start_conf = os.path.join(config_dir, 'common/kolla-start-config.json') # override container_config_directory cont_conf_dir = 'zk://%s' % (CONF.zookeeper.host) jinja_vars['container_config_directory'] = cont_conf_dir jinja_vars['deployment_id'] = CONF.kolla.deployment_id kolla_config = jinja_utils.jinja_render(start_conf, jinja_vars) kolla_config = kolla_config.replace('"', '\\"').replace('\n', '') return kolla_config
def process_service_config(self, zk, proj, conf_path, jinja_vars, kolla_config): conf = yaml.load(jinja_utils.jinja_render(conf_path, jinja_vars)) if 'service' in conf: runner = MarathonApp(conf) else: runner = ChronosTask(conf) base_node = os.path.join('kolla', self.deployment_id) runner.write_to_zookeeper(zk, base_node) runner.generate_deployment_files(kolla_config, jinja_vars, self.temp_dir)
def get_tasks(deploy_id): """Get list of tasks Reads through all the kolla mesos services config files and parses the requirements and resister options. Returns a dictionary of all the values registered by tasks { taskpath1: { 'requires': [require1, require2...] 'register': register_path } } taskpath examples - 'keystone/keystone/db_sync', 'keystone/keystone_ansible_tasks/create_database', """ def get_task_from_cmd(role, cmd, cmd_info): reg = '/kolla/%s/status/%s/%s/.done' % (deploy_id, role, cmd) task = {'register': reg, 'requires': []} for dep in cmd_info.get('dependencies', []): task['requires'].append( '/kolla/%s/status/%s/.done' % (deploy_id, dep)) return task tasks = {} config_dir = os.path.join(file_utils.find_base_dir(), 'services') for root, _, files in os.walk(config_dir): for name in files: if 'default.' in name: continue fpath = os.path.join(root, name) mini_vars = {'cinder_volume_driver': 'lvm', 'deployment_id': deploy_id} cfg = yaml.load(jinja_utils.jinja_render(fpath, mini_vars)) def get_commands(): for cmd in cfg.get('commands', {}): yield cmd, cfg['commands'][cmd] if 'service' in cfg: yield 'daemon', cfg['service']['daemon'] _, group, role = cfg['name'].split('/') for cmd, cmd_info in get_commands(): task_name = '/%s/%s/%s' % (group, role, cmd) tasks[task_name] = get_task_from_cmd(role, cmd, cmd_info) return tasks
def validate(filename, deps): mini_vars = {'cinder_volume_driver': 'lvm', 'deployment_id': 'test'} cnf = yaml.load(jinja_utils.jinja_render(filename, mini_vars)) def get_commands(): for cmd in cnf.get('commands', {}): yield cmd, cnf['commands'][cmd] if 'service' in cnf: yield 'daemon', cnf['service']['daemon'] _, group, role = cnf['name'].split('/') for cmd, cmd_info in get_commands(): validate_command(filename, cmd, cmd_info, deps, role)
def get_jinja_vars(self): # order for per-project variables (each overrides the previous): # 1. /etc/kolla/globals.yml and passwords.yml # 2. config/all.yml # 3. config/<project>/defaults/main.yml with open(file_utils.find_config_file('passwords.yml'), 'r') as gf: global_vars = yaml.load(gf) with open(file_utils.find_config_file('globals.yml'), 'r') as gf: global_vars.update(yaml.load(gf)) # all.yml file uses some its variables to template itself by jinja2, # so its raw content is used to template the file all_yml_name = os.path.join(self.config_dir, 'all.yml') with open(all_yml_name) as af: raw_vars = yaml.load(af) raw_vars.update(global_vars) jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, raw_vars)) jvars.update(global_vars) for proj in self.get_projects(): proj_yml_name = os.path.join(self.config_dir, proj, 'defaults', 'main.yml') if os.path.exists(proj_yml_name): proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name, jvars)) jvars.update(proj_vars) else: LOG.warning('Path missing %s' % proj_yml_name) # Add deployment_id jvars.update({'deployment_id': self.deployment_id}) # override node_config_directory to empty jvars.update({'node_config_directory': ''}) return jvars
def write_common_config_to_zookeeper(config_dir, zk, jinja_vars, overwrite=True): # 1. At first write global tools to ZK. FIXME: Make it a common profile conf_path = os.path.join(config_dir, 'common', 'common_config.yml.j2') common_cfg = yaml.load(jinja_utils.jinja_render(conf_path, jinja_vars)) common_node = os.path.join('kolla', 'common') for script in common_cfg: script_node = os.path.join(common_node, script) if not overwrite and zk.exists(script_node): LOG.debug('NOT Updating "%s" node in zookeeper(overwrite=False).', script_node) continue zk.ensure_path(script_node) source_path = common_cfg[script]['source'] src_file = source_path if not source_path.startswith('/'): src_file = file_utils.find_file(source_path) with open(src_file) as fp: content = fp.read() zk.set(script_node, content.encode('utf-8'))
def _get_config_tasks(config_path, tasks): controller_nodes, compute_nodes, storage_nodes, all_nodes = \ mesos_utils.get_number_of_nodes() mini_vars = {'cinder_volume_driver': 'lvm', 'deployment_id': CONF.kolla.deployment_id, 'controller_nodes': str(controller_nodes), 'compute_nodes': str(compute_nodes), 'storage_nodes': str(storage_nodes), 'all_nodes': str(all_nodes)} config = yaml.load(jinja_utils.jinja_render(config_path, mini_vars)) def get_commands(): for cmd in config.get('commands', {}): yield cmd, config['commands'][cmd] if 'service' in config: yield 'daemon', config['service']['daemon'] _, _, role = config['name'].split('/') for cmd, cmd_info in get_commands(): task_name = '%s/%s' % (role, cmd) tasks[task_name] = _get_task_from_cmd(role, cmd, cmd_info) return tasks
def validate(service_name, service_dir, variables=None, deps=None): if variables is None: variables = {} if deps is None: deps = {} filename = find_service_file(service_name, service_dir) try: cnf = yaml.load(jinja_utils.jinja_render(filename, variables)) except jinja2.exceptions.TemplateNotFound: raise exception.KollaNotFoundException(filename, entity='service definition') def get_commands(): for cmd in cnf.get('commands', {}): yield cmd, cnf['commands'][cmd] if 'service' in cnf: yield 'daemon', cnf['service']['daemon'] LOG.debug('%s: file found at %s' % (cnf['name'], filename)) for cmd, cmd_info in get_commands(): _validate_command(filename, cmd, cmd_info, deps, cnf['name'], service_dir) return deps
def write_config_to_zookeeper(self, zk): jinja_vars = self.get_jinja_vars() self.required_vars = jinja_vars for var in jinja_vars: if not jinja_vars[var]: LOG.info('empty %s=%s' % (var, jinja_vars[var])) if 'image' in var: LOG.info('%s=%s' % (var, jinja_vars[var])) for proj in self.get_projects(): proj_dir = os.path.join(self.config_dir, proj) if not os.path.exists(proj_dir): continue conf_path = os.path.join(self.config_dir, proj, '%s_config.yml.j2' % proj) extra = yaml.load(jinja_utils.jinja_render(conf_path, jinja_vars)) dest_node = os.path.join('kolla', 'config', proj, proj) # TODO() should be service zk.ensure_path(dest_node) zk.set(dest_node, json.dumps(extra)) for service in extra['config'][proj]: # write the config files for name, item in extra['config'][proj][service].iteritems(): dest_node = os.path.join('kolla', 'config', proj, service, name) zk.ensure_path(dest_node) if isinstance(item['source'], list): content = self.merge_ini_files(item['source']) else: src_file = item['source'] if not src_file.startswith('/'): src_file = file_utils.find_file(src_file) with open(src_file) as fp: content = fp.read() zk.set(dest_node, content) # write the commands for name, item in extra['commands'][proj][service].iteritems(): dest_node = os.path.join('kolla', 'commands', proj, service, name) zk.ensure_path(dest_node) try: zk.set(dest_node, json.dumps(item)) except Exception as te: LOG.error('%s=%s -> %s' % (dest_node, item, te)) # 3. do the service's config.json (now KOLLA_CONFIG) kc_name = os.path.join(self.config_dir, 'config.json') # override container_config_directory cont_conf_dir = 'zk://%s' % ( CONF.zookeeper.host) jinja_vars['container_config_directory'] = cont_conf_dir kolla_config = jinja_utils.jinja_render(kc_name, jinja_vars) kolla_config = kolla_config.replace('"', '\\"').replace( '\n', '') # 4. parse the marathon app file and add the KOLLA_CONFIG values = { 'kolla_config': kolla_config, 'zookeeper_hosts': CONF.zookeeper.host } for app_type in ['marathon', 'chronos']: app_file = os.path.join(self.base_dir, 'deployment_files', proj, '%s.%s.j2' % (service, app_type)) if not os.path.exists(app_file): LOG.debug('potentially missing file %s' % app_file) continue content = jinja_utils.jinja_render(app_file, jinja_vars, extra=values) dest_file = os.path.join(self.temp_dir, proj, '%s.%s' % (service, app_type)) file_utils.mkdir_p(os.path.dirname(dest_file)) with open(dest_file, 'w') as f: f.write(content)
def load_from_file(cls, service_file, variables): return Runner(yaml.load( jinja_utils.jinja_render(service_file, variables)))