def arrange_setup_data(gtmpldirs, config, gworkdir): """ Arrange setup data to be embedded in kickstart files. :param gtmpldirs: Guest's template dirs :: [path] :param config: Guest's config :: dict :param gworkdir: Guest's workdir """ tmpls = config.get("setup_data", []) if not tmpls: return # Nothing to do. for t in tmpls: src = t.get("src", None) assert src is not None, "Lacks 'src'" dst = t.get("dst", src) out = os.path.join(gworkdir, "setup", dst) tpaths = gtmpldirs + \ [os.path.join(d, os.path.dirname(src)) for d in gtmpldirs] + \ [os.path.dirname(out)] logging.debug("Generating {} from {}".format(out, src)) T.renderto(tpaths, config, src, out, ask=True) return subprocess.check_output( "tar --xz -c setup | base64 > setup.tar.xz.base64", shell=True, cwd=gworkdir )
def _find_template(tmpldirs, template): for tdir in tmpldirs: tmpl = os.path.join(tdir, template) if os.path.exists(tmpl): return tmpl logging.warn("Template {} not found in paths: " "{}".format(template, ', '.join(tmpldirs))) return template
def load_guest_confs(self, name, group=None): """ :param name: Guest's name, e.g. satellite-1 :param group: Guest's group, e.g. satellite """ confs = self.list_guest_confs(name, group) logging.info("Loading guest configs: %s", name) c = anyconfig.load(confs, ac_template=True) return _add_special_confs(_guest_add_missings(c))
def load_guest_confs(self, name, group=None): """ :param name: Guest's name, e.g. satellite-1 :param group: Guest's group, e.g. satellite """ confs = self.list_guest_confs(name, group) logging.info("Loading guest config files: " + name) logging.debug("Configs: " + str(confs)) c = anyconfig.load(confs) return _add_special_confs(_guest_add_missings(c))
def _check_dups_by_ip_or_mac(nis): """ Check if duplicated IP or MAC found in host list and warns about them. :param nis: A list of network interfaces, {ip, mac, ...} """ for k, v, ns in U.find_dups_in_dicts_list_g(nis, ("ip", "mac")): if _ignorable_kv_pair(k, v): continue m = "Duplicated entries: key={}, v={}, hosts={}" logging.warn(m.format(k, v, ", ".join(n.get("host", str(n)) for n in ns)))
def _check_dups_by_ip_or_mac(nis): """ Check if duplicated IP or MAC found in host list and warns about them. :param nis: A list of network interfaces, {ip, mac, ...} """ for k, v, ns in U.find_dups_in_dicts_list_g(nis, ("ip", "mac")): if _ignorable_kv_pair(k, v): continue m = "Duplicated entries: key={}, v={}, hosts={}" logging.warn( m.format(k, v, ", ".join(n.get("host", str(n)) for n in ns)))
def compile_conf_templates(conf, tmpldirs, workdir, templates_key="templates"): """ :param conf: Config object holding templates info :param tmpldirs: Template paths :param workdir: Working dir :param template_keys: Template keys to search each templates """ for k, v in conf.get(templates_key, {}).iteritems(): src = v.get("src", None) dst = v.get("dst", src) if src is None: logging.warn("{} lacks 'src' parameter".format(k)) continue if os.path.sep in src: logging.debug("Adding subdirs of source to template paths: " + src) srcdirs = [os.path.join(d, os.path.dirname(src)) for d in tmpldirs] else: srcdirs = tmpldirs # strip dir part as it will be searched from srcdir. src = os.path.basename(src) dst = os.path.join(workdir, dst) tpaths = srcdirs + [workdir] logging.debug("Generating {} from {} [{}]".format(dst, src, k)) logging.debug("Template path: " + ", ".join(tpaths)) renderto(tpaths, conf, src, dst)
def compile_conf_templates(conf, tmpldirs, workdir, templates_key="templates"): """ Compile template config files. :param conf: Config object holding templates info :param tmpldirs: Template paths :param workdir: Working dir :param template_keys: Template keys to search each templates """ for k, v in conf.get(templates_key, {}).iteritems(): src = v.get("src", None) dst = v.get("dst", src) if src is None: logging.warn("{} lacks 'src' parameter".format(k)) continue if os.path.sep in src: logging.debug("Adding subdirs of source to template paths: " + src) srcdirs = [os.path.join(d, os.path.dirname(src)) for d in tmpldirs] else: srcdirs = tmpldirs # strip dir part as it will be searched from srcdir. src = os.path.basename(src) dst = os.path.join(workdir, dst) tpaths = srcdirs + [workdir] logging.debug("Generating {} from {} [{}]".format(dst, src, k)) logging.debug("Template path: " + ", ".join(tpaths)) anytemplate.render_to(src, conf, dst, at_paths=tpaths, at_engine="jinja2")
def arrange_setup_data(gtmpldirs, config, gworkdir): """ Arrange setup data to be embedded in kickstart files. :param gtmpldirs: Guest's template dirs :: [path] :param config: Guest's config :: dict :param gworkdir: Guest's workdir """ tmpls = config.get("setup_data", []) if not tmpls: return # Nothing to do. for t in tmpls: src = t.get("src", None) content = t.get("content", None) if content is None: assert src is not None, "'src' must not be empty if content isn't!" dst = t.get("dst", src) out = os.path.join(gworkdir, "setup", dst) tpaths = gtmpldirs + \ [os.path.join(d, os.path.dirname(src)) for d in gtmpldirs] + \ [os.path.dirname(out)] logging.debug("Generating %s from %s", out, src) miniascape.template.render_to(src, config, out, tpaths) else: dst = t.get("dst", None) assert dst is not None, "'dst' must be given if content is set!" out = os.path.join(gworkdir, "setup", dst) tpaths = gtmpldirs + [os.path.dirname(out)] logging.debug("Generating %s from given content and paths: %s", out, ','.join(tpaths)) miniascape.template.renders_to(content, config, out, tpaths) return subprocess.check_output( "tar --xz --owner=root --group=root -c setup | base64 > " "setup.tar.xz.base64", shell=True, cwd=gworkdir )
def arrange_setup_data(gtmpldirs, config, gworkdir): """ Arrange setup data to be embedded in kickstart files. :param gtmpldirs: Guest's template dirs :: [path] :param config: Guest's config :: dict :param gworkdir: Guest's workdir """ tmpls = config.get("setup_data", []) if not tmpls: return # Nothing to do. for t in tmpls: src = t.get("src", None) content = t.get("content", None) if content is None: assert src is not None, "'src' must not be empty if content isn't!" dst = t.get("dst", src) out = os.path.join(gworkdir, "setup", dst) tpaths = gtmpldirs + \ [os.path.join(d, os.path.dirname(src)) for d in gtmpldirs] + \ [os.path.dirname(out)] logging.debug("Generating %s from %s", out, src) miniascape.template.render_to(src, config, out, tpaths) else: dst = t.get("dst", None) assert dst is not None, "'dst' must be given if content is set!" out = os.path.join(gworkdir, "setup", dst) tpaths = gtmpldirs + [os.path.dirname(out)] logging.debug("Generating %s from given content and paths: %s", out, ','.join(tpaths)) miniascape.template.renders_to(content, config, out, tpaths) return subprocess.check_output( "tar --xz -c setup | base64 > setup.tar.xz.base64", shell=True, cwd=gworkdir)
def load_site_ctx(ctxpath): """ Load context (conf) file from ``ctxpath``. ``ctxpath`` may be a path to context file, glob pattern of context files or a dir. :param ctxpath: A ctx file, glob pattern of ctx files or dir :: str """ if os.path.isdir(ctxpath): ctxpath = os.path.join(ctxpath, G.M_CONF_PATTERN) else: if '*' not in ctxpath and not os.path.exists(ctxpath): logging.info("Not exist and skipped: %s", ctxpath) return None ctx = anyconfig.load(ctxpath, ac_template=True) if not ctx: logging.warn("No config loaded from: %s", ctxpath) return ctx
def load_site_ctxs(ctxs): """ Load context (conf) files from ``ctxs``. :param ctxs: List of context file[s], glob patterns of context files or dirs :: [str] """ conf = anyconfig.to_container() for ctxpath in ctxs: diff = load_site_ctx(ctxpath) if diff: conf.update(diff) else: logging.warn("No config loaded from: %s", ctxpath) if not conf: raise EmptyConfigError("No config available from: " + ','.join(ctxs)) return conf
def gen_all(cf, tmpldirs, workdir): """ Generate files to build VM for all VMs. :param conffiles: config.ConfFiles object :param tmpldirs: Template dirs :: [path] :param workdir: Working top dir """ guests = cf.list_guest_names() for name in guests: gen_guest_files(name, cf, tmpldirs, _workdir(workdir, name)) conf = cf.load_host_confs() conf["guests_build_datadir"] = G.M_GUESTS_BUILDDATA_TOPDIR conf["timestamp"] = G.timestamp() conf["distdata"] = list(mk_distdata_g(guests)) logging.debug("Generating guests common build aux files...") T.compile_conf_templates(conf, tmpldirs, _guests_workdir(workdir), "guests_templates")
def gen_guest_files(name, conffiles, tmpldirs, workdir, subdir=_AUTOINST_SUBDIR): """ Generate files (vmbuild.sh and ks.cfg) to build VM `name`. :param name: Guest's name :param conffiles: ConfFiles object :param tmpldirs: Template dirs :: [path] :param workdir: Working top dir """ logging.info("Generating files for: %s", name) conf = conffiles.load_guest_confs(name) gtmpldirs = [os.path.join(d, subdir) for d in tmpldirs] if not os.path.exists(workdir): logging.debug("Creating working dir: %s", workdir) os.makedirs(workdir) logging.debug("Generating setup data: %s", name) arrange_setup_data(gtmpldirs, conf, workdir) miniascape.template.compile_conf_templates(conf, tmpldirs, workdir, "templates")
def gen_vnet_files(cf, tmpldirs, workdir, force): """ Generate libvirt network def. XML files. :param cf: An instance of miniascape.config.ConfFiles :param tmpldirs: Template search paths :param workdir: Working dir to save generated XML files :param force: Existing ones may be overwritten if True """ nets = cf.load_nets_confs() outdir = _netoutdir(workdir) tmpl = _find_template(tmpldirs, _netxml_path()) tpaths = [os.path.dirname(tmpl)] logging.debug("Network XML: tpaths={}, tmpl={}".format(tpaths, tmpl)) if not os.path.exists(outdir): os.makedirs(outdir) for name in nets: netconf = os.path.join(outdir, "{}.yml".format(name)) if os.path.exists(netconf) and not force: logging.warn("Net conf already exists: " + netconf) return logging.debug("Dump conf for the net: " + name) anyconfig.dump(nets[name], netconf) netxml = os.path.join(outdir, "{}.xml".format(name)) if os.path.exists(netxml) and not force: logging.warn("Net xml already exists: " + netxml) return nc = anyconfig.load(netconf, ac_template=True) nc["hosts"] = hosts_w_unique_ips(nc) nc["hosts_w_unique_macs"] = hosts_w_unique_macs(nc) logging.debug("Generating network xml: " + netxml) miniascape.template.render_to(tmpl, nc, netxml, tpaths)
def gen_all(cf, tmpldirs, workdir): """ Generate files to build VM for all VMs. :param conffiles: config.ConfFiles object :param tmpldirs: Template dirs :: [path] :param workdir: Working top dir """ guests = cf.list_guest_names() for name in guests: gen_guest_files(name, cf, tmpldirs, _workdir(workdir, name)) conf = cf.load_host_confs() conf["guests_build_datadir"] = G.M_GUESTS_BUILDDATA_TOPDIR conf["timestamp"] = G.timestamp() conf["distdata"] = list(mk_distdata_g(guests)) logging.debug("Generating common aux files...") miniascape.template.compile_conf_templates(conf, tmpldirs, _guests_workdir(workdir), "guests_templates")
def bootstrap(site, workdir, site_template=G.M_SITE_DEFAULT, site_ctxdir=G.M_CONF_TOPDIR, tpaths=[]): """ Bootstrap the site by arranging default configuration files. :param site: Site name, ex. "default". :param workdir: Working dir to save results. :param site_template: Site template name, ex. "default". :param site_ctxdir: Top dir to hold sites' template configuration files :param tpaths: Templates path list """ ctx = dict(site=site, workdir=workdir, site_template=site_template) ctx_outdir = os.path.join(workdir, site) if not os.path.exists(ctx_outdir): logging.info("Creating site ctx dir: " + ctx_outdir) os.makedirs(ctx_outdir) for conf in U.sglob(os.path.join(site_ctxdir, site_template, "*.yml")): os.symlink(os.path.abspath(conf), os.path.join(ctx_outdir, os.path.basename(conf))) for tmpldir in (os.path.join(d, "bootstrap", site_template) for d in tpaths): if not tmpldir.endswith(os.path.sep): tmpldir += os.path.sep for dirpath, dirnames, filenames in os.walk(tmpldir): reldir = dirpath.replace(tmpldir, '') logging.debug("tmpldir={}, reldir={}, " "dirpath={}".format(tmpldir, reldir, dirpath)) for fn in filenames: (fn_base, ext) = os.path.splitext(fn) if ext == ".j2": logging.debug("Jinja2 template found: " + fn) ofn = os.path.join(workdir, reldir, fn_base) else: ofn = os.path.join(workdir, reldir, fn) miniascape.template.render_to(fn, ctx, ofn, [dirpath] + tpaths)
def bootstrap(site, workdir, site_template=G.M_SITE_DEFAULT, site_ctxdir=G.M_CONF_TOPDIR, tpaths=[]): """ Bootstrap the site by arranging default configuration files. :param site: Site name, ex. "default". :param workdir: Working dir to save results. :param site_template: Site template name, ex. "default". :param site_ctxdir: Top dir to hold sites' template configuration files :param tpaths: Templates path list """ ctx = dict(site=site, workdir=workdir, site_template=site_template) ctx_outdir = os.path.join(workdir, site) if not os.path.exists(ctx_outdir): logging.info("Creating site ctx dir: " + ctx_outdir) os.makedirs(ctx_outdir) for conf in U.sglob(os.path.join(site_ctxdir, site_template, "*.yml")): os.symlink(os.path.abspath(conf), os.path.join(ctx_outdir, os.path.basename(conf))) for tmpldir in (os.path.join(d, "bootstrap", site_template) for d in tpaths): if not tmpldir.endswith(os.path.sep): tmpldir += os.path.sep for dirpath, dirnames, filenames in os.walk(tmpldir): reldir = dirpath.replace(tmpldir, '') logging.debug("tmpldir={}, reldir={}, " "dirpath={}".format(tmpldir, reldir, dirpath)) for fn in filenames: (fn_base, ext) = os.path.splitext(fn) if ext == ".j2": logging.debug("Jinja2 template found: " + fn) T.renderto([dirpath] + tpaths, ctx, fn, os.path.join(workdir, reldir, fn_base)) else: T.renderto([dirpath] + tpaths, {}, fn, os.path.join(workdir, reldir, fn))
def gen_guest_files(name, conffiles, tmpldirs, workdir, subdir=_AUTOINST_SUBDIR): """ Generate files (vmbuild.sh and ks.cfg) to build VM `name`. :param name: Guest's name :param conffiles: ConfFiles object :param tmpldirs: Template dirs :: [path] :param workdir: Working top dir """ logging.info("Generating files for the guest: " + name) conf = conffiles.load_guest_confs(name) gtmpldirs = [os.path.join(d, subdir) for d in tmpldirs] if not os.path.exists(workdir): logging.debug("Creating working dir: " + workdir) os.makedirs(workdir) logging.debug("Generating setup data archive to embedded: " + name) arrange_setup_data(gtmpldirs, conf, workdir) T.compile_conf_templates(conf, tmpldirs, workdir, "templates")
def load_host_confs(self): confs = self.list_host_confs() logging.info("Loading host configs") return anyconfig.load(confs, ac_template=True)
def gen_site_conf_files(conf, tmpldirs, workdir): """ Generate site specific config files for host, networks and guests from a config dict. conf :: dict -> .../{common,host,networks.d,guests}/**/*.yml :param conf: Object holding config parameters :param tmpldirs: Template path list :param workdir: Working top dir, e.g. miniascape-workdir-201303121 :return: Configuration topdir where generated config files under """ outdir = os.path.join(workdir, conf.get("site", G.M_SITE_DEFAULT)) if not os.path.exists(outdir): os.makedirs(outdir) logging.info("Generating site config: %s", outdir) baseyml = "00_base.yml" # Config file loaded first. common_conf = conf.get("common", {}) common_conf["site"] = conf.get("site", G.M_SITE_DEFAULT) for (cnf, subdir) in ((common_conf, "common"), (conf.get("host", {}), "host.d")): anyconfig.dump(cnf, os.path.join(outdir, subdir, baseyml)) # ex. /usr/share/miniascape/templates/config/ tpaths = [os.path.join(d, "config") for d in tmpldirs] logging.debug("Template paths for site confs: %r", tpaths) for net in conf.get("networks", []): noutdir = os.path.join(outdir, "networks.d", net["name"]) miniascape.template.render_to("network.j2", context=net, output=os.path.join(noutdir, baseyml), tpaths=tpaths) guests_key = "guests" for ggroup in conf.get("guests", []): ggoutdir = os.path.join(outdir, "guests.d", ggroup["name"]) if not os.path.exists(ggoutdir): os.makedirs(ggoutdir) ggroup_conf = dict() for k, v in ggroup.iteritems(): if k != guests_key: ggroup_conf[k] = v anyconfig.dump(ggroup_conf, os.path.join(ggoutdir, baseyml)) for guest in ggroup["guests"]: for k in ("name", "hostname", "fqdn"): name = guest.get(k, None) if name is not None: break else: raise NoNameGuestError("Guest must have a name or hostname or " "fqdn: guest=" + str(guest)) goutdir = os.path.join(ggoutdir, name) if not os.path.exists(goutdir): os.makedirs(goutdir) anyconfig.dump(guest, os.path.join(goutdir, baseyml)) return outdir
def load_host_confs(self): confs = self.list_host_confs() logging.info("Loading host config files") return anyconfig.load(confs)
def _default_ctxs(site): dctxs = miniascape.globals.site_src_ctx(site) logging.info("Site default ctxs: site={}, default={}".format(site, dctxs)) return dctxs
def _default_ctxs(site): dctxs = G.site_src_ctx(site) logging.info("Site default ctxs: site={}, default={}".format(site, dctxs)) return dctxs