def arrange_setup_data(gtmpldirs, config, gworkdir):
"""
Arrange setup data to be embedded in kickstart files.
:param gtmpldirs: Guest's template dirs :: [path]
:param config: Guest's config :: dict
:param gworkdir: Guest's workdir
"""
tmpls = config.get("setup_data", [])
if not tmpls:
return # Nothing to do.
for t in tmpls:
src = t.get("src", None)
assert src is not None, "Lacks 'src'"
dst = t.get("dst", src)
out = os.path.join(gworkdir, "setup", dst)
tpaths = gtmpldirs + \
[os.path.join(d, os.path.dirname(src)) for d in gtmpldirs] + \
[os.path.dirname(out)]
logging.debug("Generating {} from {}".format(out, src))
T.renderto(tpaths, config, src, out, ask=True)
return subprocess.check_output(
"tar --xz -c setup | base64 > setup.tar.xz.base64",
shell=True, cwd=gworkdir
)
def _find_template(tmpldirs, template):
for tdir in tmpldirs:
tmpl = os.path.join(tdir, template)
if os.path.exists(tmpl):
return tmpl
logging.warn("Template {} not found in paths: "
"{}".format(template, ', '.join(tmpldirs)))
return template
def _find_template(tmpldirs, template):
for tdir in tmpldirs:
tmpl = os.path.join(tdir, template)
if os.path.exists(tmpl):
return tmpl
logging.warn("Template {} not found in paths: "
"{}".format(template, ', '.join(tmpldirs)))
return template
def load_guest_confs(self, name, group=None):
"""
:param name: Guest's name, e.g. satellite-1
:param group: Guest's group, e.g. satellite
"""
confs = self.list_guest_confs(name, group)
logging.info("Loading guest configs: %s", name)
c = anyconfig.load(confs, ac_template=True)
return _add_special_confs(_guest_add_missings(c))
def load_guest_confs(self, name, group=None):
"""
:param name: Guest's name, e.g. satellite-1
:param group: Guest's group, e.g. satellite
"""
confs = self.list_guest_confs(name, group)
logging.info("Loading guest configs: %s", name)
c = anyconfig.load(confs, ac_template=True)
return _add_special_confs(_guest_add_missings(c))
def load_guest_confs(self, name, group=None):
"""
:param name: Guest's name, e.g. satellite-1
:param group: Guest's group, e.g. satellite
"""
confs = self.list_guest_confs(name, group)
logging.info("Loading guest config files: " + name)
logging.debug("Configs: " + str(confs))
c = anyconfig.load(confs)
return _add_special_confs(_guest_add_missings(c))
def _check_dups_by_ip_or_mac(nis):
"""
Check if duplicated IP or MAC found in host list and warns about them.
:param nis: A list of network interfaces, {ip, mac, ...}
"""
for k, v, ns in U.find_dups_in_dicts_list_g(nis, ("ip", "mac")):
if _ignorable_kv_pair(k, v):
continue
m = "Duplicated entries: key={}, v={}, hosts={}"
logging.warn(m.format(k, v, ", ".join(n.get("host", str(n)) for n
in ns)))
def _check_dups_by_ip_or_mac(nis):
"""
Check if duplicated IP or MAC found in host list and warns about them.
:param nis: A list of network interfaces, {ip, mac, ...}
"""
for k, v, ns in U.find_dups_in_dicts_list_g(nis, ("ip", "mac")):
if _ignorable_kv_pair(k, v):
continue
m = "Duplicated entries: key={}, v={}, hosts={}"
logging.warn(
m.format(k, v, ", ".join(n.get("host", str(n)) for n in ns)))
def compile_conf_templates(conf, tmpldirs, workdir, templates_key="templates"):
"""
:param conf: Config object holding templates info
:param tmpldirs: Template paths
:param workdir: Working dir
:param template_keys: Template keys to search each templates
"""
for k, v in conf.get(templates_key, {}).iteritems():
src = v.get("src", None)
dst = v.get("dst", src)
if src is None:
logging.warn("{} lacks 'src' parameter".format(k))
continue
if os.path.sep in src:
logging.debug("Adding subdirs of source to template paths: " + src)
srcdirs = [os.path.join(d, os.path.dirname(src)) for d in tmpldirs]
else:
srcdirs = tmpldirs
# strip dir part as it will be searched from srcdir.
src = os.path.basename(src)
dst = os.path.join(workdir, dst)
tpaths = srcdirs + [workdir]
logging.debug("Generating {} from {} [{}]".format(dst, src, k))
logging.debug("Template path: " + ", ".join(tpaths))
renderto(tpaths, conf, src, dst)
def compile_conf_templates(conf, tmpldirs, workdir, templates_key="templates"):
"""
Compile template config files.
:param conf: Config object holding templates info
:param tmpldirs: Template paths
:param workdir: Working dir
:param template_keys: Template keys to search each templates
"""
for k, v in conf.get(templates_key, {}).iteritems():
src = v.get("src", None)
dst = v.get("dst", src)
if src is None:
logging.warn("{} lacks 'src' parameter".format(k))
continue
if os.path.sep in src:
logging.debug("Adding subdirs of source to template paths: " + src)
srcdirs = [os.path.join(d, os.path.dirname(src)) for d in tmpldirs]
else:
srcdirs = tmpldirs
# strip dir part as it will be searched from srcdir.
src = os.path.basename(src)
dst = os.path.join(workdir, dst)
tpaths = srcdirs + [workdir]
logging.debug("Generating {} from {} [{}]".format(dst, src, k))
logging.debug("Template path: " + ", ".join(tpaths))
anytemplate.render_to(src,
conf,
dst,
at_paths=tpaths,
at_engine="jinja2")
def arrange_setup_data(gtmpldirs, config, gworkdir):
"""
Arrange setup data to be embedded in kickstart files.
:param gtmpldirs: Guest's template dirs :: [path]
:param config: Guest's config :: dict
:param gworkdir: Guest's workdir
"""
tmpls = config.get("setup_data", [])
if not tmpls:
return # Nothing to do.
for t in tmpls:
src = t.get("src", None)
content = t.get("content", None)
if content is None:
assert src is not None, "'src' must not be empty if content isn't!"
dst = t.get("dst", src)
out = os.path.join(gworkdir, "setup", dst)
tpaths = gtmpldirs + \
[os.path.join(d, os.path.dirname(src)) for d in gtmpldirs] + \
[os.path.dirname(out)]
logging.debug("Generating %s from %s", out, src)
miniascape.template.render_to(src, config, out, tpaths)
else:
dst = t.get("dst", None)
assert dst is not None, "'dst' must be given if content is set!"
out = os.path.join(gworkdir, "setup", dst)
tpaths = gtmpldirs + [os.path.dirname(out)]
logging.debug("Generating %s from given content and paths: %s",
out, ','.join(tpaths))
miniascape.template.renders_to(content, config, out, tpaths)
return subprocess.check_output(
"tar --xz --owner=root --group=root -c setup | base64 > "
"setup.tar.xz.base64",
shell=True, cwd=gworkdir
)
def arrange_setup_data(gtmpldirs, config, gworkdir):
"""
Arrange setup data to be embedded in kickstart files.
:param gtmpldirs: Guest's template dirs :: [path]
:param config: Guest's config :: dict
:param gworkdir: Guest's workdir
"""
tmpls = config.get("setup_data", [])
if not tmpls:
return # Nothing to do.
for t in tmpls:
src = t.get("src", None)
content = t.get("content", None)
if content is None:
assert src is not None, "'src' must not be empty if content isn't!"
dst = t.get("dst", src)
out = os.path.join(gworkdir, "setup", dst)
tpaths = gtmpldirs + \
[os.path.join(d, os.path.dirname(src)) for d in gtmpldirs] + \
[os.path.dirname(out)]
logging.debug("Generating %s from %s", out, src)
miniascape.template.render_to(src, config, out, tpaths)
else:
dst = t.get("dst", None)
assert dst is not None, "'dst' must be given if content is set!"
out = os.path.join(gworkdir, "setup", dst)
tpaths = gtmpldirs + [os.path.dirname(out)]
logging.debug("Generating %s from given content and paths: %s",
out, ','.join(tpaths))
miniascape.template.renders_to(content, config, out, tpaths)
return subprocess.check_output(
"tar --xz -c setup | base64 > setup.tar.xz.base64",
shell=True,
cwd=gworkdir)
def load_site_ctx(ctxpath):
"""
Load context (conf) file from ``ctxpath``. ``ctxpath`` may be a path to
context file, glob pattern of context files or a dir.
:param ctxpath: A ctx file, glob pattern of ctx files or dir :: str
"""
if os.path.isdir(ctxpath):
ctxpath = os.path.join(ctxpath, G.M_CONF_PATTERN)
else:
if '*' not in ctxpath and not os.path.exists(ctxpath):
logging.info("Not exist and skipped: %s", ctxpath)
return None
ctx = anyconfig.load(ctxpath, ac_template=True)
if not ctx:
logging.warn("No config loaded from: %s", ctxpath)
return ctx
def load_site_ctx(ctxpath):
"""
Load context (conf) file from ``ctxpath``. ``ctxpath`` may be a path to
context file, glob pattern of context files or a dir.
:param ctxpath: A ctx file, glob pattern of ctx files or dir :: str
"""
if os.path.isdir(ctxpath):
ctxpath = os.path.join(ctxpath, G.M_CONF_PATTERN)
else:
if '*' not in ctxpath and not os.path.exists(ctxpath):
logging.info("Not exist and skipped: %s", ctxpath)
return None
ctx = anyconfig.load(ctxpath, ac_template=True)
if not ctx:
logging.warn("No config loaded from: %s", ctxpath)
return ctx
def load_site_ctxs(ctxs):
"""
Load context (conf) files from ``ctxs``.
:param ctxs: List of context file[s], glob patterns of context files
or dirs :: [str]
"""
conf = anyconfig.to_container()
for ctxpath in ctxs:
diff = load_site_ctx(ctxpath)
if diff:
conf.update(diff)
else:
logging.warn("No config loaded from: %s", ctxpath)
if not conf:
raise EmptyConfigError("No config available from: " + ','.join(ctxs))
return conf
def load_site_ctxs(ctxs):
"""
Load context (conf) files from ``ctxs``.
:param ctxs: List of context file[s], glob patterns of context files
or dirs :: [str]
"""
conf = anyconfig.to_container()
for ctxpath in ctxs:
diff = load_site_ctx(ctxpath)
if diff:
conf.update(diff)
else:
logging.warn("No config loaded from: %s", ctxpath)
if not conf:
raise EmptyConfigError("No config available from: " + ','.join(ctxs))
return conf
def gen_all(cf, tmpldirs, workdir):
"""
Generate files to build VM for all VMs.
:param conffiles: config.ConfFiles object
:param tmpldirs: Template dirs :: [path]
:param workdir: Working top dir
"""
guests = cf.list_guest_names()
for name in guests:
gen_guest_files(name, cf, tmpldirs, _workdir(workdir, name))
conf = cf.load_host_confs()
conf["guests_build_datadir"] = G.M_GUESTS_BUILDDATA_TOPDIR
conf["timestamp"] = G.timestamp()
conf["distdata"] = list(mk_distdata_g(guests))
logging.debug("Generating guests common build aux files...")
T.compile_conf_templates(conf, tmpldirs,
_guests_workdir(workdir), "guests_templates")
def gen_guest_files(name,
conffiles,
tmpldirs,
workdir,
subdir=_AUTOINST_SUBDIR):
"""
Generate files (vmbuild.sh and ks.cfg) to build VM `name`.
:param name: Guest's name
:param conffiles: ConfFiles object
:param tmpldirs: Template dirs :: [path]
:param workdir: Working top dir
"""
logging.info("Generating files for: %s", name)
conf = conffiles.load_guest_confs(name)
gtmpldirs = [os.path.join(d, subdir) for d in tmpldirs]
if not os.path.exists(workdir):
logging.debug("Creating working dir: %s", workdir)
os.makedirs(workdir)
logging.debug("Generating setup data: %s", name)
arrange_setup_data(gtmpldirs, conf, workdir)
miniascape.template.compile_conf_templates(conf, tmpldirs, workdir,
"templates")
def gen_vnet_files(cf, tmpldirs, workdir, force):
"""
Generate libvirt network def. XML files.
:param cf: An instance of miniascape.config.ConfFiles
:param tmpldirs: Template search paths
:param workdir: Working dir to save generated XML files
:param force: Existing ones may be overwritten if True
"""
nets = cf.load_nets_confs()
outdir = _netoutdir(workdir)
tmpl = _find_template(tmpldirs, _netxml_path())
tpaths = [os.path.dirname(tmpl)]
logging.debug("Network XML: tpaths={}, tmpl={}".format(tpaths, tmpl))
if not os.path.exists(outdir):
os.makedirs(outdir)
for name in nets:
netconf = os.path.join(outdir, "{}.yml".format(name))
if os.path.exists(netconf) and not force:
logging.warn("Net conf already exists: " + netconf)
return
logging.debug("Dump conf for the net: " + name)
anyconfig.dump(nets[name], netconf)
netxml = os.path.join(outdir, "{}.xml".format(name))
if os.path.exists(netxml) and not force:
logging.warn("Net xml already exists: " + netxml)
return
nc = anyconfig.load(netconf, ac_template=True)
nc["hosts"] = hosts_w_unique_ips(nc)
nc["hosts_w_unique_macs"] = hosts_w_unique_macs(nc)
logging.debug("Generating network xml: " + netxml)
miniascape.template.render_to(tmpl, nc, netxml, tpaths)
def gen_vnet_files(cf, tmpldirs, workdir, force):
"""
Generate libvirt network def. XML files.
:param cf: An instance of miniascape.config.ConfFiles
:param tmpldirs: Template search paths
:param workdir: Working dir to save generated XML files
:param force: Existing ones may be overwritten if True
"""
nets = cf.load_nets_confs()
outdir = _netoutdir(workdir)
tmpl = _find_template(tmpldirs, _netxml_path())
tpaths = [os.path.dirname(tmpl)]
logging.debug("Network XML: tpaths={}, tmpl={}".format(tpaths, tmpl))
if not os.path.exists(outdir):
os.makedirs(outdir)
for name in nets:
netconf = os.path.join(outdir, "{}.yml".format(name))
if os.path.exists(netconf) and not force:
logging.warn("Net conf already exists: " + netconf)
return
logging.debug("Dump conf for the net: " + name)
anyconfig.dump(nets[name], netconf)
netxml = os.path.join(outdir, "{}.xml".format(name))
if os.path.exists(netxml) and not force:
logging.warn("Net xml already exists: " + netxml)
return
nc = anyconfig.load(netconf, ac_template=True)
nc["hosts"] = hosts_w_unique_ips(nc)
nc["hosts_w_unique_macs"] = hosts_w_unique_macs(nc)
logging.debug("Generating network xml: " + netxml)
miniascape.template.render_to(tmpl, nc, netxml, tpaths)
def gen_all(cf, tmpldirs, workdir):
"""
Generate files to build VM for all VMs.
:param conffiles: config.ConfFiles object
:param tmpldirs: Template dirs :: [path]
:param workdir: Working top dir
"""
guests = cf.list_guest_names()
for name in guests:
gen_guest_files(name, cf, tmpldirs, _workdir(workdir, name))
conf = cf.load_host_confs()
conf["guests_build_datadir"] = G.M_GUESTS_BUILDDATA_TOPDIR
conf["timestamp"] = G.timestamp()
conf["distdata"] = list(mk_distdata_g(guests))
logging.debug("Generating common aux files...")
miniascape.template.compile_conf_templates(conf, tmpldirs,
_guests_workdir(workdir),
"guests_templates")
def bootstrap(site,
workdir,
site_template=G.M_SITE_DEFAULT,
site_ctxdir=G.M_CONF_TOPDIR,
tpaths=[]):
"""
Bootstrap the site by arranging default configuration files.
:param site: Site name, ex. "default".
:param workdir: Working dir to save results.
:param site_template: Site template name, ex. "default".
:param site_ctxdir: Top dir to hold sites' template configuration files
:param tpaths: Templates path list
"""
ctx = dict(site=site, workdir=workdir, site_template=site_template)
ctx_outdir = os.path.join(workdir, site)
if not os.path.exists(ctx_outdir):
logging.info("Creating site ctx dir: " + ctx_outdir)
os.makedirs(ctx_outdir)
for conf in U.sglob(os.path.join(site_ctxdir, site_template, "*.yml")):
os.symlink(os.path.abspath(conf),
os.path.join(ctx_outdir, os.path.basename(conf)))
for tmpldir in (os.path.join(d, "bootstrap", site_template)
for d in tpaths):
if not tmpldir.endswith(os.path.sep):
tmpldir += os.path.sep
for dirpath, dirnames, filenames in os.walk(tmpldir):
reldir = dirpath.replace(tmpldir, '')
logging.debug("tmpldir={}, reldir={}, "
"dirpath={}".format(tmpldir, reldir, dirpath))
for fn in filenames:
(fn_base, ext) = os.path.splitext(fn)
if ext == ".j2":
logging.debug("Jinja2 template found: " + fn)
ofn = os.path.join(workdir, reldir, fn_base)
else:
ofn = os.path.join(workdir, reldir, fn)
miniascape.template.render_to(fn, ctx, ofn, [dirpath] + tpaths)
def bootstrap(site, workdir, site_template=G.M_SITE_DEFAULT,
site_ctxdir=G.M_CONF_TOPDIR, tpaths=[]):
"""
Bootstrap the site by arranging default configuration files.
:param site: Site name, ex. "default".
:param workdir: Working dir to save results.
:param site_template: Site template name, ex. "default".
:param site_ctxdir: Top dir to hold sites' template configuration files
:param tpaths: Templates path list
"""
ctx = dict(site=site, workdir=workdir, site_template=site_template)
ctx_outdir = os.path.join(workdir, site)
if not os.path.exists(ctx_outdir):
logging.info("Creating site ctx dir: " + ctx_outdir)
os.makedirs(ctx_outdir)
for conf in U.sglob(os.path.join(site_ctxdir, site_template, "*.yml")):
os.symlink(os.path.abspath(conf),
os.path.join(ctx_outdir, os.path.basename(conf)))
for tmpldir in (os.path.join(d, "bootstrap", site_template) for d
in tpaths):
if not tmpldir.endswith(os.path.sep):
tmpldir += os.path.sep
for dirpath, dirnames, filenames in os.walk(tmpldir):
reldir = dirpath.replace(tmpldir, '')
logging.debug("tmpldir={}, reldir={}, "
"dirpath={}".format(tmpldir, reldir, dirpath))
for fn in filenames:
(fn_base, ext) = os.path.splitext(fn)
if ext == ".j2":
logging.debug("Jinja2 template found: " + fn)
T.renderto([dirpath] + tpaths, ctx, fn,
os.path.join(workdir, reldir, fn_base))
else:
T.renderto([dirpath] + tpaths, {}, fn,
os.path.join(workdir, reldir, fn))
def gen_guest_files(name, conffiles, tmpldirs, workdir,
subdir=_AUTOINST_SUBDIR):
"""
Generate files (vmbuild.sh and ks.cfg) to build VM `name`.
:param name: Guest's name
:param conffiles: ConfFiles object
:param tmpldirs: Template dirs :: [path]
:param workdir: Working top dir
"""
logging.info("Generating files for the guest: " + name)
conf = conffiles.load_guest_confs(name)
gtmpldirs = [os.path.join(d, subdir) for d in tmpldirs]
if not os.path.exists(workdir):
logging.debug("Creating working dir: " + workdir)
os.makedirs(workdir)
logging.debug("Generating setup data archive to embedded: " + name)
arrange_setup_data(gtmpldirs, conf, workdir)
T.compile_conf_templates(conf, tmpldirs, workdir, "templates")
def load_host_confs(self):
confs = self.list_host_confs()
logging.info("Loading host configs")
return anyconfig.load(confs, ac_template=True)
def gen_site_conf_files(conf, tmpldirs, workdir):
"""
Generate site specific config files for host, networks and guests from a
config dict.
conf :: dict -> .../{common,host,networks.d,guests}/**/*.yml
:param conf: Object holding config parameters
:param tmpldirs: Template path list
:param workdir: Working top dir, e.g. miniascape-workdir-201303121
:return: Configuration topdir where generated config files under
"""
outdir = os.path.join(workdir, conf.get("site", G.M_SITE_DEFAULT))
if not os.path.exists(outdir):
os.makedirs(outdir)
logging.info("Generating site config: %s", outdir)
baseyml = "00_base.yml" # Config file loaded first.
common_conf = conf.get("common", {})
common_conf["site"] = conf.get("site", G.M_SITE_DEFAULT)
for (cnf, subdir) in ((common_conf, "common"), (conf.get("host",
{}), "host.d")):
anyconfig.dump(cnf, os.path.join(outdir, subdir, baseyml))
# ex. /usr/share/miniascape/templates/config/
tpaths = [os.path.join(d, "config") for d in tmpldirs]
logging.debug("Template paths for site confs: %r", tpaths)
for net in conf.get("networks", []):
noutdir = os.path.join(outdir, "networks.d", net["name"])
miniascape.template.render_to("network.j2",
context=net,
output=os.path.join(noutdir, baseyml),
tpaths=tpaths)
guests_key = "guests"
for ggroup in conf.get("guests", []):
ggoutdir = os.path.join(outdir, "guests.d", ggroup["name"])
if not os.path.exists(ggoutdir):
os.makedirs(ggoutdir)
ggroup_conf = dict()
for k, v in ggroup.iteritems():
if k != guests_key:
ggroup_conf[k] = v
anyconfig.dump(ggroup_conf, os.path.join(ggoutdir, baseyml))
for guest in ggroup["guests"]:
for k in ("name", "hostname", "fqdn"):
name = guest.get(k, None)
if name is not None:
break
else:
raise NoNameGuestError("Guest must have a name or hostname or "
"fqdn: guest=" + str(guest))
goutdir = os.path.join(ggoutdir, name)
if not os.path.exists(goutdir):
os.makedirs(goutdir)
anyconfig.dump(guest, os.path.join(goutdir, baseyml))
return outdir
def gen_site_conf_files(conf, tmpldirs, workdir):
"""
Generate site specific config files for host, networks and guests from a
config dict.
conf :: dict -> .../{common,host,networks.d,guests}/**/*.yml
:param conf: Object holding config parameters
:param tmpldirs: Template path list
:param workdir: Working top dir, e.g. miniascape-workdir-201303121
:return: Configuration topdir where generated config files under
"""
outdir = os.path.join(workdir, conf.get("site", G.M_SITE_DEFAULT))
if not os.path.exists(outdir):
os.makedirs(outdir)
logging.info("Generating site config: %s", outdir)
baseyml = "00_base.yml" # Config file loaded first.
common_conf = conf.get("common", {})
common_conf["site"] = conf.get("site", G.M_SITE_DEFAULT)
for (cnf, subdir) in ((common_conf, "common"),
(conf.get("host", {}), "host.d")):
anyconfig.dump(cnf, os.path.join(outdir, subdir, baseyml))
# ex. /usr/share/miniascape/templates/config/
tpaths = [os.path.join(d, "config") for d in tmpldirs]
logging.debug("Template paths for site confs: %r", tpaths)
for net in conf.get("networks", []):
noutdir = os.path.join(outdir, "networks.d", net["name"])
miniascape.template.render_to("network.j2", context=net,
output=os.path.join(noutdir, baseyml),
tpaths=tpaths)
guests_key = "guests"
for ggroup in conf.get("guests", []):
ggoutdir = os.path.join(outdir, "guests.d", ggroup["name"])
if not os.path.exists(ggoutdir):
os.makedirs(ggoutdir)
ggroup_conf = dict()
for k, v in ggroup.iteritems():
if k != guests_key:
ggroup_conf[k] = v
anyconfig.dump(ggroup_conf, os.path.join(ggoutdir, baseyml))
for guest in ggroup["guests"]:
for k in ("name", "hostname", "fqdn"):
name = guest.get(k, None)
if name is not None:
break
else:
raise NoNameGuestError("Guest must have a name or hostname or "
"fqdn: guest=" + str(guest))
goutdir = os.path.join(ggoutdir, name)
if not os.path.exists(goutdir):
os.makedirs(goutdir)
anyconfig.dump(guest, os.path.join(goutdir, baseyml))
return outdir
def load_host_confs(self):
confs = self.list_host_confs()
logging.info("Loading host configs")
return anyconfig.load(confs, ac_template=True)
def load_host_confs(self):
confs = self.list_host_confs()
logging.info("Loading host config files")
return anyconfig.load(confs)
def _default_ctxs(site):
dctxs = miniascape.globals.site_src_ctx(site)
logging.info("Site default ctxs: site={}, default={}".format(site, dctxs))
return dctxs
def _default_ctxs(site):
dctxs = G.site_src_ctx(site)
logging.info("Site default ctxs: site={}, default={}".format(site, dctxs))
return dctxs
