def test_render_explict_default(self):
blob = '\n'.join((
'## template: basic',
'$a,$b',
))
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEqual("1,2", c)
def test_jinja_nonascii_render_to_string(self):
"""Test jinja render_to_string with non-ascii content."""
self.assertEqual(
templater.render_string(self.add_header("jinja", self.jinja_utf8),
{"name": "bob"}),
self.jinja_utf8_rbob,
)
def test_render_basic_deeper(self):
hn = "myfoohost.yahoo.com"
expected_data = "h=%s\nc=d\n" % hn
in_data = "h=$hostname.canonical_name\nc=d\n"
params = {"hostname": {"canonical_name": hn}}
out_data = templater.render_string(in_data, params)
self.assertEqual(expected_data, out_data)
def test_jinja_nonascii_render_undefined_variables_to_default_py3(self):
"""Test py3 jinja render_to_string with undefined variable default."""
self.assertEqual(
templater.render_string(self.add_header("jinja", self.jinja_utf8),
{}),
self.jinja_utf8_rbob.replace("bob", "CI_MISSING_JINJA_VAR/name"),
)
def add_sources(srclist, template_params=None, aa_repo_match=None):
"""
add entries in /etc/apt/sources.list.d for each abbreviated
sources.list entry in 'srclist'. When rendering template, also
include the values in dictionary searchList
"""
if template_params is None:
template_params = {}
if aa_repo_match is None:
aa_repo_match = lambda f: False
errorlist = []
for ent in srclist:
if 'source' not in ent:
errorlist.append(["", "missing source"])
continue
source = ent['source']
source = templater.render_string(source, template_params)
if aa_repo_match(source):
try:
util.subp(["add-apt-repository", source])
except util.ProcessExecutionError as e:
errorlist.append(
[source, ("add-apt-repository failed. " + str(e))])
continue
if 'filename' not in ent:
ent['filename'] = 'cloud_config_sources.list'
if not ent['filename'].startswith("/"):
ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
ent['filename'])
if ('keyid' in ent and 'key' not in ent):
ks = "keyserver.ubuntu.com"
if 'keyserver' in ent:
ks = ent['keyserver']
try:
ent['key'] = getkeybyid(ent['keyid'], ks)
except:
errorlist.append([source, "failed to get key from %s" % ks])
continue
if 'key' in ent:
try:
util.subp(('apt-key', 'add', '-'), ent['key'])
except:
errorlist.append([source, "failed add key"])
try:
contents = "%s\n" % (source)
util.write_file(ent['filename'], contents, omode="ab")
except:
errorlist.append(
[source, "failed write to file %s" % ent['filename']])
return errorlist
def add_sources(srclist, template_params=None, aa_repo_match=None):
"""
add entries in /etc/apt/sources.list.d for each abbreviated
sources.list entry in 'srclist'. When rendering template, also
include the values in dictionary searchList
"""
if template_params is None:
template_params = {}
if aa_repo_match is None:
aa_repo_match = lambda f: False
errorlist = []
for ent in srclist:
if 'source' not in ent:
errorlist.append(["", "missing source"])
continue
source = ent['source']
source = templater.render_string(source, template_params)
if aa_repo_match(source):
try:
util.subp(["add-apt-repository", source])
except util.ProcessExecutionError as e:
errorlist.append([source,
("add-apt-repository failed. " + str(e))])
continue
if 'filename' not in ent:
ent['filename'] = 'cloud_config_sources.list'
if not ent['filename'].startswith("/"):
ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
ent['filename'])
if ('keyid' in ent and 'key' not in ent):
ks = "keyserver.ubuntu.com"
if 'keyserver' in ent:
ks = ent['keyserver']
try:
ent['key'] = getkeybyid(ent['keyid'], ks)
except:
errorlist.append([source, "failed to get key from %s" % ks])
continue
if 'key' in ent:
try:
util.subp(('apt-key', 'add', '-'), ent['key'])
except:
errorlist.append([source, "failed add key"])
try:
contents = "%s\n" % (source)
util.write_file(ent['filename'], contents, omode="ab")
except:
errorlist.append([source,
"failed write to file %s" % ent['filename']])
return errorlist
def generate_sources_list(cfg, release, mirrors, cloud):
"""generate_sources_list
create a source.list file based on a custom or default template
by replacing mirrors and release in the template"""
aptsrc = "/etc/apt/sources.list"
params = {'RELEASE': release, 'codename': release}
for k in mirrors:
params[k] = mirrors[k]
params[k.lower()] = mirrors[k]
tmpl = cfg.get('sources_list', None)
if tmpl is None:
LOG.info("No custom template provided, fall back to builtin")
template_fn = cloud.get_template_filename('sources.list.%s' %
(cloud.distro.name))
if not template_fn:
template_fn = cloud.get_template_filename('sources.list')
if not template_fn:
LOG.warning("No template found, "
"not rendering /etc/apt/sources.list")
return
tmpl = util.load_file(template_fn)
rendered = templater.render_string(tmpl, params)
disabled = disable_suites(cfg.get('disable_suites'), rendered, release)
util.write_file(aptsrc, disabled, mode=0o644)
def generate_sources_list(cfg, release, mirrors, cloud):
"""generate_sources_list
create a source.list file based on a custom or default template
by replacing mirrors and release in the template"""
aptsrc = "/etc/apt/sources.list"
params = {'RELEASE': release, 'codename': release}
for k in mirrors:
params[k] = mirrors[k]
params[k.lower()] = mirrors[k]
tmpl = cfg.get('sources_list', None)
if tmpl is None:
LOG.info("No custom template provided, fall back to builtin")
template_fn = cloud.get_template_filename('sources.list.%s' %
(cloud.distro.name))
if not template_fn:
template_fn = cloud.get_template_filename('sources.list')
if not template_fn:
LOG.warn("No template found, not rendering /etc/apt/sources.list")
return
tmpl = util.load_file(template_fn)
rendered = templater.render_string(tmpl, params)
disabled = disable_suites(cfg.get('disable_suites'), rendered, release)
util.write_file(aptsrc, disabled, mode=0o644)
def render_jinja_payload(payload, payload_fn, instance_data, debug=False):
instance_jinja_vars = convert_jinja_instance_data(
instance_data,
decode_paths=instance_data.get("base64-encoded-keys", []),
include_key_aliases=True,
)
if debug:
LOG.debug("Converted jinja variables\n%s",
json_dumps(instance_jinja_vars))
try:
rendered_payload = render_string(payload, instance_jinja_vars)
except (TypeError, JUndefinedError) as e:
LOG.warning("Ignoring jinja template for %s: %s", payload_fn, str(e))
return None
warnings = [
"'%s'" % var.replace(MISSING_JINJA_PREFIX, "")
for var in re.findall(r"%s[^\s]+" %
MISSING_JINJA_PREFIX, rendered_payload)
]
if warnings:
LOG.warning(
"Could not render jinja template variables in file '%s': %s",
payload_fn,
", ".join(warnings),
)
return rendered_payload
def welcome_format(action):
tpl_params = {
'version': version.version_string(),
'uptime': util.uptime(),
'timestamp': util.time_rfc2822(),
'action': action,
}
return templater.render_string(WELCOME_MSG_TPL, tpl_params)
def welcome_format(action):
tpl_params = {
'version': version.version_string(),
'uptime': util.uptime(),
'timestamp': util.time_rfc2822(),
'action': action,
}
return templater.render_string(WELCOME_MSG_TPL, tpl_params)
def add_apt_sources(srcdict,
cloud,
target=None,
template_params=None,
aa_repo_match=None):
"""
add entries in /etc/apt/sources.list.d for each abbreviated
sources.list entry in 'srcdict'. When rendering template, also
include the values in dictionary searchList
"""
if template_params is None:
template_params = {}
if aa_repo_match is None:
raise ValueError('did not get a valid repo matcher')
if not isinstance(srcdict, dict):
raise TypeError('unknown apt format: %s' % (srcdict))
for filename in srcdict:
ent = srcdict[filename]
LOG.debug("adding source/key '%s'", ent)
if 'filename' not in ent:
ent['filename'] = filename
add_apt_key(ent, target)
if 'source' not in ent:
continue
source = ent['source']
source = templater.render_string(source, template_params)
if not ent['filename'].startswith("/"):
ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
ent['filename'])
if not ent['filename'].endswith(".list"):
ent['filename'] += ".list"
if aa_repo_match(source):
try:
util.subp(["add-apt-repository", source], target=target)
except util.ProcessExecutionError:
LOG.exception("add-apt-repository failed.")
raise
continue
sourcefn = util.target_path(target, ent['filename'])
try:
contents = "%s\n" % (source)
util.write_file(sourcefn, contents, omode="a")
except IOError as detail:
LOG.exception("failed write to file %s: %s", sourcefn, detail)
raise
update_packages(cloud)
return
def add_sources(srclist, template_params=None):
"""
add entries in /etc/apt/sources.list.d for each abbreviated
sources.list entry in 'srclist'. When rendering template, also
include the values in dictionary searchList
"""
if template_params is None:
template_params = {}
errorlist = []
for ent in srclist:
if "source" not in ent:
errorlist.append(["", "missing source"])
continue
source = ent["source"]
if source.startswith("ppa:"):
try:
util.subp(["add-apt-repository", source])
except:
errorlist.append([source, "add-apt-repository failed"])
continue
source = templater.render_string(source, template_params)
if "filename" not in ent:
ent["filename"] = "cloud_config_sources.list"
if not ent["filename"].startswith("/"):
ent["filename"] = os.path.join("/etc/apt/sources.list.d/", ent["filename"])
if "keyid" in ent and "key" not in ent:
ks = "keyserver.ubuntu.com"
if "keyserver" in ent:
ks = ent["keyserver"]
try:
ent["key"] = getkeybyid(ent["keyid"], ks)
except:
errorlist.append([source, "failed to get key from %s" % ks])
continue
if "key" in ent:
try:
util.subp(("apt-key", "add", "-"), ent["key"])
except:
errorlist.append([source, "failed add key"])
try:
contents = "%s\n" % (source)
util.write_file(ent["filename"], contents, omode="ab")
except:
errorlist.append([source, "failed write to file %s" % ent["filename"]])
return errorlist
def test_jinja_do_extension_render_to_string(self):
"""Test jinja render_to_string using do extension."""
expected_result = "[1, 2, 3]"
jinja_template = (
"{% set r = [] %} {% set input = [1,2,3] %} "
"{% for i in input %} {% do r.append(i) %} {% endfor %} {{r}}")
self.assertEqual(
templater.render_string(self.add_header("jinja", jinja_template),
{}).strip(),
expected_result,
)
def test_render_basic_deeper(self):
hn = 'myfoohost.yahoo.com'
expected_data = "h=%s\nc=d\n" % hn
in_data = "h=$hostname.canonical_name\nc=d\n"
params = {
"hostname": {
"canonical_name": hn,
},
}
out_data = templater.render_string(in_data, params)
self.assertEqual(expected_data, out_data)
def add_apt_sources(srcdict, cloud, target=None, template_params=None,
aa_repo_match=None):
"""
add entries in /etc/apt/sources.list.d for each abbreviated
sources.list entry in 'srcdict'. When rendering template, also
include the values in dictionary searchList
"""
if template_params is None:
template_params = {}
if aa_repo_match is None:
raise ValueError('did not get a valid repo matcher')
if not isinstance(srcdict, dict):
raise TypeError('unknown apt format: %s' % (srcdict))
for filename in srcdict:
ent = srcdict[filename]
LOG.debug("adding source/key '%s'", ent)
if 'filename' not in ent:
ent['filename'] = filename
add_apt_key(ent, target)
if 'source' not in ent:
continue
source = ent['source']
source = templater.render_string(source, template_params)
if not ent['filename'].startswith("/"):
ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
ent['filename'])
if not ent['filename'].endswith(".list"):
ent['filename'] += ".list"
if aa_repo_match(source):
try:
util.subp(["add-apt-repository", source], target=target)
except util.ProcessExecutionError:
LOG.exception("add-apt-repository failed.")
raise
continue
sourcefn = util.target_path(target, ent['filename'])
try:
contents = "%s\n" % (source)
util.write_file(sourcefn, contents, omode="a")
except IOError as detail:
LOG.exception("failed write to file %s: %s", sourcefn, detail)
raise
update_packages(cloud)
return
def handle(_name, cfg, cloud, log, args):
msg_in = ""
if len(args) != 0:
msg_in = str(args[0])
else:
msg_in = util.get_cfg_option_str(cfg, "final_message", "")
msg_in = msg_in.strip()
if not msg_in:
msg_in = FINAL_MESSAGE_DEF
uptime = util.uptime()
ts = util.time_rfc2822()
cver = version.version_string()
try:
subs = {
"uptime": uptime,
"timestamp": ts,
"version": cver,
"datasource": str(cloud.datasource),
}
subs.update(dict([(k.upper(), v) for k, v in subs.items()]))
util.multi_log(
"%s\n" % (templater.render_string(msg_in, subs)),
console=False,
stderr=True,
log=log,
)
except Exception:
util.logexc(log, "Failed to render final message template")
boot_fin_fn = cloud.paths.boot_finished
try:
contents = "%s - %s - v. %s\n" % (uptime, ts, cver)
util.write_file(boot_fin_fn, contents, ensure_dir_exists=False)
except Exception:
util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn)
if cloud.datasource.is_disconnected:
log.warning("Used fallback datasource")
def render_jinja_payload(payload, payload_fn, instance_data, debug=False):
instance_jinja_vars = convert_jinja_instance_data(
instance_data,
decode_paths=instance_data.get('base64-encoded-keys', []))
if debug:
LOG.debug('Converted jinja variables\n%s',
json_dumps(instance_jinja_vars))
try:
rendered_payload = render_string(payload, instance_jinja_vars)
except (TypeError, JUndefinedError) as e:
LOG.warning(
'Ignoring jinja template for %s: %s', payload_fn, str(e))
return None
warnings = [
"'%s'" % var.replace(MISSING_JINJA_PREFIX, '')
for var in re.findall(
r'%s[^\s]+' % MISSING_JINJA_PREFIX, rendered_payload)]
if warnings:
LOG.warning(
"Could not render jinja template variables in file '%s': %s",
payload_fn, ', '.join(warnings))
return rendered_payload
def handle(_name, cfg, cloud, log, args):
msg_in = ''
if len(args) != 0:
msg_in = str(args[0])
else:
msg_in = util.get_cfg_option_str(cfg, "final_message", "")
msg_in = msg_in.strip()
if not msg_in:
msg_in = FINAL_MESSAGE_DEF
uptime = util.uptime()
ts = util.time_rfc2822()
cver = version.version_string()
try:
subs = {
'uptime': uptime,
'timestamp': ts,
'version': cver,
'datasource': str(cloud.datasource),
}
util.multi_log("%s\n" % (templater.render_string(msg_in, subs)),
console=False,
stderr=True,
log=log)
except Exception:
util.logexc(log, "Failed to render final message template")
boot_fin_fn = cloud.paths.boot_finished
try:
contents = "%s - %s - v. %s\n" % (uptime, ts, cver)
util.write_file(boot_fin_fn, contents)
except:
util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn)
if cloud.datasource.is_disconnected:
log.warn("Used fallback datasource")
def handle(_name, cfg, cloud, log, args):
msg_in = ''
if len(args) != 0:
msg_in = str(args[0])
else:
msg_in = util.get_cfg_option_str(cfg, "final_message", "")
msg_in = msg_in.strip()
if not msg_in:
msg_in = FINAL_MESSAGE_DEF
uptime = util.uptime()
ts = util.time_rfc2822()
cver = version.version_string()
try:
subs = {
'uptime': uptime,
'timestamp': ts,
'version': cver,
'datasource': str(cloud.datasource),
}
subs.update(dict([(k.upper(), v) for k, v in subs.items()]))
util.multi_log("%s\n" % (templater.render_string(msg_in, subs)),
console=False, stderr=True, log=log)
except Exception:
util.logexc(log, "Failed to render final message template")
boot_fin_fn = cloud.paths.boot_finished
try:
contents = "%s - %s - v. %s\n" % (uptime, ts, cver)
util.write_file(boot_fin_fn, contents)
except Exception:
util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn)
if cloud.datasource.is_disconnected:
log.warn("Used fallback datasource")
def disable_suites(disabled, src, release):
"""reads the config for suites to be disabled and removes those
from the template"""
if not disabled:
return src
retsrc = src
for suite in disabled:
suite = map_known_suites(suite)
releasesuite = templater.render_string(suite, {'RELEASE': release})
LOG.debug("Disabling suite %s as %s", suite, releasesuite)
newsrc = ""
for line in retsrc.splitlines(True):
if line.startswith("#"):
newsrc += line
continue
# sources.list allow options in cols[1] which can have spaces
# so the actual suite can be [2] or later. example:
# deb [ arch=amd64,armel k=v ] http://example.com/debian
cols = line.split()
if len(cols) > 1:
pcol = 2
if cols[1].startswith("["):
for col in cols[1:]:
pcol += 1
if col.endswith("]"):
break
if cols[pcol] == releasesuite:
line = '# suite disabled by cloud-init: %s' % line
newsrc += line
retsrc = newsrc
return retsrc
def disable_suites(disabled, src, release):
"""reads the config for suites to be disabled and removes those
from the template"""
if not disabled:
return src
retsrc = src
for suite in disabled:
suite = map_known_suites(suite)
releasesuite = templater.render_string(suite, {'RELEASE': release})
LOG.debug("Disabling suite %s as %s", suite, releasesuite)
newsrc = ""
for line in retsrc.splitlines(True):
if line.startswith("#"):
newsrc += line
continue
# sources.list allow options in cols[1] which can have spaces
# so the actual suite can be [2] or later. example:
# deb [ arch=amd64,armel k=v ] http://example.com/debian
cols = line.split()
if len(cols) > 1:
pcol = 2
if cols[1].startswith("["):
for col in cols[1:]:
pcol += 1
if col.endswith("]"):
break
if cols[pcol] == releasesuite:
line = '# suite disabled by cloud-init: %s' % line
newsrc += line
retsrc = newsrc
return retsrc
def test_render_default(self):
blob = '''$a,$b'''
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEquals("1,2", c)
def test_render_jinja_crlf(self):
blob = '\r\n'.join(("## template:jinja", "{{a}},{{b}}"))
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEqual("1,2", c)
def test_render_cheetah(self):
blob = '''## template:cheetah
$a,$b'''
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEquals("1,2", c)
def test_render_default(self):
blob = """$a,$b"""
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEqual("1,2", c)
def handle(name, cfg, cloud, log, args):
if len(args) != 0:
ph_cfg = util.read_conf(args[0])
else:
if "phone_home" not in cfg:
log.debug(
"Skipping module named %s, "
"no 'phone_home' configuration found",
name,
)
return
ph_cfg = cfg["phone_home"]
if "url" not in ph_cfg:
log.warning(
"Skipping module named %s, "
"no 'url' found in 'phone_home' configuration",
name,
)
return
url = ph_cfg["url"]
post_list = ph_cfg.get("post", "all")
tries = ph_cfg.get("tries")
try:
tries = int(tries) # type: ignore
except ValueError:
tries = 10
util.logexc(
log,
"Configuration entry 'tries' is not an integer, using %s instead",
tries,
)
if post_list == "all":
post_list = POST_LIST_ALL
all_keys = {}
all_keys["instance_id"] = cloud.get_instance_id()
all_keys["hostname"] = cloud.get_hostname()
all_keys["fqdn"] = cloud.get_hostname(fqdn=True)
pubkeys = {
"pub_key_dsa": "/etc/ssh/ssh_host_dsa_key.pub",
"pub_key_rsa": "/etc/ssh/ssh_host_rsa_key.pub",
"pub_key_ecdsa": "/etc/ssh/ssh_host_ecdsa_key.pub",
"pub_key_ed25519": "/etc/ssh/ssh_host_ed25519_key.pub",
}
for (n, path) in pubkeys.items():
try:
all_keys[n] = util.load_file(path)
except Exception:
util.logexc(log,
"%s: failed to open, can not phone home that data!",
path)
submit_keys = {}
for k in post_list:
if k in all_keys:
submit_keys[k] = all_keys[k]
else:
submit_keys[k] = None
log.warning(
"Requested key %s from 'post'"
" configuration list not available",
k,
)
# Get them read to be posted
real_submit_keys = {}
for (k, v) in submit_keys.items():
if v is None:
real_submit_keys[k] = "N/A"
else:
real_submit_keys[k] = str(v)
# Incase the url is parameterized
url_params = {
"INSTANCE_ID": all_keys["instance_id"],
}
url = templater.render_string(url, url_params)
try:
url_helper.read_file_or_url(
url,
data=real_submit_keys,
retries=tries,
sec_between=3,
ssl_details=util.fetch_ssl_details(cloud.paths),
)
except Exception:
util.logexc(log, "Failed to post phone home data to %s in %s tries",
url, tries)
def add_apt_sources(srcdict,
cloud,
target=None,
template_params=None,
aa_repo_match=None):
"""
install keys and repo source .list files defined in 'sources'
for each 'source' entry in the config:
1. expand template variables and write source .list file in
/etc/apt/sources.list.d/
2. install defined keys
3. update packages via distro-specific method (i.e. apt-key update)
@param srcdict: a dict containing elements required
@param cloud: cloud instance object
Example srcdict value:
{
'rio-grande-repo': {
'source': 'deb [signed-by=$KEY_FILE] $MIRROR $RELEASE main',
'keyid': 'B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77',
'keyserver': 'pgp.mit.edu'
}
}
Note: Deb822 format is not supported
"""
if template_params is None:
template_params = {}
if aa_repo_match is None:
raise ValueError("did not get a valid repo matcher")
if not isinstance(srcdict, dict):
raise TypeError("unknown apt format: %s" % (srcdict))
for filename in srcdict:
ent = srcdict[filename]
LOG.debug("adding source/key '%s'", ent)
if "filename" not in ent:
ent["filename"] = filename
if "source" in ent and "$KEY_FILE" in ent["source"]:
key_file = add_apt_key(ent, target, hardened=True)
template_params["KEY_FILE"] = key_file
else:
key_file = add_apt_key(ent, target)
if "source" not in ent:
continue
source = ent["source"]
source = templater.render_string(source, template_params)
if not ent["filename"].startswith("/"):
ent["filename"] = os.path.join("/etc/apt/sources.list.d/",
ent["filename"])
if not ent["filename"].endswith(".list"):
ent["filename"] += ".list"
if aa_repo_match(source):
try:
subp.subp(
["add-apt-repository", "--no-update", source],
target=target,
)
except subp.ProcessExecutionError:
LOG.exception("add-apt-repository failed.")
raise
continue
sourcefn = subp.target_path(target, ent["filename"])
try:
contents = "%s\n" % (source)
util.write_file(sourcefn, contents, omode="a")
except IOError as detail:
LOG.exception("failed write to file %s: %s", sourcefn, detail)
raise
update_packages(cloud)
return
def test_render_default(self):
blob = '''$a,$b'''
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEqual("1,2", c)
def test_render_jinja(self):
c = templater.render_string(self.jinja_tmpl, self.jinja_params)
self.assertEqual(self.jinja_expected, c)
def test_render_jinja(self):
blob = """## template:jinja
{{a}},{{b}}"""
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEqual("1,2", c)
def test_render_jinja(self):
c = templater.render_string(self.jinja_tmpl, self.jinja_params)
self.assertEqual(self.jinja_expected, c)
def test_render_jinja(self):
blob = '''## template:jinja
{{a}},{{b}}'''
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEquals("1,2", c)
def test_render_jinja_crlf(self):
blob = '\r\n'.join((
"## template:jinja",
"{{a}},{{b}}"))
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEqual("1,2", c)
def handle(name, cfg, cloud, log, args):
if len(args) != 0:
ph_cfg = util.read_conf(args[0])
else:
if not 'phone_home' in cfg:
log.debug(("Skipping module named %s, "
"no 'phone_home' configuration found"), name)
return
ph_cfg = cfg['phone_home']
if 'url' not in ph_cfg:
log.warn(("Skipping module named %s, "
"no 'url' found in 'phone_home' configuration"), name)
return
url = ph_cfg['url']
post_list = ph_cfg.get('post', 'all')
tries = ph_cfg.get('tries')
try:
tries = int(tries)
except:
tries = 10
util.logexc(log, "Configuration entry 'tries' is not an integer, "
"using %s instead", tries)
if post_list == "all":
post_list = POST_LIST_ALL
all_keys = {}
all_keys['instance_id'] = cloud.get_instance_id()
all_keys['hostname'] = cloud.get_hostname()
pubkeys = {
'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub',
'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub',
'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub',
}
for (n, path) in pubkeys.iteritems():
try:
all_keys[n] = util.load_file(path)
except:
util.logexc(log, "%s: failed to open, can not phone home that "
"data!", path)
submit_keys = {}
for k in post_list:
if k in all_keys:
submit_keys[k] = all_keys[k]
else:
submit_keys[k] = None
log.warn(("Requested key %s from 'post'"
" configuration list not available"), k)
# Get them read to be posted
real_submit_keys = {}
for (k, v) in submit_keys.iteritems():
if v is None:
real_submit_keys[k] = 'N/A'
else:
real_submit_keys[k] = str(v)
# Incase the url is parameterized
url_params = {
'INSTANCE_ID': all_keys['instance_id'],
}
url = templater.render_string(url, url_params)
try:
util.read_file_or_url(url, data=real_submit_keys,
retries=tries, sec_between=3,
ssl_details=util.fetch_ssl_details(cloud.paths))
except:
util.logexc(log, "Failed to post phone home data to %s in %s tries",
url, tries)
def test_render_explict_default(self):
blob = '\n'.join(('## template: basic', '$a,$b',))
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEqual("1,2", c)
def handle(name, _cfg, _cloud, log, _args):
svc_drop_in_path = "/etc/systemd/system/consul.service.d/10-cloud-init.conf"
join_drop_in_path = "/etc/systemd/system/consul-join.service.d/10-cloud-init.conf"
consul_wan_svc = "/etc/consul.d/service-consul-wan.json"
## consul config separated into two files because we may not have the acl
## token and encryption key at this time; it may be retrieved later from
## vault. the separate config files should make it easier to manage with
## more granularity.
## primary consul config file
main_consul_conf_path = "/etc/consul.conf"
## add'l config file that contains only acl_token and encrypt values
consul_creds_path = "/etc/consul.d/creds.json"
cfg = _cfg.get("consul")
if not cfg:
log.info("no consul config; not writing")
else:
main_consul_conf = {}
consul_creds = {}
for k, v in cfg["config"].items():
if k in ("acl_token", "encrypt"):
consul_creds[k] = v
else:
main_consul_conf[k] = v
## @todo add GCE support here
template_params = {
"private_ipv4": _cloud.datasource.metadata.get("local-ipv4"),
"public_ipv4": _cloud.datasource.metadata.get("public-ipv4"),
}
## render main config file
util.write_file(
main_consul_conf_path,
templater.render_string(json.dumps(main_consul_conf, indent=4),
template_params),
mode=0400,
)
util.chownbyname(main_consul_conf_path, "consul", "consul")
## render credentials file, if there are any credentials in there.
## consul.service will not start until the file exists, but even in the
## "legacy" environments we have the gossip encryption key.
if consul_creds:
util.write_file(
consul_creds_path,
templater.render_string(json.dumps(consul_creds, indent=4),
template_params),
mode=0400,
)
util.chownbyname(consul_creds_path, "consul", "consul")
## if this agent's a consul server, create a consul-wan service that
## advertises the wan address (which may be different than the lan
## address) and wan port
if main_consul_conf.get("server", False) is True:
wan_cfg = {
"name": "consul-wan",
"port": 8302,
}
## wow, they've certainly made the addresses … configurable.
serf_wan = main_consul_conf.get("advertise_addrs",
{}).get("serf_wan")
if serf_wan:
## advertise_addrs.serf_wan is "ip:port"
serf_wan_addr, serf_wan_port = serf_wan.split(":", 1)
wan_cfg["address"] = serf_wan_addr
wan_cfg["port"] = int(serf_wan_port)
else:
serf_wan_port = main_consul_conf.get("ports",
{}).get("serf_wan")
advertise_addr_wan = main_consul_conf.get("advertise_addr_wan")
if serf_wan_port:
wan_cfg["port"] = serf_wan_port
if advertise_addr_wan:
wan_cfg["address"] = advertise_addr_wan
if wan_cfg.get("address") is None:
log.warning("no wan address found in config")
## run through render_string as main_consul_conf is not rendered
util.write_file(
consul_wan_svc,
templater.render_string(
json.dumps({"service": wan_cfg}, indent=4),
template_params),
mode=0400,
)
util.chownbyname(consul_wan_svc, "consul", "consul")
## consul.service config
drop_in_buf = StringIO.StringIO()
print >> drop_in_buf, "[Service]"
## set GOMAXPROC
print >> drop_in_buf, 'Environment="GOMAXPROCS=%s"' % subprocess.check_output(
["nproc"]).strip()
util.write_file(svc_drop_in_path, drop_in_buf.getvalue(), 0400)
## consul-join.service config
drop_in_buf = StringIO.StringIO()
print >> drop_in_buf, "[Service]"
## set join method
print >> drop_in_buf, 'Environment="JOIN_METHOD=%s"' % cfg[
"join_method"]
util.write_file(join_drop_in_path, drop_in_buf.getvalue(), 0400)
def test_render_cheetah(self):
blob = """## template:cheetah
$a,$b"""
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEqual("1,2", c)
def handle(name, cfg, cloud, log, args):
if len(args) != 0:
ph_cfg = util.read_conf(args[0])
else:
if 'phone_home' not in cfg:
log.debug(("Skipping module named %s, "
"no 'phone_home' configuration found"), name)
return
ph_cfg = cfg['phone_home']
if 'url' not in ph_cfg:
log.warn(("Skipping module named %s, "
"no 'url' found in 'phone_home' configuration"), name)
return
url = ph_cfg['url']
post_list = ph_cfg.get('post', 'all')
tries = ph_cfg.get('tries')
try:
tries = int(tries)
except Exception:
tries = 10
util.logexc(
log, "Configuration entry 'tries' is not an integer, "
"using %s instead", tries)
if post_list == "all":
post_list = POST_LIST_ALL
all_keys = {}
all_keys['instance_id'] = cloud.get_instance_id()
all_keys['hostname'] = cloud.get_hostname()
all_keys['fqdn'] = cloud.get_hostname(fqdn=True)
pubkeys = {
'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub',
'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub',
'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub',
}
for (n, path) in pubkeys.items():
try:
all_keys[n] = util.load_file(path)
except Exception:
util.logexc(log, "%s: failed to open, can not phone home that "
"data!", path)
submit_keys = {}
for k in post_list:
if k in all_keys:
submit_keys[k] = all_keys[k]
else:
submit_keys[k] = None
log.warn(("Requested key %s from 'post'"
" configuration list not available"), k)
# Get them read to be posted
real_submit_keys = {}
for (k, v) in submit_keys.items():
if v is None:
real_submit_keys[k] = 'N/A'
else:
real_submit_keys[k] = str(v)
# Incase the url is parameterized
url_params = {
'INSTANCE_ID': all_keys['instance_id'],
}
url = templater.render_string(url, url_params)
try:
util.read_file_or_url(url,
data=real_submit_keys,
retries=tries,
sec_between=3,
ssl_details=util.fetch_ssl_details(cloud.paths))
except Exception:
util.logexc(log, "Failed to post phone home data to %s in %s tries",
url, tries)