def load_test_config(path):
"""
load a test config file by either abs path or rel path
"""
return merge_config(
c_util.read_conf(TESTCASE_CONF)['base_test_data'],
c_util.read_conf(name_to_path(path)))
def load_os_config(platform_name,
os_name,
require_enabled=False,
feature_overrides={}):
"""Load configuration for os.
@param platform_name: platform name to load os config for
@param os_name: name of os to retrieve config for
@param require_enabled: if true, raise error if 'enabled' not True
@param feature_overrides: feature flag overrides to merge with features
@return_value: config dict
"""
main_conf = c_util.read_conf(RELEASES_CONF)
default = main_conf['default_release_config']
image = main_conf['releases'][os_name]
conf = merge_config(
merge_config(get(default, 'default'), get(default, platform_name)),
merge_config(get(image, 'default'), get(image, platform_name)))
feature_conf = main_conf['features']
feature_groups = conf.get('feature_groups', [])
overrides = merge_config(get(conf, 'features'), feature_overrides)
conf['arch'] = c_util.get_architecture()
conf['features'] = merge_feature_groups(feature_conf, feature_groups,
overrides)
if require_enabled and not enabled(conf):
raise ValueError('OS is not enabled')
return conf
def __init__(self,cfgfile):
self.cloud = cloudinit.CloudInit()
self.cfg = util.read_conf(cfgfile)
self.cloud.get_data_source()
self.prepare_handlers()
def load_platform_config(platform):
"""
load configuration for platform
"""
main_conf = c_util.read_conf(PLATFORM_CONF)
return merge_config(main_conf.get('default_platform_config'),
main_conf.get('platforms')[platform])
def load_os_config(os_name):
"""
load configuration for os
"""
main_conf = c_util.read_conf(RELEASES_CONF)
return merge_config(main_conf.get('default_release_config'),
main_conf.get('releases')[os_name])
def _get_instance_configs(self):
i_cfgs = []
# If cloud-config was written, pick it up as
# a configuration file to use when running...
if not self._paths:
return i_cfgs
cc_paths = ['cloud_config']
if self._include_vendor:
# the order is important here: we want vendor2
# (dynamic vendor data from OpenStack)
# to override vendor (static data from OpenStack)
cc_paths.append('vendor2_cloud_config')
cc_paths.append('vendor_cloud_config')
for cc_p in cc_paths:
cc_fn = self._paths.get_ipath_cur(cc_p)
if cc_fn and os.path.isfile(cc_fn):
try:
i_cfgs.append(util.read_conf(cc_fn))
except PermissionError:
LOG.debug(
'Skipped loading cloud-config from %s due to'
' non-root.', cc_fn)
except Exception:
util.logexc(LOG, 'Failed loading of cloud-config from %s',
cc_fn)
return i_cfgs
def load_os_config(platform_name, os_name, require_enabled=False,
feature_overrides=None):
"""Load configuration for os.
@param platform_name: platform name to load os config for
@param os_name: name of os to retrieve config for
@param require_enabled: if true, raise error if 'enabled' not True
@param feature_overrides: feature flag overrides to merge with features
@return_value: config dict
"""
if feature_overrides is None:
feature_overrides = {}
main_conf = c_util.read_conf(RELEASES_CONF)
default = main_conf['default_release_config']
image = main_conf['releases'][os_name]
conf = merge_config(merge_config(get(default, 'default'),
get(default, platform_name)),
merge_config(get(image, 'default'),
get(image, platform_name)))
feature_conf = main_conf['features']
feature_groups = conf.get('feature_groups', [])
overrides = merge_config(get(conf, 'features'), feature_overrides)
conf['arch'] = c_util.get_architecture()
conf['features'] = merge_feature_groups(
feature_conf, feature_groups, overrides)
if require_enabled and not enabled(conf):
raise ValueError('OS is not enabled')
return conf
def from_state_file(state_file):
network_state = None
state = util.read_conf(state_file)
network_state = NetworkState()
network_state.load(state)
return network_state
def _get_env_configs(self):
e_cfgs = []
if CFG_ENV_NAME in os.environ:
e_fn = os.environ[CFG_ENV_NAME]
try:
e_cfgs.append(util.read_conf(e_fn))
except Exception:
util.logexc(LOG, 'Failed loading of env. config from %s', e_fn)
return e_cfgs
def _get_env_configs(self):
e_cfgs = []
if CFG_ENV_NAME in os.environ:
e_fn = os.environ[CFG_ENV_NAME]
try:
e_cfgs.append(util.read_conf(e_fn))
except Exception:
util.logexc(LOG, "Failed loading of env. config from %s", e_fn)
return e_cfgs
def get_config_obj(self,cfgfile):
try:
cfg = util.read_conf(cfgfile)
except:
cloudinit.log.critical("Failed loading of cloud config '%s'. Continuing with empty config\n" % cfgfile)
cloudinit.log.debug(traceback.format_exc() + "\n")
cfg = None
if cfg is None: cfg = { }
return(util.mergedict(cfg,self.cloud.cfg))
def parse_net_config(path, skip_broken=True):
"""Parses a curtin network configuration file and
return network state"""
ns = None
net_config = util.read_conf(path)
if 'network' in net_config:
ns = parse_net_config_data(net_config.get('network'),
skip_broken=skip_broken)
return ns
def parse_net_config(path):
"""Parses a curtin network configuration file and
return network state"""
ns = None
net_config = util.read_conf(path)
if 'network' in net_config:
ns = parse_net_config_data(net_config.get('network'))
return ns
def list_enabled_distros(platforms):
"""List all distros enabled for testing on specified platforms."""
def platform_has_enabled(config):
"""List if platform is enabled."""
return any(
enabled(merge_config(get(config, 'default'), get(
config, platform))) for platform in platforms)
releases = get(c_util.read_conf(RELEASES_CONF), 'releases')
return [k for k, v in releases.items() if platform_has_enabled(v)]
def list_enabled_distros(platforms):
"""List all distros enabled for testing on specified platforms."""
def platform_has_enabled(config):
"""List if platform is enabled."""
return any(enabled(merge_config(get(config, 'default'),
get(config, platform)))
for platform in platforms)
releases = get(c_util.read_conf(RELEASES_CONF), 'releases')
return [k for k, v in releases.items() if platform_has_enabled(v)]
def get_config_obj(self, cfgfile):
try:
cfg = util.read_conf(cfgfile)
except:
cloudinit.log.critical(
"Failed loading of cloud config '%s'. Continuing with empty config\n"
% cfgfile)
cloudinit.log.debug(traceback.format_exc() + "\n")
cfg = None
if cfg is None: cfg = {}
return (util.mergedict(cfg, self.cloud.cfg))
def load_platform_config(platform_name, require_enabled=False):
"""Load configuration for platform.
@param platform_name: name of platform to retrieve config for
@param require_enabled: if true, raise error if 'enabled' not True
@return_value: config dict
"""
main_conf = c_util.read_conf(PLATFORM_CONF)
conf = merge_config(main_conf['default_platform_config'],
main_conf['platforms'][platform_name])
if require_enabled and not enabled(conf):
raise ValueError('Platform is not enabled')
return conf
def load_platform_config(platform_name, require_enabled=False):
"""Load configuration for platform.
@param platform_name: name of platform to retrieve config for
@param require_enabled: if true, raise error if 'enabled' not True
@return_value: config dict
"""
main_conf = c_util.read_conf(PLATFORM_CONF)
conf = merge_config(main_conf['default_platform_config'],
main_conf['platforms'][platform_name])
if require_enabled and not enabled(conf):
raise ValueError('Platform is not enabled')
return conf
def _get_instance_configs(self):
i_cfgs = []
# If cloud-config was written, pick it up as
# a configuration file to use when running...
if not self._paths:
return i_cfgs
cc_fn = self._paths.get_ipath_cur('cloud_config')
if cc_fn and os.path.isfile(cc_fn):
try:
i_cfgs.append(util.read_conf(cc_fn))
except:
util.logexc(LOG, 'Failed loading of cloud-config from %s',
cc_fn)
return i_cfgs
def _get_instance_configs(self):
i_cfgs = []
# If cloud-config was written, pick it up as
# a configuration file to use when running...
if not self._paths:
return i_cfgs
cc_fn = self._paths.get_ipath_cur('cloud_config')
if cc_fn and os.path.isfile(cc_fn):
try:
i_cfgs.append(util.read_conf(cc_fn))
except:
util.logexc(LOG, 'Failed loading of cloud-config from %s',
cc_fn)
return i_cfgs
class DynamicTestSubclass(test_class):
_realclass = test_class
data = test_data
conf = test_conf
release_conf = read_conf(config.RELEASES_CONF)['releases']
def __str__(self):
return "%s (%s)" % (self._testMethodName,
unittest.util.strclass(self._realclass))
@classmethod
def setUpClass(cls):
cls.maybeSkipTest()
def get_config_obj(self, cfgfile):
try:
cfg = util.read_conf(cfgfile)
except:
# TODO: this 'log' could/should be passed in
cloudinit.log.critical("Failed loading of cloud config '%s'. "
"Continuing with empty config\n" % cfgfile)
cloudinit.log.debug(traceback.format_exc() + "\n")
cfg = None
if cfg is None:
cfg = {}
try:
ds_cfg = self.cloud.datasource.get_config_obj()
except:
ds_cfg = {}
cfg = util.mergedict(cfg, ds_cfg)
return (util.mergedict(cfg, self.cloud.cfg))
def get_config_obj(self, cfgfile):
try:
cfg = util.read_conf(cfgfile)
except:
# TODO: this 'log' could/should be passed in
cloudinit.log.critical("Failed loading of cloud config '%s'. "
"Continuing with empty config\n" % cfgfile)
cloudinit.log.debug(traceback.format_exc() + "\n")
cfg = None
if cfg is None:
cfg = {}
try:
ds_cfg = self.cloud.datasource.get_config_obj()
except:
ds_cfg = {}
cfg = util.mergedict(cfg, ds_cfg)
return(util.mergedict(cfg, self.cloud.cfg))
def _get_instance_configs(self):
i_cfgs = []
# If cloud-config was written, pick it up as
# a configuration file to use when running...
if not self._paths:
return i_cfgs
cc_paths = ["cloud_config"]
if self._include_vendor:
cc_paths.append("vendor_cloud_config")
for cc_p in cc_paths:
cc_fn = self._paths.get_ipath_cur(cc_p)
if cc_fn and os.path.isfile(cc_fn):
try:
i_cfgs.append(util.read_conf(cc_fn))
except Exception:
util.logexc(LOG, "Failed loading of cloud-config from %s", cc_fn)
return i_cfgs
def update_templates(self, template_config, template_data):
"""Update the image's template configuration.
Note, this will replace and delete the current pylxd_image
@param template_config: config overrides for template metadata
@param template_data: template data to place into templates/
"""
# set up tmp files
export_dir = tempfile.mkdtemp(prefix='cloud_test_util_')
extract_dir = tempfile.mkdtemp(prefix='cloud_test_util_')
new_metadata = os.path.join(export_dir, 'new-meta.tar.xz')
metadata_yaml = os.path.join(extract_dir, 'metadata.yaml')
template_dir = os.path.join(extract_dir, 'templates')
try:
# extract old data
(metadata, rootfs) = self.export_image(export_dir)
shutil.unpack_archive(metadata, extract_dir)
# update metadata
metadata = c_util.read_conf(metadata_yaml)
templates = metadata.get('templates', {})
templates.update(template_config)
metadata['templates'] = templates
util.yaml_dump(metadata, metadata_yaml)
# write out template files
for name, content in template_data.items():
path = os.path.join(template_dir, name)
c_util.write_file(path, content)
# store new data, mark new image as modified
util.flat_tar(new_metadata, extract_dir)
self.import_image(new_metadata, rootfs)
self.modified = True
finally:
# remove tmpfiles
shutil.rmtree(export_dir)
shutil.rmtree(extract_dir)
def update_templates(self, template_config, template_data):
"""Update the image's template configuration.
Note, this will replace and delete the current pylxd_image
@param template_config: config overrides for template metadata
@param template_data: template data to place into templates/
"""
# set up tmp files
export_dir = tempfile.mkdtemp(prefix='cloud_test_util_')
extract_dir = tempfile.mkdtemp(prefix='cloud_test_util_')
new_metadata = os.path.join(export_dir, 'new-meta.tar.xz')
metadata_yaml = os.path.join(extract_dir, 'metadata.yaml')
template_dir = os.path.join(extract_dir, 'templates')
try:
# extract old data
(metadata, rootfs) = self.export_image(export_dir)
shutil.unpack_archive(metadata, extract_dir)
# update metadata
metadata = c_util.read_conf(metadata_yaml)
templates = metadata.get('templates', {})
templates.update(template_config)
metadata['templates'] = templates
util.yaml_dump(metadata, metadata_yaml)
# write out template files
for name, content in template_data.items():
path = os.path.join(template_dir, name)
c_util.write_file(path, content)
# store new data, mark new image as modified
util.flat_tar(new_metadata, extract_dir)
self.import_image(new_metadata, rootfs)
self.modified = True
finally:
# remove tmpfiles
shutil.rmtree(export_dir)
shutil.rmtree(extract_dir)
def _read_cfg(self):
# Input config files override
# env config files which
# override instance configs
# which override datasource
# configs which override
# base configuration
cfgs = []
if self._fns:
for c_fn in self._fns:
try:
cfgs.append(util.read_conf(c_fn))
except Exception:
util.logexc(LOG, "Failed loading of configuration from %s", c_fn)
cfgs.extend(self._get_env_configs())
cfgs.extend(self._get_instance_configs())
cfgs.extend(self._get_datasource_configs())
if self._base_cfg:
cfgs.append(self._base_cfg)
return util.mergemanydict(cfgs)
def _read_cfg(self):
# Input config files override
# env config files which
# override instance configs
# which override datasource
# configs which override
# base configuration
cfgs = []
if self._fns:
for c_fn in self._fns:
try:
cfgs.append(util.read_conf(c_fn))
except Exception:
util.logexc(LOG, "Failed loading of configuration from %s",
c_fn)
cfgs.extend(self._get_env_configs())
cfgs.extend(self._get_instance_configs())
cfgs.extend(self._get_datasource_configs())
if self._base_cfg:
cfgs.append(self._base_cfg)
return util.mergemanydict(cfgs)
def handle(name, cfg, cloud, log, args):
if len(args) != 0:
caasp_cfg = util.read_conf(args[0])
else:
if 'suse_caasp' not in cfg:
log.debug(("Skipping module named %s, "
"no 'suse_caasp' configuration found"), name)
return
caasp_cfg = cfg['suse_caasp']
if 'role' not in caasp_cfg:
log.warn(("Skipping module named %s, "
"no 'role' found in 'suse_caasp' configuration"), name)
return
system_role = caasp_cfg['role']
if system_role == 'admin':
log.debug(("role administration node found"))
enable_service('admin-node-init')
enable_service('admin-node-setup')
enable_service('docker')
enable_service('etcd')
enable_service('kubelet')
enable_service('salt-minion')
elif system_role == 'cluster':
log.debug(("role cluster node found"))
if 'admin_node' not in caasp_cfg:
log.warn(("Skipping module named %s, "
"no 'admin_node' found for cluster system role"), name)
return
admin_node = caasp_cfg['admin_node']
contents = "master: %s" % (admin_node)
util.write_file('/etc/salt/minion.d/master.conf', contents, mode=DEFAULT_PERMS)
enable_service('docker')
enable_service('salt-minion')
else:
log.warn(("Unknown role %s, skipping module named %s"), role, name)
def _get_instance_configs(self):
i_cfgs = []
# If cloud-config was written, pick it up as
# a configuration file to use when running...
if not self._paths:
return i_cfgs
cc_paths = ['cloud_config']
if self._include_vendor:
cc_paths.append('vendor_cloud_config')
for cc_p in cc_paths:
cc_fn = self._paths.get_ipath_cur(cc_p)
if cc_fn and os.path.isfile(cc_fn):
try:
i_cfgs.append(util.read_conf(cc_fn))
except PermissionError:
LOG.debug(
'Skipped loading cloud-config from %s due to'
' non-root.', cc_fn)
except Exception:
util.logexc(LOG, 'Failed loading of cloud-config from %s',
cc_fn)
return i_cfgs
def handle(name, cfg, cloud, log, args):
if len(args) != 0:
ph_cfg = util.read_conf(args[0])
else:
if "phone_home" not in cfg:
log.debug(
"Skipping module named %s, "
"no 'phone_home' configuration found",
name,
)
return
ph_cfg = cfg["phone_home"]
if "url" not in ph_cfg:
log.warning(
"Skipping module named %s, "
"no 'url' found in 'phone_home' configuration",
name,
)
return
url = ph_cfg["url"]
post_list = ph_cfg.get("post", "all")
tries = ph_cfg.get("tries")
try:
tries = int(tries) # type: ignore
except ValueError:
tries = 10
util.logexc(
log,
"Configuration entry 'tries' is not an integer, using %s instead",
tries,
)
if post_list == "all":
post_list = POST_LIST_ALL
all_keys = {}
all_keys["instance_id"] = cloud.get_instance_id()
all_keys["hostname"] = cloud.get_hostname()
all_keys["fqdn"] = cloud.get_hostname(fqdn=True)
pubkeys = {
"pub_key_dsa": "/etc/ssh/ssh_host_dsa_key.pub",
"pub_key_rsa": "/etc/ssh/ssh_host_rsa_key.pub",
"pub_key_ecdsa": "/etc/ssh/ssh_host_ecdsa_key.pub",
"pub_key_ed25519": "/etc/ssh/ssh_host_ed25519_key.pub",
}
for (n, path) in pubkeys.items():
try:
all_keys[n] = util.load_file(path)
except Exception:
util.logexc(log,
"%s: failed to open, can not phone home that data!",
path)
submit_keys = {}
for k in post_list:
if k in all_keys:
submit_keys[k] = all_keys[k]
else:
submit_keys[k] = None
log.warning(
"Requested key %s from 'post'"
" configuration list not available",
k,
)
# Get them read to be posted
real_submit_keys = {}
for (k, v) in submit_keys.items():
if v is None:
real_submit_keys[k] = "N/A"
else:
real_submit_keys[k] = str(v)
# Incase the url is parameterized
url_params = {
"INSTANCE_ID": all_keys["instance_id"],
}
url = templater.render_string(url, url_params)
try:
url_helper.read_file_or_url(
url,
data=real_submit_keys,
retries=tries,
sec_between=3,
ssl_details=util.fetch_ssl_details(cloud.paths),
)
except Exception:
util.logexc(log, "Failed to post phone home data to %s in %s tries",
url, tries)
def main():
"""
Call with single argument of directory or http or https url.
If url is given additional arguments are allowed, which will be
interpreted as consumer_key, token_key, token_secret, consumer_secret
"""
import argparse
import pprint
parser = argparse.ArgumentParser(description='Interact with MAAS DS')
parser.add_argument("--config",
metavar="file",
help="specify DS config file",
default=None)
parser.add_argument("--ckey",
metavar="key",
help="the consumer key to auth with",
default=None)
parser.add_argument("--tkey",
metavar="key",
help="the token key to auth with",
default=None)
parser.add_argument("--csec",
metavar="secret",
help="the consumer secret (likely '')",
default="")
parser.add_argument("--tsec",
metavar="secret",
help="the token secret to auth with",
default=None)
parser.add_argument("--apiver",
metavar="version",
help="the apiver to use ("
" can be used)",
default=MD_VERSION)
subcmds = parser.add_subparsers(title="subcommands", dest="subcmd")
subcmds.add_parser('crawl', help="crawl the datasource")
subcmds.add_parser('get', help="do a single GET of provided url")
subcmds.add_parser('check-seed', help="read andn verify seed at url")
parser.add_argument("url", help="the data source to query")
args = parser.parse_args()
creds = {
'consumer_key': args.ckey,
'token_key': args.tkey,
'token_secret': args.tsec,
'consumer_secret': args.csec
}
if args.config:
cfg = util.read_conf(args.config)
if 'datasource' in cfg:
cfg = cfg['datasource']['MAAS']
for key in creds.keys():
if key in cfg and creds[key] is None:
creds[key] = cfg[key]
def geturl(url, headers_cb):
req = Request(url, data=None, headers=headers_cb(url))
return urlopen(req).read()
def printurl(url, headers_cb):
print("== %s ==\n%s\n" % (url, geturl(url, headers_cb)))
def crawl(url, headers_cb=None):
if url.endswith("/"):
for line in geturl(url, headers_cb).splitlines():
if line.endswith("/"):
crawl("%s%s" % (url, line), headers_cb)
else:
printurl("%s%s" % (url, line), headers_cb)
else:
printurl(url, headers_cb)
def my_headers(url):
headers = {}
if creds.get('consumer_key', None) is not None:
headers = oauth_headers(url, **creds)
return headers
if args.subcmd == "check-seed":
if args.url.startswith("http"):
(userdata, metadata) = read_maas_seed_url(args.url,
header_cb=my_headers,
version=args.apiver)
else:
(userdata, metadata) = read_maas_seed_url(args.url)
print("=== userdata ===")
print(userdata)
print("=== metadata ===")
pprint.pprint(metadata)
elif args.subcmd == "get":
printurl(args.url, my_headers)
elif args.subcmd == "crawl":
if not args.url.endswith("/"):
args.url = "%s/" % args.url
crawl(args.url, my_headers)
def main():
"""
Call with single argument of directory or http or https url.
If url is given additional arguments are allowed, which will be
interpreted as consumer_key, token_key, token_secret, consumer_secret
"""
import argparse
import pprint
import sys
parser = argparse.ArgumentParser(description="Interact with MAAS DS")
parser.add_argument(
"--config",
metavar="file",
help="specify DS config file",
default=None,
)
parser.add_argument(
"--ckey",
metavar="key",
help="the consumer key to auth with",
default=None,
)
parser.add_argument(
"--tkey",
metavar="key",
help="the token key to auth with",
default=None,
)
parser.add_argument(
"--csec",
metavar="secret",
help="the consumer secret (likely '')",
default="",
)
parser.add_argument(
"--tsec",
metavar="secret",
help="the token secret to auth with",
default=None,
)
parser.add_argument(
"--apiver",
metavar="version",
help="the apiver to use ( can be used)",
default=MD_VERSION,
)
subcmds = parser.add_subparsers(title="subcommands", dest="subcmd")
for (name, help) in (
("crawl", "crawl the datasource"),
("get", "do a single GET of provided url"),
("check-seed", "read and verify seed at url"),
):
p = subcmds.add_parser(name, help=help)
p.add_argument("url",
help="the datasource url",
nargs="?",
default=None)
args = parser.parse_args()
creds = {
"consumer_key": args.ckey,
"token_key": args.tkey,
"token_secret": args.tsec,
"consumer_secret": args.csec,
}
if args.config is None:
for fname in ("91_kernel_cmdline_url", "90_dpkg_maas"):
fpath = "/etc/cloud/cloud.cfg.d/" + fname + ".cfg"
if os.path.exists(fpath) and os.access(fpath, os.R_OK):
sys.stderr.write("Used config in %s.\n" % fpath)
args.config = fpath
if args.config:
cfg = util.read_conf(args.config)
if "datasource" in cfg:
cfg = cfg["datasource"]["MAAS"]
for key in creds.keys():
if key in cfg and creds[key] is None:
creds[key] = cfg[key]
if args.url is None and "metadata_url" in cfg:
args.url = cfg["metadata_url"]
if args.url is None:
sys.stderr.write("Must provide a url or a config with url.\n")
sys.exit(1)
oauth_helper = get_oauth_helper(creds)
def geturl(url):
# the retry is to ensure that oauth timestamp gets fixed
return oauth_helper.readurl(url, retries=1).contents
def printurl(url):
print("== %s ==\n%s\n" % (url, geturl(url).decode()))
def crawl(url):
if url.endswith("/"):
for line in geturl(url).decode().splitlines():
if line.endswith("/"):
crawl("%s%s" % (url, line))
elif line == "meta-data":
# meta-data is a dir, it *should* end in a /
crawl("%s%s" % (url, "meta-data/"))
else:
printurl("%s%s" % (url, line))
else:
printurl(url)
if args.subcmd == "check-seed":
sys.stderr.write("Checking seed at %s\n" % args.url)
readurl = oauth_helper.readurl
if args.url[0] == "/" or args.url.startswith("file://"):
(userdata, metadata, vd) = read_maas_seed_dir(args.url)
else:
(userdata, metadata, vd) = read_maas_seed_url(
args.url,
version=args.apiver,
read_file_or_url=readurl,
retries=2,
)
print("=== user-data ===")
print("N/A" if userdata is None else userdata.decode())
print("=== meta-data ===")
pprint.pprint(metadata)
print("=== vendor-data ===")
pprint.pprint("N/A" if vd is None else vd)
elif args.subcmd == "get":
printurl(args.url)
elif args.subcmd == "crawl":
if not args.url.endswith("/"):
args.url = "%s/" % args.url
crawl(args.url)
def read_runtime_config():
return util.read_conf(RUN_CLOUD_CONFIG)
def list_feature_flags():
"""List all supported feature flags."""
feature_conf = get(c_util.read_conf(RELEASES_CONF), 'features')
return feature_conf.get('all', [])
def handle(_name, cfg, cloud, log, args):
if len(args) != 0:
ph_cfg = util.read_conf(args[0])
else:
if not 'phone_home' in cfg:
return
ph_cfg = cfg['phone_home']
if 'url' not in ph_cfg:
log.warn("no 'url' token in phone_home")
return
url = ph_cfg['url']
post_list = ph_cfg.get('post', 'all')
tries = ph_cfg.get('tries', 10)
try:
tries = int(tries)
except:
log.warn("tries is not an integer. using 10")
tries = 10
if post_list == "all":
post_list = post_list_all
all_keys = {}
all_keys['instance_id'] = cloud.get_instance_id()
all_keys['hostname'] = cloud.get_hostname()
pubkeys = {
'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub',
'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub',
'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub',
}
for n, path in pubkeys.iteritems():
try:
fp = open(path, "rb")
all_keys[n] = fp.read()
fp.close()
except:
log.warn("%s: failed to open in phone_home" % path)
submit_keys = {}
for k in post_list:
if k in all_keys:
submit_keys[k] = all_keys[k]
else:
submit_keys[k] = "N/A"
log.warn("requested key %s from 'post' list not available")
url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']})
null_exc = object()
last_e = null_exc
for i in range(0, tries):
try:
util.readurl(url, submit_keys)
log.debug("succeeded submit to %s on try %i" % (url, i + 1))
return
except Exception as e:
log.debug("failed to post to %s on try %i" % (url, i + 1))
last_e = e
sleep(3)
log.warn("failed to post to %s in %i tries" % (url, tries))
if last_e is not null_exc:
raise(last_e)
return
def from_state_file(state_file):
state = util.read_conf(state_file)
nsi = NetworkStateInterpreter()
nsi.load(state)
return nsi
def main():
"""
Call with single argument of directory or http or https url.
If url is given additional arguments are allowed, which will be
interpreted as consumer_key, token_key, token_secret, consumer_secret
"""
import argparse
import pprint
parser = argparse.ArgumentParser(description='Interact with MAAS DS')
parser.add_argument("--config",
metavar="file",
help="specify DS config file",
default=None)
parser.add_argument("--ckey",
metavar="key",
help="the consumer key to auth with",
default=None)
parser.add_argument("--tkey",
metavar="key",
help="the token key to auth with",
default=None)
parser.add_argument("--csec",
metavar="secret",
help="the consumer secret (likely '')",
default="")
parser.add_argument("--tsec",
metavar="secret",
help="the token secret to auth with",
default=None)
parser.add_argument("--apiver",
metavar="version",
help="the apiver to use ("
" can be used)",
default=MD_VERSION)
subcmds = parser.add_subparsers(title="subcommands", dest="subcmd")
subcmds.add_parser('crawl', help="crawl the datasource")
subcmds.add_parser('get', help="do a single GET of provided url")
subcmds.add_parser('check-seed', help="read andn verify seed at url")
parser.add_argument("url", help="the data source to query")
args = parser.parse_args()
creds = {
'consumer_key': args.ckey,
'token_key': args.tkey,
'token_secret': args.tsec,
'consumer_secret': args.csec
}
if args.config:
cfg = util.read_conf(args.config)
if 'datasource' in cfg:
cfg = cfg['datasource']['MAAS']
for key in creds.keys():
if key in cfg and creds[key] is None:
creds[key] = cfg[key]
oauth_helper = url_helper.OauthUrlHelper(**creds)
def geturl(url):
# the retry is to ensure that oauth timestamp gets fixed
return oauth_helper.readurl(url, retries=1).contents
def printurl(url):
print("== %s ==\n%s\n" % (url, geturl(url).decode()))
def crawl(url):
if url.endswith("/"):
for line in geturl(url).decode().splitlines():
if line.endswith("/"):
crawl("%s%s" % (url, line))
elif line == "meta-data":
# meta-data is a dir, it *should* end in a /
crawl("%s%s" % (url, "meta-data/"))
else:
printurl("%s%s" % (url, line))
else:
printurl(url)
if args.subcmd == "check-seed":
readurl = oauth_helper.readurl
if args.url[0] == "/" or args.url.startswith("file://"):
readurl = None
(userdata, metadata) = read_maas_seed_url(args.url,
version=args.apiver,
read_file_or_url=readurl,
retries=2)
print("=== userdata ===")
print(userdata.decode())
print("=== metadata ===")
pprint.pprint(metadata)
elif args.subcmd == "get":
printurl(args.url)
elif args.subcmd == "crawl":
if not args.url.endswith("/"):
args.url = "%s/" % args.url
crawl(args.url)
def main():
"""
Call with single argument of directory or http or https url.
If url is given additional arguments are allowed, which will be
interpreted as consumer_key, token_key, token_secret, consumer_secret
"""
import argparse
import pprint
parser = argparse.ArgumentParser(description='Interact with MAAS DS')
parser.add_argument("--config", metavar="file",
help="specify DS config file", default=None)
parser.add_argument("--ckey", metavar="key",
help="the consumer key to auth with", default=None)
parser.add_argument("--tkey", metavar="key",
help="the token key to auth with", default=None)
parser.add_argument("--csec", metavar="secret",
help="the consumer secret (likely '')", default="")
parser.add_argument("--tsec", metavar="secret",
help="the token secret to auth with", default=None)
parser.add_argument("--apiver", metavar="version",
help="the apiver to use ("" can be used)", default=MD_VERSION)
subcmds = parser.add_subparsers(title="subcommands", dest="subcmd")
subcmds.add_parser('crawl', help="crawl the datasource")
subcmds.add_parser('get', help="do a single GET of provided url")
subcmds.add_parser('check-seed', help="read andn verify seed at url")
parser.add_argument("url", help="the data source to query")
args = parser.parse_args()
creds = {'consumer_key': args.ckey, 'token_key': args.tkey,
'token_secret': args.tsec, 'consumer_secret': args.csec}
if args.config:
cfg = util.read_conf(args.config)
if 'datasource' in cfg:
cfg = cfg['datasource']['MAAS']
for key in creds.keys():
if key in cfg and creds[key] is None:
creds[key] = cfg[key]
def geturl(url, headers_cb):
req = urllib2.Request(url, data=None, headers=headers_cb(url))
return (urllib2.urlopen(req).read())
def printurl(url, headers_cb):
print "== %s ==\n%s\n" % (url, geturl(url, headers_cb))
def crawl(url, headers_cb=None):
if url.endswith("/"):
for line in geturl(url, headers_cb).splitlines():
if line.endswith("/"):
crawl("%s%s" % (url, line), headers_cb)
else:
printurl("%s%s" % (url, line), headers_cb)
else:
printurl(url, headers_cb)
def my_headers(url):
headers = {}
if creds.get('consumer_key', None) is not None:
headers = oauth_headers(url, **creds)
return headers
if args.subcmd == "check-seed":
if args.url.startswith("http"):
(userdata, metadata) = read_maas_seed_url(args.url,
header_cb=my_headers,
version=args.apiver)
else:
(userdata, metadata) = read_maas_seed_url(args.url)
print "=== userdata ==="
print userdata
print "=== metadata ==="
pprint.pprint(metadata)
elif args.subcmd == "get":
printurl(args.url, my_headers)
elif args.subcmd == "crawl":
if not args.url.endswith("/"):
args.url = "%s/" % args.url
crawl(args.url, my_headers)
def list_enabled_platforms():
"""
list all platforms enabled for testing
"""
platforms = c_util.read_conf(PLATFORM_CONF).get('platforms')
return [k for k, v in platforms.items() if v.get('enabled')]
def main():
"""
Call with single argument of directory or http or https url.
If url is given additional arguments are allowed, which will be
interpreted as consumer_key, token_key, token_secret, consumer_secret
"""
import argparse
import pprint
parser = argparse.ArgumentParser(description='Interact with MAAS DS')
parser.add_argument("--config", metavar="file",
help="specify DS config file", default=None)
parser.add_argument("--ckey", metavar="key",
help="the consumer key to auth with", default=None)
parser.add_argument("--tkey", metavar="key",
help="the token key to auth with", default=None)
parser.add_argument("--csec", metavar="secret",
help="the consumer secret (likely '')", default="")
parser.add_argument("--tsec", metavar="secret",
help="the token secret to auth with", default=None)
parser.add_argument("--apiver", metavar="version",
help="the apiver to use ("" can be used)", default=MD_VERSION)
subcmds = parser.add_subparsers(title="subcommands", dest="subcmd")
subcmds.add_parser('crawl', help="crawl the datasource")
subcmds.add_parser('get', help="do a single GET of provided url")
subcmds.add_parser('check-seed', help="read andn verify seed at url")
parser.add_argument("url", help="the data source to query")
args = parser.parse_args()
creds = {'consumer_key': args.ckey, 'token_key': args.tkey,
'token_secret': args.tsec, 'consumer_secret': args.csec}
if args.config:
cfg = util.read_conf(args.config)
if 'datasource' in cfg:
cfg = cfg['datasource']['MAAS']
for key in creds.keys():
if key in cfg and creds[key] is None:
creds[key] = cfg[key]
oauth_helper = url_helper.OauthUrlHelper(**creds)
def geturl(url):
# the retry is to ensure that oauth timestamp gets fixed
return oauth_helper.readurl(url, retries=1).contents
def printurl(url):
print("== %s ==\n%s\n" % (url, geturl(url).decode()))
def crawl(url):
if url.endswith("/"):
for line in geturl(url).decode().splitlines():
if line.endswith("/"):
crawl("%s%s" % (url, line))
elif line == "meta-data":
# meta-data is a dir, it *should* end in a /
crawl("%s%s" % (url, "meta-data/"))
else:
printurl("%s%s" % (url, line))
else:
printurl(url)
if args.subcmd == "check-seed":
readurl = oauth_helper.readurl
if args.url[0] == "/" or args.url.startswith("file://"):
readurl = None
(userdata, metadata) = read_maas_seed_url(
args.url, version=args.apiver, read_file_or_url=readurl,
retries=2)
print("=== userdata ===")
print(userdata.decode())
print("=== metadata ===")
pprint.pprint(metadata)
elif args.subcmd == "get":
printurl(args.url)
elif args.subcmd == "crawl":
if not args.url.endswith("/"):
args.url = "%s/" % args.url
crawl(args.url)
def list_feature_flags():
"""List all supported feature flags."""
feature_conf = get(c_util.read_conf(RELEASES_CONF), 'features')
return feature_conf.get('all', [])
def from_state_file(state_file):
state = util.read_conf(state_file)
nsi = NetworkStateInterpreter()
nsi.load(state)
return nsi
def handle(name, cfg, cloud, log, args):
if len(args) != 0:
ph_cfg = util.read_conf(args[0])
else:
if not 'phone_home' in cfg:
log.debug(("Skipping module named %s, "
"no 'phone_home' configuration found"), name)
return
ph_cfg = cfg['phone_home']
if 'url' not in ph_cfg:
log.warn(("Skipping module named %s, "
"no 'url' found in 'phone_home' configuration"), name)
return
url = ph_cfg['url']
post_list = ph_cfg.get('post', 'all')
tries = ph_cfg.get('tries')
try:
tries = int(tries)
except:
tries = 10
util.logexc(log, "Configuration entry 'tries' is not an integer, "
"using %s instead", tries)
if post_list == "all":
post_list = POST_LIST_ALL
all_keys = {}
all_keys['instance_id'] = cloud.get_instance_id()
all_keys['hostname'] = cloud.get_hostname()
pubkeys = {
'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub',
'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub',
'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub',
}
for (n, path) in pubkeys.iteritems():
try:
all_keys[n] = util.load_file(path)
except:
util.logexc(log, "%s: failed to open, can not phone home that "
"data!", path)
submit_keys = {}
for k in post_list:
if k in all_keys:
submit_keys[k] = all_keys[k]
else:
submit_keys[k] = None
log.warn(("Requested key %s from 'post'"
" configuration list not available"), k)
# Get them read to be posted
real_submit_keys = {}
for (k, v) in submit_keys.iteritems():
if v is None:
real_submit_keys[k] = 'N/A'
else:
real_submit_keys[k] = str(v)
# Incase the url is parameterized
url_params = {
'INSTANCE_ID': all_keys['instance_id'],
}
url = templater.render_string(url, url_params)
try:
util.read_file_or_url(url, data=real_submit_keys,
retries=tries, sec_between=3,
ssl_details=util.fetch_ssl_details(cloud.paths))
except:
util.logexc(log, "Failed to post phone home data to %s in %s tries",
url, tries)
def load_test_config(path):
"""Load a test config file by either abs path or rel path."""
return merge_config(c_util.read_conf(TESTCASE_CONF)['base_test_data'],
c_util.read_conf(name_to_path(path)))
def list_enabled_platforms():
"""List all platforms enabled for testing."""
platforms = get(c_util.read_conf(PLATFORM_CONF), 'platforms')
return [k for k, v in platforms.items() if enabled(v)]
def handle(name, cfg, cloud, log, args):
if len(args) != 0:
ph_cfg = util.read_conf(args[0])
else:
if 'phone_home' not in cfg:
log.debug(("Skipping module named %s, "
"no 'phone_home' configuration found"), name)
return
ph_cfg = cfg['phone_home']
if 'url' not in ph_cfg:
log.warn(("Skipping module named %s, "
"no 'url' found in 'phone_home' configuration"), name)
return
url = ph_cfg['url']
post_list = ph_cfg.get('post', 'all')
tries = ph_cfg.get('tries')
try:
tries = int(tries)
except Exception:
tries = 10
util.logexc(
log, "Configuration entry 'tries' is not an integer, "
"using %s instead", tries)
if post_list == "all":
post_list = POST_LIST_ALL
all_keys = {}
all_keys['instance_id'] = cloud.get_instance_id()
all_keys['hostname'] = cloud.get_hostname()
all_keys['fqdn'] = cloud.get_hostname(fqdn=True)
pubkeys = {
'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub',
'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub',
'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub',
}
for (n, path) in pubkeys.items():
try:
all_keys[n] = util.load_file(path)
except Exception:
util.logexc(log, "%s: failed to open, can not phone home that "
"data!", path)
submit_keys = {}
for k in post_list:
if k in all_keys:
submit_keys[k] = all_keys[k]
else:
submit_keys[k] = None
log.warn(("Requested key %s from 'post'"
" configuration list not available"), k)
# Get them read to be posted
real_submit_keys = {}
for (k, v) in submit_keys.items():
if v is None:
real_submit_keys[k] = 'N/A'
else:
real_submit_keys[k] = str(v)
# Incase the url is parameterized
url_params = {
'INSTANCE_ID': all_keys['instance_id'],
}
url = templater.render_string(url, url_params)
try:
util.read_file_or_url(url,
data=real_submit_keys,
retries=tries,
sec_between=3,
ssl_details=util.fetch_ssl_details(cloud.paths))
except Exception:
util.logexc(log, "Failed to post phone home data to %s in %s tries",
url, tries)
def list_enabled_platforms():
"""List all platforms enabled for testing."""
platforms = get(c_util.read_conf(PLATFORM_CONF), 'platforms')
return [k for k, v in platforms.items() if enabled(v)]
post_onboard_enabled = False
if 'post_onboard_enabled' in cloud_config[name]:
post_onboard_enabled = bool(
cloud_config[name]['post_onboard_enabled'])
if post_onboard_enabled and onboard_status == SUCCESS:
if not runs_post_onboard_commands(name):
onboard_status = ERROR
# call phone_home
if phone_home_url:
tmos_onboard_utils.phone_home(
phone_home_url, True, True, onboard_status, phone_home_url_verify_tls)
if phone_home_cli and onboard_status == SUCCESS:
tmos_onboard_utils.run_cmd(phone_home_cli)
clean()
LOG.info('onboarding ended with status: %s', onboard_status)
if __name__ == "__main__":
# Running the cloud-init module from the CLI python interpreter
CLOUD_CONFIG_FILE = '/opt/cloud/instance/cloud-config.txt'
CLOUD_CONFIG = {}
if os.path.exists(CLOUD_CONFIG_FILE):
CLOUD_CONFIG = util.read_conf(CLOUD_CONFIG_FILE)
handle(sys.argv[1], CLOUD_CONFIG, None, logging, [])
def main():
"""
Call with single argument of directory or http or https url.
If url is given additional arguments are allowed, which will be
interpreted as consumer_key, token_key, token_secret, consumer_secret
"""
import argparse
import pprint
import sys
parser = argparse.ArgumentParser(description='Interact with MAAS DS')
parser.add_argument("--config", metavar="file",
help="specify DS config file", default=None)
parser.add_argument("--ckey", metavar="key",
help="the consumer key to auth with", default=None)
parser.add_argument("--tkey", metavar="key",
help="the token key to auth with", default=None)
parser.add_argument("--csec", metavar="secret",
help="the consumer secret (likely '')", default="")
parser.add_argument("--tsec", metavar="secret",
help="the token secret to auth with", default=None)
parser.add_argument("--apiver", metavar="version",
help="the apiver to use ("" can be used)",
default=MD_VERSION)
subcmds = parser.add_subparsers(title="subcommands", dest="subcmd")
for (name, help) in (('crawl', 'crawl the datasource'),
('get', 'do a single GET of provided url'),
('check-seed', 'read and verify seed at url')):
p = subcmds.add_parser(name, help=help)
p.add_argument("url", help="the datasource url", nargs='?',
default=None)
args = parser.parse_args()
creds = {'consumer_key': args.ckey, 'token_key': args.tkey,
'token_secret': args.tsec, 'consumer_secret': args.csec}
if args.config is None:
for fname in ('91_kernel_cmdline_url', '90_dpkg_maas'):
fpath = "/etc/cloud/cloud.cfg.d/" + fname + ".cfg"
if os.path.exists(fpath) and os.access(fpath, os.R_OK):
sys.stderr.write("Used config in %s.\n" % fpath)
args.config = fpath
if args.config:
cfg = util.read_conf(args.config)
if 'datasource' in cfg:
cfg = cfg['datasource']['MAAS']
for key in creds.keys():
if key in cfg and creds[key] is None:
creds[key] = cfg[key]
if args.url is None and 'metadata_url' in cfg:
args.url = cfg['metadata_url']
if args.url is None:
sys.stderr.write("Must provide a url or a config with url.\n")
sys.exit(1)
oauth_helper = get_oauth_helper(creds)
def geturl(url):
# the retry is to ensure that oauth timestamp gets fixed
return oauth_helper.readurl(url, retries=1).contents
def printurl(url):
print("== %s ==\n%s\n" % (url, geturl(url).decode()))
def crawl(url):
if url.endswith("/"):
for line in geturl(url).decode().splitlines():
if line.endswith("/"):
crawl("%s%s" % (url, line))
elif line == "meta-data":
# meta-data is a dir, it *should* end in a /
crawl("%s%s" % (url, "meta-data/"))
else:
printurl("%s%s" % (url, line))
else:
printurl(url)
if args.subcmd == "check-seed":
sys.stderr.write("Checking seed at %s\n" % args.url)
readurl = oauth_helper.readurl
if args.url[0] == "/" or args.url.startswith("file://"):
(userdata, metadata, vd) = read_maas_seed_dir(args.url)
else:
(userdata, metadata, vd) = read_maas_seed_url(
args.url, version=args.apiver, read_file_or_url=readurl,
retries=2)
print("=== user-data ===")
print("N/A" if userdata is None else userdata.decode())
print("=== meta-data ===")
pprint.pprint(metadata)
print("=== vendor-data ===")
pprint.pprint("N/A" if vd is None else vd)
elif args.subcmd == "get":
printurl(args.url)
elif args.subcmd == "crawl":
if not args.url.endswith("/"):
args.url = "%s/" % args.url
crawl(args.url)
def read_runtime_config():
return util.read_conf(RUN_CLOUD_CONFIG)
ns1_state = "/tmp/ns1.state"
with open(ns1_state, "w+") as f:
f.write(ns1_dump)
print("Loading state from file")
ns2 = from_state_file(ns1_state)
print("NS1 == NS2 ?=> {}".format(
ns1.network_state == ns2.network_state))
def test_output(network_config):
(version, config) = load_config(network_config)
ns1 = NetworkState(version=version, config=config)
ns1.parse_config()
random.shuffle(config)
ns2 = NetworkState(version=version, config=config)
ns2.parse_config()
print("NS1 == NS2 ?=> {}".format(
ns1.network_state == ns2.network_state))
eni_1 = net.render_interfaces(ns1.network_state)
eni_2 = net.render_interfaces(ns2.network_state)
print(eni_1)
print(eni_2)
print("eni_1 == eni_2 ?=> {}".format(
eni_1 == eni_2))
y = util.read_conf(sys.argv[1])
network_config = y.get('network')
test_parse(network_config)
test_dump_and_load(network_config)
test_output(network_config)
