def load_test_config(path): """ load a test config file by either abs path or rel path """ return merge_config( c_util.read_conf(TESTCASE_CONF)['base_test_data'], c_util.read_conf(name_to_path(path)))
def load_os_config(platform_name, os_name, require_enabled=False, feature_overrides={}): """Load configuration for os. @param platform_name: platform name to load os config for @param os_name: name of os to retrieve config for @param require_enabled: if true, raise error if 'enabled' not True @param feature_overrides: feature flag overrides to merge with features @return_value: config dict """ main_conf = c_util.read_conf(RELEASES_CONF) default = main_conf['default_release_config'] image = main_conf['releases'][os_name] conf = merge_config( merge_config(get(default, 'default'), get(default, platform_name)), merge_config(get(image, 'default'), get(image, platform_name))) feature_conf = main_conf['features'] feature_groups = conf.get('feature_groups', []) overrides = merge_config(get(conf, 'features'), feature_overrides) conf['arch'] = c_util.get_architecture() conf['features'] = merge_feature_groups(feature_conf, feature_groups, overrides) if require_enabled and not enabled(conf): raise ValueError('OS is not enabled') return conf
def __init__(self,cfgfile): self.cloud = cloudinit.CloudInit() self.cfg = util.read_conf(cfgfile) self.cloud.get_data_source() self.prepare_handlers()
def load_platform_config(platform): """ load configuration for platform """ main_conf = c_util.read_conf(PLATFORM_CONF) return merge_config(main_conf.get('default_platform_config'), main_conf.get('platforms')[platform])
def load_os_config(os_name): """ load configuration for os """ main_conf = c_util.read_conf(RELEASES_CONF) return merge_config(main_conf.get('default_release_config'), main_conf.get('releases')[os_name])
def _get_instance_configs(self): i_cfgs = [] # If cloud-config was written, pick it up as # a configuration file to use when running... if not self._paths: return i_cfgs cc_paths = ['cloud_config'] if self._include_vendor: # the order is important here: we want vendor2 # (dynamic vendor data from OpenStack) # to override vendor (static data from OpenStack) cc_paths.append('vendor2_cloud_config') cc_paths.append('vendor_cloud_config') for cc_p in cc_paths: cc_fn = self._paths.get_ipath_cur(cc_p) if cc_fn and os.path.isfile(cc_fn): try: i_cfgs.append(util.read_conf(cc_fn)) except PermissionError: LOG.debug( 'Skipped loading cloud-config from %s due to' ' non-root.', cc_fn) except Exception: util.logexc(LOG, 'Failed loading of cloud-config from %s', cc_fn) return i_cfgs
def load_os_config(platform_name, os_name, require_enabled=False, feature_overrides=None): """Load configuration for os. @param platform_name: platform name to load os config for @param os_name: name of os to retrieve config for @param require_enabled: if true, raise error if 'enabled' not True @param feature_overrides: feature flag overrides to merge with features @return_value: config dict """ if feature_overrides is None: feature_overrides = {} main_conf = c_util.read_conf(RELEASES_CONF) default = main_conf['default_release_config'] image = main_conf['releases'][os_name] conf = merge_config(merge_config(get(default, 'default'), get(default, platform_name)), merge_config(get(image, 'default'), get(image, platform_name))) feature_conf = main_conf['features'] feature_groups = conf.get('feature_groups', []) overrides = merge_config(get(conf, 'features'), feature_overrides) conf['arch'] = c_util.get_architecture() conf['features'] = merge_feature_groups( feature_conf, feature_groups, overrides) if require_enabled and not enabled(conf): raise ValueError('OS is not enabled') return conf
def from_state_file(state_file): network_state = None state = util.read_conf(state_file) network_state = NetworkState() network_state.load(state) return network_state
def _get_env_configs(self): e_cfgs = [] if CFG_ENV_NAME in os.environ: e_fn = os.environ[CFG_ENV_NAME] try: e_cfgs.append(util.read_conf(e_fn)) except Exception: util.logexc(LOG, 'Failed loading of env. config from %s', e_fn) return e_cfgs
def _get_env_configs(self): e_cfgs = [] if CFG_ENV_NAME in os.environ: e_fn = os.environ[CFG_ENV_NAME] try: e_cfgs.append(util.read_conf(e_fn)) except Exception: util.logexc(LOG, "Failed loading of env. config from %s", e_fn) return e_cfgs
def get_config_obj(self,cfgfile): try: cfg = util.read_conf(cfgfile) except: cloudinit.log.critical("Failed loading of cloud config '%s'. Continuing with empty config\n" % cfgfile) cloudinit.log.debug(traceback.format_exc() + "\n") cfg = None if cfg is None: cfg = { } return(util.mergedict(cfg,self.cloud.cfg))
def parse_net_config(path, skip_broken=True): """Parses a curtin network configuration file and return network state""" ns = None net_config = util.read_conf(path) if 'network' in net_config: ns = parse_net_config_data(net_config.get('network'), skip_broken=skip_broken) return ns
def parse_net_config(path): """Parses a curtin network configuration file and return network state""" ns = None net_config = util.read_conf(path) if 'network' in net_config: ns = parse_net_config_data(net_config.get('network')) return ns
def list_enabled_distros(platforms): """List all distros enabled for testing on specified platforms.""" def platform_has_enabled(config): """List if platform is enabled.""" return any( enabled(merge_config(get(config, 'default'), get( config, platform))) for platform in platforms) releases = get(c_util.read_conf(RELEASES_CONF), 'releases') return [k for k, v in releases.items() if platform_has_enabled(v)]
def list_enabled_distros(platforms): """List all distros enabled for testing on specified platforms.""" def platform_has_enabled(config): """List if platform is enabled.""" return any(enabled(merge_config(get(config, 'default'), get(config, platform))) for platform in platforms) releases = get(c_util.read_conf(RELEASES_CONF), 'releases') return [k for k, v in releases.items() if platform_has_enabled(v)]
def get_config_obj(self, cfgfile): try: cfg = util.read_conf(cfgfile) except: cloudinit.log.critical( "Failed loading of cloud config '%s'. Continuing with empty config\n" % cfgfile) cloudinit.log.debug(traceback.format_exc() + "\n") cfg = None if cfg is None: cfg = {} return (util.mergedict(cfg, self.cloud.cfg))
def load_platform_config(platform_name, require_enabled=False): """Load configuration for platform. @param platform_name: name of platform to retrieve config for @param require_enabled: if true, raise error if 'enabled' not True @return_value: config dict """ main_conf = c_util.read_conf(PLATFORM_CONF) conf = merge_config(main_conf['default_platform_config'], main_conf['platforms'][platform_name]) if require_enabled and not enabled(conf): raise ValueError('Platform is not enabled') return conf
def _get_instance_configs(self): i_cfgs = [] # If cloud-config was written, pick it up as # a configuration file to use when running... if not self._paths: return i_cfgs cc_fn = self._paths.get_ipath_cur('cloud_config') if cc_fn and os.path.isfile(cc_fn): try: i_cfgs.append(util.read_conf(cc_fn)) except: util.logexc(LOG, 'Failed loading of cloud-config from %s', cc_fn) return i_cfgs
class DynamicTestSubclass(test_class): _realclass = test_class data = test_data conf = test_conf release_conf = read_conf(config.RELEASES_CONF)['releases'] def __str__(self): return "%s (%s)" % (self._testMethodName, unittest.util.strclass(self._realclass)) @classmethod def setUpClass(cls): cls.maybeSkipTest()
def get_config_obj(self, cfgfile): try: cfg = util.read_conf(cfgfile) except: # TODO: this 'log' could/should be passed in cloudinit.log.critical("Failed loading of cloud config '%s'. " "Continuing with empty config\n" % cfgfile) cloudinit.log.debug(traceback.format_exc() + "\n") cfg = None if cfg is None: cfg = {} try: ds_cfg = self.cloud.datasource.get_config_obj() except: ds_cfg = {} cfg = util.mergedict(cfg, ds_cfg) return (util.mergedict(cfg, self.cloud.cfg))
def get_config_obj(self, cfgfile): try: cfg = util.read_conf(cfgfile) except: # TODO: this 'log' could/should be passed in cloudinit.log.critical("Failed loading of cloud config '%s'. " "Continuing with empty config\n" % cfgfile) cloudinit.log.debug(traceback.format_exc() + "\n") cfg = None if cfg is None: cfg = {} try: ds_cfg = self.cloud.datasource.get_config_obj() except: ds_cfg = {} cfg = util.mergedict(cfg, ds_cfg) return(util.mergedict(cfg, self.cloud.cfg))
def _get_instance_configs(self): i_cfgs = [] # If cloud-config was written, pick it up as # a configuration file to use when running... if not self._paths: return i_cfgs cc_paths = ["cloud_config"] if self._include_vendor: cc_paths.append("vendor_cloud_config") for cc_p in cc_paths: cc_fn = self._paths.get_ipath_cur(cc_p) if cc_fn and os.path.isfile(cc_fn): try: i_cfgs.append(util.read_conf(cc_fn)) except Exception: util.logexc(LOG, "Failed loading of cloud-config from %s", cc_fn) return i_cfgs
def update_templates(self, template_config, template_data): """Update the image's template configuration. Note, this will replace and delete the current pylxd_image @param template_config: config overrides for template metadata @param template_data: template data to place into templates/ """ # set up tmp files export_dir = tempfile.mkdtemp(prefix='cloud_test_util_') extract_dir = tempfile.mkdtemp(prefix='cloud_test_util_') new_metadata = os.path.join(export_dir, 'new-meta.tar.xz') metadata_yaml = os.path.join(extract_dir, 'metadata.yaml') template_dir = os.path.join(extract_dir, 'templates') try: # extract old data (metadata, rootfs) = self.export_image(export_dir) shutil.unpack_archive(metadata, extract_dir) # update metadata metadata = c_util.read_conf(metadata_yaml) templates = metadata.get('templates', {}) templates.update(template_config) metadata['templates'] = templates util.yaml_dump(metadata, metadata_yaml) # write out template files for name, content in template_data.items(): path = os.path.join(template_dir, name) c_util.write_file(path, content) # store new data, mark new image as modified util.flat_tar(new_metadata, extract_dir) self.import_image(new_metadata, rootfs) self.modified = True finally: # remove tmpfiles shutil.rmtree(export_dir) shutil.rmtree(extract_dir)
def _read_cfg(self): # Input config files override # env config files which # override instance configs # which override datasource # configs which override # base configuration cfgs = [] if self._fns: for c_fn in self._fns: try: cfgs.append(util.read_conf(c_fn)) except Exception: util.logexc(LOG, "Failed loading of configuration from %s", c_fn) cfgs.extend(self._get_env_configs()) cfgs.extend(self._get_instance_configs()) cfgs.extend(self._get_datasource_configs()) if self._base_cfg: cfgs.append(self._base_cfg) return util.mergemanydict(cfgs)
def handle(name, cfg, cloud, log, args): if len(args) != 0: caasp_cfg = util.read_conf(args[0]) else: if 'suse_caasp' not in cfg: log.debug(("Skipping module named %s, " "no 'suse_caasp' configuration found"), name) return caasp_cfg = cfg['suse_caasp'] if 'role' not in caasp_cfg: log.warn(("Skipping module named %s, " "no 'role' found in 'suse_caasp' configuration"), name) return system_role = caasp_cfg['role'] if system_role == 'admin': log.debug(("role administration node found")) enable_service('admin-node-init') enable_service('admin-node-setup') enable_service('docker') enable_service('etcd') enable_service('kubelet') enable_service('salt-minion') elif system_role == 'cluster': log.debug(("role cluster node found")) if 'admin_node' not in caasp_cfg: log.warn(("Skipping module named %s, " "no 'admin_node' found for cluster system role"), name) return admin_node = caasp_cfg['admin_node'] contents = "master: %s" % (admin_node) util.write_file('/etc/salt/minion.d/master.conf', contents, mode=DEFAULT_PERMS) enable_service('docker') enable_service('salt-minion') else: log.warn(("Unknown role %s, skipping module named %s"), role, name)
def _get_instance_configs(self): i_cfgs = [] # If cloud-config was written, pick it up as # a configuration file to use when running... if not self._paths: return i_cfgs cc_paths = ['cloud_config'] if self._include_vendor: cc_paths.append('vendor_cloud_config') for cc_p in cc_paths: cc_fn = self._paths.get_ipath_cur(cc_p) if cc_fn and os.path.isfile(cc_fn): try: i_cfgs.append(util.read_conf(cc_fn)) except PermissionError: LOG.debug( 'Skipped loading cloud-config from %s due to' ' non-root.', cc_fn) except Exception: util.logexc(LOG, 'Failed loading of cloud-config from %s', cc_fn) return i_cfgs
def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: if "phone_home" not in cfg: log.debug( "Skipping module named %s, " "no 'phone_home' configuration found", name, ) return ph_cfg = cfg["phone_home"] if "url" not in ph_cfg: log.warning( "Skipping module named %s, " "no 'url' found in 'phone_home' configuration", name, ) return url = ph_cfg["url"] post_list = ph_cfg.get("post", "all") tries = ph_cfg.get("tries") try: tries = int(tries) # type: ignore except ValueError: tries = 10 util.logexc( log, "Configuration entry 'tries' is not an integer, using %s instead", tries, ) if post_list == "all": post_list = POST_LIST_ALL all_keys = {} all_keys["instance_id"] = cloud.get_instance_id() all_keys["hostname"] = cloud.get_hostname() all_keys["fqdn"] = cloud.get_hostname(fqdn=True) pubkeys = { "pub_key_dsa": "/etc/ssh/ssh_host_dsa_key.pub", "pub_key_rsa": "/etc/ssh/ssh_host_rsa_key.pub", "pub_key_ecdsa": "/etc/ssh/ssh_host_ecdsa_key.pub", "pub_key_ed25519": "/etc/ssh/ssh_host_ed25519_key.pub", } for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) except Exception: util.logexc(log, "%s: failed to open, can not phone home that data!", path) submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] else: submit_keys[k] = None log.warning( "Requested key %s from 'post'" " configuration list not available", k, ) # Get them read to be posted real_submit_keys = {} for (k, v) in submit_keys.items(): if v is None: real_submit_keys[k] = "N/A" else: real_submit_keys[k] = str(v) # Incase the url is parameterized url_params = { "INSTANCE_ID": all_keys["instance_id"], } url = templater.render_string(url, url_params) try: url_helper.read_file_or_url( url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths), ) except Exception: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries)
def main(): """ Call with single argument of directory or http or https url. If url is given additional arguments are allowed, which will be interpreted as consumer_key, token_key, token_secret, consumer_secret """ import argparse import pprint parser = argparse.ArgumentParser(description='Interact with MAAS DS') parser.add_argument("--config", metavar="file", help="specify DS config file", default=None) parser.add_argument("--ckey", metavar="key", help="the consumer key to auth with", default=None) parser.add_argument("--tkey", metavar="key", help="the token key to auth with", default=None) parser.add_argument("--csec", metavar="secret", help="the consumer secret (likely '')", default="") parser.add_argument("--tsec", metavar="secret", help="the token secret to auth with", default=None) parser.add_argument("--apiver", metavar="version", help="the apiver to use (" " can be used)", default=MD_VERSION) subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") subcmds.add_parser('crawl', help="crawl the datasource") subcmds.add_parser('get', help="do a single GET of provided url") subcmds.add_parser('check-seed', help="read andn verify seed at url") parser.add_argument("url", help="the data source to query") args = parser.parse_args() creds = { 'consumer_key': args.ckey, 'token_key': args.tkey, 'token_secret': args.tsec, 'consumer_secret': args.csec } if args.config: cfg = util.read_conf(args.config) if 'datasource' in cfg: cfg = cfg['datasource']['MAAS'] for key in creds.keys(): if key in cfg and creds[key] is None: creds[key] = cfg[key] def geturl(url, headers_cb): req = Request(url, data=None, headers=headers_cb(url)) return urlopen(req).read() def printurl(url, headers_cb): print("== %s ==\n%s\n" % (url, geturl(url, headers_cb))) def crawl(url, headers_cb=None): if url.endswith("/"): for line in geturl(url, headers_cb).splitlines(): if line.endswith("/"): crawl("%s%s" % (url, line), headers_cb) else: printurl("%s%s" % (url, line), headers_cb) else: printurl(url, headers_cb) def my_headers(url): headers = {} if creds.get('consumer_key', None) is not None: headers = oauth_headers(url, **creds) return headers if args.subcmd == "check-seed": if args.url.startswith("http"): (userdata, metadata) = read_maas_seed_url(args.url, header_cb=my_headers, version=args.apiver) else: (userdata, metadata) = read_maas_seed_url(args.url) print("=== userdata ===") print(userdata) print("=== metadata ===") pprint.pprint(metadata) elif args.subcmd == "get": printurl(args.url, my_headers) elif args.subcmd == "crawl": if not args.url.endswith("/"): args.url = "%s/" % args.url crawl(args.url, my_headers)
def main(): """ Call with single argument of directory or http or https url. If url is given additional arguments are allowed, which will be interpreted as consumer_key, token_key, token_secret, consumer_secret """ import argparse import pprint import sys parser = argparse.ArgumentParser(description="Interact with MAAS DS") parser.add_argument( "--config", metavar="file", help="specify DS config file", default=None, ) parser.add_argument( "--ckey", metavar="key", help="the consumer key to auth with", default=None, ) parser.add_argument( "--tkey", metavar="key", help="the token key to auth with", default=None, ) parser.add_argument( "--csec", metavar="secret", help="the consumer secret (likely '')", default="", ) parser.add_argument( "--tsec", metavar="secret", help="the token secret to auth with", default=None, ) parser.add_argument( "--apiver", metavar="version", help="the apiver to use ( can be used)", default=MD_VERSION, ) subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") for (name, help) in ( ("crawl", "crawl the datasource"), ("get", "do a single GET of provided url"), ("check-seed", "read and verify seed at url"), ): p = subcmds.add_parser(name, help=help) p.add_argument("url", help="the datasource url", nargs="?", default=None) args = parser.parse_args() creds = { "consumer_key": args.ckey, "token_key": args.tkey, "token_secret": args.tsec, "consumer_secret": args.csec, } if args.config is None: for fname in ("91_kernel_cmdline_url", "90_dpkg_maas"): fpath = "/etc/cloud/cloud.cfg.d/" + fname + ".cfg" if os.path.exists(fpath) and os.access(fpath, os.R_OK): sys.stderr.write("Used config in %s.\n" % fpath) args.config = fpath if args.config: cfg = util.read_conf(args.config) if "datasource" in cfg: cfg = cfg["datasource"]["MAAS"] for key in creds.keys(): if key in cfg and creds[key] is None: creds[key] = cfg[key] if args.url is None and "metadata_url" in cfg: args.url = cfg["metadata_url"] if args.url is None: sys.stderr.write("Must provide a url or a config with url.\n") sys.exit(1) oauth_helper = get_oauth_helper(creds) def geturl(url): # the retry is to ensure that oauth timestamp gets fixed return oauth_helper.readurl(url, retries=1).contents def printurl(url): print("== %s ==\n%s\n" % (url, geturl(url).decode())) def crawl(url): if url.endswith("/"): for line in geturl(url).decode().splitlines(): if line.endswith("/"): crawl("%s%s" % (url, line)) elif line == "meta-data": # meta-data is a dir, it *should* end in a / crawl("%s%s" % (url, "meta-data/")) else: printurl("%s%s" % (url, line)) else: printurl(url) if args.subcmd == "check-seed": sys.stderr.write("Checking seed at %s\n" % args.url) readurl = oauth_helper.readurl if args.url[0] == "/" or args.url.startswith("file://"): (userdata, metadata, vd) = read_maas_seed_dir(args.url) else: (userdata, metadata, vd) = read_maas_seed_url( args.url, version=args.apiver, read_file_or_url=readurl, retries=2, ) print("=== user-data ===") print("N/A" if userdata is None else userdata.decode()) print("=== meta-data ===") pprint.pprint(metadata) print("=== vendor-data ===") pprint.pprint("N/A" if vd is None else vd) elif args.subcmd == "get": printurl(args.url) elif args.subcmd == "crawl": if not args.url.endswith("/"): args.url = "%s/" % args.url crawl(args.url)
def read_runtime_config(): return util.read_conf(RUN_CLOUD_CONFIG)
def list_feature_flags(): """List all supported feature flags.""" feature_conf = get(c_util.read_conf(RELEASES_CONF), 'features') return feature_conf.get('all', [])
def handle(_name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: if not 'phone_home' in cfg: return ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: log.warn("no 'url' token in phone_home") return url = ph_cfg['url'] post_list = ph_cfg.get('post', 'all') tries = ph_cfg.get('tries', 10) try: tries = int(tries) except: log.warn("tries is not an integer. using 10") tries = 10 if post_list == "all": post_list = post_list_all all_keys = {} all_keys['instance_id'] = cloud.get_instance_id() all_keys['hostname'] = cloud.get_hostname() pubkeys = { 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', } for n, path in pubkeys.iteritems(): try: fp = open(path, "rb") all_keys[n] = fp.read() fp.close() except: log.warn("%s: failed to open in phone_home" % path) submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] else: submit_keys[k] = "N/A" log.warn("requested key %s from 'post' list not available") url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) null_exc = object() last_e = null_exc for i in range(0, tries): try: util.readurl(url, submit_keys) log.debug("succeeded submit to %s on try %i" % (url, i + 1)) return except Exception as e: log.debug("failed to post to %s on try %i" % (url, i + 1)) last_e = e sleep(3) log.warn("failed to post to %s in %i tries" % (url, tries)) if last_e is not null_exc: raise(last_e) return
def from_state_file(state_file): state = util.read_conf(state_file) nsi = NetworkStateInterpreter() nsi.load(state) return nsi
def main(): """ Call with single argument of directory or http or https url. If url is given additional arguments are allowed, which will be interpreted as consumer_key, token_key, token_secret, consumer_secret """ import argparse import pprint parser = argparse.ArgumentParser(description='Interact with MAAS DS') parser.add_argument("--config", metavar="file", help="specify DS config file", default=None) parser.add_argument("--ckey", metavar="key", help="the consumer key to auth with", default=None) parser.add_argument("--tkey", metavar="key", help="the token key to auth with", default=None) parser.add_argument("--csec", metavar="secret", help="the consumer secret (likely '')", default="") parser.add_argument("--tsec", metavar="secret", help="the token secret to auth with", default=None) parser.add_argument("--apiver", metavar="version", help="the apiver to use (" " can be used)", default=MD_VERSION) subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") subcmds.add_parser('crawl', help="crawl the datasource") subcmds.add_parser('get', help="do a single GET of provided url") subcmds.add_parser('check-seed', help="read andn verify seed at url") parser.add_argument("url", help="the data source to query") args = parser.parse_args() creds = { 'consumer_key': args.ckey, 'token_key': args.tkey, 'token_secret': args.tsec, 'consumer_secret': args.csec } if args.config: cfg = util.read_conf(args.config) if 'datasource' in cfg: cfg = cfg['datasource']['MAAS'] for key in creds.keys(): if key in cfg and creds[key] is None: creds[key] = cfg[key] oauth_helper = url_helper.OauthUrlHelper(**creds) def geturl(url): # the retry is to ensure that oauth timestamp gets fixed return oauth_helper.readurl(url, retries=1).contents def printurl(url): print("== %s ==\n%s\n" % (url, geturl(url).decode())) def crawl(url): if url.endswith("/"): for line in geturl(url).decode().splitlines(): if line.endswith("/"): crawl("%s%s" % (url, line)) elif line == "meta-data": # meta-data is a dir, it *should* end in a / crawl("%s%s" % (url, "meta-data/")) else: printurl("%s%s" % (url, line)) else: printurl(url) if args.subcmd == "check-seed": readurl = oauth_helper.readurl if args.url[0] == "/" or args.url.startswith("file://"): readurl = None (userdata, metadata) = read_maas_seed_url(args.url, version=args.apiver, read_file_or_url=readurl, retries=2) print("=== userdata ===") print(userdata.decode()) print("=== metadata ===") pprint.pprint(metadata) elif args.subcmd == "get": printurl(args.url) elif args.subcmd == "crawl": if not args.url.endswith("/"): args.url = "%s/" % args.url crawl(args.url)
def main(): """ Call with single argument of directory or http or https url. If url is given additional arguments are allowed, which will be interpreted as consumer_key, token_key, token_secret, consumer_secret """ import argparse import pprint parser = argparse.ArgumentParser(description='Interact with MAAS DS') parser.add_argument("--config", metavar="file", help="specify DS config file", default=None) parser.add_argument("--ckey", metavar="key", help="the consumer key to auth with", default=None) parser.add_argument("--tkey", metavar="key", help="the token key to auth with", default=None) parser.add_argument("--csec", metavar="secret", help="the consumer secret (likely '')", default="") parser.add_argument("--tsec", metavar="secret", help="the token secret to auth with", default=None) parser.add_argument("--apiver", metavar="version", help="the apiver to use ("" can be used)", default=MD_VERSION) subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") subcmds.add_parser('crawl', help="crawl the datasource") subcmds.add_parser('get', help="do a single GET of provided url") subcmds.add_parser('check-seed', help="read andn verify seed at url") parser.add_argument("url", help="the data source to query") args = parser.parse_args() creds = {'consumer_key': args.ckey, 'token_key': args.tkey, 'token_secret': args.tsec, 'consumer_secret': args.csec} if args.config: cfg = util.read_conf(args.config) if 'datasource' in cfg: cfg = cfg['datasource']['MAAS'] for key in creds.keys(): if key in cfg and creds[key] is None: creds[key] = cfg[key] def geturl(url, headers_cb): req = urllib2.Request(url, data=None, headers=headers_cb(url)) return (urllib2.urlopen(req).read()) def printurl(url, headers_cb): print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) def crawl(url, headers_cb=None): if url.endswith("/"): for line in geturl(url, headers_cb).splitlines(): if line.endswith("/"): crawl("%s%s" % (url, line), headers_cb) else: printurl("%s%s" % (url, line), headers_cb) else: printurl(url, headers_cb) def my_headers(url): headers = {} if creds.get('consumer_key', None) is not None: headers = oauth_headers(url, **creds) return headers if args.subcmd == "check-seed": if args.url.startswith("http"): (userdata, metadata) = read_maas_seed_url(args.url, header_cb=my_headers, version=args.apiver) else: (userdata, metadata) = read_maas_seed_url(args.url) print "=== userdata ===" print userdata print "=== metadata ===" pprint.pprint(metadata) elif args.subcmd == "get": printurl(args.url, my_headers) elif args.subcmd == "crawl": if not args.url.endswith("/"): args.url = "%s/" % args.url crawl(args.url, my_headers)
def list_enabled_platforms(): """ list all platforms enabled for testing """ platforms = c_util.read_conf(PLATFORM_CONF).get('platforms') return [k for k, v in platforms.items() if v.get('enabled')]
def main(): """ Call with single argument of directory or http or https url. If url is given additional arguments are allowed, which will be interpreted as consumer_key, token_key, token_secret, consumer_secret """ import argparse import pprint parser = argparse.ArgumentParser(description='Interact with MAAS DS') parser.add_argument("--config", metavar="file", help="specify DS config file", default=None) parser.add_argument("--ckey", metavar="key", help="the consumer key to auth with", default=None) parser.add_argument("--tkey", metavar="key", help="the token key to auth with", default=None) parser.add_argument("--csec", metavar="secret", help="the consumer secret (likely '')", default="") parser.add_argument("--tsec", metavar="secret", help="the token secret to auth with", default=None) parser.add_argument("--apiver", metavar="version", help="the apiver to use ("" can be used)", default=MD_VERSION) subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") subcmds.add_parser('crawl', help="crawl the datasource") subcmds.add_parser('get', help="do a single GET of provided url") subcmds.add_parser('check-seed', help="read andn verify seed at url") parser.add_argument("url", help="the data source to query") args = parser.parse_args() creds = {'consumer_key': args.ckey, 'token_key': args.tkey, 'token_secret': args.tsec, 'consumer_secret': args.csec} if args.config: cfg = util.read_conf(args.config) if 'datasource' in cfg: cfg = cfg['datasource']['MAAS'] for key in creds.keys(): if key in cfg and creds[key] is None: creds[key] = cfg[key] oauth_helper = url_helper.OauthUrlHelper(**creds) def geturl(url): # the retry is to ensure that oauth timestamp gets fixed return oauth_helper.readurl(url, retries=1).contents def printurl(url): print("== %s ==\n%s\n" % (url, geturl(url).decode())) def crawl(url): if url.endswith("/"): for line in geturl(url).decode().splitlines(): if line.endswith("/"): crawl("%s%s" % (url, line)) elif line == "meta-data": # meta-data is a dir, it *should* end in a / crawl("%s%s" % (url, "meta-data/")) else: printurl("%s%s" % (url, line)) else: printurl(url) if args.subcmd == "check-seed": readurl = oauth_helper.readurl if args.url[0] == "/" or args.url.startswith("file://"): readurl = None (userdata, metadata) = read_maas_seed_url( args.url, version=args.apiver, read_file_or_url=readurl, retries=2) print("=== userdata ===") print(userdata.decode()) print("=== metadata ===") pprint.pprint(metadata) elif args.subcmd == "get": printurl(args.url) elif args.subcmd == "crawl": if not args.url.endswith("/"): args.url = "%s/" % args.url crawl(args.url)
def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: if not 'phone_home' in cfg: log.debug(("Skipping module named %s, " "no 'phone_home' configuration found"), name) return ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: log.warn(("Skipping module named %s, " "no 'url' found in 'phone_home' configuration"), name) return url = ph_cfg['url'] post_list = ph_cfg.get('post', 'all') tries = ph_cfg.get('tries') try: tries = int(tries) except: tries = 10 util.logexc(log, "Configuration entry 'tries' is not an integer, " "using %s instead", tries) if post_list == "all": post_list = POST_LIST_ALL all_keys = {} all_keys['instance_id'] = cloud.get_instance_id() all_keys['hostname'] = cloud.get_hostname() pubkeys = { 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', } for (n, path) in pubkeys.iteritems(): try: all_keys[n] = util.load_file(path) except: util.logexc(log, "%s: failed to open, can not phone home that " "data!", path) submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] else: submit_keys[k] = None log.warn(("Requested key %s from 'post'" " configuration list not available"), k) # Get them read to be posted real_submit_keys = {} for (k, v) in submit_keys.iteritems(): if v is None: real_submit_keys[k] = 'N/A' else: real_submit_keys[k] = str(v) # Incase the url is parameterized url_params = { 'INSTANCE_ID': all_keys['instance_id'], } url = templater.render_string(url, url_params) try: util.read_file_or_url(url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths)) except: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries)
def load_test_config(path): """Load a test config file by either abs path or rel path.""" return merge_config(c_util.read_conf(TESTCASE_CONF)['base_test_data'], c_util.read_conf(name_to_path(path)))
def list_enabled_platforms(): """List all platforms enabled for testing.""" platforms = get(c_util.read_conf(PLATFORM_CONF), 'platforms') return [k for k, v in platforms.items() if enabled(v)]
def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: if 'phone_home' not in cfg: log.debug(("Skipping module named %s, " "no 'phone_home' configuration found"), name) return ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: log.warn(("Skipping module named %s, " "no 'url' found in 'phone_home' configuration"), name) return url = ph_cfg['url'] post_list = ph_cfg.get('post', 'all') tries = ph_cfg.get('tries') try: tries = int(tries) except Exception: tries = 10 util.logexc( log, "Configuration entry 'tries' is not an integer, " "using %s instead", tries) if post_list == "all": post_list = POST_LIST_ALL all_keys = {} all_keys['instance_id'] = cloud.get_instance_id() all_keys['hostname'] = cloud.get_hostname() all_keys['fqdn'] = cloud.get_hostname(fqdn=True) pubkeys = { 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', } for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) except Exception: util.logexc(log, "%s: failed to open, can not phone home that " "data!", path) submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] else: submit_keys[k] = None log.warn(("Requested key %s from 'post'" " configuration list not available"), k) # Get them read to be posted real_submit_keys = {} for (k, v) in submit_keys.items(): if v is None: real_submit_keys[k] = 'N/A' else: real_submit_keys[k] = str(v) # Incase the url is parameterized url_params = { 'INSTANCE_ID': all_keys['instance_id'], } url = templater.render_string(url, url_params) try: util.read_file_or_url(url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths)) except Exception: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries)
post_onboard_enabled = False if 'post_onboard_enabled' in cloud_config[name]: post_onboard_enabled = bool( cloud_config[name]['post_onboard_enabled']) if post_onboard_enabled and onboard_status == SUCCESS: if not runs_post_onboard_commands(name): onboard_status = ERROR # call phone_home if phone_home_url: tmos_onboard_utils.phone_home( phone_home_url, True, True, onboard_status, phone_home_url_verify_tls) if phone_home_cli and onboard_status == SUCCESS: tmos_onboard_utils.run_cmd(phone_home_cli) clean() LOG.info('onboarding ended with status: %s', onboard_status) if __name__ == "__main__": # Running the cloud-init module from the CLI python interpreter CLOUD_CONFIG_FILE = '/opt/cloud/instance/cloud-config.txt' CLOUD_CONFIG = {} if os.path.exists(CLOUD_CONFIG_FILE): CLOUD_CONFIG = util.read_conf(CLOUD_CONFIG_FILE) handle(sys.argv[1], CLOUD_CONFIG, None, logging, [])
def main(): """ Call with single argument of directory or http or https url. If url is given additional arguments are allowed, which will be interpreted as consumer_key, token_key, token_secret, consumer_secret """ import argparse import pprint import sys parser = argparse.ArgumentParser(description='Interact with MAAS DS') parser.add_argument("--config", metavar="file", help="specify DS config file", default=None) parser.add_argument("--ckey", metavar="key", help="the consumer key to auth with", default=None) parser.add_argument("--tkey", metavar="key", help="the token key to auth with", default=None) parser.add_argument("--csec", metavar="secret", help="the consumer secret (likely '')", default="") parser.add_argument("--tsec", metavar="secret", help="the token secret to auth with", default=None) parser.add_argument("--apiver", metavar="version", help="the apiver to use ("" can be used)", default=MD_VERSION) subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") for (name, help) in (('crawl', 'crawl the datasource'), ('get', 'do a single GET of provided url'), ('check-seed', 'read and verify seed at url')): p = subcmds.add_parser(name, help=help) p.add_argument("url", help="the datasource url", nargs='?', default=None) args = parser.parse_args() creds = {'consumer_key': args.ckey, 'token_key': args.tkey, 'token_secret': args.tsec, 'consumer_secret': args.csec} if args.config is None: for fname in ('91_kernel_cmdline_url', '90_dpkg_maas'): fpath = "/etc/cloud/cloud.cfg.d/" + fname + ".cfg" if os.path.exists(fpath) and os.access(fpath, os.R_OK): sys.stderr.write("Used config in %s.\n" % fpath) args.config = fpath if args.config: cfg = util.read_conf(args.config) if 'datasource' in cfg: cfg = cfg['datasource']['MAAS'] for key in creds.keys(): if key in cfg and creds[key] is None: creds[key] = cfg[key] if args.url is None and 'metadata_url' in cfg: args.url = cfg['metadata_url'] if args.url is None: sys.stderr.write("Must provide a url or a config with url.\n") sys.exit(1) oauth_helper = get_oauth_helper(creds) def geturl(url): # the retry is to ensure that oauth timestamp gets fixed return oauth_helper.readurl(url, retries=1).contents def printurl(url): print("== %s ==\n%s\n" % (url, geturl(url).decode())) def crawl(url): if url.endswith("/"): for line in geturl(url).decode().splitlines(): if line.endswith("/"): crawl("%s%s" % (url, line)) elif line == "meta-data": # meta-data is a dir, it *should* end in a / crawl("%s%s" % (url, "meta-data/")) else: printurl("%s%s" % (url, line)) else: printurl(url) if args.subcmd == "check-seed": sys.stderr.write("Checking seed at %s\n" % args.url) readurl = oauth_helper.readurl if args.url[0] == "/" or args.url.startswith("file://"): (userdata, metadata, vd) = read_maas_seed_dir(args.url) else: (userdata, metadata, vd) = read_maas_seed_url( args.url, version=args.apiver, read_file_or_url=readurl, retries=2) print("=== user-data ===") print("N/A" if userdata is None else userdata.decode()) print("=== meta-data ===") pprint.pprint(metadata) print("=== vendor-data ===") pprint.pprint("N/A" if vd is None else vd) elif args.subcmd == "get": printurl(args.url) elif args.subcmd == "crawl": if not args.url.endswith("/"): args.url = "%s/" % args.url crawl(args.url)
ns1_state = "/tmp/ns1.state" with open(ns1_state, "w+") as f: f.write(ns1_dump) print("Loading state from file") ns2 = from_state_file(ns1_state) print("NS1 == NS2 ?=> {}".format( ns1.network_state == ns2.network_state)) def test_output(network_config): (version, config) = load_config(network_config) ns1 = NetworkState(version=version, config=config) ns1.parse_config() random.shuffle(config) ns2 = NetworkState(version=version, config=config) ns2.parse_config() print("NS1 == NS2 ?=> {}".format( ns1.network_state == ns2.network_state)) eni_1 = net.render_interfaces(ns1.network_state) eni_2 = net.render_interfaces(ns2.network_state) print(eni_1) print(eni_2) print("eni_1 == eni_2 ?=> {}".format( eni_1 == eni_2)) y = util.read_conf(sys.argv[1]) network_config = y.get('network') test_parse(network_config) test_dump_and_load(network_config) test_output(network_config)