def _get_data(self): mcfg = self.ds_cfg try: self._set_data(self.seed_dir, read_maas_seed_dir(self.seed_dir)) return True except MAASSeedDirNone: pass except MAASSeedDirMalformed as exc: LOG.warning("%s was malformed: %s", self.seed_dir, exc) raise # If there is no metadata_url, then we're not configured url = mcfg.get('metadata_url', None) if not url: return False try: # doing this here actually has a side affect of # getting oauth time-fix in place. As no where else would # retry by default, so even if we could fix the timestamp # we would not. if not self.wait_for_metadata_service(url): return False self._set_data( url, read_maas_seed_url( url, read_file_or_url=self.oauth_helper.readurl, paths=self.paths, retries=1)) return True except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", url) return False
def handle(name, cfg, cloud, log, _args): if "bootcmd" not in cfg: log.debug(("Skipping module named %s," " no 'bootcmd' key in configuration"), name) return validate_cloudconfig_schema(cfg, schema) with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) tmpf.flush() except Exception as e: util.logexc(log, "Failed to shellify bootcmd: %s", str(e)) raise try: env = os.environ.copy() iid = cloud.get_instance_id() if iid: env['INSTANCE_ID'] = str(iid) cmd = ['/bin/sh', tmpf.name] util.subp(cmd, env=env, capture=False) except Exception: util.logexc(log, "Failed to run bootcmd module %s", name) raise
def create_group(self, name, members=None): group_add_cmd = ['groupadd', name] if not members: members = [] # Check if group exists, and then add it doesn't if util.is_group(name): LOG.warn("Skipping creation of existing group '%s'" % name) else: try: util.subp(group_add_cmd) LOG.info("Created new group %s" % name) except Exception: util.logexc(LOG, "Failed to create group %s", name) # Add members to the group, if so defined if len(members) > 0: for member in members: if not util.is_user(member): LOG.warn("Unable to add group member '%s' to group '%s'" "; user does not exist.", member, name) continue util.subp(['usermod', '-a', '-G', name, member]) LOG.info("Added user '%s' to group '%s'" % (member, name))
def get_data_source(self): if self.datasource is not None: return True if self.restore_from_cache(): log.debug("restored from cache type %s" % self.datasource) return True cfglist = self.cfg['datasource_list'] dslist = list_sources(cfglist, self.ds_deps) dsnames = [f.__name__ for f in dslist] log.debug("searching for data source in %s" % dsnames) for cls in dslist: ds = cls.__name__ try: s = cls(sys_cfg=self.cfg) if s.get_data(): self.datasource = s self.datasource_name = ds log.debug("found data source %s" % ds) return True except Exception as e: log.warn("get_data of %s raised %s" % (ds, e)) util.logexc(log) msg = "Did not find data source. searched classes: %s" % dsnames log.debug(msg) raise DataSourceNotFoundException(msg)
def recycle_srcmstr_process(log): try: out = util.subp([PIDOF, SRCMSTR])[0] except util.ProcessExecutionError: util.logexc(log, 'Failed to get PID of srcmstr process.') raise srcmstr_pid_before = int(out) log.debug('Recycling srcmstr process with PID of %d.' % srcmstr_pid_before) try: os.kill(srcmstr_pid_before, 9) except: util.logexc(log, 'Failed to kill the srcmstr process.') raise # wait for srcmstr to come back up start_time = time.time() while True: time.sleep(0.5) if time.time() - start_time >= SRCMSTR_TIMEOUT_SECONDS: msg = ('srcmstr process failed to come back up within %d seconds.' % SRCMSTR_TIMEOUT_SECONDS) log.error(msg) raise Exception(msg) try: new_srcmstr_pid = int(util.subp([PIDOF, SRCMSTR])[0]) log.debug('srcmstr process came back up with PID of %d.' % new_srcmstr_pid) break except util.ProcessExecutionError: log.debug('Still waiting for srcmstr process to come ' 'back up...') continue
def wait_for_metadata_service(self, url): mcfg = self.ds_cfg max_wait = 120 try: max_wait = int(mcfg.get("max_wait", max_wait)) except Exception: util.logexc(LOG, "Failed to get max wait. using %s", max_wait) if max_wait == 0: return False timeout = 50 try: if timeout in mcfg: timeout = int(mcfg.get("timeout", timeout)) except Exception: LOG.warn("Failed to get timeout, using %s" % timeout) starttime = time.time() check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) urls = [check_url] url = self.oauth_helper.wait_for_url( urls=urls, max_wait=max_wait, timeout=timeout) if url: LOG.debug("Using metadata source: '%s'", url) else: LOG.critical("Giving up on md from %s after %i seconds", urls, int(time.time() - starttime)) return bool(url)
def get_node_id(log): try: node_id = util.subp(['/usr/bin/head', '-n1', NODE_ID_FILE])[0].strip() return node_id except util.ProcessExecutionError: util.logexc(log, 'Failed to get node ID from file %s.' % NODE_ID_FILE) raise
def read_user_data_callback(mount_dir): ''' Description: This callback will be applied by util.mount_cb() on the mounted file. Deltacloud file name contains deltacloud. Those not using Deltacloud but instead instrumenting the injection, could drop deltacloud from the file name. Input: mount_dir - Mount directory Returns: User Data ''' deltacloud_user_data_file = mount_dir + '/deltacloud-user-data.txt' user_data_file = mount_dir + '/user-data.txt' # First try deltacloud_user_data_file. On failure try user_data_file. try: user_data = util.load_file(deltacloud_user_data_file).strip() except IOError: try: user_data = util.load_file(user_data_file).strip() except IOError: util.logexc(LOG, 'Failed accessing user data file.') return None return user_data
def is_powerkvm(log): try: out = util.subp(['cat', CPUINFO])[0] return QEMU_STRING in out.lower() except: util.logexc(log, 'Failed to determine if VM is running on PowerKVM.') raise
def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): ds_list = list_sources(cfg_list, ds_deps, pkg_list) ds_names = [type_utils.obj_name(f) for f in ds_list] mode = "network" if DEP_NETWORK in ds_deps else "local" LOG.debug("Searching for %s data source in: %s", mode, ds_names) for name, cls in zip(ds_names, ds_list): myrep = events.ReportEventStack( name="search-%s" % name.replace("DataSource", ""), description="searching for %s data from %s" % (mode, name), message="no %s data found from %s" % (mode, name), parent=reporter) try: with myrep: LOG.debug("Seeing if we can get any data from %s", cls) s = cls(sys_cfg, distro, paths) if s.get_data(): myrep.message = "found %s data from %s" % (mode, name) return (s, type_utils.obj_name(cls)) except Exception: util.logexc(LOG, "Getting data from %s failed", cls) msg = ("Did not find any data source," " searched classes: (%s)") % (", ".join(ds_names)) raise DataSourceNotFoundException(msg)
def setup_user_keys(keys, user, key_prefix, log=None): import pwd saved_umask = os.umask(077) pwent = pwd.getpwnam(user) ssh_dir = '%s/.ssh' % pwent.pw_dir if not os.path.exists(ssh_dir): os.mkdir(ssh_dir) os.chown(ssh_dir, pwent.pw_uid, pwent.pw_gid) try: ssh_cfg = parse_ssh_config() akeys = ssh_cfg.get("AuthorizedKeysFile", "%h/.ssh/authorized_keys") akeys = akeys.replace("%h", pwent.pw_dir) akeys = akeys.replace("%u", user) authorized_keys = akeys except Exception: authorized_keys = '%s/.ssh/authorized_keys' % pwent.pw_dir if log: util.logexc(log) key_entries = [] for k in keys: ke = AuthKeyEntry(k, def_opt=key_prefix) key_entries.append(ke) content = update_authorized_keys(authorized_keys, key_entries) util.write_file(authorized_keys, content, 0600) os.chown(authorized_keys, pwent.pw_uid, pwent.pw_gid) util.restorecon_if_possible(ssh_dir, recursive=True) os.umask(saved_umask)
def get_data(self): """ Metadata is the whole server context and /meta/cloud-config is used as userdata. """ dsmode = None try: server_context = self.cepko.all().result server_meta = server_context['meta'] except: util.logexc(LOG, "Failed reading from the serial port") return False dsmode = server_meta.get('cloudinit-dsmode', self.dsmode) if dsmode not in VALID_DSMODES: LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode) dsmode = 'net' if dsmode == "disabled" or dsmode != self.dsmode: return False base64_fields = server_meta.get('base64_fields', '').split(',') self.userdata_raw = server_meta.get('cloudinit-user-data', "") if 'cloudinit-user-data' in base64_fields: self.userdata_raw = b64decode(self.userdata_raw) self.metadata = server_context self.ssh_public_key = server_meta['ssh_public_key'] return True
def on_first_boot(data, distro=None): """Performs any first-boot actions using data read from a config-drive.""" if not isinstance(data, dict): raise TypeError("Config-drive data expected to be a dict; not %s" % (type(data))) networkapplied = False jsonnet_conf = data.get('vendordata', {}).get('network_info') if jsonnet_conf: try: LOG.debug("Updating network interfaces from JSON in config drive") distro_user_config = distro.apply_network_json(jsonnet_conf) networkapplied = True except NotImplementedError: LOG.debug( "Distro does not implement networking setup via Vendor JSON.") pass net_conf = data.get("network_config", '') if networkapplied is False and net_conf and distro: LOG.debug("Updating network interfaces from config drive") distro.apply_network(net_conf) files = data.get('files', {}) if files: LOG.debug("Writing %s injected files", len(files)) for (filename, content) in files.items(): if not filename.startswith(os.sep): filename = os.sep + filename try: util.write_file(filename, content, mode=0o660) except IOError: util.logexc(LOG, "Failed writing file: %s", filename)
def run_part(mod, data, filename, payload, frequency, headers): mod_freq = mod.frequency if not (mod_freq == PER_ALWAYS or (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): return # Sanity checks on version (should be an int convertable) try: mod_ver = mod.handler_version mod_ver = int(mod_ver) except (TypeError, ValueError, AttributeError): mod_ver = 1 content_type = headers['Content-Type'] try: LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s", mod, content_type, filename, mod_ver, frequency) if mod_ver == 3: # Treat as v. 3 which does get a frequency + headers mod.handle_part(data, content_type, filename, payload, frequency, headers) elif mod_ver == 2: # Treat as v. 2 which does get a frequency mod.handle_part(data, content_type, filename, payload, frequency) elif mod_ver == 1: # Treat as v. 1 which gets no frequency mod.handle_part(data, content_type, filename, payload) else: raise ValueError("Unknown module version %s" % (mod_ver)) except: util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with " "frequency %s", mod, content_type, filename, mod_ver, frequency)
def add_snap_user(self, name, **kwargs): """ Add a snappy user to the system using snappy tools """ snapuser = kwargs.get('snapuser') known = kwargs.get('known', False) adduser_cmd = ["snap", "create-user", "--sudoer", "--json"] if known: adduser_cmd.append("--known") adduser_cmd.append(snapuser) # Run the command LOG.debug("Adding snap user %s", name) try: (out, err) = util.subp(adduser_cmd, logstring=adduser_cmd, capture=True) LOG.debug("snap create-user returned: %s:%s", out, err) jobj = util.load_json(out) username = jobj.get('username', None) except Exception as e: util.logexc(LOG, "Failed to create snap user %s", name) raise e return username
def write_sudo_rules(self, user, rules, sudo_file=None): if not sudo_file: sudo_file = self.ci_sudoers_fn lines = ["", "# User rules for %s" % user] if isinstance(rules, (list, tuple)): for rule in rules: lines.append("%s %s" % (user, rule)) elif isinstance(rules, (basestring, str)): lines.append("%s %s" % (user, rules)) else: msg = "Can not create sudoers rule addition with type %r" raise TypeError(msg % (type_utils.obj_name(rules))) content = "\n".join(lines) content += "\n" # trailing newline self.ensure_sudo_dir(os.path.dirname(sudo_file)) if not os.path.exists(sudo_file): contents = [util.make_header(), content] try: util.write_file(sudo_file, "\n".join(contents), 0440) except IOError as e: util.logexc(LOG, "Failed to write sudoers file %s", sudo_file) raise e else: try: util.append_file(sudo_file, content) except IOError as e: util.logexc(LOG, "Failed to append sudoers file %s", sudo_file) raise e
def get_instance_metadata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5, leaf_decoder=None): md_url = url_helper.combine_url(metadata_address, api_version) # Note, 'meta-data' explicitly has trailing /. # this is required for CloudStack (LP: #1356855) md_url = url_helper.combine_url(md_url, 'meta-data/') caller = functools.partial(util.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) def mcaller(url): return caller(url).contents try: response = caller(md_url) materializer = MetadataMaterializer(response.contents, md_url, mcaller, leaf_decoder=leaf_decoder) md = materializer.materialize() if not isinstance(md, (dict)): md = {} return md except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", md_url) return {}
def _run_modules(self, mostly_mods): cc = self.init.cloudify() # Return which ones ran # and which ones failed + the exception of why it failed failures = [] which_ran = [] for (mod, name, freq, args) in mostly_mods: try: # Try the modules frequency, otherwise fallback to a known one if not freq: freq = mod.frequency if freq not in FREQUENCIES: freq = PER_INSTANCE LOG.debug("Running module %s (%s) with frequency %s", name, mod, freq) # Use the configs logger and not our own # TODO(harlowja): possibly check the module # for having a LOG attr and just give it back # its own logger? func_args = [name, self.cfg, cc, config.LOG, args] # Mark it as having started running which_ran.append(name) # This name will affect the semaphore name created run_name = "config-%s" % (name) cc.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: util.logexc(LOG, "Running module %s (%s) failed", name, mod) failures.append((name, e)) return (which_ran, failures)
def user_data_rhevm(self): ''' RHEVM specific userdata read If on RHEV-M the user data will be contained on the floppy device in file <user_data_file> To access it: modprobe floppy Leverage util.mount_cb to: mkdir <tmp mount dir> mount /dev/fd0 <tmp mount dir> The call back passed to util.mount_cb will do: read <tmp mount dir>/<user_data_file> ''' return_str = None # modprobe floppy try: cmd = CMD_PROBE_FLOPPY (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) except ProcessExecutionError, _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False
def get_data(self): mcfg = self.ds_cfg try: (userdata, metadata) = read_maas_seed_dir(self.seeddir) self.userdata_raw = userdata self.metadata = metadata self.baseurl = self.seeddir return True except MAASSeedDirNone: pass except MAASSeedDirMalformed as exc: log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) raise try: # if there is no metadata_url, then we're not configured url = mcfg.get('metadata_url', None) if url == None: return False if not self.wait_for_metadata_service(url): return False self.baseurl = url (userdata, metadata) = read_maas_seed_url(self.baseurl, self.md_headers) self.userdata_raw = userdata self.metadata = metadata return True except Exception: util.logexc(log) return False
def crawl_metadata(self): """Crawl metadata service when available. @returns: Dictionary of crawled metadata content containing the keys: meta-data, user-data and dynamic. """ if not self.wait_for_metadata_service(): return {} api_version = self.get_metadata_api_version() crawled_metadata = {} try: crawled_metadata['user-data'] = ec2.get_instance_userdata( api_version, self.metadata_address) crawled_metadata['meta-data'] = ec2.get_instance_metadata( api_version, self.metadata_address) if self.cloud_name == CloudNames.AWS: identity = ec2.get_instance_identity( api_version, self.metadata_address) crawled_metadata['dynamic'] = {'instance-identity': identity} except Exception: util.logexc( LOG, "Failed reading from metadata address %s", self.metadata_address) return {} crawled_metadata['_metadata_api_version'] = api_version return crawled_metadata
def wait_for_metadata_service(self, url): mcfg = self.ds_cfg max_wait = 120 try: max_wait = int(mcfg.get("max_wait", max_wait)) except Exception: util.logexc(log) log.warn("Failed to get max wait. using %s" % max_wait) if max_wait == 0: return False timeout = 50 try: timeout = int(mcfg.get("timeout", timeout)) except Exception: util.logexc(log) log.warn("Failed to get timeout, using %s" % timeout) starttime = time.time() check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) url = util.wait_for_url(urls=[check_url], max_wait=max_wait, timeout=timeout, status_cb=log.warn, headers_cb=self.md_headers) if url: log.debug("Using metadata source: '%s'" % url) else: log.critical("giving up on md after %i seconds\n" % int(time.time() - starttime)) return (bool(url))
def apply_locale(self, locale, out_fn=None): # Adjust the locals value to the new value newconf = StringIO() for line in util.load_file(self.login_conf_fn).splitlines(): newconf.write(re.sub(r'^default:', r'default:lang=%s:' % locale, line)) newconf.write("\n") # Make a backup of login.conf. util.copy(self.login_conf_fn, self.login_conf_fn_bak) # And write the new login.conf. util.write_file(self.login_conf_fn, newconf.getvalue()) try: LOG.debug("Running cap_mkdb for %s", locale) util.subp(['cap_mkdb', self.login_conf_fn]) except util.ProcessExecutionError: # cap_mkdb failed, so restore the backup. util.logexc(LOG, "Failed to apply locale %s", locale) try: util.copy(self.login_conf_fn_bak, self.login_conf_fn) except IOError: util.logexc(LOG, "Failed to restore %s backup", self.login_conf_fn)
def extract_authorized_keys(username): (ssh_dir, pw_ent) = users_ssh_info(username) auth_key_fn = None with util.SeLinuxGuard(ssh_dir, recursive=True): try: # The 'AuthorizedKeysFile' may contain tokens # of the form %T which are substituted during connection set-up. # The following tokens are defined: %% is replaced by a literal # '%', %h is replaced by the home directory of the user being # authenticated and %u is replaced by the username of that user. ssh_cfg = parse_ssh_config_map(DEF_SSHD_CFG) auth_key_fn = ssh_cfg.get("authorizedkeysfile", '').strip() if not auth_key_fn: auth_key_fn = "%h/.ssh/authorized_keys" auth_key_fn = auth_key_fn.replace("%h", pw_ent.pw_dir) auth_key_fn = auth_key_fn.replace("%u", username) auth_key_fn = auth_key_fn.replace("%%", '%') if not auth_key_fn.startswith('/'): auth_key_fn = os.path.join(pw_ent.pw_dir, auth_key_fn) except (IOError, OSError): # Give up and use a default key filename auth_key_fn = os.path.join(ssh_dir, 'authorized_keys') util.logexc(LOG, "Failed extracting 'AuthorizedKeysFile' in ssh " "config from %r, using 'AuthorizedKeysFile' file " "%r instead", DEF_SSHD_CFG, auth_key_fn) return (auth_key_fn, parse_authorized_keys(auth_key_fn))
def user_data_vsphere(self): ''' vSphere specific userdata read If on vSphere the user data will be contained on the cdrom device in file <user_data_file> To access it: Leverage util.mount_cb to: mkdir <tmp mount dir> mount /dev/fd0 <tmp mount dir> The call back passed to util.mount_cb will do: read <tmp mount dir>/<user_data_file> ''' return_str = None cdrom_list = util.find_devs_with('LABEL=CDROM') for cdrom_dev in cdrom_list: try: return_str = util.mount_cb(cdrom_dev, read_user_data_callback) if return_str: break except OSError as err: if err.errno != errno.ENOENT: raise except util.MountFailedError: util.logexc(LOG, "Failed to mount %s when looking for user " "data", cdrom_dev) self.userdata_raw = return_str self.metadata = META_DATA_NOT_SUPPORTED if return_str: return True else: return False
def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," " not setting the hostname in module %s"), name) return (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) # Check for previous successful invocation of set-hostname # set-hostname artifact file accounts for both hostname and fqdn # deltas. As such, it's format is different than cc_update_hostname's # previous-hostname file which only contains the base hostname. # TODO consolidate previous-hostname and set-hostname artifact files and # distro._read_hostname implementation so we only validate one artifact. prev_fn = os.path.join(cloud.get_cpath('data'), "set-hostname") prev_hostname = {} if os.path.exists(prev_fn): prev_hostname = util.load_json(util.load_file(prev_fn)) hostname_changed = (hostname != prev_hostname.get('hostname') or fqdn != prev_hostname.get('fqdn')) if not hostname_changed: log.debug('No hostname changes. Skipping set-hostname') return log.debug("Setting the hostname to %s (%s)", fqdn, hostname) try: cloud.distro.set_hostname(hostname, fqdn) except Exception as e: msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname) util.logexc(log, msg) raise SetHostnameError("%s: %s" % (msg, e)) write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn})
def get_data(self): mcfg = self.ds_cfg try: (userdata, metadata) = read_maas_seed_dir(self.seed_dir) self.userdata_raw = userdata self.metadata = metadata self.base_url = self.seed_dir return True except MAASSeedDirNone: pass except MAASSeedDirMalformed as exc: LOG.warn("%s was malformed: %s" % (self.seed_dir, exc)) raise # If there is no metadata_url, then we're not configured url = mcfg.get('metadata_url', None) if not url: return False try: if not self.wait_for_metadata_service(url): return False self.base_url = url (userdata, metadata) = read_maas_seed_url(self.base_url, self._md_headers, paths=self.paths) self.userdata_raw = userdata self.metadata = metadata return True except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", url) return False
def handle(name, _cfg, _cloud, log, _args): required_tools = [RMCCTRL, RECFGCT] for tool in required_tools: if not os.path.isfile(tool): log.debug('%s is not found but is required, therefore not ' 'attempting to reset RMC.' % tool) return log.debug('Attempting to reset RMC.') system_info = util.system_info() node_id_before = get_node_id(log) log.debug('Node ID at beginning of module: %s' % node_id_before) # Stop the RMC subsystem and all resource managers so that we can make # some changes to it try: util.subp([RMCCTRL, '-z']) except: util.logexc(log, 'Failed to stop the RMC subsystem.') raise if 'linux' in system_info['platform'].lower(): recycle_srcmstr_process(log) reconfigure_rsct_subsystems(log) node_id_after = get_node_id(log) log.debug('Node ID at end of module: %s' % node_id_after) if node_id_after == node_id_before: msg = 'New node ID did not get generated.' log.error(msg) raise Exception(msg)
def resize(self, diskdev, partnum, partdev): """ GPT disks store metadata at the beginning (primary) and at the end (secondary) of the disk. When launching an image with a larger disk compared to the original image, the secondary copy is lost. Thus, the metadata will be marked CORRUPT, and need to be recovered. """ try: util.subp(["gpart", "recover", diskdev]) except util.ProcessExecutionError as e: if e.exit_code != 0: util.logexc(LOG, "Failed: gpart recover %s", diskdev) raise ResizeFailedException(e) before = get_size(partdev) try: util.subp(["gpart", "resize", "-i", partnum, diskdev]) except util.ProcessExecutionError as e: util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) raise ResizeFailedException(e) # Since growing the FS requires a reboot, make sure we reboot # first when this module has finished. open('/var/run/reboot-required', 'a').close() return (before, get_size(partdev))
def run_commands(commands): """Run the provided commands provided in snap:commands configuration. Commands are run individually. Any errors are collected and reported after attempting all commands. @param commands: A list or dict containing commands to run. Keys of a dict will be used to order the commands provided as dict values. """ if not commands: return LOG.debug('Running user-provided snap commands') if isinstance(commands, dict): # Sort commands based on dictionary key commands = [v for _, v in sorted(commands.items())] elif not isinstance(commands, list): raise TypeError( 'commands parameter was not a list or dict: {commands}'.format( commands=commands)) fixed_snap_commands = prepend_base_command('snap', commands) cmd_failures = [] for command in fixed_snap_commands: shell = isinstance(command, str) try: util.subp(command, shell=shell, status_cb=sys.stderr.write) except util.ProcessExecutionError as e: cmd_failures.append(str(e)) if cmd_failures: msg = 'Failures running snap commands:\n{cmd_failures}'.format( cmd_failures=cmd_failures) util.logexc(LOG, msg) raise RuntimeError(msg)
def update_resolve_conf_file(fn, dns_servers, search_servers): try: r_conf = ResolvConf(util.load_file(fn)) r_conf.parse() except IOError: util.logexc(LOG, "Failed at parsing %s reverting to an empty " "instance", fn) r_conf = ResolvConf('') r_conf.parse() if dns_servers: for s in dns_servers: try: r_conf.add_nameserver(s) except ValueError: util.logexc(LOG, "Failed at adding nameserver %s", s) if search_servers: for s in search_servers: try: r_conf.add_search_domain(s) except ValueError: util.logexc(LOG, "Failed at adding search domain %s", s) write_resolv_conf_file(fn, r_conf)
def apt_key_add(): """apt-key add <file> returns filepath to new keyring, or '/dev/null' when an error occurs """ file_name = "/dev/null" if not output_file: util.logexc( LOG, 'Unknown filename, failed to add key: "{}"'.format(data)) else: try: key_dir = (CLOUD_INIT_GPG_DIR if hardened else APT_TRUSTED_GPG_DIR) stdout = gpg.dearmor(data) file_name = "{}{}.gpg".format(key_dir, output_file) util.write_file(file_name, stdout) except subp.ProcessExecutionError: util.logexc(LOG, "Gpg error, failed to add key: {}".format(data)) except UnicodeDecodeError: util.logexc(LOG, "Decode error, failed to add key: {}".format(data)) return file_name
def _resolve_conf(self, settings, target=None): nameservers = settings.dns_nameservers searchdomains = settings.dns_searchdomains for interface in settings.iter_interfaces(): for subnet in interface.get("subnets", []): if 'dns_nameservers' in subnet: nameservers.extend(subnet['dns_nameservers']) if 'dns_search' in subnet: searchdomains.extend(subnet['dns_search']) # Try to read the /etc/resolv.conf or just start from scratch if that # fails. try: resolvconf = ResolvConf(util.load_file(subp.target_path( target, self.resolv_conf_fn))) resolvconf.parse() except IOError: util.logexc(LOG, "Failed to parse %s, use new empty file", subp.target_path(target, self.resolv_conf_fn)) resolvconf = ResolvConf('') resolvconf.parse() # Add some nameservers for server in nameservers: try: resolvconf.add_nameserver(server) except ValueError: util.logexc(LOG, "Failed to add nameserver %s", server) # And add any searchdomains. for domain in searchdomains: try: resolvconf.add_search_domain(domain) except ValueError: util.logexc(LOG, "Failed to add search domain %s", domain) util.write_file( subp.target_path(target, self.resolv_conf_fn), str(resolvconf), 0o644)
def get_data(self): # azure removes/ejects the cdrom containing the ovf-env.xml # file on reboot. So, in order to successfully reboot we # need to look in the datadir and consider that valid ddir = self.ds_cfg['data_dir'] candidates = [self.seed_dir] candidates.extend(list_possible_azure_ds_devs()) if ddir: candidates.append(ddir) found = None for cdev in candidates: try: if cdev.startswith("/dev/"): ret = util.mount_cb(cdev, load_azure_ds_dir) else: ret = load_azure_ds_dir(cdev) except NonAzureDataSource: continue except BrokenAzureDataSource as exc: raise exc except util.MountFailedError: LOG.warn("%s was not mountable", cdev) continue (md, self.userdata_raw, cfg, files) = ret self.seed = cdev self.metadata = util.mergemanydict([md, IB_DEFAULT_METADATA]) self.cfg = util.mergemanydict([cfg, IB_BUILTIN_CLOUD_CONFIG]) found = cdev LOG.debug("found datasource in %s", cdev) break if not found: return False if found == ddir: LOG.debug("using files cached in %s", ddir) # azure / hyper-v provides random data here seed = util.load_file("/sys/firmware/acpi/tables/OEM0", quiet=True) if seed: self.metadata['random_seed'] = seed # now update ds_cfg to reflect contents pass in config user_ds_cfg = util.get_cfg_by_path(self.cfg, IB_DS_CFG_PATH, {}) self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg]) mycfg = self.ds_cfg ddir = mycfg['data_dir'] if found != ddir: cached_ovfenv = util.load_file(os.path.join(ddir, 'ovf-env.xml'), quiet=True) if cached_ovfenv != files['ovf-env.xml']: # source was not walinux-agent's datadir, so we have to clean # up so 'wait_for_files' doesn't return early due to stale data cleaned = [] for f in [ os.path.join(ddir, f) for f in IB_DATA_DIR_CLEAN_LIST ]: if os.path.exists(f): util.del_file(f) cleaned.append(f) if cleaned: LOG.info("removed stale file(s) in '%s': %s", ddir, str(cleaned)) # walinux agent writes files world readable, but expects # the directory to be protected. write_files(ddir, files, dirmode=0700) # handle the hostname 'publishing' try: handle_set_hostname(mycfg.get('set_hostname'), self.metadata.get('local-hostname'), mycfg['hostname_bounce']) except Exception as e: LOG.warn("Failed publishing hostname: %s", e) util.logexc(LOG, "handling set_hostname failed") try: invoke_agent(mycfg['agent_command']) except util.ProcessExecutionError: # claim the datasource even if the command failed util.logexc(LOG, "agent command '%s' failed.", mycfg['agent_command']) # code is commented out because we have no WAAgent and # we do not use SSH keys. Thus, no files to wait for. # shcfgxml = os.path.join(ddir, "SharedConfig.xml") # wait_for = [shcfgxml] # fp_files = [] # for pk in self.cfg.get('_pubkeys', []): # bname = str(pk['fingerprint'] + ".crt") # fp_files += [os.path.join(ddir, bname)] # missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", # func=wait_for_files, # args=(wait_for + fp_files,)) # if len(missing): # LOG.warn("Did not find files, but going on: %s", missing) # if shcfgxml in missing: # LOG.warn("SharedConfig.xml missing, using static instance-id") # else: # try: # self.metadata['instance-id'] = \ # iid_from_shared_config(shcfgxml) # except ValueError as e: # LOG.warn("failed to get instance id in %s: %s", shcfgxml, e) # pubkeys = pubkeys_from_crt_files(fp_files) # self.metadata['public-keys'] = pubkeys found_ephemeral = find_ephemeral_disk() if found_ephemeral: self.ds_cfg['disk_aliases']['ephemeral0'] = found_ephemeral LOG.debug("using detected ephemeral0 of %s", found_ephemeral) cc_modules_override = support_new_ephemeral(self.sys_cfg) if cc_modules_override: self.cfg['cloud_config_modules'] = cc_modules_override return True
def handle(name, cfg, cloud, log, args): # Get config lxd_cfg = cfg.get('lxd') if not lxd_cfg: log.debug("Skipping module named %s, not present or disabled by cfg") return if not isinstance(lxd_cfg, dict): log.warn("lxd config must be a dictionary. found a '%s'", type(lxd_cfg)) return # Grab the configuration init_cfg = lxd_cfg.get('init') if not isinstance(init_cfg, dict): log.warn("lxd/init config must be a dictionary. found a '%s'", type(init_cfg)) init_cfg = {} bridge_cfg = lxd_cfg.get('bridge') if not isinstance(bridge_cfg, dict): log.warn("lxd/bridge config must be a dictionary. found a '%s'", type(bridge_cfg)) bridge_cfg = {} # Install the needed packages packages = [] if not util.which("lxd"): packages.append('lxd') if init_cfg.get("storage_backend") == "zfs" and not util.which('zfs'): packages.append('zfs') if len(packages): try: cloud.distro.install_packages(packages) except util.ProcessExecutionError as exc: log.warn("failed to install packages %s: %s", packages, exc) return # Set up lxd if init config is given if init_cfg: init_keys = ('network_address', 'network_port', 'storage_backend', 'storage_create_device', 'storage_create_loop', 'storage_pool', 'trust_password') cmd = ['lxd', 'init', '--auto'] for k in init_keys: if init_cfg.get(k): cmd.extend( ["--%s=%s" % (k.replace('_', '-'), str(init_cfg[k]))]) util.subp(cmd) # Set up lxd-bridge if bridge config is given dconf_comm = "debconf-communicate" if bridge_cfg and util.which(dconf_comm): debconf = bridge_to_debconf(bridge_cfg) # Update debconf database try: log.debug("Setting lxd debconf via " + dconf_comm) data = "\n".join( ["set %s %s" % (k, v) for k, v in debconf.items()]) + "\n" util.subp(['debconf-communicate'], data) except Exception: util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) # Remove the existing configuration file (forces re-generation) util.del_file("/etc/default/lxd-bridge") # Run reconfigure log.debug("Running dpkg-reconfigure for lxd") util.subp(['dpkg-reconfigure', 'lxd', '--frontend=noninteractive']) elif bridge_cfg: raise RuntimeError("Unable to configure lxd bridge without %s." + dconf_comm)
def unlock_passwd(self, name): try: util.subp(['usermod', '-C', 'no', name]) except Exception: util.logexc(LOG, "Failed to unlock user %s", name) raise
def clear_all(self): try: util.del_dir(self.sem_path) except (IOError, OSError): util.logexc(LOG, "Failed deleting semaphore directory %s", self.sem_path)
def main_single(name, args): # Cloud-init single stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Attempt to fetch the datasource (warn if it doesn't work) # 3. Construct the modules object # 4. Adjust any subsequent logging/output redirections using # the modules objects configuration # 5. Run the single module # 6. Done! mod_name = args.name w_msg = welcome_format(name) init = stages.Init(ds_deps=[], reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 try: init.fetch(existing="trust") except sources.DataSourceNotFoundException: # There was no datasource found, # that might be bad (or ok) depending on # the module being ran (so continue on) util.logexc(LOG, ("Failed to fetch your datasource," " likely bad things to come!")) print_exc(("Failed to fetch your datasource," " likely bad things to come!")) if not args.force: return 1 # Stage 3 mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) mod_args = args.module_args if mod_args: LOG.debug("Using passed in arguments %s", mod_args) mod_freq = args.frequency if mod_freq: LOG.debug("Using passed in frequency %s", mod_freq) mod_freq = FREQ_SHORT_NAMES.get(mod_freq) # Stage 4 try: LOG.debug("Closing stdin") util.close_stdin() util.fixup_output(mods.cfg, None) except Exception: util.logexc(LOG, "Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" " longer be active shortly")) logging.resetLogging() logging.setupLogging(mods.cfg) apply_reporting_cfg(init.cfg) # now that logging is setup and stdout redirected, send welcome welcome(name, msg=w_msg) # Stage 5 (which_ran, failures) = mods.run_single(mod_name, mod_args, mod_freq) if failures: LOG.warn("Ran %s but it failed!", mod_name) return 1 elif not which_ran: LOG.warn("Did not run %s, does it exist?", mod_name) return 1 else: # Guess it worked return 0
def handle(_name, cfg, cloud, log, _args): # remove the static keys from the pristine image if cfg.get("ssh_deletekeys", True): key_pth = os.path.join("/etc/ssh/", "ssh_host_*key*") for f in glob.glob(key_pth): try: util.del_file(f) except: util.logexc(log, "Failed deleting key file %s", f) if "ssh_keys" in cfg: # if there are keys in cloud-config, use them for (key, val) in cfg["ssh_keys"].items(): if key in KEY_2_FILE: tgt_fn = KEY_2_FILE[key][0] tgt_perms = KEY_2_FILE[key][1] util.write_file(tgt_fn, val, tgt_perms) for (priv, pub) in PRIV_2_PUB.items(): if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: continue pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0]) cmd = ['sh', '-xc', KEY_GEN_TPL % pair] try: # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except: util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) else: # if not, generate them genkeys = util.get_cfg_option_list(cfg, 'ssh_genkeytypes', GENERATE_KEY_NAMES) for keytype in genkeys: keyfile = KEY_FILE_TPL % (keytype) util.ensure_dir(os.path.dirname(keyfile)) if not os.path.exists(keyfile): cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] try: # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) except: util.logexc(log, "Failed generating key type %s to " "file %s", keytype, keyfile) try: (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) (user, _user_config) = ds.extract_default(users) disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", DISABLE_ROOT_OPTS) keys = cloud.get_public_ssh_keys() or [] if "ssh_authorized_keys" in cfg: cfgkeys = cfg["ssh_authorized_keys"] keys.extend(cfgkeys) apply_credentials(keys, user, disable_root, disable_root_opts) except: util.logexc(log, "Applying ssh credentials failed!")
def lock_passwd(self, name): try: subp.subp(["pw", "usermod", name, "-h", "-"]) except Exception: util.logexc(LOG, "Failed to lock user %s", name) raise
def force_passwd_change(self, user): try: util.subp(['usermod', '-F', user]) except Exception: util.logexc(LOG, "Failed to set pw expiration for %s", user) raise
def _get_data(self): found = None md = {} results = {} for sdir in (self.seed_dir, "/config-drive"): if not os.path.isdir(sdir): continue try: results = read_config_drive(sdir) found = sdir break except openstack.NonReadable: util.logexc(LOG, "Failed reading config drive from %s", sdir) if not found: dslist = self.sys_cfg.get('datasource_list') for dev in find_candidate_devs(dslist=dslist): try: # Set mtype if freebsd and turn off sync if dev.startswith("/dev/cd"): mtype = "cd9660" sync = False else: mtype = None sync = True results = util.mount_cb(dev, read_config_drive, mtype=mtype, sync=sync) found = dev except openstack.NonReadable: pass except util.MountFailedError: pass except openstack.BrokenMetadata: util.logexc(LOG, "Broken config drive: %s", dev) if found: break if not found: return False md = results.get('metadata', {}) md = util.mergemanydict([md, DEFAULT_METADATA]) self.dsmode = self._determine_dsmode([ results.get('dsmode'), self.ds_cfg.get('dsmode'), sources.DSMODE_PASS if results['version'] == 1 else None ]) if self.dsmode == sources.DSMODE_DISABLED: return False prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] if prev_iid != cur_iid: # better would be to handle this centrally, allowing # the datasource to do something on new instance id # note, networking is only rendered here if dsmode is DSMODE_PASS # which means "DISABLED, but render files and networking" on_first_boot(results, distro=self.distro, network=self.dsmode == sources.DSMODE_PASS) # This is legacy and sneaky. If dsmode is 'pass' then do not claim # the datasource was used, even though we did run on_first_boot above. if self.dsmode == sources.DSMODE_PASS: LOG.debug("%s: not claiming datasource, dsmode=%s", self, self.dsmode) return False self.source = found self.metadata = md self.ec2_metadata = results.get('ec2-metadata') self.userdata_raw = results.get('userdata') self.version = results['version'] self.files.update(results.get('files', {})) vd = results.get('vendordata') self.vendordata_pure = vd try: self.vendordata_raw = sources.convert_vendordata(vd) except ValueError as e: LOG.warning("Invalid content in vendor-data: %s", e) self.vendordata_raw = None # network_config is an /etc/network/interfaces formated file and is # obsolete compared to networkdata (from network_data.json) but both # might be present. self.network_eni = results.get("network_config") self.network_json = results.get('networkdata') return True
def handle(name, cfg, cloud, log, args): # Get config lxd_cfg = cfg.get("lxd") if not lxd_cfg: log.debug( "Skipping module named %s, not present or disabled by cfg", name ) return if not isinstance(lxd_cfg, dict): log.warning( "lxd config must be a dictionary. found a '%s'", type(lxd_cfg) ) return # Grab the configuration init_cfg = lxd_cfg.get("init") if not isinstance(init_cfg, dict): log.warning( "lxd/init config must be a dictionary. found a '%s'", type(init_cfg), ) init_cfg = {} bridge_cfg = lxd_cfg.get("bridge", {}) if not isinstance(bridge_cfg, dict): log.warning( "lxd/bridge config must be a dictionary. found a '%s'", type(bridge_cfg), ) bridge_cfg = {} # Install the needed packages packages = [] if not subp.which("lxd"): packages.append("lxd") if init_cfg.get("storage_backend") == "zfs" and not subp.which("zfs"): packages.append("zfsutils-linux") if len(packages): try: cloud.distro.install_packages(packages) except subp.ProcessExecutionError as exc: log.warning("failed to install packages %s: %s", packages, exc) return # Set up lxd if init config is given if init_cfg: init_keys = ( "network_address", "network_port", "storage_backend", "storage_create_device", "storage_create_loop", "storage_pool", "trust_password", ) subp.subp(["lxd", "waitready", "--timeout=300"]) cmd = ["lxd", "init", "--auto"] for k in init_keys: if init_cfg.get(k): cmd.extend( ["--%s=%s" % (k.replace("_", "-"), str(init_cfg[k]))] ) subp.subp(cmd) # Set up lxd-bridge if bridge config is given dconf_comm = "debconf-communicate" if bridge_cfg: net_name = bridge_cfg.get("name", _DEFAULT_NETWORK_NAME) if os.path.exists("/etc/default/lxd-bridge") and subp.which( dconf_comm ): # Bridge configured through packaging debconf = bridge_to_debconf(bridge_cfg) # Update debconf database try: log.debug("Setting lxd debconf via " + dconf_comm) data = ( "\n".join( ["set %s %s" % (k, v) for k, v in debconf.items()] ) + "\n" ) subp.subp(["debconf-communicate"], data) except Exception: util.logexc( log, "Failed to run '%s' for lxd with" % dconf_comm ) # Remove the existing configuration file (forces re-generation) util.del_file("/etc/default/lxd-bridge") # Run reconfigure log.debug("Running dpkg-reconfigure for lxd") subp.subp(["dpkg-reconfigure", "lxd", "--frontend=noninteractive"]) else: # Built-in LXD bridge support cmd_create, cmd_attach = bridge_to_cmd(bridge_cfg) maybe_cleanup_default( net_name=net_name, did_init=bool(init_cfg), create=bool(cmd_create), attach=bool(cmd_attach), ) if cmd_create: log.debug("Creating lxd bridge: %s" % " ".join(cmd_create)) _lxc(cmd_create) if cmd_attach: log.debug( "Setting up default lxd bridge: %s" % " ".join(cmd_attach) ) _lxc(cmd_attach) elif bridge_cfg: raise RuntimeError( "Unable to configure lxd bridge without %s." + dconf_comm )
def get_data(self): found = None meta_data = {} results = {} # read from the seed directory if os.path.isdir(self.seed_dir): try: results = read_config_drive(self.seed_dir) found = self.seed_dir except openstack.NonReadable: found = None util.logexc(LOG, "Failed reading config drive from %s", self.seed_dir) else: found = None util.logexc(LOG, "The seed directory %s not exists", self.seed_dir) if not found: # read from the opt device directory if os.path.isdir(self.opt_dir): try: results = read_config_drive(self.opt_dir) found = self.opt_dir except openstack.NonReadable: found = None util.logexc(LOG, "Failed reading config drive from %s", self.opt_dir) else: util.logexc(LOG, "The opt device directory %s not exists", self.opt_dir) if not found: # read from the opt device backup directory if os.path.isdir(self.backup_dir): try: util.logexc(LOG, "reading config drive from %s", self.backup_dir) results = read_config_drive(self.backup_dir) found = self.backup_dir except openstack.NonReadable: found = None util.logexc(LOG, "Failed reading config drive from %s", self.backup_dir) else: util.logexc(LOG, "The opt device backup directory %s not exists", self.backup_dir) found = None if not found: return False meta_data = results.get('metadata', {}) meta_data = util.mergemanydict([meta_data, DEFAULT_METADATA]) user_dsmode = results.get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None, ): LOG.warning("User specified invalid mode: %s", user_dsmode) user_dsmode = None dsmode = get_ds_mode(cfgdrv_ver=results['version'], ds_cfg=self.ds_cfg.get('dsmode'), user=user_dsmode) if dsmode == "disabled": # most likely user specified return False LOG.debug("DataSourceConfigDrive.py get_data results=%s", results) # we want to do some things (writing files and network config) # only on first boot, and even then, we want to do so in the # local datasource (so they happen earlier) even if the configured # dsmode is 'net' or 'pass'. To do this, we check the previous # instance-id prev_iid = get_previous_iid(self.paths) LOG.debug("DataSourceConfigDrive.py get_data prev_iid=%s", prev_iid) cur_iid = meta_data['instance-id'] LOG.debug("DataSourceConfigDrive.py get_data cur_iid=%s", cur_iid) LOG.debug("DataSourceConfigDrive.py get_data dsmode=%s,self.dsmode=%s", dsmode, self.dsmode) # Comment out the below line if prev_iid != cur_iid and self.dsmode == "local": on_first_boot(results, distro=self.distro) # dsmode != self.dsmode here if: # * dsmode = "pass", pass means it should only copy files and then # pass to another datasource # * dsmode = "net" and self.dsmode = "local" # so that user boothooks would be applied with network, the # local datasource just gets out of the way, and lets the net claim if dsmode != self.dsmode: LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) return False self.source = found self.metadata = meta_data self.ec2_metadata = results.get('ec2-metadata') self.userdata_raw = results.get('userdata') self.version = results['version'] self.files.update(results.get('files', {})) self.vendordata_raw = results.get('vendordata') return True
def _get_data(self): defaults = { "instance-id": "nocloud", "dsmode": self.dsmode, } found = [] mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': "", 'network-config': None} try: # Parse the system serial label from dmi. If not empty, try parsing # like the commandline md = {} serial = util.read_dmi_data('system-serial-number') if serial and load_cmdline_data(md, serial): found.append("dmi") mydata = _merge_new_seed(mydata, {'meta-data': md}) except Exception: util.logexc(LOG, "Unable to parse dmi data") return False try: # Parse the kernel command line, getting data passed in md = {} if load_cmdline_data(md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) except Exception: util.logexc(LOG, "Unable to parse command line data") return False # Check to see if the seed dir has data. pp2d_kwargs = {'required': ['user-data', 'meta-data'], 'optional': ['vendor-data', 'network-config']} for path in self.seed_dirs: try: seeded = util.pathprefix2dict(path, **pp2d_kwargs) found.append(path) LOG.debug("Using seeded data from %s", path) mydata = _merge_new_seed(mydata, seeded) break except ValueError: pass # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem # but not over external media if self.ds_cfg.get('seedfrom'): found.append("ds_config_seedfrom") mydata['meta-data']["seedfrom"] = self.ds_cfg['seedfrom'] # fields appropriately named can also just come from the datasource # config (ie, 'user-data', 'meta-data', 'vendor-data' there) if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: mydata = _merge_new_seed(mydata, self.ds_cfg) found.append("ds_config") def _pp2d_callback(mp, data): return util.pathprefix2dict(mp, **data) label = self.ds_cfg.get('fs_label', "cidata") if label is not None: for dev in self._get_devices(label): try: LOG.debug("Attempting to use data from %s", dev) try: seeded = util.mount_cb(dev, _pp2d_callback, pp2d_kwargs) except ValueError: LOG.warning("device %s with label=%s not a" "valid seed.", dev, label) continue mydata = _merge_new_seed(mydata, seeded) LOG.debug("Using data from %s", dev) found.append(dev) break except OSError as e: if e.errno != errno.ENOENT: raise except util.MountFailedError: util.logexc(LOG, "Failed to mount %s when looking for " "data", dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. if len(found) == 0: return False # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg if "seedfrom" in mydata['meta-data']: seedfrom = mydata['meta-data']["seedfrom"] seedfound = False for proto in self.supported_seed_starts: if seedfrom.startswith(proto): seedfound = proto break if not seedfound: LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) LOG.debug("Using seeded cache data from %s", seedfrom) # Values in the command line override those from the seed mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], md_seed]) mydata['user-data'] = ud found.append(seedfrom) # Now that we have exhausted any other places merge in the defaults mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], defaults]) self.dsmode = self._determine_dsmode( [mydata['meta-data'].get('dsmode')]) if self.dsmode == sources.DSMODE_DISABLED: LOG.debug("%s: not claiming datasource, dsmode=%s", self, self.dsmode) return False self.seed = ",".join(found) self.metadata = mydata['meta-data'] self.userdata_raw = mydata['user-data'] self.vendordata_raw = mydata['vendor-data'] self._network_config = mydata['network-config'] self._network_eni = mydata['meta-data'].get('network-interfaces') return True
def expire_passwd(self, user): try: subp.subp(["pw", "usermod", user, "-p", "01-Jan-1970"]) except Exception: util.logexc(LOG, "Failed to set pw expiration for %s", user) raise
def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: if "phone_home" not in cfg: log.debug( "Skipping module named %s, " "no 'phone_home' configuration found", name, ) return ph_cfg = cfg["phone_home"] if "url" not in ph_cfg: log.warning( "Skipping module named %s, " "no 'url' found in 'phone_home' configuration", name, ) return url = ph_cfg["url"] post_list = ph_cfg.get("post", "all") tries = ph_cfg.get("tries") try: tries = int(tries) # type: ignore except ValueError: tries = 10 util.logexc( log, "Configuration entry 'tries' is not an integer, using %s instead", tries, ) if post_list == "all": post_list = POST_LIST_ALL all_keys = {} all_keys["instance_id"] = cloud.get_instance_id() all_keys["hostname"] = cloud.get_hostname() all_keys["fqdn"] = cloud.get_hostname(fqdn=True) pubkeys = { "pub_key_dsa": "/etc/ssh/ssh_host_dsa_key.pub", "pub_key_rsa": "/etc/ssh/ssh_host_rsa_key.pub", "pub_key_ecdsa": "/etc/ssh/ssh_host_ecdsa_key.pub", "pub_key_ed25519": "/etc/ssh/ssh_host_ed25519_key.pub", } for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) except Exception: util.logexc(log, "%s: failed to open, can not phone home that data!", path) submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] else: submit_keys[k] = None log.warning( "Requested key %s from 'post'" " configuration list not available", k, ) # Get them read to be posted real_submit_keys = {} for (k, v) in submit_keys.items(): if v is None: real_submit_keys[k] = "N/A" else: real_submit_keys[k] = str(v) # Incase the url is parameterized url_params = { "INSTANCE_ID": all_keys["instance_id"], } url = templater.render_string(url, url_params) try: url_helper.read_file_or_url( url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths), ) except Exception: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries)
def do_resize(resize_cmd, log): try: subp.subp(resize_cmd) except subp.ProcessExecutionError: util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd) raise
def user_data_rhevm(self): ''' RHEVM specific userdata read If on RHEV-M the user data will be contained on the floppy device in file <user_data_file> To access it: modprobe floppy Leverage util.mount_cb to: mkdir <tmp mount dir> mount /dev/fd0 <tmp mount dir> The call back passed to util.mount_cb will do: read <tmp mount dir>/<user_data_file> ''' return_str = None # modprobe floppy try: cmd = CMD_PROBE_FLOPPY (cmd_out, _err) = util.subp(cmd) LOG.debug('Command: %s\nOutput%s', ' '.join(cmd), cmd_out) except ProcessExecutionError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False floppy_dev = '/dev/fd0' # udevadm settle for floppy device try: cmd = CMD_UDEVADM_SETTLE cmd.append('--exit-if-exists=' + floppy_dev) (cmd_out, _err) = util.subp(cmd) LOG.debug('Command: %s\nOutput%s', ' '.join(cmd), cmd_out) except ProcessExecutionError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False try: return_str = util.mount_cb(floppy_dev, read_user_data_callback) except OSError as err: if err.errno != errno.ENOENT: raise except util.MountFailedError: util.logexc(LOG, "Failed to mount %s when looking for user data", floppy_dev) self.userdata_raw = return_str self.metadata = META_DATA_NOT_SUPPORTED if return_str: return True else: return False
def update_hostname(self, hostname, fqdn, prev_hostname_fn): applying_hostname = hostname # Determine what the actual written hostname should be hostname = self._select_hostname(hostname, fqdn) # If the previous hostname file exists lets see if we # can get a hostname from it if prev_hostname_fn and os.path.exists(prev_hostname_fn): prev_hostname = self._read_hostname(prev_hostname_fn) else: prev_hostname = None # Lets get where we should write the system hostname # and what the system hostname is (sys_fn, sys_hostname) = self._read_system_hostname() update_files = [] # If there is no previous hostname or it differs # from what we want, lets update it or create the # file in the first place if not prev_hostname or prev_hostname != hostname: update_files.append(prev_hostname_fn) # If the system hostname is different than the previous # one or the desired one lets update it as well if (not sys_hostname) or ( sys_hostname == prev_hostname and sys_hostname != hostname ): update_files.append(sys_fn) # If something else has changed the hostname after we set it # initially, we should not overwrite those changes (we should # only be setting the hostname once per instance) if sys_hostname and prev_hostname and sys_hostname != prev_hostname: LOG.info( "%s differs from %s, assuming user maintained hostname.", prev_hostname_fn, sys_fn, ) return # Remove duplicates (incase the previous config filename) # is the same as the system config filename, don't bother # doing it twice update_files = set([f for f in update_files if f]) LOG.debug( "Attempting to update hostname to %s in %s files", hostname, len(update_files), ) for fn in update_files: try: self._write_hostname(hostname, fn) except IOError: util.logexc( LOG, "Failed to write hostname %s to %s", hostname, fn ) # If the system hostname file name was provided set the # non-fqdn as the transient hostname. if sys_fn in update_files: self._apply_hostname(applying_hostname)
def main_init(name, args): deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] if args.local: deps = [sources.DEP_FILESYSTEM] if not args.local: # See doc/kernel-cmdline.txt # # This is used in maas datasource, in "ephemeral" (read-only root) # environment where the instance netboots to iscsi ro root. # and the entity that controls the pxe config has to configure # the maas datasource. # # Could be used elsewhere, only works on network based (not local). root_name = "%s.d" % (CLOUD_CONFIG) target_fn = os.path.join(root_name, "91_kernel_cmdline_url.cfg") util.read_write_cmdline_url(target_fn) # Cloud-init 'init' stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Setup logging/output redirections with resultant config (if any) # 3. Initialize the cloud-init filesystem # 4. Check if we can stop early by looking for various files # 5. Fetch the datasource # 6. Connect to the current instance location + update the cache # 7. Consume the userdata (handlers get activated here) # 8. Construct the modules object # 9. Adjust any subsequent logging/output redirections using the modules # objects config as it may be different from init object # 10. Run the modules for the 'init' stage # 11. Done! if not args.local: w_msg = welcome_format(name) else: w_msg = welcome_format("%s-local" % (name)) init = stages.Init(ds_deps=deps, reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 outfmt = None errfmt = None try: LOG.debug("Closing stdin") util.close_stdin() (outfmt, errfmt) = util.fixup_output(init.cfg, name) except Exception: util.logexc(LOG, "Failed to setup output redirection!") print_exc("Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" " longer be active shortly")) logging.resetLogging() logging.setupLogging(init.cfg) apply_reporting_cfg(init.cfg) # Any log usage prior to setupLogging above did not have local user log # config applied. We send the welcome message now, as stderr/out have # been redirected and log now configured. welcome(name, msg=w_msg) # Stage 3 try: init.initialize() except Exception: util.logexc(LOG, "Failed to initialize, likely bad things to come!") # Stage 4 path_helper = init.paths mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK if mode == sources.DSMODE_NETWORK: existing = "trust" sys.stderr.write("%s\n" % (netinfo.debug_info())) LOG.debug(("Checking to see if files that we need already" " exist from a previous run that would allow us" " to stop early.")) # no-net is written by upstart cloud-init-nonet when network failed # to come up stop_files = [ os.path.join(path_helper.get_cpath("data"), "no-net"), ] existing_files = [] for fn in stop_files: if os.path.isfile(fn): existing_files.append(fn) if existing_files: LOG.debug("[%s] Exiting. stop file %s existed", mode, existing_files) return (None, []) else: LOG.debug("Execution continuing, no previous run detected that" " would allow us to stop early.") else: existing = "check" if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): existing = "trust" init.purge_cache() # Delete the non-net file as well util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net")) # Stage 5 try: init.fetch(existing=existing) # if in network mode, and the datasource is local # then work was done at that stage. if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: LOG.debug("[%s] Exiting. datasource %s in local mode", mode, init.datasource) return (None, []) except sources.DataSourceNotFoundException: # In the case of 'cloud-init init' without '--local' it is a bit # more likely that the user would consider it failure if nothing was # found. When using upstart it will also mentions job failure # in console log if exit code is != 0. if mode == sources.DSMODE_LOCAL: LOG.debug("No local datasource found") else: util.logexc(LOG, ("No instance datasource found!" " Likely bad things to come!")) if not args.force: init.apply_network_config(bring_up=not args.local) LOG.debug("[%s] Exiting without datasource in local mode", mode) if mode == sources.DSMODE_LOCAL: return (None, []) else: return (None, ["No instance datasource found."]) else: LOG.debug("[%s] barreling on in force mode without datasource", mode) # Stage 6 iid = init.instancify() LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", mode, name, iid, init.is_new_instance()) init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) if mode == sources.DSMODE_LOCAL: if init.datasource.dsmode != mode: LOG.debug("[%s] Exiting. datasource %s not in local mode.", mode, init.datasource) return (init.datasource, []) else: LOG.debug("[%s] %s is in local mode, will apply init modules now.", mode, init.datasource) # update fully realizes user-data (pulling in #include if necessary) init.update() # Stage 7 try: # Attempt to consume the data per instance. # This may run user-data handlers and/or perform # url downloads and such as needed. (ran, _results) = init.cloudify().run('consume_data', init.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) if not ran: # Just consume anything that is set to run per-always # if nothing ran in the per-instance code # # See: https://bugs.launchpad.net/bugs/819507 for a little # reason behind this... init.consume_data(PER_ALWAYS) except Exception: util.logexc(LOG, "Consuming user data failed!") return (init.datasource, ["Consuming user data failed!"]) apply_reporting_cfg(init.cfg) # Stage 8 - re-read and apply relevant cloud-config to include user-data mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) # Stage 9 try: outfmt_orig = outfmt errfmt_orig = errfmt (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) if outfmt_orig != outfmt or errfmt_orig != errfmt: LOG.warn("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) (outfmt, errfmt) = util.fixup_output(mods.cfg, name) except Exception: util.logexc(LOG, "Failed to re-adjust output redirection!") logging.setupLogging(mods.cfg) # Stage 10 return (init.datasource, run_module_section(mods, name, name))
def add_user(self, name, **kwargs): """ Add a user to the system using standard GNU tools """ # XXX need to make add_user idempotent somehow as we # still want to add groups or modify SSH keys on pre-existing # users in the image. if util.is_user(name): LOG.info("User %s already exists, skipping.", name) return if 'create_groups' in kwargs: create_groups = kwargs.pop('create_groups') else: create_groups = True useradd_cmd = ['useradd', name] log_useradd_cmd = ['useradd', name] if util.system_is_snappy(): useradd_cmd.append('--extrausers') log_useradd_cmd.append('--extrausers') # Since we are creating users, we want to carefully validate the # inputs. If something goes wrong, we can end up with a system # that nobody can login to. useradd_opts = { "gecos": '--comment', "homedir": '--home', "primary_group": '--gid', "uid": '--uid', "groups": '--groups', "passwd": '--password', "shell": '--shell', "expiredate": '--expiredate', "inactive": '--inactive', "selinux_user": '******', } useradd_flags = { "no_user_group": '--no-user-group', "system": '--system', "no_log_init": '--no-log-init', } redact_opts = ['passwd'] # support kwargs having groups=[list] or groups="g1,g2" groups = kwargs.get('groups') if groups: if isinstance(groups, str): groups = groups.split(",") # remove any white spaces in group names, most likely # that came in as a string like: groups: group1, group2 groups = [g.strip() for g in groups] # kwargs.items loop below wants a comma delimeted string # that can go right through to the command. kwargs['groups'] = ",".join(groups) primary_group = kwargs.get('primary_group') if primary_group: groups.append(primary_group) if create_groups and groups: for group in groups: if not util.is_group(group): self.create_group(group) LOG.debug("created group '%s' for user '%s'", group, name) # Check the values and create the command for key, val in sorted(kwargs.items()): if key in useradd_opts and val and isinstance(val, str): useradd_cmd.extend([useradd_opts[key], val]) # Redact certain fields from the logs if key in redact_opts: log_useradd_cmd.extend([useradd_opts[key], 'REDACTED']) else: log_useradd_cmd.extend([useradd_opts[key], val]) elif key in useradd_flags and val: useradd_cmd.append(useradd_flags[key]) log_useradd_cmd.append(useradd_flags[key]) # Don't create the home directory if directed so or if the user is a # system user if kwargs.get('no_create_home') or kwargs.get('system'): useradd_cmd.append('-M') log_useradd_cmd.append('-M') else: useradd_cmd.append('-m') log_useradd_cmd.append('-m') # Run the command LOG.debug("Adding user %s", name) try: subp.subp(useradd_cmd, logstring=log_useradd_cmd) except Exception as e: util.logexc(LOG, "Failed to create user %s", name) raise e
def status_wrapper(name, args, data_d=None, link_d=None): if data_d is None: data_d = os.path.normpath("/var/lib/cloud/data") if link_d is None: link_d = os.path.normpath("/run/cloud-init") status_path = os.path.join(data_d, "status.json") status_link = os.path.join(link_d, "status.json") result_path = os.path.join(data_d, "result.json") result_link = os.path.join(link_d, "result.json") util.ensure_dirs(( data_d, link_d, )) (_name, functor) = args.action if name == "init": if args.local: mode = "init-local" else: mode = "init" elif name == "modules": mode = "modules-%s" % args.mode else: raise ValueError("unknown name: %s" % name) modes = ('init', 'init-local', 'modules-config', 'modules-final') status = None if mode == 'init-local': for f in (status_link, result_link, status_path, result_path): util.del_file(f) else: try: status = json.loads(util.load_file(status_path)) except Exception: pass if status is None: nullstatus = { 'errors': [], 'start': None, 'finished': None, } status = {'v1': {}} for m in modes: status['v1'][m] = nullstatus.copy() status['v1']['datasource'] = None v1 = status['v1'] v1['stage'] = mode v1[mode]['start'] = time.time() atomic_helper.write_json(status_path, status) util.sym_link(os.path.relpath(status_path, link_d), status_link, force=True) try: ret = functor(name, args) if mode in ('init', 'init-local'): (datasource, errors) = ret if datasource is not None: v1['datasource'] = str(datasource) else: errors = ret v1[mode]['errors'] = [str(e) for e in errors] except Exception as e: util.logexc(LOG, "failed stage %s", mode) print_exc("failed run of stage %s" % mode) v1[mode]['errors'] = [str(e)] v1[mode]['finished'] = time.time() v1['stage'] = None atomic_helper.write_json(status_path, status) if mode == "modules-final": # write the 'finished' file errors = [] for m in modes: if v1[m]['errors']: errors.extend(v1[m].get('errors', [])) atomic_helper.write_json( result_path, {'v1': { 'datasource': v1['datasource'], 'errors': errors }}) util.sym_link(os.path.relpath(result_path, link_d), result_link, force=True) return len(v1[mode]['errors'])
def expire_passwd(self, user): try: subp.subp(['passwd', '--expire', user]) except Exception as e: util.logexc(LOG, "Failed to set 'expire' for %s", user) raise e
def lock_passwd(self, name): try: util.subp(['pw', 'usermod', name, '-h', '-']) except Exception as e: util.logexc(LOG, "Failed to lock user %s", name) raise e
def handle(_name, cfg, cloud, log, _args): # remove the static keys from the pristine image if cfg.get("ssh_deletekeys", True): key_pth = os.path.join("/etc/ssh/", "ssh_host_*key*") for f in glob.glob(key_pth): try: util.del_file(f) except Exception: util.logexc(log, "Failed deleting key file %s", f) if "ssh_keys" in cfg: # if there are keys in cloud-config, use them for (key, val) in cfg["ssh_keys"].items(): if key in CONFIG_KEY_TO_FILE: tgt_fn = CONFIG_KEY_TO_FILE[key][0] tgt_perms = CONFIG_KEY_TO_FILE[key][1] util.write_file(tgt_fn, val, tgt_perms) for (priv, pub) in PRIV_TO_PUB.items(): if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: continue pair = (CONFIG_KEY_TO_FILE[priv][0], CONFIG_KEY_TO_FILE[pub][0]) cmd = ['sh', '-xc', KEY_GEN_TPL % pair] try: # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): subp.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except Exception: util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) else: # if not, generate them genkeys = util.get_cfg_option_list(cfg, 'ssh_genkeytypes', GENERATE_KEY_NAMES) lang_c = os.environ.copy() lang_c['LANG'] = 'C' for keytype in genkeys: keyfile = KEY_FILE_TPL % (keytype) if os.path.exists(keyfile): continue util.ensure_dir(os.path.dirname(keyfile)) cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): try: out, err = subp.subp(cmd, capture=True, env=lang_c) sys.stdout.write(util.decode_binary(out)) except subp.ProcessExecutionError as e: err = util.decode_binary(e.stderr).lower() if (e.exit_code == 1 and err.lower().startswith("unknown key")): log.debug("ssh-keygen: unknown key type '%s'", keytype) else: util.logexc( log, "Failed generating key type %s to " "file %s", keytype, keyfile) if "ssh_publish_hostkeys" in cfg: host_key_blacklist = util.get_cfg_option_list( cfg["ssh_publish_hostkeys"], "blacklist", HOST_KEY_PUBLISH_BLACKLIST) publish_hostkeys = util.get_cfg_option_bool( cfg["ssh_publish_hostkeys"], "enabled", PUBLISH_HOST_KEYS) else: host_key_blacklist = HOST_KEY_PUBLISH_BLACKLIST publish_hostkeys = PUBLISH_HOST_KEYS if publish_hostkeys: hostkeys = get_public_host_keys(blacklist=host_key_blacklist) try: cloud.datasource.publish_host_keys(hostkeys) except Exception: util.logexc(log, "Publishing host keys failed!") try: (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) (user, _user_config) = ug_util.extract_default(users) disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", ssh_util.DISABLE_USER_OPTS) keys = [] if util.get_cfg_option_bool(cfg, 'allow_public_ssh_keys', True): keys = cloud.get_public_ssh_keys() or [] else: log.debug('Skipping import of publish SSH keys per ' 'config setting: allow_public_ssh_keys=False') if "ssh_authorized_keys" in cfg: cfgkeys = cfg["ssh_authorized_keys"] keys.extend(cfgkeys) apply_credentials(keys, user, disable_root, disable_root_opts) except Exception: util.logexc(log, "Applying SSH credentials failed!")
def get_data(self): found = None md = {} results = {} if os.path.isdir(self.seed_dir): try: results = read_config_drive(self.seed_dir) found = self.seed_dir except openstack.NonReadable: util.logexc(LOG, "Failed reading config drive from %s", self.seed_dir) if not found: for dev in find_candidate_devs(): try: results = util.mount_cb(dev, read_config_drive) found = dev except openstack.NonReadable: pass except util.MountFailedError: pass except openstack.BrokenMetadata: util.logexc(LOG, "Broken config drive: %s", dev) if found: break if not found: return False md = results.get('metadata', {}) md = util.mergemanydict([md, DEFAULT_METADATA]) user_dsmode = results.get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): LOG.warn("User specified invalid mode: %s", user_dsmode) user_dsmode = None dsmode = get_ds_mode(cfgdrv_ver=results['version'], ds_cfg=self.ds_cfg.get('dsmode'), user=user_dsmode) if dsmode == "disabled": # most likely user specified return False # TODO(smoser): fix this, its dirty. # we want to do some things (writing files and network config) # only on first boot, and even then, we want to do so in the # local datasource (so they happen earlier) even if the configured # dsmode is 'net' or 'pass'. To do this, we check the previous # instance-id prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] if prev_iid != cur_iid and self.dsmode == "local": on_first_boot(results, distro=self.distro) # dsmode != self.dsmode here if: # * dsmode = "pass", pass means it should only copy files and then # pass to another datasource # * dsmode = "net" and self.dsmode = "local" # so that user boothooks would be applied with network, the # local datasource just gets out of the way, and lets the net claim if dsmode != self.dsmode: LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) return False self.source = found self.metadata = md self.ec2_metadata = results.get('ec2-metadata') self.userdata_raw = results.get('userdata') self.version = results['version'] self.files.update(results.get('files', {})) vd = results.get('vendordata') self.vendordata_pure = vd try: self.vendordata_raw = openstack.convert_vendordata_json(vd) except ValueError as e: LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None return True
def main_init(name, args): deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] if args.local: deps = [sources.DEP_FILESYSTEM] early_logs = [ attempt_cmdline_url( path=os.path.join( "%s.d" % CLOUD_CONFIG, "91_kernel_cmdline_url.cfg" ), network=not args.local, ) ] # Cloud-init 'init' stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Setup logging/output redirections with resultant config (if any) # 3. Initialize the cloud-init filesystem # 4. Check if we can stop early by looking for various files # 5. Fetch the datasource # 6. Connect to the current instance location + update the cache # 7. Consume the userdata (handlers get activated here) # 8. Construct the modules object # 9. Adjust any subsequent logging/output redirections using the modules # objects config as it may be different from init object # 10. Run the modules for the 'init' stage # 11. Done! if not args.local: w_msg = welcome_format(name) else: w_msg = welcome_format("%s-local" % (name)) init = stages.Init(ds_deps=deps, reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 outfmt = None errfmt = None try: early_logs.append((logging.DEBUG, "Closing stdin.")) util.close_stdin() (outfmt, errfmt) = util.fixup_output(init.cfg, name) except Exception: msg = "Failed to setup output redirection!" util.logexc(LOG, msg) print_exc(msg) early_logs.append((logging.WARN, msg)) if args.debug: # Reset so that all the debug handlers are closed out LOG.debug( "Logging being reset, this logger may no longer be active shortly" ) logging.resetLogging() logging.setupLogging(init.cfg) apply_reporting_cfg(init.cfg) # Any log usage prior to setupLogging above did not have local user log # config applied. We send the welcome message now, as stderr/out have # been redirected and log now configured. welcome(name, msg=w_msg) # re-play early log messages before logging was setup for lvl, msg in early_logs: LOG.log(lvl, msg) # Stage 3 try: init.initialize() except Exception: util.logexc(LOG, "Failed to initialize, likely bad things to come!") # Stage 4 path_helper = init.paths purge_cache_on_python_version_change(init) mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK if mode == sources.DSMODE_NETWORK: existing = "trust" sys.stderr.write("%s\n" % (netinfo.debug_info())) else: existing = "check" mcfg = util.get_cfg_option_bool(init.cfg, "manual_cache_clean", False) if mcfg: LOG.debug("manual cache clean set from config") existing = "trust" else: mfile = path_helper.get_ipath_cur("manual_clean_marker") if os.path.exists(mfile): LOG.debug("manual cache clean found from marker: %s", mfile) existing = "trust" init.purge_cache() # Stage 5 bring_up_interfaces = _should_bring_up_interfaces(init, args) try: init.fetch(existing=existing) # if in network mode, and the datasource is local # then work was done at that stage. if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: LOG.debug( "[%s] Exiting. datasource %s in local mode", mode, init.datasource, ) return (None, []) except sources.DataSourceNotFoundException: # In the case of 'cloud-init init' without '--local' it is a bit # more likely that the user would consider it failure if nothing was # found. if mode == sources.DSMODE_LOCAL: LOG.debug("No local datasource found") else: util.logexc( LOG, "No instance datasource found! Likely bad things to come!" ) if not args.force: init.apply_network_config(bring_up=bring_up_interfaces) LOG.debug("[%s] Exiting without datasource", mode) if mode == sources.DSMODE_LOCAL: return (None, []) else: return (None, ["No instance datasource found."]) else: LOG.debug( "[%s] barreling on in force mode without datasource", mode ) _maybe_persist_instance_data(init) # Stage 6 iid = init.instancify() LOG.debug( "[%s] %s will now be targeting instance id: %s. new=%s", mode, name, iid, init.is_new_instance(), ) if mode == sources.DSMODE_LOCAL: # Before network comes up, set any configured hostname to allow # dhcp clients to advertize this hostname to any DDNS services # LP: #1746455. _maybe_set_hostname(init, stage="local", retry_stage="network") init.apply_network_config(bring_up=bring_up_interfaces) if mode == sources.DSMODE_LOCAL: if init.datasource.dsmode != mode: LOG.debug( "[%s] Exiting. datasource %s not in local mode.", mode, init.datasource, ) return (init.datasource, []) else: LOG.debug( "[%s] %s is in local mode, will apply init modules now.", mode, init.datasource, ) # Give the datasource a chance to use network resources. # This is used on Azure to communicate with the fabric over network. init.setup_datasource() # update fully realizes user-data (pulling in #include if necessary) init.update() _maybe_set_hostname(init, stage="init-net", retry_stage="modules:config") # Stage 7 try: # Attempt to consume the data per instance. # This may run user-data handlers and/or perform # url downloads and such as needed. (ran, _results) = init.cloudify().run( "consume_data", init.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) if not ran: # Just consume anything that is set to run per-always # if nothing ran in the per-instance code # # See: https://bugs.launchpad.net/bugs/819507 for a little # reason behind this... init.consume_data(PER_ALWAYS) except Exception: util.logexc(LOG, "Consuming user data failed!") return (init.datasource, ["Consuming user data failed!"]) # Validate user-data adheres to schema definition if os.path.exists(init.paths.get_ipath_cur("userdata_raw")): validate_cloudconfig_schema( config=init.cfg, strict=False, log_details=False ) else: LOG.debug("Skipping user-data validation. No user-data found.") apply_reporting_cfg(init.cfg) # Stage 8 - re-read and apply relevant cloud-config to include user-data mods = Modules(init, extract_fns(args), reporter=args.reporter) # Stage 9 try: outfmt_orig = outfmt errfmt_orig = errfmt (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) if outfmt_orig != outfmt or errfmt_orig != errfmt: LOG.warning("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) (outfmt, errfmt) = util.fixup_output(mods.cfg, name) except Exception: util.logexc(LOG, "Failed to re-adjust output redirection!") logging.setupLogging(mods.cfg) # give the activated datasource a chance to adjust init.activate_datasource() di_report_warn(datasource=init.datasource, cfg=init.cfg) # Stage 10 return (init.datasource, run_module_section(mods, name, name))
def handle(_name, cfg, cloud, log, _args): # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno def_mnt_opts = "defaults,nobootwait" uses_systemd = cloud.distro.uses_systemd() if uses_systemd: def_mnt_opts = "defaults,nofail,x-systemd.requires=cloud-init.service" defvals = [None, None, "auto", def_mnt_opts, "0", "2"] defvals = cfg.get("mount_default_fields", defvals) # these are our default set of mounts defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], ["swap", "none", "swap", "sw", "0", "0"]] cfgmnt = [] if "mounts" in cfg: cfgmnt = cfg["mounts"] LOG.debug("mounts configuration is %s", cfgmnt) fstab_lines = [] fstab_devs = {} fstab_removed = [] for line in util.load_file(FSTAB_PATH).splitlines(): if MNT_COMMENT in line: fstab_removed.append(line) continue try: toks = WS.split(line) except Exception: pass fstab_devs[toks[0]] = line fstab_lines.append(line) for i in range(len(cfgmnt)): # skip something that wasn't a list if not isinstance(cfgmnt[i], list): log.warning("Mount option %s not a list, got a %s instead", (i + 1), type_utils.obj_name(cfgmnt[i])) continue start = str(cfgmnt[i][0]) sanitized = sanitize_devname(start, cloud.device_name_to_device, log) if sanitized != start: log.debug("changed %s => %s" % (start, sanitized)) if sanitized is None: log.debug("Ignoring nonexistent named mount %s", start) continue elif sanitized in fstab_devs: log.info("Device %s already defined in fstab: %s", sanitized, fstab_devs[sanitized]) continue cfgmnt[i][0] = sanitized # in case the user did not quote a field (likely fs-freq, fs_passno) # but do not convert None to 'None' (LP: #898365) for j in range(len(cfgmnt[i])): if cfgmnt[i][j] is None: continue else: cfgmnt[i][j] = str(cfgmnt[i][j]) for i in range(len(cfgmnt)): # fill in values with defaults from defvals above for j in range(len(defvals)): if len(cfgmnt[i]) <= j: cfgmnt[i].append(defvals[j]) elif cfgmnt[i][j] is None: cfgmnt[i][j] = defvals[j] # if the second entry in the list is 'None' this # clears all previous entries of that same 'fs_spec' # (fs_spec is the first field in /etc/fstab, ie, that device) if cfgmnt[i][1] is None: for j in range(i): if cfgmnt[j][0] == cfgmnt[i][0]: cfgmnt[j][1] = None # for each of the "default" mounts, add them only if no other # entry has the same device name for defmnt in defmnts: start = defmnt[0] sanitized = sanitize_devname(start, cloud.device_name_to_device, log) if sanitized != start: log.debug("changed default device %s => %s" % (start, sanitized)) if sanitized is None: log.debug("Ignoring nonexistent default named mount %s", start) continue elif sanitized in fstab_devs: log.debug("Device %s already defined in fstab: %s", sanitized, fstab_devs[sanitized]) continue defmnt[0] = sanitized cfgmnt_has = False for cfgm in cfgmnt: if cfgm[0] == defmnt[0]: cfgmnt_has = True break if cfgmnt_has: log.debug(("Not including %s, already" " previously included"), start) continue cfgmnt.append(defmnt) # now, each entry in the cfgmnt list has all fstab values # if the second field is None (not the string, the value) we skip it actlist = [] for x in cfgmnt: if x[1] is None: log.debug("Skipping nonexistent device named %s", x[0]) else: actlist.append(x) swapret = handle_swapcfg(cfg.get('swap', {})) if swapret: actlist.append([swapret, "none", "swap", "sw", "0", "0"]) if len(actlist) == 0: log.debug("No modifications to fstab needed") return cc_lines = [] needswap = False need_mount_all = False dirs = [] for line in actlist: # write 'comment' in the fs_mntops, entry, claiming this line[3] = "%s,%s" % (line[3], MNT_COMMENT) if line[2] == "swap": needswap = True if line[1].startswith("/"): dirs.append(line[1]) cc_lines.append('\t'.join(line)) mount_points = [ v['mountpoint'] for k, v in util.mounts().items() if 'mountpoint' in v ] for d in dirs: try: util.ensure_dir(d) except Exception: util.logexc(log, "Failed to make '%s' config-mount", d) # dirs is list of directories on which a volume should be mounted. # If any of them does not already show up in the list of current # mount points, we will definitely need to do mount -a. if not need_mount_all and d not in mount_points: need_mount_all = True sadds = [WS.sub(" ", n) for n in cc_lines] sdrops = [WS.sub(" ", n) for n in fstab_removed] sops = (["- " + drop for drop in sdrops if drop not in sadds] + ["+ " + add for add in sadds if add not in sdrops]) fstab_lines.extend(cc_lines) contents = "%s\n" % ('\n'.join(fstab_lines)) util.write_file(FSTAB_PATH, contents) activate_cmds = [] if needswap: activate_cmds.append(["swapon", "-a"]) if len(sops) == 0: log.debug("No changes to /etc/fstab made.") else: log.debug("Changes to fstab: %s", sops) need_mount_all = True if need_mount_all: activate_cmds.append(["mount", "-a"]) if uses_systemd: activate_cmds.append(["systemctl", "daemon-reload"]) fmt = "Activating swap and mounts with: %s" for cmd in activate_cmds: fmt = "Activate mounts: %s:" + ' '.join(cmd) try: util.subp(cmd) log.debug(fmt, "PASS") except util.ProcessExecutionError: log.warning(fmt, "FAIL") util.logexc(log, fmt, "FAIL")
def status_wrapper(name, args, data_d=None, link_d=None): if data_d is None: data_d = os.path.normpath("/var/lib/cloud/data") if link_d is None: link_d = os.path.normpath("/run/cloud-init") status_path = os.path.join(data_d, "status.json") status_link = os.path.join(link_d, "status.json") result_path = os.path.join(data_d, "result.json") result_link = os.path.join(link_d, "result.json") util.ensure_dirs( ( data_d, link_d, ) ) (_name, functor) = args.action if name == "init": if args.local: mode = "init-local" else: mode = "init" elif name == "modules": mode = "modules-%s" % args.mode else: raise ValueError("unknown name: %s" % name) modes = ( "init", "init-local", "modules-init", "modules-config", "modules-final", ) if mode not in modes: raise ValueError( "Invalid cloud init mode specified '{0}'".format(mode) ) status = None if mode == "init-local": for f in (status_link, result_link, status_path, result_path): util.del_file(f) else: try: status = json.loads(util.load_file(status_path)) except Exception: pass nullstatus = { "errors": [], "start": None, "finished": None, } if status is None: status = {"v1": {}} status["v1"]["datasource"] = None for m in modes: if m not in status["v1"]: status["v1"][m] = nullstatus.copy() v1 = status["v1"] v1["stage"] = mode v1[mode]["start"] = time.time() atomic_helper.write_json(status_path, status) util.sym_link( os.path.relpath(status_path, link_d), status_link, force=True ) try: ret = functor(name, args) if mode in ("init", "init-local"): (datasource, errors) = ret if datasource is not None: v1["datasource"] = str(datasource) else: errors = ret v1[mode]["errors"] = [str(e) for e in errors] except Exception as e: util.logexc(LOG, "failed stage %s", mode) print_exc("failed run of stage %s" % mode) v1[mode]["errors"] = [str(e)] v1[mode]["finished"] = time.time() v1["stage"] = None atomic_helper.write_json(status_path, status) if mode == "modules-final": # write the 'finished' file errors = [] for m in modes: if v1[m]["errors"]: errors.extend(v1[m].get("errors", [])) atomic_helper.write_json( result_path, {"v1": {"datasource": v1["datasource"], "errors": errors}}, ) util.sym_link( os.path.relpath(result_path, link_d), result_link, force=True ) return len(v1[mode]["errors"])