def main_modules(action_name, args): name = args.mode # Cloud-init 'modules' stages are broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Get the datasource from the init object, if it does # not exist then that means the main_init stage never # worked, and thus this stage can not run. # 3. Construct the modules object # 4. Adjust any subsequent logging/output redirections using # the modules objects configuration # 5. Run the modules for the given stage name # 6. Done! w_msg = welcome_format("%s:%s" % (action_name, name)) init = stages.Init(ds_deps=[], reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 try: init.fetch(existing="trust") except sources.DataSourceNotFoundException: # There was no datasource found, theres nothing to do msg = ( "Can not apply stage %s, no datasource found! Likely bad " "things to come!" % name ) util.logexc(LOG, msg) print_exc(msg) if not args.force: return [(msg)] _maybe_persist_instance_data(init) # Stage 3 mods = Modules(init, extract_fns(args), reporter=args.reporter) # Stage 4 try: LOG.debug("Closing stdin") util.close_stdin() util.fixup_output(mods.cfg, name) except Exception: util.logexc(LOG, "Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out LOG.debug( "Logging being reset, this logger may no longer be active shortly" ) logging.resetLogging() logging.setupLogging(mods.cfg) apply_reporting_cfg(init.cfg) # now that logging is setup and stdout redirected, send welcome welcome(name, msg=w_msg) # Stage 5 return run_module_section(mods, name, name)
def main_modules(action_name, args): name = args.mode # Cloud-init 'modules' stages are broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Get the datasource from the init object, if it does # not exist then that means the main_init stage never # worked, and thus this stage can not run. # 3. Construct the modules object # 4. Adjust any subsequent logging/output redirections using # the modules objects configuration # 5. Run the modules for the given stage name # 6. Done! w_msg = welcome_format("%s:%s" % (action_name, name)) init = stages.Init(ds_deps=[], reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 try: init.fetch(existing="trust") except sources.DataSourceNotFoundException: # There was no datasource found, theres nothing to do msg = ('Can not apply stage %s, no datasource found! Likely bad ' 'things to come!' % name) util.logexc(LOG, msg) print_exc(msg) if not args.force: return [(msg)] _maybe_persist_instance_data(init) # Stage 3 mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) # Stage 4 try: LOG.debug("Closing stdin") util.close_stdin() util.fixup_output(mods.cfg, name) except Exception: util.logexc(LOG, "Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" " longer be active shortly")) logging.resetLogging() logging.setupLogging(mods.cfg) apply_reporting_cfg(init.cfg) # now that logging is setup and stdout redirected, send welcome welcome(name, msg=w_msg) # Stage 5 return run_module_section(mods, name, name)
def main_single(name, args): # Cloud-init single stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Attempt to fetch the datasource (warn if it doesn't work) # 3. Construct the modules object # 4. Adjust any subsequent logging/output redirections using # the modules objects configuration # 5. Run the single module # 6. Done! mod_name = args.name w_msg = welcome_format(name) init = stages.Init(ds_deps=[], reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 try: init.fetch(existing="trust") except sources.DataSourceNotFoundException: # There was no datasource found, # that might be bad (or ok) depending on # the module being ran (so continue on) util.logexc(LOG, ("Failed to fetch your datasource," " likely bad things to come!")) print_exc(("Failed to fetch your datasource," " likely bad things to come!")) if not args.force: return 1 # Stage 3 mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) mod_args = args.module_args if mod_args: LOG.debug("Using passed in arguments %s", mod_args) mod_freq = args.frequency if mod_freq: LOG.debug("Using passed in frequency %s", mod_freq) mod_freq = FREQ_SHORT_NAMES.get(mod_freq) # Stage 4 try: LOG.debug("Closing stdin") util.close_stdin() util.fixup_output(mods.cfg, None) except Exception: util.logexc(LOG, "Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" " longer be active shortly")) logging.resetLogging() logging.setupLogging(mods.cfg) apply_reporting_cfg(init.cfg) # now that logging is setup and stdout redirected, send welcome welcome(name, msg=w_msg) # Stage 5 (which_ran, failures) = mods.run_single(mod_name, mod_args, mod_freq) if failures: LOG.warn("Ran %s but it failed!", mod_name) return 1 elif not which_ran: LOG.warn("Did not run %s, does it exist?", mod_name) return 1 else: # Guess it worked return 0
def main_init(name, args): deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] if args.local: deps = [sources.DEP_FILESYSTEM] if not args.local: # See doc/kernel-cmdline.txt # # This is used in maas datasource, in "ephemeral" (read-only root) # environment where the instance netboots to iscsi ro root. # and the entity that controls the pxe config has to configure # the maas datasource. # # Could be used elsewhere, only works on network based (not local). root_name = "%s.d" % (CLOUD_CONFIG) target_fn = os.path.join(root_name, "91_kernel_cmdline_url.cfg") util.read_write_cmdline_url(target_fn) # Cloud-init 'init' stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Setup logging/output redirections with resultant config (if any) # 3. Initialize the cloud-init filesystem # 4. Check if we can stop early by looking for various files # 5. Fetch the datasource # 6. Connect to the current instance location + update the cache # 7. Consume the userdata (handlers get activated here) # 8. Construct the modules object # 9. Adjust any subsequent logging/output redirections using the modules # objects config as it may be different from init object # 10. Run the modules for the 'init' stage # 11. Done! if not args.local: w_msg = welcome_format(name) else: w_msg = welcome_format("%s-local" % (name)) init = stages.Init(ds_deps=deps, reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 outfmt = None errfmt = None try: LOG.debug("Closing stdin") util.close_stdin() (outfmt, errfmt) = util.fixup_output(init.cfg, name) except Exception: util.logexc(LOG, "Failed to setup output redirection!") print_exc("Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" " longer be active shortly")) logging.resetLogging() logging.setupLogging(init.cfg) apply_reporting_cfg(init.cfg) # Any log usage prior to setupLogging above did not have local user log # config applied. We send the welcome message now, as stderr/out have # been redirected and log now configured. welcome(name, msg=w_msg) # Stage 3 try: init.initialize() except Exception: util.logexc(LOG, "Failed to initialize, likely bad things to come!") # Stage 4 path_helper = init.paths mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK if mode == sources.DSMODE_NETWORK: existing = "trust" sys.stderr.write("%s\n" % (netinfo.debug_info())) LOG.debug(("Checking to see if files that we need already" " exist from a previous run that would allow us" " to stop early.")) # no-net is written by upstart cloud-init-nonet when network failed # to come up stop_files = [ os.path.join(path_helper.get_cpath("data"), "no-net"), ] existing_files = [] for fn in stop_files: if os.path.isfile(fn): existing_files.append(fn) if existing_files: LOG.debug("[%s] Exiting. stop file %s existed", mode, existing_files) return (None, []) else: LOG.debug("Execution continuing, no previous run detected that" " would allow us to stop early.") else: existing = "check" if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): existing = "trust" init.purge_cache() # Delete the non-net file as well util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net")) # Stage 5 try: init.fetch(existing=existing) # if in network mode, and the datasource is local # then work was done at that stage. if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: LOG.debug("[%s] Exiting. datasource %s in local mode", mode, init.datasource) return (None, []) except sources.DataSourceNotFoundException: # In the case of 'cloud-init init' without '--local' it is a bit # more likely that the user would consider it failure if nothing was # found. When using upstart it will also mentions job failure # in console log if exit code is != 0. if mode == sources.DSMODE_LOCAL: LOG.debug("No local datasource found") else: util.logexc(LOG, ("No instance datasource found!" " Likely bad things to come!")) if not args.force: init.apply_network_config(bring_up=not args.local) LOG.debug("[%s] Exiting without datasource in local mode", mode) if mode == sources.DSMODE_LOCAL: return (None, []) else: return (None, ["No instance datasource found."]) else: LOG.debug("[%s] barreling on in force mode without datasource", mode) # Stage 6 iid = init.instancify() LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", mode, name, iid, init.is_new_instance()) init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) if mode == sources.DSMODE_LOCAL: if init.datasource.dsmode != mode: LOG.debug("[%s] Exiting. datasource %s not in local mode.", mode, init.datasource) return (init.datasource, []) else: LOG.debug("[%s] %s is in local mode, will apply init modules now.", mode, init.datasource) # update fully realizes user-data (pulling in #include if necessary) init.update() # Stage 7 try: # Attempt to consume the data per instance. # This may run user-data handlers and/or perform # url downloads and such as needed. (ran, _results) = init.cloudify().run('consume_data', init.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) if not ran: # Just consume anything that is set to run per-always # if nothing ran in the per-instance code # # See: https://bugs.launchpad.net/bugs/819507 for a little # reason behind this... init.consume_data(PER_ALWAYS) except Exception: util.logexc(LOG, "Consuming user data failed!") return (init.datasource, ["Consuming user data failed!"]) apply_reporting_cfg(init.cfg) # Stage 8 - re-read and apply relevant cloud-config to include user-data mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) # Stage 9 try: outfmt_orig = outfmt errfmt_orig = errfmt (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) if outfmt_orig != outfmt or errfmt_orig != errfmt: LOG.warn("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) (outfmt, errfmt) = util.fixup_output(mods.cfg, name) except Exception: util.logexc(LOG, "Failed to re-adjust output redirection!") logging.setupLogging(mods.cfg) # Stage 10 return (init.datasource, run_module_section(mods, name, name))
def main_init(name, args): deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] if args.local: deps = [sources.DEP_FILESYSTEM] early_logs = [ attempt_cmdline_url( path=os.path.join( "%s.d" % CLOUD_CONFIG, "91_kernel_cmdline_url.cfg" ), network=not args.local, ) ] # Cloud-init 'init' stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Setup logging/output redirections with resultant config (if any) # 3. Initialize the cloud-init filesystem # 4. Check if we can stop early by looking for various files # 5. Fetch the datasource # 6. Connect to the current instance location + update the cache # 7. Consume the userdata (handlers get activated here) # 8. Construct the modules object # 9. Adjust any subsequent logging/output redirections using the modules # objects config as it may be different from init object # 10. Run the modules for the 'init' stage # 11. Done! if not args.local: w_msg = welcome_format(name) else: w_msg = welcome_format("%s-local" % (name)) init = stages.Init(ds_deps=deps, reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 outfmt = None errfmt = None try: early_logs.append((logging.DEBUG, "Closing stdin.")) util.close_stdin() (outfmt, errfmt) = util.fixup_output(init.cfg, name) except Exception: msg = "Failed to setup output redirection!" util.logexc(LOG, msg) print_exc(msg) early_logs.append((logging.WARN, msg)) if args.debug: # Reset so that all the debug handlers are closed out LOG.debug( "Logging being reset, this logger may no longer be active shortly" ) logging.resetLogging() logging.setupLogging(init.cfg) apply_reporting_cfg(init.cfg) # Any log usage prior to setupLogging above did not have local user log # config applied. We send the welcome message now, as stderr/out have # been redirected and log now configured. welcome(name, msg=w_msg) # re-play early log messages before logging was setup for lvl, msg in early_logs: LOG.log(lvl, msg) # Stage 3 try: init.initialize() except Exception: util.logexc(LOG, "Failed to initialize, likely bad things to come!") # Stage 4 path_helper = init.paths purge_cache_on_python_version_change(init) mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK if mode == sources.DSMODE_NETWORK: existing = "trust" sys.stderr.write("%s\n" % (netinfo.debug_info())) else: existing = "check" mcfg = util.get_cfg_option_bool(init.cfg, "manual_cache_clean", False) if mcfg: LOG.debug("manual cache clean set from config") existing = "trust" else: mfile = path_helper.get_ipath_cur("manual_clean_marker") if os.path.exists(mfile): LOG.debug("manual cache clean found from marker: %s", mfile) existing = "trust" init.purge_cache() # Stage 5 bring_up_interfaces = _should_bring_up_interfaces(init, args) try: init.fetch(existing=existing) # if in network mode, and the datasource is local # then work was done at that stage. if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: LOG.debug( "[%s] Exiting. datasource %s in local mode", mode, init.datasource, ) return (None, []) except sources.DataSourceNotFoundException: # In the case of 'cloud-init init' without '--local' it is a bit # more likely that the user would consider it failure if nothing was # found. if mode == sources.DSMODE_LOCAL: LOG.debug("No local datasource found") else: util.logexc( LOG, "No instance datasource found! Likely bad things to come!" ) if not args.force: init.apply_network_config(bring_up=bring_up_interfaces) LOG.debug("[%s] Exiting without datasource", mode) if mode == sources.DSMODE_LOCAL: return (None, []) else: return (None, ["No instance datasource found."]) else: LOG.debug( "[%s] barreling on in force mode without datasource", mode ) _maybe_persist_instance_data(init) # Stage 6 iid = init.instancify() LOG.debug( "[%s] %s will now be targeting instance id: %s. new=%s", mode, name, iid, init.is_new_instance(), ) if mode == sources.DSMODE_LOCAL: # Before network comes up, set any configured hostname to allow # dhcp clients to advertize this hostname to any DDNS services # LP: #1746455. _maybe_set_hostname(init, stage="local", retry_stage="network") init.apply_network_config(bring_up=bring_up_interfaces) if mode == sources.DSMODE_LOCAL: if init.datasource.dsmode != mode: LOG.debug( "[%s] Exiting. datasource %s not in local mode.", mode, init.datasource, ) return (init.datasource, []) else: LOG.debug( "[%s] %s is in local mode, will apply init modules now.", mode, init.datasource, ) # Give the datasource a chance to use network resources. # This is used on Azure to communicate with the fabric over network. init.setup_datasource() # update fully realizes user-data (pulling in #include if necessary) init.update() _maybe_set_hostname(init, stage="init-net", retry_stage="modules:config") # Stage 7 try: # Attempt to consume the data per instance. # This may run user-data handlers and/or perform # url downloads and such as needed. (ran, _results) = init.cloudify().run( "consume_data", init.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) if not ran: # Just consume anything that is set to run per-always # if nothing ran in the per-instance code # # See: https://bugs.launchpad.net/bugs/819507 for a little # reason behind this... init.consume_data(PER_ALWAYS) except Exception: util.logexc(LOG, "Consuming user data failed!") return (init.datasource, ["Consuming user data failed!"]) # Validate user-data adheres to schema definition if os.path.exists(init.paths.get_ipath_cur("userdata_raw")): validate_cloudconfig_schema( config=init.cfg, strict=False, log_details=False ) else: LOG.debug("Skipping user-data validation. No user-data found.") apply_reporting_cfg(init.cfg) # Stage 8 - re-read and apply relevant cloud-config to include user-data mods = Modules(init, extract_fns(args), reporter=args.reporter) # Stage 9 try: outfmt_orig = outfmt errfmt_orig = errfmt (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) if outfmt_orig != outfmt or errfmt_orig != errfmt: LOG.warning("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) (outfmt, errfmt) = util.fixup_output(mods.cfg, name) except Exception: util.logexc(LOG, "Failed to re-adjust output redirection!") logging.setupLogging(mods.cfg) # give the activated datasource a chance to adjust init.activate_datasource() di_report_warn(datasource=init.datasource, cfg=init.cfg) # Stage 10 return (init.datasource, run_module_section(mods, name, name))