def validate_cloudconfig_file(config_path, schema, annotate=False): """Validate cloudconfig file adheres to a specific jsonschema. @param config_path: Path to the yaml cloud-config file to parse, or None to default to system userdata from Paths object. @param schema: Dict describing a valid jsonschema to validate against. @param annotate: Boolean set True to print original config file with error annotations on the offending lines. @raises SchemaValidationError containing any of schema_errors encountered. @raises RuntimeError when config_path does not exist. """ if config_path is None: # Use system's raw userdata path if os.getuid() != 0: raise RuntimeError( "Unable to read system userdata as non-root user." " Try using sudo") paths = read_cfg_paths() user_data_file = paths.get_ipath_cur("userdata_raw") content = load_file(user_data_file, decode=False) else: if not os.path.exists(config_path): raise RuntimeError( 'Configfile {0} does not exist'.format(config_path)) content = load_file(config_path, decode=False) if not content.startswith(CLOUD_CONFIG_HEADER): errors = (('format-l1.c1', 'File {0} needs to begin with "{1}"'.format( config_path, CLOUD_CONFIG_HEADER.decode())), ) error = SchemaValidationError(errors) if annotate: print(annotated_cloudconfig_file({}, content, error.schema_errors)) raise error try: cloudconfig = yaml.safe_load(content) except (yaml.YAMLError) as e: line = column = 1 mark = None if hasattr(e, 'context_mark') and getattr(e, 'context_mark'): mark = getattr(e, 'context_mark') elif hasattr(e, 'problem_mark') and getattr(e, 'problem_mark'): mark = getattr(e, 'problem_mark') if mark: line = mark.line + 1 column = mark.column + 1 errors = (('format-l{line}.c{col}'.format(line=line, col=column), 'File {0} is not valid yaml. {1}'.format( config_path, str(e))), ) error = SchemaValidationError(errors) if annotate: print(annotated_cloudconfig_file({}, content, error.schema_errors)) raise error from e try: validate_cloudconfig_schema(cloudconfig, schema, strict=True) except SchemaValidationError as e: if annotate: print( annotated_cloudconfig_file(cloudconfig, content, e.schema_errors)) raise
def _read_instance_data(instance_data, user_data, vendor_data) -> dict: """Return a dict of merged instance-data, vendordata and userdata. The dict will contain supplemental userdata and vendordata keys sourced from default user-data and vendor-data files. Non-root users will have redacted INSTANCE_JSON_FILE content and redacted vendordata and userdata values. :raise: IOError/OSError on absence of instance-data.json file or invalid access perms. """ paths = None uid = os.getuid() if not all([instance_data, user_data, vendor_data]): paths = read_cfg_paths() if instance_data: instance_data_fn = instance_data else: redacted_data_fn = os.path.join(paths.run_dir, INSTANCE_JSON_FILE) if uid == 0: sensitive_data_fn = os.path.join(paths.run_dir, INSTANCE_JSON_SENSITIVE_FILE) if os.path.exists(sensitive_data_fn): instance_data_fn = sensitive_data_fn else: LOG.warning( 'Missing root-readable %s. Using redacted %s instead.', sensitive_data_fn, redacted_data_fn) instance_data_fn = redacted_data_fn else: instance_data_fn = redacted_data_fn if user_data: user_data_fn = user_data else: user_data_fn = os.path.join(paths.instance_link, 'user-data.txt') if vendor_data: vendor_data_fn = vendor_data else: vendor_data_fn = os.path.join(paths.instance_link, 'vendor-data.txt') try: instance_json = util.load_file(instance_data_fn) except (IOError, OSError) as e: if e.errno == EACCES: LOG.error("No read permission on '%s'. Try sudo", instance_data_fn) else: LOG.error('Missing instance-data file: %s', instance_data_fn) raise instance_data = util.load_json(instance_json) if uid != 0: instance_data['userdata'] = ('<%s> file:%s' % (REDACT_SENSITIVE_VALUE, user_data_fn)) instance_data['vendordata'] = ( '<%s> file:%s' % (REDACT_SENSITIVE_VALUE, vendor_data_fn)) else: instance_data['userdata'] = load_userdata(user_data_fn) instance_data['vendordata'] = load_userdata(vendor_data_fn) return instance_data
def get_status_details(paths=None) -> Tuple[UXAppStatus, str, str]: """Return a 3-tuple of status, status_details and time of last event. @param paths: An initialized cloudinit.helpers.paths object. Values are obtained from parsing paths.run_dir/status.json. """ paths = paths or read_cfg_paths() status = UXAppStatus.NOT_RUN status_detail = "" status_v1 = {} status_file = os.path.join(paths.run_dir, "status.json") result_file = os.path.join(paths.run_dir, "result.json") (is_disabled, reason) = _is_cloudinit_disabled( CLOUDINIT_DISABLED_FILE, paths ) if is_disabled: status = UXAppStatus.DISABLED status_detail = reason if os.path.exists(status_file): if not os.path.exists(result_file): status = UXAppStatus.RUNNING status_v1 = load_json(load_file(status_file)).get("v1", {}) errors = [] latest_event = 0 for key, value in sorted(status_v1.items()): if key == "stage": if value: status = UXAppStatus.RUNNING status_detail = "Running in stage: {0}".format(value) elif key == "datasource": status_detail = value elif isinstance(value, dict): errors.extend(value.get("errors", [])) start = value.get("start") or 0 finished = value.get("finished") or 0 if finished == 0 and start != 0: status = UXAppStatus.RUNNING event_time = max(start, finished) if event_time > latest_event: latest_event = event_time if errors: status = UXAppStatus.ERROR status_detail = "\n".join(errors) elif status == UXAppStatus.NOT_RUN and latest_event > 0: status = UXAppStatus.DONE if latest_event: time = strftime("%a, %d %b %Y %H:%M:%S %z", gmtime(latest_event)) else: time = "" return status, status_detail, time
def handle_status_args(name, args) -> int: """Handle calls to 'cloud-init status' as a subcommand.""" # Read configured paths paths = read_cfg_paths() status, status_detail, time = get_status_details(paths) if args.wait: while status in (UXAppStatus.NOT_RUN, UXAppStatus.RUNNING): sys.stdout.write(".") sys.stdout.flush() status, status_detail, time = get_status_details(paths) sleep(0.25) sys.stdout.write("\n") print("status: {0}".format(status.value)) if args.long: if time: print("time: {0}".format(time)) print("detail:\n{0}".format(status_detail)) return 1 if status == UXAppStatus.ERROR else 0
def handle_args(name, args): """Handle calls to 'cloud-init query' as a subcommand.""" paths = None addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING) if not any([args.list_keys, args.varname, args.format, args.dump_all]): LOG.error('Expected one of the options: --all, --format,' ' --list-keys or varname') get_parser().print_help() return 1 uid = os.getuid() if not all([args.instance_data, args.user_data, args.vendor_data]): paths = read_cfg_paths() if args.instance_data: instance_data_fn = args.instance_data else: redacted_data_fn = os.path.join(paths.run_dir, INSTANCE_JSON_FILE) if uid == 0: sensitive_data_fn = os.path.join(paths.run_dir, INSTANCE_JSON_SENSITIVE_FILE) if os.path.exists(sensitive_data_fn): instance_data_fn = sensitive_data_fn else: LOG.warning( 'Missing root-readable %s. Using redacted %s instead.', sensitive_data_fn, redacted_data_fn) instance_data_fn = redacted_data_fn else: instance_data_fn = redacted_data_fn if args.user_data: user_data_fn = args.user_data else: user_data_fn = os.path.join(paths.instance_link, 'user-data.txt') if args.vendor_data: vendor_data_fn = args.vendor_data else: vendor_data_fn = os.path.join(paths.instance_link, 'vendor-data.txt') try: instance_json = util.load_file(instance_data_fn) except (IOError, OSError) as e: if e.errno == EACCES: LOG.error("No read permission on '%s'. Try sudo", instance_data_fn) else: LOG.error('Missing instance-data file: %s', instance_data_fn) return 1 instance_data = util.load_json(instance_json) if uid != 0: instance_data['userdata'] = ('<%s> file:%s' % (REDACT_SENSITIVE_VALUE, user_data_fn)) instance_data['vendordata'] = ( '<%s> file:%s' % (REDACT_SENSITIVE_VALUE, vendor_data_fn)) else: instance_data['userdata'] = load_userdata(user_data_fn) instance_data['vendordata'] = load_userdata(vendor_data_fn) if args.format: payload = '## template: jinja\n{fmt}'.format(fmt=args.format) rendered_payload = render_jinja_payload( payload=payload, payload_fn='query commandline', instance_data=instance_data, debug=True if args.debug else False) if rendered_payload: print(rendered_payload) return 0 return 1 response = convert_jinja_instance_data(instance_data) if args.varname: try: for var in args.varname.split('.'): response = response[var] except KeyError: LOG.error('Undefined instance-data key %s', args.varname) return 1 if args.list_keys: if not isinstance(response, dict): LOG.error("--list-keys provided but '%s' is not a dict", var) return 1 response = '\n'.join(sorted(response.keys())) elif args.list_keys: response = '\n'.join(sorted(response.keys())) if not isinstance(response, str): response = util.json_dumps(response) print(response) return 0
def do_test_frequency(self, frequency): ci_paths = read_cfg_paths() scripts_dir = ci_paths.get_cpath("scripts") testFolder = os.path.join(scripts_dir, path_map[frequency]) folder = get_script_folder_by_frequency(frequency, scripts_dir) assert testFolder == folder
def _get_user_data_file() -> str: paths = read_cfg_paths() return paths.get_ipath_cur("userdata_raw")
def handle_args(name, args): """Handle calls to 'cloud-init query' as a subcommand.""" paths = None addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING) if not any([args.list_keys, args.varname, args.format, args.dump_all]): LOG.error( 'Expected one of the options: --all, --format,' ' --list-keys or varname') get_parser().print_help() return 1 uid = os.getuid() if not all([args.instance_data, args.user_data, args.vendor_data]): paths = read_cfg_paths() if not args.instance_data: if uid == 0: default_json_fn = INSTANCE_JSON_SENSITIVE_FILE else: default_json_fn = INSTANCE_JSON_FILE # World readable instance_data_fn = os.path.join(paths.run_dir, default_json_fn) else: instance_data_fn = args.instance_data if not args.user_data: user_data_fn = os.path.join(paths.instance_link, 'user-data.txt') else: user_data_fn = args.user_data if not args.vendor_data: vendor_data_fn = os.path.join(paths.instance_link, 'vendor-data.txt') else: vendor_data_fn = args.vendor_data try: instance_json = util.load_file(instance_data_fn) except IOError: LOG.error('Missing instance-data.json file: %s', instance_data_fn) return 1 instance_data = util.load_json(instance_json) if uid != 0: instance_data['userdata'] = ( '<%s> file:%s' % (REDACT_SENSITIVE_VALUE, user_data_fn)) instance_data['vendordata'] = ( '<%s> file:%s' % (REDACT_SENSITIVE_VALUE, vendor_data_fn)) else: instance_data['userdata'] = util.load_file(user_data_fn) instance_data['vendordata'] = util.load_file(vendor_data_fn) if args.format: payload = '## template: jinja\n{fmt}'.format(fmt=args.format) rendered_payload = render_jinja_payload( payload=payload, payload_fn='query commandline', instance_data=instance_data, debug=True if args.debug else False) if rendered_payload: print(rendered_payload) return 0 return 1 response = convert_jinja_instance_data(instance_data) if args.varname: try: for var in args.varname.split('.'): response = response[var] except KeyError: LOG.error('Undefined instance-data key %s', args.varname) return 1 if args.list_keys: if not isinstance(response, dict): LOG.error("--list-keys provided but '%s' is not a dict", var) return 1 response = '\n'.join(sorted(response.keys())) elif args.list_keys: response = '\n'.join(sorted(response.keys())) if not isinstance(response, six.string_types): response = util.json_dumps(response) print(response) return 0
def status_wrapper(name, args, data_d=None, link_d=None): if data_d is None: paths = read_cfg_paths() data_d = paths.get_cpath("data") if link_d is None: link_d = os.path.normpath("/run/cloud-init") status_path = os.path.join(data_d, "status.json") status_link = os.path.join(link_d, "status.json") result_path = os.path.join(data_d, "result.json") result_link = os.path.join(link_d, "result.json") util.ensure_dirs(( data_d, link_d, )) (_name, functor) = args.action if name == "init": if args.local: mode = "init-local" else: mode = "init" elif name == "modules": mode = "modules-%s" % args.mode else: raise ValueError("unknown name: %s" % name) modes = ( "init", "init-local", "modules-init", "modules-config", "modules-final", ) if mode not in modes: raise ValueError( "Invalid cloud init mode specified '{0}'".format(mode)) status = None if mode == "init-local": for f in (status_link, result_link, status_path, result_path): util.del_file(f) else: try: status = json.loads(util.load_file(status_path)) except Exception: pass nullstatus = { "errors": [], "start": None, "finished": None, } if status is None: status = {"v1": {}} status["v1"]["datasource"] = None for m in modes: if m not in status["v1"]: status["v1"][m] = nullstatus.copy() v1 = status["v1"] v1["stage"] = mode v1[mode]["start"] = time.time() atomic_helper.write_json(status_path, status) util.sym_link(os.path.relpath(status_path, link_d), status_link, force=True) try: ret = functor(name, args) if mode in ("init", "init-local"): (datasource, errors) = ret if datasource is not None: v1["datasource"] = str(datasource) else: errors = ret v1[mode]["errors"] = [str(e) for e in errors] except Exception as e: util.logexc(LOG, "failed stage %s", mode) print_exc("failed run of stage %s" % mode) v1[mode]["errors"] = [str(e)] v1[mode]["finished"] = time.time() v1["stage"] = None atomic_helper.write_json(status_path, status) if mode == "modules-final": # write the 'finished' file errors = [] for m in modes: if v1[m]["errors"]: errors.extend(v1[m].get("errors", [])) atomic_helper.write_json( result_path, {"v1": { "datasource": v1["datasource"], "errors": errors }}, ) util.sym_link(os.path.relpath(result_path, link_d), result_link, force=True) return len(v1[mode]["errors"])