def config_args(modargs): cliargs = modargs.get("__cli_options", {}) kwargs = modargs.copy() config_file = modargs.get('conf', dexy.utils.defaults['config_file']) # Update from config file if file_exists(config_file): with open(config_file, "r") as f: if config_file.endswith(".conf"): try: conf_args = parse_yaml(f.read()) except dexy.exceptions.UserFeedback as yaml_exception: try: conf_args = parse_json(f.read()) except dexy.exceptions.UserFeedback as json_exception: print("--------------------------------------------------") print("Tried to parse YAML:") print(yaml_exception) print("--------------------------------------------------") print("Tried to parse JSON:") print(json_exception) print("--------------------------------------------------") raise dexy.exceptions.UserFeedback("Unable to parse config file '%s' as YAML or as JSON." % config_file) elif config_file.endswith(".yaml"): conf_args = parse_yaml(f.read()) elif config_file.endswith(".json"): conf_args = parse_json(f.read()) else: raise dexy.exceptions.UserFeedback("Don't know how to load config from '%s'" % config_file) if conf_args: # TODO raise error if kwargs.update(conf_args) if cliargs: # cliargs may be False for k in list(cliargs.keys()): try: kwargs[k] = modargs[k] except KeyError: msg = "This command does not take a '--%s' argument." % k raise dexy.exceptions.UserFeedback(msg) # TODO allow updating from env variables, e.g. DEXY_ARTIFACTS_DIR return kwargs
def config_args(modargs): cliargs = modargs.get("__cli_options", {}) kwargs = modargs.copy() config_file = modargs.get('conf', dexy.utils.defaults['config_file']) # Update from config file if file_exists(config_file): with open(config_file, "rb") as f: if config_file.endswith(".conf"): try: conf_args = parse_yaml(f.read()) except dexy.exceptions.UserFeedback as yaml_exception: try: conf_args = parse_json(f.read()) except dexy.exceptions.UserFeedback as json_exception: print "--------------------------------------------------" print "Tried to parse YAML:" print yaml_exception print "--------------------------------------------------" print "Tried to parse JSON:" print json_exception print "--------------------------------------------------" raise dexy.exceptions.UserFeedback("Unable to parse config file '%s' as YAML or as JSON." % config_file) elif config_file.endswith(".yaml"): conf_args = parse_yaml(f.read()) elif config_file.endswith(".json"): conf_args = parse_json(f.read()) else: raise dexy.exceptions.UserFeedback("Don't know how to load config from '%s'" % config_file) if conf_args: kwargs.update(conf_args) if cliargs: # cliargs may be False for k in cliargs.keys(): try: kwargs[k] = modargs[k] except KeyError: msg = "This command does not take a '--%s' argument." % k raise dexy.exceptions.UserFeedback(msg) # TODO allow updating from env variables, e.g. DEXY_ARTIFACTS_DIR return kwargs
def process_text(self, input_text): regex = "\r?\n---\r?\n" if re.search(regex, input_text): raw_yamlargs, content = re.split(regex, input_text) yamlargs = parse_yaml(raw_yamlargs) self.add_runtime_args(yamlargs) return content else: return input_text
def config_args(modargs): cliargs = modargs.get("__cli_options", {}) kwargs = modargs.copy() config_file = modargs.get('conf', dexy.utils.defaults['config_file']) # Update from config file if file_exists(config_file): with open(config_file, "rb") as f: if config_file.endswith(".conf"): try: conf_args = parse_yaml(f.read()) except dexy.exceptions.UserFeedback as yaml_exception: try: conf_args = parse_json(f.read()) except dexy.exceptions.UserFeedback as json_exception: print "--------------------------------------------------" print "Tried to parse YAML:" print yaml_exception print "--------------------------------------------------" print "Tried to parse JSON:" print json_exception print "--------------------------------------------------" raise dexy.exceptions.UserFeedback( "Unable to parse config file '%s' as YAML or as JSON." % config_file) elif config_file.endswith(".yaml"): conf_args = parse_yaml(f.read()) elif config_file.endswith(".json"): conf_args = parse_json(f.read()) else: raise dexy.exceptions.UserFeedback( "Don't know how to load config from '%s'" % config_file) if conf_args: kwargs.update(conf_args) if cliargs: # cliargs may be False for k in cliargs.keys(): kwargs[k] = modargs[k] # TODO allow updating from env variables, e.g. DEXY_ARTIFACTS_DIR return kwargs
def process_text(self, input_text): regex = "\r?\n---\r?\n" if re.search(regex, input_text): self.log_debug("Found yaml content.") raw_yamlargs, content = re.split(regex, input_text) yamlargs = parse_yaml(raw_yamlargs) self.log_debug("Adding yaml: %s" % yamlargs) self.add_runtime_args(yamlargs) return content else: self.log_debug("No yaml content found.") return input_text
def parse(self, directory, input_text): def parse_key_mapping(mapping): for original_node_key, v in mapping.iteritems(): # handle things which aren't nodes if original_node_key == 'defaults': self.ast.default_args_for_directory.append((directory, v,)) continue # handle nodes original_file = original_node_key.split("|")[0] orig_exists = self.file_exists(directory, original_file) star_in_key = "*" in original_node_key dot_in_key = "." in original_node_key pipe_in_key = "|" in original_node_key treat_key_as_bundle_name = not orig_exists and not star_in_key and not dot_in_key and not pipe_in_key if treat_key_as_bundle_name: node_key = original_node_key else: node_key = self.wrapper.join_dir(directory, original_node_key) # v is a sequence whose members may be children or kwargs if not v: raise dexy.exceptions.UserFeedback("Empty doc config for %s" % node_key) if hasattr(v, 'keys'): raise dexy.exceptions.UserFeedback("You passed a dict to %s, please pass a sequence" % node_key) siblings = [] for element in v: if hasattr(element, 'keys'): # This is a dict of length 1 kk = element.keys()[0] vv = element[kk] if isinstance(vv, list): # This is a sequence. It probably represents a # child task but if starts with 'args' or if it # matches a filter alias for the parent doc, then # it is nested complex kwargs. if kk == "args" or (kk in node_key.split("|")): # nested complex kwargs for vvv in vv: self.ast.add_node(node_key, **vvv) else: # child task. we note the dependency, add # dependencies on prior siblings, and recurse # to process the child. self.ast.add_dependency(node_key, self.wrapper.join_dir(directory, kk)) if self.wrapper.siblings: for s in siblings: self.ast.add_dependency(self.wrapper.join_dir(directory, kk), s) siblings.append(self.wrapper.join_dir(directory, kk)) parse_key_mapping(element) else: # This is a key:value argument for this task self.ast.add_node(node_key, **element) else: # This is a child task with no args, we only have to # note the dependencies self.ast.add_dependency(node_key, self.wrapper.join_dir(directory, element)) if self.wrapper.siblings: for s in siblings: self.ast.add_dependency(self.wrapper.join_dir(directory, element), s) siblings.append(self.wrapper.join_dir(directory, element)) def parse_keys(data, top=False): if hasattr(data, 'keys'): parse_key_mapping(data) elif isinstance(data, basestring): self.ast.add_node(self.wrapper.join_dir(directory, data)) elif isinstance(data, list): if top: self.ast.root_nodes_ordered = True for element in data: parse_keys(element) else: raise Exception("invalid input %s" % data) config = parse_yaml(input_text) parse_keys(config, top=True)