def dumps(instance: object, out: str): conf = __generate(instance, "") if out: if out.lower() == "hocon": return HOCONConverter.to_hocon(conf) if out.lower() == "yaml": return HOCONConverter.to_yaml(conf) if out.lower() == "json": return HOCONConverter.to_json(conf) if out.lower() == "properties": return HOCONConverter.to_properties(conf) return conf
def to_dict(self, with_include=True): """Convert HOCON string into dict. Args: with_include: If True then double-quote-escaped `include` statements will be kept as a plain string under key HOCONSTRING_INCLUDE_KEY. Otherwise, `include` statements will be excluded. """ if with_include: hocon_str = self._hocon_str else: hocon_str = self.get_contents(with_include=False) c = ConfigFactory.parse_string(hocon_str) j = HOCONConverter.to_json(c) return json.loads(j)
def load_config(cwd=os.getcwd(), debug=False): """ Tries to find HOCON files named "iss4e.conf" using the paths returned by find_files(). The found files are then parsed and merged together, so that a single configuration dict is returned. For details on HOCON syntax, see https://github.com/chimpler/pyhocon and https://github.com/typesafehub/config/ Example configuration: - default config in home dir (~/iss4e.conf): datasources { influx { host = ${HOSTNAME} # also set your passwords (e.g. from env with ${MYSQL_PASSWD} here } mysql { host = localhost } } - local config in cwd (./iss4e.conf): webike { # use the generic information from ${datasources.influx} (should be defined in ~/iss4e.conf and contain # host, password, ...) and extend it to use the (non-generic) database "webike" influx = ${datasources.influx} { db = "webike" } } - merged config that will be returned: { "datasources": { "influx": { "host": "SD959-LT" }, "mysql": { "host": "localhost" } }, "webike": { "influx": { "host": "SD959-LT", # copied from ~/iss4e.conf: datasources.influx "db": "webike" } } } """ # find "iss4e.conf" file in current working dir or parent directories files = find_files("iss4e.conf", cwd) configs = [ ConfigFactory.parse_file(file, required=False, resolve=False) for file in files if os.path.isfile(file) ] if debug: print("Config files:\n" + "\n".join(file + " [" + ("not " if not os.path.isfile(file) else "") + "found]" for file in files)) # merge all levels of config config = ConfigTree(root=True) config.put( "__main__", os.path.basename( getattr(sys.modules['__main__'], "__file__", "__cli__"))) config.put("__cwd__", os.path.abspath(cwd)) for c in configs: config = ConfigTree.merge_configs(c, config) config = ConfigParser.resolve_substitutions(config) if debug: print("Loaded config:\n" + HOCONConverter.to_json(config)) # if config contains a key "logging", use it to reconfigure python logging if "logging" in config: if debug: print("Reconfiguring logging from config") if config.get("capture_exceptions", True): sys.excepthook = log_uncaught_exception logging.captureWarnings(config.get("capture_warnings", True)) logging.config.dictConfig(config["logging"].as_plain_ordered_dict()) # check python version # iss4e lib is using some syntax features and functions which were only introduced in python 3.5 rec_ver = tuple(config.get("min_py_version", [3, 5])) if sys.version_info < rec_ver: warnings.warn( "Using outdated python version {}, a version >= {} would be recommended for use with iss4e lib. " "Try using a newer python binary, e.g. by calling `python{}.{}` instead of the default `python`." .format(sys.version_info, rec_ver, rec_ver[0], rec_ver[1])) return config
def to_dict(self): """Convert contents without include to dict. """ c = ConfigFactory.parse_string(self._contents_wo_include) j = HOCONConverter.to_json(c) return json.loads(j)
def dumps(obj: Any, **kwargs) -> str: return HOCONConverter.to_json(obj, **kwargs)
def parse_string(self, s): conf = ConfigFactory.parse_string(s) return json.loads(HOCONConverter.to_json(conf))
def parse_fp(self, fp): conf = ConfigFactory.parse_file(fp) return json.loads(HOCONConverter.to_json(conf))
def save_config(args: Args, cfg: ConfigTree): config_path = args.run_dir / 'config.json' with open(config_path, 'w') as f: f.write(HOCONConverter.to_json(cfg))
from pyhocon import ConfigFactory, HOCONConverter conf = ConfigFactory.parse_file('samples/animal.conf') print("cat says :", conf.get('cat.garfield.say')) print("dog says :", conf.get('dog.mutt.say')) ## dog.conf has include "cat.conf" and both have key garfield.say. ## The value from cat (meow) will override that of dog (bobooooo) print("After merge :", conf.get('dog.mutt.hates.garfield.say')) with open('samples/expanded_animal.json', "w") as output_json: output_json.writelines(HOCONConverter.to_json(conf)) # we can also use json.dumps # fd.write(json.dumps(confTree,indent=4))
def __get_backend_conf_str(self): """ Initializes the following backend stanzas, which are defined in "backend" {} in a Cromwell's backend configuration file: 1) local: local backend 2) gc: Google Cloud backend (optional) 3) aws: AWS backend (optional) 4) slurm: SLURM (optional) 5) sge: SGE (optional) 6) pbs: PBS (optional) Also, initializes the following common non-"backend" stanzas: a) common: base stanzas b) mysql: connect to MySQL (optional) Then converts it to a HOCON string """ # init backend dict backend_dict = {} # common stanza for backend conf file merge_dict( backend_dict, CaperBackendCommon( port=self._port, disable_call_caching=self._disable_call_caching, max_concurrent_workflows=self._max_concurrent_workflows)) # local backend merge_dict( backend_dict, CaperBackendLocal(out_dir=self._out_dir, concurrent_job_limit=self._max_concurrent_tasks)) # GC if self._gcp_prj is not None and self._out_gcs_bucket is not None: merge_dict( backend_dict, CaperBackendGCP( gcp_prj=self._gcp_prj, out_gcs_bucket=self._out_gcs_bucket, concurrent_job_limit=self._max_concurrent_tasks)) # AWS if self._aws_batch_arn is not None and self._aws_region is not None \ and self._out_s3_bucket is not None: merge_dict( backend_dict, CaperBackendAWS( aws_batch_arn=self._aws_batch_arn, aws_region=self._aws_region, out_s3_bucket=self._out_s3_bucket, concurrent_job_limit=self._max_concurrent_tasks)) # SLURM merge_dict( backend_dict, CaperBackendSLURM(out_dir=self._out_dir, partition=self._slurm_partition, account=self._slurm_account, extra_param=self._slurm_extra_param, concurrent_job_limit=self._max_concurrent_tasks)) # SGE merge_dict( backend_dict, CaperBackendSGE(out_dir=self._out_dir, pe=self._sge_pe, queue=self._sge_queue, extra_param=self._sge_extra_param, concurrent_job_limit=self._max_concurrent_tasks)) # PBS merge_dict( backend_dict, CaperBackendPBS(out_dir=self._out_dir, queue=self._pbs_queue, extra_param=self._pbs_extra_param, concurrent_job_limit=self._max_concurrent_tasks)) # Database if self._no_file_db is not None and self._no_file_db: file_db = None else: file_db = self._file_db merge_dict( backend_dict, CaperBackendDatabase(file_db=file_db, mysql_ip=self._mysql_db_ip, mysql_port=self._mysql_db_port, mysql_user=self._mysql_db_user, mysql_password=self._mysql_db_password, db_timeout=self._db_timeout)) # set header for conf ("include ...") assert (Caper.BACKEND_CONF_HEADER.endswith('\n')) lines_header = [Caper.BACKEND_CONF_HEADER] # override with user-specified backend.conf if exists if self._backend_file is not None: lines_wo_header = [] with open(CaperURI(self._backend_file).get_local_file(), 'r') as fp: for line in fp.readlines(): # find header and exclude if re.findall(Caper.RE_PATTERN_BACKEND_CONF_HEADER, line): if line not in lines_header: lines_header.append(line) else: lines_wo_header.append(line) # parse HOCON to JSON to dict c = ConfigFactory.parse_string(''.join(lines_wo_header)) j = HOCONConverter.to_json(c) d = json.loads(j) # apply to backend conf merge_dict(backend_dict, d) # use default backend (local) if not specified if self._backend is not None: backend_dict['backend']['default'] = self._backend else: backend_dict['backend']['default'] = Caper.DEFAULT_BACKEND # dict to HOCON (excluding header) backend_hocon = ConfigFactory.from_dict(backend_dict) # write header to HOCON string backend_str = ''.join(lines_header) # convert HOCON to string backend_str += HOCONConverter.to_hocon(backend_hocon) return backend_str
def loads(s: AnyStr, **kwargs: Any) -> Dict[str, Any]: hocon_obj = ConfigFactory.parse_string(s, **kwargs) return json.loads(HOCONConverter.to_json(hocon_obj))