def dumps(instance: object, out: str): conf = __generate(instance, "") if out: if out.lower() == "hocon": return HOCONConverter.to_hocon(conf) if out.lower() == "yaml": return HOCONConverter.to_yaml(conf) if out.lower() == "json": return HOCONConverter.to_json(conf) if out.lower() == "properties": return HOCONConverter.to_properties(conf) return conf
def _config_dict_to_text(config): if not isinstance(config, dict): raise ValueError("Model configuration only supports dictionary objects") try: # hack, pyhocon is not very good with dict conversion so we pass through json try: import json text = json.dumps(config) text = HOCONConverter.convert(ConfigFactory.parse_string(text), 'hocon') except Exception: # fallback pyhocon text = HOCONConverter.convert(ConfigFactory.from_dict(config), 'hocon') except Exception: raise ValueError("Could not serialize configuration dictionary:\n", config) return text
def single_run(policy, trace, size=4, changes={}, name=None, save=True, reuse=False, verbose=False, readonly=False): name = name if name else policy policy = Policy[policy] trace = Trace[trace] if 0 < size < 9: size = trace.typical_caches()[size - 1] conf_path = caffeine_root + 'simulator{0}src{0}main{0}resources{0}'.format( os.sep) conf_file = conf_path + 'application.conf' if not os.path.exists(output_csvs_path): os.makedirs(output_csvs_path) run_simulator = './gradlew simulator:run -x caffeine:compileJava -x caffeine:compileCodeGenJava' # run_simulator = './gradlew simulator:run' if os.path.exists(conf_file): conf = ConfigFactory.parse_file(conf_file) else: conf = ConfigFactory.parse_string(""" caffeine { simulator { } } """) simulator = conf['caffeine']['simulator'] simulator.put('files.paths', [resources_path + trace.format() + os.sep + trace.file()]) simulator.put('files.format', trace.value['format']) simulator.put('maximum-size', size) simulator.put('policies', [policy.value]) simulator.put('admission', [Admission.ALWAYS.value]) simulator.put('report.format', 'csv') simulator.put( 'report.output', output_csvs_path + '{}-{}-{}.csv'.format(trace.name, size, name)) for k, v in changes.items(): simulator.put(k, v) with open(conf_file, 'w') as f: f.write(HOCONConverter.to_hocon(conf)) if (not reuse or not os.path.isfile( simulator['report']['output'])) and not readonly: call(run_simulator, shell=True, cwd=caffeine_root, stdout=subprocess.DEVNULL if not verbose else None) with open(simulator['report']['output'], 'r') as csvfile: reader = csv.DictReader(csvfile) results = {line['Policy']: float(line['Hit rate']) for line in reader} if not save: os.remove(simulator['report']['output']) return results if len(results) != 1 else results[0]
def merge(self, b, update=False): """Merge self with b and then returns a plain string of merged. Args: b: HOCONString, dict, str to be merged. b's `include` statement will always be ignored. update: If True then replace self with a merged one. Returns: String of merged HOCONs. """ if isinstance(b, HOCONString): d = b.to_dict() elif isinstance(b, str): d = HOCONString(b).to_dict() elif isinstance(b, dict): d = b else: raise TypeError('Unsupported type {t}'.format(t=type(b))) self_d = self.to_dict() merge_dict(self_d, d) hocon = ConfigFactory.from_dict(self_d) hocon_str = HOCONConverter.to_hocon(hocon) if update: self._hocon_str = hocon_str return HOCONString(hocon_str).get_contents()
def print_configuration(self, remove_secret_keys=("secret", "pass", "token", "account_key")): # remove all the secrets from the print def recursive_remove_secrets(dictionary, secret_keys=()): for k in list(dictionary): for s in secret_keys: if s in k: dictionary.pop(k) break if isinstance(dictionary.get(k, None), dict): recursive_remove_secrets(dictionary[k], secret_keys=secret_keys) elif isinstance(dictionary.get(k, None), (list, tuple)): for item in dictionary[k]: if isinstance(item, dict): recursive_remove_secrets(item, secret_keys=secret_keys) config = deepcopy(self.config.to_dict()) # remove the env variable, it's not important config.pop('env', None) if remove_secret_keys: recursive_remove_secrets(config, secret_keys=remove_secret_keys) config = ConfigFactory.from_dict(config) self.log.debug("Run by interpreter: %s", sys.executable) print("Current configuration (trains_agent v{}, location: {}):\n" "----------------------\n{}\n".format( self.version, self._config_file, HOCONConverter.convert(config, "properties")))
def gen_conf(scale, app_name, query): conf = ConfigFactory.parse_file('../conf/application.conf') conf.put("all.query-num", query) conf.put("all.data-scale", scale) conf.put("all.app-suffix", app_name) with open('../conf/application-run.conf', 'w') as f: f.write(HOCONConverter.convert(conf, 'hocon'))
def save(self, output_dir: Union[Path, str]) -> bool: d = Path(output_dir) / instance_full_classname(self) if not os.path.exists(d): os.makedirs(d) logger.info(f"Saving Config to {d / 'config.conf'}") with open(d / "config.conf", "w") as f: f.write(HOCONConverter.to_hocon(self.config)) return True
def oci_image_bundle_conf(args, component_name, oci_manifest, oci_config): conf = ConfigFactory.parse_string('') load_bundle_args_into_conf(conf, args) if 'annotations' in oci_manifest and oci_manifest[ 'annotations'] is not None: annotations_tree = conf.get('annotations') for key in sorted(oci_manifest['annotations']): annotations_tree.put(key, oci_manifest['annotations'][key]) endpoints_tree = ConfigTree() oci_tree = ConfigTree() oci_tree.put('description', args.component_description) oci_tree.put('file-system-type', 'oci-image') oci_tree.put('start-command', ['ociImageTag', args.tag]) oci_tree.put('endpoints', endpoints_tree) components_tree = ConfigTree() components_tree.put(component_name, oci_tree) if args.use_default_endpoints and 'config' in oci_config and 'ExposedPorts' in oci_config[ 'config']: check_arguments = [] for exposed_port in sorted(oci_config['config']['ExposedPorts']): type_parts = exposed_port.split('/', 1) port = int(type_parts[0]) protocol = type_parts[1] if len(type_parts) > 1 else 'tcp' name = '{}-{}-{}'.format(component_name, protocol, port) check_arguments.append('${}_HOST'.format( re.sub('\\W', '_', name.upper()))) entry_tree = ConfigTree() entry_tree.put('bind-protocol', protocol) entry_tree.put('bind-port', port) entry_tree.put('service-name', name) endpoints_tree.put(name, entry_tree) if len(check_arguments) > 0: oci_check_tree = ConfigTree() oci_check_tree.put('description', 'Status check for oci-image component') oci_check_tree.put('file-system-type', 'universal') oci_check_tree.put('start-command', ['check'] + check_arguments) oci_check_tree.put('endpoints', {}) components_tree = ConfigTree() components_tree.put(component_name, oci_tree) components_tree.put('{}-status'.format(component_name), oci_check_tree) conf.put('components', components_tree) return HOCONConverter.to_hocon(conf)
def _load_custom_config(run_config): """Load custom configuration input HOCON file for cromwell. """ from pyhocon import ConfigFactory, HOCONConverter, ConfigTree conf = ConfigFactory.parse_file(run_config) out = {} if "database" in conf: out["database"] = HOCONConverter.to_hocon(ConfigTree({"database": conf.get_config("database")})) return out
def parse(hocon_file): """ This function takes a hocon file and returns a dictionary. :param hocon_file: Path to hocon file. :returns conf: Dictionary version of passed hocon file """ conf = ConfigFactory.parse_file(hocon_file) conf = json.loads(HOCONConverter.convert(conf, 'json')) return conf
def gen_conf(t1, t2, scale, app_name, query): conf = ConfigFactory.parse_file('../conf/application.conf') conf.put("Q23.table-list", [t1, t2]) conf.put("all.data-scale", scale) conf.put("all.hdfs", 'hdfs://%s:8020/'%HDFS) conf.put("all.app-suffix", app_name) conf.put("Q23.query", QUERYS[query]) with open('../conf/application-run.conf', 'w') as f: f.write(HOCONConverter.convert(conf, 'hocon'))
def run_one_task(self, queue: Text, task_id: Text, worker_args=None): task_data = self._session.api_client.tasks.get_all(id=[task_id])[0] # push task into the k8s queue, so we have visibility on pending tasks in the k8s scheduler try: self._session.api_client.tasks.enqueue( task_id, queue=self.k8s_pending_queue_name, status_reason='k8s pending scheduler') except Exception as e: self.log.error( "ERROR: Could not push back task [{}] to k8s pending queue [{}], error: {}" .format(task_id, self.k8s_pending_queue_name, e)) return if task_data.execution.docker_cmd: docker_image = task_data.execution.docker_cmd else: docker_image = str( os.environ.get("TRAINS_DOCKER_IMAGE") or self._session.config.get("agent.default_docker.image", "nvidia/cuda")) # take the first part, this is the docker image name (not arguments) docker_image = docker_image.split()[0] create_trains_conf = "echo '{}' >> ~/trains.conf && ".format( HOCONConverter.to_hocon(self._session.config._config)) if callable(self.kubectl_cmd): kubectl_cmd = self.kubectl_cmd(task_id, docker_image, queue, task_data) else: kubectl_cmd = self.kubectl_cmd.format(task_id=task_id, docker_image=docker_image, queue_id=queue) # make sure we gave a list if isinstance(kubectl_cmd, str): kubectl_cmd = kubectl_cmd.split() kubectl_cmd += [ "--labels=TRAINS=agent", "--command", "--", "/bin/sh", "-c", create_trains_conf + self.container_bash_script.format(task_id) ] process = subprocess.Popen(kubectl_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = process.communicate() self.log.info("K8s scheduling experiment task id={}".format(task_id)) if error: self.log.error("Running kubectl encountered an error: {}".format( error if isinstance(error, str) else error.decode()))
def _to_config_object(self, s) -> 'Configuration': if isinstance(s, ConfigTree): if self.__use_json: return Configuration( json.loads(HOCONConverter.convert(s, 'json'))) return Configuration(s) elif isinstance(s, str): fallback_config = ConfigFactory.from_dict({}) res = ConfigFactory.parse_string( s, resolve=False).with_fallback(fallback_config) if self.__use_json: return json.loads(HOCONConverter.convert(res, 'json')) return Configuration(res) elif isinstance(s, bytes): s = s.decode() fallback_config = ConfigFactory.from_dict({}) res = ConfigFactory.parse_string( s, resolve=False).with_fallback(fallback_config) if self.__use_json: return Configuration( json.loads(HOCONConverter.convert(res, 'json'))) return Configuration(res) raise ValueError("Cannot parse value", s)
def append(self, c: Union['Configuration', Dict]) -> 'Configuration': try: return super(ConfigurationHocon, self).append(c) except Exception as e: fallback_config = ConfigFactory.from_dict(self.as_dict()) res = ConfigFactory.parse_string( json.dumps(c.as_dict()), resolve=True).with_fallback(fallback_config) if self.__use_json: _c = Configuration( json.loads(HOCONConverter.convert(res, 'json'))) return super(ConfigurationHocon, self).append(_c) _c = Configuration(res) return super(ConfigurationHocon, self).append(_c)
def merge(self, b): if isinstance(b, HOCONString): d = b.to_dict() elif isinstance(b, str): d = HOCONString(b).to_dict() elif isinstance(b, dict): d = b else: raise TypeError('Unsupported type {t}'.format(t=type(b))) self_d = self.to_dict() merge_dict(self_d, d) hocon = ConfigFactory.from_dict(self_d) return self._include + '\n' + HOCONConverter.to_hocon(hocon)
def from_dict(cls, d, include=''): """Create HOCONString from dict. Args: include: `include` statement to be added to the top of the HOCONString. """ hocon = ConfigFactory.from_dict(d) hocon_str = HOCONConverter.to_hocon(hocon) if include: if not is_valid_include(include): raise ValueError( 'Wrong HOCON include format. {include}'.format( include=include)) hocon_str = NEW_LINE.join([include, hocon_str]) return cls(hocon_str=hocon_str)
def to_dict(self, with_include=True): """Convert HOCON string into dict. Args: with_include: If True then double-quote-escaped `include` statements will be kept as a plain string under key HOCONSTRING_INCLUDE_KEY. Otherwise, `include` statements will be excluded. """ if with_include: hocon_str = self._hocon_str else: hocon_str = self.get_contents(with_include=False) c = ConfigFactory.parse_string(hocon_str) j = HOCONConverter.to_json(c) return json.loads(j)
def load_config(cwd=os.getcwd(), debug=False): """ Tries to find HOCON files named "iss4e.conf" using the paths returned by find_files(). The found files are then parsed and merged together, so that a single configuration dict is returned. For details on HOCON syntax, see https://github.com/chimpler/pyhocon and https://github.com/typesafehub/config/ Example configuration: - default config in home dir (~/iss4e.conf): datasources { influx { host = ${HOSTNAME} # also set your passwords (e.g. from env with ${MYSQL_PASSWD} here } mysql { host = localhost } } - local config in cwd (./iss4e.conf): webike { # use the generic information from ${datasources.influx} (should be defined in ~/iss4e.conf and contain # host, password, ...) and extend it to use the (non-generic) database "webike" influx = ${datasources.influx} { db = "webike" } } - merged config that will be returned: { "datasources": { "influx": { "host": "SD959-LT" }, "mysql": { "host": "localhost" } }, "webike": { "influx": { "host": "SD959-LT", # copied from ~/iss4e.conf: datasources.influx "db": "webike" } } } """ # find "iss4e.conf" file in current working dir or parent directories files = find_files("iss4e.conf", cwd) configs = [ ConfigFactory.parse_file(file, required=False, resolve=False) for file in files if os.path.isfile(file) ] if debug: print("Config files:\n" + "\n".join(file + " [" + ("not " if not os.path.isfile(file) else "") + "found]" for file in files)) # merge all levels of config config = ConfigTree(root=True) config.put( "__main__", os.path.basename( getattr(sys.modules['__main__'], "__file__", "__cli__"))) config.put("__cwd__", os.path.abspath(cwd)) for c in configs: config = ConfigTree.merge_configs(c, config) config = ConfigParser.resolve_substitutions(config) if debug: print("Loaded config:\n" + HOCONConverter.to_json(config)) # if config contains a key "logging", use it to reconfigure python logging if "logging" in config: if debug: print("Reconfiguring logging from config") if config.get("capture_exceptions", True): sys.excepthook = log_uncaught_exception logging.captureWarnings(config.get("capture_warnings", True)) logging.config.dictConfig(config["logging"].as_plain_ordered_dict()) # check python version # iss4e lib is using some syntax features and functions which were only introduced in python 3.5 rec_ver = tuple(config.get("min_py_version", [3, 5])) if sys.version_info < rec_ver: warnings.warn( "Using outdated python version {}, a version >= {} would be recommended for use with iss4e lib. " "Try using a newer python binary, e.g. by calling `python{}.{}` instead of the default `python`." .format(sys.version_info, rec_ver, rec_ver[0], rec_ver[1])) return config
env_dict = parse_env_variables(env_dict) nested_set(env_dict, ['waves', 'directory'], '/waves') nested_set(env_dict, ['waves', 'data-directory'], '/waves/data') nested_set(env_dict, ['waves', 'wallet', 'seed'], wallet_data['seed']) nested_set(env_dict, ['waves', 'wallet', 'password'], wallet_data['password']) WAVES_AUTODETECT_ADDRESS = os.getenv('WAVES_AUTODETECT_ADDRESS', DEFAULT_AUTODETECT) WAVES_DECLARED_ADDRESS = os.getenv('WAVES_DECLARED_ADDRESS') print("WAVES_DECLARED_ADDRESS", WAVES_DECLARED_ADDRESS) if WAVES_AUTODETECT_ADDRESS.lower( ) == 'yes' and WAVES_DECLARED_ADDRESS is None: WAVES_DECLARED_ADDRESS = get_external_ip() + ':' + get_port_number( NETWORK) print("Detected address is " + WAVES_DECLARED_ADDRESS) nested_set(env_dict, ['waves', 'network', 'declared-address'], WAVES_DECLARED_ADDRESS) elif WAVES_DECLARED_ADDRESS is not None: nested_set(env_dict, ['waves', 'network', 'declared-address'], WAVES_DECLARED_ADDRESS) config = ConfigFactory.from_dict(env_dict) local_conf = HOCONConverter.convert(config, 'hocon') print(local_conf) with open(LOCAL_FILE_PATH, 'w') as file: file.write(local_conf) download_jar_file(WAVES_VERSION, VERSION)
def to_str(self): return HOCONConverter.convert(self._config_tree, 'hocon')
def write(self, stream, output_format='hocon'): config = ConfigFactory.from_dict(self.as_dict()) res = HOCONConverter.convert(config, output_format='hocon', compact=True) stream.write(res)
def print(self): print() print("Configuration:") print(HOCONConverter.convert(self.data, 'hocon')) print()
config["parameters"] = complete_parameters template_without_dashes, parameters_without_dashes = convert_dashes_to_underscores( template, config["parameters"], template_path) if config['parameters'] != parameters_without_dashes: parameters_changed = True config['parameters'] = parameters_without_dashes if parameters_changed: print("Overwriting template.conf: %s" % config_path) # We can use the HOCONConverter for dumping the json as well but we have more options to control # the output with the default json writer if opts.output_format == "hocon": open(config_path, "w+").write(HOCONConverter.convert(config, "hocon")) else: open(config_path, "w+").write( # This is a config tree but json.dumps uses iterators to go over it thinking it is a dict. # Since configtree already supports these iterators it works correctly for it. json.dumps(config, indent=2, separators=( ',', ': ', ), ensure_ascii=False) + "\n") if template_without_dashes != template: print("Overwriting template: %s" % template_path) open(template_path, "w+").write(template_without_dashes)
def parse_fp(self, fp): conf = ConfigFactory.parse_file(fp) return json.loads(HOCONConverter.to_json(conf))
def to_dict(self): """Convert contents without include to dict. """ c = ConfigFactory.parse_string(self._contents_wo_include) j = HOCONConverter.to_json(c) return json.loads(j)
def to_json(self): return json.loads(HOCONConverter.convert(self._config_tree, 'json'))
def parse_string(self, s): conf = ConfigFactory.parse_string(s) return json.loads(HOCONConverter.to_json(conf))
def save_config(args: Args, cfg: ConfigTree): config_path = args.run_dir / 'config.json' with open(config_path, 'w') as f: f.write(HOCONConverter.to_json(cfg))
template_path, opts.parameter_type) parameters_changed = False if config["parameters"] != complete_parameters: parameters_changed = True config["parameters"] = complete_parameters template_without_dashes, parameters_without_dashes = convert_dashes_to_underscores(template, config["parameters"], template_path) if config['parameters'] != parameters_without_dashes: parameters_changed = True config['parameters'] = parameters_without_dashes if parameters_changed: print("Overwriting template.conf: %s" % config_path) # We can use the HOCONConverter for dumping the json as well but we have more options to control # the output with the default json writer if opts.output_format == "hocon": open(config_path, "w+").write(HOCONConverter.convert(config, "hocon")) else: open(config_path, "w+").write( # This is a config tree but json.dumps uses iterators to go over it thinking it is a dict. # Since configtree already supports these iterators it works correctly for it. json.dumps(config, indent=2, separators=(',', ': ', ), ensure_ascii=False) + "\n" ) if template_without_dashes != template: print("Overwriting template: %s" % template_path) open(template_path, "w+").write(template_without_dashes)
def saveHoconFile(pathFileNameExt, parameterDict): paramHocon = ConfigFactory.from_dict(parameterDict) paramStr = HOCONConverter.to_hocon(paramHocon,indent=4) with open(pathFileNameExt, "w", encoding='utf-8') as fd: fd.write(paramStr)
def dumps(obj: Any, **kwargs) -> str: return HOCONConverter.to_json(obj, **kwargs)
def generate_compose(genesis_path): if not os.path.isdir("/waves-mnt/output"): os.mkdir("/waves-mnt/output") compose_content = "version: '3'\n\nservices:" nodes_list = [] known_peers_list = [] for k in range(0, accounts_count): known_peers_list.append(f"node{k}:6864") for k in range(0, accounts_count): if k > 0: if int(network_port) < 10000: instance_network_port = f"{k}{network_port}" instance_api_port = f"{k}{api_port}" else: instance_network_port = f"{str(int(network_port) + k)}" instance_api_port = f"{str(int(api_port) + k)}" else: instance_network_port = network_port instance_api_port = api_port node_config_path = f"/waves-mnt/output/node{k}/configs" if not os.path.isdir(node_config_path): os.makedirs(node_config_path, exist_ok=True) config_content = ConfigFactory.parse_file(genesis_path) config_content['waves']['network'] = ConfigFactory.from_dict( {'known-peers': known_peers_list}) config_content = HOCONConverter.convert(config_content, 'hocon') config_path = f"/waves-mnt/output/node{k}/configs/local.conf" with open(config_path, 'w') as config_file: config_file.write(config_content) nodes_list.append(f"node{k}:6869") compose_content += f""" node{k}: hostname: node{k} image: wavesplatform/node environment: - WAVES__NETWORK__NODE_NAME=node{k} - WAVES_WALLET_SEED={accounts[k]['seed']} - WAVES_WALLET_PASSWORD={accounts[k]['password']} - WAVES_VERSION=latest - WAVES_NETWORK={network} - WAVES_LOG_LEVEL={log_level} - WAVES_HEAP_SIZE={heap_size} - WAVES_AUTODETECT_ADDRESS={autodetect_address} - WAVES_AUTODETECT_ADDRESS_PORT={instance_network_port} - WAVES__NETWORK__DECLARED_ADDRESS=node{k}:6864 - WAVES__REST_API__ENABLE={WAVES__REST_API__ENABLE} - WAVES__REST_API__PORT=6869 - WAVES__REST_API__API_KEY_HASH=3z7LKtx9DwNVtEZ2whjZqrdZH8iWTZsdQYwWbgsxeXhm - WAVES__BLOCKCHAIN__CUSTOM__ADDRESS_SCHEME_CHARACTER={network_byte} - WAVES__MINER__QUORUM={accounts_count - 1} - WAVES__MINER__MIN_MICRO_BLOCK_AGE=2s - WAVES__MINER__MICRO_BLOCK_INTERVAL=3s - WAVES__MINER__INTERVAL_AFTER_LAST_BLOCK_THEN_GENERATION_IS_ALLOWED=999d restart: always networks: default: aliases: - node{k} - node{k}.waves ports: - "{instance_api_port}:6869" - "{instance_network_port}:6864" volumes: - ./node{k}:/waves""" compose_content = append_data_service(compose_content, nodes_list) compose_path = "/waves-mnt/output/docker-compose.yml" with open(compose_path, 'w') as compose_file: compose_file.write(compose_content)
def from_dict(cls, d, include=''): hocon = ConfigFactory.from_dict(d) hocon_str = HOCONConverter.to_hocon(hocon) if include: hocon_str = include + '\n' + hocon_str return cls(hocon_str=hocon_str)