def _config_dict_to_text(config): if not isinstance(config, dict): raise ValueError("Model configuration only supports dictionary objects") try: # hack, pyhocon is not very good with dict conversion so we pass through json try: import json text = json.dumps(config) text = HOCONConverter.convert(ConfigFactory.parse_string(text), 'hocon') except Exception: # fallback pyhocon text = HOCONConverter.convert(ConfigFactory.from_dict(config), 'hocon') except Exception: raise ValueError("Could not serialize configuration dictionary:\n", config) return text
def print_configuration(self, remove_secret_keys=("secret", "pass", "token", "account_key")): # remove all the secrets from the print def recursive_remove_secrets(dictionary, secret_keys=()): for k in list(dictionary): for s in secret_keys: if s in k: dictionary.pop(k) break if isinstance(dictionary.get(k, None), dict): recursive_remove_secrets(dictionary[k], secret_keys=secret_keys) elif isinstance(dictionary.get(k, None), (list, tuple)): for item in dictionary[k]: if isinstance(item, dict): recursive_remove_secrets(item, secret_keys=secret_keys) config = deepcopy(self.config.to_dict()) # remove the env variable, it's not important config.pop('env', None) if remove_secret_keys: recursive_remove_secrets(config, secret_keys=remove_secret_keys) config = ConfigFactory.from_dict(config) self.log.debug("Run by interpreter: %s", sys.executable) print("Current configuration (trains_agent v{}, location: {}):\n" "----------------------\n{}\n".format( self.version, self._config_file, HOCONConverter.convert(config, "properties")))
def gen_conf(scale, app_name, query): conf = ConfigFactory.parse_file('../conf/application.conf') conf.put("all.query-num", query) conf.put("all.data-scale", scale) conf.put("all.app-suffix", app_name) with open('../conf/application-run.conf', 'w') as f: f.write(HOCONConverter.convert(conf, 'hocon'))
def gen_conf(t1, t2, scale, app_name, query): conf = ConfigFactory.parse_file('../conf/application.conf') conf.put("Q23.table-list", [t1, t2]) conf.put("all.data-scale", scale) conf.put("all.hdfs", 'hdfs://%s:8020/'%HDFS) conf.put("all.app-suffix", app_name) conf.put("Q23.query", QUERYS[query]) with open('../conf/application-run.conf', 'w') as f: f.write(HOCONConverter.convert(conf, 'hocon'))
def parse(hocon_file): """ This function takes a hocon file and returns a dictionary. :param hocon_file: Path to hocon file. :returns conf: Dictionary version of passed hocon file """ conf = ConfigFactory.parse_file(hocon_file) conf = json.loads(HOCONConverter.convert(conf, 'json')) return conf
def append(self, c: Union['Configuration', Dict]) -> 'Configuration': try: return super(ConfigurationHocon, self).append(c) except Exception as e: fallback_config = ConfigFactory.from_dict(self.as_dict()) res = ConfigFactory.parse_string( json.dumps(c.as_dict()), resolve=True).with_fallback(fallback_config) if self.__use_json: _c = Configuration( json.loads(HOCONConverter.convert(res, 'json'))) return super(ConfigurationHocon, self).append(_c) _c = Configuration(res) return super(ConfigurationHocon, self).append(_c)
def _to_config_object(self, s) -> 'Configuration': if isinstance(s, ConfigTree): if self.__use_json: return Configuration( json.loads(HOCONConverter.convert(s, 'json'))) return Configuration(s) elif isinstance(s, str): fallback_config = ConfigFactory.from_dict({}) res = ConfigFactory.parse_string( s, resolve=False).with_fallback(fallback_config) if self.__use_json: return json.loads(HOCONConverter.convert(res, 'json')) return Configuration(res) elif isinstance(s, bytes): s = s.decode() fallback_config = ConfigFactory.from_dict({}) res = ConfigFactory.parse_string( s, resolve=False).with_fallback(fallback_config) if self.__use_json: return Configuration( json.loads(HOCONConverter.convert(res, 'json'))) return Configuration(res) raise ValueError("Cannot parse value", s)
def format_hocon(value): """" This convert a dict to hocon(STR) type. """ config = ConfigFactory().from_dict(value) return HOCONConverter.convert(config, 'hocon')
def to_json(self): return json.loads(HOCONConverter.convert(self._config_tree, 'json'))
def to_str(self): return HOCONConverter.convert(self._config_tree, 'hocon')
env_dict = parse_env_variables(env_dict) nested_set(env_dict, ['waves', 'directory'], '/waves') nested_set(env_dict, ['waves', 'data-directory'], '/waves/data') nested_set(env_dict, ['waves', 'wallet', 'seed'], wallet_data['seed']) nested_set(env_dict, ['waves', 'wallet', 'password'], wallet_data['password']) WAVES_AUTODETECT_ADDRESS = os.getenv('WAVES_AUTODETECT_ADDRESS', DEFAULT_AUTODETECT) WAVES_DECLARED_ADDRESS = os.getenv('WAVES_DECLARED_ADDRESS') print("WAVES_DECLARED_ADDRESS", WAVES_DECLARED_ADDRESS) if WAVES_AUTODETECT_ADDRESS.lower( ) == 'yes' and WAVES_DECLARED_ADDRESS is None: WAVES_DECLARED_ADDRESS = get_external_ip() + ':' + get_port_number( NETWORK) print("Detected address is " + WAVES_DECLARED_ADDRESS) nested_set(env_dict, ['waves', 'network', 'declared-address'], WAVES_DECLARED_ADDRESS) elif WAVES_DECLARED_ADDRESS is not None: nested_set(env_dict, ['waves', 'network', 'declared-address'], WAVES_DECLARED_ADDRESS) config = ConfigFactory.from_dict(env_dict) local_conf = HOCONConverter.convert(config, 'hocon') print(local_conf) with open(LOCAL_FILE_PATH, 'w') as file: file.write(local_conf) download_jar_file(WAVES_VERSION, VERSION)
def write(self, stream, output_format='hocon'): config = ConfigFactory.from_dict(self.as_dict()) res = HOCONConverter.convert(config, output_format='hocon', compact=True) stream.write(res)
config["parameters"] = complete_parameters template_without_dashes, parameters_without_dashes = convert_dashes_to_underscores( template, config["parameters"], template_path) if config['parameters'] != parameters_without_dashes: parameters_changed = True config['parameters'] = parameters_without_dashes if parameters_changed: print("Overwriting template.conf: %s" % config_path) # We can use the HOCONConverter for dumping the json as well but we have more options to control # the output with the default json writer if opts.output_format == "hocon": open(config_path, "w+").write(HOCONConverter.convert(config, "hocon")) else: open(config_path, "w+").write( # This is a config tree but json.dumps uses iterators to go over it thinking it is a dict. # Since configtree already supports these iterators it works correctly for it. json.dumps(config, indent=2, separators=( ',', ': ', ), ensure_ascii=False) + "\n") if template_without_dashes != template: print("Overwriting template: %s" % template_path) open(template_path, "w+").write(template_without_dashes)
def generate_compose(genesis_path): if not os.path.isdir("/waves-mnt/output"): os.mkdir("/waves-mnt/output") compose_content = "version: '3'\n\nservices:" nodes_list = [] known_peers_list = [] for k in range(0, accounts_count): known_peers_list.append(f"node{k}:6864") for k in range(0, accounts_count): if k > 0: if int(network_port) < 10000: instance_network_port = f"{k}{network_port}" instance_api_port = f"{k}{api_port}" else: instance_network_port = f"{str(int(network_port) + k)}" instance_api_port = f"{str(int(api_port) + k)}" else: instance_network_port = network_port instance_api_port = api_port node_config_path = f"/waves-mnt/output/node{k}/configs" if not os.path.isdir(node_config_path): os.makedirs(node_config_path, exist_ok=True) config_content = ConfigFactory.parse_file(genesis_path) config_content['waves']['network'] = ConfigFactory.from_dict( {'known-peers': known_peers_list}) config_content = HOCONConverter.convert(config_content, 'hocon') config_path = f"/waves-mnt/output/node{k}/configs/local.conf" with open(config_path, 'w') as config_file: config_file.write(config_content) nodes_list.append(f"node{k}:6869") compose_content += f""" node{k}: hostname: node{k} image: wavesplatform/node environment: - WAVES__NETWORK__NODE_NAME=node{k} - WAVES_WALLET_SEED={accounts[k]['seed']} - WAVES_WALLET_PASSWORD={accounts[k]['password']} - WAVES_VERSION=latest - WAVES_NETWORK={network} - WAVES_LOG_LEVEL={log_level} - WAVES_HEAP_SIZE={heap_size} - WAVES_AUTODETECT_ADDRESS={autodetect_address} - WAVES_AUTODETECT_ADDRESS_PORT={instance_network_port} - WAVES__NETWORK__DECLARED_ADDRESS=node{k}:6864 - WAVES__REST_API__ENABLE={WAVES__REST_API__ENABLE} - WAVES__REST_API__PORT=6869 - WAVES__REST_API__API_KEY_HASH=3z7LKtx9DwNVtEZ2whjZqrdZH8iWTZsdQYwWbgsxeXhm - WAVES__BLOCKCHAIN__CUSTOM__ADDRESS_SCHEME_CHARACTER={network_byte} - WAVES__MINER__QUORUM={accounts_count - 1} - WAVES__MINER__MIN_MICRO_BLOCK_AGE=2s - WAVES__MINER__MICRO_BLOCK_INTERVAL=3s - WAVES__MINER__INTERVAL_AFTER_LAST_BLOCK_THEN_GENERATION_IS_ALLOWED=999d restart: always networks: default: aliases: - node{k} - node{k}.waves ports: - "{instance_api_port}:6869" - "{instance_network_port}:6864" volumes: - ./node{k}:/waves""" compose_content = append_data_service(compose_content, nodes_list) compose_path = "/waves-mnt/output/docker-compose.yml" with open(compose_path, 'w') as compose_file: compose_file.write(compose_content)
def render_to_template(**data): config = ConfigFactory.from_dict(data) return HOCONConverter.convert(config, 'hocon')
def print(self): print() print("Configuration:") print(HOCONConverter.convert(self.data, 'hocon')) print()
def to_str(self): return HOCONConverter.convert(self._config_tree, 'hocon')
def config2json(conf): lines = HOCONConverter.convert(conf, indent=0) return ''.join(lines).replace('\n', ' ')
template_path, opts.parameter_type) parameters_changed = False if config["parameters"] != complete_parameters: parameters_changed = True config["parameters"] = complete_parameters template_without_dashes, parameters_without_dashes = convert_dashes_to_underscores(template, config["parameters"], template_path) if config['parameters'] != parameters_without_dashes: parameters_changed = True config['parameters'] = parameters_without_dashes if parameters_changed: print("Overwriting template.conf: %s" % config_path) # We can use the HOCONConverter for dumping the json as well but we have more options to control # the output with the default json writer if opts.output_format == "hocon": open(config_path, "w+").write(HOCONConverter.convert(config, "hocon")) else: open(config_path, "w+").write( # This is a config tree but json.dumps uses iterators to go over it thinking it is a dict. # Since configtree already supports these iterators it works correctly for it. json.dumps(config, indent=2, separators=(',', ': ', ), ensure_ascii=False) + "\n" ) if template_without_dashes != template: print("Overwriting template: %s" % template_path) open(template_path, "w+").write(template_without_dashes)
def write_config(conf, filename, output_format): lines = HOCONConverter.convert(conf, output_format=output_format, indent=4) with open(filename, 'w') as fh: fh.writelines(lines)