def _config_dict_to_text(config):
if not isinstance(config, dict):
raise ValueError("Model configuration only supports dictionary objects")
try:
# hack, pyhocon is not very good with dict conversion so we pass through json
try:
import json
text = json.dumps(config)
text = HOCONConverter.convert(ConfigFactory.parse_string(text), 'hocon')
except Exception:
# fallback pyhocon
text = HOCONConverter.convert(ConfigFactory.from_dict(config), 'hocon')
except Exception:
raise ValueError("Could not serialize configuration dictionary:\n", config)
return text
def print_configuration(self,
remove_secret_keys=("secret", "pass", "token",
"account_key")):
# remove all the secrets from the print
def recursive_remove_secrets(dictionary, secret_keys=()):
for k in list(dictionary):
for s in secret_keys:
if s in k:
dictionary.pop(k)
break
if isinstance(dictionary.get(k, None), dict):
recursive_remove_secrets(dictionary[k],
secret_keys=secret_keys)
elif isinstance(dictionary.get(k, None), (list, tuple)):
for item in dictionary[k]:
if isinstance(item, dict):
recursive_remove_secrets(item,
secret_keys=secret_keys)
config = deepcopy(self.config.to_dict())
# remove the env variable, it's not important
config.pop('env', None)
if remove_secret_keys:
recursive_remove_secrets(config, secret_keys=remove_secret_keys)
config = ConfigFactory.from_dict(config)
self.log.debug("Run by interpreter: %s", sys.executable)
print("Current configuration (trains_agent v{}, location: {}):\n"
"----------------------\n{}\n".format(
self.version, self._config_file,
HOCONConverter.convert(config, "properties")))
def gen_conf(scale, app_name, query):
conf = ConfigFactory.parse_file('../conf/application.conf')
conf.put("all.query-num", query)
conf.put("all.data-scale", scale)
conf.put("all.app-suffix", app_name)
with open('../conf/application-run.conf', 'w') as f:
f.write(HOCONConverter.convert(conf, 'hocon'))
def gen_conf(t1, t2, scale, app_name, query):
conf = ConfigFactory.parse_file('../conf/application.conf')
conf.put("Q23.table-list", [t1, t2])
conf.put("all.data-scale", scale)
conf.put("all.hdfs", 'hdfs://%s:8020/'%HDFS)
conf.put("all.app-suffix", app_name)
conf.put("Q23.query", QUERYS[query])
with open('../conf/application-run.conf', 'w') as f:
f.write(HOCONConverter.convert(conf, 'hocon'))
def parse(hocon_file):
"""
This function takes a hocon file
and returns a dictionary.
:param hocon_file: Path to hocon file.
:returns conf: Dictionary version of passed hocon file
"""
conf = ConfigFactory.parse_file(hocon_file)
conf = json.loads(HOCONConverter.convert(conf, 'json'))
return conf
def append(self, c: Union['Configuration', Dict]) -> 'Configuration':
try:
return super(ConfigurationHocon, self).append(c)
except Exception as e:
fallback_config = ConfigFactory.from_dict(self.as_dict())
res = ConfigFactory.parse_string(
json.dumps(c.as_dict()),
resolve=True).with_fallback(fallback_config)
if self.__use_json:
_c = Configuration(
json.loads(HOCONConverter.convert(res, 'json')))
return super(ConfigurationHocon, self).append(_c)
_c = Configuration(res)
return super(ConfigurationHocon, self).append(_c)
def _to_config_object(self, s) -> 'Configuration':
if isinstance(s, ConfigTree):
if self.__use_json:
return Configuration(
json.loads(HOCONConverter.convert(s, 'json')))
return Configuration(s)
elif isinstance(s, str):
fallback_config = ConfigFactory.from_dict({})
res = ConfigFactory.parse_string(
s, resolve=False).with_fallback(fallback_config)
if self.__use_json:
return json.loads(HOCONConverter.convert(res, 'json'))
return Configuration(res)
elif isinstance(s, bytes):
s = s.decode()
fallback_config = ConfigFactory.from_dict({})
res = ConfigFactory.parse_string(
s, resolve=False).with_fallback(fallback_config)
if self.__use_json:
return Configuration(
json.loads(HOCONConverter.convert(res, 'json')))
return Configuration(res)
raise ValueError("Cannot parse value", s)
def format_hocon(value):
""""
This convert a dict to hocon(STR) type.
"""
config = ConfigFactory().from_dict(value)
return HOCONConverter.convert(config, 'hocon')
def to_json(self):
return json.loads(HOCONConverter.convert(self._config_tree, 'json'))
def to_str(self):
return HOCONConverter.convert(self._config_tree, 'hocon')
env_dict = parse_env_variables(env_dict)
nested_set(env_dict, ['waves', 'directory'], '/waves')
nested_set(env_dict, ['waves', 'data-directory'], '/waves/data')
nested_set(env_dict, ['waves', 'wallet', 'seed'], wallet_data['seed'])
nested_set(env_dict, ['waves', 'wallet', 'password'],
wallet_data['password'])
WAVES_AUTODETECT_ADDRESS = os.getenv('WAVES_AUTODETECT_ADDRESS',
DEFAULT_AUTODETECT)
WAVES_DECLARED_ADDRESS = os.getenv('WAVES_DECLARED_ADDRESS')
print("WAVES_DECLARED_ADDRESS", WAVES_DECLARED_ADDRESS)
if WAVES_AUTODETECT_ADDRESS.lower(
) == 'yes' and WAVES_DECLARED_ADDRESS is None:
WAVES_DECLARED_ADDRESS = get_external_ip() + ':' + get_port_number(
NETWORK)
print("Detected address is " + WAVES_DECLARED_ADDRESS)
nested_set(env_dict, ['waves', 'network', 'declared-address'],
WAVES_DECLARED_ADDRESS)
elif WAVES_DECLARED_ADDRESS is not None:
nested_set(env_dict, ['waves', 'network', 'declared-address'],
WAVES_DECLARED_ADDRESS)
config = ConfigFactory.from_dict(env_dict)
local_conf = HOCONConverter.convert(config, 'hocon')
print(local_conf)
with open(LOCAL_FILE_PATH, 'w') as file:
file.write(local_conf)
download_jar_file(WAVES_VERSION, VERSION)
def write(self, stream, output_format='hocon'):
config = ConfigFactory.from_dict(self.as_dict())
res = HOCONConverter.convert(config,
output_format='hocon',
compact=True)
stream.write(res)
config["parameters"] = complete_parameters
template_without_dashes, parameters_without_dashes = convert_dashes_to_underscores(
template, config["parameters"], template_path)
if config['parameters'] != parameters_without_dashes:
parameters_changed = True
config['parameters'] = parameters_without_dashes
if parameters_changed:
print("Overwriting template.conf: %s" % config_path)
# We can use the HOCONConverter for dumping the json as well but we have more options to control
# the output with the default json writer
if opts.output_format == "hocon":
open(config_path,
"w+").write(HOCONConverter.convert(config, "hocon"))
else:
open(config_path, "w+").write(
# This is a config tree but json.dumps uses iterators to go over it thinking it is a dict.
# Since configtree already supports these iterators it works correctly for it.
json.dumps(config,
indent=2,
separators=(
',',
': ',
),
ensure_ascii=False) + "\n")
if template_without_dashes != template:
print("Overwriting template: %s" % template_path)
open(template_path, "w+").write(template_without_dashes)
def generate_compose(genesis_path):
if not os.path.isdir("/waves-mnt/output"):
os.mkdir("/waves-mnt/output")
compose_content = "version: '3'\n\nservices:"
nodes_list = []
known_peers_list = []
for k in range(0, accounts_count):
known_peers_list.append(f"node{k}:6864")
for k in range(0, accounts_count):
if k > 0:
if int(network_port) < 10000:
instance_network_port = f"{k}{network_port}"
instance_api_port = f"{k}{api_port}"
else:
instance_network_port = f"{str(int(network_port) + k)}"
instance_api_port = f"{str(int(api_port) + k)}"
else:
instance_network_port = network_port
instance_api_port = api_port
node_config_path = f"/waves-mnt/output/node{k}/configs"
if not os.path.isdir(node_config_path):
os.makedirs(node_config_path, exist_ok=True)
config_content = ConfigFactory.parse_file(genesis_path)
config_content['waves']['network'] = ConfigFactory.from_dict(
{'known-peers': known_peers_list})
config_content = HOCONConverter.convert(config_content, 'hocon')
config_path = f"/waves-mnt/output/node{k}/configs/local.conf"
with open(config_path, 'w') as config_file:
config_file.write(config_content)
nodes_list.append(f"node{k}:6869")
compose_content += f"""
node{k}:
hostname: node{k}
image: wavesplatform/node
environment:
- WAVES__NETWORK__NODE_NAME=node{k}
- WAVES_WALLET_SEED={accounts[k]['seed']}
- WAVES_WALLET_PASSWORD={accounts[k]['password']}
- WAVES_VERSION=latest
- WAVES_NETWORK={network}
- WAVES_LOG_LEVEL={log_level}
- WAVES_HEAP_SIZE={heap_size}
- WAVES_AUTODETECT_ADDRESS={autodetect_address}
- WAVES_AUTODETECT_ADDRESS_PORT={instance_network_port}
- WAVES__NETWORK__DECLARED_ADDRESS=node{k}:6864
- WAVES__REST_API__ENABLE={WAVES__REST_API__ENABLE}
- WAVES__REST_API__PORT=6869
- WAVES__REST_API__API_KEY_HASH=3z7LKtx9DwNVtEZ2whjZqrdZH8iWTZsdQYwWbgsxeXhm
- WAVES__BLOCKCHAIN__CUSTOM__ADDRESS_SCHEME_CHARACTER={network_byte}
- WAVES__MINER__QUORUM={accounts_count - 1}
- WAVES__MINER__MIN_MICRO_BLOCK_AGE=2s
- WAVES__MINER__MICRO_BLOCK_INTERVAL=3s
- WAVES__MINER__INTERVAL_AFTER_LAST_BLOCK_THEN_GENERATION_IS_ALLOWED=999d
restart: always
networks:
default:
aliases:
- node{k}
- node{k}.waves
ports:
- "{instance_api_port}:6869"
- "{instance_network_port}:6864"
volumes:
- ./node{k}:/waves"""
compose_content = append_data_service(compose_content, nodes_list)
compose_path = "/waves-mnt/output/docker-compose.yml"
with open(compose_path, 'w') as compose_file:
compose_file.write(compose_content)
def render_to_template(**data):
config = ConfigFactory.from_dict(data)
return HOCONConverter.convert(config, 'hocon')
def print(self):
print()
print("Configuration:")
print(HOCONConverter.convert(self.data, 'hocon'))
print()
def to_str(self):
return HOCONConverter.convert(self._config_tree, 'hocon')
def config2json(conf):
lines = HOCONConverter.convert(conf, indent=0)
return ''.join(lines).replace('\n', ' ')
template_path, opts.parameter_type)
parameters_changed = False
if config["parameters"] != complete_parameters:
parameters_changed = True
config["parameters"] = complete_parameters
template_without_dashes, parameters_without_dashes = convert_dashes_to_underscores(template,
config["parameters"],
template_path)
if config['parameters'] != parameters_without_dashes:
parameters_changed = True
config['parameters'] = parameters_without_dashes
if parameters_changed:
print("Overwriting template.conf: %s" % config_path)
# We can use the HOCONConverter for dumping the json as well but we have more options to control
# the output with the default json writer
if opts.output_format == "hocon":
open(config_path, "w+").write(HOCONConverter.convert(config, "hocon"))
else:
open(config_path, "w+").write(
# This is a config tree but json.dumps uses iterators to go over it thinking it is a dict.
# Since configtree already supports these iterators it works correctly for it.
json.dumps(config, indent=2, separators=(',', ': ', ), ensure_ascii=False) + "\n"
)
if template_without_dashes != template:
print("Overwriting template: %s" % template_path)
open(template_path, "w+").write(template_without_dashes)
def write_config(conf, filename, output_format):
lines = HOCONConverter.convert(conf, output_format=output_format, indent=4)
with open(filename, 'w') as fh:
fh.writelines(lines)
