def dumps(instance: object, out: str):
conf = __generate(instance, "")
if out:
if out.lower() == "hocon":
return HOCONConverter.to_hocon(conf)
if out.lower() == "yaml":
return HOCONConverter.to_yaml(conf)
if out.lower() == "json":
return HOCONConverter.to_json(conf)
if out.lower() == "properties":
return HOCONConverter.to_properties(conf)
return conf
def _config_dict_to_text(config):
if not isinstance(config, dict):
raise ValueError("Model configuration only supports dictionary objects")
try:
# hack, pyhocon is not very good with dict conversion so we pass through json
try:
import json
text = json.dumps(config)
text = HOCONConverter.convert(ConfigFactory.parse_string(text), 'hocon')
except Exception:
# fallback pyhocon
text = HOCONConverter.convert(ConfigFactory.from_dict(config), 'hocon')
except Exception:
raise ValueError("Could not serialize configuration dictionary:\n", config)
return text
def single_run(policy,
trace,
size=4,
changes={},
name=None,
save=True,
reuse=False,
verbose=False,
readonly=False):
name = name if name else policy
policy = Policy[policy]
trace = Trace[trace]
if 0 < size < 9:
size = trace.typical_caches()[size - 1]
conf_path = caffeine_root + 'simulator{0}src{0}main{0}resources{0}'.format(
os.sep)
conf_file = conf_path + 'application.conf'
if not os.path.exists(output_csvs_path):
os.makedirs(output_csvs_path)
run_simulator = './gradlew simulator:run -x caffeine:compileJava -x caffeine:compileCodeGenJava'
# run_simulator = './gradlew simulator:run'
if os.path.exists(conf_file):
conf = ConfigFactory.parse_file(conf_file)
else:
conf = ConfigFactory.parse_string("""
caffeine {
simulator {
}
}
""")
simulator = conf['caffeine']['simulator']
simulator.put('files.paths',
[resources_path + trace.format() + os.sep + trace.file()])
simulator.put('files.format', trace.value['format'])
simulator.put('maximum-size', size)
simulator.put('policies', [policy.value])
simulator.put('admission', [Admission.ALWAYS.value])
simulator.put('report.format', 'csv')
simulator.put(
'report.output',
output_csvs_path + '{}-{}-{}.csv'.format(trace.name, size, name))
for k, v in changes.items():
simulator.put(k, v)
with open(conf_file, 'w') as f:
f.write(HOCONConverter.to_hocon(conf))
if (not reuse or not os.path.isfile(
simulator['report']['output'])) and not readonly:
call(run_simulator,
shell=True,
cwd=caffeine_root,
stdout=subprocess.DEVNULL if not verbose else None)
with open(simulator['report']['output'], 'r') as csvfile:
reader = csv.DictReader(csvfile)
results = {line['Policy']: float(line['Hit rate']) for line in reader}
if not save:
os.remove(simulator['report']['output'])
return results if len(results) != 1 else results[0]
def merge(self, b, update=False):
"""Merge self with b and then returns a plain string of merged.
Args:
b:
HOCONString, dict, str to be merged.
b's `include` statement will always be ignored.
update:
If True then replace self with a merged one.
Returns:
String of merged HOCONs.
"""
if isinstance(b, HOCONString):
d = b.to_dict()
elif isinstance(b, str):
d = HOCONString(b).to_dict()
elif isinstance(b, dict):
d = b
else:
raise TypeError('Unsupported type {t}'.format(t=type(b)))
self_d = self.to_dict()
merge_dict(self_d, d)
hocon = ConfigFactory.from_dict(self_d)
hocon_str = HOCONConverter.to_hocon(hocon)
if update:
self._hocon_str = hocon_str
return HOCONString(hocon_str).get_contents()
def print_configuration(self,
remove_secret_keys=("secret", "pass", "token",
"account_key")):
# remove all the secrets from the print
def recursive_remove_secrets(dictionary, secret_keys=()):
for k in list(dictionary):
for s in secret_keys:
if s in k:
dictionary.pop(k)
break
if isinstance(dictionary.get(k, None), dict):
recursive_remove_secrets(dictionary[k],
secret_keys=secret_keys)
elif isinstance(dictionary.get(k, None), (list, tuple)):
for item in dictionary[k]:
if isinstance(item, dict):
recursive_remove_secrets(item,
secret_keys=secret_keys)
config = deepcopy(self.config.to_dict())
# remove the env variable, it's not important
config.pop('env', None)
if remove_secret_keys:
recursive_remove_secrets(config, secret_keys=remove_secret_keys)
config = ConfigFactory.from_dict(config)
self.log.debug("Run by interpreter: %s", sys.executable)
print("Current configuration (trains_agent v{}, location: {}):\n"
"----------------------\n{}\n".format(
self.version, self._config_file,
HOCONConverter.convert(config, "properties")))
def gen_conf(scale, app_name, query):
conf = ConfigFactory.parse_file('../conf/application.conf')
conf.put("all.query-num", query)
conf.put("all.data-scale", scale)
conf.put("all.app-suffix", app_name)
with open('../conf/application-run.conf', 'w') as f:
f.write(HOCONConverter.convert(conf, 'hocon'))
def save(self, output_dir: Union[Path, str]) -> bool:
d = Path(output_dir) / instance_full_classname(self)
if not os.path.exists(d):
os.makedirs(d)
logger.info(f"Saving Config to {d / 'config.conf'}")
with open(d / "config.conf", "w") as f:
f.write(HOCONConverter.to_hocon(self.config))
return True
def oci_image_bundle_conf(args, component_name, oci_manifest, oci_config):
conf = ConfigFactory.parse_string('')
load_bundle_args_into_conf(conf, args)
if 'annotations' in oci_manifest and oci_manifest[
'annotations'] is not None:
annotations_tree = conf.get('annotations')
for key in sorted(oci_manifest['annotations']):
annotations_tree.put(key, oci_manifest['annotations'][key])
endpoints_tree = ConfigTree()
oci_tree = ConfigTree()
oci_tree.put('description', args.component_description)
oci_tree.put('file-system-type', 'oci-image')
oci_tree.put('start-command', ['ociImageTag', args.tag])
oci_tree.put('endpoints', endpoints_tree)
components_tree = ConfigTree()
components_tree.put(component_name, oci_tree)
if args.use_default_endpoints and 'config' in oci_config and 'ExposedPorts' in oci_config[
'config']:
check_arguments = []
for exposed_port in sorted(oci_config['config']['ExposedPorts']):
type_parts = exposed_port.split('/', 1)
port = int(type_parts[0])
protocol = type_parts[1] if len(type_parts) > 1 else 'tcp'
name = '{}-{}-{}'.format(component_name, protocol, port)
check_arguments.append('${}_HOST'.format(
re.sub('\\W', '_', name.upper())))
entry_tree = ConfigTree()
entry_tree.put('bind-protocol', protocol)
entry_tree.put('bind-port', port)
entry_tree.put('service-name', name)
endpoints_tree.put(name, entry_tree)
if len(check_arguments) > 0:
oci_check_tree = ConfigTree()
oci_check_tree.put('description',
'Status check for oci-image component')
oci_check_tree.put('file-system-type', 'universal')
oci_check_tree.put('start-command', ['check'] + check_arguments)
oci_check_tree.put('endpoints', {})
components_tree = ConfigTree()
components_tree.put(component_name, oci_tree)
components_tree.put('{}-status'.format(component_name),
oci_check_tree)
conf.put('components', components_tree)
return HOCONConverter.to_hocon(conf)
def _load_custom_config(run_config):
"""Load custom configuration input HOCON file for cromwell.
"""
from pyhocon import ConfigFactory, HOCONConverter, ConfigTree
conf = ConfigFactory.parse_file(run_config)
out = {}
if "database" in conf:
out["database"] = HOCONConverter.to_hocon(ConfigTree({"database": conf.get_config("database")}))
return out
def parse(hocon_file):
"""
This function takes a hocon file
and returns a dictionary.
:param hocon_file: Path to hocon file.
:returns conf: Dictionary version of passed hocon file
"""
conf = ConfigFactory.parse_file(hocon_file)
conf = json.loads(HOCONConverter.convert(conf, 'json'))
return conf
def gen_conf(t1, t2, scale, app_name, query):
conf = ConfigFactory.parse_file('../conf/application.conf')
conf.put("Q23.table-list", [t1, t2])
conf.put("all.data-scale", scale)
conf.put("all.hdfs", 'hdfs://%s:8020/'%HDFS)
conf.put("all.app-suffix", app_name)
conf.put("Q23.query", QUERYS[query])
with open('../conf/application-run.conf', 'w') as f:
f.write(HOCONConverter.convert(conf, 'hocon'))
def run_one_task(self, queue: Text, task_id: Text, worker_args=None):
task_data = self._session.api_client.tasks.get_all(id=[task_id])[0]
# push task into the k8s queue, so we have visibility on pending tasks in the k8s scheduler
try:
self._session.api_client.tasks.enqueue(
task_id,
queue=self.k8s_pending_queue_name,
status_reason='k8s pending scheduler')
except Exception as e:
self.log.error(
"ERROR: Could not push back task [{}] to k8s pending queue [{}], error: {}"
.format(task_id, self.k8s_pending_queue_name, e))
return
if task_data.execution.docker_cmd:
docker_image = task_data.execution.docker_cmd
else:
docker_image = str(
os.environ.get("TRAINS_DOCKER_IMAGE")
or self._session.config.get("agent.default_docker.image",
"nvidia/cuda"))
# take the first part, this is the docker image name (not arguments)
docker_image = docker_image.split()[0]
create_trains_conf = "echo '{}' >> ~/trains.conf && ".format(
HOCONConverter.to_hocon(self._session.config._config))
if callable(self.kubectl_cmd):
kubectl_cmd = self.kubectl_cmd(task_id, docker_image, queue,
task_data)
else:
kubectl_cmd = self.kubectl_cmd.format(task_id=task_id,
docker_image=docker_image,
queue_id=queue)
# make sure we gave a list
if isinstance(kubectl_cmd, str):
kubectl_cmd = kubectl_cmd.split()
kubectl_cmd += [
"--labels=TRAINS=agent", "--command", "--", "/bin/sh", "-c",
create_trains_conf + self.container_bash_script.format(task_id)
]
process = subprocess.Popen(kubectl_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, error = process.communicate()
self.log.info("K8s scheduling experiment task id={}".format(task_id))
if error:
self.log.error("Running kubectl encountered an error: {}".format(
error if isinstance(error, str) else error.decode()))
def _to_config_object(self, s) -> 'Configuration':
if isinstance(s, ConfigTree):
if self.__use_json:
return Configuration(
json.loads(HOCONConverter.convert(s, 'json')))
return Configuration(s)
elif isinstance(s, str):
fallback_config = ConfigFactory.from_dict({})
res = ConfigFactory.parse_string(
s, resolve=False).with_fallback(fallback_config)
if self.__use_json:
return json.loads(HOCONConverter.convert(res, 'json'))
return Configuration(res)
elif isinstance(s, bytes):
s = s.decode()
fallback_config = ConfigFactory.from_dict({})
res = ConfigFactory.parse_string(
s, resolve=False).with_fallback(fallback_config)
if self.__use_json:
return Configuration(
json.loads(HOCONConverter.convert(res, 'json')))
return Configuration(res)
raise ValueError("Cannot parse value", s)
def append(self, c: Union['Configuration', Dict]) -> 'Configuration':
try:
return super(ConfigurationHocon, self).append(c)
except Exception as e:
fallback_config = ConfigFactory.from_dict(self.as_dict())
res = ConfigFactory.parse_string(
json.dumps(c.as_dict()),
resolve=True).with_fallback(fallback_config)
if self.__use_json:
_c = Configuration(
json.loads(HOCONConverter.convert(res, 'json')))
return super(ConfigurationHocon, self).append(_c)
_c = Configuration(res)
return super(ConfigurationHocon, self).append(_c)
def merge(self, b):
if isinstance(b, HOCONString):
d = b.to_dict()
elif isinstance(b, str):
d = HOCONString(b).to_dict()
elif isinstance(b, dict):
d = b
else:
raise TypeError('Unsupported type {t}'.format(t=type(b)))
self_d = self.to_dict()
merge_dict(self_d, d)
hocon = ConfigFactory.from_dict(self_d)
return self._include + '\n' + HOCONConverter.to_hocon(hocon)
def from_dict(cls, d, include=''):
"""Create HOCONString from dict.
Args:
include:
`include` statement to be added to the top of the HOCONString.
"""
hocon = ConfigFactory.from_dict(d)
hocon_str = HOCONConverter.to_hocon(hocon)
if include:
if not is_valid_include(include):
raise ValueError(
'Wrong HOCON include format. {include}'.format(
include=include))
hocon_str = NEW_LINE.join([include, hocon_str])
return cls(hocon_str=hocon_str)
def to_dict(self, with_include=True):
"""Convert HOCON string into dict.
Args:
with_include:
If True then double-quote-escaped `include` statements will be kept as a plain string
under key HOCONSTRING_INCLUDE_KEY.
Otherwise, `include` statements will be excluded.
"""
if with_include:
hocon_str = self._hocon_str
else:
hocon_str = self.get_contents(with_include=False)
c = ConfigFactory.parse_string(hocon_str)
j = HOCONConverter.to_json(c)
return json.loads(j)
def load_config(cwd=os.getcwd(), debug=False):
"""
Tries to find HOCON files named "iss4e.conf" using the paths returned by find_files().
The found files are then parsed and merged together, so that a single configuration dict is returned.
For details on HOCON syntax, see https://github.com/chimpler/pyhocon and https://github.com/typesafehub/config/
Example configuration:
- default config in home dir (~/iss4e.conf):
datasources {
influx {
host = ${HOSTNAME}
# also set your passwords (e.g. from env with ${MYSQL_PASSWD} here
}
mysql {
host = localhost
}
}
- local config in cwd (./iss4e.conf):
webike {
# use the generic information from ${datasources.influx} (should be defined in ~/iss4e.conf and contain
# host, password, ...) and extend it to use the (non-generic) database "webike"
influx = ${datasources.influx} {
db = "webike"
}
}
- merged config that will be returned:
{
"datasources": {
"influx": {
"host": "SD959-LT"
},
"mysql": {
"host": "localhost"
}
},
"webike": {
"influx": {
"host": "SD959-LT", # copied from ~/iss4e.conf: datasources.influx
"db": "webike"
}
}
}
"""
# find "iss4e.conf" file in current working dir or parent directories
files = find_files("iss4e.conf", cwd)
configs = [
ConfigFactory.parse_file(file, required=False, resolve=False)
for file in files if os.path.isfile(file)
]
if debug:
print("Config files:\n" +
"\n".join(file + " [" +
("not " if not os.path.isfile(file) else "") + "found]"
for file in files))
# merge all levels of config
config = ConfigTree(root=True)
config.put(
"__main__",
os.path.basename(
getattr(sys.modules['__main__'], "__file__", "__cli__")))
config.put("__cwd__", os.path.abspath(cwd))
for c in configs:
config = ConfigTree.merge_configs(c, config)
config = ConfigParser.resolve_substitutions(config)
if debug:
print("Loaded config:\n" + HOCONConverter.to_json(config))
# if config contains a key "logging", use it to reconfigure python logging
if "logging" in config:
if debug:
print("Reconfiguring logging from config")
if config.get("capture_exceptions", True):
sys.excepthook = log_uncaught_exception
logging.captureWarnings(config.get("capture_warnings", True))
logging.config.dictConfig(config["logging"].as_plain_ordered_dict())
# check python version
# iss4e lib is using some syntax features and functions which were only introduced in python 3.5
rec_ver = tuple(config.get("min_py_version", [3, 5]))
if sys.version_info < rec_ver:
warnings.warn(
"Using outdated python version {}, a version >= {} would be recommended for use with iss4e lib. "
"Try using a newer python binary, e.g. by calling `python{}.{}` instead of the default `python`."
.format(sys.version_info, rec_ver, rec_ver[0], rec_ver[1]))
return config
env_dict = parse_env_variables(env_dict)
nested_set(env_dict, ['waves', 'directory'], '/waves')
nested_set(env_dict, ['waves', 'data-directory'], '/waves/data')
nested_set(env_dict, ['waves', 'wallet', 'seed'], wallet_data['seed'])
nested_set(env_dict, ['waves', 'wallet', 'password'],
wallet_data['password'])
WAVES_AUTODETECT_ADDRESS = os.getenv('WAVES_AUTODETECT_ADDRESS',
DEFAULT_AUTODETECT)
WAVES_DECLARED_ADDRESS = os.getenv('WAVES_DECLARED_ADDRESS')
print("WAVES_DECLARED_ADDRESS", WAVES_DECLARED_ADDRESS)
if WAVES_AUTODETECT_ADDRESS.lower(
) == 'yes' and WAVES_DECLARED_ADDRESS is None:
WAVES_DECLARED_ADDRESS = get_external_ip() + ':' + get_port_number(
NETWORK)
print("Detected address is " + WAVES_DECLARED_ADDRESS)
nested_set(env_dict, ['waves', 'network', 'declared-address'],
WAVES_DECLARED_ADDRESS)
elif WAVES_DECLARED_ADDRESS is not None:
nested_set(env_dict, ['waves', 'network', 'declared-address'],
WAVES_DECLARED_ADDRESS)
config = ConfigFactory.from_dict(env_dict)
local_conf = HOCONConverter.convert(config, 'hocon')
print(local_conf)
with open(LOCAL_FILE_PATH, 'w') as file:
file.write(local_conf)
download_jar_file(WAVES_VERSION, VERSION)
def to_str(self):
return HOCONConverter.convert(self._config_tree, 'hocon')
def write(self, stream, output_format='hocon'):
config = ConfigFactory.from_dict(self.as_dict())
res = HOCONConverter.convert(config,
output_format='hocon',
compact=True)
stream.write(res)
def print(self):
print()
print("Configuration:")
print(HOCONConverter.convert(self.data, 'hocon'))
print()
config["parameters"] = complete_parameters
template_without_dashes, parameters_without_dashes = convert_dashes_to_underscores(
template, config["parameters"], template_path)
if config['parameters'] != parameters_without_dashes:
parameters_changed = True
config['parameters'] = parameters_without_dashes
if parameters_changed:
print("Overwriting template.conf: %s" % config_path)
# We can use the HOCONConverter for dumping the json as well but we have more options to control
# the output with the default json writer
if opts.output_format == "hocon":
open(config_path,
"w+").write(HOCONConverter.convert(config, "hocon"))
else:
open(config_path, "w+").write(
# This is a config tree but json.dumps uses iterators to go over it thinking it is a dict.
# Since configtree already supports these iterators it works correctly for it.
json.dumps(config,
indent=2,
separators=(
',',
': ',
),
ensure_ascii=False) + "\n")
if template_without_dashes != template:
print("Overwriting template: %s" % template_path)
open(template_path, "w+").write(template_without_dashes)
def parse_fp(self, fp):
conf = ConfigFactory.parse_file(fp)
return json.loads(HOCONConverter.to_json(conf))
def to_dict(self):
"""Convert contents without include to dict.
"""
c = ConfigFactory.parse_string(self._contents_wo_include)
j = HOCONConverter.to_json(c)
return json.loads(j)
def to_json(self):
return json.loads(HOCONConverter.convert(self._config_tree, 'json'))
def parse_string(self, s):
conf = ConfigFactory.parse_string(s)
return json.loads(HOCONConverter.to_json(conf))
def save_config(args: Args, cfg: ConfigTree):
config_path = args.run_dir / 'config.json'
with open(config_path, 'w') as f:
f.write(HOCONConverter.to_json(cfg))
template_path, opts.parameter_type)
parameters_changed = False
if config["parameters"] != complete_parameters:
parameters_changed = True
config["parameters"] = complete_parameters
template_without_dashes, parameters_without_dashes = convert_dashes_to_underscores(template,
config["parameters"],
template_path)
if config['parameters'] != parameters_without_dashes:
parameters_changed = True
config['parameters'] = parameters_without_dashes
if parameters_changed:
print("Overwriting template.conf: %s" % config_path)
# We can use the HOCONConverter for dumping the json as well but we have more options to control
# the output with the default json writer
if opts.output_format == "hocon":
open(config_path, "w+").write(HOCONConverter.convert(config, "hocon"))
else:
open(config_path, "w+").write(
# This is a config tree but json.dumps uses iterators to go over it thinking it is a dict.
# Since configtree already supports these iterators it works correctly for it.
json.dumps(config, indent=2, separators=(',', ': ', ), ensure_ascii=False) + "\n"
)
if template_without_dashes != template:
print("Overwriting template: %s" % template_path)
open(template_path, "w+").write(template_without_dashes)
def saveHoconFile(pathFileNameExt, parameterDict):
paramHocon = ConfigFactory.from_dict(parameterDict)
paramStr = HOCONConverter.to_hocon(paramHocon,indent=4)
with open(pathFileNameExt, "w", encoding='utf-8') as fd:
fd.write(paramStr)
def to_str(self):
return HOCONConverter.convert(self._config_tree, 'hocon')
def dumps(obj: Any, **kwargs) -> str:
return HOCONConverter.to_json(obj, **kwargs)
def generate_compose(genesis_path):
if not os.path.isdir("/waves-mnt/output"):
os.mkdir("/waves-mnt/output")
compose_content = "version: '3'\n\nservices:"
nodes_list = []
known_peers_list = []
for k in range(0, accounts_count):
known_peers_list.append(f"node{k}:6864")
for k in range(0, accounts_count):
if k > 0:
if int(network_port) < 10000:
instance_network_port = f"{k}{network_port}"
instance_api_port = f"{k}{api_port}"
else:
instance_network_port = f"{str(int(network_port) + k)}"
instance_api_port = f"{str(int(api_port) + k)}"
else:
instance_network_port = network_port
instance_api_port = api_port
node_config_path = f"/waves-mnt/output/node{k}/configs"
if not os.path.isdir(node_config_path):
os.makedirs(node_config_path, exist_ok=True)
config_content = ConfigFactory.parse_file(genesis_path)
config_content['waves']['network'] = ConfigFactory.from_dict(
{'known-peers': known_peers_list})
config_content = HOCONConverter.convert(config_content, 'hocon')
config_path = f"/waves-mnt/output/node{k}/configs/local.conf"
with open(config_path, 'w') as config_file:
config_file.write(config_content)
nodes_list.append(f"node{k}:6869")
compose_content += f"""
node{k}:
hostname: node{k}
image: wavesplatform/node
environment:
- WAVES__NETWORK__NODE_NAME=node{k}
- WAVES_WALLET_SEED={accounts[k]['seed']}
- WAVES_WALLET_PASSWORD={accounts[k]['password']}
- WAVES_VERSION=latest
- WAVES_NETWORK={network}
- WAVES_LOG_LEVEL={log_level}
- WAVES_HEAP_SIZE={heap_size}
- WAVES_AUTODETECT_ADDRESS={autodetect_address}
- WAVES_AUTODETECT_ADDRESS_PORT={instance_network_port}
- WAVES__NETWORK__DECLARED_ADDRESS=node{k}:6864
- WAVES__REST_API__ENABLE={WAVES__REST_API__ENABLE}
- WAVES__REST_API__PORT=6869
- WAVES__REST_API__API_KEY_HASH=3z7LKtx9DwNVtEZ2whjZqrdZH8iWTZsdQYwWbgsxeXhm
- WAVES__BLOCKCHAIN__CUSTOM__ADDRESS_SCHEME_CHARACTER={network_byte}
- WAVES__MINER__QUORUM={accounts_count - 1}
- WAVES__MINER__MIN_MICRO_BLOCK_AGE=2s
- WAVES__MINER__MICRO_BLOCK_INTERVAL=3s
- WAVES__MINER__INTERVAL_AFTER_LAST_BLOCK_THEN_GENERATION_IS_ALLOWED=999d
restart: always
networks:
default:
aliases:
- node{k}
- node{k}.waves
ports:
- "{instance_api_port}:6869"
- "{instance_network_port}:6864"
volumes:
- ./node{k}:/waves"""
compose_content = append_data_service(compose_content, nodes_list)
compose_path = "/waves-mnt/output/docker-compose.yml"
with open(compose_path, 'w') as compose_file:
compose_file.write(compose_content)
def from_dict(cls, d, include=''):
hocon = ConfigFactory.from_dict(d)
hocon_str = HOCONConverter.to_hocon(hocon)
if include:
hocon_str = include + '\n' + hocon_str
return cls(hocon_str=hocon_str)
