def load_from_folder(config_folder_location: str) -> Configuration:
"""Return a configuration based on a folder location.
:param config_folder_location:
String of folder path to be read.
:returns:
Configuration object.
"""
logger = logging.getLogger(__name__)
logger.info(
f'Attempting to load config from folder {config_folder_location}')
if os.path.isdir(config_folder_location) is False:
logger.critical(f'{config_folder_location} is not a directory')
raise ValueError('Invalid configuration folder specified.')
os.chdir(config_folder_location)
files = glob.glob('*.yaml')
files.extend(glob.glob('*.yml'))
files = set(files) - set(glob.glob('*example*'))
for file in files:
logger.info(f'Found file {file}')
# Forcing INFO level logging here, debug will print file contents which might leak secrets
conf = hiyapyco.load(list(files),
method=hiyapyco.METHOD_MERGE,
mergelists=False,
loglevel='INFO')
configuration = _build_configuration(yaml.safe_load(hiyapyco.dump(conf)))
return configuration
def dump_configs(self, out_dir=None, with_json=False):
if not self.config:
logger.error("No configurations found")
return
if not out_dir:
for filename, config in self.config.items():
logger.info("{}:\n---\n{}".format(
filename, odyldo.safe_dump(config,
default_flow_style=False)))
if with_json:
logger.info("{}\n".format(json.dumps(config)))
else:
for filename, config in self.config.items():
output_name = os.path.sep.join([out_dir, filename])
output_dir = os.path.dirname(output_name)
mkdir_p(output_dir)
with open(output_name, 'w') as f:
logger.info(output_name)
f.write('---\n')
f.write(hiyapyco.dump(config, default_flow_style=False))
json_output_name = YAML_SUFFIX_PATTERN.sub(
'.json', output_name)
if with_json:
with open(json_output_name, 'w') as f:
logger.info(json_output_name)
json.dump(config, f)
def yaml_dump_to_file(input, path):
with open(path, "w") as fp:
fp.write(hiyapyco.dump(input))
if log_level == "debug" or log_level == "DEBUG":
with open(path, 'r') as f:
print("[yaml-merge][yaml_dump_to_file][debug] content : {}".format(
f.read()))
def save_config(config, path=local_config_path):
""" saves a CONFIG dict into disk """
try:
with open(path, "w") as f:
f.write(hiyapyco.dump(config))
except Exception:
print("No access to local configuration file {}".format(
local_config_path))
def merge_profile(ndis_id):
filename = f"{ndis_id}.yaml"
local_filepath = os.path.join(WORKING_DIR, "profiles", filename)
remote_filepath = os.path.join(
WORKING_DIR, "remote",
filename) # TODO(divv) This path needs to be configurable
with open(local_filepath, 'r') as f:
local_yaml = f.read()
logger.debug(f"**** LOCAL YAML ****:\n{local_yaml}")
if os.path.isfile(remote_filepath):
logger.info("Synchronising profiles")
with open(remote_filepath, 'r') as f:
remote_yaml = f.read()
logger.debug(f"**** REMOTE YAML ****:\n{remote_yaml}")
merged_hypc = hiyapyco.load([remote_yaml, local_yaml],
method=hiyapyco.METHOD_MERGE,
loglevel=logging.INFO,
mergelists=False)
merged_yaml = hiyapyco.dump(merged_hypc)
logger.debug(f"**** MERGED YAML ****:\n{merged_yaml}")
merged_dict = yaml.load(merged_yaml)
for survey, risks in merged_dict["surveys"].items():
logger.debug(f"Survey: {survey}")
logger.debug(f"Risks: {risks}")
for risk, observations in risks.items():
logger.debug(f"Risk: {risk}")
logger.debug(f"observations: {observations}")
deduped_observations = dedupe_list_of_dicts(observations)
logger.debug(f"Deduped observations: {deduped_observations}")
merged_dict["surveys"][survey][risk] = deduped_observations
logger.debug(f"Overwriting local and remote profiles for {ndis_id}")
with open(local_filepath, 'w') as f:
f.write(yaml.dump(merged_dict))
with open(remote_filepath, 'w') as f:
f.write(yaml.dump(merged_dict))
# cleanup
del (remote_yaml)
del (merged_dict)
del (merged_hypc)
del (merged_yaml)
logger.debug("deleted remote, merged dictionaries")
else: # This is a new profile that does not yet exist on the remote server
with open(remote_filepath, 'w') as f:
f.write(local_yaml)
# cleanup
del (local_yaml)
logger.debug("deleted local dictionary")
def merge_yaml(self,
out_file,
generated_yaml,
yaml_path,
yaml_mask='*.yml'):
print ">> Merge yaml files : gen:%s + inp:%s/%s => %s" % (
generated_yaml, yaml_path, yaml_mask, out_file)
yaml_list = []
for fname in glob.glob(os.path.join(yaml_path, yaml_mask)):
with open(fname) as fp:
yaml_file = fp.read()
yaml_list.append(yaml_file)
yaml_list.append(generated_yaml)
merged_yaml = hiyapyco.load(yaml_list,
method=hiyapyco.METHOD_MERGE)
#print(hiyapyco.dump(merged_yaml))
domain = open(out_file, "w+")
domain.writelines(hiyapyco.dump(merged_yaml))
def parseTemplate(self, data, offset):
y = template.from_buffer(data, offset)
self.templates[y.ID] = {}
self.templates[y.ID]["struct"] = None
self.templates[y.ID]["spec"] = []
self.templates[y.ID]["yaml"] = None
offset += 4
fieldList = []
fieldSpecList = []
dFields = collections.OrderedDict()
dFields[y.ID] = collections.OrderedDict()
for x in xrange(1, y.fieldCount + 1):
z = fieldSpec.from_buffer(data, offset)
m = nfTypes.metaIE.registry[z.fieldType](z.fieldLength)
dFields[y.ID]["Field" + str(x)] = collections.OrderedDict([
("elementID", z.fieldType), ("name", m.nf9name),
("length", z.fieldLength)
])
fieldSpecList.append(m)
fieldList.append((m.nf9name, m.seeType))
offset += 4
newdump = hiyapyco.dump(dFields)
if y.ID in self.templates:
if newdump == self.templates[y.ID]["yaml"]:
return
self.templates[y.ID]["struct"] = type(str(y.ID),
(ctypes.BigEndianStructure, ), {
"_pack_": 1,
"_fields_": fieldList
})
self.templates[y.ID]["spec"] = fieldSpecList
self.templates[y.ID]["yaml"] = newdump
return y.ID
def load_config_files(*files):
"""Load and merge YAML config files.
Files that come earlier in the list take precedence over files
that come later in the list.
Args:
*files (list) : Variable number of file paths.
Example::
load_config_files(file1, file2, file3, ...):
"""
files = [file for file in files if file is not None and file != '']
for file in files:
sys.path.insert(0, os.path.dirname(file))
LOG.debug('Loading config files: {}'.format(files))
# hiyapyco merges in order least important to most important
files.reverse()
expanded_files = process_templates(files)
hiyapyco.jinja2env = NativeEnvironment(variable_start_string='(',
variable_end_string=')',
undefined=LogUndefined)
cfg_dict = hiyapyco.load(expanded_files,
method=hiyapyco.METHOD_MERGE,
interpolate=True,
failonmissingfiles=True)
if LOG.getEffectiveLevel() == logLevelFromName("DEBUG"):
LOG.debug("Merged YAML config:\n\n%s\n",
hiyapyco.dump(cfg_dict, default_flow_style=False))
return cfg_dict
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='Modify Conan settings.yml configuration')
parser.add_argument('--merge-file',
nargs='*',
default=[],
help='YAML config file to merge')
parser.add_argument('--method', default='METHOD_MERGE')
bool_arg(parser, 'mergelists', True)
bool_arg(parser, 'interpolate', False)
bool_arg(parser, 'castinterpolated', False)
bool_arg(parser, 'usedefaultyamlloader', False)
bool_arg(parser, 'failonmissingfiles', True)
args = parser.parse_args()
in_data = get_stdin() or ''
in_data += "\n" # newline is used to distinguish yaml from filename
output = ConanOutput(sys.stdout, sys.stderr, True)
conan_cache = cache.ClientCache(
os.path.join(get_conan_user_home(), '.conan'), output)
path = conan_cache.settings_path
existing = cache.load(path) \
if os.path.exists(path) \
else default_settings()
method = hiyapyco.METHODS[args.method]
settings = hiyapyco.load([existing, in_data],
*args.merge_file,
mergelists=args.mergelists,
method=method,
interpolate=args.interpolate,
castinterpolated=args.castinterpolated,
usedefaultyamlloader=args.usedefaultyamlloader,
failonmissingfiles=args.failonmissingfiles)
settings_yml = hiyapyco.dump(settings)
cache.save(path, normalize(settings_yml), only_if_modified=True)
def parseTemplate(self, data, offset):
y = template.from_buffer(data, offset)
self.templates[y.ID] = {}
self.templates[y.ID]["struct"] = None
self.templates[y.ID]["spec"] = []
self.templates[y.ID]["yaml"] = None
offset += 4
fieldList = []
fieldSpecList = []
dFields = collections.OrderedDict()
dFields[y.ID] = collections.OrderedDict()
for x in xrange(1, y.fieldCount + 1):
z = fieldSpec.from_buffer(data, offset)
m = nfTypes.metaIE.registry[z.fieldType](z.fieldLength)
dFields[y.ID]["Field" + str(x)] = collections.OrderedDict(
[("elementID", z.fieldType), ("name", m.nf9name), ("length", z.fieldLength)]
)
fieldSpecList.append(m)
fieldList.append((m.nf9name, m.seeType))
offset += 4
newdump = hiyapyco.dump(dFields)
if y.ID in self.templates:
if newdump == self.templates[y.ID]["yaml"]:
return
self.templates[y.ID]["struct"] = type(
str(y.ID), (ctypes.BigEndianStructure,), {"_pack_": 1, "_fields_": fieldList}
)
self.templates[y.ID]["spec"] = fieldSpecList
self.templates[y.ID]["yaml"] = newdump
return y.ID
basepath = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.dirname(basepath))
import hiyapyco
print('='*10, 'method=hiyapyco.METHOD_SIMPLE', '='*10)
conf = hiyapyco.load(
os.path.join(basepath, 'list1.yaml'),
os.path.join(basepath, 'list2.yaml'),
method=hiyapyco.METHOD_SIMPLE,
failonmissingfiles=True
)
print(conf)
print('-'*10, 'YAML', '-'*10)
print(hiyapyco.dump(conf))
print('='*10, 'method=hiyapyco.METHOD_MERGE', '='*10)
conf = hiyapyco.load(
os.path.join(basepath, 'list1.yaml'),
os.path.join(basepath, 'list2.yaml'),
method=hiyapyco.METHOD_MERGE,
failonmissingfiles=True
)
print(conf)
print('-'*10, 'YAML', '-'*10)
print(hiyapyco.dump(conf))
assert conf == {'l': {'a': ['laa', 'lab', 'lac', 'lad'], 'b': ['lba', 'lbb', 'lcc', 'lbd'], 'c': [{'csub': ['lcc1', 'lcc2', 'lcc4', 'lcc5']}, 'lcc 2 only', {'csub2': ['csub2 1', 'csub2 2']}, 'lca', 'lcb', 'lcc', 'lcd']}}
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 smartindent nu
import hiyapyco
options = {"swagger_ui": True} # turn off ui
connex_app = connexion.FlaskApp(__name__, specification_dir='swagger-doc/', options=options)
swagger_file = hiyapyco.load(
[
# main
'swagger-doc/main.yaml',
'swagger-doc/auth.yaml',
'swagger-doc/upload.yaml',
# swagger-doc
'swagger-doc/user.yaml',
'swagger-doc/account.yaml',
'swagger-doc/product.yaml',
'swagger-doc/type_member.yaml',
'swagger-doc/bill.yaml',
'swagger-doc/bill_detail.yaml',
'swagger-doc/money_to_point.yaml',
'swagger-doc/point_to_money.yaml',
'swagger-doc/extract_point.yaml',
'swagger-doc/cart.yaml',
'swagger-doc/my_order.yaml',
],
method=hiyapyco.METHOD_MERGE
)
swagger_file_dumped = hiyapyco.dump(swagger_file)
#!/usr/bin/env python
import hiyapyco
import re
# Currently the entrypoint.sh uses arguments that are ignored in this file.
# https://github.com/zerwes/hiyapyco/blob/master/examples/hiyapyco_example.py
merged_yaml = hiyapyco.load('/etc/home-assistant/default-configuration.yaml',
'/config/custom-configuration.yaml',
method=hiyapyco.METHOD_MERGE)
config = re.sub(pattern=r'\'!(.*)\'',
repl='!\\1',
string=hiyapyco.dump(merged_yaml))
with open("/config/configuration.yaml", "w") as configuration:
configuration.write(
"# DO NOT EDIT THIS FILE. It will be regenerated upon container startup. Instead, edit /config/custom-configuration.yaml. \n"
)
configuration.write(config)
def dump(self, defaultFlowStyle=True):
r"""Returns the YAML dump.
"""
return dump(self.Data, defaultFlowStyle)
import hiyapyco
import sys
import os
import glob
basepath = os.path.dirname(os.path.realpath(__file__))
hiyapyco._usedefaultyamlloader = True
src_path = sys.argv[1]
merged_file_path = sys.argv[2]
stacks = os.path.join(src_path, '*.yaml')
print(stacks)
conf = hiyapyco.load(glob.glob(stacks), method=hiyapyco.METHOD_MERGE)
print(hiyapyco.dump(conf))
with open(merged_file_path, 'w') as yaml_file:
hiyapyco.dump(conf, stream=yaml_file)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import hiyapyco
import os
import sys
# load the yaml files
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
with open(sys.argv[1]) as fp:
yaml1 = fp.read()
with open(sys.argv[2]) as fp:
yaml2 = fp.read()
merged_yaml = hiyapyco.load([yaml1, yaml2], method=hiyapyco.METHOD_MERGE)
out_file = open("/buildout/merged.yml", "w")
out_file.write(hiyapyco.dump(merged_yaml))
out_file.close()
def combinate_requirements(ibis, ci, res):
merged_yaml = hiyapyco.load([ibis, ci], method=hiyapyco.METHOD_MERGE)
with open(res, "w") as f_res:
hiyapyco.dump(merged_yaml, stream=f_res)
# until the `extennd` will not be supported without the "working in my computer, but not working in the server", here we should run this instead
import hiyapyco
import os
COMMON_SERVICES_FILE_PATH="common-services.yaml"
directory = r'./'
for entry in os.scandir(directory):
if entry.path.startswith("./observer-") is False:
continue
base_docker_compose_path = "/".join([entry.path, "base-docker-compose.yaml"])
print(base_docker_compose_path)
docker_compose_conf = hiyapyco.load(COMMON_SERVICES_FILE_PATH, base_docker_compose_path, method=hiyapyco.METHOD_MERGE)
base_docker_compose_conf = hiyapyco.load(base_docker_compose_path)
base_services = base_docker_compose_conf["services"]
to_remove = []
for key, value in docker_compose_conf["services"].items():
if key not in base_services.keys():
to_remove.append(key)
if "extends" in value.keys():
del value["extends"]
for not_included_service in to_remove:
del docker_compose_conf["services"][not_included_service]
target_docker_compose_path = "/".join([entry.path, "docker-compose.yaml"])
with open(target_docker_compose_path, 'w') as yaml_file:
yaml_file.write(hiyapyco.dump(docker_compose_conf, default_flow_style=False))
# print(hiyapyco.dump(conf, default_flow_style=False))
base = """
key_one: 1
key_two: 2
array_of_dicts:
- dict_key_one: a
dict_key_two: b
dict_key_three: c
- dict_key_one: a1
dict_key_two: b1
dict_key_three: c1
"""
layer = """
key_two: 2222
array_of_dicts:
- dict_key_one: a2
dict_key_two: b2
dict_key_three: c2
"""
CONF = hiyapyco.load([base, layer], method=hiyapyco.METHOD_MERGE)
print(hiyapyco.dump(CONF))
print("... using mergelists=False ...")
CONF = hiyapyco.load([base, layer],
method=hiyapyco.METHOD_MERGE,
mergelists=False)
print(hiyapyco.dump(CONF))
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 smartindent nu
import os
import glob
import hiyapyco
path = 'data/domain'
yaml_list = []
for filename in glob.glob(os.path.join(path, '*.yml')):
with open(filename, encoding='utf-8') as fp:
yaml_file = fp.read()
yaml_list.append(yaml_file)
merged_yaml = hiyapyco.load(yaml_list, method=hiyapyco.METHOD_MERGE)
print(hiyapyco.dump(merged_yaml))
domain_yml_file = open("../domain.yml", "w+")
domain_yml_file.writelines(hiyapyco.dump(merged_yaml))
level=logging.DEBUG,
format='%(levelname)s\t[%(name)s] %(funcName)s: %(message)s'
)
print('='*10, 'method=hiyapyco.METHOD_SIMPLE', '='*10)
conf = hiyapyco.load(
os.path.join(basepath, 'listl31.yaml'),
os.path.join(basepath, 'listl32.yaml'),
failonmissingfiles=True
)
print('-'*10, 'DATA', '-'*10)
print(conf)
print('-'*10, 'PPRINT', '-'*10)
pprint.PrettyPrinter(indent=4).pprint(conf)
print('-'*10, 'YAML', '-'*10)
print(hiyapyco.dump(conf))
print('-'*10, 'EOF', '-'*10)
assert conf['l3'] == [{'a': 'aaa', 'c': 'c'}, {'b': 'BBB', 'c': 'CCC'}, {'a': 'x', 'b': 'y', 'c': 'z'}]
print('='*10, 'method=hiyapyco.METHOD_MERGE', '='*10)
conf = hiyapyco.load(
os.path.join(basepath, 'listl31.yaml'),
os.path.join(basepath, 'listl32.yaml'),
method=hiyapyco.METHOD_MERGE,
failonmissingfiles=True
)
print('-'*10, 'DATA', '-'*10)
print(conf)
print('-'*10, 'PPRINT', '-'*10)
pprint.PrettyPrinter(indent=4).pprint(conf)
print('-'*10, 'YAML', '-'*10)
if merge_files :
logger.info("configuring "+ f + " with files: "+str(merge_files))
merged_f = hiyapyco.load(
*merge_files,
interpolate=True,
method=hiyapyco.METHOD_MERGE,
failonmissingfiles=True
)
logger.info("merged "+f+" yaml-->"+str(merged_f)+"<--")
outfile = os.path.basename(f)
target = os.path.join(spack_config_dir, outfile)
logger.info(" output config_file " + outfile + "<-- ")
if not os.path.exists(target):
out=hiyapyco.dump(merged_f, default_flow_style=False)
templ = mytemplate.stringtemplate(out)
out = mytemplate.stringtemplate(out).safe_substitute(subst)
logger.info("WRITING config_file " + outfile + " -->" + target + "<-- ")
open(target, "w").write(out)
else :
logger.info("no template file for "+ f + " : skipping ")
util.source(os.path.join(dest,'share','spack','setup-env.sh'))
if args.runconfig :
for p in reversed(config_path_list):
initfile=os.path.join(p,'config.sh')
if os.path.exists(initfile):
logger.info("parsing init file-->" + initfile + "<-- ")
import hiyapyco
with open('./jupyterhub/values.yaml.orig') as fp:
values = fp.read()
with open('./config.yaml') as fp:
config = fp.read()
merged_yaml = hiyapyco.load([values, config], method=hiyapyco.METHOD_MERGE)
print(hiyapyco.dump(merged_yaml))
def generate_robot_launch_description(robot_namespace: str, simulation=False):
"""Generate robot launch description based on namespace."""
map = LaunchConfiguration("map")
namespace = LaunchConfiguration("namespace")
use_namespace = LaunchConfiguration("use_namespace")
use_sim_time = LaunchConfiguration("use_sim_time")
autostart = LaunchConfiguration("autostart")
default_bt_xml_filename = LaunchConfiguration("default_bt_xml_filename")
params = tempfile.NamedTemporaryFile(mode="w", delete=False)
robot_params = os.path.join(get_package_share_directory(robot_namespace),
"param", f"{robot_namespace}.yml")
navigation_params = os.path.join(get_package_share_directory("robot"),
"param", "robot.yml")
merged_params = hiyapyco.load([navigation_params, robot_params],
method=hiyapyco.METHOD_MERGE)
params.file.write(hiyapyco.dump(merged_params))
urdf = os.path.join(get_package_share_directory(robot_namespace), "robot",
f"{robot_namespace}.urdf")
robot_launch_file_dir = os.path.dirname(__file__)
nav2_bt_xml_file = os.path.join(
get_package_share_directory("robot"),
"behavior_trees",
"navigate_w_replanning_time.xml",
)
remappings = [("/tf", "tf"), ("/tf_static", "tf_static")]
return launch.LaunchDescription([
DeclareLaunchArgument(
"namespace",
default_value=robot_namespace,
description="Robot global namespace",
),
DeclareLaunchArgument(
"use_namespace",
default_value="true",
description=
"Whether to apply a namespace to the robot stack including navigation2",
),
DeclareLaunchArgument("autostart",
default_value="true",
description="Autostart the nav stack"),
DeclareLaunchArgument("use_sim_time",
default_value="false",
description="Use /clock if true"),
DeclareLaunchArgument(
"default_bt_xml_filename",
default_value=nav2_bt_xml_file,
description="Full path to the behavior tree xml file to use",
),
GroupAction([
PushRosNamespace(condition=IfCondition(use_namespace),
namespace=namespace),
Node(
package="lcd_driver",
executable="lcd_driver",
output="screen",
parameters=[params.name],
remappings=remappings,
),
Node(
package="robot_state_publisher",
executable="robot_state_publisher",
output="screen",
parameters=[params.name],
arguments=[urdf],
remappings=remappings,
),
Node(
package="localisation",
executable="localisation",
output="screen",
parameters=[params.name],
remappings=remappings,
),
Node(
package="cetautomatix",
executable="cetautomatix",
output="screen",
parameters=[params.name],
remappings=remappings,
),
Node(
package="teb_obstacles",
executable="teb_obstacles",
output="screen",
parameters=[],
),
IncludeLaunchDescription(
PythonLaunchDescriptionSource(
[robot_launch_file_dir, "/navigation_launch.py"]),
launch_arguments={
"autostart": autostart,
"namespace": namespace,
"use_sim_time": use_sim_time,
"default_bt_xml_filename": default_bt_xml_filename,
"params_file": params.name,
}.items(),
),
]),
IncludeLaunchDescription(
PythonLaunchDescriptionSource(
[robot_launch_file_dir, "/interfaces.py"]),
condition=IfCondition(str(not simulation)),
launch_arguments={
"namespace": namespace,
"use_namespace": use_namespace,
"use_sim_time": use_sim_time,
"params_file": params.name,
}.items(),
),
])
base = """
key_one: 1
key_two: 2
array_of_dicts:
- dict_key_one: a
dict_key_two: b
dict_key_three: c
- dict_key_one: a1
dict_key_two: b1
dict_key_three: c1
"""
layer = """
key_two: 2222
array_of_dicts:
- dict_key_one: a2
dict_key_two: b2
dict_key_three: c2
"""
CONF = hiyapyco.load([base, layer], method=hiyapyco.METHOD_MERGE)
print (hiyapyco.dump(CONF))
print ("... using mergelists=False ...")
CONF = hiyapyco.load([base, layer], method=hiyapyco.METHOD_MERGE, mergelists=False)
print (hiyapyco.dump(CONF))
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 smartindent nu