def load_config():
YamlIncludeConstructor.add_to_loader_class(loader_class=yaml.FullLoader)
with open("config.yaml", "r") as file:
yaml_as_dictionary = yaml.load(file, Loader=yaml.FullLoader)
return from_dict(data_class=Config, data=yaml_as_dictionary)
def main(prog_args):
YamlIncludeConstructor.add_to_loader_class(loader_class=yaml.FullLoader)
print('read_config...')
dtp_config = read_config(prog_args)
print(dtp_config)
print('load_driver...')
dtp_driver = load_driver(dtp_config)
print(dtp_driver)
print('load_data_source...')
data_source = load_data_source(dtp_config, dtp_driver)
print(list(data_source))
print('transform_data_source...')
pygm_source = transform_data_source(dtp_config, data_source)
print(pygm_source)
print('translate_into_yaml...')
pygm_yaml = translate_into_yaml(dtp_config, pygm_source)
print(pygm_yaml)
print('process_info_schema...')
metadata_formatted = process_info_schema(dtp_config, pygm_yaml)
print(metadata_formatted)
print('Exiting...')
pass
def load_yaml_data(conf_path: Path, quiet: bool = False) -> AnyByStrDict:
"""Load the `copier.yml` file.
This is like a simple YAML load, but applying all specific quirks needed
for [the `copier.yml` file][the-copieryml-file].
For example, it supports the `!include` tag with glob includes, and
merges multiple sections.
Params:
conf_path: The path to the `copier.yml` file.
quiet: Used to configure the exception.
Raises:
InvalidConfigFileError: When the file is formatted badly.
"""
YamlIncludeConstructor.add_to_loader_class(loader_class=yaml.FullLoader,
base_dir=conf_path.parent)
try:
with open(conf_path) as f:
flattened_result = deepflatten(
yaml.load_all(f, Loader=yaml.FullLoader),
depth=2,
types=(list, ),
)
# HACK https://bugs.python.org/issue32792#msg311822
# I'd use ChainMap, but it doesn't respect order in Python 3.6
result = {}
for part in flattened_result:
result.update(part)
return result
except yaml.parser.ParserError as e:
raise InvalidConfigFileError(conf_path, quiet) from e
def load_manifest_from_file(filename: str, type: Type[T]) -> T:
_logger.debug("Loading manifest file (%s)", filename)
filepath = os.path.abspath(filename)
_logger.debug("filepath: %s", filepath)
filedir: str = os.path.dirname(filepath)
utils.print_dir(dir=filedir)
YamlIncludeConstructor.add_to_loader_class(
loader_class=yaml.SafeLoader, base_dir=filedir)
_add_ssm_param_injector()
_add_env_var_injector()
with open(filepath, "r") as f:
raw: Dict[str, Any] = cast(Dict[str, Any], yaml.safe_load(f))
_logger.debug("raw: %s", raw)
if type is Manifest:
raw["SsmParameterName"] = f"/orbit/{raw['Name']}/manifest"
manifest: T = cast(
T,
Manifest.Schema().load(data=raw,
many=False,
partial=False,
unknown=EXCLUDE))
elif type is FoundationManifest:
raw["SsmParameterName"] = f"/orbit-f/{raw['Name']}/manifest"
manifest = cast(
T,
FoundationManifest.Schema().load(data=raw,
many=False,
partial=False,
unknown=EXCLUDE))
else:
raise ValueError("Unknown 'manifest' Type")
ManifestSerDe.dump_manifest_to_ssm(manifest=manifest)
return manifest
def _load_yaml(self, file_yaml):
""" Load a yaml file """
yaml_dir = os.path.dirname(file_yaml)
self._debug(f"Loading YAML file '{file_yaml}', base dir '{yaml_dir}'")
YamlIncludeConstructor.add_to_loader_class(
loader_class=yaml.FullLoader, base_dir=yaml_dir)
with open(file_yaml) as yaml_file:
return yaml.load(yaml_file, Loader=yaml.FullLoader)
def open_config(config_path: str,
is_yaml: bool = False) -> Tuple[Union[Dict, dict], bool]:
"""Open yaml config
Args:
config_path (str): yaml config path
is_yaml (bool): If True, return yaml dict rather than addict.Dict
Returns: CONFIG
"""
# * Check for existence
if not osp.isfile(config_path):
raise RuntimeError(f"No config file found at {config_path}")
# * Get extension
name, ext = osp.splitext(config_path)
is_py = False
# * Read yml/yaml config
if ext == '.yml' or ext == '.yaml':
# * Read and set config
config_dir, _ = osp.split(config_path)
YamlIncludeConstructor.add_to_loader_class(
loader_class=yaml.FullLoader, base_dir=config_dir)
with open(config_path, 'r') as f:
yaml_config = yaml.load(f, Loader=yaml.FullLoader)
if is_yaml:
config = yaml_config
else:
config = Dict(yaml_config)
elif ext == '.py':
is_py = True
# # * Get import name
# def get_import_name(name):
# name = ntpath.normpath(name)
# return '.'.join(name.split(ntpath.sep))
# * Return config
spec = spec_from_file_location("foo", config_path)
module = module_from_spec(spec)
spec.loader.exec_module(module)
config: Dict = module.config
# config: Dict = import_module(get_import_name(name)).config
elif ext == '.json':
is_py = True
with open(config_path, 'r') as json_f:
config = Dict(json.load(json_f))
elif ext == ".pkl":
is_py = True
with open(config_path, 'rb') as pkl_f:
config = Dict(pickle.load(pkl_f))
else:
raise ValueError(
f"config_path = {config_path} must be python, json, pkl, yml or yaml file."
)
return config, is_py
def setUp(self):
if yaml.__version__ >= '5.0':
self.LOADER = yaml.FullLoader
else:
self.LOADER = yaml.Loader
YamlIncludeConstructor.add_to_loader_class()
self.assertIn(YamlIncludeConstructor.DEFAULT_TAG_NAME,
self.LOADER.yaml_constructors)
def main(in_, template_path, root, set_, output_file, output_dir):
YamlIncludeConstructor.add_to_loader_class(loader_class=FullLoader)
with open(in_, "r") as stream:
yaml = load(stream, Loader=FullLoader)
template_args = get_template_args(yaml, root, set_)
output_path = make_output_path(output_file, output_dir)
secrender(template_path, template_args, output_path)
def load_yaml_data(conf_path: Path,
quiet: bool = False,
_warning: bool = True) -> AnyByStrDict:
YamlIncludeConstructor.add_to_loader_class(loader_class=yaml.FullLoader,
base_dir=conf_path.parent)
try:
with open(conf_path) as f:
return yaml.load(f, Loader=yaml.FullLoader)
except yaml.parser.ParserError as e:
raise InvalidConfigFileError(conf_path, quiet) from e
def __init__(self, config_path: str = "config/"):
self.configured = False
YamlIncludeConstructor.add_to_loader_class(
loader_class=yaml.FullLoader, base_dir=config_path)
with open(config_path + 'config.yaml', 'r') as config_yaml:
config = yaml.load(config_yaml, Loader=yaml.FullLoader)
self.sensors = [
QSensor(s["name"], s["units"]) for s in config["sensors"]
]
self.relays = [QRelay(r["name"], False) for r in config["relays"]]
self.configured = True
def __analyze_file_yaml(self):
self.__logger.debug(f"CONFIG_YAML:\n{json.dumps(self.yaml_file_name, indent=2)}")
YamlIncludeConstructor.add_to_loader_class(loader_class=yaml.FullLoader, base_dir='.')
with open(self.yaml_file_name) as yml:
conf_yaml = yaml.load(yml, Loader=yaml.FullLoader)
self.__logger.debug(f"CONFIG_YAML:\n{json.dumps(conf_yaml, indent=2)}")
# Analyze images info
for key, val in conf_yaml["vm"].items():
for nm, inf in val.items():
init_img = inf.get('init_img') or ""
name_intrf = inf.get('name_intrf') or ""
plat = inf.get('platform') or ""
# if 'init_img' in inf:
# init_img = inf['init_img']
self.imgs_info[nm] = ImageInfo(
name=nm,
flavor_id=inf.get('flavor'),
image=inf.get('image'),
dev_type=key,
init_img=inf.get('init_img'),
name_intrf=inf.get('name_intrf'),
plat=inf.get('platform'),
vol_en = inf.get('volume_enable'),
vol_img = inf.get('volume_image'),
vol_dev = inf.get('volume_device')
)
# Analyze network links
for key, val in conf_yaml["networks"].items():
# print(f'{key}')
name = "link_" + str(key)
self.links_info[key] = LinkInfo(name=name, num=key, val=val)
self.__logger.debug(f"LinkInfo: {self.links_info[key]}")
used_net_links = list()
# Analyze routers
if 'routers' in conf_yaml:
for key, val in conf_yaml["routers"].items():
# self.__logger.info(f"net: {str(self.links_info)}")
self.rtrs_info[key] = RouterInfo(name=key, val=val, net=self.links_info)
self.__logger.debug(f'{self.rtrs_info[key]}')
# Analyze servers
if 'servers' in conf_yaml:
for key, val in conf_yaml["servers"].items():
self.srvs_info[key] = ServerInfo(name=key, val=val, net=self.links_info)
self.__logger.debug(f'{self.srvs_info[key]}')
self.__logger.debug(str(self.__str__()))
def read_config_file(fpath):
"""
Read configuration from a YAML file at path `fpath`. The file
can include other files using the `!include` directive.
"""
basedir = fpath.parent
YamlIncludeConstructor.add_to_loader_class(
loader_class=SafeLoader, base_dir=basedir)
with open(fpath, 'r') as class_conf:
config = yaml.load(class_conf, Loader=SafeLoader)
return config
def main():
args = argument_parser()
print('base_dir: {}'.format(args.base_dir))
print('field_template_file: {}'.format(args.field_template_file))
print('output_file: {}'.format(args.output_file))
YamlIncludeConstructor.add_to_loader_class(loader_class=yaml.FullLoader,
base_dir=args.base_dir)
with open(args.field_template_file) as f:
data = yaml.load(f, Loader=yaml.FullLoader)
with open(args.output_file, 'w') as file:
yaml.dump(data, file, default_flow_style=False, sort_keys=False)
def __init__(self, *args, **kwargs):
super(cls, self).__init__(*args, **kwargs)
def yaml_join(loader, node):
seq = loader.construct_sequence(node)
return ' '.join([str(i) for i in seq])
def yaml_remove_arg(loader, node):
orig, remove = node.value
args = loader.construct_scalar(orig)
return " ".join(a for a in args.split() if a != remove.value)
def folder_constructor(loader, node):
return Folder(node.value)
def union_constructor(loader, node):
return Union(node.value)
self.add_constructor(u'tag:yaml.org,2002:map', from_yaml)
self.add_constructor(u'tag:yaml.org,2002:omap', from_yaml)
self.add_constructor('!join', yaml_join)
self.add_constructor('!remove_arg', yaml_remove_arg)
self.add_constructor("!folder", folder_constructor)
self.add_constructor("!union", union_constructor)
self.add_constructor('!include',
YamlIncludeConstructor(base_dir=base_dir))
def setUpClass(cls) -> None:
reader_table = [
(re.compile(r'^.+\.(([yY][mM][lL])|([Yy][aA][mM][lL]))$'), YamlReader),
(re.compile(r'^.+\.([j][2])$'), YamlReader),
]
constructor = YamlIncludeConstructor(reader_map=reader_table)
yaml.FullLoader.add_constructor('!inc', constructor)
def setUp(self):
from yaml import SafeLoader, Loader
self.LOADER_CLASSES = [SafeLoader, Loader]
try:
from yaml import CSafeLoader
except ImportError as err:
print(err, file=stderr)
else:
self.LOADER_CLASSES.append(CSafeLoader)
try:
from yaml import CLoader
except ImportError as err:
print(err, file=stderr)
else:
self.LOADER_CLASSES.append(CLoader)
for loader_cls in self.LOADER_CLASSES:
YamlIncludeConstructor.add_to_loader_class(loader_cls)
def _add_constructors(self, base_path: str) -> None:
_ = self # Fake usage
yaml.SafeLoader.add_constructor( # type: ignore
"!engine", partial(_custom_yaml_constructor, clstype=RealEngine))
yaml.SafeLoader.add_constructor( # type: ignore
"!retry", partial(_custom_yaml_constructor, clstype=RetryHandler))
yaml.SafeLoader.add_constructor( # type: ignore
"!include", YamlIncludeConstructor(base_dir=base_path))
yaml.SafeLoader.add_constructor( # type: ignore
"!env", _custom_env_tag)
def run_config(config_path: Path) -> None:
"""Parse a YAML config file, returning the validated ILLIXR system config."""
YamlIncludeConstructor.add_to_loader_class(
loader_class=yaml.FullLoader, base_dir=config_path.parent,
)
with config_path.open() as f:
config = yaml.full_load(f)
with (root_dir / "runner/config_schema.yaml").open() as f:
config_schema = yaml.safe_load(f)
jsonschema.validate(instance=config, schema=config_schema)
fill_defaults(config, config_schema)
loader = config["loader"]["name"]
if loader not in loaders:
raise RuntimeError(f"No such loader: {loader}")
loaders[loader](config)
def __init__(self, config_path: Path):
self.batch_index = None
self.yaml_path = None
self.db_path = None
self.config_path = config_path.absolute()
YamlIncludeConstructor.add_to_loader_class(
loader_class=yaml.FullLoader,
base_dir=config_path.absolute().parent)
try:
self.data = self.apply_schema(self.config_path)
except SchemaError as exc:
sys.exit(exc.code)
try:
self.do_checks(self.data)
except Error as exc:
sys.exit(exc)
self.batch_mode = 'batch_simulations' in self.data
if self.batch_mode:
self.batch_simulations = self.data['batch_simulations']
class MultiLoaderTestCase(unittest.TestCase):
constructor = YamlIncludeConstructor()
def setUp(self):
yaml.add_constructor('!include', self.constructor)
def test_full_load_all_yaml(self):
txt = dedent('''
---
file1: !include tests/data/include.d/1.yaml
---
file2: !include tests/data/include.d/2.yaml
''').strip()
iterable = yaml.full_load_all(txt)
for i, data in enumerate(iterable):
if i == 0:
self.assertEqual(data, {'file1': YAML1})
elif i == 1:
self.assertEqual(data, {'file2': YAML2})
else:
raise RuntimeError()
def containers(self, yaml_config, render_context=None, base_dir=None):
if isfile(u"{}".format(yaml_config)):
if base_dir:
YamlIncludeConstructor.add_to_loader_class(base_dir=base_dir)
else:
base_dir = u"{}".format(dirname(yaml_config))
YamlIncludeConstructor.add_to_loader_class(base_dir=base_dir)
handle = open(yaml_config)
config = handle.read()
handle.close()
else:
YamlIncludeConstructor.add_to_loader_class(base_dir=base_dir)
config = yaml_config
config = Environment(loader=FileSystemLoader(base_dir or '.')).\
from_string(config).render(render_context or {})
for doc in yaml.load_all(config, Loader=yaml.FullLoader):
try:
yield SparcConfigContainer(doc)
except BrokenImplementation:
raise ValueError(
"expected yaml_config to contain valid yaml file path or string: {}"
.format(yaml_config))
import requests
from sp_api.tests.base_api import BaseApi
import json
from yamlinclude import YamlIncludeConstructor
from sp_api.tests.base_test import BaseTest
from yaml import load_all
class SmsCampaign(BaseApi):
def __init__(self):
super().__init__()
self.url = '{}/{}'.format(self.api_settings['url_api'], 'sms/send')
# self.url = '{}/{}'.format(self.api_settings['url_api'], 'sms/campaigns')
def add_campaign(self, requests_info):
self.headers.update({'Content-Type': 'application/json'})
response = requests.post(self.url, headers=self.headers, json=requests_info)
return response.json()
if __name__ == '__main__':
# add_campaign
resource_dir = BaseTest().conf['resourses']
YamlIncludeConstructor.add_to_loader_class(base_dir=resource_dir + '/requests/sms/campaign')
with open('{}/{}'.format(resource_dir, "requests/sms/campaign/add_campaigns.yaml")) as f:
for request_data in load_all(f):
print(json.dumps(request_data, indent=4, ensure_ascii=False))
added_campaign = SmsCampaign().add_campaign(request_data)
print(json.dumps(added_campaign, indent=4, ensure_ascii=False))
def setUp(self):
for loader_cls in YAML_LOADERS:
YamlIncludeConstructor.add_to_loader_class(loader_cls)
def read_vars(vars_file):
YamlIncludeConstructor.add_to_loader_class(
loader_class=BuildLoader, base_dir=os.path.dirname(vars_file))
with open(vars_file, 'r') as file_open:
used = yaml.load(file_open.read(), BuildLoader)
used.setdefault('environment', [])
used.setdefault('runtime_environment', [])
used.setdefault('runtime_interpreted', {})
used.setdefault('runtime_postprocess', [])
current_vars = {}
if 'extends' in used:
current_vars, config = read_vars(used['extends'])
no_interpreted = set()
no_interpreted.update(config.get('no_interpreted', []))
no_interpreted.update(used.get('no_interpreted', []))
used['no_interpreted'] = list(no_interpreted)
environment = config['environment']
runtime_environment = config['runtime_environment']
for e in used['runtime_environment']:
if e in environment:
environment.remove(e)
runtime_environment.append(e)
for name, interpreted in config.get('runtime_interpreted', {}).items():
if name in used['runtime_interpreted']:
if interpreted is list and used['runtime_interpreted'][
name] is list:
used['runtime_interpreted'][name] += interpreted
else:
value = {'vars': []}
if interpreted is list:
value['vars'] += interpreted
else:
clone = copy.copy(interpreted)
if 'vars' in clone:
value['vars'] += clone['vars']
del clone['vars']
value.update(clone)
if used['runtime_interpreted'][name] is list:
value['vars'] += used['runtime_interpreted'][name]
else:
clone = copy.copy(used['runtime_interpreted'][name])
if 'vars' in clone:
value['vars'] += clone['vars']
del clone['vars']
value.update(clone)
else:
used['runtime_interpreted'][name] = interpreted
used['runtime_postprocess'] += config.get('runtime_postprocess', [])
for e in used['environment']:
if e in runtime_environment:
runtime_environment.remove(e)
environment.append(e)
used['environment'] = environment
used['runtime_environment'] = runtime_environment
new_vars = do_process(used, used.get('vars', {}))
update_paths = []
for update_path in used.get('update_paths', []):
split_path = update_path.split('.')
for i in range(len(split_path)):
update_paths.append('.'.join(split_path[:i + 1]))
update_vars(current_vars, new_vars, set(update_paths))
return current_vars, used
# Ravestate class which represents a cluster of module configurations
# for a single context instance.
from ravestate import module
from collections import defaultdict
from typing import List, Any
import yaml
from yamlinclude import YamlIncludeConstructor
YamlIncludeConstructor.add_to_loader_class(yaml.SafeLoader)
from reggol import get_logger
logger = get_logger(__name__)
class Configuration:
"""
The Configuration class maintains a dictionary of key-value stores, which
represent configuration entries for specific named modules. The key-value
stores may be successively updated with consecutive yaml files, where each
yaml document has the following content:
---
module: module-name
config:
key-a: value-a
key-b: value-b
# etc
---
import os
import logging
import logging.config
import argparse
from datetime import datetime
from zipfile import ZipFile, ZIP_DEFLATED
from dotenv import load_dotenv
import yaml
from yamlinclude import YamlIncludeConstructor
from plugins import plugin_loader
VERSION = '1.0'
BASE_DIR = os.path.dirname(__file__)
CONF_DIR = os.path.join(BASE_DIR, 'config')
OUTPUT_DIR = os.path.join(BASE_DIR, 'output')
YamlIncludeConstructor.add_to_loader_class(loader_class = yaml.SafeLoader, base_dir = CONF_DIR)
load_dotenv()
os.makedirs(OUTPUT_DIR, exist_ok = True)
logging.config.fileConfig(os.path.join(CONF_DIR, 'logging.conf'))
logger = logging.getLogger('report')
def read_config(filename: str):
"""
Read the configuration file (YAML)
"""
logger.info(f"reading configuration file {filename}")
with open(filename, 'r') as config:
try:
return yaml.safe_load(config)
except yaml.YAMLError as ex:
def __new__(cls):
if cls._instance is None:
parser = argparse.ArgumentParser()
parser.add_argument('-i',
'--id',
type=str,
help='Unique client ID.')
parser.add_argument('-p',
'--port',
type=str,
help='The port number for running a server.')
parser.add_argument('-c',
'--config',
type=str,
default='./config.yml',
help='Federated learning configuration file.')
parser.add_argument('-s',
'--server',
type=str,
default=None,
help='The server hostname and port number.')
parser.add_argument(
'-d',
'--download',
action='store_true',
help='Download the dataset to prepare for a training session.')
parser.add_argument('-l',
'--log',
type=str,
default='info',
help='Log messages level.')
args = parser.parse_args()
Config.args = args
if Config.args.id is not None:
Config.args.id = int(args.id)
if Config.args.port is not None:
Config.args.port = int(args.port)
numeric_level = getattr(logging, args.log.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError(f'Invalid log level: {args.log}')
logging.basicConfig(
format='[%(levelname)s][%(asctime)s]: %(message)s',
datefmt='%H:%M:%S')
root_logger = logging.getLogger()
root_logger.setLevel(numeric_level)
cls._instance = super(Config, cls).__new__(cls)
if 'config_file' in os.environ:
filename = os.environ['config_file']
else:
filename = args.config
YamlIncludeConstructor.add_to_loader_class(
loader_class=yaml.SafeLoader, base_dir='./configs')
if os.path.isfile(filename):
with open(filename, 'r', encoding="utf8") as config_file:
config = yaml.load(config_file, Loader=yaml.SafeLoader)
else:
# if the configuration file does not exist, use a default one
config = Config.default_config()
Config.clients = Config.namedtuple_from_dict(config['clients'])
Config.server = Config.namedtuple_from_dict(config['server'])
Config.data = Config.namedtuple_from_dict(config['data'])
Config.trainer = Config.namedtuple_from_dict(config['trainer'])
Config.algorithm = Config.namedtuple_from_dict(config['algorithm'])
if Config.args.server is not None:
Config.server = Config.server._replace(
address=args.server.split(':')[0])
Config.server = Config.server._replace(
port=args.server.split(':')[1])
if Config.args.download:
Config.clients = Config.clients._replace(total_clients=1)
Config.clients = Config.clients._replace(per_round=1)
if 'results' in config:
Config.results = Config.namedtuple_from_dict(config['results'])
if hasattr(Config().results, 'results_dir'):
Config.result_dir = Config.results.results_dir
else:
datasource = Config.data.datasource
model = Config.trainer.model_name
server_type = Config.algorithm.type
Config.result_dir = f'./results/{datasource}/{model}/{server_type}/'
if 'model' in config:
Config.model = Config.namedtuple_from_dict(config['model'])
if hasattr(Config().trainer, 'max_concurrency'):
# Using a temporary SQLite database to limit the maximum number of concurrent
# trainers
Config.sql_connection = sqlite3.connect(
"/tmp/running_trainers.sqlitedb")
Config().cursor = Config.sql_connection.cursor()
# Customizable dictionary of global parameters
Config.params: dict = {}
# A run ID is unique to each client in an experiment
Config.params['run_id'] = os.getpid()
# Pretrained models
Config.params['model_dir'] = "./models/pretrained/"
Config.params['pretrained_model_dir'] = "./models/pretrained/"
return cls._instance
def setUp(self):
YamlIncludeConstructor.add_to_loader_class()
#!/usr/bin/env python
import logging
try:
from mpi4py import MPI
H5FLOW_MPI = True
except Exception as e:
logging.warning(f'Running without mpi4py because {e}')
H5FLOW_MPI = False
from .core import H5FlowManager, resources
import argparse
import yaml
import sys
from yamlinclude import YamlIncludeConstructor
YamlIncludeConstructor.add_to_loader_class(loader_class=yaml.FullLoader,
base_dir='./')
def run(configs,
output_filename,
input_filename=None,
start_position=None,
end_position=None,
verbose=0):
'''
Execute a workflow specified by ``config`` writing to ``output_filename``.
:param configs: ``list``, paths to configuration yamls to run in sequence
:param output_filename: ``str``, path to output hdf5 file
:param input_filename: ``str``, path to optional input file (default: ``None``)
def __init__(self):
DottedDict.__init__(self)
YamlIncludeConstructor.add_to_loader_class(
loader_class=yaml.FullLoader) # , base_dir='/your/conf/dir')
