def __init__(self, fname, plot_conf, layout_conf, read_yaml = True):
'''
:param str fname: file name of log
:param str/dict plot_conf: plot yaml file name / dict loaded from plot.yaml
:param str/dict layout_conf: layout yaml file name / dcit loaded from layout.yaml
'''
self.fname = fname
if read_yaml:
self.plot_dict = metayaml.read(plot_conf)
self.layout_dict = metayaml.read(layout_conf)["main"]
else:
self.plot_dict = plot_conf
self.layout_dict = layout_conf
# expand [0-33] => [0,1,2,...,33]
for leg_info in self.plot_dict.values():
for log_info in leg_info['data']:
if type(log_info['column'][0]) == str:
log_info['column'] = expand_str_to_list(log_info['column'][0])
for dict_title in self.layout_dict:
for leg in self.layout_dict[dict_title]['legends']:
if type(leg['id'][0]) == str:
leg['id'] = expand_str_to_list(leg['id'][0])
self.layout_dict[dict_title].setdefault('newline', True)
self.layout_dict[dict_title].setdefault('left_label', False)
self.layout_dict[dict_title].setdefault('bottom_label', "time [s]")
self.dataListDict = {}
def __init__(self, fname, plot_conf_name, layout_conf_name, title):
self.fname = fname
self.plot_dict = metayaml.read(plot_conf_name)
self.layout_dict = metayaml.read(layout_conf_name)["main"]
# expand [0-33] => [0,1,2,...,33]
for leg_info in self.plot_dict.values():
for log_info in leg_info['data']:
if type(log_info['column'][0]) == str:
log_info['column'] = expand_str_to_list(
log_info['column'][0])
for title in self.layout_dict:
for leg in self.layout_dict[title]['legends']:
if type(leg['id'][0]) == str:
leg['id'] = expand_str_to_list(leg['id'][0])
self.layout_dict[title].setdefault('newline', True)
self.layout_dict[title].setdefault('label', False)
# setup view
self.view = pyqtgraph.GraphicsLayoutWidget()
self.view.setBackground('w')
self.view.setWindowTitle(title if title else fname.split('/')[-1])
# self.dateListDict is set by self.readData()
self.dataListDict = {} # todo: to list of dictionary
# back up for plot items
self.plotItemOrig = {}
# default font style
self.font_type = 'Times New Roman'
self.font_size = 12
self.font_color = 'black'
def __init__(self):
#Sensor setup radar
sensors = []
full_conf = metayaml.read('network_sink_recorder.yml')
SensorInstance.setup(sensors, full_conf)
self.model = Model(sensors)
self.folder_location = QtWidgets.QFileDialog()
self.timer_for_graph_1 = Timer.create_timer(self.update_range_plot,
600)
self.timer_for_graph_2 = Timer.create_timer(self.to_be_implemented,
1000)
self.timer_for_graph_3 = Timer.create_timer(self.update_vmax, 7)
self.splitter1, self.splitter2 = self.create_splitters()
self.configuration_tree, self.parameters = self.create_configuration_tree_and_parameter(
self.splitter1)
self.create_actions_for_buttons(self.timer_for_graph_1,
self.timer_for_graph_2,
self.timer_for_graph_3)
self.plotwidget1, self.plotwidget2, self.plotwidget3 = self.create_layout(
self.splitter2)
self.mw.show()
self.P1 = self.plotwidget1.plot()
self.P2 = self.plotwidget2.plot()
self.P3 = self.plotwidget3.plot()
self.P1.setPen(width=4, color='#ffcc33')
self.P3.setPen(width=4, color='#004966')
#Start the program in a stopped state
self.state_stopped()
def _get_config_data():
"""
Internal function that returns data of the given configuration
:return: dict
"""
if not package_name:
logger.error(
'Impossible to find configuration if package is not given!')
return None
if not config_name:
logger.error(
'Impossible to to find configuration if configuration name is not given!'
)
return None
if package_name not in _PACKAGE_CONFIGS:
logger.error(
'No configurations find for package "{}"'.format(package_name))
return None
valid_package_configs = get_all_package_configs(
package_name=package_name,
root_package_name=root_package_name,
environment=environment)
if not valid_package_configs or config_name not in valid_package_configs:
# tp.logger.info(
# 'Impossible to load configuration "{}" for package "{}" because it does not exists in '
# 'configuration folders!'.format(config_name, package_name))
return
module_configs = valid_package_configs[config_name]
# We read the last configuration found: dcc_version > dcc > base
config_path = module_configs[-1]
config_data = dict()
try:
config_data = metayaml.read(module_configs, config_dict)
except Exception:
logger.error(
'Error while reading configuration files: {} | {}'.format(
module_configs, traceback.format_exc()))
if not config_data:
raise RuntimeError(
'Configuration file "{}" is empty!'.format(config_path))
# We store path where configuration file is located in disk
if 'config' in config_data and 'path' in config_data['config']:
raise RuntimeError(
'Configuration file cannot contains section with path attribute! {}'
.format(config_path))
if 'config' in config_data:
config_data['config']['path'] = config_path
else:
config_data['config'] = {'path': config_path}
return config_data
def load_configs(file_names):
"""
Loads the yaml config files.
"""
# with open(file_name, 'r') as f:
# config_str = f.read()
# config = Box.from_yaml(config_str)
config = Box(metayaml.read(file_names))
return config
def load_variant(variant_name: str, variants_dir: str) -> VariantDefinition:
if variants_dir is None:
variants_dir = get_resource_path("config/variants")
subpaths = [variants_dir, variant_name+".yaml"]
yaml_fname = os.path.join(*subpaths)
if os.path.exists(yaml_fname):
definition = metayaml.read(yaml_fname)
else:
raise UnknownVariant(f"Cannot find product with name '{variant_name}'")
return VariantDefinition(variant_name, definition)
def main():
import docopt
import metayaml
import os
import lib
args = docopt.docopt(__doc__)
stage_config = args['<stage_config>']
if stage_config:
config_path = stage_config
if not os.path.exists(config_path):
config_path = os.path.join(lib.config_stage_directory(),
config_path)
if not os.path.exists(config_path):
config_path += '.yaml'
if not os.path.exists(config_path):
print('Stage config "{}" not found.'.format(stage_config))
return
configs = [
os.path.join(lib.root_directory(), "backend", "configs",
"backend.yaml"), config_path
]
def fix_me():
print("fixme")
def replace_db(uri, database):
from sqlalchemy.engine.url import make_url
u = make_url(uri)
u.database = database
return u.__to_string__(hide_password=False)
config = metayaml.read(configs,
defaults={
"__FIX_ME__": fix_me,
"STAGE_DIRECTORY":
lib.config_stage_directory(),
"join": os.path.join,
"ROOT": lib.root_directory(),
"environ": os.environ,
"replace_db": replace_db
})
bootstrap = BackendBootstrap(
config['backend']['entry_point'],
config['user']['default_users'][0]['email'],
config['user']['default_users'][0]['password'], args['--update'],
args['--verbose'])
else:
bootstrap = BackendBootstrap(args['--url'], args['--email'],
args['--password'], args['--update'],
args['--verbose'])
bootstrap.process_doc(args['<entity_config>'])
def read(path=None, set_globals=True):
import os
import metayaml
from attrdict import AttrDict
from lib import root_directory, config_stage_directory
def fix_me():
print("fixme")
raise Exception("Please fill parameter")
stage_config = os.environ.get('BOSS_CONFIG', None)
custom_stage_config = pytest.config.getoption('stage_config')
test_config = os.path.join(root_directory(), 'tests', 'configs', 'default.yaml')
custom_test_config = pytest.config.getoption('test_config')
configs = list()
configs.append(test_config)
if path:
configs.append(path)
elif custom_stage_config:
configs.append(custom_stage_config)
elif stage_config:
configs.append(stage_config)
else:
raise ValueError("No stage config specified. Ether set env BOSS_CONFIG or specify argument --stage_config for tests")
if custom_test_config:
configs.append(custom_test_config)
config = metayaml.read(configs,
defaults={
"__FIX_ME__": fix_me,
"STAGE_DIRECTORY": config_stage_directory(),
"join": os.path.join,
"ROOT": root_directory()
})
if not config['backend']['entry_point']:
config['backend']['entry_point'] = 'http://localhost:8080'
config = AttrDict(config, recursive=True)
if set_globals:
for k in config.keys():
if k == "__FIX_ME__":
continue
v = getattr(config, k)
globals()[k] = v
return config
def init_conf(local_conf=''):
global conf
global logger
if not conf:
configs = ["./trusty.yaml"]
local_conf = local_conf or os.environ.get("LOCAL_CONF", None)
if local_conf:
configs.append(local_conf)
conf = read(configs)
if not logger:
logger = get_logger()
if not get_environment_params(conf, environment2configuration):
sys.exit(1)
return conf, logger
def __init__(self, yml=None, *, obj=None, baseunit="mm", **kwargs):
"""Given a YAML file that's a toplevel list or dict,
this turns it into nested Params all the way down.
The other args and kwargs are for internal use with recursion.
"""
self.__dict__ = self
if yml is not None:
if isinstance(yml, list):
obj = read(yml)
else:
obj = read([yml])
kwargs = obj
for k, v in list(obj.items()):
if isinstance(v, str):
obj[k] = self.__convert(v, baseunit=baseunit, **kwargs)
if isinstance(v, dict):
obj[k] = Params(obj=v, baseunit=baseunit, **kwargs)
self.update(obj)
def init_conf(local_conf=''):
global conf
global logger
if not conf:
# Read configuration
# FIXME: search configuration under virtualenv
path_list = [
os.path.join('configs', 'trep.yaml'),
os.path.join(sys.prefix, 'local', 'etc', 'trep', 'trep.yaml'),
os.path.join(sys.prefix, 'etc', 'trep', 'trep.yaml'),
os.environ.get(
"TREP_CONFIG",
os.path.join(tools.config_stage_directory(), 'trep',
'trep.yaml')),
pkg_resources.resource_filename('trep', 'etc/trep/trep.yaml'),
]
from pprint import pprint
pprint(path_list)
for path in path_list:
if os.path.isfile(path):
print('Configuration found under:%s' % path)
stage_config = path
break
else:
print("Stage configuration not found")
local_conf = local_conf or os.environ.get(
"{}LOCAL_CONF".format(ENV_VARIABLES_PREFIX), None)
configs = [stage_config]
if local_conf:
configs.append(local_conf)
if not configs:
return None, None
for cfg in configs:
if not os.path.isfile(cfg):
print('Config does not exist: %s' % cfg)
conf = metayaml.read(configs,
defaults={
"__FIX_ME__": fix_me,
"join": os.path.join,
"ROOT": tools.root_directory(),
"env": os.environ,
})
if not get_environment_params(conf, environment2configuration):
sys.exit(1)
if not logger:
logger = get_logger()
return conf, logger
def read():
import os
import metayaml
from attrdict import AttrDict
from lib import root_directory, config_stage_directory
from os import environ
GLOBAL_STAGE_CONFIG = "/etc/boss.yaml"
class PleaseFillRequiredParameter(Exception):
pass
def replace_db(uri, database):
from sqlalchemy.engine.url import make_url
u = make_url(uri)
u.database = database
return u.__to_string__(hide_password=False)
def fix_me():
raise PleaseFillRequiredParameter("Please fill parameter")
stage_config = GLOBAL_STAGE_CONFIG if os.path.isfile(GLOBAL_STAGE_CONFIG) else \
os.environ.get("BOSS_CONFIG", os.path.join(config_stage_directory(), "dev_local.yaml"))
configs = [
os.path.join(root_directory(), "backend", "configs", "backend.yaml"),
stage_config
]
config = metayaml.read(configs,
defaults={
"__FIX_ME__": fix_me,
"STAGE_DIRECTORY": config_stage_directory(),
"join": os.path.join,
"ROOT": root_directory(),
"environ": environ,
"replace_db": replace_db
})
config = AttrDict(config, recursive=True)
for k in config.keys():
if k == "__FIX_ME__":
continue
v = getattr(config, k)
globals()[k] = v
return config
def _get_config_data(self, config_name, config_dict):
"""
Returns the config data of the given project name
:return: dict
"""
if not config_name:
LOGGER.error('Project Configuration File not found! {}'.format(
self, config_name))
return
module_config_name = config_name
if not module_config_name.endswith('.yml'):
module_config_name = config_name + '.yml'
all_config_paths = self._manager.get_config_paths(
module_config_name=module_config_name, skip_non_existent=False)
valid_config_paths = self._manager.get_config_paths(
module_config_name=module_config_name)
if not valid_config_paths:
raise RuntimeError(
'Impossible to load configuration "{}" because it does not exists in any of '
'the configuration folders: {}'.format(
config_name, ''.join(all_config_paths)))
root_config_path = valid_config_paths[-1]
config_data = metayaml.read(valid_config_paths,
config_dict) or OrderedDict()
if config_data is None:
raise RuntimeError(
'Project Configuration File {} is empty! {}'.format(
self, root_config_path))
# We store path where configuration file is located in disk
if 'config' in config_data and 'path' in config_data['config']:
raise RuntimeError(
'Project Configuration File for {} Project cannot contains '
'config section with path attribute! {}'.format(
self, root_config_path))
return
if 'config' in config_data:
config_data['config']['path'] = root_config_path
else:
config_data['config'] = {'path': root_config_path}
return config_data
def config_validate(path:str) -> dict:
if path.endswith('.yaml'):
config_reader = lambda path: metayaml.read(path)
elif path.endswith('.json'):
config_reader = lambda path: json.load(open(path, 'r'))
else:
raise argparse.ArgumentTypeError('unsupported config file format: {}'.format(path.split('.')[-1]))
try:
config = config_reader(path)
except OSError as e:
raise argparse.ArgumentTypeError("can't open '%s': %s" % (path, e))
base_shema = shema.copy()
base_shema.pop('openstack', None)
_validate_shema(base_shema, config)
return config
def init_conf(local_conf=''):
global conf
global logger
if not conf:
# Read configuration
# FIXME: search configuration under virtualenv
path_list = [
os.path.join('configs', 'trep.yaml'),
os.path.join(sys.prefix, 'local', 'etc', 'trep', 'trep.yaml'),
os.path.join(sys.prefix, 'etc', 'trep', 'trep.yaml'),
os.environ.get("TREP_CONFIG",
os.path.join(tools.config_stage_directory(),
'trep', 'trep.yaml')),
pkg_resources.resource_filename('trep', 'etc/trep/trep.yaml'),
]
from pprint import pprint
pprint(path_list)
for path in path_list:
if os.path.isfile(path):
print('Configuration found under:%s' % path)
stage_config = path
break
else:
print("Stage configuration not found")
local_conf = local_conf or os.environ.get("{}LOCAL_CONF".format(ENV_VARIABLES_PREFIX), None)
configs = [stage_config]
if local_conf:
configs.append(local_conf)
if not configs:
return None, None
for cfg in configs:
if not os.path.isfile(cfg):
print('Config does not exist: %s' % cfg)
conf = metayaml.read(configs,
defaults={
"__FIX_ME__": fix_me,
"join": os.path.join,
"ROOT": tools.root_directory(),
"env": os.environ,
})
if not get_environment_params(conf, environment2configuration):
sys.exit(1)
if not logger:
logger = get_logger()
return conf, logger
def _get_arguments() -> dict:
shema = metayaml.read(os.path.join(os.path.dirname(__file__), 'config_template.yaml'))
shema = _convert_ordered_dict(shema)
parser = argparse.ArgumentParser('BOSS Light Tests', formatter_class=argparse.RawDescriptionHelpFormatter,
description=_get_doc_from_shema(shema))
parser.add_argument('-o', '--openstack', help='Enable openstack test', action='store_true')
parser.add_argument('-v', '--verbose', help='Be verbose', action='store_true')
parser.add_argument('-t', '--teamcity', help='Enable teamcity messages', action='store_true')
parser.add_argument('--test_label', type=str, help='Tests label', default=None, required=False)
def config_validate(path:str) -> dict:
if path.endswith('.yaml'):
config_reader = lambda path: metayaml.read(path)
elif path.endswith('.json'):
config_reader = lambda path: json.load(open(path, 'r'))
else:
raise argparse.ArgumentTypeError('unsupported config file format: {}'.format(path.split('.')[-1]))
try:
config = config_reader(path)
except OSError as e:
raise argparse.ArgumentTypeError("can't open '%s': %s" % (path, e))
base_shema = shema.copy()
base_shema.pop('openstack', None)
_validate_shema(base_shema, config)
return config
parser.add_argument('config', type=config_validate, metavar='config_file', help='Config file (yaml, json)')
args = parser.parse_args()
if args.openstack:
_validate_shema(shema, args.config, False, parser)
try:
import light_tests.clients.openstack
except ImportError as e:
raise ValueError('Openstack tests are enabled but import error occured: {}'.format(e))
args.config = _convert_ordered_dict(args.config)
return vars(args)
def main():
import docopt
import metayaml
import os
import lib
args = docopt.docopt(__doc__)
stage_config = args['<stage_config>']
if stage_config:
config_path = stage_config
if not os.path.exists(config_path):
config_path = os.path.join(lib.config_stage_directory(), config_path)
if not os.path.exists(config_path):
config_path += '.yaml'
if not os.path.exists(config_path):
print('Stage config "{}" not found.'.format(stage_config))
return
configs = [os.path.join(lib.root_directory(), "backend", "configs", "backend.yaml"), config_path]
def fix_me():
print("fixme")
def replace_db(uri, database):
from sqlalchemy.engine.url import make_url
u = make_url(uri)
u.database = database
return u.__to_string__(hide_password=False)
config = metayaml.read(configs, defaults={
"__FIX_ME__": fix_me,
"STAGE_DIRECTORY": lib.config_stage_directory(),
"join": os.path.join,
"ROOT": lib.root_directory(),
"environ": os.environ,
"replace_db": replace_db})
bootstrap = BackendBootstrap(config['backend']['entry_point'], config['user']['default_users'][0]['email'],
config['user']['default_users'][0]['password'],
args['--update'], args['--verbose'])
else:
bootstrap = BackendBootstrap(args['--url'], args['--email'], args['--password'],
args['--update'], args['--verbose'])
bootstrap.process_doc(args['<entity_config>'])
def config_validate(path: str) -> dict:
if path.endswith('.yaml'):
config_reader = lambda path: metayaml.read(path)
elif path.endswith('.json'):
config_reader = lambda path: json.load(open(path, 'r'))
else:
raise argparse.ArgumentTypeError(
'unsupported config file format: {}'.format(
path.split('.')[-1]))
try:
config = config_reader(path)
except OSError as e:
raise argparse.ArgumentTypeError("can't open '%s': %s" % (path, e))
base_shema = shema.copy()
base_shema.pop('openstack', None)
_validate_shema(base_shema, config)
return config
def init_conf(local_conf=""):
global conf
global logger
if not conf:
# Read configuration
# FIXME: search configuration under virtualenv
path_list = [
os.path.join("configs", "trep.yaml"),
os.path.join(sys.prefix, "local", "etc", "trep", "trep.yaml"),
os.path.join(sys.prefix, "etc", "trep", "trep.yaml"),
os.environ.get("TREP_CONFIG", os.path.join(helpers.config_stage_directory(), "trep", "trep.yaml")),
pkg_resources.resource_filename("trep", "etc/trep/trep.yaml"),
]
from pprint import pprint
pprint(path_list)
for path in path_list:
if os.path.isfile(path):
print("Configuration found under:%s" % path)
stage_config = path
break
else:
print("Stage configuration not found")
local_conf = local_conf or os.environ.get("LOCAL_CONF", None)
configs = [stage_config]
if local_conf:
configs.append(local_conf)
if not configs:
return None, None
for cfg in configs:
if not os.path.isfile(cfg):
print("Config does not exist: %s" % cfg)
conf = metayaml.read(
configs,
defaults={"__FIX_ME__": fix_me, "join": os.path.join, "ROOT": helpers.root_directory(), "env": os.environ},
)
if not logger:
logger = get_logger()
if not get_environment_params(conf, environment2configuration):
sys.exit(1)
return conf, logger
def read():
import os
import metayaml
from attrdict import AttrDict
def root_directory(application_path=None):
root_path = application_path or os.path.dirname(__file__)
while root_path and "bin" not in os.listdir(root_path):
root_path = os.path.dirname(root_path)
return root_path
def config_directory():
root_path = root_directory()
return os.path.join(root_path, "configs")
def fix_me():
print("fix me")
raise Exception("Required field is empty")
stage_config = os.environ.get("EFA_CONFIG", os.path.join(config_directory(), "dev.yaml"))
configs = [os.path.join(root_directory(), "scripts", "configs", "efa.yaml"),
stage_config]
config = metayaml.read(configs,
defaults={
"__FIX_ME__": fix_me,
"STAGE_DIRECTORY": config_directory(),
"join": os.path.join,
"ROOT": root_directory()
})
config = AttrDict(config, recursive=True)
for k in config.keys():
if k == "__FIX_ME__":
continue
v = getattr(config, k)
globals()[k] = v
return config
def read(path: str, defaults: dict=None) -> dict:
defaults = defaults or dict()
defaults['join'] = os.path.join
defaults['BASE_DIR'] = BASE_DIR
return metayaml.read(path, defaults=defaults, disable_order_dict=True)
def read_config(filename=None):
filename = filename or os.environ.get(
'NEWSBOT_CONFIG',
os.path.join(os.path.dirname(os.path.dirname(__file__)),
'newsbot.yaml'))
return metayaml.read(filename)
def read_config(filename):
return metayaml.read(filename)
def read_config(filename):
return metayaml.read(filename)
def load_configs(file_names): """ Loads the yaml config files. """ config = Box(metayaml.read(file_names)) return config
def read(path: str, defaults: dict = None) -> dict:
defaults = defaults or dict()
defaults['join'] = os.path.join
defaults['BASE_DIR'] = BASE_DIR
return metayaml.read(path, defaults=defaults, disable_order_dict=True)
def _get_arguments() -> dict:
shema = metayaml.read(
os.path.join(os.path.dirname(__file__), 'config_template.yaml'))
shema = _convert_ordered_dict(shema)
parser = argparse.ArgumentParser(
'BOSS Light Tests',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=_get_doc_from_shema(shema))
parser.add_argument('-o',
'--openstack',
help='Enable openstack test',
action='store_true')
parser.add_argument('-v',
'--verbose',
help='Be verbose',
action='store_true')
parser.add_argument('-t',
'--teamcity',
help='Enable teamcity messages',
action='store_true')
parser.add_argument('--test_label',
type=str,
help='Tests label',
default=None,
required=False)
def config_validate(path: str) -> dict:
if path.endswith('.yaml'):
config_reader = lambda path: metayaml.read(path)
elif path.endswith('.json'):
config_reader = lambda path: json.load(open(path, 'r'))
else:
raise argparse.ArgumentTypeError(
'unsupported config file format: {}'.format(
path.split('.')[-1]))
try:
config = config_reader(path)
except OSError as e:
raise argparse.ArgumentTypeError("can't open '%s': %s" % (path, e))
base_shema = shema.copy()
base_shema.pop('openstack', None)
_validate_shema(base_shema, config)
return config
parser.add_argument('config',
type=config_validate,
metavar='config_file',
help='Config file (yaml, json)')
args = parser.parse_args()
if args.openstack:
_validate_shema(shema, args.config, False, parser)
try:
import light_tests.clients.openstack
except ImportError as e:
raise ValueError(
'Openstack tests are enabled but import error occured: {}'.
format(e))
args.config = _convert_ordered_dict(args.config)
return vars(args)
