def test_update_param_info(self):
pi = {}
ConfigurationLoader.update_param_info(
pi, {
'parameters': {
'p1': 1,
'p2': u'2',
'p3': '3',
'p4': ['1', '2', '3', '4'],
'p5': False,
'p6': -3.33,
'p7': {
'val': '34',
'type': 'str',
'desc': 'Some desc'
}
}
})
for p in ('p1', 'p2', 'p3', 'p4', 'p5', 'p6', 'p7'):
self.assertIn(p, pi)
for f in ('val', 'type', 'desc'):
self.assertIn(f, pi[p])
for s in (('p1', 'int', 1), ('p2', 'str', u'2'), ('p3', 'str', '3'),
('p4', 'str', ['1', '2', '3', '4']), ('p5', 'bool', False),
('p6', 'float', -3.33), ('p7', 'str', '34')):
self.assertEqual(pi[s[0]]['type'], s[1])
self.assertEqual(pi[s[0]]['val'], s[2])
def load_configuration(self):
"""Loads configuration specified by a user on a command line.
At this moment, DLBS has already loaded standard configuration (if `discard_default_config` flag is not
present). DLBS will try to load user configuration from `config` file (if not None) overwriting default
parameters. Then, it will try to load user provided parameters (`params`, `vars` and `extensions`) that will
overwrite existing configuration.
If `plan` file is present, it will be loaded if `action` is `run`.
"""
if self.config_file is not None:
logging.debug('Loading configuration from: %s', self.config_file)
with open(self.config_file) as file_obj:
user_config = json.load(file_obj)
# Update parameter information from user configuration.
ConfigurationLoader.update_param_info(self.param_info,
user_config,
is_user_config=True)
# Update existing benchmark configuration.
ConfigurationLoader.update(
self.config, ConfigurationLoader.remove_info(user_config))
if self.plan_file is not None and self.action == 'run':
logging.debug('Loading plan from: %s', self.plan_file)
with open(self.plan_file) as plan_file:
self.plan = json.load(plan_file)
def test(self):
"""dlbs -> TestConfigurationLoader::test [Loading default configuration.]"""
files, config, param_info = ConfigurationLoader.load(self.config_path)
# Check method returns object of expected type
self.assertIs(type(files), list)
self.assertIs(type(config), dict)
self.assertIs(type(param_info), dict)
# Check we load all configuration files
file_names = [os.path.basename(f) for f in files]
file_names.sort()
self.assertEqual(file_names, self.config_files)
# Check we have parameters and extensions sections
self.assertIn('parameters', config)
self.assertIn('extensions', config)
# Check presence of standard parameters
for ns in TestConfigurationLoader.params:
for param in TestConfigurationLoader.params[ns]:
self.assertIn(ns + '.' + param, config['parameters'])
# Check that values in configuration are not dictionaries and always have
# a parameter info object for every parameter
for param in config['parameters']:
self.assertFalse(
isinstance(config['parameters'][param], dict),
"In configuration dictionary parameter value cannot be a ditionary"
)
self.assertIn(param, param_info,
"Missing parameter in parameter info dictionary.")
# Check values in paramter info object are always dictionaries containing
# three mandatory fields.
for param in param_info:
self.assertTrue(
isinstance(param_info[param], dict),
"In parameter info dictionary a value must be a ditionary.")
for field in ('val', 'type', 'desc'):
self.assertIn(field, param_info[param])
def setUpBase(self, files=None):
if files is None:
files = ['base.json']
_, self.config, self.param_info = ConfigurationLoader.load(
os.path.join(os.path.dirname(dlbs.__file__), 'configs'),
files=files)
self.plan = None
def load_configuration(self):
"""Loads configuration specified by a user on a command line."""
if self.config_file is not None:
logging.debug('Loading configuration from: %s', self.config_file)
with open(self.config_file) as file_obj:
user_config = json.load(file_obj)
# Update parameter information from user configuration.
ConfigurationLoader.update_param_info(self.param_info,
user_config,
is_user_config=True)
# Update existing benchmark configuration.
ConfigurationLoader.update(
self.config, ConfigurationLoader.remove_info(user_config))
if self.plan_file is not None and self.action == 'run':
logging.debug('Loading plan from: %s', self.plan_file)
with open(self.plan_file) as plan_file:
self.plan = json.load(plan_file)
def __init__(self):
"""Initialize helper by parsing command line arguments.
It's easier to do it manually. The format of command line is:
experimenter.py action command command_parameters
"""
_, _, self.param_info = ConfigurationLoader.load(
os.path.join(os.path.dirname(__file__), '..', 'configs'))
for key in self.param_info:
pi = self.param_info[key]
if 'desc' in pi and isinstance(pi['desc'], Six.string_types):
pi['desc'] = [pi['desc']]
with open(os.path.join(os.path.dirname(__file__),
'frameworks.json')) as file_obj:
self.frameworks_help = json.load(file_obj)
def test_path_none(self):
# None path must throw error
with self.assertRaises(ValueError):
ConfigurationLoader.load(None)
# Non existing directory must trigger Value Error
with self.assertRaises(ValueError):
ConfigurationLoader.load('/dr3/f2t23f/tfwegh5/sgh3gw4/hh/')
# Existing directory and non existing file must trigger ValueError
with self.assertRaises(ValueError):
ConfigurationLoader.load('/', files=['sfasdf23r23r23r2r23r.json'])
def init(self, **kwargs):
"""Initializes experimenter.
Args:
**kwargs (dict): Optional initialization parameters:
- action (str): Action to perform.
- config (str): A user-provided configuration file.
- plan (str): A file for generated benchmark plan.
- no_validation (bool): If true, do not perform validation
- progress_file (str): A path to progress file (if not None, enables progress reporting).
- params (dict): User defined parameters.
- vars (dict): User defined variables.
- discard_default_config (bool): If True, do not load standard DLBS config.
- extensions (dict): User provided extensions.
User provided parameters (`params`), variables (`vars`) and extensions (`extensions`) overwrite values defined
in user configuration files (`config`) if it is present.
Information defined in a uses-provided configuration file (`config`) overwrites standard DLBS configuration.
"""
if self.__initialized:
raise RuntimeError("Experimenter can only be initialized once.")
self.action = DictUtils.get(kwargs, 'action', 'run')
self.config_file = DictUtils.get(kwargs, 'config', None)
self.plan_file = DictUtils.get(kwargs, 'plan', None)
self.validation = not DictUtils.get(kwargs, 'no_validation', False)
self.__progress_file = DictUtils.get(kwargs, 'progress_file', None)
# Get parameters and variables from a command line/user-provided
self.params.update(DictUtils.get(kwargs, 'params', {}))
self.variables.update(DictUtils.get(kwargs, 'vars', {}))
# Load default configuration
if not DictUtils.get(kwargs, 'discard_default_config', False):
logging.debug("Loading default configuration")
_, self.config, self.param_info = ConfigurationLoader.load(
os.path.join(os.path.dirname(__file__), 'configs'))
# Load configurations specified on a command line
self.load_configuration()
# Add extensions from command line
DictUtils.ensure_exists(self.config, 'extensions', [])
self.config['extensions'].extend(
DictUtils.get(kwargs, 'extensions', []))
# All's done
self.__initialized = True
def __init__(self, tmp_folder):
# Check folder does not exist or empty
if not os.path.isdir(tmp_folder):
os.makedirs(tmp_folder)
else:
assert len(os.listdir(tmp_folder)) == 0,\
"Folder %s must be empty." % tmp_folder
self.tmp_folder = tmp_folder
# Load configuration
_, _, self.param_info = ConfigurationLoader.load(
os.path.join(os.path.dirname(__file__), '..', 'configs'))
# If a parameter contains description and it is string, convert to array
for key in self.param_info:
pi = self.param_info[key]
if 'desc' in pi and isinstance(pi['desc'], basestring):
pi['desc'] = [pi['desc']]
# Load common and framework specific params
with open(os.path.join(os.path.dirname(__file__),
'frameworks.json')) as file_obj:
self.framework_info = json.load(file_obj)
def test_remove_info(self):
config = ConfigurationLoader.remove_info({
'parameters': {
'p1': 1,
'p2': u'2',
'p3': '3',
'p4': ['1', '2', '3', '4'],
'p5': False,
'p6': -3.33,
'p7': {
'val': '34',
'type': 's3tr',
'desc': 'Some desc'
}
}
})
for p in ('p1', 'p2', 'p3', 'p4', 'p5', 'p6', 'p7'):
self.assertIn(p, config['parameters'])
for s in (('p1', 1), ('p2', u'2'), ('p3', '3'),
('p4', ['1', '2', '3',
'4']), ('p5', False), ('p6', -3.33), ('p7', '34')):
self.assertEqual(config['parameters'][s[0]], s[1])
def init(self,
init_logger=False,
load_default_config=True,
load_config=True):
"""Initializes experimenter.
:param bool init_logger: If True, initializes loggers
:param bool load_default_config: If false, does not load standard configuration.
:param bool load_config: If true, loads configuration specified on a command line
"""
# Parse command line arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'action',
type=str,
help=
'Action to perform. Valid actions: "print-config", "run", "build" and "analyze-plan".'
)
parser.add_argument('--config',
required=False,
type=str,
help='Configuration file (json) of an experiment.\
Will override values from default configuration.'
)
parser.add_argument('--plan',
required=False,
type=str,
help='Pre-built plan of an experiment (json).\
If action is "build", a file name to write plan to.\
If action is "run", a file name to read plan from.'
)
parser.add_argument('--progress_file', '--progress-file', required=False, type=str, default=None,
help='A JSON file that experimenter will be updating on its progress.'\
'If not present, no progress info will be available.'\
'Put it somewhere in /dev/shm')
parser.add_argument(
'-P',
action='append',
required=False,
default=[],
help='Parameters that override parameters in configuration file.\
For instance, -Pexp.phase=2. Values must be json parsable (json.loads()).'
)
parser.add_argument(
'-V',
action='append',
required=False,
default=[],
help=
'Variables that override variables in configuration file in section "variables". \
These variables are used to generate different combinations of experiments.\
For instance: -Vexp.framework=\'["tensorflow", "caffe2"]\'.\
Values must be json parsable (json.loads()).'
)
parser.add_argument(
'--log_level',
'--log-level',
required=False,
default='info',
help=
'Python logging level. Valid values: "critical", "error", "warning", "info" and "debug"'
)
parser.add_argument('--discard_default_config',
'--discard-default-config',
required=False,
default=False,
action='store_true',
help='Do not load default configuration.')
parser.add_argument(
'--no_validation',
'--no-validation',
required=False,
default=False,
action='store_true',
help='Do not perform config validation before running benchmarks.')
parser.add_argument(
'-E',
action='append',
required=False,
default=[],
help=
'Extensions to add. Can be usefull to quickly customize experiments.\
Must be valid json parsable array element for "extension" array.'
)
args = parser.parse_args()
log_level = logging.getLevelName(args.log_level.upper())
self.action = args.action
self.config_file = args.config
self.plan_file = args.plan
self.validation = not args.no_validation
self.__progress_file = args.progress_file
# Initialize logger
if init_logger:
logging.debug("Initializing logger to level %s", args.log_level)
root = logging.getLogger()
root.setLevel(log_level)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(log_level)
root.addHandler(handler)
logging.debug("Parsing parameters on a command line")
DictUtils.add(self.params,
args.P,
pattern='(.+?(?=[=]))=(.+)',
must_match=True)
logging.debug("Parsing variables on a command line")
DictUtils.add(self.variables,
args.V,
pattern='(.+?(?=[=]))=(.+)',
must_match=True)
# Load default configuration
if load_default_config and not args.discard_default_config:
logging.debug("Loading default configuration")
_, self.config, self.param_info = ConfigurationLoader.load(
os.path.join(os.path.dirname(__file__), 'configs'))
# Load configurations specified on a command line
if load_config:
logging.debug("Loading user configuration")
self.load_configuration()
# Add extensions from command line
DictUtils.ensure_exists(self.config, 'extensions', [])
if len(args.E) > 0:
logging.debug("Parsing extensions on a command line")
for extension in args.E:
try:
ext = json.loads(extension)
logging.debug('Found extension: %s', str(ext))
self.config['extensions'].append(ext)
except Exception as err:
logging.warn("Found non-json parsable extension: %s",
extension)
raise err