def test(self):
"""dlbs -> TestConfigurationLoader::test [Loading default configuration.]"""
files, config, param_info = ConfigurationLoader.load(self.config_path)
# Check method returns object of expected type
self.assertIs(type(files), list)
self.assertIs(type(config), dict)
self.assertIs(type(param_info), dict)
# Check we load all configuration files
file_names = [os.path.basename(f) for f in files]
file_names.sort()
self.assertEqual(file_names, self.config_files)
# Check we have parameters and extensions sections
self.assertIn('parameters', config)
self.assertIn('extensions', config)
# Check presence of standard parameters
for ns in TestConfigurationLoader.params:
for param in TestConfigurationLoader.params[ns]:
self.assertIn(ns + '.' + param, config['parameters'])
# Check that values in configuration are not dictionaries and always have
# a parameter info object for every parameter
for param in config['parameters']:
self.assertFalse(
isinstance(config['parameters'][param], dict),
"In configuration dictionary parameter value cannot be a ditionary"
)
self.assertIn(param, param_info,
"Missing parameter in parameter info dictionary.")
# Check values in paramter info object are always dictionaries containing
# three mandatory fields.
for param in param_info:
self.assertTrue(
isinstance(param_info[param], dict),
"In parameter info dictionary a value must be a ditionary.")
for field in ('val', 'type', 'desc'):
self.assertIn(field, param_info[param])
def setUpBase(self, files=None):
if files is None:
files = ['base.json']
_, self.config, self.param_info = ConfigurationLoader.load(
os.path.join(os.path.dirname(dlbs.__file__), 'configs'),
files=files)
self.plan = None
def __init__(self):
"""Initialize helper by parsing command line arguments.
It's easier to do it manually. The format of command line is:
experimenter.py action command command_parameters
"""
_, _, self.param_info = ConfigurationLoader.load(
os.path.join(os.path.dirname(__file__), '..', 'configs'))
for key in self.param_info:
pi = self.param_info[key]
if 'desc' in pi and isinstance(pi['desc'], Six.string_types):
pi['desc'] = [pi['desc']]
with open(os.path.join(os.path.dirname(__file__),
'frameworks.json')) as file_obj:
self.frameworks_help = json.load(file_obj)
def test_path_none(self):
# None path must throw error
with self.assertRaises(ValueError):
ConfigurationLoader.load(None)
# Non existing directory must trigger Value Error
with self.assertRaises(ValueError):
ConfigurationLoader.load('/dr3/f2t23f/tfwegh5/sgh3gw4/hh/')
# Existing directory and non existing file must trigger ValueError
with self.assertRaises(ValueError):
ConfigurationLoader.load('/', files=['sfasdf23r23r23r2r23r.json'])
def init(self, **kwargs):
"""Initializes experimenter.
Args:
**kwargs (dict): Optional initialization parameters:
- action (str): Action to perform.
- config (str): A user-provided configuration file.
- plan (str): A file for generated benchmark plan.
- no_validation (bool): If true, do not perform validation
- progress_file (str): A path to progress file (if not None, enables progress reporting).
- params (dict): User defined parameters.
- vars (dict): User defined variables.
- discard_default_config (bool): If True, do not load standard DLBS config.
- extensions (dict): User provided extensions.
User provided parameters (`params`), variables (`vars`) and extensions (`extensions`) overwrite values defined
in user configuration files (`config`) if it is present.
Information defined in a uses-provided configuration file (`config`) overwrites standard DLBS configuration.
"""
if self.__initialized:
raise RuntimeError("Experimenter can only be initialized once.")
self.action = DictUtils.get(kwargs, 'action', 'run')
self.config_file = DictUtils.get(kwargs, 'config', None)
self.plan_file = DictUtils.get(kwargs, 'plan', None)
self.validation = not DictUtils.get(kwargs, 'no_validation', False)
self.__progress_file = DictUtils.get(kwargs, 'progress_file', None)
# Get parameters and variables from a command line/user-provided
self.params.update(DictUtils.get(kwargs, 'params', {}))
self.variables.update(DictUtils.get(kwargs, 'vars', {}))
# Load default configuration
if not DictUtils.get(kwargs, 'discard_default_config', False):
logging.debug("Loading default configuration")
_, self.config, self.param_info = ConfigurationLoader.load(
os.path.join(os.path.dirname(__file__), 'configs'))
# Load configurations specified on a command line
self.load_configuration()
# Add extensions from command line
DictUtils.ensure_exists(self.config, 'extensions', [])
self.config['extensions'].extend(
DictUtils.get(kwargs, 'extensions', []))
# All's done
self.__initialized = True
def __init__(self, tmp_folder):
# Check folder does not exist or empty
if not os.path.isdir(tmp_folder):
os.makedirs(tmp_folder)
else:
assert len(os.listdir(tmp_folder)) == 0,\
"Folder %s must be empty." % tmp_folder
self.tmp_folder = tmp_folder
# Load configuration
_, _, self.param_info = ConfigurationLoader.load(
os.path.join(os.path.dirname(__file__), '..', 'configs'))
# If a parameter contains description and it is string, convert to array
for key in self.param_info:
pi = self.param_info[key]
if 'desc' in pi and isinstance(pi['desc'], basestring):
pi['desc'] = [pi['desc']]
# Load common and framework specific params
with open(os.path.join(os.path.dirname(__file__),
'frameworks.json')) as file_obj:
self.framework_info = json.load(file_obj)
def init(self,
init_logger=False,
load_default_config=True,
load_config=True):
"""Initializes experimenter.
:param bool init_logger: If True, initializes loggers
:param bool load_default_config: If false, does not load standard configuration.
:param bool load_config: If true, loads configuration specified on a command line
"""
# Parse command line arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'action',
type=str,
help=
'Action to perform. Valid actions: "print-config", "run", "build" and "analyze-plan".'
)
parser.add_argument('--config',
required=False,
type=str,
help='Configuration file (json) of an experiment.\
Will override values from default configuration.'
)
parser.add_argument('--plan',
required=False,
type=str,
help='Pre-built plan of an experiment (json).\
If action is "build", a file name to write plan to.\
If action is "run", a file name to read plan from.'
)
parser.add_argument('--progress_file', '--progress-file', required=False, type=str, default=None,
help='A JSON file that experimenter will be updating on its progress.'\
'If not present, no progress info will be available.'\
'Put it somewhere in /dev/shm')
parser.add_argument(
'-P',
action='append',
required=False,
default=[],
help='Parameters that override parameters in configuration file.\
For instance, -Pexp.phase=2. Values must be json parsable (json.loads()).'
)
parser.add_argument(
'-V',
action='append',
required=False,
default=[],
help=
'Variables that override variables in configuration file in section "variables". \
These variables are used to generate different combinations of experiments.\
For instance: -Vexp.framework=\'["tensorflow", "caffe2"]\'.\
Values must be json parsable (json.loads()).'
)
parser.add_argument(
'--log_level',
'--log-level',
required=False,
default='info',
help=
'Python logging level. Valid values: "critical", "error", "warning", "info" and "debug"'
)
parser.add_argument('--discard_default_config',
'--discard-default-config',
required=False,
default=False,
action='store_true',
help='Do not load default configuration.')
parser.add_argument(
'--no_validation',
'--no-validation',
required=False,
default=False,
action='store_true',
help='Do not perform config validation before running benchmarks.')
parser.add_argument(
'-E',
action='append',
required=False,
default=[],
help=
'Extensions to add. Can be usefull to quickly customize experiments.\
Must be valid json parsable array element for "extension" array.'
)
args = parser.parse_args()
log_level = logging.getLevelName(args.log_level.upper())
self.action = args.action
self.config_file = args.config
self.plan_file = args.plan
self.validation = not args.no_validation
self.__progress_file = args.progress_file
# Initialize logger
if init_logger:
logging.debug("Initializing logger to level %s", args.log_level)
root = logging.getLogger()
root.setLevel(log_level)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(log_level)
root.addHandler(handler)
logging.debug("Parsing parameters on a command line")
DictUtils.add(self.params,
args.P,
pattern='(.+?(?=[=]))=(.+)',
must_match=True)
logging.debug("Parsing variables on a command line")
DictUtils.add(self.variables,
args.V,
pattern='(.+?(?=[=]))=(.+)',
must_match=True)
# Load default configuration
if load_default_config and not args.discard_default_config:
logging.debug("Loading default configuration")
_, self.config, self.param_info = ConfigurationLoader.load(
os.path.join(os.path.dirname(__file__), 'configs'))
# Load configurations specified on a command line
if load_config:
logging.debug("Loading user configuration")
self.load_configuration()
# Add extensions from command line
DictUtils.ensure_exists(self.config, 'extensions', [])
if len(args.E) > 0:
logging.debug("Parsing extensions on a command line")
for extension in args.E:
try:
ext = json.loads(extension)
logging.debug('Found extension: %s', str(ext))
self.config['extensions'].append(ext)
except Exception as err:
logging.warn("Found non-json parsable extension: %s",
extension)
raise err