def run(self): """Runs subprocess with Popen. This method must not be called directly. Use blocking :py:meth:`~dlbs.Worker.work` method instead. """ try: # Dump parameters to a log file or to standard output DictUtils.ensure_exists(self.params, 'exp.log_file', default_value='') if self.params['exp.log_file'].strip() == '': self.params['exp.log_file'] = '/dev/stdout' IOUtils.mkdirf(self.params['exp.log_file']) with open(self.params['exp.log_file'], 'a+') as log_file: self.__dump_parameters(log_file) # This is where we launch process. Keep in mind, that the log file that's # supposed to be created is exp.log_file or exp_log_file in the script. # Other output of the launching script will be printed by this pyhton code # to a stanard output. self.process = subprocess.Popen(self.command, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=self.environ) while True: output = self.process.stdout.readline() if output == '' and self.process.poll() is not None: break if output: sys.stdout.write(output) sys.stdout.flush() self.ret_code = self.process.poll() except Exception as err: logging.warn('Exception has been caught for experiment %s: %s', self.params.get('exp.id'), str(err)) logging.warn(traceback.format_exc()) self.ret_code = -1
def parse_json_arguments(args): """Parse parameters, variables and extensions. Args: args (dict): Dictionary of command line arguments returned by `parse_arguments`. Is not modified. Returns: A tuple of (params, variables, extensions): - `params` is a dictionary of parameters (all params in args['P']) - `variables` is a dictionary of variables (all vars in args['V']) - `extensions` is a list of dictionaries (all extensions in in args['E']) """ for param in ('P', 'V', 'E'): DictUtils.ensure_exists(args, param, []) params, variables, extensions = ({}, {}, []) DictUtils.add(params, args['P'], pattern='(.+?(?=[=]))=(.+)', must_match=True) DictUtils.add(variables, args['V'], pattern='(.+?(?=[=]))=(.+)', must_match=True) for extension in args['E']: try: extensions.append(json.loads(extension)) except Exception as err: logging.warn("Found non-json parsable extension: %s", extension) raise err return params, variables, extensions
def test_ensure_exists_1(self): """dlbs -> TestDictUtils::test_ensure_exists_1 [Testing dictionary helpers #1]""" DictUtils.ensure_exists(self.dictionary, 'exp.framework') self.assertEqual('exp.framework' in self.dictionary, True) self.assertEqual('exp.model' in self.dictionary, True) self.assertEqual('exp.device_batch' in self.dictionary, True) self.assertEqual(self.dictionary['exp.framework'], self.framework) self.assertEqual(self.dictionary['exp.model'], self.model) self.assertEqual(self.dictionary['exp.device_batch'], self.device_batch) self.assertEqual(len(self.dictionary), 3)
def report(self, inputs=None, output=None, output_cols=None, report_speedup=False, report_efficiency=False, **kwargs): DictUtils.ensure_exists(kwargs, 'report_batch_times', True) DictUtils.ensure_exists(kwargs, 'report_input_specs', True) # Build cache that will map benchmarks keys to benchmark objects. self.build_cache(inputs, output, output_cols) # Iterate over column values and build table with batch times and throughput cols = [] times = [] throughputs = [] benchmark_keys = [input_col['vals'] for input_col in self.input_cols] # Build tables for batch times and benchmarks throughputs # The `benchmark_key` is a tuple of column values e.g. ('ResNet50', 256) for benchmark_key in itertools.product(*benchmark_keys): cols.append(copy.deepcopy(benchmark_key)) times.append([None] * len(self.output_cols)) throughputs.append([None] * len(self.output_cols)) for output_col in self.output_cols: benchmark_key = [str(key) for key in benchmark_key] key = '.'.join(benchmark_key + [str(output_col['value'])]) if key in self.cache: times[-1][output_col['index']] = self.cache[key]['results.time'] throughputs[-1][output_col['index']] = self.cache[key]['results.throughput'] # Determine minimal widths for columns self.compute_column_widths(times, throughputs) # header = self.get_header() if kwargs['report_batch_times']: self.print_table("Batch time (milliseconds)", header, cols, times) self.print_table("Throughput (instances per second e.g. images/sec)", header, cols, throughputs) if report_speedup: speedups = self.compute_speedups(throughputs) self.print_table("Speedup (based on instances per second table, " "relative to first output column ({} = {}))".format(self.output_param, self.output_cols[0]['value']), header, cols, speedups) if report_efficiency: efficiency = self.compute_efficiency(times) self.print_table("Efficiency (based on batch times table, " "relative to first output column ({} = {}))".format(self.output_param, self.output_cols[0]['value']), header, cols, efficiency) if kwargs['report_input_specs']: print("This report is configured with the following parameters:") print(" inputs = %s" % str(inputs)) print(" output = %s" % output) print(" output_cols = %s" % str(output_cols)) print(" report_speedup = %s" % str(report_speedup)) print(" report_efficiency = %s" % str(report_efficiency))
def test_ensure_exists_3(self): """dlbs -> TestDictUtils::test_ensure_exists_3 [Testing dictionary helpers #3]""" DictUtils.ensure_exists(self.dictionary, 'exp.data_dir', '/nfs/imagenet') self.assertEqual('exp.framework' in self.dictionary, True) self.assertEqual('exp.model' in self.dictionary, True) self.assertEqual('exp.device_batch' in self.dictionary, True) self.assertEqual('exp.data_dir' in self.dictionary, True) self.assertEqual(self.dictionary['exp.framework'], self.framework) self.assertEqual(self.dictionary['exp.model'], self.model) self.assertEqual(self.dictionary['exp.device_batch'], self.device_batch) self.assertEqual(self.dictionary['exp.data_dir'], '/nfs/imagenet') self.assertEqual(len(self.dictionary), 4)
def __init__(self): self.__validation = True # Validate config before running benchmarks self.__action = None # Action to perform (build, run, ...) self.__config_file = None # Configuration file to load self.__progress_file = None # A JSON file with current progress self.__config = {} # Loaded configuration self.__param_info = { } # Parameter meta-info such as type and value domain self.__plan_file = None # File with pre-built plan self.__plan = [] # Loaded or generated plan self.__params = {} # Override env variables from files self.__variables = {} # Override variables from files # Dirty hacks DictUtils.ensure_exists(os.environ, 'CUDA_CACHE_PATH', '') DictUtils.ensure_exists( os.environ, 'DLBS_ROOT', os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../'))
def init(self, **kwargs): """Initializes experimenter. Args: **kwargs (dict): Optional initialization parameters: - action (str): Action to perform. - config (str): A user-provided configuration file. - plan (str): A file for generated benchmark plan. - no_validation (bool): If true, do not perform validation - progress_file (str): A path to progress file (if not None, enables progress reporting). - params (dict): User defined parameters. - vars (dict): User defined variables. - discard_default_config (bool): If True, do not load standard DLBS config. - extensions (dict): User provided extensions. User provided parameters (`params`), variables (`vars`) and extensions (`extensions`) overwrite values defined in user configuration files (`config`) if it is present. Information defined in a uses-provided configuration file (`config`) overwrites standard DLBS configuration. """ if self.__initialized: raise RuntimeError("Experimenter can only be initialized once.") self.action = DictUtils.get(kwargs, 'action', 'run') self.config_file = DictUtils.get(kwargs, 'config', None) self.plan_file = DictUtils.get(kwargs, 'plan', None) self.validation = not DictUtils.get(kwargs, 'no_validation', False) self.__progress_file = DictUtils.get(kwargs, 'progress_file', None) # Get parameters and variables from a command line/user-provided self.params.update(DictUtils.get(kwargs, 'params', {})) self.variables.update(DictUtils.get(kwargs, 'vars', {})) # Load default configuration if not DictUtils.get(kwargs, 'discard_default_config', False): logging.debug("Loading default configuration") _, self.config, self.param_info = ConfigurationLoader.load( os.path.join(os.path.dirname(__file__), 'configs')) # Load configurations specified on a command line self.load_configuration() # Add extensions from command line DictUtils.ensure_exists(self.config, 'extensions', []) self.config['extensions'].extend( DictUtils.get(kwargs, 'extensions', [])) # All's done self.__initialized = True
def parse_log_files(filenames, opts=None): """ Parses files and returns their parameters. :param list filenames: List of file names to parse. :param dict opts: Dictionary of options. :rtype: tuple<list, list> :return: A tuple of two lists - succeeded and failed benchmarks """ opts = {} if opts is None else opts for key in ('filter_params', 'filter_query', 'output_params'): DictUtils.ensure_exists(opts, key) DictUtils.ensure_exists(opts, 'failed_benchmarks', 'discard') DictUtils.ensure_exists(opts, '_extended_params', {}) DictUtils.ensure_exists(opts, 'ignore_errors', False) succeeded_benchmarks = [] failed_benchmarks = [] for filename in filenames: # Parse log file params = LogParser.parse_log_file( filename, ignore_errors=opts['ignore_errors']) # Check if this benchmark does not match filter if len(params) == 0 or \ not DictUtils.contains(params, opts['filter_params']) or \ not DictUtils.match(params, opts['filter_query']): continue # Add extended parameters and compute them if len(opts['_extended_params']) > 0: params.update(opts['_extended_params']) Processor().compute_variables([params]) # params = params[0] # Identify is this benchmark succeeded of failed. succeeded = 'results.throughput' in params and \ isinstance(params['results.throughput'], (int, long, float)) and \ params['results.throughput'] > 0 # Get only those key/values that need to be serialized params = DictUtils.subdict(params, opts['output_params']) # Append benchmark either to succeeded or failed list if succeeded: succeeded_benchmarks.append(params) else: if opts['failed_benchmarks'] == 'keep': succeeded_benchmarks.append(params) elif opts['failed_benchmarks'] == 'keep_separately': failed_benchmarks.append(params) # return succeeded_benchmarks, failed_benchmarks
def build(config, params, variables): """ Given input configuration and comamnd line parameters/variables build experiments :param dict config: Dictionary of parameters/variables/extensions :param dict params: Dictionary of command line parameters :param dict variables: Dictionary of command line variables :return: Array of experiments. Each experiment is defined by a set of parameters. A high level overview of what buidler does is: :: 1. Add **variables** to 'variables' section of a configuration ('config'). 2. Override variables in 'parameters' section in 'config' with those specified on a command line ('params'). 3. For every combination (Cartesian product) of variables in 'config': a. Create copy of parameters. b. Add combination to those parameters c. Apply extensions possibly generating multiple experiments d. Add them to a list of experiments. In case input configuration contains extensions, this algorithm applies: :: 1. Set experiments <- [experiment] 2. For extension in extension_list: 3. Set active_experiments <- [] 4. For experiment in experiments: 5. If not match(experiment, extension.condition): 6. active_experiments.append(experiment) 7. active_experiments.extend( extend(extended_config, extension) ) 8. experiments <- active_experiments 9. Return experiments """ # This makes parsing life easier. for section in ['parameters', 'variables', 'extensions']: DictUtils.ensure_exists(config, section, {}) for extension in config['extensions']: for section in ['condition', 'parameters', 'cases']: DictUtils.ensure_exists(extension, section, {}) # First, we need to update variables that contribute to creating # different experiments for var_name in variables: config['variables'][var_name] = copy.deepcopy(variables[var_name]) # We also need to ensure that those values are lists for variable in config['variables']: if not isinstance(config['variables'][variable], list): config['variables'][variable] = [config['variables'][variable]] # Now, we need to override environmental varaibles for param in params: config['parameters'][param] = copy.deepcopy(params[param]) plan = [] # Get order of variables in experiments # These are all variables that we will vary var_keys = config['variables'].keys() # Filter oder of those variables - remove ones that we do not actually have if 'sys.plan_builder.var_order' in config['parameters']: var_order = [ k for k in config['parameters']['sys.plan_builder.var_order'] if k in var_keys ] else: var_order = [] # Add those that are not in an order array for k in var_keys: if k not in var_order: var_order.append(k) var_values = [config['variables'][var_key] for var_key in var_order] # This loop will work just once if var_values is empty. for variables_combination in itertools.product(*var_values): # Create base set of variables. experiment = copy.deepcopy(config['parameters']) # Add current combination of variables experiment.update( dict((k, v) for (k, v) in zip(var_order, variables_combination))) # Apply extensions possibly generating many experiment configurations extended_experiments = Builder.apply_extensions(experiment, config) # Add to plan plan.extend(extended_experiments) return plan
def init(self, init_logger=False, load_default_config=True, load_config=True): """Initializes experimenter. :param bool init_logger: If True, initializes loggers :param bool load_default_config: If false, does not load standard configuration. :param bool load_config: If true, loads configuration specified on a command line """ # Parse command line arguments parser = argparse.ArgumentParser() parser.add_argument( 'action', type=str, help= 'Action to perform. Valid actions: "print-config", "run", "build" and "analyze-plan".' ) parser.add_argument('--config', required=False, type=str, help='Configuration file (json) of an experiment.\ Will override values from default configuration.' ) parser.add_argument('--plan', required=False, type=str, help='Pre-built plan of an experiment (json).\ If action is "build", a file name to write plan to.\ If action is "run", a file name to read plan from.' ) parser.add_argument('--progress_file', '--progress-file', required=False, type=str, default=None, help='A JSON file that experimenter will be updating on its progress.'\ 'If not present, no progress info will be available.'\ 'Put it somewhere in /dev/shm') parser.add_argument( '-P', action='append', required=False, default=[], help='Parameters that override parameters in configuration file.\ For instance, -Pexp.phase=2. Values must be json parsable (json.loads()).' ) parser.add_argument( '-V', action='append', required=False, default=[], help= 'Variables that override variables in configuration file in section "variables". \ These variables are used to generate different combinations of experiments.\ For instance: -Vexp.framework=\'["tensorflow", "caffe2"]\'.\ Values must be json parsable (json.loads()).' ) parser.add_argument( '--log_level', '--log-level', required=False, default='info', help= 'Python logging level. Valid values: "critical", "error", "warning", "info" and "debug"' ) parser.add_argument('--discard_default_config', '--discard-default-config', required=False, default=False, action='store_true', help='Do not load default configuration.') parser.add_argument( '--no_validation', '--no-validation', required=False, default=False, action='store_true', help='Do not perform config validation before running benchmarks.') parser.add_argument( '-E', action='append', required=False, default=[], help= 'Extensions to add. Can be usefull to quickly customize experiments.\ Must be valid json parsable array element for "extension" array.' ) args = parser.parse_args() log_level = logging.getLevelName(args.log_level.upper()) self.action = args.action self.config_file = args.config self.plan_file = args.plan self.validation = not args.no_validation self.__progress_file = args.progress_file # Initialize logger if init_logger: logging.debug("Initializing logger to level %s", args.log_level) root = logging.getLogger() root.setLevel(log_level) handler = logging.StreamHandler(sys.stdout) handler.setLevel(log_level) root.addHandler(handler) logging.debug("Parsing parameters on a command line") DictUtils.add(self.params, args.P, pattern='(.+?(?=[=]))=(.+)', must_match=True) logging.debug("Parsing variables on a command line") DictUtils.add(self.variables, args.V, pattern='(.+?(?=[=]))=(.+)', must_match=True) # Load default configuration if load_default_config and not args.discard_default_config: logging.debug("Loading default configuration") _, self.config, self.param_info = ConfigurationLoader.load( os.path.join(os.path.dirname(__file__), 'configs')) # Load configurations specified on a command line if load_config: logging.debug("Loading user configuration") self.load_configuration() # Add extensions from command line DictUtils.ensure_exists(self.config, 'extensions', []) if len(args.E) > 0: logging.debug("Parsing extensions on a command line") for extension in args.E: try: ext = json.loads(extension) logging.debug('Found extension: %s', str(ext)) self.config['extensions'].append(ext) except Exception as err: logging.warn("Found non-json parsable extension: %s", extension) raise err