def run_batch(self, batch_name): if not batch_name in self.batches: raise RuntimeError("Can't find batch '%s' to run." % batch_name) batch = self.batches[batch_name].copy() batch.update(self.settings.BATCH_OVERRIDE) # A batch declared 'abstract' is not runnable if batch.get('abstract', False): sys.stderr.write(" Batch marked as abstract. Not running.\n") return False elif batch.get('disabled', False) or not batch.get('enabled', True): sys.stderr.write(" Batch disabled.\n") return False argsets = self.get_argsets(batch) pause = int(batch.get('pause', 0)) batch_time = None if self.settings.BATCH_RESUME is not None and os.path.isdir( self.settings.BATCH_RESUME): # We're resuming a batch run. Try to find a data file we can get the # original batch run time from. for dirpath, dirnames, filenames in os.walk( self.settings.BATCH_RESUME): datafiles = [ f for f in filenames if f.endswith(resultset.SUFFIX) ] if datafiles: f = datafiles[0] r = resultset.load(os.path.join(dirpath, f)) batch_time = r.meta("BATCH_TIME") break if batch_time is None: raise RuntimeError( "No data files found in resume directory %s." % self.settings.BATCH_RESUME) elif self.settings.BATCH_RESUME: raise RuntimeError("Batch resume directory %s doesn't exist!\n" % self.settings.BATCH_RESUME) else: batch_time = self.settings.TIME for b, settings in self.expand_argsets(batch, argsets, batch_time, batch_name): if 'output_path' in b: output_path = clean_path(b['output_path'], allow_dirs=True) else: output_path = settings.DATA_DIR if settings.BATCH_RESUME is not None: if os.path.commonprefix([ os.path.abspath(output_path), os.path.abspath(settings.BATCH_RESUME) ]) != os.path.abspath(settings.BATCH_RESUME): raise RuntimeError( "Batch-specified output path is not a subdirectory of resume path. Bailing." ) if os.path.exists( os.path.join( output_path, "%s%s" % (settings.DATA_FILENAME, resultset.SUFFIX))): sys.stderr.write(" Previous result exists, skipping.\n") continue if settings.BATCH_DRY and settings.BATCH_VERBOSE: sys.stderr.write(" Would output to: %s.\n" % output_path) elif not settings.BATCH_DRY and not os.path.exists(output_path): try: os.makedirs(output_path) except OSError as e: raise RuntimeError( "Unable to create output path '%s': %s." % (output_path, e)) commands = self.commands_for(b, settings) if not settings.BATCH_DRY: self.log_fd = io.open( os.path.join(output_path, "%s.log" % settings.DATA_FILENAME), "at") if b.get('debug_log', False): settings.LOG_FILE = os.path.join( output_path, "%s.debug.log" % settings.DATA_FILENAME) self.run_commands(commands, 'pre') self.run_commands(commands, 'monitor') try: if settings.BATCH_VERBOSE: if settings.BATCH_DRY: sys.stderr.write(" Would run test '%s'.\n" % settings.NAME) else: sys.stderr.write(" Running test '%s'.\n" % settings.NAME) sys.stderr.write(" data_filename=%s\n" % settings.DATA_FILENAME) for k in sorted([i.lower() for i in CONFIG_TYPES.keys()]): if k in b: sys.stderr.write(" %s=%s\n" % (k, b[k])) if settings.BATCH_DRY: self.tests_run += 1 else: self.run_test(settings, output_path) except KeyboardInterrupt: self.run_commands(commands, 'post', essential_only=True) raise except: self.run_commands(commands, 'post', essential_only=True) sys.stderr.write(" Error running test: %s\n" % " ".join( traceback.format_exception_only(*sys.exc_info()[:2]))) else: try: self.run_commands(commands, 'post') except: self.run_commands(commands, 'post', essential_only=True) sys.stderr.write(" Error running post-commands: %s\n" % " ".join( traceback.format_exception_only( *sys.exc_info()[:2]))) finally: self.kill_children() if self.log_fd: self.log_fd.close() self.log_fd = None if settings.BATCH_DRY and settings.BATCH_VERBOSE: sys.stderr.write(" Would sleep for %d seconds.\n" % pause) elif not settings.BATCH_DRY: time.sleep(pause)
def run_batch(self, batch_name): if not batch_name in self.batches: raise RuntimeError("Can't find batch '%s' to run." % batch_name) batch = self.batches[batch_name].copy() batch.update(self.settings.BATCH_OVERRIDE) # A batch declared 'abstract' is not runnable if batch.get('abstract', False): sys.stderr.write(" Batch marked as abstract. Not running.\n") return False elif batch.get('disabled', False) or not batch.get('enabled', True): sys.stderr.write(" Batch disabled.\n") return False argsets = self.get_argsets(batch) pause = int(batch.get('pause', 0)) batch_time = None if self.settings.BATCH_RESUME is not None and os.path.isdir(self.settings.BATCH_RESUME): # We're resuming a batch run. Try to find a data file we can get the # original batch run time from. for dirpath, dirnames, filenames in os.walk(self.settings.BATCH_RESUME): datafiles = [f for f in filenames if f.endswith(resultset.SUFFIX)] if datafiles: f = datafiles[0] r = resultset.load(os.path.join(dirpath, f)) batch_time = r.meta("BATCH_TIME") try: self.settings.BATCH_UUID = r.meta("BATCH_UUID") sys.stderr.write(" Using previous UUID %s.\n" % self.settings.BATCH_UUID) except KeyError: pass break if batch_time is None: raise RuntimeError("No data files found in resume directory %s." % self.settings.BATCH_RESUME) elif self.settings.BATCH_RESUME: raise RuntimeError("Batch resume directory %s doesn't exist!\n" % self.settings.BATCH_RESUME) else: batch_time = self.settings.TIME filenames_seen = set() for b,settings in self.expand_argsets(batch, argsets, batch_time, batch_name): if 'output_path' in b: output_path = clean_path(b['output_path'], allow_dirs=True) else: output_path = settings.DATA_DIR if settings.BATCH_RESUME is not None: if os.path.commonprefix([os.path.abspath(output_path), os.path.abspath(settings.BATCH_RESUME)]) != os.path.abspath(settings.BATCH_RESUME): raise RuntimeError("Batch-specified output path is not a subdirectory of resume path. Bailing.") if os.path.exists(os.path.join(output_path, "%s%s" % (settings.DATA_FILENAME, resultset.SUFFIX))): sys.stderr.write(" Previous result exists, skipping.\n") continue if settings.BATCH_DRY and settings.BATCH_VERBOSE: sys.stderr.write(" Would output to: %s.\n" % output_path) elif not settings.BATCH_DRY and not os.path.exists(output_path): try: os.makedirs(output_path) except OSError as e: raise RuntimeError("Unable to create output path '%s': %s." % (output_path,e)) commands = self.commands_for(b, settings) if not settings.BATCH_DRY: self.log_fd = io.open(os.path.join(output_path,"%s.log" % settings.DATA_FILENAME), "at") if b.get('debug_log', False): settings.LOG_FILE = os.path.join(output_path,"%s.debug.log" % settings.DATA_FILENAME) if settings.DATA_FILENAME in filenames_seen: sys.stderr.write(" Warning: Filename already seen in this run: %s\n" % settings.DATA_FILENAME) filenames_seen.add(settings.DATA_FILENAME) self.run_commands(commands, 'pre') self.run_commands(commands, 'monitor') try: if settings.BATCH_VERBOSE: if settings.BATCH_DRY: sys.stderr.write(" Would run test '%s'.\n" % settings.NAME) else: sys.stderr.write(" Running test '%s'.\n" % settings.NAME) sys.stderr.write(" data_filename=%s\n" % settings.DATA_FILENAME) for k in sorted([i.lower() for i in CONFIG_TYPES.keys()]): if k in b: sys.stderr.write(" %s=%s\n" % (k, b[k])) if settings.BATCH_DRY: self.tests_run += 1 else: self.run_test(settings, output_path) except KeyboardInterrupt: self.run_commands(commands, 'post', essential_only=True) raise except: self.run_commands(commands, 'post', essential_only=True) sys.stderr.write(" Error running test: %s\n" % " ".join(traceback.format_exception_only(*sys.exc_info()[:2]))) if settings.DEBUG_ERROR: traceback.print_exc() else: try: self.run_commands(commands, 'post') except: self.run_commands(commands, 'post', essential_only=True) sys.stderr.write(" Error running post-commands: %s\n" % " ".join(traceback.format_exception_only(*sys.exc_info()[:2]))) if settings.DEBUG_ERROR: traceback.print_exc() finally: self.kill_children() if self.log_fd: self.log_fd.close() self.log_fd = None if settings.BATCH_DRY and settings.BATCH_VERBOSE: sys.stderr.write(" Would sleep for %d seconds.\n" % pause) elif not settings.BATCH_DRY: time.sleep(pause)
def run_batch(self, batchname): if not batchname in self.batches: raise RuntimeError("Can't find batch '%s' to run." % batchname) batch = self.batches[batchname] batch.update(self.settings.BATCH_OVERRIDE) # A batch declared 'abstract' is not runnable if batch.get('abstract', False): sys.stderr.write(" Batch marked as abstract. Not running.\n") return False elif batch.get('disabled', False) or not batch.get('enabled', True): sys.stderr.write(" Batch disabled.\n") return False argsets = [] for k in batch.keys(): if k.startswith("for_"): argset = [] for a in batch[k].split(','): a = a.strip().lower() matches = [arg for arg in self.args if fnmatch(arg, a)] if not matches: raise RuntimeError("No matches for arg: '%s'." % a) argset.extend(matches) argsets.append(argset) reps = range(1, int(batch.get('repetitions', 1)) + 1) argsets.append(reps) pause = int(batch.get('pause', 0)) batch_time = None if self.settings.BATCH_RESUME is not None and os.path.isdir( self.settings.BATCH_RESUME): # We're resuming a batch run. Try to find a data file we can get the # original batch run time from. for dirpath, dirnames, filenames in os.walk( self.settings.BATCH_RESUME): datafiles = [ f for f in filenames if f.endswith(resultset.SUFFIX) ] if datafiles: f = datafiles[0] r = resultset.load(os.path.join(dirpath, f)) batch_time = r.meta("BATCH_TIME") break if batch_time is None: raise RuntimeError( "No data files found in resume directory %s." % self.settings.BATCH_RESUME) elif self.settings.BATCH_RESUME: raise RuntimeError("Batch resume directory %s doesn't exist!\n" % self.settings.BATCH_RESUME) else: batch_time = self.settings.TIME for argset in itertools.product(*argsets): rep = argset[-1] argset = argset[:-1] settings = self.settings.copy() sys.stderr.write(" args:%s rep:%02d" % (",".join(argset), rep)) if settings.BATCH_DRY: sys.stderr.write(" (dry run)") sys.stderr.write(".\n") settings.FORMAT = 'null' settings.BATCH_NAME = batchname settings.BATCH_TIME = batch_time settings.TIME = datetime.now() expand_vars = { 'repetition': "%02d" % rep, 'batch_time': settings.BATCH_TIME.strftime("%Y-%m-%dT%H%M%S") } for arg in argset: if not arg in self.args: raise RuntimeError("Invalid arg: '%s'." % arg) expand_vars.update(self.args[arg]) b = self.apply_args(batch, expand_vars, settings) if not 'test_name' in b: raise RuntimeError("Missing test name.") settings.load_rcvalues(b.items(), override=True) settings.NAME = b['test_name'] settings.load_test(informational=settings.BATCH_DRY) settings.DATA_FILENAME = self.gen_filename(settings, b, argset, rep) if 'output_path' in b: output_path = clean_path(b['output_path'], allow_dirs=True) else: output_path = settings.DATA_DIR if settings.BATCH_RESUME is not None: if os.path.commonprefix([ os.path.abspath(output_path), os.path.abspath(settings.BATCH_RESUME) ]) != os.path.abspath(settings.BATCH_RESUME): raise RuntimeError( "Batch-specified output path is not a subdirectory of resume path. Bailing." ) if os.path.exists( os.path.join( output_path, "%s%s" % (settings.DATA_FILENAME, resultset.SUFFIX))): sys.stderr.write(" Previous result exists, skipping.\n") continue if settings.BATCH_DRY and settings.BATCH_VERBOSE: sys.stderr.write(" Would output to: %s.\n" % output_path) elif not settings.BATCH_DRY and not os.path.exists(output_path): try: os.makedirs(output_path) except OSError as e: raise RuntimeError( "Unable to create output path '%s': %s." % (output_path, e)) commands = self.commands_for(b, settings) if not settings.BATCH_DRY: self.log_fd = io.open( os.path.join(output_path, "%s.log" % settings.DATA_FILENAME), "at") if b.get('debug_log', False): settings.LOG_FILE = os.path.join( output_path, "%s.debug.log" % settings.DATA_FILENAME) self.run_commands(commands, 'pre') self.run_commands(commands, 'monitor') try: if settings.BATCH_VERBOSE: if settings.BATCH_DRY: sys.stderr.write(" Would run test '%s'.\n" % settings.NAME) else: sys.stderr.write(" Running test '%s'.\n" % settings.NAME) sys.stderr.write(" data_filename=%s\n" % settings.DATA_FILENAME) for k in sorted([i.lower() for i in CONFIG_TYPES.keys()]): if k in b: sys.stderr.write(" %s=%s\n" % (k, b[k])) if settings.BATCH_DRY: self.tests_run += 1 else: self.run_test(settings, output_path) except KeyboardInterrupt: self.run_commands(commands, 'post', essential_only=True) raise except: self.run_commands(commands, 'post', essential_only=True) sys.stderr.write(" Error running test: %s\n" % " ".join( traceback.format_exception_only(*sys.exc_info()[:2]))) else: try: self.run_commands(commands, 'post') except: self.run_commands(commands, 'post', essential_only=True) sys.stderr.write(" Error running post-commands: %s\n" % " ".join( traceback.format_exception_only( *sys.exc_info()[:2]))) finally: self.kill_children() if self.log_fd: self.log_fd.close() self.log_fd = None if settings.BATCH_DRY and settings.BATCH_VERBOSE: sys.stderr.write(" Would sleep for %d seconds.\n" % pause) elif not settings.BATCH_DRY: time.sleep(pause)
def run_batch(self, batchname): if not batchname in self.batches: raise RuntimeError("Can't find batch '%s' to run." % batchname) batch = self.batches[batchname] batch.update(self.settings.BATCH_OVERRIDE) # A batch declared 'abstract' is not runnable if batch.get('abstract', False): sys.stderr.write(" Batch marked as abstract. Not running.\n") return False elif batch.get('disabled', False) or not batch.get('enabled', True): sys.stderr.write(" Batch disabled.\n") return False argsets = [] for k in batch.keys(): if k.startswith("for_"): argset = [] for a in batch[k].split(','): a = a.strip().lower() matches = [arg for arg in self.args if fnmatch(arg,a)] if not matches: raise RuntimeError("No matches for arg: '%s'." % a) argset.extend(matches) argsets.append(argset) reps = range(1,int(batch.get('repetitions', 1))+1) argsets.append(reps) pause = int(batch.get('pause', 0)) batch_time = None if self.settings.BATCH_RESUME is not None and os.path.isdir(self.settings.BATCH_RESUME): # We're resuming a batch run. Try to find a data file we can get the # original batch run time from. for dirpath, dirnames, filenames in os.walk(self.settings.BATCH_RESUME): datafiles = [f for f in filenames if f.endswith(resultset.SUFFIX)] if datafiles: f = datafiles[0] r = resultset.load(os.path.join(dirpath, f)) batch_time = r.meta("BATCH_TIME") break if batch_time is None: raise RuntimeError("No data files found in resume directory %s." % self.settings.BATCH_RESUME) elif self.settings.BATCH_RESUME: raise RuntimeError("Batch resume directory %s doesn't exist!\n" % self.settings.BATCH_RESUME) else: batch_time = self.settings.TIME for argset in itertools.product(*argsets): rep = argset[-1] argset = argset[:-1] settings = self.settings.copy() sys.stderr.write(" args:%s rep:%02d" % (",".join(argset),rep)) if settings.BATCH_DRY: sys.stderr.write(" (dry run)") sys.stderr.write(".\n") settings.FORMAT = 'null' settings.BATCH_NAME = batchname settings.BATCH_TIME = batch_time settings.TIME = datetime.now() expand_vars = {'repetition': "%02d" % rep, 'batch_time': settings.BATCH_TIME.strftime("%Y-%m-%dT%H%M%S")} for arg in argset: if not arg in self.args: raise RuntimeError("Invalid arg: '%s'." % arg) expand_vars.update(self.args[arg]) b = self.apply_args(batch, expand_vars, settings) if not 'test_name' in b: raise RuntimeError("Missing test name.") settings.load_rcvalues(b.items(), override=True) settings.NAME = b['test_name'] settings.load_test(informational=settings.BATCH_DRY) settings.DATA_FILENAME = self.gen_filename(settings, b, argset, rep) if 'output_path' in b: output_path = clean_path(b['output_path'], allow_dirs=True) else: output_path = settings.DATA_DIR if settings.BATCH_RESUME is not None: if os.path.commonprefix([os.path.abspath(output_path), os.path.abspath(settings.BATCH_RESUME)]) != os.path.abspath(settings.BATCH_RESUME): raise RuntimeError("Batch-specified output path is not a subdirectory of resume path. Bailing.") if os.path.exists(os.path.join(output_path, "%s%s" % (settings.DATA_FILENAME, resultset.SUFFIX))): sys.stderr.write(" Previous result exists, skipping.\n") continue if settings.BATCH_DRY and settings.BATCH_VERBOSE: sys.stderr.write(" Would output to: %s.\n" % output_path) elif not settings.BATCH_DRY and not os.path.exists(output_path): try: os.makedirs(output_path) except OSError as e: raise RuntimeError("Unable to create output path '%s': %s." % (output_path,e)) commands = self.commands_for(b, settings) if not settings.BATCH_DRY: self.log_fd = io.open(os.path.join(output_path,"%s.log" % settings.DATA_FILENAME), "at") if b.get('debug_log', False): settings.LOG_FILE = os.path.join(output_path,"%s.debug.log" % settings.DATA_FILENAME) self.run_commands(commands, 'pre') self.run_commands(commands, 'monitor') try: if settings.BATCH_VERBOSE: if settings.BATCH_DRY: sys.stderr.write(" Would run test '%s'.\n" % settings.NAME) else: sys.stderr.write(" Running test '%s'.\n" % settings.NAME) sys.stderr.write(" data_filename=%s\n" % settings.DATA_FILENAME) for k in sorted([i.lower() for i in CONFIG_TYPES.keys()]): if k in b: sys.stderr.write(" %s=%s\n" % (k, b[k])) if settings.BATCH_DRY: self.tests_run += 1 else: self.run_test(settings, output_path) except KeyboardInterrupt: self.run_commands(commands, 'post', essential_only=True) raise except: self.run_commands(commands, 'post', essential_only=True) sys.stderr.write(" Error running test: %s\n" % " ".join(traceback.format_exception_only(*sys.exc_info()[:2]))) else: try: self.run_commands(commands, 'post') except: self.run_commands(commands, 'post', essential_only=True) sys.stderr.write(" Error running post-commands: %s\n" % " ".join(traceback.format_exception_only(*sys.exc_info()[:2]))) finally: self.kill_children() if self.log_fd: self.log_fd.close() self.log_fd = None if settings.BATCH_DRY and settings.BATCH_VERBOSE: sys.stderr.write(" Would sleep for %d seconds.\n" % pause) elif not settings.BATCH_DRY: time.sleep(pause)