def select_tests(all_tests, test_categories, selected_categories, prefix=''): '''Return the set of tests contained by the selected test categories.''' test_categories['_all_'] = [test.path for test in all_tests] if ('_default_' in selected_categories and '_default_' not in test_categories): selected_categories = ['_all_'] # Recursively expand job categories. while compat.compat_any(cat in test_categories for cat in selected_categories): tmp = [] for cat in selected_categories: if cat in test_categories: tmp.extend(test_categories[cat]) else: # cat has been fully expanded and now refers to a test # contained within the directory named cat. tmp.append(cat) selected_categories = tmp # Select tests to run. tests = [] parent = lambda pdir, cdir: \ not os.path.relpath(cdir, start=pdir).startswith(os.pardir) for cat in selected_categories: # test paths are relative to the config directory but absolute paths # are stored . found = False cat_paths = glob.glob(os.path.join(prefix, cat)) for test in all_tests: if cat == test.name: found = True tests.append(test) elif compat.compat_any( os.path.exists(path) and os.path.samefile(path, test.path) for path in cat_paths): found = True tests.append(test) elif compat.compat_any( parent(path, test.path) for path in cat_paths): # test contained within a subdirectory of a cat_path. found = True tests.append(test) if not found: print('WARNING: %s test/category not found.\n' % cat) # Only want to run each test once. tests = list(compat.compat_set(tests)) return tests
def select_tests(all_tests, test_categories, selected_categories, prefix=''): '''Return the set of tests contained by the selected test categories.''' test_categories['_all_'] = [test.path for test in all_tests] if ('_default_' in selected_categories and '_default_' not in test_categories): selected_categories = ['_all_'] # Recursively expand job categories. while compat.compat_any( cat in test_categories for cat in selected_categories ): tmp = [] for cat in selected_categories: if cat in test_categories: tmp.extend(test_categories[cat]) else: # cat has been fully expanded and now refers to a test # contained within the directory named cat. tmp.append(cat) selected_categories = tmp # Select tests to run. tests = [] parent = lambda pdir, cdir: \ not os.path.relpath(cdir, start=pdir).startswith(os.pardir) for cat in selected_categories: # test paths are relative to the config directory but absolute paths # are stored . found = False cat_paths = glob.glob(os.path.join(prefix, cat)) for test in all_tests: if cat == test.name: found = True tests.append(test) elif compat.compat_any(os.path.exists(path) and os.path.samefile(path, test.path) for path in cat_paths): found = True tests.append(test) elif compat.compat_any(parent(path, test.path) for path in cat_paths): # test contained within a subdirectory of a cat_path. found = True tests.append(test) if not found: print('WARNING: %s test/category not found.\n' % cat) # Only want to run each test once. tests = list(compat.compat_set(tests)) return tests
def __init__(self, bools=None, status=None, name=None): (self._unknown, self._skipped) = (-2, -1) (self._passed, self._partial, self._failed) = (0, 1, 2) if name is not None: setattr(self, 'status', getattr(self, '_' + name)) elif status is not None: self.status = status elif bools: if compat.compat_all(bools): self.status = self._passed elif compat.compat_any(bools): self.status = self._partial else: self.status = self._failed else: self.status = self._unknown
def __init__(self, bools=None, status=None, name=None): (self._unknown, self._skipped) = (-2, -1) (self._passed, self._partial, self._failed) = (0, 1, 2) if name is not None: setattr(self, 'status', getattr(self, '_'+name)) elif status is not None: self.status = status elif bools: if compat.compat_all(bools): self.status = self._passed elif compat.compat_any(bools): self.status = self._partial else: self.status = self._failed else: self.status = self._unknown
def run_test(self, verbose=1, cluster_queue=None, rundir=None): '''Run all jobs in test.''' try: # Construct tests. test_cmds = [] test_files = [] for (test_input, test_arg) in self.inputs_args: if (test_input and not os.path.exists( os.path.join(self.path, test_input))): err = 'Input file does not exist: %s' % (test_input, ) raise exceptions.RunError(err) test_cmds.append( self.test_program.run_cmd(test_input, test_arg, self.nprocs)) test_files.append( util.testcode_filename(FILESTEM['test'], self.test_program.test_id, test_input, test_arg)) # Move files matching output pattern out of the way. self.move_old_output_files(verbose) # Run tests one-at-a-time locally or submit job in single submit # file to a queueing system. if cluster_queue: if self.output: for (ind, test) in enumerate(test_cmds): # Don't quote self.output if it contains any wildcards # (assume the user set it up correctly!) out = self.output if not compat.compat_any( wild in self.output for wild in ['*', '?', '[', '{']): out = pipes.quote(self.output) test_cmds[ind] = '%s; mv %s %s' % ( test_cmds[ind], out, pipes.quote(test_files[ind])) test_cmds = ['\n'.join(test_cmds)] for (ind, test) in enumerate(test_cmds): job = self.start_job(test, cluster_queue, verbose) job.wait() # Analyse tests as they finish. if cluster_queue: # Did all of them at once. for (test_input, test_arg) in self.inputs_args: self.verify_job(test_input, test_arg, verbose, rundir) else: # Did one job at a time. (test_input, test_arg) = self.inputs_args[ind] err = [] if self.output: try: self.move_output_to_test_output(test_files[ind]) except exceptions.RunError: err.append(sys.exc_info()[1]) status = validation.Status() if job.returncode != 0: err.insert( 0, 'Error running job. Return code: %i' % job.returncode) (status, msg) = self.skip_job(test_input, test_arg, verbose) if status.skipped(): self._update_status(status, (test_input, test_arg)) if verbose > 0 and verbose < 3: sys.stdout.write( util.info_line(self.path, test_input, test_arg, rundir)) status.print_status(msg, verbose) elif err: # re-raise first error we hit. raise exceptions.RunError(err[0]) else: self.verify_job(test_input, test_arg, verbose, rundir) except exceptions.RunError: err = sys.exc_info()[1] if verbose > 2: err = 'Test(s) in %s failed.\n%s' % (self.path, err) status = validation.Status([False]) self._update_status(status, (test_input, test_arg)) if verbose > 0 and verbose < 3: info_line = util.info_line(self.path, test_input, test_arg, rundir) sys.stdout.write(info_line) status.print_status(err, verbose) # Shouldn't run remaining tests after such a catastrophic failure. # Mark all remaining tests as skipped so the user knows that they # weren't run. err = 'Previous test in %s caused a system failure.' % (self.path) status = validation.Status(name='skipped') for ((test_input, test_arg), stat) in self.status.items(): if not self.status[(test_input, test_arg)]: self._update_status(status, (test_input, test_arg)) if verbose > 2: cmd = self.test_program.run_cmd( test_input, test_arg, self.nprocs) print('Test using %s in %s' % (cmd, self.path)) elif verbose > 0: info_line = util.info_line(self.path, test_input, test_arg, rundir) sys.stdout.write(info_line) status.print_status(err, verbose)
def run_test(self, verbose=1, cluster_queue=None, rundir=None): '''Run all jobs in test.''' try: # Construct tests. test_cmds = [] test_files = [] for (test_input, test_arg) in self.inputs_args: if (test_input and not os.path.exists(os.path.join(self.path,test_input))): err = 'Input file does not exist: %s' % (test_input,) raise exceptions.RunError(err) test_cmds.append(self.test_program.run_cmd(test_input, test_arg, self.nprocs)) test_files.append(util.testcode_filename(FILESTEM['test'], self.test_program.test_id, test_input, test_arg)) # Move files matching output pattern out of the way. self.move_old_output_files(verbose) # Run tests one-at-a-time locally or submit job in single submit # file to a queueing system. if cluster_queue: if self.output: for (ind, test) in enumerate(test_cmds): # Don't quote self.output if it contains any wildcards # (assume the user set it up correctly!) out = self.output if not compat.compat_any(wild in self.output for wild in ['*', '?', '[', '{']): out = pipes.quote(self.output) test_cmds[ind] = '%s; mv %s %s' % (test_cmds[ind], out, pipes.quote(test_files[ind])) test_cmds = ['\n'.join(test_cmds)] for (ind, test) in enumerate(test_cmds): job = self.start_job(test, cluster_queue, verbose) job.wait() # Analyse tests as they finish. if cluster_queue: # Did all of them at once. for (test_input, test_arg) in self.inputs_args: self.verify_job(test_input, test_arg, verbose, rundir) else: # Did one job at a time. (test_input, test_arg) = self.inputs_args[ind] err = [] if self.output: try: self.move_output_to_test_output(test_files[ind]) except exceptions.RunError: err.append(sys.exc_info()[1]) status = validation.Status() if job.returncode != 0: err.insert(0, 'Error running job. Return code: %i' % job.returncode) (status, msg) = self.skip_job(test_input, test_arg, verbose) if status.skipped(): self._update_status(status, (test_input, test_arg)) if verbose > 0 and verbose < 3: sys.stdout.write( util.info_line(self.path, test_input, test_arg, rundir) ) status.print_status(msg, verbose) elif err: # re-raise first error we hit. raise exceptions.RunError(err[0]) else: self.verify_job(test_input, test_arg, verbose, rundir) except exceptions.RunError: err = sys.exc_info()[1] if verbose > 2: err = 'Test(s) in %s failed.\n%s' % (self.path, err) status = validation.Status([False]) self._update_status(status, (test_input, test_arg)) if verbose > 0 and verbose < 3: info_line = util.info_line(self.path, test_input, test_arg, rundir) sys.stdout.write(info_line) status.print_status(err, verbose) # Shouldn't run remaining tests after such a catastrophic failure. # Mark all remaining tests as skipped so the user knows that they # weren't run. err = 'Previous test in %s caused a system failure.' % (self.path) status = validation.Status(name='skipped') for ((test_input, test_arg), stat) in self.status.items(): if not self.status[(test_input,test_arg)]: self._update_status(status, (test_input, test_arg)) if verbose > 2: cmd = self.test_program.run_cmd(test_input, test_arg, self.nprocs) print('Test using %s in %s' % (cmd, self.path)) elif verbose > 0: info_line = util.info_line(self.path, test_input, test_arg, rundir) sys.stdout.write(info_line) status.print_status(err, verbose)