def analyse(args): """ Analyses the result for one, the most recent, or all tests """ # Parse test name, or get next test to run if args.all: names = sorted(pfunk.tests.tests()) elif args.last: names = [pfunk.find_previous_test()] elif args.name is None: names = [pfunk.find_next_test()] else: names = _parse_pattern(args.name) if not names: return failed = 0 for name in names: print('Analysing ' + name + ' ... ', end='') result = pfunk.tests.analyse(name) failed += 0 if result else 1 print('ok' if result else 'FAIL') if not result: print('{} failed'.format(name), file=sys.stderr) print() print('-' * 60) print('Ran ' + str(len(names)) + ' tests') if failed: print('Failed: ' + str(failed))
def run(args): """ Runs a test. """ # Parse test name, or get next test to run if args.name is None: names = [pfunk.find_next_test()] else: names = _parse_pattern(args.name) if not names: return # Update pints repository if not args.no_refresh: print('Refreshing Pints repository') pfunk.pintsrepo.pull() # Allow testing of older pints versions if args.t: pints_checkout, results_dir = args.t # Check analysing and plotting is disabled if args.analyse or args.plot or args.show: print('When testing specific commits or branches, plots and/or' ' analysis cannot be run.') sys.exit(1) # Change result directory results_dir = os.path.abspath(results_dir) if results_dir == pfunk._DIR_RESULT_DEFAULT: print('When testing specific commits or branches, an alternative' ' results directory must be specified.') sys.exit(1) pfunk.DIR_RESULT = results_dir # Check out alternative pints version print('Checking out ' + pints_checkout) pfunk.pintsrepo.checkout(pints_checkout) print(pfunk.pintsrepo.info()) # Prepare module pfunk.pintsrepo.prepare_module() # Run tests for name in names: for i in range(args.r): print('Running test ' + name) pfunk.tests.run(name) if args.analyse: print('Analysing ' + name + ' ... ', end='') result = pfunk.tests.analyse(name) print('ok' if result else 'FAIL') if args.plot or args.show: print('Creating plot for ' + name) pfunk.tests.plot(name, args.show) print('Done')
def weekend(args): """ Keep running tests, generating reports, and committing results. """ while True: pfunk.pfunkrepo.pull() pfunk.pfunkrepo.prepare_module() for i in range(10): name = pfunk.find_next_test() print('Running test ' + name) pfunk.tests.run(name) pfunk.tests.plot(name) print('Done') pfunk.generate_report() pfunk.resultsrepo.commit_results()
def list_tests(args): """ Shows all available tests and the date they were last run. """ if args.next: # Show next test only print(pfunk.find_next_test(args.database)) return # Show table of tests dates = pfunk.find_test_dates(args.database) w = max(4, max([len(k) for k in dates.keys()])) print('| Name' + ' ' * (w - 4) + ' | Last run |') print('-' * (w + 26)) for test in sorted(dates.items(), key=lambda x: x[1]): name, date = test print('| ' + name + ' ' * (w - len(name)) + ' | ' + pfunk.date(date) + ' |')
def run(args): """ Runs a test. """ # Parse test name, or get next test to run if args.name is None: names = [pfunk.find_next_test(args.database)] else: names = _parse_pattern(args.name) if not names: return # Update pints repository if not args.no_refresh: print('Refreshing Pints repository') pfunk.pintsrepo.pull() # Allow testing of older pints versions if args.commit: # Check analysing and plotting is disabled if args.analyse or args.plot or args.show: print('When testing specific commits, plots and/or' ' analysis cannot be run.') sys.exit(1) # Check out alternative pints version print(f'Checking out {args.commit}') pfunk.pintsrepo.checkout(args.commit) print(pfunk.pintsrepo.info()) # Prepare module pfunk.pintsrepo.prepare_module() pfunk.pfunkrepo.prepare_module() # Multi-processing nproc = min(args.r, multiprocessing.cpu_count() - 2) # Run tests for name in names: # Run the test args.r times if nproc > 1: # Run in parallel with multiprocessing.Pool(processes=nproc) as pool: print(f'Running {name} {args.r} times with {nproc} processes:', flush=True) # Starmap with product of name and # range: -> [(name, 0), (name, 1), ...] pool.starmap(pfunk.tests.run, product([name], [args.database], range(args.r))) else: # Run without multiprocessing print(f'Running {name} {args.r} times without multiprocessing') for i in range(args.r): pfunk.tests.run(name, args.database, i) if args.analyse: print('Analysing ' + name + ' ... ', end='') result = pfunk.tests.analyse(name, args.database) print('ok' if result else 'FAIL') if args.plot or args.show: print('Creating plot for ' + name) pfunk.tests.plot(name, args.database, args.show) print('Done')