def cprint(msg, color, bold=False, file=None): if file is None: file = sys.stderr if color == 'red' else sys.stdout if USE_COLORS: print_color(msg, color, bold=bold, file=file) else: print(msg, file=file)
def main(): # collect tot_procs = 0 tot_ads = 0 tot_calls = 0 signaler = object() d = defaultdict(int) start = time.time() for p in psutil.process_iter(attrs=[], ad_value=signaler): tot_procs += 1 for methname, value in p.info.items(): tot_calls += 1 if value is signaler: tot_ads += 1 d[methname] += 1 else: d[methname] += 0 elapsed = time.time() - start # print templ = "%-20s %-5s %-9s %s" s = templ % ("API", "AD", "Percent", "Outcome") print_color(s, color=None, bold=True) for methname, ads in sorted(d.items(), key=lambda x: (x[1], x[0])): perc = (ads / tot_procs) * 100 outcome = "SUCCESS" if not ads else "ACCESS DENIED" s = templ % (methname, ads, "%6.1f%%" % perc, outcome) print_color(s, "red" if ads else None) tot_perc = round((tot_ads / tot_calls) * 100, 1) print("-" * 50) print("Totals: access-denied=%s (%s%%), calls=%s, processes=%s, " "elapsed=%ss" % (tot_ads, tot_perc, tot_calls, tot_procs, round(elapsed, 2)))
def run(options): safe_rmtree('dist') urls = get_file_urls(options) completed = 0 exc = None with concurrent.futures.ThreadPoolExecutor() as e: fut_to_url = {e.submit(download_file, url): url for url in urls} for fut in concurrent.futures.as_completed(fut_to_url): url = fut_to_url[fut] try: local_fname = fut.result() except Exception: print_color("error while downloading %s" % (url), 'red') raise else: completed += 1 print("downloaded %-45s %s" % (local_fname, bytes2human(os.path.getsize(local_fname)))) # 2 wheels (32 and 64 bit) per supported python version expected = len(PY_VERSIONS) * 2 if expected != completed: return exit("expected %s files, got %s" % (expected, completed)) if exc: return exit() rename_27_wheels()
def main(): _setup() usage = "python3 -m psutil.tests [opts] [test-name]" parser = optparse.OptionParser(usage=usage, description="run unit tests") parser.add_option("--last-failed", action="store_true", default=False, help="only run last failed tests") parser.add_option("--parallel", action="store_true", default=False, help="run tests in parallel") opts, args = parser.parse_args() if not opts.last_failed: safe_rmpath(FAILED_TESTS_FNAME) # test-by-name if args: if len(args) > 1: parser.print_usage() return sys.exit(1) return runner.run_from_name(args[0]) elif opts.last_failed: runner.run_last_failed() elif not opts.parallel: runner.run() # parallel elif concurrencytest is None: print_color("concurrencytest module is not installed; " "running serial tests instead", "red") runner.run() elif NWORKERS == 1: print_color("only 1 CPU; running serial tests instead", "red") runner.run() else: runner.run_parallel()
def _finalize(self, success): if success: safe_rmpath(FAILED_TESTS_FNAME) else: self._write_last_failed() print_color("FAILED", "red") sys.exit(1)
def _exit(self, success): if success: print_color("SUCCESS", "green", bold=True) safe_rmpath(FAILED_TESTS_FNAME) sys.exit(0) else: print_color("FAILED", "red", bold=True) self._write_last_failed() sys.exit(1)
def print_timings(): timings.sort(key=lambda x: x[1]) i = 0 while timings[:]: title, elapsed = timings.pop(0) s = templ % (title, "%f" % elapsed) if i > len(timings) - 5: print_color(s, color="red") else: print(s)
def main(): groups = collections.defaultdict(list) for path in glob.glob('dist/*.whl'): wheel = Wheel(path) groups[wheel.platform()].append(wheel) tot_files = 0 tot_size = 0 templ = "%-54s %7s %7s %7s" for platf, wheels in groups.items(): ppn = "%s (total = %s)" % (platf, len(wheels)) s = templ % (ppn, "size", "arch", "pyver") print_color('\n' + s, color=None, bold=True) for wheel in sorted(wheels, key=lambda x: x.name): tot_files += 1 tot_size += wheel.size() s = templ % (wheel.name, bytes2human(wheel.size()), wheel.arch(), wheel.pyver()) if 'pypy' in wheel.pyver(): print_color(s, color='violet') else: print_color(s, color='brown') print_color("\ntotals: files=%s, size=%s" % ( tot_files, bytes2human(tot_size)), bold=True)
def run(self, suite): ser_suite, par_suite = self._split_suite(suite) par_suite = self._parallelize(par_suite) # run parallel print_color("starting parallel tests using %s workers" % NWORKERS, "green", bold=True) t = time.time() par = self._run(par_suite) par_elapsed = time.time() - t # At this point we should have N zombies (the workers), which # will disappear with wait(). orphans = psutil.Process().children() gone, alive = psutil.wait_procs(orphans, timeout=1) if alive: print_color("alive processes %s" % alive, "red") reap_children() # run serial t = time.time() ser = self._run(ser_suite) ser_elapsed = time.time() - t # print if not par.wasSuccessful() and ser_suite.countTestCases() > 0: par.printErrors() # print them again at the bottom par_fails, par_errs, par_skips = map( len, (par.failures, par.errors, par.skipped)) ser_fails, ser_errs, ser_skips = map( len, (ser.failures, ser.errors, ser.skipped)) print( textwrap.dedent(""" +----------+----------+----------+----------+----------+----------+ | | total | failures | errors | skipped | time | +----------+----------+----------+----------+----------+----------+ | parallel | %3s | %3s | %3s | %3s | %.2fs | +----------+----------+----------+----------+----------+----------+ | serial | %3s | %3s | %3s | %3s | %.2fs | +----------+----------+----------+----------+----------+----------+ """ % (par.testsRun, par_fails, par_errs, par_skips, par_elapsed, ser.testsRun, ser_fails, ser_errs, ser_skips, ser_elapsed))) print( "Ran %s tests in %.3fs using %s workers" % (par.testsRun + ser.testsRun, par_elapsed + ser_elapsed, NWORKERS)) ok = par.wasSuccessful() and ser.wasSuccessful() self._exit(ok)
def run_parallel(self): """Run tests in parallel.""" ser_suite, par_suite = self.loader.parallel() par_suite = self._parallelize_suite(par_suite) # run parallel print("starting parallel tests using %s workers" % NWORKERS) t = time.time() par = self._run(par_suite) par_elapsed = time.time() - t # cleanup workers and test subprocesses orphans = psutil.Process().children() gone, alive = psutil.wait_procs(orphans, timeout=1) if alive: print_color("alive processes %s" % alive, "red") reap_children() # run serial t = time.time() ser = self._run(ser_suite) ser_elapsed = time.time() - t # print if not par.wasSuccessful(): par.printErrors() # print them again at the bottom par_fails, par_errs, par_skips = map( len, (par.failures, par.errors, par.skipped)) ser_fails, ser_errs, ser_skips = map( len, (ser.failures, ser.errors, ser.skipped)) print("-" * 70) print( textwrap.dedent(""" +----------+----------+----------+----------+----------+----------+ | | total | failures | errors | skipped | time | +----------+----------+----------+----------+----------+----------+ | parallel | %3s | %3s | %3s | %3s | %.2fs | +----------+----------+----------+----------+----------+----------+ | serial | %3s | %3s | %3s | %3s | %.2fs | +----------+----------+----------+----------+----------+----------+ """ % (par.testsRun, par_fails, par_errs, par_skips, par_elapsed, ser.testsRun, ser_fails, ser_errs, ser_skips, ser_elapsed))) print( "Ran %s tests in %.3fs using %s workers" % (par.testsRun + ser.testsRun, par_elapsed + ser_elapsed, NWORKERS)) ok = par.wasSuccessful() and ser.wasSuccessful() self._finalize(ok)
def main(): # --- system public_apis = [] ignore = [ 'wait_procs', 'process_iter', 'win_service_get', 'win_service_iter' ] if psutil.MACOS: ignore.append('net_connections') # raises AD for name in psutil.__all__: obj = getattr(psutil, name, None) if inspect.isfunction(obj): if name not in ignore: public_apis.append(name) print_color(templ % ("SYSTEM APIS", "SECONDS"), color=None, bold=True) print("-" * 34) for name in public_apis: fun = getattr(psutil, name) args = () if name == 'pid_exists': args = (os.getpid(), ) elif name == 'disk_usage': args = (os.getcwd(), ) timecall(name, fun, *args) timecall('cpu_count (cores)', psutil.cpu_count, logical=False) timecall('process_iter (all)', lambda: list(psutil.process_iter())) print_timings() # --- process print("") print_color(templ % ("PROCESS APIS", "SECONDS"), color=None, bold=True) print("-" * 34) ignore = [ 'send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', 'as_dict', 'parent', 'parents', 'memory_info_ex', 'oneshot', 'pid', 'rlimit' ] if psutil.MACOS: ignore.append('memory_maps') # XXX p = psutil.Process() for name in sorted(dir(p)): if not name.startswith('_') and name not in ignore: fun = getattr(p, name) timecall(name, fun) print_timings()
def main(): def is64bit(name): return name.endswith(('x86_64.whl', 'amd64.whl')) groups = collections.defaultdict(list) for path in glob.glob('dist/*.whl'): name = os.path.basename(path) plat = name.split('-')[-1] pyimpl = name.split('-')[3] ispypy = 'pypy' in pyimpl if 'linux' in plat: if ispypy: groups['pypy_on_linux'].append(name) else: groups['linux'].append(name) elif 'win' in plat: if ispypy: groups['pypy_on_windows'].append(name) else: groups['windows'].append(name) elif 'macosx' in plat: if ispypy: groups['pypy_on_macos'].append(name) else: groups['macos'].append(name) else: assert 0, name tot_files = 0 tot_size = 0 templ = "%-54s %7s %7s %7s" for platf, names in groups.items(): ppn = "%s (total = %s)" % (platf.replace('_', ' '), len(names)) s = templ % (ppn, "size", "arch", "pyver") print_color('\n' + s, color=None, bold=True) for name in sorted(names): tot_files += 1 path = os.path.join('dist', name) size = os.path.getsize(path) tot_size += size arch = '64' if is64bit(name) else '32' pyver = 'pypy' if name.split('-')[3].startswith('pypy') else 'py' pyver += name.split('-')[2][2:] s = templ % (name, bytes2human(size), arch, pyver) if 'pypy' in pyver: print_color(s, color='violet') else: print_color(s, color='brown') print_color("\ntotals: files=%s, size=%s" % (tot_files, bytes2human(tot_size)), bold=1)
def get_file_urls(): with requests.Session() as session: data = session.get(BASE_URL + '/projects/' + USER + '/' + PROJECT, timeout=TIMEOUT) data = data.json() urls = [] for job in (job['jobId'] for job in data['build']['jobs']): job_url = BASE_URL + '/buildjobs/' + job + '/artifacts' data = session.get(job_url, timeout=TIMEOUT) data = data.json() for item in data: file_url = job_url + '/' + item['fileName'] urls.append(file_url) if not urls: print_color("no artifacts found", 'red') sys.exit(1) else: for url in sorted(urls, key=lambda x: os.path.basename(x)): yield url
def warn(msg): print_color(msg + " Running serial tests instead.", "red", file=sys.stderr)
def _print_color(self, s, color, bold=False): print_color(s, color, bold=bold, file=self.stream)
def _log(self, msg): if self.verbose: print_color(msg, color="yellow", file=sys.stderr)
def _print_color(self, s, color, bold=False): file = sys.stderr if color == "red" else sys.stdout print_color(s, color, bold=bold, file=file)