def main(argv=None): global notebook_limit global quiet tornado.log.enable_pretty_logging() parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--run-dir", type=str, default="./runs", help="directory in which to store results", ) parser.add_argument( "-q", "--quiet", action="store_true", help="Run more quietly: don't echo build & test output to stderr", ) parser.add_argument( "-n", "--limit", default=notebook_limit, type=int, help="Limit to this many notebook tests per repo", ) parser.add_argument( "--force-build", action="store_true", help="Force rebuild of images, even if an image already exists", ) parser.add_argument("repos", nargs="+", help="repos to test") opts = parser.parse_args(argv) quiet = opts.quiet notebook_limit = opts.limit for repo in opts.repos: if "://" not in repo: # allow a/b shortcuts for github repo = "https://github.com/" + repo if "@" in repo: repo, ref = repo.split("@") else: ref = "master" try: result_file, results = test_one_repo(repo, ref=ref, run_dir=opts.run_dir, force_build=opts.force_build) except Exception: log.exception(f"Error testing {repo}@{ref}") else: print_summary(results, result_file, opts.run_dir)
def send_ping(self, *args, **kwargs): """ Sends ping message Runs periodically. """ log.debug("Sending ping") message = PING_MESSAGE try: self.udp.send(message) except Exception: log.exception("Failed to send ping") self.stop()
def run_tests(image, checkout_path, run_dir): """Find tests to run and run them""" notebooks = list(find_notebooks(checkout_path)) count = len(notebooks) log.info(f"Found {count} to test") if notebook_limit and count > notebook_limit: log.info(f"Limiting to first {notebook_limit}/{count} notebooks") notebooks = notebooks[:notebook_limit] for nb_path in notebooks: test_log_file = os.path.join( run_dir, "logs", f"test-notebook-{nb_path.replace('/', '-')}-{run_id}.txt") try: yield run_one_test(image, "notebook", nb_path, run_dir, test_log_file) except Exception: log.exception(f"Error running test {nb_path}") yield { "kind": "notebook", "success": False, "test_id": nb_path, "path": test_log_file, }
def test_one_repo(repo, ref="master", run_dir="./runs", force_build=False): slug = repo_slug(repo).lower() slug_parts = slug.split("/") path = "/".join([slug_parts[0], slug_parts[1][0]] + slug_parts[1:]) repo_run_dir = os.path.join(run_dir, path) log_dir = os.path.join(repo_run_dir, "logs") result_dir = os.path.join(repo_run_dir, "results") for d in [log_dir, result_dir]: try: os.makedirs(d) except FileExistsError: pass result_file = os.path.join(result_dir, f"results-{ref}-{run_id}.csv") with open(result_file, "w") as f: writer = csv.writer(f) writer.writerow(TestResult._fields) build_log_file = os.path.join(log_dir, f"build-{ref}-{run_id}.txt") checkout_path, resolved_ref, last_modified = clone_repo(repo, ref) log.info(f"Building {repo}@{ref} in {repo_run_dir} with run id {run_id}") results = [] def add_result(kind, test_id, success, path): path = os.path.relpath(path, run_dir) log.info( f"Recording test result: repo={repo}, kind={kind}, test_id={test_id}, {'success' if success else 'failure'}" ) result = TestResult( repo, ref, resolved_ref, last_modified, kind, test_id, success, path, timestamp, run_id, repo2docker.__version__, ) results.append(result) with open(result_file, "a") as f: writer = csv.writer(f) writer.writerow(result) try: image, checkout_path = build_repo( repo, resolved_ref=resolved_ref, checkout_path=checkout_path, build_log_file=build_log_file, force_build=force_build, ) except Exception: # log errors that won't be in the build log # (these will usually be bugs in our script!) if not isinstance(Exception, CalledProcessError): log.exception("Build failure") with open(build_log_file, "a") as f: traceback.print_exc(file=f) # record build failure add_result(kind="build", test_id="build", success=False, path=build_log_file) return result_file, results else: add_result(kind="build", test_id="build", success=True, path=build_log_file) for result in run_tests(image, checkout_path, repo_run_dir): add_result(**result) return result_file, results