def main(argv=None):
global notebook_limit
global quiet
tornado.log.enable_pretty_logging()
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--run-dir",
type=str,
default="./runs",
help="directory in which to store results",
)
parser.add_argument(
"-q",
"--quiet",
action="store_true",
help="Run more quietly: don't echo build & test output to stderr",
)
parser.add_argument(
"-n",
"--limit",
default=notebook_limit,
type=int,
help="Limit to this many notebook tests per repo",
)
parser.add_argument(
"--force-build",
action="store_true",
help="Force rebuild of images, even if an image already exists",
)
parser.add_argument("repos", nargs="+", help="repos to test")
opts = parser.parse_args(argv)
quiet = opts.quiet
notebook_limit = opts.limit
for repo in opts.repos:
if "://" not in repo:
# allow a/b shortcuts for github
repo = "https://github.com/" + repo
if "@" in repo:
repo, ref = repo.split("@")
else:
ref = "master"
try:
result_file, results = test_one_repo(repo,
ref=ref,
run_dir=opts.run_dir,
force_build=opts.force_build)
except Exception:
log.exception(f"Error testing {repo}@{ref}")
else:
print_summary(results, result_file, opts.run_dir)
def send_ping(self, *args, **kwargs):
"""
Sends ping message
Runs periodically.
"""
log.debug("Sending ping")
message = PING_MESSAGE
try:
self.udp.send(message)
except Exception:
log.exception("Failed to send ping")
self.stop()
def run_tests(image, checkout_path, run_dir):
"""Find tests to run and run them"""
notebooks = list(find_notebooks(checkout_path))
count = len(notebooks)
log.info(f"Found {count} to test")
if notebook_limit and count > notebook_limit:
log.info(f"Limiting to first {notebook_limit}/{count} notebooks")
notebooks = notebooks[:notebook_limit]
for nb_path in notebooks:
test_log_file = os.path.join(
run_dir, "logs",
f"test-notebook-{nb_path.replace('/', '-')}-{run_id}.txt")
try:
yield run_one_test(image, "notebook", nb_path, run_dir,
test_log_file)
except Exception:
log.exception(f"Error running test {nb_path}")
yield {
"kind": "notebook",
"success": False,
"test_id": nb_path,
"path": test_log_file,
}
def test_one_repo(repo, ref="master", run_dir="./runs", force_build=False):
slug = repo_slug(repo).lower()
slug_parts = slug.split("/")
path = "/".join([slug_parts[0], slug_parts[1][0]] + slug_parts[1:])
repo_run_dir = os.path.join(run_dir, path)
log_dir = os.path.join(repo_run_dir, "logs")
result_dir = os.path.join(repo_run_dir, "results")
for d in [log_dir, result_dir]:
try:
os.makedirs(d)
except FileExistsError:
pass
result_file = os.path.join(result_dir, f"results-{ref}-{run_id}.csv")
with open(result_file, "w") as f:
writer = csv.writer(f)
writer.writerow(TestResult._fields)
build_log_file = os.path.join(log_dir, f"build-{ref}-{run_id}.txt")
checkout_path, resolved_ref, last_modified = clone_repo(repo, ref)
log.info(f"Building {repo}@{ref} in {repo_run_dir} with run id {run_id}")
results = []
def add_result(kind, test_id, success, path):
path = os.path.relpath(path, run_dir)
log.info(
f"Recording test result: repo={repo}, kind={kind}, test_id={test_id}, {'success' if success else 'failure'}"
)
result = TestResult(
repo,
ref,
resolved_ref,
last_modified,
kind,
test_id,
success,
path,
timestamp,
run_id,
repo2docker.__version__,
)
results.append(result)
with open(result_file, "a") as f:
writer = csv.writer(f)
writer.writerow(result)
try:
image, checkout_path = build_repo(
repo,
resolved_ref=resolved_ref,
checkout_path=checkout_path,
build_log_file=build_log_file,
force_build=force_build,
)
except Exception:
# log errors that won't be in the build log
# (these will usually be bugs in our script!)
if not isinstance(Exception, CalledProcessError):
log.exception("Build failure")
with open(build_log_file, "a") as f:
traceback.print_exc(file=f)
# record build failure
add_result(kind="build",
test_id="build",
success=False,
path=build_log_file)
return result_file, results
else:
add_result(kind="build",
test_id="build",
success=True,
path=build_log_file)
for result in run_tests(image, checkout_path, repo_run_dir):
add_result(**result)
return result_file, results