def build_and_upload(run_attrs=parse_args([])):
"""
Build a list of framework scheduler and put them at URLs so a cluster can use it.
build() and upload()should be two different functions, but that's a
project for another day.
run_attrs takes defaults from the argument parser with no arguments
"""
for framework_name in fwinfo.get_framework_names():
framework = fwinfo.get_framework(framework_name)
func = build_and_upload_single
args = framework, run_attrs
_action_wrapper("build %s" % framework.name, framework, func, *args)
def _multicluster_linear_per_cluster(run_attrs, repo_root):
test_list = list(fwinfo.get_framework_names())
next_test = None
all_ok = False # only one completion state out of the loop
try:
while True:
# acquire the next test, if there is one
if not next_test and test_list:
next_test = test_list.pop(0)
logger.info("Next test to run: %s", next_test)
if next_test:
# we have a test to run, find a cluster to run it on.
avail_cluster = clustinfo.get_idle_cluster()
logger.debug("avail_cluster=%s", avail_cluster)
if not avail_cluster and clustinfo.running_count(
) < run_attrs.cluster_count:
# we're supposed to start more clusters, so do so
human_count = clustinfo.running_count() + 1
logger.info("Launching cluster %s towards count %s",
human_count, run_attrs.cluster_count)
# TODO: retry cluster launches
start_config = launch_ccm_cluster.StartConfig(
private_agents=6)
avail_cluster = clustinfo.start_cluster(
start_config,
reporting_name="Cluster %s" % human_count)
elif not avail_cluster:
# We're not supposed to start more clusters, so wait, and
# check for test completion.
info_bits = []
for cluster in clustinfo._clusters:
template = "cluster_id=%s in use by frameworks=%s"
info_bits.append(
template % (cluster.cluster_id, cluster.in_use()))
logger.info("Waiting for cluster to launch %s; %s",
next_test, ", ".join(info_bits))
# TODO: report .out sizes for running tests
time.sleep(30) # waiting for an available cluster
# meanwhile, a test might finish
all_ok = _handle_test_completions()
if not all_ok:
logger.info("Some tests failed; aborting early"
) # TODO paramaterize
break
continue
# At this point, we have a cluster and a test, so start it.
logger.info(
"Testing framework=%s in background on cluster=%s.",
next_test, avail_cluster.cluster_id)
framework = fwinfo.get_framework(next_test)
func = start_test_background
args = framework, avail_cluster, repo_root
_action_wrapper("Launch %s tests" % framework.name, framework,
func, *args)
next_test = None
avail_cluster = None
else:
# No tests left, handle completion and waiting for completion.
if not fwinfo.running_frameworks():
logger.info("No framework tests running. All done.")
all_ok = True
break # all tests done
logger.info(
"No framework tests to launch, waiting for completions.")
# echo status
time.sleep(30) # waiting for tests to complete
# after launching a test, or waiting, check for test completion.
all_ok = _handle_test_completions()
if not all_ok:
logger.info(
"Some tests failed; aborting early") # TODO paramaterize
break
finally:
# TODO probably should also make this teardown optional
for framework_name in fwinfo.get_framework_names():
logger.info("Terminating subprocess for framework=%s",
framework_name)
framework = fwinfo.get_framework(framework_name)
if framework.popen:
framework.popen.terminate(
) # does nothing if already completed
return all_ok