def _one_cluster_linear_tests(run_attrs, repo_root, continue_on_error):
fail_fast = not continue_on_error
if run_attrs.cluster_url and run_attrs.cluster_token:
clustinfo.add_running_cluster(run_attrs.cluster_url,
run_attrs.cluster_token)
else:
start_config = launch_ccm_cluster.StartConfig(private_agents=6)
clustinfo.start_cluster(start_config)
cluster = clustinfo._clusters[0]
all_ok = True
for framework in fwinfo.get_frameworks():
func = run_test
args = framework, cluster, repo_root, fail_fast
try:
_action_wrapper("Run %s tests" % framework.name, framework, func,
*args)
except Exception as e:
all_ok = False
if fail_fast:
logger.info(
"Some tests failed; aborting early") # TODO paramaterize
raise
else:
logger.info("Tests for % tests failed.", framework.name)
return all_ok
def _launch_cluster(launch_config=None):
ccm_token = os.environ['CCM_AUTH_TOKEN']
if not launch_config:
launch_config = launch_ccm_cluster.StartConfig(private_agents=6)
cluster_info = launch_ccm_cluster.start_cluster(ccm_token, launch_config)
cluster = ClusterInfo(cluster_info["url"],
cluster_info["auth_token"],
cluster_id=cluster_info["id"])
return cluster
def _one_cluster_linear_tests(run_attrs, repo_root):
start_config = launch_ccm_cluster.StartConfig(private_agents=6)
clustinfo.start_cluster(start_config)
cluster = clustinfo._clusters[0]
for framework in fwinfo.get_frameworks():
func = run_test
args = framework, cluster, repo_root
_action_wrapper("Run %s tests" % framework.name, framework, func,
*args)
def _one_cluster_linear_tests(run_attrs, repo_root):
if run_attrs.cluster_url and run_attrs.cluster_token:
clustinfo.add_running_cluster(run_attrs.cluster_url,
run_attrs.cluster_token)
else:
start_config = launch_ccm_cluster.StartConfig(private_agents=6)
clustinfo.start_cluster(start_config)
cluster = clustinfo._clusters[0]
for framework in fwinfo.get_frameworks():
func = run_test
args = framework, cluster, repo_root
_action_wrapper("Run %s tests" % framework.name, framework, func,
*args)
# we don't handle exceptions here, so any failures will stop us from
# getting this far.
all_passed = True
return all_passed
def _launch_cluster(launch_config=None):
github_label = launch_ccm_cluster.determine_github_label()
ccm_token = os.environ['CCM_AUTH_TOKEN']
launcher = launch_ccm_cluster.CCMLauncher(ccm_token, github_label)
start_stop_attempts = launch_ccm_cluster.CCMLauncher.DEFAULT_ATTEMPTS
if 'CCM_ATTEMPTS' in os.environ:
start_stop_attempts = int(os.environ['CCM_ATTEMPTS'])
if not launch_config:
launch_config = launch_ccm_cluster.StartConfig(private_agents=6)
cluster_info = launch_ccm_cluster.start_cluster(launcher, github_label,
start_stop_attempts,
launch_config)
cluster = ClusterInfo(cluster_info["url"],
cluster_info["auth_token"],
cluster_id=cluster_info["id"])
return cluster
def _multicluster_linear_per_cluster(run_attrs, repo_root):
test_list = list(fwinfo.get_framework_names())
next_test = None
all_ok = False # only one completion state out of the loop
try:
while True:
# acquire the next test, if there is one
if not next_test and test_list:
next_test = test_list.pop(0)
logger.info("Next test to run: %s", next_test)
if next_test:
# we have a test to run, find a cluster to run it on.
avail_cluster = clustinfo.get_idle_cluster()
logger.debug("avail_cluster=%s", avail_cluster)
if not avail_cluster and clustinfo.running_count(
) < run_attrs.cluster_count:
# we're supposed to start more clusters, so do so
human_count = clustinfo.running_count() + 1
logger.info("Launching cluster %s towards count %s",
human_count, run_attrs.cluster_count)
# TODO: retry cluster launches
start_config = launch_ccm_cluster.StartConfig(
private_agents=6)
avail_cluster = clustinfo.start_cluster(
start_config,
reporting_name="Cluster %s" % human_count)
elif not avail_cluster:
# We're not supposed to start more clusters, so wait, and
# check for test completion.
info_bits = []
for cluster in clustinfo._clusters:
template = "cluster_id=%s in use by frameworks=%s"
info_bits.append(
template % (cluster.cluster_id, cluster.in_use()))
logger.info("Waiting for cluster to launch %s; %s",
next_test, ", ".join(info_bits))
# TODO: report .out sizes for running tests
time.sleep(30) # waiting for an available cluster
# meanwhile, a test might finish
all_ok = _handle_test_completions()
if not all_ok:
logger.info("Some tests failed; aborting early"
) # TODO paramaterize
break
continue
# At this point, we have a cluster and a test, so start it.
logger.info(
"Testing framework=%s in background on cluster=%s.",
next_test, avail_cluster.cluster_id)
framework = fwinfo.get_framework(next_test)
func = start_test_background
args = framework, avail_cluster, repo_root
_action_wrapper("Launch %s tests" % framework.name, framework,
func, *args)
next_test = None
avail_cluster = None
else:
# No tests left, handle completion and waiting for completion.
if not fwinfo.running_frameworks():
logger.info("No framework tests running. All done.")
all_ok = True
break # all tests done
logger.info(
"No framework tests to launch, waiting for completions.")
# echo status
time.sleep(30) # waiting for tests to complete
# after launching a test, or waiting, check for test completion.
all_ok = _handle_test_completions()
if not all_ok:
logger.info(
"Some tests failed; aborting early") # TODO paramaterize
break
finally:
# TODO probably should also make this teardown optional
for framework_name in fwinfo.get_framework_names():
logger.info("Terminating subprocess for framework=%s",
framework_name)
framework = fwinfo.get_framework(framework_name)
if framework.popen:
framework.popen.terminate(
) # does nothing if already completed
return all_ok