def setup_frameworks(run_attrs):
# We should get the cluster version from the running cluster, when it
# exists. However there are two problems:
# 1 - the busted certs we use give some python installs indigestion (this
# could be worked around)
# 2 - during dev cycles, the version is often wrong :-(
# As a result, just use the env var supplied to most continuous
# integration jobs, CCM_CHANNEL
ccm_channel = os.environ.get("CCM_CHANNEL")
version = None
if ccm_channel and ccm_channel.startswith("testing/1"):
testing, version = ccm_channel.split("/", 1)
version_chars = set(version)
valid_version_chars = set(string.digits + '.')
if not version_chars.issubset(valid_version_chars):
version = None
if not run_attrs.test:
fwinfo.autodiscover_frameworks(dcos_version=version)
else:
for framework in run_attrs.test:
fwinfo.add_framework(framework, dcos_version=version)
if run_attrs.order == "random":
fwinfo.shuffle_order()
fw_names = fwinfo.get_framework_names()
logger.info("Frameworks initialized: %s", ", ".join(fw_names))
def setup_frameworks(run_attrs):
if not run_attrs.test:
fwinfo.autodiscover_frameworks()
else:
for framework in run_attrs.test:
fwinfo.add_framework(framework)
if run_attrs.order == "random":
fwinfo.shuffle_order()
fw_names = fwinfo.get_framework_names()
logger.info("Frameworks initialized: %s", ", ".join(fw_names))
def build_and_upload(run_attrs=parse_args([])):
"""
Build a list of framework scheduler and put them at URLs so a cluster can use it.
build() and upload()should be two different functions, but that's a
project for another day.
run_attrs takes defaults from the argument parser with no arguments
"""
for framework_name in fwinfo.get_framework_names():
framework = fwinfo.get_framework(framework_name)
func = build_and_upload_single
args = framework, run_attrs
_action_wrapper("build %s" % framework.name, framework, func, *args)
def _multicluster_linear_per_cluster(run_attrs, repo_root):
test_list = list(fwinfo.get_framework_names())
next_test = None
all_ok = False # only one completion state out of the loop
try:
while True:
# acquire the next test, if there is one
if not next_test and test_list:
next_test = test_list.pop(0)
logger.info("Next test to run: %s", next_test)
if next_test:
# we have a test to run, find a cluster to run it on.
avail_cluster = clustinfo.get_idle_cluster()
logger.debug("avail_cluster=%s", avail_cluster)
if not avail_cluster and clustinfo.running_count(
) < run_attrs.cluster_count:
# we're supposed to start more clusters, so do so
human_count = clustinfo.running_count() + 1
logger.info("Launching cluster %s towards count %s",
human_count, run_attrs.cluster_count)
# TODO: retry cluster launches
start_config = launch_ccm_cluster.StartConfig(
private_agents=6)
avail_cluster = clustinfo.start_cluster(
start_config,
reporting_name="Cluster %s" % human_count)
elif not avail_cluster:
# We're not supposed to start more clusters, so wait, and
# check for test completion.
info_bits = []
for cluster in clustinfo._clusters:
template = "cluster_id=%s in use by frameworks=%s"
info_bits.append(
template % (cluster.cluster_id, cluster.in_use()))
logger.info("Waiting for cluster to launch %s; %s",
next_test, ", ".join(info_bits))
# TODO: report .out sizes for running tests
time.sleep(30) # waiting for an available cluster
# meanwhile, a test might finish
all_ok = _handle_test_completions()
if not all_ok:
logger.info("Some tests failed; aborting early"
) # TODO paramaterize
break
continue
# At this point, we have a cluster and a test, so start it.
logger.info(
"Testing framework=%s in background on cluster=%s.",
next_test, avail_cluster.cluster_id)
framework = fwinfo.get_framework(next_test)
func = start_test_background
args = framework, avail_cluster, repo_root
_action_wrapper("Launch %s tests" % framework.name, framework,
func, *args)
next_test = None
avail_cluster = None
else:
# No tests left, handle completion and waiting for completion.
if not fwinfo.running_frameworks():
logger.info("No framework tests running. All done.")
all_ok = True
break # all tests done
logger.info(
"No framework tests to launch, waiting for completions.")
# echo status
time.sleep(30) # waiting for tests to complete
# after launching a test, or waiting, check for test completion.
all_ok = _handle_test_completions()
if not all_ok:
logger.info(
"Some tests failed; aborting early") # TODO paramaterize
break
finally:
# TODO probably should also make this teardown optional
for framework_name in fwinfo.get_framework_names():
logger.info("Terminating subprocess for framework=%s",
framework_name)
framework = fwinfo.get_framework(framework_name)
if framework.popen:
framework.popen.terminate(
) # does nothing if already completed
return all_ok