def test_build_test_execution_commands_spot_bid(self):
"""Test building commands using spot bids instead of flat rates."""
exp = (["starcluster -c sc_config start -b 0.50 --force-spot-master "
"nightly_tests"],
["starcluster -c sc_config sshmaster -u root nightly_tests "
"'source /bin/setup.sh; cd /bin; ./tests.py'",
"starcluster -c sc_config sshmaster -u root nightly_tests "
"'/bin/cogent_tests'"],
["starcluster -c sc_config terminate -c nightly_tests"])
test_suites = parse_config_file(self.config)
obs = _build_test_execution_commands(test_suites, 'sc_config',
'nightly_tests', spot_bid=0.50)
self.assertEqual(obs, exp)
# With custom cluster template and user.
exp = (["starcluster -c sc_config start -c some_cluster_template -b "
"1.00 --force-spot-master nightly_tests"],
["starcluster -c sc_config sshmaster -u ubuntu nightly_tests "
"'source /bin/setup.sh; cd /bin; ./tests.py'",
"starcluster -c sc_config sshmaster -u ubuntu nightly_tests "
"'/bin/cogent_tests'"],
["starcluster -c sc_config terminate -c nightly_tests"])
obs = _build_test_execution_commands(test_suites, 'sc_config',
'nightly_tests', cluster_template='some_cluster_template',
user='******', spot_bid=1)
self.assertEqual(obs, exp)
def test_build_test_execution_commands_standard(self):
"""Test building commands based on standard, valid input."""
exp = (["starcluster -c sc_config start nightly_tests"],
["starcluster -c sc_config sshmaster -u root nightly_tests "
"'source /bin/setup.sh; cd /bin; ./tests.py'",
"starcluster -c sc_config sshmaster -u root nightly_tests "
"'/bin/cogent_tests'"],
["starcluster -c sc_config terminate -c nightly_tests"])
test_suites = parse_config_file(self.config)
obs = _build_test_execution_commands(test_suites, 'sc_config',
'nightly_tests')
self.assertEqual(obs, exp)
def test_build_test_execution_commands_standard(self):
"""Test building commands based on standard, valid input."""
exp = (["starcluster -c sc_config start nightly_tests"], [
"starcluster -c sc_config sshmaster -u ubuntu nightly_tests "
"'source /bin/setup.sh; cd /bin; ./tests.py'",
"starcluster -c sc_config sshmaster -u ubuntu nightly_tests "
"'/bin/cogent_tests'"
], ["starcluster -c sc_config terminate -c nightly_tests"])
test_suites = parse_config_file(self.config)
obs = _build_test_execution_commands(test_suites, 'sc_config',
'ubuntu', 'nightly_tests')
self.assertEqual(obs, exp)
def test_build_test_execution_commands_custom_cluster_template(self):
"""Test building commands using a non-default cluster template."""
exp = (["starcluster -c sc_config start -c some_cluster_template "
"nightly_tests"],
["starcluster -c sc_config sshmaster -u ubuntu nightly_tests "
"'source /bin/setup.sh; cd /bin; ./tests.py'",
"starcluster -c sc_config sshmaster -u ubuntu nightly_tests "
"'/bin/cogent_tests'"],
["starcluster -c sc_config terminate -c nightly_tests"])
test_suites = parse_config_file(self.config)
obs = _build_test_execution_commands(test_suites, 'sc_config',
'nightly_tests', 'some_cluster_template', 'ubuntu')
self.assertEqual(obs, exp)
def test_build_test_execution_commands_custom_starcluster_exe_fp(self):
"""Test building commands using a non-default starcluster exec."""
exp = (["/usr/local/bin/starcluster -c sc_config start -c "
"some_cluster_template nightly_tests"],
["/usr/local/bin/starcluster -c sc_config sshmaster -u ubuntu "
"nightly_tests 'source /bin/setup.sh; cd /bin; ./tests.py'",
"/usr/local/bin/starcluster -c sc_config sshmaster -u ubuntu "
"nightly_tests '/bin/cogent_tests'"],
["/usr/local/bin/starcluster -c sc_config terminate -c "
"nightly_tests"])
test_suites = parse_config_file(self.config)
obs = _build_test_execution_commands(test_suites, 'sc_config',
'nightly_tests', 'some_cluster_template', 'ubuntu',
sc_exe_fp='/usr/local/bin/starcluster')
self.assertEqual(obs, exp)
def test_build_test_execution_commands_custom_cluster_template(self):
"""Test building commands using a non-default cluster template."""
exp = ([
"starcluster -c sc_config start -c some_cluster_template "
"nightly_tests"
], [
"starcluster -c sc_config sshmaster -u ubuntu nightly_tests "
"'source /bin/setup.sh; cd /bin; ./tests.py'",
"starcluster -c sc_config sshmaster -u ubuntu nightly_tests "
"'/bin/cogent_tests'"
], ["starcluster -c sc_config terminate -c nightly_tests"])
test_suites = parse_config_file(self.config)
obs = _build_test_execution_commands(test_suites, 'sc_config',
'ubuntu', 'nightly_tests',
'some_cluster_template')
self.assertEqual(obs, exp)
def test_parse_config_file_standard(self):
"""Test parsing a standard config file."""
exp = [['QIIME', 'source /bin/setup.sh; cd /bin; ./tests.py'],
['PyCogent', '/bin/cogent_tests']]
obs = parse_config_file(self.config1)
self.assertEqual(obs, exp)
def run_test_suites(config_f, sc_config_fp, recipients_f, email_settings_f,
cluster_tag, cluster_template=None,
user='******', setup_timeout=20.0, test_suites_timeout=240.0,
teardown_timeout=20.0, sc_exe_fp='starcluster'):
"""Runs the suite(s) of tests and emails the results to the recipients.
This function does not return anything. This function is not unit-tested
because there isn't a clean way to test it since it sends an email, starts
up a cluster on Amazon EC2, etc. Nearly every other 'private' function that
this function calls has been extensively unit-tested (whenever possible).
Thus, the amount of untested code has been minimized and contained here.
Arguments:
config_f - the input configuration file describing the test suites to
be run
sc_config_fp - the starcluster config filepath that will be used to
start/terminate the remote cluster that the tests will be run on
recipients_f - the file containing email addresses of those who should
receive the test suite results
email_settings_f - the file containing email (SMTP) settings to allow
the script to send an email
cluster_tag - the starcluster cluster tag to use when creating the
remote cluster (a string)
cluster_template - the starcluster cluster template to use in the
starcluster config file. If not provided, the default cluster
template in the starcluster config file will be used
user - the user who the tests should be run as on the remote cluster (a
string)
setup_timeout - the number of minutes to allow the cluster to be set up
before aborting and attempting to terminate it. Must be a float, to
allow for fractions of a minute
test_suites_timeout - the number of minutes to allow *all* test suites
to run before terminating the cluster. Must be a float, to allow
for fractions of a minute
teardown_timeout - the number of minutes to allow the cluster to be
terminated before aborting. Must be a float, to allow for fractions
of a minute
sc_exe_fp - path to the starcluster executable
"""
if setup_timeout <= 0 or test_suites_timeout <= 0 or teardown_timeout <= 0:
raise ValueError("The timeout (in minutes) must be greater than zero.")
# Parse the various configuration files first so that we know if there's
# any outstanding problems with file formats before continuing.
test_suites = parse_config_file(config_f)
recipients = parse_email_list(recipients_f)
email_settings = parse_email_settings(email_settings_f)
# Get the commands that need to be executed (these include launching a
# cluster, running the test suites, and terminating the cluster).
setup_cmds, test_suites_cmds, teardown_cmds = \
_build_test_execution_commands(test_suites, sc_config_fp,
cluster_tag, cluster_template, user,
sc_exe_fp)
# Execute the commands and build up the body of an email with the
# summarized results as well as the output in log file attachments.
email_body, attachments = _execute_commands_and_build_email(
test_suites, setup_cmds, test_suites_cmds, teardown_cmds,
setup_timeout, test_suites_timeout, teardown_timeout, cluster_tag)
# Send the email.
# TODO: this should be configurable by the user.
subject = "Test suite results [Clout testing system]"
send_email(email_settings['smtp_server'], email_settings['smtp_port'],
email_settings['sender'], email_settings['password'],
recipients, subject, email_body, attachments)
def run_test_suites(config_f,
sc_config_fp,
recipients_f,
email_settings_f,
cluster_tag,
cluster_template=None,
user='******',
spot_bid=None,
setup_timeout=20.0,
test_suites_timeout=240.0,
teardown_timeout=20.0,
sc_exe_fp='starcluster',
suppress_spot_bid_check=False):
"""Runs the test suites and emails the results to the recipients.
This function does not return anything. This function is not unit-tested
because there isn't a clean way to test it since it sends an email, starts
up a cluster on Amazon EC2, etc. Nearly every other 'private' function that
this function calls has been extensively unit-tested (whenever possible).
Thus, the amount of untested code has been minimized and contained here.
Arguments:
config_f - the input configuration file describing the test suites to
be run
sc_config_fp - the starcluster config filepath that will be used to
start/terminate the cluster that the tests will be run on
recipients_f - the file containing email addresses of those who should
receive the test suite results
email_settings_f - the file containing email (SMTP) settings to allow
the script to send an email
cluster_tag - the starcluster cluster tag to use when creating the
cluster (a string)
cluster_template - the starcluster cluster template to use in the
starcluster config file. If not provided, the default cluster
template in the starcluster config file will be used
user - the user who the tests should be run as on the cluster
(a string)
spot_bid - the maximum bid in USD to use for spot instances (a float).
If None, "on-demand" flat rates will be used for all instances
setup_timeout - the number of minutes to allow the cluster to be set up
before aborting and attempting to terminate it. Must be a float, to
allow for fractions of a minute
test_suites_timeout - the number of minutes to allow *all* test suites
to run before terminating the cluster. Must be a float, to allow
for fractions of a minute
teardown_timeout - the number of minutes to allow the cluster to be
terminated before aborting. Must be a float, to allow for fractions
of a minute
sc_exe_fp - path to the starcluster executable
suppress_spot_bid_check - if True, suppress sanity checking of
spot_bid. By default, if spot_bid is greater than
clout.static.MAX_SPOT_BID, an error will be raised
"""
if setup_timeout <= 0 or test_suites_timeout <= 0 or teardown_timeout <= 0:
raise ValueError("The timeout (in minutes) must be greater than zero.")
if spot_bid is not None:
try:
spot_bid = float(spot_bid)
except ValueError:
raise ValueError("Could not convert max spot bid to a float. Max "
"spot bid must be numeric.")
if spot_bid <= 0:
raise ValueError("Max spot bid of $%.2f must be greater than zero."
% spot_bid)
if not suppress_spot_bid_check and spot_bid > MAX_SPOT_BID:
raise ValueError("Max spot bid of $%.2f seems very high. If you "
"are sure this is the max spot bid that you want "
"to use, you can suppress this check with "
"--supprress_spot_bid_check." % spot_bid)
# Parse the various configuration files first so that we know if there's
# any outstanding problems with file formats before continuing.
test_suites = parse_config_file(config_f)
recipients = parse_email_list(recipients_f)
email_settings = parse_email_settings(email_settings_f)
# Get the commands that need to be executed (these include launching a
# cluster, running the test suites, and terminating the cluster).
setup_cmds, test_suites_cmds, teardown_cmds = \
_build_test_execution_commands(test_suites, sc_config_fp,
cluster_tag, cluster_template, user,
spot_bid, sc_exe_fp)
# Execute the commands and build up the body of an email with the
# summarized results as well as the output in log file attachments.
email_body, attachments = _execute_commands_and_build_email(
test_suites, setup_cmds, test_suites_cmds, teardown_cmds,
setup_timeout, test_suites_timeout, teardown_timeout, cluster_tag)
# Send the email.
# TODO: this should be configurable by the user.
subject = "Test suite results [Clout testing system]"
send_email(email_settings['smtp_server'], email_settings['smtp_port'],
email_settings['sender'], email_settings['password'],
recipients, subject, email_body, attachments)
def test_parse_config_file_standard(self):
"""Test parsing a standard config file."""
exp = [['QIIME', 'source /bin/setup.sh; cd /bin; ./tests.py'],
['PyCogent', '/bin/cogent_tests']]
obs = parse_config_file(self.config1)
self.assertEqual(obs, exp)
def run_test_suites(config_f,
sc_config_fp,
recipients_f,
email_settings_f,
cluster_tag,
cluster_template=None,
user='******',
spot_bid=None,
setup_timeout=20.0,
test_suites_timeout=240.0,
teardown_timeout=20.0,
sc_exe_fp='starcluster',
suppress_spot_bid_check=False):
"""Runs the test suites and emails the results to the recipients.
This function does not return anything. This function is not unit-tested
because there isn't a clean way to test it since it sends an email, starts
up a cluster on Amazon EC2, etc. Nearly every other 'private' function that
this function calls has been extensively unit-tested (whenever possible).
Thus, the amount of untested code has been minimized and contained here.
Arguments:
config_f - the input configuration file describing the test suites to
be run
sc_config_fp - the starcluster config filepath that will be used to
start/terminate the cluster that the tests will be run on
recipients_f - the file containing email addresses of those who should
receive the test suite results
email_settings_f - the file containing email (SMTP) settings to allow
the script to send an email
cluster_tag - the starcluster cluster tag to use when creating the
cluster (a string)
cluster_template - the starcluster cluster template to use in the
starcluster config file. If not provided, the default cluster
template in the starcluster config file will be used
user - the user who the tests should be run as on the cluster
(a string)
spot_bid - the maximum bid in USD to use for spot instances (a float).
If None, "on-demand" flat rates will be used for all instances
setup_timeout - the number of minutes to allow the cluster to be set up
before aborting and attempting to terminate it. Must be a float, to
allow for fractions of a minute
test_suites_timeout - the number of minutes to allow *all* test suites
to run before terminating the cluster. Must be a float, to allow
for fractions of a minute
teardown_timeout - the number of minutes to allow the cluster to be
terminated before aborting. Must be a float, to allow for fractions
of a minute
sc_exe_fp - path to the starcluster executable
suppress_spot_bid_check - if True, suppress sanity checking of
spot_bid. By default, if spot_bid is greater than
clout.static.MAX_SPOT_BID, an error will be raised
"""
if setup_timeout <= 0 or test_suites_timeout <= 0 or teardown_timeout <= 0:
raise ValueError("The timeout (in minutes) must be greater than zero.")
if spot_bid is not None:
try:
spot_bid = float(spot_bid)
except ValueError:
raise ValueError("Could not convert max spot bid to a float. Max "
"spot bid must be numeric.")
if spot_bid <= 0:
raise ValueError(
"Max spot bid of $%.2f must be greater than zero." % spot_bid)
if not suppress_spot_bid_check and spot_bid > MAX_SPOT_BID:
raise ValueError("Max spot bid of $%.2f seems very high. If you "
"are sure this is the max spot bid that you want "
"to use, you can suppress this check with "
"--supprress_spot_bid_check." % spot_bid)
# Parse the various configuration files first so that we know if there's
# any outstanding problems with file formats before continuing.
test_suites = parse_config_file(config_f)
recipients = parse_email_list(recipients_f)
email_settings = parse_email_settings(email_settings_f)
# Get the commands that need to be executed (these include launching a
# cluster, running the test suites, and terminating the cluster).
setup_cmds, test_suites_cmds, teardown_cmds = \
_build_test_execution_commands(test_suites, sc_config_fp,
cluster_tag, cluster_template, user,
spot_bid, sc_exe_fp)
# Execute the commands and build up the body of an email with the
# summarized results as well as the output in log file attachments.
email_body, attachments = _execute_commands_and_build_email(
test_suites, setup_cmds, test_suites_cmds, teardown_cmds,
setup_timeout, test_suites_timeout, teardown_timeout, cluster_tag)
# Send the email.
# TODO: this should be configurable by the user.
subject = "Test suite results [Clout testing system]"
send_email(email_settings['smtp_server'], email_settings['smtp_port'],
email_settings['sender'], email_settings['password'],
recipients, subject, email_body, attachments)