def do_test_cloud(cmd, cmd_args={}, cmd_format="gnu", repeat=1, timeout=60):
global test_references
global use_local_retester
if use_local_retester:
return do_test(cmd, cmd_args, cmd_format, repeat, timeout)
# Build up the command line
command = cmd
cmd_args_keys = [k for k in cmd_args]
cmd_args_keys.sort()
for arg in cmd_args_keys:
command += " "
# GNU cmd line builder
if cmd_format == "gnu":
if (isinstance(cmd_args[arg], types.BooleanType)):
if cmd_args[arg]:
command += "--%s" % arg
else:
command += "--%s \"%s\"" % (
arg, retester.shell_escape(str(cmd_args[arg])))
# Make cmd line builder
elif cmd_format == "make":
command += "%s=%s" % (arg, str(cmd_args[arg]))
# Invalid cmd line builder
else:
print "Invalid command line formatter"
raise NameError()
# Run the test
if repeat == 1: print "Running %r..." % command
else: print "Running %r (repeating %d times)..." % (command, repeat)
if timeout > 60:
print "(This test may take up to %d seconds each time.)" % timeout
test_reference = TestReference(command)
for i in xrange(repeat):
test_reference.single_runs.append(
issue_test_to_some_node(command, timeout))
test_references.append(test_reference)
def do_test_cloud(cmd, cmd_args={}, cmd_format="gnu", repeat=1, timeout=60):
global test_references
global use_local_retester
if use_local_retester:
return do_test(cmd, cmd_args, cmd_format, repeat, timeout)
# Build up the command line
command = cmd
cmd_args_keys = [k for k in cmd_args]
cmd_args_keys.sort()
for arg in cmd_args_keys:
command += " "
# GNU cmd line builder
if cmd_format == "gnu":
if(isinstance(cmd_args[arg], types.BooleanType)):
if cmd_args[arg]:
command += "--%s" % arg
else:
command += "--%s \"%s\"" % (arg, retester.shell_escape(str(cmd_args[arg])))
# Make cmd line builder
elif cmd_format == "make":
command += "%s=%s" % (arg, str(cmd_args[arg]))
# Invalid cmd line builder
else:
print "Invalid command line formatter"
raise NameError()
# Run the test
if repeat == 1: print "Running %r..." % command
else: print "Running %r (repeating %d times)..." % (command, repeat)
if timeout > 60: print "(This test may take up to %d seconds each time.)" % timeout
test_reference = TestReference(command)
for i in xrange(repeat):
test_reference.single_runs.append(issue_test_to_some_node(command, timeout))
test_references.append(test_reference)
def start_test_on_node(node,
test_command,
test_timeout=None,
locking_timeout=0):
global testing_nodes
global remaining_nodes_to_allocate
if locking_timeout == None:
locking_timeout = 0
#print ("trying to acquire lock with timeout %i" % locking_timeout)
if node.acquire_lock(locking_timeout) == False:
return False
#print ("Got lock!")
# Check if we can retrieve previous test results for this node now
retrieve_results_from_node(node)
try:
# Initialize node if not happened before...
if node.basedata_installed == False:
copy_basedata_to_testing_node(node)
# Generate random reference
system_random = random.SystemRandom()
test_reference = "cloudtest_" + str(
system_random.randint(10000000, 99999999))
# Create test directory and check that it isn't taken
directory_created = False
while not directory_created:
node.make_directory_recursively("cloud_retest")
try:
node.make_directory("cloud_retest/%s" % test_reference)
directory_created = True
except IOError:
directory_created = False
test_reference = "cloudtest_" + str(
system_random.randint(10000000,
99999999)) # Try another reference
print "Starting test with test reference %s on node %s" % (
test_reference, node.hostname)
# Prepare for test...
copy_per_test_data_to_testing_node(node, test_reference)
# Store test_command and test_timeout into files on the remote node for the wrapper script to pick it up
command_result = node.run_command("echo -n %s > cloud_retest/%s/test/test_command" % \
(retester.shell_escape(test_command), test_reference))
if command_result[0] != 0:
print "Unable to store command"
# TODO: Throw an exception
if test_timeout == None:
command_result = node.run_command(
"echo -n \"\" > cloud_retest/%s/test/test_timeout" %
(test_reference))
else:
command_result = node.run_command(
"echo -n %i > cloud_retest/%s/test/test_timeout" %
(test_timeout, test_reference))
if command_result[0] != 0:
print "Unable to store timeout"
# TODO: Throw an exception
# Run test and release lock after it has finished
command_result = node.run_command(
"sh -c \"nohup sh -c \\\"(cd %s; LD_LIBRARY_PATH=/tmp/cloudtest_libs:$LD_LIBRARY_PATH PATH=/tmp/cloudtest_bin:$PATH PYTHONPATH=/tmp/cloudtest_python:$PYTHONPATH VALGRIND_LIB=/tmp/cloudtest_libs/valgrind python %s; %s)&\\\" > /dev/null 2> /dev/null\""
% ("cloud_retest/%s/test" % test_reference,
wrapper_script_filename.replace(
" ", "\\ "), node.get_release_lock_command()))
except (IOError, EOFError, paramiko.SSHException, Exception) as e:
print "Starting test failed: %s" % e
test_reference = "Failed"
try:
node.release_lock()
except (IOError, EOFError, paramiko.SSHException, Exception):
print "Unable to release lock on node %s. Node is now defunct." % node.hostname
testing_nodes.remove(node)
remaining_nodes_to_allocate += 1
return (node, test_reference)
def start_test_on_node(node, test_command, test_timeout = None, locking_timeout = 0):
global testing_nodes
global remaining_nodes_to_allocate
if locking_timeout == None:
locking_timeout = 0
#print ("trying to acquire lock with timeout %i" % locking_timeout)
if node.acquire_lock(locking_timeout) == False:
return False
#print ("Got lock!")
# Check if we can retrieve previous test results for this node now
retrieve_results_from_node(node)
try:
# Initialize node if not happened before...
if node.basedata_installed == False:
copy_basedata_to_testing_node(node)
# Generate random reference
system_random = random.SystemRandom()
test_reference = "cloudtest_" + str(system_random.randint(10000000, 99999999))
# Create test directory and check that it isn't taken
directory_created = False
while not directory_created:
node.make_directory_recursively("cloud_retest")
try:
node.make_directory("cloud_retest/%s" % test_reference)
directory_created = True
except IOError:
directory_created = False
test_reference = "cloudtest_" + str(system_random.randint(10000000, 99999999)) # Try another reference
print "Starting test with test reference %s on node %s" % (test_reference, node.hostname)
# Prepare for test...
copy_per_test_data_to_testing_node(node, test_reference)
# Store test_command and test_timeout into files on the remote node for the wrapper script to pick it up
command_result = node.run_command("echo -n %s > cloud_retest/%s/test/test_command" % \
(retester.shell_escape(test_command), test_reference))
if command_result[0] != 0:
print "Unable to store command"
# TODO: Throw an exception
if test_timeout == None:
command_result = node.run_command("echo -n \"\" > cloud_retest/%s/test/test_timeout" % (test_reference))
else:
command_result = node.run_command("echo -n %i > cloud_retest/%s/test/test_timeout" % (test_timeout, test_reference))
if command_result[0] != 0:
print "Unable to store timeout"
# TODO: Throw an exception
# Run test and release lock after it has finished
command_result = node.run_command("sh -c \"nohup sh -c \\\"(cd %s; LD_LIBRARY_PATH=/tmp/cloudtest_libs:$LD_LIBRARY_PATH PATH=/tmp/cloudtest_bin:$PATH PYTHONPATH=/tmp/cloudtest_python:$PYTHONPATH VALGRIND_LIB=/tmp/cloudtest_libs/valgrind python %s; %s)&\\\" > /dev/null 2> /dev/null\"" % ("cloud_retest/%s/test" % test_reference, wrapper_script_filename.replace(" ", "\\ "), node.get_release_lock_command()))
except (IOError, EOFError, paramiko.SSHException, Exception) as e:
print "Starting test failed: %s" % e
test_reference = "Failed"
try:
node.release_lock()
except (IOError, EOFError, paramiko.SSHException, Exception):
print "Unable to release lock on node %s. Node is now defunct." % node.hostname
testing_nodes.remove(node)
remaining_nodes_to_allocate += 1
return (node, test_reference)