def _test(self, spotInstances=False):
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
self.createClusterUtil()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = ['virtualenv', '--system-site-packages', '--never-download',
'/home/venv']
self.sshUtil(venv_command)
upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1']
self.sshUtil(upgrade_command)
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
self.sshUtil(yaml_command)
# install toil scripts
install_command = ['/home/venv/bin/pip', 'install', 'toil-scripts==%s' % self.toilScripts]
self.sshUtil(install_command)
toilOptions = ['--batchSystem=mesos',
'--workDir=/var/lib/toil',
'--mesosMaster=%s:5050' % leader.private_ip_address,
'--clean=always',
'--retryCount=2']
toilOptions.extend(['--provisioner=aws',
'--nodeType=' + self.instanceType,
'--maxNodes=%s' % self.numWorkers,
'--logDebug'])
if spotInstances:
toilOptions.extend([
'--preemptableNodeType=%s:%s' % (self.instanceType, self.spotBid),
# The RNASeq pipeline does not specify a preemptability requirement so we
# need to specify a default, otherwise jobs would never get scheduled.
'--defaultPreemptable',
'--maxPreemptableNodes=%s' % self.numWorkers])
toilOptions = ' '.join(toilOptions)
# TOIL_AWS_NODE_DEBUG prevents the provisioner from killing nodes that
# fail a status check. This allows for easier debugging of
# https://github.com/BD2KGenomics/toil/issues/1141
runCommand = ['bash', '-c',
'PATH=/home/venv/bin/:$PATH '
'TOIL_AWS_NODE_DEBUG=True '
'TOIL_SCRIPTS_TEST_NUM_SAMPLES='+str(self.numSamples)+
' TOIL_SCRIPTS_TEST_TOIL_OPTIONS=' + pipes.quote(toilOptions) +
' TOIL_SCRIPTS_TEST_JOBSTORE=' + self.jobStore +
' /home/venv/bin/python -m unittest -v' +
' toil_scripts.rnaseq_cgl.test.test_rnaseq_cgl.RNASeqCGLTest.test_manifest']
self.sshUtil(runCommand)
assert len(self.getMatchingRoles(self.clusterName)) == 1
AWSProvisioner.destroyCluster(self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 0
def _test(self, spotInstances=False, fulfillableBid=True):
"""
Does the work of the testing. Many features' test are thrown in here is no particular
order
:param spotInstances: Specify if you want to use spotInstances
:param fulfillableBid: If false, the bid will never succeed. Used to test bid failure
"""
if not fulfillableBid:
self.spotBid = '0.01'
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
self.launchCluster()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
self.leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName)
ctx = AWSProvisioner._buildContext(self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = ['virtualenv', '--system-site-packages', '--never-download',
'/home/venv']
self.sshUtil(venv_command)
upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1']
self.sshUtil(upgrade_command)
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
self.sshUtil(yaml_command)
self._getScript()
toilOptions = [self.jobStore,
'--batchSystem=mesos',
'--workDir=/var/lib/toil',
'--clean=always',
'--retryCount=2',
'--clusterStats=/home/',
'--logDebug',
'--logFile=/home/sort.log',
'--provisioner=aws']
if spotInstances:
toilOptions.extend([
'--preemptableNodeType=%s:%s' % (self.instanceType, self.spotBid),
# The RNASeq pipeline does not specify a preemptability requirement so we
# need to specify a default, otherwise jobs would never get scheduled.
'--defaultPreemptable',
'--maxPreemptableNodes=%s' % self.numWorkers])
else:
toilOptions.extend(['--nodeType=' + self.instanceType,
'--maxNodes=%s' % self.numWorkers])
self._runScript(toilOptions)
assert len(self.getMatchingRoles(self.clusterName)) == 1
checkStatsCommand = ['/home/venv/bin/python', '-c',
'import json; import os; '
'json.load(open("/home/" + [f for f in os.listdir("/home/") '
'if f.endswith(".json")].pop()))'
]
self.sshUtil(checkStatsCommand)
volumeID = self.getRootVolID()
ctx = AWSProvisioner._buildContext(self.clusterName)
AWSProvisioner.destroyCluster(self.clusterName)
self.leader.update()
for attempt in range(6):
# https://github.com/BD2KGenomics/toil/issues/1567
# retry this for up to 1 minute until the volume disappears
try:
ctx.ec2.get_all_volumes(volume_ids=[volumeID])
time.sleep(10)
except EC2ResponseError as e:
if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
break
else:
raise
else:
self.fail('Volume with ID %s was not cleaned up properly' % volumeID)
assert len(self.getMatchingRoles(self.clusterName)) == 0
def _test(self, spotInstances=False, fulfillableBid=True):
"""
Does the work of the testing. Many features' test are thrown in here is no particular
order
:param spotInstances: Specify if you want to use spotInstances
:param fulfillableBid: If false, the bid will never succeed. Used to test bid failure
"""
if not fulfillableBid:
self.spotBid = '0.01'
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
self.launchCluster()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
self.leader = AWSProvisioner._getLeader(wait=False,
clusterName=self.clusterName)
ctx = AWSProvisioner._buildContext(self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = [
'virtualenv', '--system-site-packages', '--never-download',
'/home/venv'
]
self.sshUtil(venv_command)
upgrade_command = [
'/home/venv/bin/pip', 'install', 'setuptools==28.7.1'
]
self.sshUtil(upgrade_command)
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
self.sshUtil(yaml_command)
self._getScript()
toilOptions = [
self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil',
'--clean=always', '--retryCount=2', '--clusterStats=/home/',
'--logDebug', '--logFile=/home/sort.log', '--provisioner=aws'
]
if spotInstances:
toilOptions.extend([
'--preemptableNodeType=%s:%s' %
(self.instanceType, self.spotBid),
# The RNASeq pipeline does not specify a preemptability requirement so we
# need to specify a default, otherwise jobs would never get scheduled.
'--defaultPreemptable',
'--maxPreemptableNodes=%s' % self.numWorkers
])
else:
toilOptions.extend([
'--nodeType=' + self.instanceType,
'--maxNodes=%s' % self.numWorkers
])
self._runScript(toilOptions)
assert len(self.getMatchingRoles(self.clusterName)) == 1
checkStatsCommand = [
'/home/venv/bin/python', '-c', 'import json; import os; '
'json.load(open("/home/" + [f for f in os.listdir("/home/") '
'if f.endswith(".json")].pop()))'
]
self.sshUtil(checkStatsCommand)
volumeID = self.getRootVolID()
ctx = AWSProvisioner._buildContext(self.clusterName)
AWSProvisioner.destroyCluster(self.clusterName)
self.leader.update()
for attempt in range(6):
# https://github.com/BD2KGenomics/toil/issues/1567
# retry this for up to 1 minute until the volume disappears
try:
ctx.ec2.get_all_volumes(volume_ids=[volumeID])
time.sleep(10)
except EC2ResponseError as e:
if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
break
else:
raise
else:
self.fail('Volume with ID %s was not cleaned up properly' %
volumeID)
assert len(self.getMatchingRoles(self.clusterName)) == 0
def testAWSProvisionerUtils(self):
clusterName = 'cluster-utils-test' + str(uuid.uuid4())
keyName = os.getenv('TOIL_AWS_KEYNAME')
try:
# --provisioner flag should default to aws, so we're not explicitly
# specifying that here
system([self.toilMain, 'launch-cluster', '--nodeType=t2.micro',
'--keyPairName=' + keyName, clusterName])
finally:
system([self.toilMain, 'destroy-cluster', '--provisioner=aws', clusterName])
try:
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
userTags = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}
tags = {'Name': clusterName, 'Owner': keyName}
tags.update(userTags)
# launch preemptable master with same name
system([self.toilMain, 'launch-cluster', '-t', 'key1=value1', '-t', 'key2=value2', '--tag', 'key3=value3',
'--nodeType=m3.medium:0.2', '--keyPairName=' + keyName, clusterName,
'--provisioner=aws', '--logLevel=DEBUG'])
# test leader tags
leaderTags = AWSProvisioner._getLeader(clusterName).tags
self.assertEqual(tags, leaderTags)
# Test strict host key checking
# Doesn't work when run locally.
if(keyName == 'jenkins@jenkins-master'):
try:
AWSProvisioner.sshLeader(clusterName=clusterName, strict=True)
except RuntimeError:
pass
else:
self.fail("Host key verification passed where it should have failed")
# Add the host key to known_hosts so that the rest of the tests can
# pass without choking on the verification prompt.
AWSProvisioner.sshLeader(clusterName=clusterName, strict=True, sshOptions=['-oStrictHostKeyChecking=no'])
system([self.toilMain, 'ssh-cluster', '--provisioner=aws', clusterName])
testStrings = ["'foo'",
'"foo"',
' foo',
'$PATH',
'"',
"'",
'\\',
'| cat',
'&& cat',
'; cat'
]
for test in testStrings:
logger.info('Testing SSH with special string: %s', test)
compareTo = "import sys; assert sys.argv[1]==%r" % test
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['python', '-', test],
input=compareTo)
try:
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['nonsenseShouldFail'])
except RuntimeError:
pass
else:
self.fail('The remote command failed silently where it should have '
'raised an error')
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['python', '-c', "import os; assert os.environ['TOIL_WORKDIR']=='/var/lib/toil'"])
# `toil rsync-cluster`
# Testing special characters - string.punctuation
fname = '!"#$%&\'()*+,-.;<=>:\ ?@[\\\\]^_`{|}~'
testData = os.urandom(3 * (10**6))
with tempfile.NamedTemporaryFile(suffix=fname) as tmpFile:
relpath = os.path.basename(tmpFile.name)
tmpFile.write(testData)
tmpFile.flush()
# Upload file to leader
AWSProvisioner.rsyncLeader(clusterName=clusterName, args=[tmpFile.name, ":"])
# Ensure file exists
AWSProvisioner.sshLeader(clusterName=clusterName, args=["test", "-e", relpath])
tmpDir = tempfile.mkdtemp()
# Download the file again and make sure it's the same file
# `--protect-args` needed because remote bash chokes on special characters
AWSProvisioner.rsyncLeader(clusterName=clusterName, args=["--protect-args", ":" + relpath, tmpDir])
with open(os.path.join(tmpDir, relpath), "r") as f:
self.assertEqual(f.read(), testData, "Downloaded file does not match original file")
finally:
system([self.toilMain, 'destroy-cluster', '--provisioner=aws', clusterName])
try:
shutil.rmtree(tmpDir)
except NameError:
pass
def _test(self, preemptableJobs=False):
"""
Does the work of the testing. Many features' test are thrown in here is no particular
order
"""
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
self.launchCluster()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
self.leader = AWSProvisioner._getLeader(wait=False,
clusterName=self.clusterName)
ctx = AWSProvisioner._buildContext(self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = [
'virtualenv', '--system-site-packages', '--never-download',
'/home/venv'
]
self.sshUtil(venv_command)
upgrade_command = [
'/home/venv/bin/pip', 'install', 'setuptools==28.7.1'
]
self.sshUtil(upgrade_command)
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
self.sshUtil(yaml_command)
self._getScript()
toilOptions = [
self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil',
'--clean=always', '--retryCount=2', '--clusterStats=/home/',
'--logDebug', '--logFile=/home/sort.log', '--provisioner=aws'
]
toilOptions.extend([
'--nodeTypes=' + ",".join(self.instanceTypes),
'--maxNodes=%s' % ",".join(self.numWorkers)
])
if preemptableJobs:
toilOptions.extend(['--defaultPreemptable'])
self._runScript(toilOptions)
assert len(self.getMatchingRoles(self.clusterName)) == 1
checkStatsCommand = [
'/home/venv/bin/python', '-c', 'import json; import os; '
'json.load(open("/home/" + [f for f in os.listdir("/home/") '
'if f.endswith(".json")].pop()))'
]
self.sshUtil(checkStatsCommand)
from boto.exception import EC2ResponseError
volumeID = self.getRootVolID()
ctx = AWSProvisioner._buildContext(self.clusterName)
AWSProvisioner.destroyCluster(self.clusterName)
self.leader.update()
for attempt in range(6):
# https://github.com/BD2KGenomics/toil/issues/1567
# retry this for up to 1 minute until the volume disappears
try:
ctx.ec2.get_all_volumes(volume_ids=[volumeID])
time.sleep(10)
except EC2ResponseError as e:
if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
break
else:
raise
else:
self.fail('Volume with ID %s was not cleaned up properly' %
volumeID)
assert len(self.getMatchingRoles(self.clusterName)) == 0
def testAWSProvisionerUtils(self):
clusterName = 'cluster-utils-test' + str(uuid.uuid4())
keyName = os.getenv('TOIL_AWS_KEYNAME')
try:
# --provisioner flag should default to aws, so we're not explicitly
# specifying that here
system([
self.toilMain, 'launch-cluster', '--leaderNodeType=t2.micro',
'--keyPairName=' + keyName, clusterName
])
finally:
system([
self.toilMain, 'destroy-cluster', '--provisioner=aws',
clusterName
])
try:
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
userTags = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}
tags = {'Name': clusterName, 'Owner': keyName}
tags.update(userTags)
# launch preemptable master with same name
system([
self.toilMain, 'launch-cluster', '-t', 'key1=value1', '-t',
'key2=value2', '--tag', 'key3=value3',
'--leaderNodeType=m3.medium:0.2', '--keyPairName=' + keyName,
clusterName, '--provisioner=aws', '--logLevel=DEBUG'
])
# test leader tags
leaderTags = AWSProvisioner._getLeader(clusterName).tags
self.assertEqual(tags, leaderTags)
# Test strict host key checking
# Doesn't work when run locally.
if (keyName == 'jenkins@jenkins-master'):
try:
AWSProvisioner.sshLeader(clusterName=clusterName,
strict=True)
except RuntimeError:
pass
else:
self.fail(
"Host key verification passed where it should have failed"
)
# Add the host key to known_hosts so that the rest of the tests can
# pass without choking on the verification prompt.
AWSProvisioner.sshLeader(clusterName=clusterName,
strict=True,
sshOptions=['-oStrictHostKeyChecking=no'])
system([
self.toilMain, 'ssh-cluster', '--provisioner=aws', clusterName
])
testStrings = [
"'foo'", '"foo"', ' foo', '$PATH', '"', "'", '\\', '| cat',
'&& cat', '; cat'
]
for test in testStrings:
logger.info('Testing SSH with special string: %s', test)
compareTo = "import sys; assert sys.argv[1]==%r" % test
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['python', '-', test],
input=compareTo)
try:
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['nonsenseShouldFail'])
except RuntimeError:
pass
else:
self.fail(
'The remote command failed silently where it should have '
'raised an error')
AWSProvisioner.sshLeader(
clusterName=clusterName,
args=[
'python', '-c',
"import os; assert os.environ['TOIL_WORKDIR']=='/var/lib/toil'"
])
# `toil rsync-cluster`
# Testing special characters - string.punctuation
fname = '!"#$%&\'()*+,-.;<=>:\ ?@[\\\\]^_`{|}~'
testData = os.urandom(3 * (10**6))
with tempfile.NamedTemporaryFile(suffix=fname) as tmpFile:
relpath = os.path.basename(tmpFile.name)
tmpFile.write(testData)
tmpFile.flush()
# Upload file to leader
AWSProvisioner.rsyncLeader(clusterName=clusterName,
args=[tmpFile.name, ":"])
# Ensure file exists
AWSProvisioner.sshLeader(clusterName=clusterName,
args=["test", "-e", relpath])
tmpDir = tempfile.mkdtemp()
# Download the file again and make sure it's the same file
# `--protect-args` needed because remote bash chokes on special characters
AWSProvisioner.rsyncLeader(
clusterName=clusterName,
args=["--protect-args", ":" + relpath, tmpDir])
with open(os.path.join(tmpDir, relpath), "r") as f:
self.assertEqual(
f.read(), testData,
"Downloaded file does not match original file")
finally:
system([
self.toilMain, 'destroy-cluster', '--provisioner=aws',
clusterName
])
try:
shutil.rmtree(tmpDir)
except NameError:
pass
