def _test(self, spotInstances=False):
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
self.createClusterUtil()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = ['virtualenv', '--system-site-packages', '--never-download',
'/home/venv']
self.sshUtil(venv_command)
upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1']
self.sshUtil(upgrade_command)
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
self.sshUtil(yaml_command)
# install toil scripts
install_command = ['/home/venv/bin/pip', 'install', 'toil-scripts==%s' % self.toilScripts]
self.sshUtil(install_command)
toilOptions = ['--batchSystem=mesos',
'--workDir=/var/lib/toil',
'--mesosMaster=%s:5050' % leader.private_ip_address,
'--clean=always',
'--retryCount=2']
toilOptions.extend(['--provisioner=aws',
'--nodeType=' + self.instanceType,
'--maxNodes=%s' % self.numWorkers,
'--logDebug'])
if spotInstances:
toilOptions.extend([
'--preemptableNodeType=%s:%s' % (self.instanceType, self.spotBid),
# The RNASeq pipeline does not specify a preemptability requirement so we
# need to specify a default, otherwise jobs would never get scheduled.
'--defaultPreemptable',
'--maxPreemptableNodes=%s' % self.numWorkers])
toilOptions = ' '.join(toilOptions)
# TOIL_AWS_NODE_DEBUG prevents the provisioner from killing nodes that
# fail a status check. This allows for easier debugging of
# https://github.com/BD2KGenomics/toil/issues/1141
runCommand = ['bash', '-c',
'PATH=/home/venv/bin/:$PATH '
'TOIL_AWS_NODE_DEBUG=True '
'TOIL_SCRIPTS_TEST_NUM_SAMPLES='+str(self.numSamples)+
' TOIL_SCRIPTS_TEST_TOIL_OPTIONS=' + pipes.quote(toilOptions) +
' TOIL_SCRIPTS_TEST_JOBSTORE=' + self.jobStore +
' /home/venv/bin/python -m unittest -v' +
' toil_scripts.rnaseq_cgl.test.test_rnaseq_cgl.RNASeqCGLTest.test_manifest']
self.sshUtil(runCommand)
assert len(self.getMatchingRoles(self.clusterName)) == 1
AWSProvisioner.destroyCluster(self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 0
def test_read_write_global_files(self):
"""
Make sure the `_write_file_to_cloud()` and `_read_file_from_cloud()`
functions of the AWS provisioner work as intended.
"""
provisioner = AWSProvisioner(f'aws-provisioner-test-{uuid4()}',
'mesos', 'us-west-2a', 50, None, None)
key = 'config/test.txt'
contents = b"Hello, this is a test."
try:
url = provisioner._write_file_to_cloud(key, contents=contents)
self.assertTrue(url.startswith("s3://"))
self.assertEqual(contents, provisioner._read_file_from_cloud(key))
finally:
# the cluster was never launched, but we need to clean up the s3 bucket
provisioner.destroyCluster()
def _test(self, spotInstances=False, fulfillableBid=True):
"""
Does the work of the testing. Many features' test are thrown in here is no particular
order
:param spotInstances: Specify if you want to use spotInstances
:param fulfillableBid: If false, the bid will never succeed. Used to test bid failure
"""
if not fulfillableBid:
self.spotBid = '0.01'
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
self.launchCluster()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
self.leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName)
ctx = AWSProvisioner._buildContext(self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = ['virtualenv', '--system-site-packages', '--never-download',
'/home/venv']
self.sshUtil(venv_command)
upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1']
self.sshUtil(upgrade_command)
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
self.sshUtil(yaml_command)
self._getScript()
toilOptions = [self.jobStore,
'--batchSystem=mesos',
'--workDir=/var/lib/toil',
'--clean=always',
'--retryCount=2',
'--clusterStats=/home/',
'--logDebug',
'--logFile=/home/sort.log',
'--provisioner=aws']
if spotInstances:
toilOptions.extend([
'--preemptableNodeType=%s:%s' % (self.instanceType, self.spotBid),
# The RNASeq pipeline does not specify a preemptability requirement so we
# need to specify a default, otherwise jobs would never get scheduled.
'--defaultPreemptable',
'--maxPreemptableNodes=%s' % self.numWorkers])
else:
toilOptions.extend(['--nodeType=' + self.instanceType,
'--maxNodes=%s' % self.numWorkers])
self._runScript(toilOptions)
assert len(self.getMatchingRoles(self.clusterName)) == 1
checkStatsCommand = ['/home/venv/bin/python', '-c',
'import json; import os; '
'json.load(open("/home/" + [f for f in os.listdir("/home/") '
'if f.endswith(".json")].pop()))'
]
self.sshUtil(checkStatsCommand)
volumeID = self.getRootVolID()
ctx = AWSProvisioner._buildContext(self.clusterName)
AWSProvisioner.destroyCluster(self.clusterName)
self.leader.update()
for attempt in range(6):
# https://github.com/BD2KGenomics/toil/issues/1567
# retry this for up to 1 minute until the volume disappears
try:
ctx.ec2.get_all_volumes(volume_ids=[volumeID])
time.sleep(10)
except EC2ResponseError as e:
if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
break
else:
raise
else:
self.fail('Volume with ID %s was not cleaned up properly' % volumeID)
assert len(self.getMatchingRoles(self.clusterName)) == 0
def _test(self, spotInstances=False, fulfillableBid=True):
"""
Does the work of the testing. Many features' test are thrown in here is no particular
order
:param spotInstances: Specify if you want to use spotInstances
:param fulfillableBid: If false, the bid will never succeed. Used to test bid failure
"""
if not fulfillableBid:
self.spotBid = '0.01'
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
self.launchCluster()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
self.leader = AWSProvisioner._getLeader(wait=False,
clusterName=self.clusterName)
ctx = AWSProvisioner._buildContext(self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = [
'virtualenv', '--system-site-packages', '--never-download',
'/home/venv'
]
self.sshUtil(venv_command)
upgrade_command = [
'/home/venv/bin/pip', 'install', 'setuptools==28.7.1'
]
self.sshUtil(upgrade_command)
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
self.sshUtil(yaml_command)
self._getScript()
toilOptions = [
self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil',
'--clean=always', '--retryCount=2', '--clusterStats=/home/',
'--logDebug', '--logFile=/home/sort.log', '--provisioner=aws'
]
if spotInstances:
toilOptions.extend([
'--preemptableNodeType=%s:%s' %
(self.instanceType, self.spotBid),
# The RNASeq pipeline does not specify a preemptability requirement so we
# need to specify a default, otherwise jobs would never get scheduled.
'--defaultPreemptable',
'--maxPreemptableNodes=%s' % self.numWorkers
])
else:
toilOptions.extend([
'--nodeType=' + self.instanceType,
'--maxNodes=%s' % self.numWorkers
])
self._runScript(toilOptions)
assert len(self.getMatchingRoles(self.clusterName)) == 1
checkStatsCommand = [
'/home/venv/bin/python', '-c', 'import json; import os; '
'json.load(open("/home/" + [f for f in os.listdir("/home/") '
'if f.endswith(".json")].pop()))'
]
self.sshUtil(checkStatsCommand)
volumeID = self.getRootVolID()
ctx = AWSProvisioner._buildContext(self.clusterName)
AWSProvisioner.destroyCluster(self.clusterName)
self.leader.update()
for attempt in range(6):
# https://github.com/BD2KGenomics/toil/issues/1567
# retry this for up to 1 minute until the volume disappears
try:
ctx.ec2.get_all_volumes(volume_ids=[volumeID])
time.sleep(10)
except EC2ResponseError as e:
if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
break
else:
raise
else:
self.fail('Volume with ID %s was not cleaned up properly' %
volumeID)
assert len(self.getMatchingRoles(self.clusterName)) == 0
def _test(self, preemptableJobs=False):
"""
Does the work of the testing. Many features' test are thrown in here is no particular
order
"""
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
self.launchCluster()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
self.leader = AWSProvisioner._getLeader(wait=False,
clusterName=self.clusterName)
ctx = AWSProvisioner._buildContext(self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = [
'virtualenv', '--system-site-packages', '--never-download',
'/home/venv'
]
self.sshUtil(venv_command)
upgrade_command = [
'/home/venv/bin/pip', 'install', 'setuptools==28.7.1'
]
self.sshUtil(upgrade_command)
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
self.sshUtil(yaml_command)
self._getScript()
toilOptions = [
self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil',
'--clean=always', '--retryCount=2', '--clusterStats=/home/',
'--logDebug', '--logFile=/home/sort.log', '--provisioner=aws'
]
toilOptions.extend([
'--nodeTypes=' + ",".join(self.instanceTypes),
'--maxNodes=%s' % ",".join(self.numWorkers)
])
if preemptableJobs:
toilOptions.extend(['--defaultPreemptable'])
self._runScript(toilOptions)
assert len(self.getMatchingRoles(self.clusterName)) == 1
checkStatsCommand = [
'/home/venv/bin/python', '-c', 'import json; import os; '
'json.load(open("/home/" + [f for f in os.listdir("/home/") '
'if f.endswith(".json")].pop()))'
]
self.sshUtil(checkStatsCommand)
from boto.exception import EC2ResponseError
volumeID = self.getRootVolID()
ctx = AWSProvisioner._buildContext(self.clusterName)
AWSProvisioner.destroyCluster(self.clusterName)
self.leader.update()
for attempt in range(6):
# https://github.com/BD2KGenomics/toil/issues/1567
# retry this for up to 1 minute until the volume disappears
try:
ctx.ec2.get_all_volumes(volume_ids=[volumeID])
time.sleep(10)
except EC2ResponseError as e:
if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
break
else:
raise
else:
self.fail('Volume with ID %s was not cleaned up properly' %
volumeID)
assert len(self.getMatchingRoles(self.clusterName)) == 0
def tearDown(self):
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
AWSProvisioner.destroyCluster(self.clusterName)
