def launchCluster(self):
from boto.ec2.blockdevicemapping import BlockDeviceType
self.createClusterUtil(args=[
'--leaderStorage',
str(self.requestedLeaderStorage), '--nodeTypes', ",".join(
self.instanceTypes), '-w', ",".join(self.numWorkers),
'--nodeStorage',
str(self.requestedLeaderStorage)
])
ctx = AWSProvisioner._buildContext(self.clusterName)
nodes = AWSProvisioner._getNodesInCluster(ctx,
self.clusterName,
both=True)
nodes.sort(key=lambda x: x.launch_time)
# assuming that leader is first
workers = nodes[1:]
# test that two worker nodes were created
self.assertEqual(2, len(workers))
# test that workers have expected storage size
# just use the first worker
worker = workers[0]
worker = next(wait_instances_running(ctx.ec2, [worker]))
rootBlockDevice = worker.block_device_mapping["/dev/xvda"]
self.assertTrue(isinstance(rootBlockDevice, BlockDeviceType))
rootVolume = ctx.ec2.get_all_volumes(
volume_ids=[rootBlockDevice.volume_id])[0]
self.assertGreaterEqual(rootVolume.size, self.requestedNodeStorage)
def launchCluster(self):
self.createClusterUtil(args=['-w', '2'])
ctx = AWSProvisioner._buildContext(self.clusterName)
# test that two worker nodes were created + 1 for leader
self.assertEqual(
2 + 1,
len(
AWSProvisioner._getNodesInCluster(ctx,
self.clusterName,
both=True)))
def getRootVolID(self):
"""
Adds in test to check that EBS volume is build with adequate size.
Otherwise is functionally equivalent to parent.
:return: volumeID
"""
volumeID = super(AWSAutoscaleTest, self).getRootVolID()
ctx = AWSProvisioner._buildContext(self.clusterName)
rootVolume = ctx.ec2.get_all_volumes(volume_ids=[volumeID])[0]
# test that the leader is given adequate storage
self.assertGreaterEqual(rootVolume.size, self.requestedLeaderStorage)
return volumeID
def getRootVolID(self):
"""
Adds in test to check that EBS volume is build with adequate size.
Otherwise is functionally equivalent to parent.
:return: volumeID
"""
volumeID = super(AWSAutoscaleTest, self).getRootVolID()
ctx = AWSProvisioner._buildContext(self.clusterName)
rootVolume = ctx.ec2.get_all_volumes(volume_ids=[volumeID])[0]
# test that the leader is given adequate storage
self.assertGreaterEqual(rootVolume.size, self.requestedLeaderStorage)
return volumeID
def launchCluster(self):
self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage),
'-w', '2', '--nodeStorage', str(self.requestedLeaderStorage)])
ctx = AWSProvisioner._buildContext(self.clusterName)
nodes = AWSProvisioner._getNodesInCluster(ctx, self.clusterName, both=True)
nodes.sort(key=lambda x: x.launch_time)
# assuming that leader is first
workers = nodes[1:]
# test that two worker nodes were created
self.assertEqual(2, len(workers))
# test that workers have expected storage size
# just use the first worker
worker = workers[0]
worker = next(wait_instances_running(ctx.ec2, [worker]))
rootBlockDevice = worker.block_device_mapping["/dev/xvda"]
self.assertTrue(isinstance(rootBlockDevice, BlockDeviceType))
rootVolume = ctx.ec2.get_all_volumes(volume_ids=[rootBlockDevice.volume_id])[0]
self.assertGreaterEqual(rootVolume.size, self.requestedNodeStorage)
def getMatchingRoles(self, clusterName):
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
ctx = AWSProvisioner._buildContext(clusterName)
roles = list(ctx.local_roles())
return roles
def _test(self, spotInstances=False, fulfillableBid=True):
"""
Does the work of the testing. Many features' test are thrown in here is no particular
order
:param spotInstances: Specify if you want to use spotInstances
:param fulfillableBid: If false, the bid will never succeed. Used to test bid failure
"""
if not fulfillableBid:
self.spotBid = '0.01'
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
self.launchCluster()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
self.leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName)
ctx = AWSProvisioner._buildContext(self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = ['virtualenv', '--system-site-packages', '--never-download',
'/home/venv']
self.sshUtil(venv_command)
upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1']
self.sshUtil(upgrade_command)
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
self.sshUtil(yaml_command)
self._getScript()
toilOptions = [self.jobStore,
'--batchSystem=mesos',
'--workDir=/var/lib/toil',
'--clean=always',
'--retryCount=2',
'--clusterStats=/home/',
'--logDebug',
'--logFile=/home/sort.log',
'--provisioner=aws']
if spotInstances:
toilOptions.extend([
'--preemptableNodeType=%s:%s' % (self.instanceType, self.spotBid),
# The RNASeq pipeline does not specify a preemptability requirement so we
# need to specify a default, otherwise jobs would never get scheduled.
'--defaultPreemptable',
'--maxPreemptableNodes=%s' % self.numWorkers])
else:
toilOptions.extend(['--nodeType=' + self.instanceType,
'--maxNodes=%s' % self.numWorkers])
self._runScript(toilOptions)
assert len(self.getMatchingRoles(self.clusterName)) == 1
checkStatsCommand = ['/home/venv/bin/python', '-c',
'import json; import os; '
'json.load(open("/home/" + [f for f in os.listdir("/home/") '
'if f.endswith(".json")].pop()))'
]
self.sshUtil(checkStatsCommand)
volumeID = self.getRootVolID()
ctx = AWSProvisioner._buildContext(self.clusterName)
AWSProvisioner.destroyCluster(self.clusterName)
self.leader.update()
for attempt in range(6):
# https://github.com/BD2KGenomics/toil/issues/1567
# retry this for up to 1 minute until the volume disappears
try:
ctx.ec2.get_all_volumes(volume_ids=[volumeID])
time.sleep(10)
except EC2ResponseError as e:
if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
break
else:
raise
else:
self.fail('Volume with ID %s was not cleaned up properly' % volumeID)
assert len(self.getMatchingRoles(self.clusterName)) == 0
def getMatchingRoles(self, clusterName):
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
ctx = AWSProvisioner._buildContext(clusterName)
roles = list(ctx.local_roles())
return roles
def _test(self, spotInstances=False, fulfillableBid=True):
"""
Does the work of the testing. Many features' test are thrown in here is no particular
order
:param spotInstances: Specify if you want to use spotInstances
:param fulfillableBid: If false, the bid will never succeed. Used to test bid failure
"""
if not fulfillableBid:
self.spotBid = '0.01'
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
self.launchCluster()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
self.leader = AWSProvisioner._getLeader(wait=False,
clusterName=self.clusterName)
ctx = AWSProvisioner._buildContext(self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = [
'virtualenv', '--system-site-packages', '--never-download',
'/home/venv'
]
self.sshUtil(venv_command)
upgrade_command = [
'/home/venv/bin/pip', 'install', 'setuptools==28.7.1'
]
self.sshUtil(upgrade_command)
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
self.sshUtil(yaml_command)
self._getScript()
toilOptions = [
self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil',
'--clean=always', '--retryCount=2', '--clusterStats=/home/',
'--logDebug', '--logFile=/home/sort.log', '--provisioner=aws'
]
if spotInstances:
toilOptions.extend([
'--preemptableNodeType=%s:%s' %
(self.instanceType, self.spotBid),
# The RNASeq pipeline does not specify a preemptability requirement so we
# need to specify a default, otherwise jobs would never get scheduled.
'--defaultPreemptable',
'--maxPreemptableNodes=%s' % self.numWorkers
])
else:
toilOptions.extend([
'--nodeType=' + self.instanceType,
'--maxNodes=%s' % self.numWorkers
])
self._runScript(toilOptions)
assert len(self.getMatchingRoles(self.clusterName)) == 1
checkStatsCommand = [
'/home/venv/bin/python', '-c', 'import json; import os; '
'json.load(open("/home/" + [f for f in os.listdir("/home/") '
'if f.endswith(".json")].pop()))'
]
self.sshUtil(checkStatsCommand)
volumeID = self.getRootVolID()
ctx = AWSProvisioner._buildContext(self.clusterName)
AWSProvisioner.destroyCluster(self.clusterName)
self.leader.update()
for attempt in range(6):
# https://github.com/BD2KGenomics/toil/issues/1567
# retry this for up to 1 minute until the volume disappears
try:
ctx.ec2.get_all_volumes(volume_ids=[volumeID])
time.sleep(10)
except EC2ResponseError as e:
if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
break
else:
raise
else:
self.fail('Volume with ID %s was not cleaned up properly' %
volumeID)
assert len(self.getMatchingRoles(self.clusterName)) == 0
def _test(self, preemptableJobs=False):
"""
Does the work of the testing. Many features' test are thrown in here is no particular
order
"""
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
self.launchCluster()
# get the leader so we know the IP address - we don't need to wait since create cluster
# already insures the leader is running
self.leader = AWSProvisioner._getLeader(wait=False,
clusterName=self.clusterName)
ctx = AWSProvisioner._buildContext(self.clusterName)
assert len(self.getMatchingRoles(self.clusterName)) == 1
# --never-download prevents silent upgrades to pip, wheel and setuptools
venv_command = [
'virtualenv', '--system-site-packages', '--never-download',
'/home/venv'
]
self.sshUtil(venv_command)
upgrade_command = [
'/home/venv/bin/pip', 'install', 'setuptools==28.7.1'
]
self.sshUtil(upgrade_command)
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
self.sshUtil(yaml_command)
self._getScript()
toilOptions = [
self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil',
'--clean=always', '--retryCount=2', '--clusterStats=/home/',
'--logDebug', '--logFile=/home/sort.log', '--provisioner=aws'
]
toilOptions.extend([
'--nodeTypes=' + ",".join(self.instanceTypes),
'--maxNodes=%s' % ",".join(self.numWorkers)
])
if preemptableJobs:
toilOptions.extend(['--defaultPreemptable'])
self._runScript(toilOptions)
assert len(self.getMatchingRoles(self.clusterName)) == 1
checkStatsCommand = [
'/home/venv/bin/python', '-c', 'import json; import os; '
'json.load(open("/home/" + [f for f in os.listdir("/home/") '
'if f.endswith(".json")].pop()))'
]
self.sshUtil(checkStatsCommand)
from boto.exception import EC2ResponseError
volumeID = self.getRootVolID()
ctx = AWSProvisioner._buildContext(self.clusterName)
AWSProvisioner.destroyCluster(self.clusterName)
self.leader.update()
for attempt in range(6):
# https://github.com/BD2KGenomics/toil/issues/1567
# retry this for up to 1 minute until the volume disappears
try:
ctx.ec2.get_all_volumes(volume_ids=[volumeID])
time.sleep(10)
except EC2ResponseError as e:
if e.status == 400 and 'InvalidVolume.NotFound' in e.code:
break
else:
raise
else:
self.fail('Volume with ID %s was not cleaned up properly' %
volumeID)
assert len(self.getMatchingRoles(self.clusterName)) == 0
