def launchCluster(self): from boto.ec2.blockdevicemapping import BlockDeviceType self.createClusterUtil(args=[ '--leaderStorage', str(self.requestedLeaderStorage), '--nodeTypes', ",".join( self.instanceTypes), '-w', ",".join(self.numWorkers), '--nodeStorage', str(self.requestedLeaderStorage) ]) ctx = AWSProvisioner._buildContext(self.clusterName) nodes = AWSProvisioner._getNodesInCluster(ctx, self.clusterName, both=True) nodes.sort(key=lambda x: x.launch_time) # assuming that leader is first workers = nodes[1:] # test that two worker nodes were created self.assertEqual(2, len(workers)) # test that workers have expected storage size # just use the first worker worker = workers[0] worker = next(wait_instances_running(ctx.ec2, [worker])) rootBlockDevice = worker.block_device_mapping["/dev/xvda"] self.assertTrue(isinstance(rootBlockDevice, BlockDeviceType)) rootVolume = ctx.ec2.get_all_volumes( volume_ids=[rootBlockDevice.volume_id])[0] self.assertGreaterEqual(rootVolume.size, self.requestedNodeStorage)
def launchCluster(self): self.createClusterUtil(args=['-w', '2']) ctx = AWSProvisioner._buildContext(self.clusterName) # test that two worker nodes were created + 1 for leader self.assertEqual( 2 + 1, len( AWSProvisioner._getNodesInCluster(ctx, self.clusterName, both=True)))
def getRootVolID(self): """ Adds in test to check that EBS volume is build with adequate size. Otherwise is functionally equivalent to parent. :return: volumeID """ volumeID = super(AWSAutoscaleTest, self).getRootVolID() ctx = AWSProvisioner._buildContext(self.clusterName) rootVolume = ctx.ec2.get_all_volumes(volume_ids=[volumeID])[0] # test that the leader is given adequate storage self.assertGreaterEqual(rootVolume.size, self.requestedLeaderStorage) return volumeID
def launchCluster(self): self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage), '-w', '2', '--nodeStorage', str(self.requestedLeaderStorage)]) ctx = AWSProvisioner._buildContext(self.clusterName) nodes = AWSProvisioner._getNodesInCluster(ctx, self.clusterName, both=True) nodes.sort(key=lambda x: x.launch_time) # assuming that leader is first workers = nodes[1:] # test that two worker nodes were created self.assertEqual(2, len(workers)) # test that workers have expected storage size # just use the first worker worker = workers[0] worker = next(wait_instances_running(ctx.ec2, [worker])) rootBlockDevice = worker.block_device_mapping["/dev/xvda"] self.assertTrue(isinstance(rootBlockDevice, BlockDeviceType)) rootVolume = ctx.ec2.get_all_volumes(volume_ids=[rootBlockDevice.volume_id])[0] self.assertGreaterEqual(rootVolume.size, self.requestedNodeStorage)
def getMatchingRoles(self, clusterName): from toil.provisioners.aws.awsProvisioner import AWSProvisioner ctx = AWSProvisioner._buildContext(clusterName) roles = list(ctx.local_roles()) return roles
def _test(self, spotInstances=False, fulfillableBid=True): """ Does the work of the testing. Many features' test are thrown in here is no particular order :param spotInstances: Specify if you want to use spotInstances :param fulfillableBid: If false, the bid will never succeed. Used to test bid failure """ if not fulfillableBid: self.spotBid = '0.01' from toil.provisioners.aws.awsProvisioner import AWSProvisioner self.launchCluster() # get the leader so we know the IP address - we don't need to wait since create cluster # already insures the leader is running self.leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName) ctx = AWSProvisioner._buildContext(self.clusterName) assert len(self.getMatchingRoles(self.clusterName)) == 1 # --never-download prevents silent upgrades to pip, wheel and setuptools venv_command = ['virtualenv', '--system-site-packages', '--never-download', '/home/venv'] self.sshUtil(venv_command) upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1'] self.sshUtil(upgrade_command) yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12'] self.sshUtil(yaml_command) self._getScript() toilOptions = [self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil', '--clean=always', '--retryCount=2', '--clusterStats=/home/', '--logDebug', '--logFile=/home/sort.log', '--provisioner=aws'] if spotInstances: toilOptions.extend([ '--preemptableNodeType=%s:%s' % (self.instanceType, self.spotBid), # The RNASeq pipeline does not specify a preemptability requirement so we # need to specify a default, otherwise jobs would never get scheduled. '--defaultPreemptable', '--maxPreemptableNodes=%s' % self.numWorkers]) else: toilOptions.extend(['--nodeType=' + self.instanceType, '--maxNodes=%s' % self.numWorkers]) self._runScript(toilOptions) assert len(self.getMatchingRoles(self.clusterName)) == 1 checkStatsCommand = ['/home/venv/bin/python', '-c', 'import json; import os; ' 'json.load(open("/home/" + [f for f in os.listdir("/home/") ' 'if f.endswith(".json")].pop()))' ] self.sshUtil(checkStatsCommand) volumeID = self.getRootVolID() ctx = AWSProvisioner._buildContext(self.clusterName) AWSProvisioner.destroyCluster(self.clusterName) self.leader.update() for attempt in range(6): # https://github.com/BD2KGenomics/toil/issues/1567 # retry this for up to 1 minute until the volume disappears try: ctx.ec2.get_all_volumes(volume_ids=[volumeID]) time.sleep(10) except EC2ResponseError as e: if e.status == 400 and 'InvalidVolume.NotFound' in e.code: break else: raise else: self.fail('Volume with ID %s was not cleaned up properly' % volumeID) assert len(self.getMatchingRoles(self.clusterName)) == 0
def _test(self, spotInstances=False, fulfillableBid=True): """ Does the work of the testing. Many features' test are thrown in here is no particular order :param spotInstances: Specify if you want to use spotInstances :param fulfillableBid: If false, the bid will never succeed. Used to test bid failure """ if not fulfillableBid: self.spotBid = '0.01' from toil.provisioners.aws.awsProvisioner import AWSProvisioner self.launchCluster() # get the leader so we know the IP address - we don't need to wait since create cluster # already insures the leader is running self.leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName) ctx = AWSProvisioner._buildContext(self.clusterName) assert len(self.getMatchingRoles(self.clusterName)) == 1 # --never-download prevents silent upgrades to pip, wheel and setuptools venv_command = [ 'virtualenv', '--system-site-packages', '--never-download', '/home/venv' ] self.sshUtil(venv_command) upgrade_command = [ '/home/venv/bin/pip', 'install', 'setuptools==28.7.1' ] self.sshUtil(upgrade_command) yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12'] self.sshUtil(yaml_command) self._getScript() toilOptions = [ self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil', '--clean=always', '--retryCount=2', '--clusterStats=/home/', '--logDebug', '--logFile=/home/sort.log', '--provisioner=aws' ] if spotInstances: toilOptions.extend([ '--preemptableNodeType=%s:%s' % (self.instanceType, self.spotBid), # The RNASeq pipeline does not specify a preemptability requirement so we # need to specify a default, otherwise jobs would never get scheduled. '--defaultPreemptable', '--maxPreemptableNodes=%s' % self.numWorkers ]) else: toilOptions.extend([ '--nodeType=' + self.instanceType, '--maxNodes=%s' % self.numWorkers ]) self._runScript(toilOptions) assert len(self.getMatchingRoles(self.clusterName)) == 1 checkStatsCommand = [ '/home/venv/bin/python', '-c', 'import json; import os; ' 'json.load(open("/home/" + [f for f in os.listdir("/home/") ' 'if f.endswith(".json")].pop()))' ] self.sshUtil(checkStatsCommand) volumeID = self.getRootVolID() ctx = AWSProvisioner._buildContext(self.clusterName) AWSProvisioner.destroyCluster(self.clusterName) self.leader.update() for attempt in range(6): # https://github.com/BD2KGenomics/toil/issues/1567 # retry this for up to 1 minute until the volume disappears try: ctx.ec2.get_all_volumes(volume_ids=[volumeID]) time.sleep(10) except EC2ResponseError as e: if e.status == 400 and 'InvalidVolume.NotFound' in e.code: break else: raise else: self.fail('Volume with ID %s was not cleaned up properly' % volumeID) assert len(self.getMatchingRoles(self.clusterName)) == 0
def _test(self, preemptableJobs=False): """ Does the work of the testing. Many features' test are thrown in here is no particular order """ from toil.provisioners.aws.awsProvisioner import AWSProvisioner self.launchCluster() # get the leader so we know the IP address - we don't need to wait since create cluster # already insures the leader is running self.leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName) ctx = AWSProvisioner._buildContext(self.clusterName) assert len(self.getMatchingRoles(self.clusterName)) == 1 # --never-download prevents silent upgrades to pip, wheel and setuptools venv_command = [ 'virtualenv', '--system-site-packages', '--never-download', '/home/venv' ] self.sshUtil(venv_command) upgrade_command = [ '/home/venv/bin/pip', 'install', 'setuptools==28.7.1' ] self.sshUtil(upgrade_command) yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12'] self.sshUtil(yaml_command) self._getScript() toilOptions = [ self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil', '--clean=always', '--retryCount=2', '--clusterStats=/home/', '--logDebug', '--logFile=/home/sort.log', '--provisioner=aws' ] toilOptions.extend([ '--nodeTypes=' + ",".join(self.instanceTypes), '--maxNodes=%s' % ",".join(self.numWorkers) ]) if preemptableJobs: toilOptions.extend(['--defaultPreemptable']) self._runScript(toilOptions) assert len(self.getMatchingRoles(self.clusterName)) == 1 checkStatsCommand = [ '/home/venv/bin/python', '-c', 'import json; import os; ' 'json.load(open("/home/" + [f for f in os.listdir("/home/") ' 'if f.endswith(".json")].pop()))' ] self.sshUtil(checkStatsCommand) from boto.exception import EC2ResponseError volumeID = self.getRootVolID() ctx = AWSProvisioner._buildContext(self.clusterName) AWSProvisioner.destroyCluster(self.clusterName) self.leader.update() for attempt in range(6): # https://github.com/BD2KGenomics/toil/issues/1567 # retry this for up to 1 minute until the volume disappears try: ctx.ec2.get_all_volumes(volume_ids=[volumeID]) time.sleep(10) except EC2ResponseError as e: if e.status == 400 and 'InvalidVolume.NotFound' in e.code: break else: raise else: self.fail('Volume with ID %s was not cleaned up properly' % volumeID) assert len(self.getMatchingRoles(self.clusterName)) == 0