def _test(self, spotInstances=False): from toil.provisioners.aws.awsProvisioner import AWSProvisioner self.createClusterUtil() # get the leader so we know the IP address - we don't need to wait since create cluster # already insures the leader is running leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName) assert len(self.getMatchingRoles(self.clusterName)) == 1 # --never-download prevents silent upgrades to pip, wheel and setuptools venv_command = ['virtualenv', '--system-site-packages', '--never-download', '/home/venv'] self.sshUtil(venv_command) upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1'] self.sshUtil(upgrade_command) yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12'] self.sshUtil(yaml_command) # install toil scripts install_command = ['/home/venv/bin/pip', 'install', 'toil-scripts==%s' % self.toilScripts] self.sshUtil(install_command) toilOptions = ['--batchSystem=mesos', '--workDir=/var/lib/toil', '--mesosMaster=%s:5050' % leader.private_ip_address, '--clean=always', '--retryCount=2'] toilOptions.extend(['--provisioner=aws', '--nodeType=' + self.instanceType, '--maxNodes=%s' % self.numWorkers, '--logDebug']) if spotInstances: toilOptions.extend([ '--preemptableNodeType=%s:%s' % (self.instanceType, self.spotBid), # The RNASeq pipeline does not specify a preemptability requirement so we # need to specify a default, otherwise jobs would never get scheduled. '--defaultPreemptable', '--maxPreemptableNodes=%s' % self.numWorkers]) toilOptions = ' '.join(toilOptions) # TOIL_AWS_NODE_DEBUG prevents the provisioner from killing nodes that # fail a status check. This allows for easier debugging of # https://github.com/BD2KGenomics/toil/issues/1141 runCommand = ['bash', '-c', 'PATH=/home/venv/bin/:$PATH ' 'TOIL_AWS_NODE_DEBUG=True ' 'TOIL_SCRIPTS_TEST_NUM_SAMPLES='+str(self.numSamples)+ ' TOIL_SCRIPTS_TEST_TOIL_OPTIONS=' + pipes.quote(toilOptions) + ' TOIL_SCRIPTS_TEST_JOBSTORE=' + self.jobStore + ' /home/venv/bin/python -m unittest -v' + ' toil_scripts.rnaseq_cgl.test.test_rnaseq_cgl.RNASeqCGLTest.test_manifest'] self.sshUtil(runCommand) assert len(self.getMatchingRoles(self.clusterName)) == 1 AWSProvisioner.destroyCluster(self.clusterName) assert len(self.getMatchingRoles(self.clusterName)) == 0
def test_read_write_global_files(self): """ Make sure the `_write_file_to_cloud()` and `_read_file_from_cloud()` functions of the AWS provisioner work as intended. """ provisioner = AWSProvisioner(f'aws-provisioner-test-{uuid4()}', 'mesos', 'us-west-2a', 50, None, None) key = 'config/test.txt' contents = b"Hello, this is a test." try: url = provisioner._write_file_to_cloud(key, contents=contents) self.assertTrue(url.startswith("s3://")) self.assertEqual(contents, provisioner._read_file_from_cloud(key)) finally: # the cluster was never launched, but we need to clean up the s3 bucket provisioner.destroyCluster()
def _test(self, spotInstances=False, fulfillableBid=True): """ Does the work of the testing. Many features' test are thrown in here is no particular order :param spotInstances: Specify if you want to use spotInstances :param fulfillableBid: If false, the bid will never succeed. Used to test bid failure """ if not fulfillableBid: self.spotBid = '0.01' from toil.provisioners.aws.awsProvisioner import AWSProvisioner self.launchCluster() # get the leader so we know the IP address - we don't need to wait since create cluster # already insures the leader is running self.leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName) ctx = AWSProvisioner._buildContext(self.clusterName) assert len(self.getMatchingRoles(self.clusterName)) == 1 # --never-download prevents silent upgrades to pip, wheel and setuptools venv_command = ['virtualenv', '--system-site-packages', '--never-download', '/home/venv'] self.sshUtil(venv_command) upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1'] self.sshUtil(upgrade_command) yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12'] self.sshUtil(yaml_command) self._getScript() toilOptions = [self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil', '--clean=always', '--retryCount=2', '--clusterStats=/home/', '--logDebug', '--logFile=/home/sort.log', '--provisioner=aws'] if spotInstances: toilOptions.extend([ '--preemptableNodeType=%s:%s' % (self.instanceType, self.spotBid), # The RNASeq pipeline does not specify a preemptability requirement so we # need to specify a default, otherwise jobs would never get scheduled. '--defaultPreemptable', '--maxPreemptableNodes=%s' % self.numWorkers]) else: toilOptions.extend(['--nodeType=' + self.instanceType, '--maxNodes=%s' % self.numWorkers]) self._runScript(toilOptions) assert len(self.getMatchingRoles(self.clusterName)) == 1 checkStatsCommand = ['/home/venv/bin/python', '-c', 'import json; import os; ' 'json.load(open("/home/" + [f for f in os.listdir("/home/") ' 'if f.endswith(".json")].pop()))' ] self.sshUtil(checkStatsCommand) volumeID = self.getRootVolID() ctx = AWSProvisioner._buildContext(self.clusterName) AWSProvisioner.destroyCluster(self.clusterName) self.leader.update() for attempt in range(6): # https://github.com/BD2KGenomics/toil/issues/1567 # retry this for up to 1 minute until the volume disappears try: ctx.ec2.get_all_volumes(volume_ids=[volumeID]) time.sleep(10) except EC2ResponseError as e: if e.status == 400 and 'InvalidVolume.NotFound' in e.code: break else: raise else: self.fail('Volume with ID %s was not cleaned up properly' % volumeID) assert len(self.getMatchingRoles(self.clusterName)) == 0
def _test(self, spotInstances=False, fulfillableBid=True): """ Does the work of the testing. Many features' test are thrown in here is no particular order :param spotInstances: Specify if you want to use spotInstances :param fulfillableBid: If false, the bid will never succeed. Used to test bid failure """ if not fulfillableBid: self.spotBid = '0.01' from toil.provisioners.aws.awsProvisioner import AWSProvisioner self.launchCluster() # get the leader so we know the IP address - we don't need to wait since create cluster # already insures the leader is running self.leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName) ctx = AWSProvisioner._buildContext(self.clusterName) assert len(self.getMatchingRoles(self.clusterName)) == 1 # --never-download prevents silent upgrades to pip, wheel and setuptools venv_command = [ 'virtualenv', '--system-site-packages', '--never-download', '/home/venv' ] self.sshUtil(venv_command) upgrade_command = [ '/home/venv/bin/pip', 'install', 'setuptools==28.7.1' ] self.sshUtil(upgrade_command) yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12'] self.sshUtil(yaml_command) self._getScript() toilOptions = [ self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil', '--clean=always', '--retryCount=2', '--clusterStats=/home/', '--logDebug', '--logFile=/home/sort.log', '--provisioner=aws' ] if spotInstances: toilOptions.extend([ '--preemptableNodeType=%s:%s' % (self.instanceType, self.spotBid), # The RNASeq pipeline does not specify a preemptability requirement so we # need to specify a default, otherwise jobs would never get scheduled. '--defaultPreemptable', '--maxPreemptableNodes=%s' % self.numWorkers ]) else: toilOptions.extend([ '--nodeType=' + self.instanceType, '--maxNodes=%s' % self.numWorkers ]) self._runScript(toilOptions) assert len(self.getMatchingRoles(self.clusterName)) == 1 checkStatsCommand = [ '/home/venv/bin/python', '-c', 'import json; import os; ' 'json.load(open("/home/" + [f for f in os.listdir("/home/") ' 'if f.endswith(".json")].pop()))' ] self.sshUtil(checkStatsCommand) volumeID = self.getRootVolID() ctx = AWSProvisioner._buildContext(self.clusterName) AWSProvisioner.destroyCluster(self.clusterName) self.leader.update() for attempt in range(6): # https://github.com/BD2KGenomics/toil/issues/1567 # retry this for up to 1 minute until the volume disappears try: ctx.ec2.get_all_volumes(volume_ids=[volumeID]) time.sleep(10) except EC2ResponseError as e: if e.status == 400 and 'InvalidVolume.NotFound' in e.code: break else: raise else: self.fail('Volume with ID %s was not cleaned up properly' % volumeID) assert len(self.getMatchingRoles(self.clusterName)) == 0
def _test(self, preemptableJobs=False): """ Does the work of the testing. Many features' test are thrown in here is no particular order """ from toil.provisioners.aws.awsProvisioner import AWSProvisioner self.launchCluster() # get the leader so we know the IP address - we don't need to wait since create cluster # already insures the leader is running self.leader = AWSProvisioner._getLeader(wait=False, clusterName=self.clusterName) ctx = AWSProvisioner._buildContext(self.clusterName) assert len(self.getMatchingRoles(self.clusterName)) == 1 # --never-download prevents silent upgrades to pip, wheel and setuptools venv_command = [ 'virtualenv', '--system-site-packages', '--never-download', '/home/venv' ] self.sshUtil(venv_command) upgrade_command = [ '/home/venv/bin/pip', 'install', 'setuptools==28.7.1' ] self.sshUtil(upgrade_command) yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12'] self.sshUtil(yaml_command) self._getScript() toilOptions = [ self.jobStore, '--batchSystem=mesos', '--workDir=/var/lib/toil', '--clean=always', '--retryCount=2', '--clusterStats=/home/', '--logDebug', '--logFile=/home/sort.log', '--provisioner=aws' ] toilOptions.extend([ '--nodeTypes=' + ",".join(self.instanceTypes), '--maxNodes=%s' % ",".join(self.numWorkers) ]) if preemptableJobs: toilOptions.extend(['--defaultPreemptable']) self._runScript(toilOptions) assert len(self.getMatchingRoles(self.clusterName)) == 1 checkStatsCommand = [ '/home/venv/bin/python', '-c', 'import json; import os; ' 'json.load(open("/home/" + [f for f in os.listdir("/home/") ' 'if f.endswith(".json")].pop()))' ] self.sshUtil(checkStatsCommand) from boto.exception import EC2ResponseError volumeID = self.getRootVolID() ctx = AWSProvisioner._buildContext(self.clusterName) AWSProvisioner.destroyCluster(self.clusterName) self.leader.update() for attempt in range(6): # https://github.com/BD2KGenomics/toil/issues/1567 # retry this for up to 1 minute until the volume disappears try: ctx.ec2.get_all_volumes(volume_ids=[volumeID]) time.sleep(10) except EC2ResponseError as e: if e.status == 400 and 'InvalidVolume.NotFound' in e.code: break else: raise else: self.fail('Volume with ID %s was not cleaned up properly' % volumeID) assert len(self.getMatchingRoles(self.clusterName)) == 0
def tearDown(self): from toil.provisioners.aws.awsProvisioner import AWSProvisioner AWSProvisioner.destroyCluster(self.clusterName)