def testSetEnv(self): # Parasol disobeys shell rules and stupidly splits the command at the space character # before exec'ing it, whether the space is quoted, escaped or not. This means that we # can't have escaped or quotes spaces in the command line. So we can't use bash -c # '...' or python -c '...'. The safest thing to do here is to script the test and # invoke that script rather than inline the test via -c. def assertEnv(): import os, sys sys.exit(0 if os.getenv('FOO') == 'bar' else 42) script_body = dedent('\n'.join(getsource(assertEnv).split('\n')[1:])) with tempFileContaining(script_body, suffix='.py') as script_path: # First, ensure that the test fails if the variable is *not* set command = sys.executable + ' ' + script_path jobNode4 = JobNode(command=command, jobName='test4', unitName=None, jobStoreID='4', requirements=defaultRequirements) job4 = self.batchSystem.issueBatchJob(jobNode4) jobID, exitStatus, wallTime = self.batchSystem.getUpdatedBatchJob(maxWait=1000) self.assertEqual(exitStatus, 42) self.assertEqual(jobID, job4) # Now set the variable and ensure that it is present self.batchSystem.setEnv('FOO', 'bar') jobNode5 = JobNode(command=command, jobName='test5', unitName=None, jobStoreID='5', requirements=defaultRequirements) job5 = self.batchSystem.issueBatchJob(jobNode5) jobID, exitStatus, wallTime = self.batchSystem.getUpdatedBatchJob(maxWait=1000) self.assertEqual(exitStatus, 0) self.assertEqual(jobID, job5)
def testSetEnv(self): # Parasol disobeys shell rules and stupidly splits the command at the space character # before exec'ing it, whether the space is quoted, escaped or not. This means that we # can't have escaped or quotes spaces in the command line. So we can't use bash -c # '...' or python -c '...'. The safest thing to do here is to script the test and # invoke that script rather than inline the test via -c. def assertEnv(): import os, sys sys.exit(0 if os.getenv('FOO') == 'bar' else 42) script_body = dedent('\n'.join(getsource(assertEnv).split('\n')[1:])) with tempFileContaining(script_body, suffix='.py') as script_path: # First, ensure that the test fails if the variable is *not* set command = sys.executable + ' ' + script_path jobNode4 = JobNode(command=command, jobName='test4', unitName=None, jobStoreID=None, requirements=defaultRequirements) job4 = self.batchSystem.issueBatchJob(jobNode4) jobID, exitStatus, wallTime = self.batchSystem.getUpdatedBatchJob(maxWait=1000) self.assertEqual(exitStatus, 42) self.assertEqual(jobID, job4) # Now set the variable and ensure that it is present self.batchSystem.setEnv('FOO', 'bar') jobNode5 = JobNode(command=command, jobName='test5', unitName=None, jobStoreID=None, requirements=defaultRequirements) job5 = self.batchSystem.issueBatchJob(jobNode5) jobID, exitStatus, wallTime = self.batchSystem.getUpdatedBatchJob(maxWait=1000) self.assertEqual(exitStatus, 0) self.assertEqual(jobID, job5)
def testNonPyStandAlone(self): """ Asserts that Toil enforces the user script to have a .py or .pyc extension because that's the only way auto-deployment can re-import the module on a worker. See https://github.com/BD2KGenomics/toil/issues/631 and https://github.com/BD2KGenomics/toil/issues/858 """ def script(): import argparse from toil.job import Job from toil.common import Toil def fn(): pass if __name__ == '__main__': parser = argparse.ArgumentParser() Job.Runner.addToilOptions(parser) options = parser.parse_args() job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M') with Toil(options) as toil: toil.start(job) scriptBody = dedent('\n'.join(getsource(script).split('\n')[1:])) shebang = '#! %s\n' % sys.executable with tempFileContaining(shebang + scriptBody) as scriptPath: self.assertFalse(scriptPath.endswith(('.py', '.pyc'))) os.chmod(scriptPath, 0o755) jobStorePath = scriptPath + '.jobStore' process = subprocess.Popen([scriptPath, jobStorePath], stderr=subprocess.PIPE) stdout, stderr = process.communicate() self.assertTrue( 'The name of a user script/module must end in .py or .pyc.' in stderr.decode('utf-8')) self.assertNotEqual(0, process.returncode) self.assertFalse(os.path.exists(jobStorePath))
def testNonPyStandAlone(self): """ Asserts that Toil enforces the user script to have a .py or .pyc extension because that's the only way hot deployment can re-import the module on a worker. See https://github.com/BD2KGenomics/toil/issues/631 and https://github.com/BD2KGenomics/toil/issues/858 """ def script(): import argparse from toil.job import Job from toil.common import Toil def fn(): pass if __name__ == '__main__': parser = argparse.ArgumentParser() Job.Runner.addToilOptions(parser) options = parser.parse_args() job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M') with Toil(options) as toil: toil.start(job) scriptBody = dedent('\n'.join(getsource(script).split('\n')[1:])) shebang = '#! %s\n' % sys.executable with tempFileContaining(shebang + scriptBody) as scriptPath: self.assertFalse(scriptPath.endswith(('.py', '.pyc'))) os.chmod(scriptPath, 0755) jobStorePath = scriptPath + '.jobStore' process = Popen([scriptPath, jobStorePath], stderr=PIPE) stdout, stderr = process.communicate() self.assertTrue('The name of a user script/module must end in .py or .pyc.' in stderr) self.assertNotEquals(0, process.returncode) self.assertFalse(os.path.exists(jobStorePath))