def testSetEnv(self):
# Parasol disobeys shell rules and stupidly splits the command at the space character
# before exec'ing it, whether the space is quoted, escaped or not. This means that we
# can't have escaped or quotes spaces in the command line. So we can't use bash -c
# '...' or python -c '...'. The safest thing to do here is to script the test and
# invoke that script rather than inline the test via -c.
def assertEnv():
import os, sys
sys.exit(0 if os.getenv('FOO') == 'bar' else 42)
script_body = dedent('\n'.join(getsource(assertEnv).split('\n')[1:]))
with tempFileContaining(script_body, suffix='.py') as script_path:
# First, ensure that the test fails if the variable is *not* set
command = sys.executable + ' ' + script_path
jobNode4 = JobNode(command=command, jobName='test4', unitName=None,
jobStoreID='4', requirements=defaultRequirements)
job4 = self.batchSystem.issueBatchJob(jobNode4)
jobID, exitStatus, wallTime = self.batchSystem.getUpdatedBatchJob(maxWait=1000)
self.assertEqual(exitStatus, 42)
self.assertEqual(jobID, job4)
# Now set the variable and ensure that it is present
self.batchSystem.setEnv('FOO', 'bar')
jobNode5 = JobNode(command=command, jobName='test5', unitName=None,
jobStoreID='5', requirements=defaultRequirements)
job5 = self.batchSystem.issueBatchJob(jobNode5)
jobID, exitStatus, wallTime = self.batchSystem.getUpdatedBatchJob(maxWait=1000)
self.assertEqual(exitStatus, 0)
self.assertEqual(jobID, job5)
def testSetEnv(self):
# Parasol disobeys shell rules and stupidly splits the command at the space character
# before exec'ing it, whether the space is quoted, escaped or not. This means that we
# can't have escaped or quotes spaces in the command line. So we can't use bash -c
# '...' or python -c '...'. The safest thing to do here is to script the test and
# invoke that script rather than inline the test via -c.
def assertEnv():
import os, sys
sys.exit(0 if os.getenv('FOO') == 'bar' else 42)
script_body = dedent('\n'.join(getsource(assertEnv).split('\n')[1:]))
with tempFileContaining(script_body, suffix='.py') as script_path:
# First, ensure that the test fails if the variable is *not* set
command = sys.executable + ' ' + script_path
jobNode4 = JobNode(command=command, jobName='test4', unitName=None,
jobStoreID=None, requirements=defaultRequirements)
job4 = self.batchSystem.issueBatchJob(jobNode4)
jobID, exitStatus, wallTime = self.batchSystem.getUpdatedBatchJob(maxWait=1000)
self.assertEqual(exitStatus, 42)
self.assertEqual(jobID, job4)
# Now set the variable and ensure that it is present
self.batchSystem.setEnv('FOO', 'bar')
jobNode5 = JobNode(command=command, jobName='test5', unitName=None,
jobStoreID=None, requirements=defaultRequirements)
job5 = self.batchSystem.issueBatchJob(jobNode5)
jobID, exitStatus, wallTime = self.batchSystem.getUpdatedBatchJob(maxWait=1000)
self.assertEqual(exitStatus, 0)
self.assertEqual(jobID, job5)
def testNonPyStandAlone(self):
"""
Asserts that Toil enforces the user script to have a .py or .pyc extension because that's
the only way auto-deployment can re-import the module on a worker. See
https://github.com/BD2KGenomics/toil/issues/631 and
https://github.com/BD2KGenomics/toil/issues/858
"""
def script():
import argparse
from toil.job import Job
from toil.common import Toil
def fn():
pass
if __name__ == '__main__':
parser = argparse.ArgumentParser()
Job.Runner.addToilOptions(parser)
options = parser.parse_args()
job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M')
with Toil(options) as toil:
toil.start(job)
scriptBody = dedent('\n'.join(getsource(script).split('\n')[1:]))
shebang = '#! %s\n' % sys.executable
with tempFileContaining(shebang + scriptBody) as scriptPath:
self.assertFalse(scriptPath.endswith(('.py', '.pyc')))
os.chmod(scriptPath, 0o755)
jobStorePath = scriptPath + '.jobStore'
process = subprocess.Popen([scriptPath, jobStorePath],
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
self.assertTrue(
'The name of a user script/module must end in .py or .pyc.' in
stderr.decode('utf-8'))
self.assertNotEqual(0, process.returncode)
self.assertFalse(os.path.exists(jobStorePath))
def testNonPyStandAlone(self):
"""
Asserts that Toil enforces the user script to have a .py or .pyc extension because that's
the only way hot deployment can re-import the module on a worker. See
https://github.com/BD2KGenomics/toil/issues/631 and
https://github.com/BD2KGenomics/toil/issues/858
"""
def script():
import argparse
from toil.job import Job
from toil.common import Toil
def fn():
pass
if __name__ == '__main__':
parser = argparse.ArgumentParser()
Job.Runner.addToilOptions(parser)
options = parser.parse_args()
job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M')
with Toil(options) as toil:
toil.start(job)
scriptBody = dedent('\n'.join(getsource(script).split('\n')[1:]))
shebang = '#! %s\n' % sys.executable
with tempFileContaining(shebang + scriptBody) as scriptPath:
self.assertFalse(scriptPath.endswith(('.py', '.pyc')))
os.chmod(scriptPath, 0755)
jobStorePath = scriptPath + '.jobStore'
process = Popen([scriptPath, jobStorePath], stderr=PIPE)
stdout, stderr = process.communicate()
self.assertTrue('The name of a user script/module must end in .py or .pyc.' in stderr)
self.assertNotEquals(0, process.returncode)
self.assertFalse(os.path.exists(jobStorePath))
