def createClusterUtil(self, args=None):
args = [] if args is None else args
subprocess.check_call([
'toil', 'launch-cluster', '-p=aws', '-z=us-west-2a',
'--keyPairName=%s' %
self.keyName, '--leaderNodeType=t2.medium', self.clusterName
] + args)
def createClusterUtil(self, args=None):
if args is None:
args = []
callCommand = ['toil', 'launch-cluster', '-p=aws', '-z=us-west-2a','--keyPairName=%s' % self.keyName,
'--leaderNodeType=%s' % self.leaderInstanceType, self.clusterName]
callCommand = callCommand + args if args else callCommand
subprocess.check_call(callCommand)
def testJobStoreContents(self):
"""Test toilDebugFile.printContentsOfJobStore().
Runs a workflow that imports 'B.txt' and 'mkFile.py' into the
jobStore. 'A.txt', 'C.txt', 'ABC.txt' are then created. This checks to
make sure these contents are found in the jobStore and printed."""
contents = ['A.txt', 'B.txt', 'C.txt', 'ABC.txt', 'mkFile.py']
subprocess.check_call([
'python',
os.path.abspath('src/toil/utils/toilDebugFile.py'),
self.jobStoreDir, '--listFilesInJobStore=True'
])
jobstoreFileContents = os.path.abspath('jobstore_files.txt')
files = []
match = 0
with open(jobstoreFileContents, 'r') as f:
for line in f:
files.append(line.strip())
for xfile in files:
for expected_file in contents:
if xfile.endswith(expected_file):
match = match + 1
logger.debug(files)
logger.debug(contents)
logger.debug(match)
# C.txt will match twice (once with 'C.txt', and once with 'ABC.txt')
assert match == 6
os.remove(jobstoreFileContents)
def testWriteGzipLogs(self):
subprocess.check_call([
sys.executable, '-m', helloWorld.__name__, './toilTest',
'--clean=always', '--logLevel=debug',
'--writeLogsGzip=%s' % self.tempDir
])
self._assertFileTypeExists(self.tempDir, '.log.gz', 'gzip')
def _fixPermissions(tool, workDir):
"""
Deprecated.
Fix permission of a mounted Docker directory by reusing the tool to change
ownership. Docker natively runs as a root inside the container, and files
written to the mounted directory are implicitly owned by root.
:param list baseDockerCall: Docker run parameters
:param str tool: Name of tool
:param str workDir: Path of work directory to recursively chown
"""
if os.geteuid() == 0:
# we're running as root so this chown is redundant
return
baseDockerCall = [
'docker', 'run', '--log-driver=none', '-v',
os.path.abspath(workDir) + ':/data', '--rm', '--entrypoint=chown'
]
stat = os.stat(workDir)
command = baseDockerCall + [tool] + [
'-R', '{}:{}'.format(stat.st_uid, stat.st_gid), '/data'
]
for attempt in retry(predicate=dockerPredicate):
with attempt:
subprocess.check_call(command)
def fetchFiles(self, symLink):
"""
Fn for testFetchJobStoreFiles() and testFetchJobStoreFilesWSymlinks().
Runs a workflow that imports 'B.txt' and 'mkFile.py' into the
jobStore. 'A.txt', 'C.txt', 'ABC.txt' are then created. This test then
attempts to get a list of these files and copy them over into ./src from
the jobStore, confirm that they are present, and then delete them.
"""
contents = ['A.txt', 'B.txt', 'C.txt', 'ABC.txt', 'mkFile.py']
outputDir = os.path.abspath('src')
cmd = [
'python',
os.path.abspath('src/toil/utils/toilDebugFile.py'),
self.jobStoreDir, '--fetch', '*A.txt', '*B.txt', '*C.txt',
'*ABC.txt', '*mkFile.py',
'--localFilePath=' + os.path.abspath('src'),
'--useSymlinks=' + str(symLink)
]
subprocess.check_call(cmd)
for xfile in contents:
matchingFilesFound = recursiveGlob(outputDir, '*' + xfile)
self.assertGreaterEqual(len(matchingFilesFound), 1)
for fileFound in matchingFilesFound:
assert fileFound.endswith(xfile) and os.path.exists(fileFound)
if fileFound.endswith('-' + xfile):
os.remove(fileFound)
def setUp(self):
"""Initial set up of variables for the test."""
subprocess.check_call([
'python',
os.path.abspath(
'src/toil/test/utils/ABCWorkflowDebug/debugWorkflow.py')
])
self.jobStoreDir = os.path.abspath('toilWorkflowRun')
self.tempDir = self._createTempDir(purpose='tempDir')
def fetch_and_unzip_from_s3(self, filename, data, data_dir):
if not os.path.exists(data):
s3_loc = os.path.join('http://toil-datasets.s3.amazonaws.com/', filename)
fetch_from_s3_cmd = ["wget", "-P", self.test_directory, s3_loc]
subprocess.check_call(fetch_from_s3_cmd)
# extract the compressed data if not already extracted
if not os.path.exists(data_dir):
with zipfile.ZipFile(data, 'r') as zip_ref:
zip_ref.extractall(self.test_directory)
def createClusterUtil(self, args=None):
if args is None:
args = []
callCommand = ['toil', 'launch-cluster', self.clusterName, '-p=gce', '--keyPairName=%s' % self.keyName,
'--leaderNodeType=%s' % self.leaderInstanceType, '--zone=%s' % self.googleZone]
if self.botoDir is not None:
callCommand += ['--boto=%s' % self.botoDir]
callCommand = callCommand + args if args else callCommand
log.info("createClusterUtil: %s" % ''.join(callCommand))
subprocess.check_call(callCommand)
def system(command):
"""
A convenience wrapper around subprocess.check_call that logs the command before passing it
on. The command can be either a string or a sequence of strings. If it is a string shell=True
will be passed to subprocess.check_call.
:type command: str | sequence[string]
"""
logger.debug('Running: %r', command)
subprocess.check_call(command, shell=isinstance(command, string_types), bufsize=-1)
def testTut04(self):
'''Test if toilwdl produces the same outputs as known good outputs for WDL's
GATK tutorial #4.'''
if self.manual_integration_tests:
wdl = os.path.abspath("src/toil/test/wdl/wdl_templates/t04/jointCallingGenotypes.wdl")
json = os.path.abspath("src/toil/test/wdl/wdl_templates/t04/jointCallingGenotypes_inputs.json")
ref_dir = os.path.abspath("src/toil/test/wdl/wdl_templates/t04/output/")
subprocess.check_call(['python', self.program, wdl, json, '-o', self.output_dir])
compare_runs(self.output_dir, ref_dir)
def testMD5sum(self):
'''Test if toilwdl produces the same outputs as known good outputs for WDL's
GATK tutorial #1.'''
wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.wdl')
inputfile = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.input')
json = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.json')
subprocess.check_call(['python', self.program, wdl, json, '-o', self.output_dir])
md5sum_output = os.path.join(self.output_dir, 'md5sum.txt')
assert os.path.exists(md5sum_output)
os.unlink(md5sum_output)
def testPipe(self):
'''Test basic bash input functionality with a pipe.'''
wdl = os.path.abspath(
"src/toil/test/wdl/wdl_templates/testPipe/call.wdl")
json = os.path.abspath(
"src/toil/test/wdl/wdl_templates/testPipe/call.json")
ref_dir = os.path.abspath(
"src/toil/test/wdl/wdl_templates/testPipe/output/")
subprocess.check_call(
['python', self.program, wdl, json, '--out_dir', self.output_dir])
compare_runs(self.output_dir, ref_dir)
def testENCODE(self):
'''Test if toilwdl produces the same outputs as known good outputs for
a short ENCODE run.'''
wdl = os.path.abspath(
"src/toil/test/wdl/wdl_templates/testENCODE/encode_mapping_workflow.wdl")
json = os.path.abspath(
"src/toil/test/wdl/wdl_templates/testENCODE/encode_mapping_workflow.wdl.json")
ref_dir = os.path.abspath(
"src/toil/test/wdl/wdl_templates/testENCODE/output/")
subprocess.check_call(
['python', self.program, wdl, json, '--docker_user=None', '--out_dir', self.output_dir])
compare_runs(self.output_dir, ref_dir)
def _dockerKill(containerName, action):
"""
Deprecated. Kills the specified container.
:param str containerName: The name of the container created by docker_call
:param int action: What action should be taken on the container?
"""
running = containerIsRunning(containerName)
if running is None:
# This means that the container doesn't exist. We will see this if the
# container was run with --rm and has already exited before this call.
logger.debug(
'The container with name "%s" appears to have already been '
'removed. Nothing to '
'do.', containerName)
else:
if action in (None, FORGO):
logger.debug(
'The container with name %s continues to exist as we '
'were asked to forgo a '
'post-job action on it.', containerName)
else:
logger.debug(
'The container with name %s exists. Running '
'user-specified defer functions.', containerName)
if running and action >= STOP:
logger.debug('Stopping container "%s".', containerName)
for attempt in retry(predicate=dockerPredicate):
with attempt:
subprocess.check_call(
['docker', 'stop', containerName])
else:
logger.debug('The container "%s" was not found to be running.',
containerName)
if action >= RM:
# If the container was run with --rm, then stop will most likely
# remove the container. We first check if it is running then
# remove it.
running = containerIsRunning(containerName)
if running is not None:
logger.debug('Removing container "%s".', containerName)
for attempt in retry(predicate=dockerPredicate):
with attempt:
subprocess.check_call(
['docker', 'rm', '-f', containerName])
else:
logger.debug(
'Container "%s" was not found on the system.'
'Nothing to remove.', containerName)
def test_bsubline_works():
command = lsf.build_bsub_line(cpu=1,
mem=2147483648,
runtime=1,
jobname="Test Job")
command.extend(["-K", "echo"])
assert subprocess.check_call(command) == 0
def _setSSH(self):
if not os.path.exists('/root/.sshSuccess'):
subprocess.check_call([
'ssh-keygen', '-f', '/root/.ssh/id_rsa', '-t', 'rsa', '-N', ''
])
with open('/root/.sshSuccess', 'w') as f:
f.write(
'written here because of restrictive permissions on .ssh dir'
)
os.chmod('/root/.ssh', 0o700)
subprocess.check_call(
['bash', '-c', 'eval $(ssh-agent) && ssh-add -k'])
with open('/root/.ssh/id_rsa.pub') as f:
masterPublicKey = f.read()
masterPublicKey = masterPublicKey.split(' ')[1] # take 'body' of key
# confirm it really is an RSA public key
assert masterPublicKey.startswith('AAAAB3NzaC1yc2E'), masterPublicKey
return masterPublicKey
def _setSSH(self):
"""
Generate a key pair, save it in /root/.ssh/id_rsa.pub, and return the public key.
The file /root/.sshSuccess is used to prevent this operation from running twice.
:return Public key.
"""
if not os.path.exists('/root/.sshSuccess'):
subprocess.check_call([
'ssh-keygen', '-f', '/root/.ssh/id_rsa', '-t', 'rsa', '-N', ''
])
with open('/root/.sshSuccess', 'w') as f:
f.write(
'written here because of restrictive permissions on .ssh dir'
)
os.chmod('/root/.ssh', 0o700)
subprocess.check_call(
['bash', '-c', 'eval $(ssh-agent) && ssh-add -k'])
with open('/root/.ssh/id_rsa.pub') as f:
masterPublicKey = f.read()
masterPublicKey = masterPublicKey.split(' ')[1] # take 'body' of key
# confirm it really is an RSA public key
assert masterPublicKey.startswith('AAAAB3NzaC1yc2E'), masterPublicKey
return masterPublicKey
def _testExternal(self, moduleName, pyFiles, virtualenv=False):
dirPath = self._createTempDir()
if virtualenv:
self.assertTrue(inVirtualEnv())
# --never-download prevents silent upgrades to pip, wheel and setuptools
subprocess.check_call(['virtualenv', '--never-download', dirPath])
sitePackages = os.path.join(dirPath, 'lib', 'python2.7',
'site-packages')
# tuple assignment is necessary to make this line immediately precede the try:
oldPrefix, sys.prefix, dirPath = sys.prefix, dirPath, sitePackages
else:
oldPrefix = None
try:
for relPath in pyFiles:
path = os.path.join(dirPath, relPath)
mkdir_p(os.path.dirname(path))
with open(path, 'w') as f:
f.write('pass\n')
sys.path.append(dirPath)
try:
userScript = importlib.import_module(moduleName)
try:
self._test(userScript.__name__,
expectedContents=pyFiles,
allowExtraContents=True)
finally:
del userScript
while moduleName:
del sys.modules[moduleName]
self.assertFalse(moduleName in sys.modules)
moduleName = '.'.join(moduleName.split('.')[:-1])
finally:
sys.path.remove(dirPath)
finally:
if oldPrefix:
sys.prefix = oldPrefix
def _coreRsync(cls, ip, args, applianceName='toil_leader', **kwargs):
remoteRsync = "docker exec -i %s rsync" % applianceName # Access rsync inside appliance
parsedArgs = []
sshCommand = "ssh"
strict = kwargs.pop('strict', False)
if not strict:
sshCommand = "ssh -oUserKnownHostsFile=/dev/null -oStrictHostKeyChecking=no"
hostInserted = False
# Insert remote host address
for i in args:
if i.startswith(":") and not hostInserted:
i = ("core@%s" % ip) + i
hostInserted = True
elif i.startswith(":") and hostInserted:
raise ValueError("Cannot rsync between two remote hosts")
parsedArgs.append(i)
if not hostInserted:
raise ValueError("No remote host found in argument list")
command = ['rsync', '-e', sshCommand, '--rsync-path', remoteRsync]
log.debug("Running %r.", command + parsedArgs)
return subprocess.check_call(command + parsedArgs)
def sshUtil(self, command):
baseCommand = ['toil', 'ssh-cluster', '--insecure', '-p=gce', self.clusterName]
callCommand = baseCommand + command
subprocess.check_call(callCommand)
def killJob(self, jobID):
subprocess.check_call(['bkill', self.getBatchSystemID(jobID)])
def _testFromMain(self):
testMethodName = self.id().split('.')[-1]
self.assertTrue(testMethodName.endswith('FromMain'))
subprocess.check_call(
[sys.executable, '-m', self.__module__, testMethodName[:-8]])
def greatGrandChild(cmd):
subprocess.check_call(cmd, shell=True)
def stop(self, fileStore):
subprocess.check_call(self.cmd + ' -1', shell=True)
def grandChildJob(job, cmd):
job.addService(Service(cmd))
job.addChildFn(greatGrandChild, cmd)
subprocess.check_call(cmd, shell=True)
def printUnicodeCharacter():
# We want to get a unicode character to stdout but we can't print it directly because of
# Python encoding issues. To work around this we print in a separate Python process. See
# http://stackoverflow.com/questions/492483/setting-the-correct-encoding-when-piping-stdout-in-python
subprocess.check_call([sys.executable, '-c', "print('\\xc3\\xbc')"])
def rsyncUtil(self, src, dest):
baseCommand = ['toil', 'rsync-cluster', '--insecure', '-p=gce', self.clusterName]
callCommand = baseCommand + [src, dest]
subprocess.check_call(callCommand)
def destroyClusterUtil(self):
callCommand = ['toil', 'destroy-cluster', '-p=gce', self.clusterName]
subprocess.check_call(callCommand)
def cleanJobStoreUtil(self):
callCommand = ['toil', 'clean', self.jobStore]
subprocess.check_call(callCommand)
