def is_singularity_available(raise_error=False, path=False):
"""
Check if singularity is available to run in the current environment.
Arguments:
raise_error (bool): flag to raise error when command is unavailable.
path (bool): flag to return location of the command in the user's path.
Returns:
bool: True if singularity is available.
str: absolute location of the file in the user's path.
Raises:
OSError: if the raise_error flag was passed as an argument and the
command is not available to execute.
"""
try:
subprocess.check_output(["singularity", "--version"])
if path:
return which("singularity")
return True
except (subprocess.CalledProcessError, OSError) as error:
if raise_error:
raise exceptions.SingularityNotAvailableError(str(error))
return False
def test_run_conformance(self, batchSystem=None):
rootDir = self._projectRootPath()
cwlSpec = os.path.join(rootDir, 'src/toil/test/cwl/spec')
workDir = os.path.join(cwlSpec, 'v1.0')
# The latest cwl git hash. Update it to get the latest tests.
testhash = "22490926651174c6cbe01c76c2ded3c9e8d0ee6f"
url = "https://github.com/common-workflow-language/common-workflow-language/archive/%s.zip" % testhash
if not os.path.exists(cwlSpec):
urlretrieve(url, "spec.zip")
with zipfile.ZipFile('spec.zip', "r") as z:
z.extractall()
shutil.move("common-workflow-language-%s" % testhash, cwlSpec)
os.remove("spec.zip")
try:
cmd = ['cwltest', '--tool', 'toil-cwl-runner', '--test=conformance_test_v1.0.yaml',
'--timeout=1800', '--basedir=' + workDir]
if batchSystem:
cmd.extend(["--batchSystem", batchSystem])
subprocess.check_output(cmd, cwd=workDir, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
only_unsupported = False
# check output -- if we failed but only have unsupported features, we're okay
p = re.compile(r"(?P<failures>\d+) failures, (?P<unsupported>\d+) unsupported features")
for line in e.output.split("\n"):
m = p.search(line)
if m:
if int(m.group("failures")) == 0 and int(m.group("unsupported")) > 0:
only_unsupported = True
break
if not only_unsupported:
print(e.output)
raise e
def test_run_conformance(self, batchSystem=None):
rootDir = self._projectRootPath()
cwlSpec = os.path.join(rootDir, 'src/toil/test/cwl/spec')
testhash = "91f108df4d4ca567e567fc65f61feb0674467a84"
url = "https://github.com/common-workflow-language/common-workflow-language/archive/%s.zip" % testhash
if not os.path.exists(cwlSpec):
urlretrieve(url, "spec.zip")
with zipfile.ZipFile('spec.zip', "r") as z:
z.extractall()
shutil.move("common-workflow-language-%s" % testhash, cwlSpec)
os.remove("spec.zip")
try:
cmd = [
"bash", "run_test.sh", "RUNNER=toil-cwl-runner", "DRAFT=v1.0",
"-j4"
]
if batchSystem:
cmd.extend(["--batchSystem", batchSystem])
subprocess.check_output(cmd, cwd=cwlSpec, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
only_unsupported = False
# check output -- if we failed but only have unsupported features, we're okay
p = re.compile(
r"(?P<failures>\d+) failures, (?P<unsupported>\d+) unsupported features"
)
for line in e.output.split("\n"):
m = p.search(line)
if m:
if int(m.group("failures")) == 0 and int(
m.group("unsupported")) > 0:
only_unsupported = True
break
if not only_unsupported:
print(e.output)
raise e
def test_run_conformance(self, batchSystem=None):
try:
cmd = [
'cwltest', '--tool', 'toil-cwl-runner',
'--test=conformance_test_v1.0.yaml', '--timeout=2400',
'--basedir=' + self.workDir
]
if batchSystem:
cmd.extend(["--batchSystem", batchSystem])
subprocess.check_output(cmd,
cwd=self.workDir,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
only_unsupported = False
# check output -- if we failed but only have unsupported features, we're okay
p = re.compile(
r"(?P<failures>\d+) failures, (?P<unsupported>\d+) unsupported features"
)
error_log = e.output
if isinstance(e.output, bytes):
# py2/3 string handling
error_log = e.output.decode('utf-8')
for line in error_log.split('\n'):
m = p.search(line)
if m:
if int(m.group("failures")) == 0 and int(
m.group("unsupported")) > 0:
only_unsupported = True
break
if not only_unsupported:
print(error_log)
raise e
def testMultipleLogToMaster(self):
toilOutput = subprocess.check_output([
sys.executable, '-m', helloWorld.__name__, './toilTest',
'--clean=always', '--logLevel=info'
],
stderr=subprocess.STDOUT)
assert helloWorld.parentMessage in toilOutput.decode('utf-8')
def testLogToMaster(self):
toilOutput = subprocess.check_output([
sys.executable, '-m', helloWorld.__name__, './toilTest',
'--clean=always', '--logLevel=info'
],
stderr=subprocess.STDOUT)
assert helloWorld.childMessage in toilOutput
def obtainSystemConstants(cls):
def byteStrip(s):
return s.encode('utf-8').strip()
lines = [_f for _f in map(byteStrip, subprocess.check_output(["qhost"]).decode('utf-8').split('\n')) if _f]
line = lines[0]
items = line.strip().split()
num_columns = len(items)
cpu_index = None
mem_index = None
for i in range(num_columns):
if items[i] == 'NCPU':
cpu_index = i
elif items[i] == 'MEMTOT':
mem_index = i
if cpu_index is None or mem_index is None:
RuntimeError('qhost command does not return NCPU or MEMTOT columns')
maxCPU = 0
maxMEM = MemoryString("0")
for line in lines[2:]:
items = line.strip().split()
if len(items) < num_columns:
RuntimeError('qhost output has a varying number of columns')
if items[cpu_index] != '-' and items[cpu_index] > maxCPU:
maxCPU = items[cpu_index]
if items[mem_index] != '-' and MemoryString(items[mem_index]) > maxMEM:
maxMEM = MemoryString(items[mem_index])
if maxCPU is 0 or maxMEM is 0:
RuntimeError('qhost returned null NCPU or MEMTOT info')
return maxCPU, maxMEM
def testRegularLog(self):
toilOutput = subprocess.check_output([
sys.executable, '-m', helloWorld.__name__, './toilTest',
'--clean=always', '--batchSystem=singleMachine', '--logLevel=debug'
],
stderr=subprocess.STDOUT)
assert "single machine batch system" in toilOutput.decode('utf-8')
def needs_appliance(test_item):
import json
test_item = _mark_test('appliance', test_item)
if less_strict_bool(os.getenv('TOIL_SKIP_DOCKER')):
return unittest.skip('Skipping docker test.')(test_item)
if next(which('docker'), None):
image = applianceSelf()
try:
images = subprocess.check_output(['docker', 'inspect', image])
except subprocess.CalledProcessError:
images = []
else:
images = {
i['Id']
for i in json.loads(images) if image in i['RepoTags']
}
if len(images) == 0:
return unittest.skip(
"Cannot find appliance image %s. Use 'make test' target to "
"automatically build appliance, or just run 'make docker' "
"prior to running this test." % image)(test_item)
elif len(images) == 1:
return test_item
else:
assert False, 'Expected `docker inspect` to return zero or one image.'
else:
return unittest.skip('Install Docker to include this test.')(test_item)
def testWriteGzipLogs(self):
toilOutput = subprocess.check_output([
sys.executable, '-m', helloWorld.__name__, './toilTest',
'--clean=always', '--logLevel=debug',
'--writeLogsGzip=%s' % self.tempDir
],
stderr=subprocess.STDOUT)
self._assertFileTypeExists(self.tempDir, '.log.gz', 'gzip')
def _getNotFinishedIDs():
return {
int(i)
for i in subprocess.check_output(["bjobs", "-o", "id"])
.decode("utf-8")
.strip()
.split("\n")[1:]
}
def apply_lsadmin(fn):
"""
apply fn to each line of lsadmin, returning the result
"""
cmd = ["lsadmin", "showconf", "lim"]
try:
output = subprocess.check_output(cmd)
except:
return None
return fn(output.split("\n"))
def apply_bparams(fn):
"""
apply fn to each line of bparams, returning the result
"""
cmd = ["bparams", "-a"]
try:
output = subprocess.check_output(cmd)
except:
return None
return fn(output.split("\n"))
def submitJob(self, subLine):
try:
output = subprocess.check_output(
subLine, stderr=subprocess.STDOUT).decode('utf-8')
# sbatch prints a line like 'Submitted batch job 2954103'
result = int(output.strip().split()[-1])
logger.debug("sbatch submitted job %d", result)
return result
except OSError as e:
logger.error("sbatch command failed")
raise e
def process_single_outfile(f, fileStore, workDir, outDir):
if os.path.exists(f):
output_f_path = f
elif os.path.exists(os.path.abspath(f)):
output_f_path = os.path.abspath(f)
elif os.path.exists(os.path.join(workDir, 'execution', f)):
output_f_path = os.path.join(workDir, 'execution', f)
elif os.path.exists(os.path.join('execution', f)):
output_f_path = os.path.join('execution', f)
elif os.path.exists(os.path.join(workDir, f)):
output_f_path = os.path.join(workDir, f)
else:
tmp = subprocess.check_output(['ls', '-lha', workDir])
exe = subprocess.check_output(['ls', '-lha', os.path.join(workDir, 'execution')])
raise RuntimeError('OUTPUT FILE: {} was not found!\n'
'{}\n\n'
'{}\n'.format(f, tmp, exe))
output_file = fileStore.writeGlobalFile(output_f_path)
preserveThisFilename = os.path.basename(output_f_path)
fileStore.exportFile(output_file, "file://" + os.path.join(os.path.abspath(outDir), preserveThisFilename))
return (output_file, preserveThisFilename)
def size(f, unit='B', d=None):
"""
Returns the size of a file in bytes.
:param f: Filename
:param d: The directory containing the file to be sized.
:param unit: Return the byte size in these units (gigabytes, etc.).
:return:
"""
if isinstance(f, tuple) and d:
f = os.path.join(d, f[0])
divisor = return_bytes(unit)
return (float(subprocess.check_output(['du', '-s', f],
env=dict(os.environ, BLOCKSIZE='512')).split()[0].decode('ascii')) * 512) / divisor
def needs_rsync3(test_item):
"""
Use as a decorator before test classes or methods that depend on any features used in rsync
version 3.0.0+
Necessary because :meth:`utilsTest.testAWSProvisionerUtils` uses option `--protect-args` which is only
available in rsync 3
"""
test_item = _mark_test('rsync', test_item)
try:
versionInfo = subprocess.check_output(['rsync', '--version']).decode('utf-8')
if int(versionInfo.split()[2].split('.')[0]) < 3: # output looks like: 'rsync version 2.6.9 ...'
return unittest.skip('This test depends on rsync version 3.0.0+.')(test_item)
except subprocess.CalledProcessError:
return unittest.skip('rsync needs to be installed to run this test.')(test_item)
return test_item
def _pbsVersion(self):
""" Determines PBS/Torque version via pbsnodes
"""
try:
out = subprocess.check_output(["pbsnodes", "--version"])
if "PBSPro" in out:
logger.debug("PBS Pro proprietary Torque version detected")
self._version = "pro"
else:
logger.debug("Torque OSS version detected")
self._version = "oss"
except subprocess.CalledProcessError as e:
if e.returncode != 0:
logger.error("Could not determine PBS/Torque version")
return self._version
def _processStatusCommandLSF(self, command, jobID):
output = subprocess.check_output(command).decode("utf-8")
cmdstr = " ".join(command)
if "Done successfully" in output:
logger.debug("Detected completed job: %s", cmdstr)
status = 0
elif "Completed <done>" in output:
logger.debug("Detected completed job: %s", cmdstr)
status = 0
elif "TERM_MEMLIMIT" in output:
status = self._customRetry(jobID, term_memlimit=True)
elif "TERM_RUNLIMIT" in output:
status = self._customRetry(jobID, term_runlimit=True)
elif "New job is waiting for scheduling" in output:
logger.debug("Detected job pending scheduling: %s", cmdstr)
status = None
elif "PENDING REASONS" in output:
logger.debug("Detected pending job: %s", cmdstr)
status = None
elif "Started on " in output:
logger.debug("Detected job started but not completed: %s", cmdstr)
status = None
elif "Completed <exit>" in output:
logger.error("Detected failed job: %s", cmdstr)
status = 1
elif "Exited with exit code" in output:
logger.error("Detected failed job: %s", cmdstr)
status = 1
else:
status = self._CANT_DETERMINE_JOB_STATUS
return status
def obtainSystemConstants(cls):
# sinfo -Ne --format '%m,%c'
# sinfo arguments:
# -N for node-oriented
# -h for no header
# -e for exact values (e.g. don't return 32+)
# --format to get memory, cpu
max_cpu = 0
max_mem = MemoryString('0')
lines = subprocess.check_output(['sinfo', '-Nhe', '--format', '%m %c']).split('\n')
for line in lines:
values = line.split()
if len(values) < 2:
continue
mem, cpu = values
max_cpu = max(max_cpu, int(cpu))
max_mem = max(max_mem, MemoryString(mem + 'M'))
if max_cpu == 0 or max_mem.byteVal() == 0:
RuntimeError('sinfo did not return memory or cpu info')
return max_cpu, max_mem
def getRunningJobIDs(self):
# Should return a dictionary of Job IDs and number of seconds
times = {}
with self.runningJobsLock:
currentjobs = dict((str(self.batchJobIDs[x][0]), x) for x in self.runningJobs)
# currentjobs is a dictionary that maps a slurm job id (string) to our own internal job id
# squeue arguments:
# -h for no header
# --format to get jobid i, state %t and time days-hours:minutes:seconds
lines = subprocess.check_output(['squeue', '-h', '--format', '%i %t %M']).split('\n')
for line in lines:
values = line.split()
if len(values) < 3:
continue
slurm_jobid, state, elapsed_time = values
if slurm_jobid in currentjobs and state == 'R':
seconds_running = self.parse_elapsed(elapsed_time)
times[currentjobs[slurm_jobid]] = seconds_running
return times
def singularity_call(
image,
args=None,
cwd=None,
env=None,
check_output=None,
working_dir=None,
volumes=None,
remove_tmp_dir=True,
):
"""
Execute parameters in a singularity container via subprocess.
Singularity will be called with the following command:
singularity -q exec
--bind <shared_fs>:<shared_fs> # if shared_fs is provided
--contain --workdir <working_dir> # if working_dir is provided
--pwd {cwd} # if cwd is provided
<image> <args>
Docker images can be run by prefacing the input image with 'docker://'.
In this case, Singularity will download, convert, and cache the image
on the fly. This cache can be set with SINGULARITY_CACHEDIR, and
defaults to the user's home directory. This cache can be a major
bottleneck when repeatedly more different images than it can hold
(not very many). So for this type of usage pattern (concurrent or short
consecutive calls to different images), it is best to run Singularity
images natively.
Arguments:
image (str): name/path of the image.
args (list): list of command line arguments passed to the tool.
cwd (str): current working directory.
env (dict): environment variables to set inside container.
check_output (bool): check_output or check_call behavior.
working_dir (str): path to a working directory. If passed, a tmpdir
will be created inside and will be mounted to /tmp.
volumes (list): list of tuples (src-path, dst-path) to be mounted,
dst-path must be absolute path.
remove_tmp_dir (bool): remove tmpdir created inside `working_dir`.
Returns:
str: (check_output=True) stdout of the system call.
int: (check_output=False) 0 if call succeed else non-0.
Raises:
toil_container.ContainerError: if the container invocation fails.
toil_container.SingularityNotAvailableError: singularity not installed.
"""
singularity_path = is_singularity_available(raise_error=True, path=True)
singularity_version = subprocess.check_output([singularity_path, "--version"])
# ensure singularity doesn't overwrite $HOME by pointing to dummy dir
# /tmp will be mapped to work_dir/scratch/tmp and removed after the call
home_dir = ".unused_home"
work_dir = mkdtemp(prefix=_TMP_PREFIX, dir=working_dir)
singularity_args = [
"--home",
"{}:/tmp/{}".format(os.getcwd(), home_dir),
"--workdir",
work_dir,
]
if singularity_version.startswith("2.4"):
os.makedirs(os.path.join(work_dir, "scratch", "tmp", home_dir))
singularity_args += ["--scratch", "/tmp"]
else:
os.makedirs(os.path.join(work_dir, "tmp", home_dir))
singularity_args += ["--contain"]
# set parameters for managing directories if options are defined
if volumes:
for src, dst in volumes:
singularity_args += ["--bind", "{}:{}".format(src, dst)]
if cwd:
singularity_args += ["--pwd", cwd]
# setup the outgoing subprocess call for singularity
command = [singularity_path, "-q", "exec"] + singularity_args
command += [image] + (args or [])
if check_output:
call = subprocess.check_output
else:
call = subprocess.check_call
try:
output = call(command, env=env or {})
error = False
except (subprocess.CalledProcessError, OSError) as error:
pass
if remove_tmp_dir:
shutil.rmtree(work_dir, ignore_errors=True)
if error:
raise get_container_error(error)
try:
return output.decode()
except AttributeError:
return output
def test_which_util():
"""Test to check the which util is working."""
std_out = subprocess.check_output([utils.which("python"), "--version"],
stderr=subprocess.STDOUT)
assert "2.7" in std_out.decode()
