def set_sff_trimpoints_with_sfftools(
sff_dir, technical_lengths, sffinfo_path='sffinfo', sfffile_path='sfffile',
debug=False):
"""Set trimpoints to end of technical read for all SFF files in directory.
This function essentially provides the reference implementation.
It uses the official sfftools from Roche to process the SFF files.
"""
if not (exists(sffinfo_path) or which(sffinfo_path)):
raise ApplicationNotFoundError(
'sffinfo executable not found. Is it installed and in your $PATH?')
if not (exists(sfffile_path) or which(sfffile_path)):
raise ApplicationNotFoundError(
'sfffile executable not found. Is it installed and in your $PATH?')
for lib_id, sff_fp in get_per_lib_sff_fps(sff_dir):
try:
readlength = technical_lengths[lib_id]
except KeyError:
continue
sffinfo_args = [sffinfo_path, '-s', sff_fp]
if debug:
print "Running sffinfo command %s" % sffinfo_args
sffinfo_output_file = TemporaryFile()
check_call(sffinfo_args, stdout=sffinfo_output_file)
sffinfo_output_file.seek(0)
seqlengths = {}
for line in sffinfo_output_file:
if line.startswith('>'):
fields = line[1:].split()
seq_len = fields[1].split('=')[1]
seqlengths[fields[0]] = seq_len
trim_fp = sff_fp + '.trim'
trim_file = open(trim_fp, 'w')
for id_, length in seqlengths.items():
curr_length = int(seqlengths[id_])
# Sfftools use 1-based index
left_trim = readlength + 1
# Key sequence not included in FASTA length
right_trim = curr_length + 4
if curr_length > left_trim:
trim_file.write(
"%s\t%s\t%s\n" % (id_, left_trim, right_trim))
else:
stderr.write(
'Rejected read %s with trim points %s and %s (orig '
'length %s)' % (id_, left_trim, curr_length, length))
trim_file.close()
trimmed_sff_fp = sff_fp + '.trimmed'
sfffile_args = [
sfffile_path, '-t', trim_fp, '-o', trimmed_sff_fp, sff_fp]
if debug:
print "Running sfffile command:", sfffile_args
check_call(sfffile_args, stdout=open(devnull, 'w'))
remove(sff_fp)
rename(trimmed_sff_fp, sff_fp)
def test_ampliconnoise_install(self):
""" AmpliconNoise install looks sane."""
url = "http://qiime.org/install/install.html#ampliconnoise-install-notes"
pyro_lookup_file = getenv('PYRO_LOOKUP_FILE')
self.assertTrue(pyro_lookup_file is not None,
"$PYRO_LOOKUP_FILE variable is not set. See %s for help." % url)
self.assertTrue(exists(pyro_lookup_file),
"$PYRO_LOOKUP_FILE variable is not set to an existing filepath.")
seq_lookup_file = getenv('SEQ_LOOKUP_FILE')
self.assertTrue(seq_lookup_file is not None,
"$SEQ_LOOKUP_FILE variable is not set. See %s for help." % url)
self.assertTrue(exists(seq_lookup_file),
"$SEQ_LOOKUP_FILE variable is not set to an existing filepath.")
self.assertTrue(which("SplitKeys.pl"),
"Couldn't find SplitKeys.pl. " +
"Perhaps AmpliconNoise Scripts directory isn't in $PATH?" +
" See %s for help." % url)
self.assertTrue(which("FCluster"),
"Couldn't find FCluster. " +
"Perhaps the AmpliconNoise bin directory isn't in $PATH?" +
" See %s for help." % url)
self.assertTrue(which("Perseus"),
"Couldn't find Perseus. " +
"Perhaps the AmpliconNoise bin directory isn't in $PATH?" +
" See %s for help." % url)
def test_ampliconnoise_install(self):
""" AmpliconNoise install looks sane."""
url = "http://qiime.org/install/install.html#ampliconnoise-install-notes"
pyro_lookup_file = getenv('PYRO_LOOKUP_FILE')
self.assertTrue(
pyro_lookup_file is not None,
"$PYRO_LOOKUP_FILE variable is not set. See %s for help." % url)
self.assertTrue(
exists(pyro_lookup_file),
"$PYRO_LOOKUP_FILE variable is not set to an existing filepath.")
seq_lookup_file = getenv('SEQ_LOOKUP_FILE')
self.assertTrue(
seq_lookup_file is not None,
"$SEQ_LOOKUP_FILE variable is not set. See %s for help." % url)
self.assertTrue(
exists(seq_lookup_file),
"$SEQ_LOOKUP_FILE variable is not set to an existing filepath.")
self.assertTrue(
which("SplitKeys.pl"), "Couldn't find SplitKeys.pl. " +
"Perhaps AmpliconNoise Scripts directory isn't in $PATH?" +
" See %s for help." % url)
self.assertTrue(
which("FCluster"), "Couldn't find FCluster. " +
"Perhaps the AmpliconNoise bin directory isn't in $PATH?" +
" See %s for help." % url)
self.assertTrue(
which("Perseus"), "Couldn't find Perseus. " +
"Perhaps the AmpliconNoise bin directory isn't in $PATH?" +
" See %s for help." % url)
def test_raxmlHPC_supported_version(self):
"""raxmlHPC is in path and version is supported """
acceptable_version = [(7, 3, 0), (7, 3, 0)]
self.assertTrue(
which('raxmlHPC'),
"raxmlHPC not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.")
command = "raxmlHPC -v | grep version"
proc = Popen(command,
shell=True,
universal_newlines=True,
stdout=PIPE,
stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[4].strip()
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported raxmlHPC version. %s is required, but running %s." %
('.'.join(map(str, acceptable_version)), version_string))
def test_clearcut_supported_version(self):
"""clearcut is in path and version is supported """
acceptable_version = (1, 0, 9)
self.assertTrue(
which('clearcut'),
"clearcut not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.")
command = "clearcut -V"
proc = Popen(command,
shell=True,
universal_newlines=True,
stdout=PIPE,
stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[2].strip()
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported clearcut version. %s is required, but running %s." %
('.'.join(map(str, acceptable_version)), version_string))
def test_R_supported_version(self):
"""R is in path and version is supported """
minimum_version = (2, 12, 0)
self.assertTrue(which('R'),
"R not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
command = "R --version | grep 'R version' | awk '{print $3}'"
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip()
try:
version = tuple(map(int, version_string.split('.')))
pass_test = False
if version[0] == minimum_version[0]:
if version[1] == minimum_version[1]:
if version[2] >= minimum_version[2]:
pass_test = True
elif version[1] > minimum_version[1]:
pass_test = True
elif version[0] > minimum_version[0]:
pass_test = True
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,
"Unsupported R version. %s or greater is required, but running %s."
% ('.'.join(map(str, minimum_version)), version_string))
def test_FastTree_supported_version(self):
"""FastTree is in path and version is supported """
acceptable_version = (2, 1, 3)
self.assertTrue(which('FastTree'),
"FastTree not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
# If FastTree is run interactively, it outputs the following line:
# Usage for FastTree version 2.1.3 SSE3:
#
# If run non-interactively:
# FastTree Version 2.1.3 SSE3
command = "FastTree 2>&1 > %s | grep -i version" % devnull
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read().strip()
version_str_matches = re.findall('ersion\s+(\S+)\s+', stdout)
self.assertEqual(len(version_str_matches), 1,
"Could not find FastTree version info in usage text "
"'%s'." % stdout)
version_str = version_str_matches[0]
try:
version = tuple(map(int, version_str.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
acceptable_version_str = '.'.join(map(str, acceptable_version))
self.assertTrue(pass_test,
"Unsupported FastTree version. %s is required, but "
"running %s." % (acceptable_version_str, version_str))
def test_rtax_supported_version(self):
"""rtax is in path and version is supported """
acceptable_version = [(0, 984)]
self.assertTrue(
which('rtax'),
"rtax not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
command = "rtax 2>&1 > %s | grep Version | awk '{print $2}'" % devnull
proc = Popen(command,
shell=True,
universal_newlines=True,
stdout=PIPE,
stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip()
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported rtax version. %s is required, but running %s." %
('.'.join(map(str, acceptable_version)), version_string))
def test_cdhit_supported_version(self):
"""cd-hit is in path and version is supported """
self.assertTrue(
which("cd-hit"),
"cd-hit not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.",
)
def test_mothur_supported_version(self):
"""mothur is in path and version is supported """
acceptable_version = (1, 25, 0)
self.assertTrue(
which("mothur"),
"mothur not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.",
)
# mothur creates a log file in cwd, so create a tmp and cd there first
log_file = join(get_qiime_temp_dir(), "mothur.log")
command = "mothur \"#set.logfile(name=%s)\" | grep '^mothur v'" % log_file
stdout, stderr, exit_Status = qiime_system_call(command)
# remove log file
remove_files([log_file], error_on_missing=False)
version_string = stdout.strip().split(" ")[1].strip("v.")
try:
version = tuple(map(int, version_string.split(".")))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported mothur version. %s is required, but running %s."
% (".".join(map(str, acceptable_version)), version_string),
)
def test_usearch_supported_version(self):
"""usearch is in path and version is supported """
acceptable_version = [(5, 2, 236), (5, 2, 236)]
self.assertTrue(
which('usearch'),
"usearch not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.")
command = "usearch --version"
proc = Popen(command,
shell=True,
universal_newlines=True,
stdout=PIPE,
stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.split('v')[1]
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported usearch version. %s is required, but running %s." %
('.'.join(map(str, acceptable_version)), version_string))
def test_R_supported_version(self):
"""R is in path and version is supported """
minimum_version = (2, 12, 0)
self.assertTrue(
which('R'),
"R not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
command = "R --version | grep 'R version' | awk '{print $3}'"
proc = Popen(command,
shell=True,
universal_newlines=True,
stdout=PIPE,
stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip()
try:
version = tuple(map(int, version_string.split('.')))
pass_test = False
if version[0] == minimum_version[0]:
if version[1] == minimum_version[1]:
if version[2] >= minimum_version[2]:
pass_test = True
elif version[1] > minimum_version[1]:
pass_test = True
elif version[0] > minimum_version[0]:
pass_test = True
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported R version. %s or greater is required, but running %s."
% ('.'.join(map(str, minimum_version)), version_string))
def test_denoise_worker(self):
"""denoiser_worker.py is where it belongs and is callable."""
DENOISE_WORKER = "denoiser_worker.py"
self.assertTrue(
which(DENOISE_WORKER) is not None,
"%s is not accessible via the PATH environment "
"variable." % DENOISE_WORKER)
# test if its callable and actually works
command = "%s -h" % DENOISE_WORKER
proc = Popen(command,
shell=True,
universal_newlines=True,
stdout=PIPE,
stderr=STDOUT)
if (proc.wait() != 0):
self.fail(
"Calling %s failed. Check permissions and that it is in fact an executable."
% DENOISE_WORKER)
result = proc.stdout.read()
# check that the help string looks correct
self.assertTrue(result.startswith("Usage"))
def test_FastTree_supported_version(self):
"""FastTree is in path and version is supported """
acceptable_version = (2, 1, 3)
self.assertTrue(which('FastTree'),
"FastTree not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
# If FastTree is run interactively, it outputs the following line:
# Usage for FastTree version 2.1.3 SSE3:
#
# If run non-interactively:
# FastTree Version 2.1.3 SSE3
command = "FastTree 2>&1 > %s | grep -i version" % devnull
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read().strip()
version_str_matches = re.findall('ersion\s+(\S+)\s+', stdout)
self.assertEqual(len(version_str_matches), 1,
"Could not find FastTree version info in usage text "
"'%s'." % stdout)
version_str = version_str_matches[0]
try:
version = tuple(map(int, version_str.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
acceptable_version_str = '.'.join(map(str, acceptable_version))
self.assertTrue(pass_test,
"Unsupported FastTree version. %s is required, but "
"running %s." % (acceptable_version_str, version_str))
def test_mothur_supported_version(self):
"""mothur is in path and version is supported """
acceptable_version = (1, 25, 0)
self.assertTrue(
which('mothur'),
"mothur not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.")
# mothur creates a log file in cwd, so create a tmp and cd there first
log_file = join(get_qiime_temp_dir(), 'mothur.log')
command = "mothur \"#set.logfile(name=%s)\" | grep '^mothur v'" % log_file
stdout, stderr, exit_Status = qiime_system_call(command)
# remove log file
remove_files([log_file], error_on_missing=False)
version_string = stdout.strip().split(' ')[1].strip('v.')
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported mothur version. %s is required, but running %s." %
('.'.join(map(str, acceptable_version)), version_string))
def check_flowgram_ali_exe():
"""Check if we have a working FlowgramAligner"""
ali_exe = get_flowgram_ali_exe()
if which(ali_exe) is None:
raise ApplicationNotFoundError("The alignment program %s is not "
"accessible via the PATH environment "
"variable." % ali_exe)
# test if its callable and actually works
command = "%s -h" % ali_exe
proc = Popen(command,
shell=True,
universal_newlines=True,
stdout=PIPE,
stderr=STDOUT)
if (proc.wait() != 0):
raise ApplicationError(
"Calling %s failed. Check permissions and that it is in fact an executable."
% ali_exe)
result = proc.stdout.read()
# check that the help string looks correct
if (not result.startswith("Usage")):
raise ApplicationError(
"Calling %s failed. Check permissions and that it is in fact an executable."
% ali_exe)
return True
def test_blastall_fp(self):
"""blastall_fp is set to a valid path"""
blastall = self.config["blastall_fp"]
if not self.config["blastall_fp"].startswith("/"):
# path is relative, figure out absolute path
blast_all = which(blastall)
if not blast_all:
raise ApplicationNotFoundError(
"blastall_fp set to %s, but is not in your PATH. Either use an absolute path to or put it in your PATH." %
blastall)
self.config["blastall_fp"] = blast_all
test_qiime_config_variable("blastall_fp", self.config, self, X_OK)
def test_blastall_fp(self):
"""blastall_fp is set to a valid path"""
blastall = self.config["blastall_fp"]
if not self.config["blastall_fp"].startswith("/"):
# path is relative, figure out absolute path
blast_all = which(blastall)
if not blast_all:
raise ApplicationNotFoundError(
"blastall_fp set to %s, but is not in your PATH. Either use an absolute path to or put it in your PATH."
% blastall)
self.config["blastall_fp"] = blast_all
test_qiime_config_variable("blastall_fp", self.config, self, X_OK)
def test_cluster_jobs_fp(self):
"""cluster_jobs_fp is set to a valid path and is executable"""
fp = self.config["cluster_jobs_fp"]
if fp:
full_path = which(fp)
if full_path:
fp = full_path
# test if file exists or is in $PATH
self.assertTrue(exists(fp), "cluster_jobs_fp set to an invalid file path or is not in $PATH: %s" % fp)
modes = {R_OK: "readable", W_OK: "writable", X_OK: "executable"}
# test if file readable
self.assertTrue(access(fp, X_OK), "cluster_jobs_fp is not %s: %s" % (modes[X_OK], fp))
def submit_jobs(filenames, verbose=False):
"""Submit jobs in filenames.
filenames: list of prepared qsub job scripts, ready to be submitted
verbose: a binary verbose flag
"""
if not which("qsub"):
raise ApplicationNotFoundError("qsub not found. Can't submit jobs.")
for file in filenames:
command = "qsub %s" % file
result = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT).stdout.read()
if verbose:
print result
def wait_for_cluster_ids(ids, interval=10):
"""Puts process to sleep until jobs with ids are done.
ids: list of ids to wait for
interval: time to sleep in seconds
NOT USED ANYMORE
"""
if which("qstat"):
for id in ids:
while(getoutput("qstat %s" % id).startswith("Job")):
sleep(interval)
else:
raise ApplicationNotFoundError("qstat not available. Is it installed?\n" +
"This test may fail if not run on a cluster.")
def submit_jobs(filenames, verbose=False):
"""Submit jobs in filenames.
filenames: list of prepared qsub job scripts, ready to be submitted
verbose: a binary verbose flag
"""
if not which("qsub"):
raise ApplicationNotFoundError("qsub not found. Can't submit jobs.")
for file in filenames:
command = 'qsub %s' % file
result = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT).stdout.read()
if verbose:
print result
def wait_for_cluster_ids(ids, interval=10):
"""Puts process to sleep until jobs with ids are done.
ids: list of ids to wait for
interval: time to sleep in seconds
NOT USED ANYMORE
"""
if which("qstat"):
for id in ids:
while(getoutput("qstat %s" % id).startswith("Job")):
sleep(interval)
else:
raise ApplicationNotFoundError("qstat not available. Is it installed?\n" +
"This test may fail if not run on a cluster.")
def test_chimeraSlayer_install(self):
"""no obvious problems with ChimeraSlayer install """
# The ChimerSalyer app requires that all its components are installed
# relative to the main program ChimeraSlayer.pl.
# We therefore check that at least one the files is there.
# However, if the directory structure of ChimeraSlayer changes, this test will most
# likely fail as well and need to be updated.
# Tested with the version of microbiomeutil_2010-04-29
chim_slay = which("ChimeraSlayer.pl")
self.assertTrue(chim_slay, "ChimeraSlayer was not found in your $PATH")
dir, app_name = split(chim_slay)
self.assertTrue(
exists(dir + "/ChimeraParentSelector/chimeraParentSelector.pl"),
"ChimeraSlayer depends on external files in directoryies relative to its "
"install directory. These do not appear to be present.")
def test_chimeraSlayer_install(self):
"""no obvious problems with ChimeraSlayer install """
# The ChimerSalyer app requires that all its components are installed
# relative to the main program ChimeraSlayer.pl.
# We therefore check that at least one the files is there.
# However, if the directory structure of ChimeraSlayer changes, this test will most
# likely fail as well and need to be updated.
# Tested with the version of microbiomeutil_2010-04-29
chim_slay = which("ChimeraSlayer.pl")
self.assertTrue(chim_slay, "ChimeraSlayer was not found in your $PATH")
dir, app_name = split(chim_slay)
self.assertTrue(
exists(dir + "/ChimeraParentSelector/chimeraParentSelector.pl"),
"ChimeraSlayer depends on external files in directoryies relative to its "
"install directory. These do not appear to be present.")
def test_cluster_jobs_script(self):
"""cluster_jobs_fp is set to a good value"""
qiime_config = load_qiime_config()
submit_script = qiime_config['cluster_jobs_fp']
if (submit_script):
full_path = which(submit_script)
if full_path:
submit_script = full_path
self.assertTrue(exists(submit_script),
"cluster_jobs_fp is not set to a valid path in qiime config: %s" % submit_script)
# check if executable
self.assertTrue(access(submit_script, X_OK),
"cluster_jobs_fp is not executable: %s" % submit_script)
else:
# Can't run in parallel, but not a critical error
pass
def test_cluster_jobs_script(self):
"""cluster_jobs_fp is set to a good value"""
qiime_config = load_qiime_config()
submit_script = qiime_config['cluster_jobs_fp']
if (submit_script):
full_path = which(submit_script)
if full_path:
submit_script = full_path
self.assertTrue(exists(submit_script),
"cluster_jobs_fp is not set to a valid path in qiime config: %s" % submit_script)
# check if executable
self.assertTrue(access(submit_script, X_OK),
"cluster_jobs_fp is not executable: %s" % submit_script)
else:
# Can't run in parallel, but not a critical error
pass
def _error_on_missing_application(self, params):
"""Raise an ApplicationNotFoundError if the app is not accessible
In this case, checks for the java runtime and the RDP jar file.
"""
if not (os.path.exists('java') or which('java')):
raise ApplicationNotFoundError(
"Cannot find java runtime. Is it installed? Is it in your "
"path?")
jar_fp = self._get_jar_fp()
if jar_fp is None:
raise ApplicationNotFoundError(
"JAR file not found in current directory and the RDP_JAR_PATH "
"environment variable is not set. Please set RDP_JAR_PATH to "
"the full pathname of the JAR file.")
if not os.path.exists(jar_fp):
raise ApplicationNotFoundError(
"JAR file %s does not exist." % jar_fp)
def make_cidx_file(fp):
"""make a CS index file.
fp: reference DB to make an index of
ChimeraSlayer implicetely performs this task when the file is not
already created and deletes it when done. Especially in a parallel
environment it mkaes sense to manually make and delete the file.
"""
if which("cdbfasta"):
# cdbfasta comes with the microbiometools/CS
# should always be installed when using CS
args = ["cdbfasta", fp]
# cdbfasta write one line to stderr
stdout, stderr = Popen(args, stderr=PIPE, stdout=PIPE).communicate()
else:
raise ApplicationNotFoundError
def make_cidx_file(fp):
"""make a CS index file.
fp: reference DB to make an index of
ChimeraSlayer implicetely performs this task when the file is not
already created and deletes it when done. Especially in a parallel
environment it mkaes sense to manually make and delete the file.
"""
if which("cdbfasta"):
# cdbfasta comes with the microbiometools/CS
# should always be installed when using CS
args = ["cdbfasta", fp]
# cdbfasta write one line to stderr
stdout, stderr = Popen(args, stderr=PIPE, stdout=PIPE).communicate()
else:
raise ApplicationNotFoundError
def test_cluster_jobs_fp(self):
"""cluster_jobs_fp is set to a valid path and is executable"""
fp = self.config["cluster_jobs_fp"]
if fp:
full_path = which(fp)
if full_path:
fp = full_path
# test if file exists or is in $PATH
self.assertTrue(exists(fp),
"cluster_jobs_fp set to an invalid file path or is not in $PATH: %s" % fp)
modes = {R_OK: "readable",
W_OK: "writable",
X_OK: "executable"}
# test if file readable
self.assertTrue(access(fp, X_OK),
"cluster_jobs_fp is not %s: %s" % (modes[X_OK], fp))
def test_rtax_supported_version(self):
"""rtax is in path and version is supported """
acceptable_version = [(0, 984)]
self.assertTrue(which('rtax'),
"rtax not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
command = "rtax 2>&1 > %s | grep Version | awk '{print $2}'" % devnull
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip()
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,
"Unsupported rtax version. %s is required, but running %s."
% ('.'.join(map(str, acceptable_version)), version_string))
def test_flowgramAli_bin(self):
"""Check if we have a working FlowgramAligner"""
ali_exe = get_flowgram_ali_exe()
self.assertTrue(which(ali_exe) is not None, "The alignment program %s "
"is not accessible via the PATH environment variable."
% ali_exe)
# test if its callable and actually works
command = "%s -h" % ali_exe
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
if (proc.wait() != 0):
self.fail("Calling %s failed. Check permissions and that it is in fact an executable."
% ali_exe)
result = proc.stdout.read()
# check that the help string looks correct
self.assertTrue(result.startswith("Usage"))
def test_denoise_worker(self):
"""denoiser_worker.py is where it belongs and is callable."""
DENOISE_WORKER = "denoiser_worker.py"
self.assertTrue(which(DENOISE_WORKER) is not None,
"%s is not accessible via the PATH environment "
"variable." % DENOISE_WORKER)
# test if its callable and actually works
command = "%s -h" % DENOISE_WORKER
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
if (proc.wait() != 0):
self.fail("Calling %s failed. Check permissions and that it is in fact an executable."
% DENOISE_WORKER)
result = proc.stdout.read()
# check that the help string looks correct
self.assertTrue(result.startswith("Usage"))
def test_raxmlHPC_supported_version(self):
"""raxmlHPC is in path and version is supported """
acceptable_version = [(7, 3, 0), (7, 3, 0)]
self.assertTrue(which('raxmlHPC'),
"raxmlHPC not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
command = "raxmlHPC -v | grep version"
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[4].strip()
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,
"Unsupported raxmlHPC version. %s is required, but running %s."
% ('.'.join(map(str, acceptable_version)), version_string))
def test_clearcut_supported_version(self):
"""clearcut is in path and version is supported """
acceptable_version = (1, 0, 9)
self.assertTrue(which('clearcut'),
"clearcut not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
command = "clearcut -V"
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[2].strip()
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,
"Unsupported clearcut version. %s is required, but running %s."
% ('.'.join(map(str, acceptable_version)), version_string))
def test_usearch_supported_version(self):
"""usearch is in path and version is supported """
acceptable_version = [(5, 2, 236), (5, 2, 236)]
self.assertTrue(which('usearch'),
"usearch not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
command = "usearch --version"
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.split('v')[1]
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,
"Unsupported usearch version. %s is required, but running %s."
% ('.'.join(map(str, acceptable_version)), version_string))
def test_flowgramAli_bin(self):
"""Check if we have a working FlowgramAligner"""
ali_exe = get_flowgram_ali_exe()
self.assertTrue(which(ali_exe) is not None, "The alignment program %s "
"is not accessible via the PATH environment variable."
% ali_exe)
# test if its callable and actually works
command = "%s -h" % ali_exe
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
if (proc.wait() != 0):
self.fail("Calling %s failed. Check permissions and that it is in fact an executable."
% ali_exe)
result = proc.stdout.read()
# check that the help string looks correct
self.assertTrue(result.startswith("Usage"))
def submit_jobs(commands, prefix):
"""submit jobs using exe pointed to by cluster_jobs_fp.
commands: List of commands (strings) that should be executed
prefix: A uniq prefix used to name submit script
"""
qiime_config = load_qiime_config()
CLUSTER_JOBS_SCRIPT = qiime_config["cluster_jobs_fp"]
if not CLUSTER_JOBS_SCRIPT:
raise ApplicationNotFoundError("cluster_jobs_fp not set in config file!")
if not (exists(CLUSTER_JOBS_SCRIPT) or which(CLUSTER_JOBS_SCRIPT)):
raise ApplicationNotFoundError("cluster_jobs_fp not in $PATH or provided as full path!")
outfilename = join(get_qiime_temp_dir(), "%s_commands.txt" % prefix)
fh = open(outfilename, "w")
fh.write("\n".join(commands))
fh.close()
cmd = "%s -ms %s %s" % (CLUSTER_JOBS_SCRIPT, outfilename, prefix)
system(cmd)
remove(outfilename)
def test_INFERNAL_supported_version(self):
"""INFERNAL is in path and version is supported """
acceptable_version = (1, 0, 2)
self.assertTrue(
which("cmbuild"),
"Infernal not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.",
)
command = "cmbuild -h | grep INF"
proc = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(" ")[2].strip()
try:
version = tuple(map(int, version_string.split(".")))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported INFERNAL version. %s is required, but running %s."
% (".".join(map(str, acceptable_version)), version_string),
)
def test_muscle_supported_version(self):
"""muscle is in path and version is supported """
acceptable_version = (3, 8, 31)
self.assertTrue(
which("muscle"),
"muscle not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.",
)
command = "muscle -version"
proc = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(" ")[1].strip("v")
try:
version = tuple(map(int, version_string.split(".")))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported muscle version. %s is required, but running %s."
% (".".join(map(str, acceptable_version)), version_string),
)
def submit_jobs(commands, prefix):
"""submit jobs using exe pointed to by cluster_jobs_fp.
commands: List of commands (strings) that should be executed
prefix: A uniq prefix used to name submit script
"""
qiime_config = load_qiime_config()
CLUSTER_JOBS_SCRIPT = qiime_config['cluster_jobs_fp']
if not CLUSTER_JOBS_SCRIPT:
raise ApplicationNotFoundError(
"cluster_jobs_fp not set in config file!")
if not (exists(CLUSTER_JOBS_SCRIPT) or which(CLUSTER_JOBS_SCRIPT)):
raise ApplicationNotFoundError(
"cluster_jobs_fp not in $PATH or provided as full path!")
outfilename = join(get_qiime_temp_dir(), "%s_commands.txt" % prefix)
fh = open(outfilename, "w")
fh.write("\n".join(commands))
fh.close()
cmd = '%s -ms %s %s' % (CLUSTER_JOBS_SCRIPT, outfilename, prefix)
system(cmd)
remove(outfilename)
def check_flowgram_ali_exe():
"""Check if we have a working FlowgramAligner"""
ali_exe = get_flowgram_ali_exe()
if which(ali_exe) is None:
raise ApplicationNotFoundError("The alignment program %s is not "
"accessible via the PATH environment "
"variable." % ali_exe)
# test if its callable and actually works
command = "%s -h" % ali_exe
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
if (proc.wait() != 0):
raise ApplicationError("Calling %s failed. Check permissions and that it is in fact an executable."
% ali_exe)
result = proc.stdout.read()
# check that the help string looks correct
if (not result.startswith("Usage")):
raise ApplicationError("Calling %s failed. Check permissions and that it is in fact an executable."
% ali_exe)
return True
else:
gdata_installed = "Installed."
try:
import h5py
h5py_lib_version = h5py.__version__ + " (HDF5 version: %s)" % h5py.version.hdf5_version
except ImportError:
h5py_lib_version = "Not installed."
pynast_lib_version = get_pynast_version()
if pynast_lib_version is None:
pynast_lib_version = "Not installed."
if which("sortmerna") is None:
sortmerna_lib_version = "Not installed."
else:
_, serr, _ = qiime_system_call("sortmerna --version")
sortmerna_lib_version = serr.strip()
if which("sumaclust") is None:
sumaclust_lib_version = "Not installed."
else:
sout, _, _ = qiime_system_call("sumaclust --help")
sout_lines = sout.split("\n")
sumaclust_lib_version = "Installed, but can't identify version."
for e in sout_lines:
e = e.strip()
if e.startswith("SUMACLUST Version"):
sumaclust_lib_version = e
bfillings_lib_version = "Installed."
# gdata doesn't have __version__ and adding that is outside of our control.
# just indicate whether it's installed or not
try:
import gdata
except ImportError:
gdata_installed = "Not installed."
else:
gdata_installed = "Installed."
pynast_lib_version = get_pynast_version()
if pynast_lib_version is None:
pynast_lib_version = "Not installed."
if which('sortmerna') is None:
sortmerna_lib_version = "Not installed."
else:
_, serr, _ = qiime_system_call("sortmerna --version")
sortmerna_lib_version = serr.strip()
if which('sumaclust') is None:
sumaclust_lib_version = "Not installed."
else:
sout, _, _ = qiime_system_call("sumaclust --help")
sout_lines = sout.split('\n')
sumaclust_lib_version = "Installed, but can't identify version."
for e in sout_lines:
e = e.strip()
if e.startswith('SUMACLUST Version'):
sumaclust_lib_version = e
def check_sfffile():
"""Raise error if sfffile is not in $PATH """
if not which('sfffile'):
raise ApplicationNotFoundError(_MISSING_APP_MESSAGE % 'sfffile')
gdata_installed = "Not installed."
else:
gdata_installed = "Installed."
try:
import h5py
h5py_lib_version = (h5py.__version__ +
' (HDF5 version: %s)' % h5py.version.hdf5_version)
except ImportError:
h5py_lib_version = "Not installed."
pynast_lib_version = get_pynast_version()
if pynast_lib_version is None:
pynast_lib_version = "Not installed."
if which('sortmerna') is None:
sortmerna_lib_version = "Not installed."
else:
_, serr, _ = qiime_system_call("sortmerna --version")
sortmerna_lib_version = serr.strip()
if which('sumaclust') is None:
sumaclust_lib_version = "Not installed."
else:
sout, _, _ = qiime_system_call("sumaclust --help")
sout_lines = sout.split('\n')
sumaclust_lib_version = "Installed, but can't identify version."
for e in sout_lines:
e = e.strip()
if e.startswith('SUMACLUST Version'):
sumaclust_lib_version = e
def test_cdhit_supported_version(self):
"""cd-hit is in path and version is supported """
self.assertTrue(
which('cd-hit'),
"cd-hit not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.")
