def test_ampliconnoise_install(self):
""" AmpliconNoise install looks sane."""
url="http://qiime.org/install/install.html#ampliconnoise-install-notes"
pyro_lookup_file = getenv('PYRO_LOOKUP_FILE')
self.assertTrue(pyro_lookup_file != None,
"$PYRO_LOOKUP_FILE variable is not set. See %s for help." % url)
self.assertTrue(exists(pyro_lookup_file),
"$PYRO_LOOKUP_FILE variable is not set to an existing filepath.")
seq_lookup_file = getenv('SEQ_LOOKUP_FILE')
self.assertTrue(seq_lookup_file != None,
"$SEQ_LOOKUP_FILE variable is not set. See %s for help." % url)
self.assertTrue(exists(seq_lookup_file),
"$SEQ_LOOKUP_FILE variable is not set to an existing filepath.")
self.assertTrue(app_path("SplitKeys.pl"),
"Couldn't find SplitKeys.pl. "+\
"Perhaps AmpliconNoise Scripts directory isn't in $PATH?"+\
" See %s for help." % url)
self.assertTrue(app_path("FCluster"),
"Couldn't find FCluster. "+\
"Perhaps the AmpliconNoise bin directory isn't in $PATH?"+\
" See %s for help." % url)
self.assertTrue(app_path("Perseus"),
"Couldn't find Perseus. "+\
"Perhaps the AmpliconNoise bin directory isn't in $PATH?"+\
" See %s for help." % url)
def set_sff_trimpoints_with_sfftools(
sff_dir, technical_lengths, sffinfo_path='sffinfo', sfffile_path='sfffile',
debug=False):
"""Set trimpoints to end of technical read for all SFF files in directory.
This function essentially provides the reference implementation.
It uses the official sfftools from Roche to process the SFF files.
"""
if not (os.path.exists(sffinfo_path) or app_path(sffinfo_path)):
raise ApplicationNotFoundError(
'sffinfo executable not found. Is it installed and in your $PATH?')
if not (os.path.exists(sfffile_path) or app_path(sfffile_path)):
raise ApplicationNotFoundError(
'sfffile executable not found. Is it installed and in your $PATH?')
for lib_id, sff_fp in get_per_lib_sff_fps(sff_dir):
try:
readlength = technical_lengths[lib_id]
except KeyError:
continue
sffinfo_args = [sffinfo_path, '-s', sff_fp]
if debug:
print "Running sffinfo command %s" % sffinfo_args
sffinfo_output_file = tempfile.TemporaryFile()
subprocess.check_call(sffinfo_args, stdout=sffinfo_output_file)
sffinfo_output_file.seek(0)
seqlengths = {}
for line in sffinfo_output_file:
if line.startswith('>'):
fields = line[1:].split()
seq_len = fields[1].split('=')[1]
seqlengths[fields[0]] = seq_len
trim_fp = sff_fp + '.trim'
trim_file = open(trim_fp, 'w')
for id_, length in seqlengths.items():
curr_length = int(seqlengths[id_])
# Sfftools use 1-based index
left_trim = readlength + 1
# Key sequence not included in FASTA length
right_trim = curr_length + 4
if curr_length > left_trim:
trim_file.write(
"%s\t%s\t%s\n" % (id_, left_trim, right_trim))
else:
sys.stderr.write(
'Rejected read %s with trim points %s and %s (orig '
'length %s)' % (id_, left_trim, curr_length, length))
trim_file.close()
trimmed_sff_fp = sff_fp + '.trimmed'
sfffile_args = [
sfffile_path, '-t', trim_fp, '-o', trimmed_sff_fp, sff_fp]
if debug:
print "Running sfffile command:", sfffile_args
subprocess.check_call(sfffile_args, stdout=open(os.devnull, 'w'))
os.remove(sff_fp)
os.rename(trimmed_sff_fp, sff_fp)
def test_ampliconnoise_install(self):
""" AmpliconNoise install looks sane."""
url = "http://www.qiime.org/install/install.html#ampliconnoise-install"
pyro_lookup_file = getenv('PYRO_LOOKUP_FILE')
self.assertTrue(pyro_lookup_file != None,
"$PYRO_LOOKUP_FILE variable is not set. See %s for help." % url)
self.assertTrue(exists(pyro_lookup_file),
"$PYRO_LOOKUP_FILE variable is not set to an existing filepath.")
seq_lookup_file = getenv('SEQ_LOOKUP_FILE')
self.assertTrue(seq_lookup_file != None,
"$SEQ_LOOKUP_FILE variable is not set. See %s for help." % url)
self.assertTrue(exists(seq_lookup_file),
"$SEQ_LOOKUP_FILE variable is not set to an existing filepath.")
self.assertTrue(app_path("SplitKeys.pl"),
"Couldn't find SplitKeys.pl. "+\
"Perhaps AmpliconNoise Scripts directory isn't in $PATH?"+\
" See %s for help." % url)
self.assertTrue(app_path("FCluster"),
"Couldn't find FCluster. "+\
"Perhaps the AmpliconNoise bin directory isn't in $PATH?"+\
" See %s for help." % url)
self.assertTrue(app_path("Perseus"),
"Couldn't find Perseus. "+\
"Perhaps the AmpliconNoise bin directory isn't in $PATH?"+\
" See %s for help." % url)
def _error_on_missing_application(self,params):
""" Raise an ApplicationNotFoundError if the app is not accessible
"""
if not app_path('RNAforester'):
raise ApplicationNotFoundError("Cannot find RNAforester. Is it installed? Is it in your path?")
if not app_path('RNAshapes'):
raise ApplicationNotFoundError("Cannot find RNAshapes. Is it installed? Is it in your path?")
def set_sff_trimpoints_with_sfftools(
sff_dir, technical_lengths, sffinfo_path='sffinfo', sfffile_path='sfffile',
debug=False):
"""Set trimpoints to end of technical read for all SFF files in directory.
This function essentially provides the reference implementation.
It uses the official sfftools from Roche to process the SFF files.
"""
if not (os.path.exists(sffinfo_path) or app_path(sffinfo_path)):
raise ApplicationNotFoundError(
'sffinfo executable not found. Is it installed and in your $PATH?')
if not (os.path.exists(sfffile_path) or app_path(sfffile_path)):
raise ApplicationNotFoundError(
'sfffile executable not found. Is it installed and in your $PATH?')
for lib_id, sff_fp in get_per_lib_sff_fps(sff_dir):
try:
readlength = technical_lengths[lib_id]
except KeyError:
continue
sffinfo_args = [sffinfo_path, '-s', sff_fp]
if debug:
print "Running sffinfo command %s" % sffinfo_args
sffinfo_output_file = tempfile.TemporaryFile()
subprocess.check_call(sffinfo_args, stdout=sffinfo_output_file)
sffinfo_output_file.seek(0)
seqlengths = {}
for line in sffinfo_output_file:
if line.startswith('>'):
fields = line[1:].split()
seq_len = fields[1].split('=')[1]
seqlengths[fields[0]] = seq_len
trim_fp = sff_fp + '.trim'
trim_file = open(trim_fp, 'w')
for id_, length in seqlengths.items():
curr_length = int(seqlengths[id_])
# Sfftools use 1-based index
left_trim = readlength + 1
# Key sequence not included in FASTA length
right_trim = curr_length + 4
if curr_length > left_trim:
trim_file.write(
"%s\t%s\t%s\n" % (id_, left_trim, right_trim))
else:
sys.stderr.write(
'Rejected read %s with trim points %s and %s (orig '
'length %s)' % (id_, left_trim, curr_length, length))
trim_file.close()
trimmed_sff_fp = sff_fp + '.trimmed'
sfffile_args = [
sfffile_path, '-t', trim_fp, '-o', trimmed_sff_fp, sff_fp]
if debug:
print "Running sfffile command:", sfffile_args
subprocess.check_call(sfffile_args, stdout=open(os.devnull, 'w'))
os.remove(sff_fp)
os.rename(trimmed_sff_fp, sff_fp)
def _error_on_missing_application(self, params):
""" Raise an ApplicationNotFoundError if the app is not accessible
"""
if not app_path('RNAforester'):
raise ApplicationNotFoundError,\
"Cannot find RNAforester. Is it installed? Is it in your path?"
if not app_path('RNAshapes'):
raise ApplicationNotFoundError,\
"Cannot find RNAshapes. Is it installed? Is it in your path?"
def test_mothur_supported_version(self):
"""mothur is in path and version is supported """
acceptable_version = (1,25,0)
self.assertTrue(app_path('mothur'),
"mothur not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
# mothur creates a log file in cwd, so create a tmp and cd there first
command = "mothur \"#set.logfile(name=mothur.log)\" | grep '^mothur v'"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout, stderr = proc.communicate()
# remove log file
remove('mothur.log')
version_string = stdout.strip().split(' ')[1].strip('v.')
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported mothur version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def setUp(self):
"""Check if Raxml version is supported for this test"""
acceptable_version = (7, 3, 0)
self.assertTrue(app_path('raxmlHPC'),
"raxmlHPC not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "raxmlHPC -v | grep version"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[4].strip()
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported raxmlHPC version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
"""Setup data for raxml tests"""
self.seqs1 = [
'ACUGCUAGCUAGUAGCGUACGUA', 'GCUACGUAGCUAC', 'GCGGCUAUUAGAUCGUA'
]
self.labels1 = ['>1', '>2', '>3']
self.lines1 = flatten(zip(self.labels1, self.seqs1))
self.test_model = "GTRCAT"
self.align1 = get_align_for_phylip(StringIO(PHYLIP_FILE))
self.test_fn1 = "/tmp/raxml_test1.txt"
self.test_fn2 = "raxml_test1.txt"
self.test_fn1_space = "/tmp/raxml test1.txt"
def test_mothur_supported_version(self):
"""mothur is in path and version is supported """
acceptable_version = (1, 25, 0)
self.assertTrue(
app_path("mothur"),
"mothur not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.",
)
# mothur creates a log file in cwd, so create a tmp and cd there first
log_file = join(get_qiime_temp_dir(), "mothur.log")
command = "mothur \"#set.logfile(name=%s)\" | grep '^mothur v'" % log_file
stdout, stderr, exit_Status = qiime_system_call(command)
# remove log file
remove_files([log_file], error_on_missing=False)
version_string = stdout.strip().split(" ")[1].strip("v.")
try:
version = tuple(map(int, version_string.split(".")))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported mothur version. %s is required, but running %s."
% (".".join(map(str, acceptable_version)), version_string),
)
def test_mothur_supported_version(self):
"""mothur is in path and version is supported """
acceptable_version = (1,25,0)
self.assertTrue(app_path('mothur'),
"mothur not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
# mothur creates a log file in cwd, so create a tmp and cd there first
command = "mothur \"#set.logfile(name=mothur.log)\" | grep '^mothur v'"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout, stderr = proc.communicate()
# remove log file
remove('mothur.log')
version_string = stdout.strip().split(' ')[1].strip('v.')
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported mothur version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def test_ParsInsert_supported_version(self):
"""ParsInsert is in path and version is supported """
acceptable_version = ["1.04"]
self.assertTrue(
app_path("ParsInsert"),
"ParsInsert not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.",
)
command = "ParsInsert -v | grep App | awk '{print $3}'"
proc = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
# remove log file generated
remove("ParsInsert.log")
version_string = stdout.strip()
try:
pass_test = version_string in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported ParsInsert version. %s is required, but running %s."
% (".".join(map(str, acceptable_version)), version_string),
)
def test_R_supported_version(self):
"""R is in path and version is supported """
minimum_version = (2,12,0)
self.assertTrue(app_path('R'),
"R not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "R --version | grep 'R version' | awk '{print $3}'"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip()
try:
version = tuple(map(int,version_string.split('.')))
pass_test = False
if version[0] == minimum_version[0]:
if version[1] == minimum_version[1]:
if version[2] >= minimum_version[2]:
pass_test = True
elif version[1] > minimum_version[1]:
pass_test = True
elif version[0] > minimum_version[0]:
pass_test = True
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported R version. %s or greater is required, but running %s." \
% ('.'.join(map(str,minimum_version)), version_string))
def test_cdhit_supported_version(self):
"""cd-hit is in path and version is supported """
self.assertTrue(
app_path("cd-hit"),
"cd-hit not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.",
)
def setUp(self):
"""Check if Raxml version is supported for this test"""
acceptable_version = (7,3,0)
self.assertTrue(app_path('raxmlHPC'),
"raxmlHPC not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "raxmlHPC -v | grep version"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[4].strip()
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported raxmlHPC version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
"""Setup data for raxml tests"""
self.seqs1 = ['ACUGCUAGCUAGUAGCGUACGUA','GCUACGUAGCUAC',
'GCGGCUAUUAGAUCGUA']
self.labels1 = ['>1','>2','>3']
self.lines1 = flatten(zip(self.labels1,self.seqs1))
self.test_model = "GTRCAT"
self.align1 = get_align_for_phylip(StringIO(PHYLIP_FILE))
self.test_fn1 = "/tmp/raxml_test1.txt"
self.test_fn2 = "raxml_test1.txt"
self.test_fn1_space = "/tmp/raxml test1.txt"
def test_mothur_supported_version(self):
"""mothur is in path and version is supported """
acceptable_version = (1,6,0)
self.assertTrue(app_path('mothur'),
"mothur not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
# mothur creates a log file in cwd, so create a tmp and cd there first
tmp_dir = get_random_directory_name(output_dir='/tmp/')
command = "cd %s ; mothur -v | grep ^mothur" % tmp_dir
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout, stderr = proc.communicate()
version_string = stdout.strip().split(' ')[1].strip('v.')
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported mothur version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
# remove the directory and the log file
rmtree(tmp_dir)
def submit_jobs(filenames, verbose=False):
"""Submit jobs in filenames."""
if(not app_path("qsub")):
raise ApplicationNotFoundError,"qsub not found. Can't submit jobs."
for file in filenames:
command = 'qsub -u wwwuser %s' % (file)
result = Popen(command, shell=True, universal_newlines=True,\
stdout=PIPE, stderr=STDOUT).stdout.read()
if verbose:
print result
def test_blastall_fp(self):
"""blastall_fp is set to a valid path"""
blastall = self.config["blastall_fp"]
if not self.config["blastall_fp"].startswith("/"):
#path is relative, figure out absolute path
blast_all = app_path(blastall)
if not blast_all:
raise ApplicationNotFoundError("blastall_fp set to %s, but is not in your PATH. Either use an absolute path to or put it in your PATH." % blastall)
self.config["blastall_fp"] = blast_all
test_qiime_config_variable("blastall_fp", self.config, self, X_OK)
def test_blastall_fp(self):
"""blastall_fp is set to a valid path"""
blastall = self.config["blastall_fp"]
if not self.config["blastall_fp"].startswith("/"):
#path is relative, figure out absolute path
blast_all = app_path(blastall)
if not blast_all:
raise ApplicationNotFoundError("blastall_fp set to %s, but is not in your PATH. Either use an absolute path to or put it in your PATH." % blastall)
self.config["blastall_fp"] = blast_all
test_qiime_config_variable("blastall_fp", self.config, self, X_OK)
def test_cluster_jobs_fp(self):
"""cluster_jobs_fp is set to a valid path and is executable"""
fp = self.config["cluster_jobs_fp"]
if fp:
full_path = app_path(fp)
if full_path:
fp = full_path
# test if file exists or is in $PATH
self.assertTrue(exists(fp), "cluster_jobs_fp set to an invalid file path or is not in $PATH: %s" % fp)
modes = {R_OK: "readable", W_OK: "writable", X_OK: "executable"}
# test if file readable
self.assertTrue(access(fp, X_OK), "cluster_jobs_fp is not %s: %s" % (modes[X_OK], fp))
def wait_for_cluster_ids(ids, interval=10):
"""Puts process to sleep until jobs with ids are done.
ids: list of ids to wait for
interval: time to sleep in seconds
NOT USED ANYMORE
"""
if app_path("qstat"):
for id in ids:
while getoutput("qstat %s" % id).startswith("Job"):
sleep(interval)
else:
raise ApplicationNotFoundError, "qstat not available. Is it installed?\n" + "This test may fail if not run on a cluster."
def test_chimeraSlayer_install(self):
"""no obvious problems with ChimeraSlayer install """
#The ChimerSalyer app requires that all its components are installed
# relative to the main program ChimeraSlayer.pl.
# We therefore check that at least one the files is there.
# However, if the directory structure of ChimeraSlayer changes, this test will most
# likely fail as well and need to be updated.
# Tested with the version of microbiomeutil_2010-04-29
chim_slay = app_path("ChimeraSlayer.pl")
self.assertTrue(chim_slay,"ChimeraSlayer was not found in your $PATH")
dir, app_name = split(chim_slay)
self.assertTrue(exists(dir+"/ChimeraParentSelector/chimeraParentSelector.pl"),
"ChimeraSlayer depends on external files in directoryies relative to its "
"install directory. Thesedo not appear to be present.")
def submit_jobs(filenames, verbose=False):
"""Submit jobs in filenames.
filenames: list of prepared qsub job scripts, ready to be submitted
verbose: a binary verbose flag
"""
if(not app_path("qsub")):
raise ApplicationNotFoundError,"qsub not found. Can't submit jobs."
for file in filenames:
command = 'qsub %s' % file
result = Popen(command, shell=True, universal_newlines=True,\
stdout=PIPE, stderr=STDOUT).stdout.read()
if verbose:
print result
def test_chimeraSlayer_install(self):
"""no obvious problems with ChimeraSlayer install """
#The ChimerSalyer app requires that all its components are installed
# relative to the main program ChimeraSlayer.pl.
# We therefore check that at least one the files is there.
# However, if the directory structure of ChimeraSlayer changes, this test will most
# likely fail as well and need to be updated.
# Tested with the version of microbiomeutil_2010-04-29
chim_slay = app_path("ChimeraSlayer.pl")
self.assertTrue(chim_slay,"ChimeraSlayer was not found in your $PATH")
dir, app_name = split(chim_slay)
self.assertTrue(exists(dir+"/ChimeraParentSelector/chimeraParentSelector.pl"),
"ChimeraSlayer depends on external files in directoryies relative to its "
"install directory. Thesedo not appear to be present.")
def submit_jobs(filenames, verbose=False):
"""Submit jobs in filenames.
filenames: list of prepared qsub job scripts, ready to be submitted
verbose: a binary verbose flag
"""
if(not app_path("qsub")):
raise ApplicationNotFoundError,"qsub not found. Can't submit jobs."
for file in filenames:
command = 'qsub %s' % file
result = Popen(command, shell=True, universal_newlines=True,\
stdout=PIPE, stderr=STDOUT).stdout.read()
if verbose:
print result
def wait_for_cluster_ids(ids, interval=10):
"""Puts process to sleep until jobs with ids are done.
ids: list of ids to wait for
interval: time to sleep in seconds
NOT USED ANYMORE
"""
if (app_path("qstat")):
for id in ids:
while (getoutput("qstat %s" % id).startswith("Job")):
sleep(interval)
else:
raise ApplicationNotFoundError,"qstat not available. Is it installed?\n"+\
"This test may fail if not run on a cluster."
def make_cidx_file(fp):
"""make a CS index file.
fp: reference DB to make an index of
ChimeraSlayer implicetely performs this task when the file is not
already created and deletes it when done. Especially in a parallel
environment it mkaes sense to manually make and delete the file.
"""
if app_path("cdbfasta"):
#cdbfasta comes with the microbiometools/CS
#should always be installed when using CS
args = ["cdbfasta", fp]
#cdbfasta write one line to stderr
stdout, stderr = Popen(args, stderr=PIPE, stdout=PIPE).communicate()
else:
raise ApplicationNotFoundError
def make_cidx_file(fp):
"""make a CS index file.
fp: reference DB to make an index of
ChimeraSlayer implicetely performs this task when the file is not
already created and deletes it when done. Especially in a parallel
environment it mkaes sense to manually make and delete the file.
"""
if app_path("cdbfasta"):
#cdbfasta comes with the microbiometools/CS
#should always be installed when using CS
args = ["cdbfasta", fp]
#cdbfasta write one line to stderr
stdout, stderr = Popen(args, stderr=PIPE, stdout=PIPE).communicate()
else:
raise ApplicationNotFoundError
def test_cluster_jobs_script(self):
"""cluster_jobs_fp is set to a good value"""
qiime_config = load_qiime_config()
submit_script = qiime_config['cluster_jobs_fp']
if (submit_script):
full_path = app_path(submit_script)
if full_path:
submit_script = full_path
self.assertTrue(exists(submit_script),
"cluster_jobs_fp is not set to a valid path in qiime config: %s" % submit_script)
#check if executable
self.assertTrue(access(submit_script, X_OK),
"cluster_jobs_fp is not executable: %s" % submit_script)
else:
#Can't run in parallel, but not a critical error
pass
def _error_on_missing_application(self,params):
"""Raise an ApplicationNotFoundError if the app is not accessible
In this case, checks for the java runtime and the RDP jar file.
"""
if not (os.path.exists('java') or app_path('java')):
raise ApplicationNotFoundError(
"Cannot find java runtime. Is it installed? Is it in your "
"path?")
jar_fp = self._get_jar_fp()
if jar_fp is None:
raise ApplicationNotFoundError(
"JAR file not found in current directory and the RDP_JAR_PATH "
"environment variable is not set. Please set RDP_JAR_PATH to "
"the full pathname of the JAR file.")
if not os.path.exists(jar_fp):
raise ApplicationNotFoundError(
"JAR file %s does not exist." % jar_fp)
def test_raxmlHPC_supported_version(self):
"""raxmlHPC is in path and version is supported """
acceptable_version = [(7,3,0),(7,3,0)]
self.assertTrue(app_path('raxmlHPC'),
"raxmlHPC not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "raxmlHPC -v | grep version"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[4].strip()
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported raxmlHPC version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def test_cluster_jobs_fp(self):
"""cluster_jobs_fp is set to a valid path and is executable"""
fp = self.config["cluster_jobs_fp"]
if fp:
full_path = app_path(fp)
if full_path:
fp = full_path
#test if file exists or is in $PATH
self.assertTrue(
exists(fp),
"cluster_jobs_fp set to an invalid file path or is not in $PATH: %s"
% fp)
modes = {R_OK: "readable", W_OK: "writable", X_OK: "executable"}
#test if file readable
self.assertTrue(
access(fp, X_OK),
"cluster_jobs_fp is not %s: %s" % (modes[X_OK], fp))
def test_pplacer_supported_version(self):
"""pplacer is in path and version is supported """
acceptable_version = [(1, 1), (1, 1)]
self.assertTrue(app_path('pplacer'),
"pplacer not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
command = "pplacer --version"
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip()[1:4]
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,
"Unsupported pplacer version. %s is required, but running %s."
% ('.'.join(map(str, acceptable_version)), version_string))
def test_clearcut_supported_version(self):
"""clearcut is in path and version is supported """
acceptable_version = (1,0,9)
self.assertTrue(app_path('clearcut'),
"clearcut not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "clearcut -V"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[2].strip()
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported clearcut version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def test_INFERNAL_supported_version(self):
"""INFERNAL is in path and version is supported """
acceptable_version = (1, 0, 2)
self.assertTrue(app_path('cmbuild'),
"Infernal not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
command = "cmbuild -h | grep INF"
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[2].strip()
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,
"Unsupported INFERNAL version. %s is required, but running %s."
% ('.'.join(map(str, acceptable_version)), version_string))
def test_rtax_supported_version(self):
"""rtax is in path and version is supported """
acceptable_version = [(0,982),(0,983)]
self.assertTrue(app_path('rtax'),
"rtax not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "rtax 2>&1 > %s | grep Version | awk '{print $2}'" % devnull
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip()
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported rtax version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def test_rtax_supported_version(self):
"""rtax is in path and version is supported """
acceptable_version = [(0,982),(0,983)]
self.assertTrue(app_path('rtax'),
"rtax not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "rtax 2>&1 > %s | grep Version | awk '{print $2}'" % devnull
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip()
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported rtax version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def test_raxmlHPC_supported_version(self):
"""raxmlHPC is in path and version is supported """
acceptable_version = [(7,3,0),(7,3,0)]
self.assertTrue(app_path('raxmlHPC'),
"raxmlHPC not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "raxmlHPC -v | grep version"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[4].strip()
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported raxmlHPC version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def test_clearcut_supported_version(self):
"""clearcut is in path and version is supported """
acceptable_version = (1,0,9)
self.assertTrue(app_path('clearcut'),
"clearcut not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "clearcut -V"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[2].strip()
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported clearcut version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def test_muscle_supported_version(self):
"""muscle is in path and version is supported """
acceptable_version = (3, 8, 31)
self.assertTrue(app_path('muscle'),
"muscle not found. This may or may not be a problem depending on " +
"which components of QIIME you plan to use.")
command = "muscle -version"
proc = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(' ')[1].strip('v')
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,
"Unsupported muscle version. %s is required, but running %s."
% ('.'.join(map(str, acceptable_version)), version_string))
def test_usearch_supported_version(self):
"""usearch is in path and version is supported """
acceptable_version = [(5,2,236),(5,2,236)]
self.assertTrue(app_path('usearch'),
"usearch not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "usearch --version"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.split('v')[1]
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported usearch version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def test_usearch_supported_version(self):
"""usearch is in path and version is supported """
acceptable_version = [(5,2,236),(5,2,236)]
self.assertTrue(app_path('usearch'),
"usearch not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "usearch --version"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.split('v')[1]
try:
version = tuple(map(int,version_string.split('.')))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported usearch version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def _error_on_missing_application(self,params):
""" Raise an ApplicationNotFoundError if the app is not accessible
This method checks in the system path (usually $PATH) or for
the existence of self._command. If self._command is not found
in either place, an ApplicationNotFoundError is raised to
inform the user that the application they are trying to access is
not available.
This method should be overwritten when self._command does not
represent the relevant executable (e.g., self._command = 'prog -a')
or in more complex cases where the file to be executed may be
passed as a parameter (e.g., with java jar files, where the
jar file is passed to java via '-jar'). It can also be overwritten
to by-pass testing for application presence by never raising an
error.
"""
command = self._command
if not (exists(command) or app_path(command)):
raise ApplicationNotFoundError("Cannot find %s. Is it installed? Is it in your path?"\
% command)
def _error_on_missing_application(self, params):
""" Raise an ApplicationNotFoundError if the app is not accessible
This method checks in the system path (usually $PATH) or for
the existence of self._command. If self._command is not found
in either place, an ApplicationNotFoundError is raised to
inform the user that the application they are trying to access is
not available.
This method should be overwritten when self._command does not
represent the relevant executable (e.g., self._command = 'prog -a')
or in more complex cases where the file to be executed may be
passed as a parameter (e.g., with java jar files, where the
jar file is passed to java via '-jar'). It can also be overwritten
to by-pass testing for application presence by never raising an
error.
"""
command = self._command
if not (exists(command) or app_path(command)):
raise ApplicationNotFoundError("Cannot find %s. Is it installed? Is it in your path?"\
% command)
def test_cluster_jobs_script(self):
"""cluster_jobs_fp is set to a good value"""
qiime_config = load_qiime_config()
submit_script = qiime_config['cluster_jobs_fp']
if (submit_script):
full_path = app_path(submit_script)
if full_path:
submit_script = full_path
self.assertTrue(
exists(submit_script),
"cluster_jobs_fp is not set to a valid path in qiime config: %s"
% submit_script)
#check if executable
self.assertTrue(
access(submit_script, X_OK),
"cluster_jobs_fp is not executable: %s" % submit_script)
else:
#Can't run in parallel, but not a critical error
pass
def submit_jobs(commands, prefix):
"""submit jobs using exe pointed to by cluster_jobs_fp.
commands: List of commands (strings) that should be executed
prefix: A uniq prefix used to name submit script
"""
qiime_config = load_qiime_config()
CLUSTER_JOBS_SCRIPT = qiime_config['cluster_jobs_fp']
if not CLUSTER_JOBS_SCRIPT:
raise ApplicationNotFoundError,"cluster_jobs_fp not set in config file!"
if not (exists(CLUSTER_JOBS_SCRIPT) or app_path(CLUSTER_JOBS_SCRIPT)):
raise ApplicationNotFoundError,"cluster_jobs_fp not in $PATH or provided as full path!"
outfilename = join(get_qiime_temp_dir(), "%s_commands.txt" % prefix)
fh = open(outfilename, "w")
fh.write("\n".join(commands))
fh.close()
cmd = '%s -ms %s %s'%(CLUSTER_JOBS_SCRIPT, outfilename, prefix)
system(cmd)
remove(outfilename)
def submit_jobs(commands, prefix):
"""submit jobs using exe pointed to by cluster_jobs_fp.
commands: List of commands (strings) that should be executed
prefix: A uniq prefix used to name submit script
"""
qiime_config = load_qiime_config()
CLUSTER_JOBS_SCRIPT = qiime_config['cluster_jobs_fp']
if not CLUSTER_JOBS_SCRIPT:
raise ApplicationNotFoundError, "cluster_jobs_fp not set in config file!"
if not (exists(CLUSTER_JOBS_SCRIPT) or app_path(CLUSTER_JOBS_SCRIPT)):
raise ApplicationNotFoundError, "cluster_jobs_fp not in $PATH or provided as full path!"
outfilename = join(get_qiime_temp_dir(), "%s_commands.txt" % prefix)
fh = open(outfilename, "w")
fh.write("\n".join(commands))
fh.close()
cmd = '%s -ms %s %s' % (CLUSTER_JOBS_SCRIPT, outfilename, prefix)
system(cmd)
remove(outfilename)
def test_R_supported_version(self):
"""R is in path and version is supported """
acceptable_version = [(2, 12, 0), (2, 12, 0)]
self.assertTrue(
app_path("R"),
"R not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.",
)
command = "R --version | grep 'R version' | awk '{print $3}'"
proc = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip()
try:
version = tuple(map(int, version_string.split(".")))
pass_test = version in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported R version. %s is required, but running %s."
% (".".join(map(str, acceptable_version)), version_string),
)
def test_ParsInsert_supported_version(self):
"""ParsInsert is in path and version is supported """
acceptable_version = ["1.04"]
self.assertTrue(app_path('ParsInsert'),
"ParsInsert not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
command = "ParsInsert -v | grep App | awk '{print $3}'"
proc = Popen(command,shell=True,universal_newlines=True,\
stdout=PIPE,stderr=STDOUT)
stdout = proc.stdout.read()
# remove log file generated
remove('ParsInsert.log')
version_string = stdout.strip()
try:
pass_test = version_string in acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported ParsInsert version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def test_cdbtools_supported_version(self):
"""cdbtools is in path and version is supported """
acceptable_version = (0, 99)
self.assertTrue(
app_path("cdbfasta"),
"cdbtools not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.",
)
command = "cdbfasta -v"
proc = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT)
stdout = proc.stdout.read()
version_string = stdout.strip().split(" ")[2].strip()
try:
version = tuple(map(int, version_string.split(".")))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported cdbtools version. %s is required, but running %s."
% (".".join(map(str, acceptable_version)), version_string),
)
def test_mothur_supported_version(self):
"""mothur is in path and version is supported """
acceptable_version = (1, 25, 0)
self.assertTrue(app_path('mothur'),
"mothur not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
# mothur creates a log file in cwd, so create a tmp and cd there first
log_file = join(get_qiime_temp_dir(), 'mothur.log')
command = "mothur \"#set.logfile(name=%s)\" | grep '^mothur v'" % log_file
stdout, stderr, exit_Status = qiime_system_call(command)
# remove log file
remove_files([log_file], error_on_missing=False)
version_string = stdout.strip().split(' ')[1].strip('v.')
try:
version = tuple(map(int, version_string.split('.')))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(pass_test,\
"Unsupported mothur version. %s is required, but running %s." \
% ('.'.join(map(str,acceptable_version)), version_string))
def test_cdhit_supported_version(self):
"""cd-hit is in path and version is supported """
self.assertTrue(app_path('cd-hit'),
"cd-hit not found. This may or may not be a problem depending on "+\
"which components of QIIME you plan to use.")
def assign_taxonomy(dataPath,
reference_sequences_fp,
id_to_taxonomy_fp,
read_1_seqs_fp,
read_2_seqs_fp,
single_ok=False,
no_single_ok_generic=False,
header_id_regex=None,
read_id_regex="\S+\s+(\S+)",
amplicon_id_regex="(\S+)\s+(\S+?)\/",
output_fp=None,
log_path=None,
HALT_EXEC=False,
base_tmp_dir='/tmp'):
"""Assign taxonomy to each sequence in data with the RTAX classifier
# data: open fasta file object or list of fasta lines
dataPath: path to a fasta file
output_fp: path to write output; if not provided, result will be
returned in a dict of {seq_id:(taxonomy_assignment,confidence)}
"""
usearch_command = "usearch"
if not (exists(usearch_command) or app_path(usearch_command)):
raise ApplicationNotFoundError,\
"Cannot find %s. Is it installed? Is it in your path?"\
% usearch_command
my_tmp_dir = get_tmp_filename(tmp_dir=base_tmp_dir,
prefix='rtax_',
suffix='',
result_constructor=str)
os.makedirs(my_tmp_dir)
try:
# RTAX classifier doesn't necessarily preserve identifiers
# it reports back only the id extracted as $1 using header_id_regex
# since rtax takes the original unclustered sequence files as input,
# the usual case is that the regex extracts the amplicon ID from the second field
# Use lookup table
read_1_id_to_orig_id = {}
readIdExtractor = re.compile(
read_id_regex) # OTU clustering produces ">clusterID read_1_id"
data = open(dataPath, 'r')
for seq_id, seq in MinimalFastaParser(data):
# apply the regex
extract = readIdExtractor.match(seq_id)
if extract is None:
stderr.write("Matched no ID with read_id_regex " +
read_id_regex + " in '" + seq_id +
"' from file " + dataPath + "\n")
else:
read_1_id_to_orig_id[extract.group(1)] = seq_id
#stderr.write(extract.group(1) + " => " + seq_id + "\n")
#seq_id_lookup[seq_id.split()[1]] = seq_id
data.close()
# make list of amplicon IDs to pass to RTAX
id_list_fp = open(my_tmp_dir + "/ampliconIdsToClassify", "w")
# Establish mapping of amplicon IDs to read_1 IDs
# simultaneously write the amplicon ID file for those IDs found in the input mapping above
amplicon_to_read_1_id = {}
ampliconIdExtractor = re.compile(
amplicon_id_regex
) # split_libraries produces >read_1_id ampliconID/1 ... // see also assign_taxonomy 631
read_1_data = open(read_1_seqs_fp, 'r')
for seq_id, seq in MinimalFastaParser(read_1_data):
# apply the regex
extract = ampliconIdExtractor.match(seq_id)
if extract is None:
stderr.write("Matched no ID with amplicon_id_regex " +
amplicon_id_regex + " in '" + seq_id +
"' from file " + read_1_seqs_fp + "\n")
else:
read_1_id = extract.group(1)
amplicon_id = extract.group(2)
try:
amplicon_to_read_1_id[amplicon_id] = read_1_id
bogus = read_1_id_to_orig_id[
read_1_id] # verify that the id is valid
id_list_fp.write('%s\n' % (amplicon_id))
except KeyError:
pass
data.close()
id_list_fp.close()
app = Rtax(HALT_EXEC=HALT_EXEC)
temp_output_file = tempfile.NamedTemporaryFile(
prefix='RtaxAssignments_', suffix='.txt')
app.Parameters['-o'].on(temp_output_file.name)
app.Parameters['-r'].on(reference_sequences_fp)
app.Parameters['-t'].on(id_to_taxonomy_fp)
# app.Parameters['-d'].on(delimiter)
app.Parameters['-l'].on(id_list_fp.name) # these are amplicon IDs
app.Parameters['-a'].on(read_1_seqs_fp)
if read_2_seqs_fp is not None:
app.Parameters['-b'].on(read_2_seqs_fp)
app.Parameters['-i'].on(header_id_regex)
app.Parameters['-m'].on(my_tmp_dir)
if single_ok: app.Parameters['-f'].on()
if no_single_ok_generic: app.Parameters['-g'].on()
#app.Parameters['-v'].on()
app_result = app()
if log_path:
f = open(log_path, 'a')
errString = ''.join(app_result['StdErr'].readlines()) + '\n'
f.write(errString)
f.close()
assignments = {}
# restore original sequence IDs with spaces
for line in app_result['Assignments']:
toks = line.strip().split('\t')
rtax_id = toks.pop(0)
if len(toks):
bestpcid = toks.pop(0) # ignored
lineage = toks
# RTAX does not provide a measure of confidence. We could pass one in,
# based on the choice of primers, or even look it up on the fly in the tables
# from the "optimal primers" paper; but it would be the same for every
# query sequence anyway.
# we could also return bestpcid, but that's not the same thing as confidence.
confidence = 1.0
read_1_id = amplicon_to_read_1_id[rtax_id]
orig_id = read_1_id_to_orig_id[read_1_id]
if lineage:
assignments[orig_id] = (';'.join(lineage), confidence)
else:
assignments[orig_id] = ('Unclassified', 1.0)
if output_fp:
try:
output_file = open(output_fp, 'w')
except OSError:
raise OSError("Can't open output file for writing: %s" %
output_fp)
for seq_id, assignment in assignments.items():
lineage, confidence = assignment
output_file.write('%s\t%s\t%1.3f\n' %
(seq_id, lineage, confidence))
output_file.close()
return None
else:
return assignments
finally:
try:
rmtree(my_tmp_dir)
except OSError:
pass
__credits__ = ["Jeremy Widmann", "Rob Knight"] __license__ = "GPL" __version__ = "1.5.3-dev" __maintainer__ = "Jeremy Widmann" __email__ = "*****@*****.**" __status__ = "Production" """Tests of the pdb_color module. Owner: Jeremy Widmann [email protected] Revision History: October 2006 Jeremy Widmann: File created """ MUSCLE_PATH = app_path('muscle') class PdbColorTests(TestCase): """Tests for pdb_color functions. """ def setUp(self): """Setup for pdb_color tests.""" #Nucleotide test data results self.test_pdb_chains_1 = { 'A': [(1, 'G'), (2, 'C'), (3, 'C'), (4, 'A'), (5, 'C'), (6, 'C'), (7, 'C'), (8, 'U'), (9, 'G')], 'B': [(10, 'C'), (11, 'A'), (12, 'G'), (13, 'G'), (14, 'G'), (15, 'U'), (16, 'C'), (17, 'G'), (18, 'G'), (19, 'C')] } self.ungapped_to_pdb_1 = {
def check_sfffile():
"""Raise error if sfffile is not in $PATH """
if not app_path('sfffile'):
raise ApplicationNotFoundError(_MISSING_APP_MESSAGE % 'sfffile')
from cogent.maths.stats.util import Freqs
from cogent.util.misc import app_path
from cogent.struct.rna2d import ViennaStructure
from cogent.util.unit_test import TestCase, main
__author__ = "Rob Knight"
__copyright__ = "Copyright 2007-2009, The Cogent Project"
__credits__ = ["Rob Knight", "Daniel McDonald"]
__license__ = "GPL"
__version__ = "1.4.1"
__maintainer__ = "Rob Knight"
__email__ = "*****@*****.**"
__status__ = "Development"
#need to skip some tests if RNAfold absent
if app_path('RNAfold'):
RNAFOLD_PRESENT = True
else:
RNAFOLD_PRESENT = False
class FunctionTests(TestCase):
"""Tests of standalone functions"""
def setUp(self):
self.standards = (0, 1, 5, 30, 173, 1000, 4382)
def test_permuations_negative_k(self):
"""permutations should raise IndexError if k negative"""
self.assertRaises(IndexError, permutations, 3, -1)
def test_permutations_k_more_than_n(self):
"""permutations should raise IndexError if k > n"""
""" Build the denoiser code binary """
cwd = getcwd()
denoiser_dir = join(cwd, 'qiime/support_files/denoiser/FlowgramAlignment')
chdir(denoiser_dir)
call(["make"])
chdir(cwd)
print "Denoiser built."
pycogent_version = tuple([int(v) \
for v in re.split("[^\d]", cogent.__version__) if v.isdigit()])
if pycogent_version < (1, 5):
print "PyCogent >= 1.5.0 required, but %s is installed." % cogent.__version__
exit(1)
if app_path("ghc"):
build_denoiser()
else:
print "GHC not installed, so cannot build the Denoiser binary."
setup(name='QIIME',
version=__version__,
description='Quantitative Insights Into Microbial Ecology',
author=__maintainer__,
author_email=__email__,
maintainer=__maintainer__,
maintainer_email=__email__,
url='http://www.qiime.org',
packages=[
'qiime', 'qiime/parallel', 'qiime/pycogent_backports',
'qiime/denoiser'
def _error_on_missing_application(self, params):
""" Raise an ApplicationNotFoundError if the app is not accessible
"""
if not app_path('blastall'):
raise ApplicationNotFoundError,\
"Cannot find blastall. Is it installed? Is it in your path?"
def suite():
modules_to_test = [
'test_recalculation.rst',
'test_phylo',
'test_dictarray.rst',
'test_align.test_align',
'test_align.test_algorithm',
'test_align.test_weights.test_methods',
'test_align.test_weights.test_util',
'test_app.test_parameters',
'test_app.test_util',
'test_cluster.test_goodness_of_fit',
'test_cluster.test_metric_scaling',
'test_cluster.test_procrustes',
'test_cluster.test_UPGMA',
'test_cluster.test_nmds',
'test_core.test_alphabet',
'test_core.test_alignment',
'test_core.test_annotation',
'test_core.test_bitvector',
'test_core.test_core_standalone',
'test_core.test_features.rst',
'test_core.test_entity',
'test_core.test_genetic_code',
'test_core.test_info',
'test_core.test_location',
'test_core.test_maps',
'test_core.test_moltype',
'test_core.test_profile',
'test_core.test_seq_aln_integration',
'test_core.test_sequence',
'test_core.test_tree',
'test_core.test_tree2',
'test_core.test_usage',
'test_data.test_molecular_weight',
'test_evolve.test_best_likelihood',
'test_evolve.test_bootstrap',
'test_evolve.test_coevolution',
'test_evolve.test_models',
'test_evolve.test_motifchange',
'test_evolve.test_substitution_model',
'test_evolve.test_scale_rules',
'test_evolve.test_likelihood_function',
'test_evolve.test_newq',
'test_evolve.test_parameter_controller',
'test_format.test_mage',
'test_format.test_fasta',
'test_format.test_pdb_color',
'test_format.test_xyzrn',
'test_maths.test_geometry',
'test_maths.test_matrix_logarithm',
'test_maths.test_matrix.test_distance',
'test_maths.test_spatial.test_ckd3',
'test_maths.test_stats.test_alpha_diversity',
'test_maths.test_stats.test_distribution',
'test_maths.test_stats.test_histogram',
'test_maths.test_stats.test_special',
'test_maths.test_stats.test_test',
'test_maths.test_stats.test_ks',
'test_maths.test_stats.test_rarefaction',
'test_maths.test_stats.test_util',
'test_maths.test_stats.test_cai.test_adaptor',
'test_maths.test_stats.test_cai.test_get_by_cai',
'test_maths.test_stats.test_cai.test_util',
'test_maths.test_optimisers',
'test_maths.test_distance_transform',
'test_maths.test_unifrac.test_fast_tree',
'test_maths.test_unifrac.test_fast_unifrac',
'test_motif.test_util',
'test_parse.test_aaindex',
'test_parse.test_agilent_microarray',
'test_parse.test_blast',
'test_parse.test_bpseq',
'test_parse.test_cigar',
'test_parse.test_clustal',
'test_parse.test_column',
'test_parse.test_comrna',
'test_parse.test_consan',
'test_parse.test_cove',
'test_parse.test_ct',
'test_parse.test_cut',
'test_parse.test_cutg',
'test_parse.test_dialign',
'test_parse.test_ebi',
'test_parse.test_fasta',
'test_parse.test_gibbs',
'test_parse.test_genbank',
'test_parse.test_gff',
'test_parse.test_ilm',
'test_parse.test_locuslink',
'test_parse.test_mage',
'test_parse.test_meme',
'test_parse.test_msms',
'test_parse.test_ncbi_taxonomy',
'test_parse.test_nexus',
'test_parse.test_nupack',
'test_parse.test_phylip',
'test_parse.test_pknotsrg',
'test_parse.test_rdb',
'test_parse.test_record',
'test_parse.test_record_finder',
'test_parse.test_rfam',
'test_parse.test_rnaalifold',
'test_parse.test_rna_fold',
'test_parse.test_rnaview',
'test_parse.test_rnaforester',
'test_parse.test_sprinzl',
'test_parse.test_tree',
'test_parse.test_unigene',
'test_seqsim.test_analysis',
'test_seqsim.test_birth_death',
'test_seqsim.test_markov',
'test_seqsim.test_microarray',
'test_seqsim.test_microarray_normalize',
'test_seqsim.test_randomization',
'test_seqsim.test_searchpath',
'test_seqsim.test_sequence_generators',
'test_seqsim.test_tree',
'test_seqsim.test_usage',
'test_struct.test_knots',
'test_struct.test_pairs_util',
'test_struct.test_rna2d',
'test_struct.test_asa',
'test_struct.test_contact',
'test_struct.test_annotation',
'test_struct.test_selection',
'test_struct.test_manipulation',
'test_util.test_unit_test',
'test_util.test_array',
'test_util.test_dict2d',
'test_util.test_misc',
'test_util.test_organizer',
'test_util.test_recode_alignment',
'test_util.test_table.rst',
'test_util.test_transform',
]
try:
import matplotlib
except:
print >> sys.stderr, "No matplotlib so not running test_draw.py"
else:
matplotlib.use('Agg') # non interactive
# test_draw not yet structured as unit tests
#modules_to_test.append('test_draw')
#Try importing modules for app controllers
apps = [('blastall', 'test_blast'),
('carnac', 'test_carnac'),
('clearcut', 'test_clearcut'),
('clustalw', 'test_clustalw'),
('cmfinder.pl', 'test_cmfinder'),
('comrna', 'test_comrna'),
('contrafold', 'test_contrafold'),
('covea', 'test_cove'),
('dialign2-2', 'test_dialign'),
('dynalign', 'test_dynalign'),
('FastTree', 'test_fasttree'),
('foldalign', 'test_foldalign'),
('ilm', 'test_ilm'),
('knetfold.pl', 'test_knetfold'),
('mafft', 'test_mafft'),
('mfold', 'test_mfold'),
('muscle', 'test_muscle'),
('msms', 'test_msms'),
('rdp_classifier-2.0.jar', 'test_rdp_classifier'),
('Fold.out', 'test_nupack'),
('findphyl', 'test_pfold'),
('pknotsRG-1.2-i386-linux-static', 'test_pknotsrg'),
('RNAalifold', 'test_rnaalifold'),
('rnaview', 'test_rnaview'),
('RNAfold', 'test_vienna_package'),
('raxmlHPC', 'test_raxml'),
('sfold.X86_64.LINUX', 'test_sfold'),
('stride', 'test_stride'),
('hybrid-ss-min', 'test_unafold'),
('cd-hit', 'test_cd_hit'),
('calculate_likelihood', 'test_gctmpca'),
('sfffile', 'test_sfffile'),
('sffinfo', 'test_sffinfo'),
('uclust','test_uclust')
]
for app, test_name in apps:
if app_path(app):
modules_to_test.append('test_app.' + test_name)
else:
print >> sys.stderr, "Can't find %s executable: skipping test" % app
if app_path('muscle'):
modules_to_test.append('test_format.test_pdb_color')
# we now toggle the db tests, based on an environment flag
if int(os.environ.get('TEST_DB', 0)):
db_tests = ['test_db.test_ncbi', 'test_db.test_pdb',
'test_db.test_rfam', 'test_db.test_util']
# we check for an environment flag for ENSEMBL
# we expect this to have the username and account for a localhost
# installation of the Ensembl MySQL databases
if 'ENSEMBL_ACCOUNT' in os.environ:
# check for cogent.db.ensembl dependencies
test_ensembl = True
for module in ['MySQLdb', 'sqlalchemy']:
if not module_present(module):
test_ensembl = False
print >> sys.stderr, \
"Module '%s' not present: skipping test" % module
if test_ensembl:
db_tests += ['test_db.test_ensembl.test_assembly',
'test_db.test_ensembl.test_database',
'test_db.test_ensembl.test_compara',
'test_db.test_ensembl.test_genome',
'test_db.test_ensembl.test_host',
'test_db.test_ensembl.test_species',
'test_db.test_ensembl.test_feature_level']
else:
print >> sys.stderr, "Environment variable ENSEMBL_ACCOUNT not "\
"set: skipping db.ensembl tests"
for db_test in db_tests:
modules_to_test.append(db_test)
else:
print >> sys.stderr, \
"Environment variable TEST_DB=1 not set: skipping db tests"
assert sys.version_info >= (2, 4)
alltests = unittest.TestSuite()
for module in modules_to_test:
if module.endswith('.rst'):
module = os.path.join(*module.split(".")[:-1]) + ".rst"
test = doctest.DocFileSuite(module, optionflags=
doctest.REPORT_ONLY_FIRST_FAILURE |
doctest.ELLIPSIS)
else:
test = unittest.findTestCases(my_import(module))
alltests.addTest(test)
return alltests
