def set_sff_trimpoints_with_sfftools(
sff_dir, technical_lengths, sffinfo_path='sffinfo', sfffile_path='sfffile',
debug=False):
"""Set trimpoints to end of technical read for all SFF files in directory.
This function essentially provides the reference implementation.
It uses the official sfftools from Roche to process the SFF files.
"""
if not (exists(sffinfo_path) or which(sffinfo_path)):
raise ApplicationNotFoundError(
'sffinfo executable not found. Is it installed and in your $PATH?')
if not (exists(sfffile_path) or which(sfffile_path)):
raise ApplicationNotFoundError(
'sfffile executable not found. Is it installed and in your $PATH?')
for lib_id, sff_fp in get_per_lib_sff_fps(sff_dir):
try:
readlength = technical_lengths[lib_id]
except KeyError:
continue
sffinfo_args = [sffinfo_path, '-s', sff_fp]
if debug:
print "Running sffinfo command %s" % sffinfo_args
sffinfo_output_file = TemporaryFile()
check_call(sffinfo_args, stdout=sffinfo_output_file)
sffinfo_output_file.seek(0)
seqlengths = {}
for line in sffinfo_output_file:
if line.startswith('>'):
fields = line[1:].split()
seq_len = fields[1].split('=')[1]
seqlengths[fields[0]] = seq_len
trim_fp = sff_fp + '.trim'
trim_file = open(trim_fp, 'w')
for id_, length in seqlengths.items():
curr_length = int(seqlengths[id_])
# Sfftools use 1-based index
left_trim = readlength + 1
# Key sequence not included in FASTA length
right_trim = curr_length + 4
if curr_length > left_trim:
trim_file.write(
"%s\t%s\t%s\n" % (id_, left_trim, right_trim))
else:
stderr.write(
'Rejected read %s with trim points %s and %s (orig '
'length %s)' % (id_, left_trim, curr_length, length))
trim_file.close()
trimmed_sff_fp = sff_fp + '.trimmed'
sfffile_args = [
sfffile_path, '-t', trim_fp, '-o', trimmed_sff_fp, sff_fp]
if debug:
print "Running sfffile command:", sfffile_args
check_call(sfffile_args, stdout=open(devnull, 'w'))
remove(sff_fp)
rename(trimmed_sff_fp, sff_fp)
def check_flowgram_ali_exe():
"""Check if we have a working FlowgramAligner"""
ali_exe = get_flowgram_ali_exe()
if which(ali_exe) is None:
raise ApplicationNotFoundError("The alignment program %s is not "
"accessible via the PATH environment "
"variable." % ali_exe)
# test if its callable and actually works
command = "%s -h" % ali_exe
proc = Popen(command,
shell=True,
universal_newlines=True,
stdout=PIPE,
stderr=STDOUT)
if (proc.wait() != 0):
raise ApplicationError(
"Calling %s failed. Check permissions and that it is in fact an executable."
% ali_exe)
result = proc.stdout.read()
# check that the help string looks correct
if (not result.startswith("Usage")):
raise ApplicationError(
"Calling %s failed. Check permissions and that it is in fact an executable."
% ali_exe)
return True
def raise_gdata_not_found_error(*args, **kwargs):
raise ApplicationNotFoundError(
"gdata cannot be found.\nIs it installed? "
"Is it in your $PYTHONPATH?\nThis is an optional QIIME "
"dependency, but is required if you plan to use QIIME's remote "
"mapping file features. For more information, please see "
"http://qiime.org/install/install.html.")
def test_blastall_fp(self):
"""blastall_fp is set to a valid path"""
blastall = self.config["blastall_fp"]
if not self.config["blastall_fp"].startswith("/"):
# path is relative, figure out absolute path
blast_all = which(blastall)
if not blast_all:
raise ApplicationNotFoundError(
"blastall_fp set to %s, but is not in your PATH. Either use an absolute path to or put it in your PATH."
% blastall)
self.config["blastall_fp"] = blast_all
test_qiime_config_variable("blastall_fp", self.config, self, X_OK)
def submit_jobs(commands, prefix):
"""submit jobs using exe pointed to by cluster_jobs_fp.
commands: List of commands (strings) that should be executed
prefix: A uniq prefix used to name submit script
"""
qiime_config = load_qiime_config()
CLUSTER_JOBS_SCRIPT = qiime_config['cluster_jobs_fp']
if not CLUSTER_JOBS_SCRIPT:
raise ApplicationNotFoundError(
"cluster_jobs_fp not set in config file!")
if not (exists(CLUSTER_JOBS_SCRIPT) or which(CLUSTER_JOBS_SCRIPT)):
raise ApplicationNotFoundError(
"cluster_jobs_fp not in $PATH or provided as full path!")
outfilename = join(get_qiime_temp_dir(), "%s_commands.txt" % prefix)
fh = open(outfilename, "w")
fh.write("\n".join(commands))
fh.close()
cmd = '%s -ms %s %s' % (CLUSTER_JOBS_SCRIPT, outfilename, prefix)
system(cmd)
remove(outfilename)
def submit_jobs(filenames, verbose=False):
"""Submit jobs in filenames.
filenames: list of prepared qsub job scripts, ready to be submitted
verbose: a binary verbose flag
"""
if not which("qsub"):
raise ApplicationNotFoundError("qsub not found. Can't submit jobs.")
for file in filenames:
command = 'qsub %s' % file
result = Popen(command, shell=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT).stdout.read()
if verbose:
print result
def wait_for_cluster_ids(ids, interval=10):
"""Puts process to sleep until jobs with ids are done.
ids: list of ids to wait for
interval: time to sleep in seconds
NOT USED ANYMORE
"""
if which("qstat"):
for id in ids:
while(getoutput("qstat %s" % id).startswith("Job")):
sleep(interval)
else:
raise ApplicationNotFoundError("qstat not available. Is it installed?\n" +
"This test may fail if not run on a cluster.")
def raise_pynast_not_found_error(*args, **kwargs):
raise ApplicationNotFoundError("PyNAST cannot be found.\nIs PyNAST installed? Is it in your $PYTHONPATH?" +
"\nYou can obtain PyNAST from http://qiime.org/pynast/.")
def check_sfffile():
"""Raise error if sfffile is not in $PATH """
if not which('sfffile'):
raise ApplicationNotFoundError(_MISSING_APP_MESSAGE % 'sfffile')