def test_ampliconnoise_install(self): """ AmpliconNoise install looks sane.""" url="http://qiime.org/install/install.html#ampliconnoise-install-notes" pyro_lookup_file = getenv('PYRO_LOOKUP_FILE') self.assertTrue(pyro_lookup_file != None, "$PYRO_LOOKUP_FILE variable is not set. See %s for help." % url) self.assertTrue(exists(pyro_lookup_file), "$PYRO_LOOKUP_FILE variable is not set to an existing filepath.") seq_lookup_file = getenv('SEQ_LOOKUP_FILE') self.assertTrue(seq_lookup_file != None, "$SEQ_LOOKUP_FILE variable is not set. See %s for help." % url) self.assertTrue(exists(seq_lookup_file), "$SEQ_LOOKUP_FILE variable is not set to an existing filepath.") self.assertTrue(app_path("SplitKeys.pl"), "Couldn't find SplitKeys.pl. "+\ "Perhaps AmpliconNoise Scripts directory isn't in $PATH?"+\ " See %s for help." % url) self.assertTrue(app_path("FCluster"), "Couldn't find FCluster. "+\ "Perhaps the AmpliconNoise bin directory isn't in $PATH?"+\ " See %s for help." % url) self.assertTrue(app_path("Perseus"), "Couldn't find Perseus. "+\ "Perhaps the AmpliconNoise bin directory isn't in $PATH?"+\ " See %s for help." % url)
def set_sff_trimpoints_with_sfftools( sff_dir, technical_lengths, sffinfo_path='sffinfo', sfffile_path='sfffile', debug=False): """Set trimpoints to end of technical read for all SFF files in directory. This function essentially provides the reference implementation. It uses the official sfftools from Roche to process the SFF files. """ if not (os.path.exists(sffinfo_path) or app_path(sffinfo_path)): raise ApplicationNotFoundError( 'sffinfo executable not found. Is it installed and in your $PATH?') if not (os.path.exists(sfffile_path) or app_path(sfffile_path)): raise ApplicationNotFoundError( 'sfffile executable not found. Is it installed and in your $PATH?') for lib_id, sff_fp in get_per_lib_sff_fps(sff_dir): try: readlength = technical_lengths[lib_id] except KeyError: continue sffinfo_args = [sffinfo_path, '-s', sff_fp] if debug: print "Running sffinfo command %s" % sffinfo_args sffinfo_output_file = tempfile.TemporaryFile() subprocess.check_call(sffinfo_args, stdout=sffinfo_output_file) sffinfo_output_file.seek(0) seqlengths = {} for line in sffinfo_output_file: if line.startswith('>'): fields = line[1:].split() seq_len = fields[1].split('=')[1] seqlengths[fields[0]] = seq_len trim_fp = sff_fp + '.trim' trim_file = open(trim_fp, 'w') for id_, length in seqlengths.items(): curr_length = int(seqlengths[id_]) # Sfftools use 1-based index left_trim = readlength + 1 # Key sequence not included in FASTA length right_trim = curr_length + 4 if curr_length > left_trim: trim_file.write( "%s\t%s\t%s\n" % (id_, left_trim, right_trim)) else: sys.stderr.write( 'Rejected read %s with trim points %s and %s (orig ' 'length %s)' % (id_, left_trim, curr_length, length)) trim_file.close() trimmed_sff_fp = sff_fp + '.trimmed' sfffile_args = [ sfffile_path, '-t', trim_fp, '-o', trimmed_sff_fp, sff_fp] if debug: print "Running sfffile command:", sfffile_args subprocess.check_call(sfffile_args, stdout=open(os.devnull, 'w')) os.remove(sff_fp) os.rename(trimmed_sff_fp, sff_fp)
def test_ampliconnoise_install(self): """ AmpliconNoise install looks sane.""" url = "http://www.qiime.org/install/install.html#ampliconnoise-install" pyro_lookup_file = getenv('PYRO_LOOKUP_FILE') self.assertTrue(pyro_lookup_file != None, "$PYRO_LOOKUP_FILE variable is not set. See %s for help." % url) self.assertTrue(exists(pyro_lookup_file), "$PYRO_LOOKUP_FILE variable is not set to an existing filepath.") seq_lookup_file = getenv('SEQ_LOOKUP_FILE') self.assertTrue(seq_lookup_file != None, "$SEQ_LOOKUP_FILE variable is not set. See %s for help." % url) self.assertTrue(exists(seq_lookup_file), "$SEQ_LOOKUP_FILE variable is not set to an existing filepath.") self.assertTrue(app_path("SplitKeys.pl"), "Couldn't find SplitKeys.pl. "+\ "Perhaps AmpliconNoise Scripts directory isn't in $PATH?"+\ " See %s for help." % url) self.assertTrue(app_path("FCluster"), "Couldn't find FCluster. "+\ "Perhaps the AmpliconNoise bin directory isn't in $PATH?"+\ " See %s for help." % url) self.assertTrue(app_path("Perseus"), "Couldn't find Perseus. "+\ "Perhaps the AmpliconNoise bin directory isn't in $PATH?"+\ " See %s for help." % url)
def _error_on_missing_application(self,params): """ Raise an ApplicationNotFoundError if the app is not accessible """ if not app_path('RNAforester'): raise ApplicationNotFoundError("Cannot find RNAforester. Is it installed? Is it in your path?") if not app_path('RNAshapes'): raise ApplicationNotFoundError("Cannot find RNAshapes. Is it installed? Is it in your path?")
def _error_on_missing_application(self, params): """ Raise an ApplicationNotFoundError if the app is not accessible """ if not app_path('RNAforester'): raise ApplicationNotFoundError,\ "Cannot find RNAforester. Is it installed? Is it in your path?" if not app_path('RNAshapes'): raise ApplicationNotFoundError,\ "Cannot find RNAshapes. Is it installed? Is it in your path?"
def test_mothur_supported_version(self): """mothur is in path and version is supported """ acceptable_version = (1,25,0) self.assertTrue(app_path('mothur'), "mothur not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.") # mothur creates a log file in cwd, so create a tmp and cd there first command = "mothur \"#set.logfile(name=mothur.log)\" | grep '^mothur v'" proc = Popen(command,shell=True,universal_newlines=True,\ stdout=PIPE,stderr=STDOUT) stdout, stderr = proc.communicate() # remove log file remove('mothur.log') version_string = stdout.strip().split(' ')[1].strip('v.') try: version = tuple(map(int,version_string.split('.'))) pass_test = version == acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test,\ "Unsupported mothur version. %s is required, but running %s." \ % ('.'.join(map(str,acceptable_version)), version_string))
def setUp(self): """Check if Raxml version is supported for this test""" acceptable_version = (7, 3, 0) self.assertTrue(app_path('raxmlHPC'), "raxmlHPC not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.") command = "raxmlHPC -v | grep version" proc = Popen(command,shell=True,universal_newlines=True,\ stdout=PIPE,stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.strip().split(' ')[4].strip() try: version = tuple(map(int, version_string.split('.'))) pass_test = version == acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test,\ "Unsupported raxmlHPC version. %s is required, but running %s." \ % ('.'.join(map(str,acceptable_version)), version_string)) """Setup data for raxml tests""" self.seqs1 = [ 'ACUGCUAGCUAGUAGCGUACGUA', 'GCUACGUAGCUAC', 'GCGGCUAUUAGAUCGUA' ] self.labels1 = ['>1', '>2', '>3'] self.lines1 = flatten(zip(self.labels1, self.seqs1)) self.test_model = "GTRCAT" self.align1 = get_align_for_phylip(StringIO(PHYLIP_FILE)) self.test_fn1 = "/tmp/raxml_test1.txt" self.test_fn2 = "raxml_test1.txt" self.test_fn1_space = "/tmp/raxml test1.txt"
def test_mothur_supported_version(self): """mothur is in path and version is supported """ acceptable_version = (1, 25, 0) self.assertTrue( app_path("mothur"), "mothur not found. This may or may not be a problem depending on " + "which components of QIIME you plan to use.", ) # mothur creates a log file in cwd, so create a tmp and cd there first log_file = join(get_qiime_temp_dir(), "mothur.log") command = "mothur \"#set.logfile(name=%s)\" | grep '^mothur v'" % log_file stdout, stderr, exit_Status = qiime_system_call(command) # remove log file remove_files([log_file], error_on_missing=False) version_string = stdout.strip().split(" ")[1].strip("v.") try: version = tuple(map(int, version_string.split("."))) pass_test = version == acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue( pass_test, "Unsupported mothur version. %s is required, but running %s." % (".".join(map(str, acceptable_version)), version_string), )
def test_ParsInsert_supported_version(self): """ParsInsert is in path and version is supported """ acceptable_version = ["1.04"] self.assertTrue( app_path("ParsInsert"), "ParsInsert not found. This may or may not be a problem depending on " + "which components of QIIME you plan to use.", ) command = "ParsInsert -v | grep App | awk '{print $3}'" proc = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT) stdout = proc.stdout.read() # remove log file generated remove("ParsInsert.log") version_string = stdout.strip() try: pass_test = version_string in acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue( pass_test, "Unsupported ParsInsert version. %s is required, but running %s." % (".".join(map(str, acceptable_version)), version_string), )
def test_R_supported_version(self): """R is in path and version is supported """ minimum_version = (2,12,0) self.assertTrue(app_path('R'), "R not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.") command = "R --version | grep 'R version' | awk '{print $3}'" proc = Popen(command,shell=True,universal_newlines=True,\ stdout=PIPE,stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.strip() try: version = tuple(map(int,version_string.split('.'))) pass_test = False if version[0] == minimum_version[0]: if version[1] == minimum_version[1]: if version[2] >= minimum_version[2]: pass_test = True elif version[1] > minimum_version[1]: pass_test = True elif version[0] > minimum_version[0]: pass_test = True except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test,\ "Unsupported R version. %s or greater is required, but running %s." \ % ('.'.join(map(str,minimum_version)), version_string))
def test_cdhit_supported_version(self): """cd-hit is in path and version is supported """ self.assertTrue( app_path("cd-hit"), "cd-hit not found. This may or may not be a problem depending on " + "which components of QIIME you plan to use.", )
def setUp(self): """Check if Raxml version is supported for this test""" acceptable_version = (7,3,0) self.assertTrue(app_path('raxmlHPC'), "raxmlHPC not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.") command = "raxmlHPC -v | grep version" proc = Popen(command,shell=True,universal_newlines=True,\ stdout=PIPE,stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.strip().split(' ')[4].strip() try: version = tuple(map(int,version_string.split('.'))) pass_test = version == acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test,\ "Unsupported raxmlHPC version. %s is required, but running %s." \ % ('.'.join(map(str,acceptable_version)), version_string)) """Setup data for raxml tests""" self.seqs1 = ['ACUGCUAGCUAGUAGCGUACGUA','GCUACGUAGCUAC', 'GCGGCUAUUAGAUCGUA'] self.labels1 = ['>1','>2','>3'] self.lines1 = flatten(zip(self.labels1,self.seqs1)) self.test_model = "GTRCAT" self.align1 = get_align_for_phylip(StringIO(PHYLIP_FILE)) self.test_fn1 = "/tmp/raxml_test1.txt" self.test_fn2 = "raxml_test1.txt" self.test_fn1_space = "/tmp/raxml test1.txt"
def test_mothur_supported_version(self): """mothur is in path and version is supported """ acceptable_version = (1,6,0) self.assertTrue(app_path('mothur'), "mothur not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.") # mothur creates a log file in cwd, so create a tmp and cd there first tmp_dir = get_random_directory_name(output_dir='/tmp/') command = "cd %s ; mothur -v | grep ^mothur" % tmp_dir proc = Popen(command,shell=True,universal_newlines=True,\ stdout=PIPE,stderr=STDOUT) stdout, stderr = proc.communicate() version_string = stdout.strip().split(' ')[1].strip('v.') try: version = tuple(map(int,version_string.split('.'))) pass_test = version == acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test,\ "Unsupported mothur version. %s is required, but running %s." \ % ('.'.join(map(str,acceptable_version)), version_string)) # remove the directory and the log file rmtree(tmp_dir)
def submit_jobs(filenames, verbose=False): """Submit jobs in filenames.""" if(not app_path("qsub")): raise ApplicationNotFoundError,"qsub not found. Can't submit jobs." for file in filenames: command = 'qsub -u wwwuser %s' % (file) result = Popen(command, shell=True, universal_newlines=True,\ stdout=PIPE, stderr=STDOUT).stdout.read() if verbose: print result
def test_blastall_fp(self): """blastall_fp is set to a valid path""" blastall = self.config["blastall_fp"] if not self.config["blastall_fp"].startswith("/"): #path is relative, figure out absolute path blast_all = app_path(blastall) if not blast_all: raise ApplicationNotFoundError("blastall_fp set to %s, but is not in your PATH. Either use an absolute path to or put it in your PATH." % blastall) self.config["blastall_fp"] = blast_all test_qiime_config_variable("blastall_fp", self.config, self, X_OK)
def test_cluster_jobs_fp(self): """cluster_jobs_fp is set to a valid path and is executable""" fp = self.config["cluster_jobs_fp"] if fp: full_path = app_path(fp) if full_path: fp = full_path # test if file exists or is in $PATH self.assertTrue(exists(fp), "cluster_jobs_fp set to an invalid file path or is not in $PATH: %s" % fp) modes = {R_OK: "readable", W_OK: "writable", X_OK: "executable"} # test if file readable self.assertTrue(access(fp, X_OK), "cluster_jobs_fp is not %s: %s" % (modes[X_OK], fp))
def wait_for_cluster_ids(ids, interval=10): """Puts process to sleep until jobs with ids are done. ids: list of ids to wait for interval: time to sleep in seconds NOT USED ANYMORE """ if app_path("qstat"): for id in ids: while getoutput("qstat %s" % id).startswith("Job"): sleep(interval) else: raise ApplicationNotFoundError, "qstat not available. Is it installed?\n" + "This test may fail if not run on a cluster."
def test_chimeraSlayer_install(self): """no obvious problems with ChimeraSlayer install """ #The ChimerSalyer app requires that all its components are installed # relative to the main program ChimeraSlayer.pl. # We therefore check that at least one the files is there. # However, if the directory structure of ChimeraSlayer changes, this test will most # likely fail as well and need to be updated. # Tested with the version of microbiomeutil_2010-04-29 chim_slay = app_path("ChimeraSlayer.pl") self.assertTrue(chim_slay,"ChimeraSlayer was not found in your $PATH") dir, app_name = split(chim_slay) self.assertTrue(exists(dir+"/ChimeraParentSelector/chimeraParentSelector.pl"), "ChimeraSlayer depends on external files in directoryies relative to its " "install directory. Thesedo not appear to be present.")
def submit_jobs(filenames, verbose=False): """Submit jobs in filenames. filenames: list of prepared qsub job scripts, ready to be submitted verbose: a binary verbose flag """ if(not app_path("qsub")): raise ApplicationNotFoundError,"qsub not found. Can't submit jobs." for file in filenames: command = 'qsub %s' % file result = Popen(command, shell=True, universal_newlines=True,\ stdout=PIPE, stderr=STDOUT).stdout.read() if verbose: print result
def wait_for_cluster_ids(ids, interval=10): """Puts process to sleep until jobs with ids are done. ids: list of ids to wait for interval: time to sleep in seconds NOT USED ANYMORE """ if (app_path("qstat")): for id in ids: while (getoutput("qstat %s" % id).startswith("Job")): sleep(interval) else: raise ApplicationNotFoundError,"qstat not available. Is it installed?\n"+\ "This test may fail if not run on a cluster."
def make_cidx_file(fp): """make a CS index file. fp: reference DB to make an index of ChimeraSlayer implicetely performs this task when the file is not already created and deletes it when done. Especially in a parallel environment it mkaes sense to manually make and delete the file. """ if app_path("cdbfasta"): #cdbfasta comes with the microbiometools/CS #should always be installed when using CS args = ["cdbfasta", fp] #cdbfasta write one line to stderr stdout, stderr = Popen(args, stderr=PIPE, stdout=PIPE).communicate() else: raise ApplicationNotFoundError
def test_cluster_jobs_script(self): """cluster_jobs_fp is set to a good value""" qiime_config = load_qiime_config() submit_script = qiime_config['cluster_jobs_fp'] if (submit_script): full_path = app_path(submit_script) if full_path: submit_script = full_path self.assertTrue(exists(submit_script), "cluster_jobs_fp is not set to a valid path in qiime config: %s" % submit_script) #check if executable self.assertTrue(access(submit_script, X_OK), "cluster_jobs_fp is not executable: %s" % submit_script) else: #Can't run in parallel, but not a critical error pass
def _error_on_missing_application(self,params): """Raise an ApplicationNotFoundError if the app is not accessible In this case, checks for the java runtime and the RDP jar file. """ if not (os.path.exists('java') or app_path('java')): raise ApplicationNotFoundError( "Cannot find java runtime. Is it installed? Is it in your " "path?") jar_fp = self._get_jar_fp() if jar_fp is None: raise ApplicationNotFoundError( "JAR file not found in current directory and the RDP_JAR_PATH " "environment variable is not set. Please set RDP_JAR_PATH to " "the full pathname of the JAR file.") if not os.path.exists(jar_fp): raise ApplicationNotFoundError( "JAR file %s does not exist." % jar_fp)
def test_raxmlHPC_supported_version(self): """raxmlHPC is in path and version is supported """ acceptable_version = [(7,3,0),(7,3,0)] self.assertTrue(app_path('raxmlHPC'), "raxmlHPC not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.") command = "raxmlHPC -v | grep version" proc = Popen(command,shell=True,universal_newlines=True,\ stdout=PIPE,stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.strip().split(' ')[4].strip() try: version = tuple(map(int,version_string.split('.'))) pass_test = version in acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test,\ "Unsupported raxmlHPC version. %s is required, but running %s." \ % ('.'.join(map(str,acceptable_version)), version_string))
def test_cluster_jobs_fp(self): """cluster_jobs_fp is set to a valid path and is executable""" fp = self.config["cluster_jobs_fp"] if fp: full_path = app_path(fp) if full_path: fp = full_path #test if file exists or is in $PATH self.assertTrue( exists(fp), "cluster_jobs_fp set to an invalid file path or is not in $PATH: %s" % fp) modes = {R_OK: "readable", W_OK: "writable", X_OK: "executable"} #test if file readable self.assertTrue( access(fp, X_OK), "cluster_jobs_fp is not %s: %s" % (modes[X_OK], fp))
def test_pplacer_supported_version(self): """pplacer is in path and version is supported """ acceptable_version = [(1, 1), (1, 1)] self.assertTrue(app_path('pplacer'), "pplacer not found. This may or may not be a problem depending on " + "which components of QIIME you plan to use.") command = "pplacer --version" proc = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.strip()[1:4] try: version = tuple(map(int, version_string.split('.'))) pass_test = version in acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test, "Unsupported pplacer version. %s is required, but running %s." % ('.'.join(map(str, acceptable_version)), version_string))
def test_clearcut_supported_version(self): """clearcut is in path and version is supported """ acceptable_version = (1,0,9) self.assertTrue(app_path('clearcut'), "clearcut not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.") command = "clearcut -V" proc = Popen(command,shell=True,universal_newlines=True,\ stdout=PIPE,stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.strip().split(' ')[2].strip() try: version = tuple(map(int,version_string.split('.'))) pass_test = version == acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test,\ "Unsupported clearcut version. %s is required, but running %s." \ % ('.'.join(map(str,acceptable_version)), version_string))
def test_INFERNAL_supported_version(self): """INFERNAL is in path and version is supported """ acceptable_version = (1, 0, 2) self.assertTrue(app_path('cmbuild'), "Infernal not found. This may or may not be a problem depending on " + "which components of QIIME you plan to use.") command = "cmbuild -h | grep INF" proc = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.strip().split(' ')[2].strip() try: version = tuple(map(int, version_string.split('.'))) pass_test = version == acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test, "Unsupported INFERNAL version. %s is required, but running %s." % ('.'.join(map(str, acceptable_version)), version_string))
def test_rtax_supported_version(self): """rtax is in path and version is supported """ acceptable_version = [(0,982),(0,983)] self.assertTrue(app_path('rtax'), "rtax not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.") command = "rtax 2>&1 > %s | grep Version | awk '{print $2}'" % devnull proc = Popen(command,shell=True,universal_newlines=True,\ stdout=PIPE,stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.strip() try: version = tuple(map(int,version_string.split('.'))) pass_test = version in acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test,\ "Unsupported rtax version. %s is required, but running %s." \ % ('.'.join(map(str,acceptable_version)), version_string))
def test_muscle_supported_version(self): """muscle is in path and version is supported """ acceptable_version = (3, 8, 31) self.assertTrue(app_path('muscle'), "muscle not found. This may or may not be a problem depending on " + "which components of QIIME you plan to use.") command = "muscle -version" proc = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.strip().split(' ')[1].strip('v') try: version = tuple(map(int, version_string.split('.'))) pass_test = version == acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test, "Unsupported muscle version. %s is required, but running %s." % ('.'.join(map(str, acceptable_version)), version_string))
def test_usearch_supported_version(self): """usearch is in path and version is supported """ acceptable_version = [(5,2,236),(5,2,236)] self.assertTrue(app_path('usearch'), "usearch not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.") command = "usearch --version" proc = Popen(command,shell=True,universal_newlines=True,\ stdout=PIPE,stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.split('v')[1] try: version = tuple(map(int,version_string.split('.'))) pass_test = version in acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test,\ "Unsupported usearch version. %s is required, but running %s." \ % ('.'.join(map(str,acceptable_version)), version_string))
def _error_on_missing_application(self,params): """ Raise an ApplicationNotFoundError if the app is not accessible This method checks in the system path (usually $PATH) or for the existence of self._command. If self._command is not found in either place, an ApplicationNotFoundError is raised to inform the user that the application they are trying to access is not available. This method should be overwritten when self._command does not represent the relevant executable (e.g., self._command = 'prog -a') or in more complex cases where the file to be executed may be passed as a parameter (e.g., with java jar files, where the jar file is passed to java via '-jar'). It can also be overwritten to by-pass testing for application presence by never raising an error. """ command = self._command if not (exists(command) or app_path(command)): raise ApplicationNotFoundError("Cannot find %s. Is it installed? Is it in your path?"\ % command)
def _error_on_missing_application(self, params): """ Raise an ApplicationNotFoundError if the app is not accessible This method checks in the system path (usually $PATH) or for the existence of self._command. If self._command is not found in either place, an ApplicationNotFoundError is raised to inform the user that the application they are trying to access is not available. This method should be overwritten when self._command does not represent the relevant executable (e.g., self._command = 'prog -a') or in more complex cases where the file to be executed may be passed as a parameter (e.g., with java jar files, where the jar file is passed to java via '-jar'). It can also be overwritten to by-pass testing for application presence by never raising an error. """ command = self._command if not (exists(command) or app_path(command)): raise ApplicationNotFoundError("Cannot find %s. Is it installed? Is it in your path?"\ % command)
def test_cluster_jobs_script(self): """cluster_jobs_fp is set to a good value""" qiime_config = load_qiime_config() submit_script = qiime_config['cluster_jobs_fp'] if (submit_script): full_path = app_path(submit_script) if full_path: submit_script = full_path self.assertTrue( exists(submit_script), "cluster_jobs_fp is not set to a valid path in qiime config: %s" % submit_script) #check if executable self.assertTrue( access(submit_script, X_OK), "cluster_jobs_fp is not executable: %s" % submit_script) else: #Can't run in parallel, but not a critical error pass
def submit_jobs(commands, prefix): """submit jobs using exe pointed to by cluster_jobs_fp. commands: List of commands (strings) that should be executed prefix: A uniq prefix used to name submit script """ qiime_config = load_qiime_config() CLUSTER_JOBS_SCRIPT = qiime_config['cluster_jobs_fp'] if not CLUSTER_JOBS_SCRIPT: raise ApplicationNotFoundError,"cluster_jobs_fp not set in config file!" if not (exists(CLUSTER_JOBS_SCRIPT) or app_path(CLUSTER_JOBS_SCRIPT)): raise ApplicationNotFoundError,"cluster_jobs_fp not in $PATH or provided as full path!" outfilename = join(get_qiime_temp_dir(), "%s_commands.txt" % prefix) fh = open(outfilename, "w") fh.write("\n".join(commands)) fh.close() cmd = '%s -ms %s %s'%(CLUSTER_JOBS_SCRIPT, outfilename, prefix) system(cmd) remove(outfilename)
def submit_jobs(commands, prefix): """submit jobs using exe pointed to by cluster_jobs_fp. commands: List of commands (strings) that should be executed prefix: A uniq prefix used to name submit script """ qiime_config = load_qiime_config() CLUSTER_JOBS_SCRIPT = qiime_config['cluster_jobs_fp'] if not CLUSTER_JOBS_SCRIPT: raise ApplicationNotFoundError, "cluster_jobs_fp not set in config file!" if not (exists(CLUSTER_JOBS_SCRIPT) or app_path(CLUSTER_JOBS_SCRIPT)): raise ApplicationNotFoundError, "cluster_jobs_fp not in $PATH or provided as full path!" outfilename = join(get_qiime_temp_dir(), "%s_commands.txt" % prefix) fh = open(outfilename, "w") fh.write("\n".join(commands)) fh.close() cmd = '%s -ms %s %s' % (CLUSTER_JOBS_SCRIPT, outfilename, prefix) system(cmd) remove(outfilename)
def test_R_supported_version(self): """R is in path and version is supported """ acceptable_version = [(2, 12, 0), (2, 12, 0)] self.assertTrue( app_path("R"), "R not found. This may or may not be a problem depending on " + "which components of QIIME you plan to use.", ) command = "R --version | grep 'R version' | awk '{print $3}'" proc = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.strip() try: version = tuple(map(int, version_string.split("."))) pass_test = version in acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue( pass_test, "Unsupported R version. %s is required, but running %s." % (".".join(map(str, acceptable_version)), version_string), )
def test_ParsInsert_supported_version(self): """ParsInsert is in path and version is supported """ acceptable_version = ["1.04"] self.assertTrue(app_path('ParsInsert'), "ParsInsert not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.") command = "ParsInsert -v | grep App | awk '{print $3}'" proc = Popen(command,shell=True,universal_newlines=True,\ stdout=PIPE,stderr=STDOUT) stdout = proc.stdout.read() # remove log file generated remove('ParsInsert.log') version_string = stdout.strip() try: pass_test = version_string in acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test,\ "Unsupported ParsInsert version. %s is required, but running %s." \ % ('.'.join(map(str,acceptable_version)), version_string))
def test_cdbtools_supported_version(self): """cdbtools is in path and version is supported """ acceptable_version = (0, 99) self.assertTrue( app_path("cdbfasta"), "cdbtools not found. This may or may not be a problem depending on " + "which components of QIIME you plan to use.", ) command = "cdbfasta -v" proc = Popen(command, shell=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT) stdout = proc.stdout.read() version_string = stdout.strip().split(" ")[2].strip() try: version = tuple(map(int, version_string.split("."))) pass_test = version == acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue( pass_test, "Unsupported cdbtools version. %s is required, but running %s." % (".".join(map(str, acceptable_version)), version_string), )
def test_mothur_supported_version(self): """mothur is in path and version is supported """ acceptable_version = (1, 25, 0) self.assertTrue(app_path('mothur'), "mothur not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.") # mothur creates a log file in cwd, so create a tmp and cd there first log_file = join(get_qiime_temp_dir(), 'mothur.log') command = "mothur \"#set.logfile(name=%s)\" | grep '^mothur v'" % log_file stdout, stderr, exit_Status = qiime_system_call(command) # remove log file remove_files([log_file], error_on_missing=False) version_string = stdout.strip().split(' ')[1].strip('v.') try: version = tuple(map(int, version_string.split('.'))) pass_test = version == acceptable_version except ValueError: pass_test = False version_string = stdout self.assertTrue(pass_test,\ "Unsupported mothur version. %s is required, but running %s." \ % ('.'.join(map(str,acceptable_version)), version_string))
def test_cdhit_supported_version(self): """cd-hit is in path and version is supported """ self.assertTrue(app_path('cd-hit'), "cd-hit not found. This may or may not be a problem depending on "+\ "which components of QIIME you plan to use.")
def assign_taxonomy(dataPath, reference_sequences_fp, id_to_taxonomy_fp, read_1_seqs_fp, read_2_seqs_fp, single_ok=False, no_single_ok_generic=False, header_id_regex=None, read_id_regex="\S+\s+(\S+)", amplicon_id_regex="(\S+)\s+(\S+?)\/", output_fp=None, log_path=None, HALT_EXEC=False, base_tmp_dir='/tmp'): """Assign taxonomy to each sequence in data with the RTAX classifier # data: open fasta file object or list of fasta lines dataPath: path to a fasta file output_fp: path to write output; if not provided, result will be returned in a dict of {seq_id:(taxonomy_assignment,confidence)} """ usearch_command = "usearch" if not (exists(usearch_command) or app_path(usearch_command)): raise ApplicationNotFoundError,\ "Cannot find %s. Is it installed? Is it in your path?"\ % usearch_command my_tmp_dir = get_tmp_filename(tmp_dir=base_tmp_dir, prefix='rtax_', suffix='', result_constructor=str) os.makedirs(my_tmp_dir) try: # RTAX classifier doesn't necessarily preserve identifiers # it reports back only the id extracted as $1 using header_id_regex # since rtax takes the original unclustered sequence files as input, # the usual case is that the regex extracts the amplicon ID from the second field # Use lookup table read_1_id_to_orig_id = {} readIdExtractor = re.compile( read_id_regex) # OTU clustering produces ">clusterID read_1_id" data = open(dataPath, 'r') for seq_id, seq in MinimalFastaParser(data): # apply the regex extract = readIdExtractor.match(seq_id) if extract is None: stderr.write("Matched no ID with read_id_regex " + read_id_regex + " in '" + seq_id + "' from file " + dataPath + "\n") else: read_1_id_to_orig_id[extract.group(1)] = seq_id #stderr.write(extract.group(1) + " => " + seq_id + "\n") #seq_id_lookup[seq_id.split()[1]] = seq_id data.close() # make list of amplicon IDs to pass to RTAX id_list_fp = open(my_tmp_dir + "/ampliconIdsToClassify", "w") # Establish mapping of amplicon IDs to read_1 IDs # simultaneously write the amplicon ID file for those IDs found in the input mapping above amplicon_to_read_1_id = {} ampliconIdExtractor = re.compile( amplicon_id_regex ) # split_libraries produces >read_1_id ampliconID/1 ... // see also assign_taxonomy 631 read_1_data = open(read_1_seqs_fp, 'r') for seq_id, seq in MinimalFastaParser(read_1_data): # apply the regex extract = ampliconIdExtractor.match(seq_id) if extract is None: stderr.write("Matched no ID with amplicon_id_regex " + amplicon_id_regex + " in '" + seq_id + "' from file " + read_1_seqs_fp + "\n") else: read_1_id = extract.group(1) amplicon_id = extract.group(2) try: amplicon_to_read_1_id[amplicon_id] = read_1_id bogus = read_1_id_to_orig_id[ read_1_id] # verify that the id is valid id_list_fp.write('%s\n' % (amplicon_id)) except KeyError: pass data.close() id_list_fp.close() app = Rtax(HALT_EXEC=HALT_EXEC) temp_output_file = tempfile.NamedTemporaryFile( prefix='RtaxAssignments_', suffix='.txt') app.Parameters['-o'].on(temp_output_file.name) app.Parameters['-r'].on(reference_sequences_fp) app.Parameters['-t'].on(id_to_taxonomy_fp) # app.Parameters['-d'].on(delimiter) app.Parameters['-l'].on(id_list_fp.name) # these are amplicon IDs app.Parameters['-a'].on(read_1_seqs_fp) if read_2_seqs_fp is not None: app.Parameters['-b'].on(read_2_seqs_fp) app.Parameters['-i'].on(header_id_regex) app.Parameters['-m'].on(my_tmp_dir) if single_ok: app.Parameters['-f'].on() if no_single_ok_generic: app.Parameters['-g'].on() #app.Parameters['-v'].on() app_result = app() if log_path: f = open(log_path, 'a') errString = ''.join(app_result['StdErr'].readlines()) + '\n' f.write(errString) f.close() assignments = {} # restore original sequence IDs with spaces for line in app_result['Assignments']: toks = line.strip().split('\t') rtax_id = toks.pop(0) if len(toks): bestpcid = toks.pop(0) # ignored lineage = toks # RTAX does not provide a measure of confidence. We could pass one in, # based on the choice of primers, or even look it up on the fly in the tables # from the "optimal primers" paper; but it would be the same for every # query sequence anyway. # we could also return bestpcid, but that's not the same thing as confidence. confidence = 1.0 read_1_id = amplicon_to_read_1_id[rtax_id] orig_id = read_1_id_to_orig_id[read_1_id] if lineage: assignments[orig_id] = (';'.join(lineage), confidence) else: assignments[orig_id] = ('Unclassified', 1.0) if output_fp: try: output_file = open(output_fp, 'w') except OSError: raise OSError("Can't open output file for writing: %s" % output_fp) for seq_id, assignment in assignments.items(): lineage, confidence = assignment output_file.write('%s\t%s\t%1.3f\n' % (seq_id, lineage, confidence)) output_file.close() return None else: return assignments finally: try: rmtree(my_tmp_dir) except OSError: pass
__credits__ = ["Jeremy Widmann", "Rob Knight"] __license__ = "GPL" __version__ = "1.5.3-dev" __maintainer__ = "Jeremy Widmann" __email__ = "*****@*****.**" __status__ = "Production" """Tests of the pdb_color module. Owner: Jeremy Widmann [email protected] Revision History: October 2006 Jeremy Widmann: File created """ MUSCLE_PATH = app_path('muscle') class PdbColorTests(TestCase): """Tests for pdb_color functions. """ def setUp(self): """Setup for pdb_color tests.""" #Nucleotide test data results self.test_pdb_chains_1 = { 'A': [(1, 'G'), (2, 'C'), (3, 'C'), (4, 'A'), (5, 'C'), (6, 'C'), (7, 'C'), (8, 'U'), (9, 'G')], 'B': [(10, 'C'), (11, 'A'), (12, 'G'), (13, 'G'), (14, 'G'), (15, 'U'), (16, 'C'), (17, 'G'), (18, 'G'), (19, 'C')] } self.ungapped_to_pdb_1 = {
def check_sfffile(): """Raise error if sfffile is not in $PATH """ if not app_path('sfffile'): raise ApplicationNotFoundError(_MISSING_APP_MESSAGE % 'sfffile')
from cogent.maths.stats.util import Freqs from cogent.util.misc import app_path from cogent.struct.rna2d import ViennaStructure from cogent.util.unit_test import TestCase, main __author__ = "Rob Knight" __copyright__ = "Copyright 2007-2009, The Cogent Project" __credits__ = ["Rob Knight", "Daniel McDonald"] __license__ = "GPL" __version__ = "1.4.1" __maintainer__ = "Rob Knight" __email__ = "*****@*****.**" __status__ = "Development" #need to skip some tests if RNAfold absent if app_path('RNAfold'): RNAFOLD_PRESENT = True else: RNAFOLD_PRESENT = False class FunctionTests(TestCase): """Tests of standalone functions""" def setUp(self): self.standards = (0, 1, 5, 30, 173, 1000, 4382) def test_permuations_negative_k(self): """permutations should raise IndexError if k negative""" self.assertRaises(IndexError, permutations, 3, -1) def test_permutations_k_more_than_n(self): """permutations should raise IndexError if k > n"""
""" Build the denoiser code binary """ cwd = getcwd() denoiser_dir = join(cwd, 'qiime/support_files/denoiser/FlowgramAlignment') chdir(denoiser_dir) call(["make"]) chdir(cwd) print "Denoiser built." pycogent_version = tuple([int(v) \ for v in re.split("[^\d]", cogent.__version__) if v.isdigit()]) if pycogent_version < (1, 5): print "PyCogent >= 1.5.0 required, but %s is installed." % cogent.__version__ exit(1) if app_path("ghc"): build_denoiser() else: print "GHC not installed, so cannot build the Denoiser binary." setup(name='QIIME', version=__version__, description='Quantitative Insights Into Microbial Ecology', author=__maintainer__, author_email=__email__, maintainer=__maintainer__, maintainer_email=__email__, url='http://www.qiime.org', packages=[ 'qiime', 'qiime/parallel', 'qiime/pycogent_backports', 'qiime/denoiser'
def _error_on_missing_application(self, params): """ Raise an ApplicationNotFoundError if the app is not accessible """ if not app_path('blastall'): raise ApplicationNotFoundError,\ "Cannot find blastall. Is it installed? Is it in your path?"
def suite(): modules_to_test = [ 'test_recalculation.rst', 'test_phylo', 'test_dictarray.rst', 'test_align.test_align', 'test_align.test_algorithm', 'test_align.test_weights.test_methods', 'test_align.test_weights.test_util', 'test_app.test_parameters', 'test_app.test_util', 'test_cluster.test_goodness_of_fit', 'test_cluster.test_metric_scaling', 'test_cluster.test_procrustes', 'test_cluster.test_UPGMA', 'test_cluster.test_nmds', 'test_core.test_alphabet', 'test_core.test_alignment', 'test_core.test_annotation', 'test_core.test_bitvector', 'test_core.test_core_standalone', 'test_core.test_features.rst', 'test_core.test_entity', 'test_core.test_genetic_code', 'test_core.test_info', 'test_core.test_location', 'test_core.test_maps', 'test_core.test_moltype', 'test_core.test_profile', 'test_core.test_seq_aln_integration', 'test_core.test_sequence', 'test_core.test_tree', 'test_core.test_tree2', 'test_core.test_usage', 'test_data.test_molecular_weight', 'test_evolve.test_best_likelihood', 'test_evolve.test_bootstrap', 'test_evolve.test_coevolution', 'test_evolve.test_models', 'test_evolve.test_motifchange', 'test_evolve.test_substitution_model', 'test_evolve.test_scale_rules', 'test_evolve.test_likelihood_function', 'test_evolve.test_newq', 'test_evolve.test_parameter_controller', 'test_format.test_mage', 'test_format.test_fasta', 'test_format.test_pdb_color', 'test_format.test_xyzrn', 'test_maths.test_geometry', 'test_maths.test_matrix_logarithm', 'test_maths.test_matrix.test_distance', 'test_maths.test_spatial.test_ckd3', 'test_maths.test_stats.test_alpha_diversity', 'test_maths.test_stats.test_distribution', 'test_maths.test_stats.test_histogram', 'test_maths.test_stats.test_special', 'test_maths.test_stats.test_test', 'test_maths.test_stats.test_ks', 'test_maths.test_stats.test_rarefaction', 'test_maths.test_stats.test_util', 'test_maths.test_stats.test_cai.test_adaptor', 'test_maths.test_stats.test_cai.test_get_by_cai', 'test_maths.test_stats.test_cai.test_util', 'test_maths.test_optimisers', 'test_maths.test_distance_transform', 'test_maths.test_unifrac.test_fast_tree', 'test_maths.test_unifrac.test_fast_unifrac', 'test_motif.test_util', 'test_parse.test_aaindex', 'test_parse.test_agilent_microarray', 'test_parse.test_blast', 'test_parse.test_bpseq', 'test_parse.test_cigar', 'test_parse.test_clustal', 'test_parse.test_column', 'test_parse.test_comrna', 'test_parse.test_consan', 'test_parse.test_cove', 'test_parse.test_ct', 'test_parse.test_cut', 'test_parse.test_cutg', 'test_parse.test_dialign', 'test_parse.test_ebi', 'test_parse.test_fasta', 'test_parse.test_gibbs', 'test_parse.test_genbank', 'test_parse.test_gff', 'test_parse.test_ilm', 'test_parse.test_locuslink', 'test_parse.test_mage', 'test_parse.test_meme', 'test_parse.test_msms', 'test_parse.test_ncbi_taxonomy', 'test_parse.test_nexus', 'test_parse.test_nupack', 'test_parse.test_phylip', 'test_parse.test_pknotsrg', 'test_parse.test_rdb', 'test_parse.test_record', 'test_parse.test_record_finder', 'test_parse.test_rfam', 'test_parse.test_rnaalifold', 'test_parse.test_rna_fold', 'test_parse.test_rnaview', 'test_parse.test_rnaforester', 'test_parse.test_sprinzl', 'test_parse.test_tree', 'test_parse.test_unigene', 'test_seqsim.test_analysis', 'test_seqsim.test_birth_death', 'test_seqsim.test_markov', 'test_seqsim.test_microarray', 'test_seqsim.test_microarray_normalize', 'test_seqsim.test_randomization', 'test_seqsim.test_searchpath', 'test_seqsim.test_sequence_generators', 'test_seqsim.test_tree', 'test_seqsim.test_usage', 'test_struct.test_knots', 'test_struct.test_pairs_util', 'test_struct.test_rna2d', 'test_struct.test_asa', 'test_struct.test_contact', 'test_struct.test_annotation', 'test_struct.test_selection', 'test_struct.test_manipulation', 'test_util.test_unit_test', 'test_util.test_array', 'test_util.test_dict2d', 'test_util.test_misc', 'test_util.test_organizer', 'test_util.test_recode_alignment', 'test_util.test_table.rst', 'test_util.test_transform', ] try: import matplotlib except: print >> sys.stderr, "No matplotlib so not running test_draw.py" else: matplotlib.use('Agg') # non interactive # test_draw not yet structured as unit tests #modules_to_test.append('test_draw') #Try importing modules for app controllers apps = [('blastall', 'test_blast'), ('carnac', 'test_carnac'), ('clearcut', 'test_clearcut'), ('clustalw', 'test_clustalw'), ('cmfinder.pl', 'test_cmfinder'), ('comrna', 'test_comrna'), ('contrafold', 'test_contrafold'), ('covea', 'test_cove'), ('dialign2-2', 'test_dialign'), ('dynalign', 'test_dynalign'), ('FastTree', 'test_fasttree'), ('foldalign', 'test_foldalign'), ('ilm', 'test_ilm'), ('knetfold.pl', 'test_knetfold'), ('mafft', 'test_mafft'), ('mfold', 'test_mfold'), ('muscle', 'test_muscle'), ('msms', 'test_msms'), ('rdp_classifier-2.0.jar', 'test_rdp_classifier'), ('Fold.out', 'test_nupack'), ('findphyl', 'test_pfold'), ('pknotsRG-1.2-i386-linux-static', 'test_pknotsrg'), ('RNAalifold', 'test_rnaalifold'), ('rnaview', 'test_rnaview'), ('RNAfold', 'test_vienna_package'), ('raxmlHPC', 'test_raxml'), ('sfold.X86_64.LINUX', 'test_sfold'), ('stride', 'test_stride'), ('hybrid-ss-min', 'test_unafold'), ('cd-hit', 'test_cd_hit'), ('calculate_likelihood', 'test_gctmpca'), ('sfffile', 'test_sfffile'), ('sffinfo', 'test_sffinfo'), ('uclust','test_uclust') ] for app, test_name in apps: if app_path(app): modules_to_test.append('test_app.' + test_name) else: print >> sys.stderr, "Can't find %s executable: skipping test" % app if app_path('muscle'): modules_to_test.append('test_format.test_pdb_color') # we now toggle the db tests, based on an environment flag if int(os.environ.get('TEST_DB', 0)): db_tests = ['test_db.test_ncbi', 'test_db.test_pdb', 'test_db.test_rfam', 'test_db.test_util'] # we check for an environment flag for ENSEMBL # we expect this to have the username and account for a localhost # installation of the Ensembl MySQL databases if 'ENSEMBL_ACCOUNT' in os.environ: # check for cogent.db.ensembl dependencies test_ensembl = True for module in ['MySQLdb', 'sqlalchemy']: if not module_present(module): test_ensembl = False print >> sys.stderr, \ "Module '%s' not present: skipping test" % module if test_ensembl: db_tests += ['test_db.test_ensembl.test_assembly', 'test_db.test_ensembl.test_database', 'test_db.test_ensembl.test_compara', 'test_db.test_ensembl.test_genome', 'test_db.test_ensembl.test_host', 'test_db.test_ensembl.test_species', 'test_db.test_ensembl.test_feature_level'] else: print >> sys.stderr, "Environment variable ENSEMBL_ACCOUNT not "\ "set: skipping db.ensembl tests" for db_test in db_tests: modules_to_test.append(db_test) else: print >> sys.stderr, \ "Environment variable TEST_DB=1 not set: skipping db tests" assert sys.version_info >= (2, 4) alltests = unittest.TestSuite() for module in modules_to_test: if module.endswith('.rst'): module = os.path.join(*module.split(".")[:-1]) + ".rst" test = doctest.DocFileSuite(module, optionflags= doctest.REPORT_ONLY_FIRST_FAILURE | doctest.ELLIPSIS) else: test = unittest.findTestCases(my_import(module)) alltests.addTest(test) return alltests