Esempio n. 1
0
    def run(self, **kwargs):

        input_dir = kwargs['input_dir']
        input_sra_pattern = join(input_dir, '*.sra')
        input_filepaths = glob(input_sra_pattern)
        output_fp = kwargs['output_fp']

        for input_filepath in input_filepaths:
            temp_files_to_remove = []
            temp_dirs_to_remove = []
            input_filename = split(input_filepath)[1]
            input_basename = splitext(input_filename)[0]

            # create and call the sra-dump command
            temp_fastq_dir = join(self.temp_dir, '%s_fastq' % input_basename)
            command = "%s %s -O %s" % (self.sra_dump_path, input_filepath,
                                       temp_fastq_dir)
            if self.verbose:
                print command
            stdout, stderr, ret_val = pyqi_system_call(command)
            temp_dirs_to_remove.append(temp_fastq_dir)
            temp_fastq_fp = glob(join(temp_fastq_dir, '*.fastq'))[0]

            # convert fastq to fasta
            temp_fasta_fp = join(self.temp_dir, '%s.fasta' % input_basename)
            command = "%s %s > %s" % (self.fastq_to_fasta, temp_fastq_fp,
                                      temp_fasta_fp)
            stdout, stderr, ret_val = pyqi_system_call(command)
            if self.verbose:
                print command
            temp_files_to_remove.append(temp_fasta_fp)

            # convert fasta to qiime-compatible headers
            stdout, stderr, ret_val = pyqi_system_call(command)
            command = "sed 's/\./_/g;s/ .*$//g' %s >> %s" % (temp_fasta_fp,
                                                             output_fp)
            stdout, stderr, ret_val = pyqi_system_call(command)
            if self.verbose:
                print command

            # clean up
            #if self.verbose:
            #   print "Removing files: %s" % " ".join(temp_files_to_remove)
            #  print "Removing directories: %s" % " ".join(temp_dirs_to_remove)
            #remove_files(temp_files_to_remove)
            #for temp_dir_to_remove in temp_dirs_to_remove:
            #   rmtree(temp_dir_to_remove)

        return {'result': output_fp}
    def run(self, **kwargs):
        
        input_dir = kwargs['input_dir']
        input_biom_table_pattern = join(input_dir,'*.biom')
        input_filepath = glob(input_biom_table_pattern)
        
        output_dir = kwargs['output_dir']
        
        sampling_depth = kwargs['sampling_depth']
        
        
        
        mapping_file = kwargs['mapping_file']
        input_mapping_file_pattern = (mapping_file + 'mapping_file.txt')
        

        for input_filepath in input_filepath:
            temp_files_to_remove = []
            temp_dirs_to_remove = []
            input_filename = split(input_filepath)[1]
            input_basename = splitext(input_filename)[0]
            
            #Create and call the core_diversity_analysis.py command and run it using a miRNAs biom table
            command = "%s -i %s -m %s -e %s -o %s  --suppress_otu_category_significance --nonphylogenetic_diversity" % (self.core_diversity_anlysis_path, input_filepath,  mapping_file, sampling_depth, output_dir)
            if self.verbose:
                print command
            stdout, stderr, ret_val = pyqi_system_call(command)
            if ret_val != 0:
            
                return {"status":ret_val,
                        "error":stderr}
    def run(self, **kwargs):
        
        input_fp = kwargs['input_file']
        
        
        output_dir = kwargs['output_dir']
        
                
        #Mapping file
        mapping_file_fp = kwargs['mapping_file']
        input_mapping_file_pattern = join(mapping_file_fp,'mapping_file.txt')
        

        temp_files_to_remove = []
        temp_dirs_to_remove = []
        input_filename = split(input_fp)[1]
        input_basename = splitext(input_filename)[0]
            
        #Create and call the core_diversity_analysis.py command and run it using a miRNAs biom table
        command = "%s -i %s -m %s -e %s -o %s -a -O %s -c %s --suppress_otu_category_significance --nonphylogenetic_diversity" % (self.core_diversity_analyses_path, input_fp, mapping_file_fp, int(kwargs["sampling_depth"]), output_dir, int(kwargs["jobs_to_start"]), str(kwargs["category"]))
        if self.verbose:
                print command
        stdout, stderr, ret_val = pyqi_system_call(command)
        if ret_val != 0:
            
                return {"status":ret_val,
                        "error":stderr}
Esempio n. 4
0
 def _build_and_upload(self):
     cmd = [sys.executable, 'setup.py', 'sdist', 'upload']
     stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
                                               dry_run=not self.RealRun)
     if retval is not 0:
         self._fail("build and upload failed,\nSTDOUT:\n%s\n\nSTDERR:\n%s", 
                    stdout, stderr)
    def run(self, **kwargs):

        input_fp = kwargs['input_file']

        output_dir = kwargs['output_dir']

        #Mapping file
        mapping_file_fp = kwargs['mapping_file']
        input_mapping_file_pattern = join(mapping_file_fp, 'mapping_file.txt')

        temp_files_to_remove = []
        temp_dirs_to_remove = []
        input_filename = split(input_fp)[1]
        input_basename = splitext(input_filename)[0]

        #Create and call the core_diversity_analysis.py command and run it using a miRNAs biom table
        command = "%s -i %s -m %s -e %s -o %s -a -O %s -c %s --suppress_otu_category_significance --nonphylogenetic_diversity" % (
            self.core_diversity_analyses_path, input_fp, mapping_file_fp,
            int(kwargs["sampling_depth"]), output_dir,
            int(kwargs["jobs_to_start"]), str(kwargs["category"]))
        if self.verbose:
            print command
        stdout, stderr, ret_val = pyqi_system_call(command)
        if ret_val != 0:

            return {"status": ret_val, "error": stderr}
Esempio n. 6
0
 def _make_git_tag(self, tag):
     self._info('Tagging "%s"', tag)
     cmd = ['git', 'tag', tag]
     stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
                                               dry_run=not self.RealRun)
     if retval is not 0:
         self._fail("Could not git tag, \nSTDOUT:\n%s\n\nSTDERR:\n%s",stdout,
              stderr)
Esempio n. 7
0
 def _make_git_commit(self, message, *args):
     message = message % args
     cmd = ['git', 'commit', '-am', message]
     stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
                                               dry_run=not self.RealRun)
     if retval is not 0:
         self._fail("Could not git commit, \nSTDOUT:\n%s\n\nSTDERR:\n%s", 
                    stdout, stderr)
Esempio n. 8
0
 def _build_and_upload(self):
     cmd = [sys.executable, 'setup.py', 'sdist', 'upload']
     stdout, stderr, retval = pyqi_system_call(cmd,
                                               shell=False,
                                               dry_run=not self.RealRun)
     if retval is not 0:
         self._fail("build and upload failed,\nSTDOUT:\n%s\n\nSTDERR:\n%s",
                    stdout, stderr)
Esempio n. 9
0
 def _git_push_tag(self, tag):
     self._info('Pushing tag "%s"', tag)
     cmd = ['git','push','upstream',tag]
     stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
                                               dry_run=not self.RealRun)
     if retval is not 0:
         self._fail("Could not push tag %s, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
                    stdout, stderr, tag)
Esempio n. 10
0
    def _get_git_branch(self):
        cmd = ['git','rev-parse','--abbrev-ref','HEAD']

        # ignoring self.RealRun, always execute
        stdout, stderr, retval = pyqi_system_call(cmd, shell=False)
        if retval is not 0:
            self._fail("Could not get git branch, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
                       stdout, stderr)
        return stdout.strip()
Esempio n. 11
0
    def _get_git_tags(self):
        cmd = ['git', 'tag']
        stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
                                                  dry_run=not self.RealRun)
        if retval is not 0:
            self._fail("Could not git tag, \nSTDOUT:\n%s\n\nSTDERR:\n%s", 
                       stdout, stderr)

        return stdout.splitlines()
Esempio n. 12
0
 def _git_push_tag(self, tag):
     self._info('Pushing tag "%s"', tag)
     cmd = ['git', 'push', 'upstream', tag]
     stdout, stderr, retval = pyqi_system_call(cmd,
                                               shell=False,
                                               dry_run=not self.RealRun)
     if retval is not 0:
         self._fail("Could not push tag %s, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
                    stdout, stderr, tag)
Esempio n. 13
0
 def _make_git_tag(self, tag):
     self._info('Tagging "%s"', tag)
     cmd = ['git', 'tag', tag]
     stdout, stderr, retval = pyqi_system_call(cmd,
                                               shell=False,
                                               dry_run=not self.RealRun)
     if retval is not 0:
         self._fail("Could not git tag, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
                    stdout, stderr)
Esempio n. 14
0
 def _git_push_branch(self):
     branch = self._get_git_branch()
     self._info('Pushing branch %s to origin', branch)
     cmd = ['git','push','upstream', branch]
     stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
                                               dry_run=not self.RealRun)
     if retval is not 0:
         self._fail("Could not push branch %s, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
                    stdout, stderr, branch)
Esempio n. 15
0
 def _make_git_commit(self, message, *args):
     message = message % args
     cmd = ['git', 'commit', '-am', message]
     stdout, stderr, retval = pyqi_system_call(cmd,
                                               shell=False,
                                               dry_run=not self.RealRun)
     if retval is not 0:
         self._fail("Could not git commit, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
                    stdout, stderr)
Esempio n. 16
0
    def _get_git_branch(self):
        cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD']

        # ignoring self.RealRun, always execute
        stdout, stderr, retval = pyqi_system_call(cmd, shell=False)
        if retval is not 0:
            self._fail(
                "Could not get git branch, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
                stdout, stderr)
        return stdout.strip()
Esempio n. 17
0
    def _get_git_tags(self):
        cmd = ['git', 'tag']
        stdout, stderr, retval = pyqi_system_call(cmd,
                                                  shell=False,
                                                  dry_run=not self.RealRun)
        if retval is not 0:
            self._fail("Could not git tag, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
                       stdout, stderr)

        return stdout.splitlines()
Esempio n. 18
0
 def _git_push_branch(self):
     branch = self._get_git_branch()
     self._info('Pushing branch %s to origin', branch)
     cmd = ['git', 'push', 'upstream', branch]
     stdout, stderr, retval = pyqi_system_call(cmd,
                                               shell=False,
                                               dry_run=not self.RealRun)
     if retval is not 0:
         self._fail(
             "Could not push branch %s, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
             stdout, stderr, branch)
Esempio n. 19
0
    def _git_is_clean(self):
        cmd = ['git', 'diff', '--quiet']

        # always execute, even in dry run
        stdout, stderr, retval = pyqi_system_call(cmd, shell=False)
        return retval == 0
Esempio n. 20
0
    def run(self, **kwargs):
        
        input_fp = kwargs['input_file']
       
        
        output_dir = kwargs['output_dir']
        
        ncrnadb_fp = kwargs['ncRNAdb_file']
        input_all_ncrna_except_mirna_database_pattern = ncrnadb_fp
        
        maturemirnadb_fp = kwargs['mature_miRNAs_database_file']
        input_human_mature_mirna_database_pattern = maturemirnadb_fp
         
         
        temp_files_to_remove = []
        temp_dirs_to_remove = []
        input_filename = split(input_fp)[1]
        input_basename = splitext(input_filename)[0]
            
        #Create and call the parallel_pick_otus_uclust_ref.py command and run it against Ensemble nc_ex_mirna database
        command = "%s -i %s -r %s -o %s -O %s --enable_rev_strand_match --max_accepts 1 --max_rejects 8 --stepwords 8 --word_length 8" % (self.parallel_pick_otus_uclust_ref_path, input_fp, input_all_ncrna_except_mirna_database_pattern, self.temp_dir, int(kwargs["jobs_to_start"]))
        if self.verbose:
                print command
        stdout, stderr, ret_val = pyqi_system_call(command)
        if ret_val != 0:
            raise Exception(stderr)


        # Filter all the sequences from the previous closed-reference picking otu that didn't hit the database (i.e. standard output from parallel_pick_otus_uclust_ref.py = *_failures.txt) using the script 'filter_fasta.py -f input_fasta -s index_list -o output) 
        temp_fasta_index_list_failing_to_hit_database_fp = join(self.temp_dir, '%s_failures.txt' % input_basename)
        temp_fasta_filtered_fp = join(self.temp_dir, '%s_filtered.fasta' % input_basename)

        command = "%s -f %s -s %s  -o %s" % (self.filter_fasta, input_fp, temp_fasta_index_list_failing_to_hit_database_fp, temp_fasta_filtered_fp)
        if self.verbose:
                print command
        stdout, stderr, ret_val = pyqi_system_call(command)
        if ret_val != 0:
            raise Exception(stderr)


        # Create and call the parallel_pick_otus_uclust_ref.py command against mirBase - human mature mirna database
            
        temp_index_of_otus_hitting_miRbase_fp= join(self.temp_dir, '%s_otus.txt' % input_basename)
        stdout, stderr, ret_val = pyqi_system_call(command)
        command = "%s -i %s -r %s -o %s --enable_rev_strand_match --max_accepts 1 --max_rejects 8 --stepwords 8 --word_length 8" % (self.parallel_pick_otus_uclust_ref_path, temp_fasta_filtered_fp, input_human_mature_mirna_database_pattern, self.temp_dir)
        if self.verbose:
                print command
        stdout, stderr, ret_val = pyqi_system_call(command)
        if ret_val != 0:
            raise Exception(stderr)

            
        # Create an otu_table using the outuput otu_map from the previous step:
            
            
        mirna_final_biom_table = join(output_dir, '%s.biom' % input_basename)
        command = "%s -i %s -o %s"  % (self.make_otu_table, temp_index_of_otus_hitting_miRbase_fp, mirna_final_biom_table)
        stdout, stderr, ret_val = pyqi_system_call(command)
        if self.verbose:
                print command
        stdout, stderr, ret_val = pyqi_system_call(command)
        if ret_val != 0:
            raise Exception(stderr)
            
        return {"status": "is ok",
                "error":None}
Esempio n. 21
0
    def _git_is_clean(self):
        cmd = ['git','diff','--quiet']

        # always execute, even in dry run
        stdout, stderr, retval = pyqi_system_call(cmd, shell=False)
        return retval == 0