def run(self, **kwargs):
input_dir = kwargs['input_dir']
input_sra_pattern = join(input_dir, '*.sra')
input_filepaths = glob(input_sra_pattern)
output_fp = kwargs['output_fp']
for input_filepath in input_filepaths:
temp_files_to_remove = []
temp_dirs_to_remove = []
input_filename = split(input_filepath)[1]
input_basename = splitext(input_filename)[0]
# create and call the sra-dump command
temp_fastq_dir = join(self.temp_dir, '%s_fastq' % input_basename)
command = "%s %s -O %s" % (self.sra_dump_path, input_filepath,
temp_fastq_dir)
if self.verbose:
print command
stdout, stderr, ret_val = pyqi_system_call(command)
temp_dirs_to_remove.append(temp_fastq_dir)
temp_fastq_fp = glob(join(temp_fastq_dir, '*.fastq'))[0]
# convert fastq to fasta
temp_fasta_fp = join(self.temp_dir, '%s.fasta' % input_basename)
command = "%s %s > %s" % (self.fastq_to_fasta, temp_fastq_fp,
temp_fasta_fp)
stdout, stderr, ret_val = pyqi_system_call(command)
if self.verbose:
print command
temp_files_to_remove.append(temp_fasta_fp)
# convert fasta to qiime-compatible headers
stdout, stderr, ret_val = pyqi_system_call(command)
command = "sed 's/\./_/g;s/ .*$//g' %s >> %s" % (temp_fasta_fp,
output_fp)
stdout, stderr, ret_val = pyqi_system_call(command)
if self.verbose:
print command
# clean up
#if self.verbose:
# print "Removing files: %s" % " ".join(temp_files_to_remove)
# print "Removing directories: %s" % " ".join(temp_dirs_to_remove)
#remove_files(temp_files_to_remove)
#for temp_dir_to_remove in temp_dirs_to_remove:
# rmtree(temp_dir_to_remove)
return {'result': output_fp}
def run(self, **kwargs):
input_dir = kwargs['input_dir']
input_biom_table_pattern = join(input_dir,'*.biom')
input_filepath = glob(input_biom_table_pattern)
output_dir = kwargs['output_dir']
sampling_depth = kwargs['sampling_depth']
mapping_file = kwargs['mapping_file']
input_mapping_file_pattern = (mapping_file + 'mapping_file.txt')
for input_filepath in input_filepath:
temp_files_to_remove = []
temp_dirs_to_remove = []
input_filename = split(input_filepath)[1]
input_basename = splitext(input_filename)[0]
#Create and call the core_diversity_analysis.py command and run it using a miRNAs biom table
command = "%s -i %s -m %s -e %s -o %s --suppress_otu_category_significance --nonphylogenetic_diversity" % (self.core_diversity_anlysis_path, input_filepath, mapping_file, sampling_depth, output_dir)
if self.verbose:
print command
stdout, stderr, ret_val = pyqi_system_call(command)
if ret_val != 0:
return {"status":ret_val,
"error":stderr}
def run(self, **kwargs):
input_fp = kwargs['input_file']
output_dir = kwargs['output_dir']
#Mapping file
mapping_file_fp = kwargs['mapping_file']
input_mapping_file_pattern = join(mapping_file_fp,'mapping_file.txt')
temp_files_to_remove = []
temp_dirs_to_remove = []
input_filename = split(input_fp)[1]
input_basename = splitext(input_filename)[0]
#Create and call the core_diversity_analysis.py command and run it using a miRNAs biom table
command = "%s -i %s -m %s -e %s -o %s -a -O %s -c %s --suppress_otu_category_significance --nonphylogenetic_diversity" % (self.core_diversity_analyses_path, input_fp, mapping_file_fp, int(kwargs["sampling_depth"]), output_dir, int(kwargs["jobs_to_start"]), str(kwargs["category"]))
if self.verbose:
print command
stdout, stderr, ret_val = pyqi_system_call(command)
if ret_val != 0:
return {"status":ret_val,
"error":stderr}
def _build_and_upload(self):
cmd = [sys.executable, 'setup.py', 'sdist', 'upload']
stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail("build and upload failed,\nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr)
def run(self, **kwargs):
input_fp = kwargs['input_file']
output_dir = kwargs['output_dir']
#Mapping file
mapping_file_fp = kwargs['mapping_file']
input_mapping_file_pattern = join(mapping_file_fp, 'mapping_file.txt')
temp_files_to_remove = []
temp_dirs_to_remove = []
input_filename = split(input_fp)[1]
input_basename = splitext(input_filename)[0]
#Create and call the core_diversity_analysis.py command and run it using a miRNAs biom table
command = "%s -i %s -m %s -e %s -o %s -a -O %s -c %s --suppress_otu_category_significance --nonphylogenetic_diversity" % (
self.core_diversity_analyses_path, input_fp, mapping_file_fp,
int(kwargs["sampling_depth"]), output_dir,
int(kwargs["jobs_to_start"]), str(kwargs["category"]))
if self.verbose:
print command
stdout, stderr, ret_val = pyqi_system_call(command)
if ret_val != 0:
return {"status": ret_val, "error": stderr}
def _make_git_tag(self, tag):
self._info('Tagging "%s"', tag)
cmd = ['git', 'tag', tag]
stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail("Could not git tag, \nSTDOUT:\n%s\n\nSTDERR:\n%s",stdout,
stderr)
def _make_git_commit(self, message, *args):
message = message % args
cmd = ['git', 'commit', '-am', message]
stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail("Could not git commit, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr)
def _build_and_upload(self):
cmd = [sys.executable, 'setup.py', 'sdist', 'upload']
stdout, stderr, retval = pyqi_system_call(cmd,
shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail("build and upload failed,\nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr)
def _git_push_tag(self, tag):
self._info('Pushing tag "%s"', tag)
cmd = ['git','push','upstream',tag]
stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail("Could not push tag %s, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr, tag)
def _get_git_branch(self):
cmd = ['git','rev-parse','--abbrev-ref','HEAD']
# ignoring self.RealRun, always execute
stdout, stderr, retval = pyqi_system_call(cmd, shell=False)
if retval is not 0:
self._fail("Could not get git branch, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr)
return stdout.strip()
def _get_git_tags(self):
cmd = ['git', 'tag']
stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail("Could not git tag, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr)
return stdout.splitlines()
def _git_push_tag(self, tag):
self._info('Pushing tag "%s"', tag)
cmd = ['git', 'push', 'upstream', tag]
stdout, stderr, retval = pyqi_system_call(cmd,
shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail("Could not push tag %s, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr, tag)
def _make_git_tag(self, tag):
self._info('Tagging "%s"', tag)
cmd = ['git', 'tag', tag]
stdout, stderr, retval = pyqi_system_call(cmd,
shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail("Could not git tag, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr)
def _git_push_branch(self):
branch = self._get_git_branch()
self._info('Pushing branch %s to origin', branch)
cmd = ['git','push','upstream', branch]
stdout, stderr, retval = pyqi_system_call(cmd, shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail("Could not push branch %s, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr, branch)
def _make_git_commit(self, message, *args):
message = message % args
cmd = ['git', 'commit', '-am', message]
stdout, stderr, retval = pyqi_system_call(cmd,
shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail("Could not git commit, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr)
def _get_git_branch(self):
cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD']
# ignoring self.RealRun, always execute
stdout, stderr, retval = pyqi_system_call(cmd, shell=False)
if retval is not 0:
self._fail(
"Could not get git branch, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr)
return stdout.strip()
def _get_git_tags(self):
cmd = ['git', 'tag']
stdout, stderr, retval = pyqi_system_call(cmd,
shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail("Could not git tag, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr)
return stdout.splitlines()
def _git_push_branch(self):
branch = self._get_git_branch()
self._info('Pushing branch %s to origin', branch)
cmd = ['git', 'push', 'upstream', branch]
stdout, stderr, retval = pyqi_system_call(cmd,
shell=False,
dry_run=not self.RealRun)
if retval is not 0:
self._fail(
"Could not push branch %s, \nSTDOUT:\n%s\n\nSTDERR:\n%s",
stdout, stderr, branch)
def _git_is_clean(self):
cmd = ['git', 'diff', '--quiet']
# always execute, even in dry run
stdout, stderr, retval = pyqi_system_call(cmd, shell=False)
return retval == 0
def run(self, **kwargs):
input_fp = kwargs['input_file']
output_dir = kwargs['output_dir']
ncrnadb_fp = kwargs['ncRNAdb_file']
input_all_ncrna_except_mirna_database_pattern = ncrnadb_fp
maturemirnadb_fp = kwargs['mature_miRNAs_database_file']
input_human_mature_mirna_database_pattern = maturemirnadb_fp
temp_files_to_remove = []
temp_dirs_to_remove = []
input_filename = split(input_fp)[1]
input_basename = splitext(input_filename)[0]
#Create and call the parallel_pick_otus_uclust_ref.py command and run it against Ensemble nc_ex_mirna database
command = "%s -i %s -r %s -o %s -O %s --enable_rev_strand_match --max_accepts 1 --max_rejects 8 --stepwords 8 --word_length 8" % (self.parallel_pick_otus_uclust_ref_path, input_fp, input_all_ncrna_except_mirna_database_pattern, self.temp_dir, int(kwargs["jobs_to_start"]))
if self.verbose:
print command
stdout, stderr, ret_val = pyqi_system_call(command)
if ret_val != 0:
raise Exception(stderr)
# Filter all the sequences from the previous closed-reference picking otu that didn't hit the database (i.e. standard output from parallel_pick_otus_uclust_ref.py = *_failures.txt) using the script 'filter_fasta.py -f input_fasta -s index_list -o output)
temp_fasta_index_list_failing_to_hit_database_fp = join(self.temp_dir, '%s_failures.txt' % input_basename)
temp_fasta_filtered_fp = join(self.temp_dir, '%s_filtered.fasta' % input_basename)
command = "%s -f %s -s %s -o %s" % (self.filter_fasta, input_fp, temp_fasta_index_list_failing_to_hit_database_fp, temp_fasta_filtered_fp)
if self.verbose:
print command
stdout, stderr, ret_val = pyqi_system_call(command)
if ret_val != 0:
raise Exception(stderr)
# Create and call the parallel_pick_otus_uclust_ref.py command against mirBase - human mature mirna database
temp_index_of_otus_hitting_miRbase_fp= join(self.temp_dir, '%s_otus.txt' % input_basename)
stdout, stderr, ret_val = pyqi_system_call(command)
command = "%s -i %s -r %s -o %s --enable_rev_strand_match --max_accepts 1 --max_rejects 8 --stepwords 8 --word_length 8" % (self.parallel_pick_otus_uclust_ref_path, temp_fasta_filtered_fp, input_human_mature_mirna_database_pattern, self.temp_dir)
if self.verbose:
print command
stdout, stderr, ret_val = pyqi_system_call(command)
if ret_val != 0:
raise Exception(stderr)
# Create an otu_table using the outuput otu_map from the previous step:
mirna_final_biom_table = join(output_dir, '%s.biom' % input_basename)
command = "%s -i %s -o %s" % (self.make_otu_table, temp_index_of_otus_hitting_miRbase_fp, mirna_final_biom_table)
stdout, stderr, ret_val = pyqi_system_call(command)
if self.verbose:
print command
stdout, stderr, ret_val = pyqi_system_call(command)
if ret_val != 0:
raise Exception(stderr)
return {"status": "is ok",
"error":None}
def _git_is_clean(self):
cmd = ['git','diff','--quiet']
# always execute, even in dry run
stdout, stderr, retval = pyqi_system_call(cmd, shell=False)
return retval == 0
