def make_filtered_and_common_dataset(i_datasets_folder: str, datasets: dict, datasets_filt: dict, datasets_read: dict, unique_datasets: list, train_test_dict: dict, mmvec_pairs: dict, filtering: dict, unique_filterings: dict, job_folder: str, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, analysis: str, filt_raref: str, filt_datasets_done: dict, common_datasets_done: dict, input_to_filtered: dict, already_computed: dict, subsets: dict, jobs: bool) -> (dict, dict): """ :param i_datasets_folder: :param datasets: list of data_sets. :param datasets_read: :param mmvec_pairs: :param mmvec_filtering: :param job_folder: :param force: :param prjct_nm: :param qiime_env: :param chmod: :return: """ print('\t-> [%s] Get datasets filtered...' % analysis) filt_datasets, filt_jobs = get_datasets_filtered( i_datasets_folder, datasets, datasets_read, datasets_filt, unique_datasets, unique_filterings, force, analysis, filt_datasets_done, input_to_filtered, train_test_dict, already_computed, subsets) common_jobs = [] common_datasets = {} if analysis == 'mmvec': print('\t-> [mmvec] Get common datasets...') common_datasets, common_jobs = get_common_datasets( i_datasets_folder, mmvec_pairs, filtering, filt_datasets, common_datasets_done, input_to_filtered, train_test_dict, force, subsets) pre_jobs = filt_jobs + common_jobs if len(pre_jobs): import_sh = '%s/2_run_%s_imports_%s%s.sh' % (job_folder, prjct_nm, analysis, filt_raref) import_pbs = '%s.pbs' % splitext(import_sh)[0] with open(import_sh, 'w') as import_o: for cmd in pre_jobs: import_o.write('\necho "%s"\n' % cmd) import_o.write('%s\n' % cmd) run_xpbs(import_sh, import_pbs, '%s.mprt.mmsb.%s%s' % (prjct_nm, analysis, filt_raref), qiime_env, '2', '1', '1', '150', 'mb', chmod, 1, '# Import datasets for %s' % analysis, None, noloc, jobs) return filt_datasets, common_datasets
def edit_taxonomies(i_datasets_folder: str, taxonomies: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, slurm: bool, run_params: dict, filt_raref: str, jobs: bool, chunkit: int): job_folder = get_job_folder(i_datasets_folder, 'taxonomy') job_folder2 = get_job_folder(i_datasets_folder, 'taxonomy/chunks') main_written = 0 to_chunk = [] run_pbs = '%s/1_run_taxonomy_edit_%s%s.sh' % (job_folder, prjct_nm, filt_raref) with open(run_pbs, 'w') as o: for dat, (_, qza, tsv) in taxonomies.items(): if not isfile(tsv): continue written = 0 out_pd = pd.read_csv(tsv, dtype=str, sep='\t') taxo = out_pd['Taxon'].tolist() taxo_edit = get_taxa_edit(taxo) if taxo != taxo_edit: out_pd['Taxon'] = taxo_edit out_pd.to_csv(tsv, index=False, sep='\t') cmd = run_import(tsv, qza, 'FeatureData[Taxonomy]') out_sh = '%s/run_taxonomy_edit_%s_%s%s.sh' % ( job_folder2, prjct_nm, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n\n' % cmd) main_written += 1 written += 1 if written: to_chunk.append(out_sh) if not chunkit: run_xpbs(out_sh, out_pbs, '%s.tx.dt.%s%s' % (prjct_nm, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if to_chunk and chunkit: simple_chunks(run_pbs, job_folder2, to_chunk, 'taxonomy_edit', prjct_nm, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit, None) if main_written: print_message('# Edit features taxonomy to not contain "," characters', 'sh', run_pbs, jobs)
def run_collapse(i_datasets_folder: str, datasets: dict, datasets_filt: dict, datasets_read: dict, datasets_features: dict, datasets_phylo: dict, split_taxa_pds: dict, taxonomies: dict, p_collapse_taxo: str, datasets_rarefs: dict, datasets_collapsed: dict, datasets_collapsed_map: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, slurm: bool, run_params: dict, filt_raref: str, jobs: bool) -> dict: collapse_taxo = get_collapse_taxo(p_collapse_taxo, datasets_filt) main_written = 0 collapsed = {} datasets_update = {} datasets_read_update = {} datasets_features_update = {} datasets_phylo_update = {} stop_for_collapse = False job_folder = get_job_folder(i_datasets_folder, 'collapsed_taxo') job_folder2 = get_job_folder(i_datasets_folder, 'collapsed_taxo/chunks') run_pbs = '%s/3_run_collapsed_taxo_%s%s.sh' % (job_folder, prjct_nm, filt_raref) with open(run_pbs, 'w') as o: for dat, tab_meta_fps in datasets.items(): if dat not in collapse_taxo: continue # get the taxonomic levels collapse_levels = collapse_taxo[dat] split_taxa_pd, split_taxa_fp = split_taxa_pds[dat] split_levels, remove_empties = get_split_levels( collapse_levels, split_taxa_pd) collapsed[dat] = split_levels # files that will be collapsed using qiime2 tax_qza, tax_fp = taxonomies[dat][1:] written = 0 out_sh = '%s/run_collapsed_taxo_%s_%s%s.sh' % ( job_folder2, prjct_nm, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: for idx, tab_meta_fp in enumerate(tab_meta_fps): tab_fp, meta_fp = tab_meta_fp tab_qza = '%s.qza' % splitext(tab_fp)[0] for tax, level in split_levels.items(): coll_paths = collapse_paths(dat, tax, tab_fp, meta_fp) dat_tax = coll_paths[0] dat_coll = coll_paths[1] coll_tsv = coll_paths[2] coll_qza = coll_paths[3] coll_meta = coll_paths[4] if isfile(coll_tsv) and isfile(coll_meta): coll_pd = pd.read_csv(coll_tsv, index_col=0, header=0, sep='\t') coll_meta_pd = pd.read_csv( coll_meta, header=0, sep='\t', dtype={'sample_name': str}) if coll_pd.shape[0] < 5: continue cmd = fix_collapsed_data(remove_empties[tax], coll_pd, coll_tsv, coll_qza, coll_meta) if cmd: cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n\n' % cmd) main_written += 1 written += 1 datasets_read_update.setdefault( dat_tax, []).append([coll_pd, coll_meta_pd]) datasets_collapsed.setdefault(dat, []).append(dat_coll) datasets_collapsed_map[dat_coll] = dat datasets_update.setdefault(dat_tax, []).append( [coll_tsv, coll_meta]) datasets_rarefs.setdefault(dat_tax, []).append( datasets_rarefs[dat][idx]) datasets_phylo_update[dat_tax] = ('', 0) else: written += 1 main_written += 1 stop_for_collapse = True cmd = write_collapse_taxo(tab_qza, tax_qza, coll_qza, coll_tsv, meta_fp, coll_meta, level, remove_empties[tax]) cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n\n' % cmd) if written: run_xpbs(out_sh, out_pbs, '%s.cllps.%s%s' % (prjct_nm, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if main_written: print_message( '# Collapse features for taxo levels defined in %s' % p_collapse_taxo, 'sh', run_pbs, jobs) if stop_for_collapse: print('Stopping here as this collapse must be run first for other ' 'analyses to work') sys.exit(0) datasets.update(datasets_update) datasets_read.update(datasets_read_update) datasets_phylo.update(datasets_phylo_update) return collapsed
def run_doc(i_datasets_folder: str, datasets: dict, p_doc_config: str, datasets_rarefs: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, slurm: bool, run_params: dict, filt_raref: str, phates: dict, doc_phate: bool, split: bool, jobs: bool, chunkit: int) -> None: job_folder2 = get_job_folder(i_datasets_folder, 'doc/chunks') doc_filtering, doc_params, main_cases_dict = get_doc_config(p_doc_config) all_sh_pbs = {} all_import_sh_pbs = {} dat_cases_tabs = {} need_to_run_phate = [] need_to_run_less_phate = [] for dat, tsv_meta_pds_ in datasets.items(): dat_cases_tabs[dat] = {} if dat in doc_filtering: filters = doc_filtering[dat] else: filters = {'0-0': ['0', '0']} for idx, tsv_meta_pds in enumerate(tsv_meta_pds_): dat_phates = [] if dat in phates: dat_phates = phates[dat][idx] tsv, meta = tsv_meta_pds meta_pd = read_meta_pd(meta) meta_pd = meta_pd.set_index('sample_name') cases_dict = check_metadata_cases_dict(meta, meta_pd, dict(main_cases_dict), 'DOC') cur_raref = datasets_rarefs[dat][idx] dat_cases_tabs[dat][cur_raref] = {} if not split: out_sh = '%s/run_doc_%s%s%s.sh' % (job_folder2, dat, filt_raref, cur_raref) out_import_sh = '%s/run_import_doc_%s%s%s.sh' % ( job_folder2, dat, filt_raref, cur_raref) odir = get_analysis_folder(i_datasets_folder, 'doc/%s' % dat) for filt, (fp, fa) in filters.items(): if split: out_sh = '%s/run_doc_%s%s%s_%s.sh' % ( job_folder2, dat, filt_raref, cur_raref, filt) out_import_sh = '%s/run_import_doc_%s%s%s_%s.sh' % ( job_folder2, dat, filt_raref, cur_raref, filt) for case_var, case_vals_list in cases_dict.items(): cur_sh = '%s/run_doc_%s_%s%s%s_%s.sh' % ( job_folder2, dat, case_var, filt_raref, cur_raref, filt) cur_sh = cur_sh.replace(' ', '-') cur_import_sh = '%s/run_import_doc_%s_%s%s%s_%s.sh' % ( job_folder2, dat, case_var, filt_raref, cur_raref, filt) cur_import_sh = cur_import_sh.replace(' ', '-') all_sh_pbs.setdefault((dat, out_sh), []).append(cur_sh) all_import_sh_pbs.setdefault((dat, out_import_sh), []).append(cur_import_sh) cases = run_single_doc( i_datasets_folder, odir, tsv, meta_pd, case_var, doc_params, case_vals_list, cur_sh, cur_import_sh, force, filt, cur_raref, fp, fa, run_params["n_nodes"], run_params["n_procs"], dat_phates, doc_phate, need_to_run_phate, need_to_run_less_phate) dat_cases_tabs[dat][cur_raref].setdefault(case_var, []).extend(cases) for need_to_run in need_to_run_phate: print(' -', need_to_run) job_folder = get_job_folder(i_datasets_folder, 'doc') main_sh = write_main_sh(job_folder, '3_run_import_doc%s' % filt_raref, all_import_sh_pbs, '%s.doc.mpt%s' % (prjct_nm, filt_raref), "4", "1", "1", "500", "mb", qiime_env, chmod, noloc, slurm, jobs, chunkit) if main_sh: if p_doc_config: if p_doc_config.startswith('/panfs'): p_doc_config = p_doc_config.replace(os.getcwd(), '') print('# Import for DOC (groups config in %s)' % p_doc_config) else: print('# Import DOC') print_message('', 'sh', main_sh, jobs) main_sh = write_main_sh(job_folder, '3_run_doc%s' % filt_raref, all_sh_pbs, '%s.doc%s' % (prjct_nm, filt_raref), run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, jobs, slurm, chunkit, '~/.') if main_sh: if p_doc_config: if p_doc_config.startswith('/panfs'): p_doc_config = p_doc_config.replace(os.getcwd(), '') print('# DOC (groups config in %s)' % p_doc_config) else: print('# DOC') print_message('', 'sh', main_sh, jobs) do_r = 1 if do_r: job_folder = get_job_folder(i_datasets_folder, 'doc/R') job_folder2 = get_job_folder(i_datasets_folder, 'doc/R/chunks') main_written = 0 main_sh = '%s/run_R_doc%s.sh' % (job_folder, filt_raref) with open(main_sh, 'w') as main_o: for dat, raref_case_var_cases in dat_cases_tabs.items(): shs = [] written = 0 odir = get_analysis_folder(i_datasets_folder, 'doc/%s' % dat) log_error = '%s/log.error' % odir for raref, case_var_cases in raref_case_var_cases.items(): for case_var, cases in case_var_cases.items(): for cdx, case in enumerate(cases): plot = '%s_%s_%s_%s' % (dat, raref, case_var, cdx) case_r = '%s/R' % case pdf = '%s/plot.pdf' % case_r do = '%s/DO.tsv' % case_r if not isfile(pdf): cur_r = '%s/run_R_doc_%s_%s_%s_vanilla.R' % ( job_folder2, dat, case_var, cdx) cur_sh = 'echo "*** %s" >> %s\n' % (plot, log_error) cur_sh += 'R -f %s --vanilla 2>> %s\n' % ( cur_r, log_error) cur_sh += 'echo "end" >> %s\n' % log_error shs.append(cur_sh) with open(cur_r, 'w') as o: o.write("library(DOC)\n") o.write("library(ggplot2)\n") if not isfile(do): o.write( "otu <- read.table('%s/tab.tsv', header=T, sep='\\t', comment.char='', check.names=F, nrows=2)\n" % case) o.write( "index_name <- colnames(otu)[1]\n") o.write( "otu <- read.table('%s/tab.tsv', header=T, sep='\\t', comment.char='', check.names=F, row.names=index_name)\n" % case) o.write("if (dim(otu)[1] > 100) {\n") o.write(" res <- DOC(otu)\n") o.write( " res.null <- DOC.null(otu)\n") o.write( " write.table(x=res$DO, file='%s/DO.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write( " write.table(x=res$LME, file='%s/LME.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write( " colnames(res$NEG) <- c('Neg_Slope', 'Data')\n" ) o.write( " write.table(x=res$NEG, file='%s/NEG.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write( " write.table(x=res$FNS, file='%s/FNS.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write( " write.table(x=res$BOOT, file='%s/BOOT.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write( " write.table(x=res$CI, file='%s/CI.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write( " write.table(x=res.null$DO, file='%s/null_DO.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write( " write.table(x=res.null$LME, file='%s/null_LME.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write( " colnames(res.null$NEG) <- c('Neg_Slope', 'Data')\n" ) o.write( " write.table(x=res.null$NEG, file='%s/null_NEG.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write( " write.table(x=res.null$FNS, file='%s/null_FNS.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write( " write.table(x=res.null$BOOT, file='%s/null_BOOT.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write( " write.table(x=res.null$CI, file='%s/null_CI.tsv', sep='\\t', quote=F, row.names=F)\n" % case_r) o.write("}\n") o.write( "res = list(BOOT=read.table('%s/BOOT.tsv', h=T, sep='\\t'), CI=read.table('%s/CI.tsv', h=T, sep='\\t'), DO=read.table('%s/DO.tsv', h=T, sep='\\t'), LME=read.table('%s/LME.tsv', h=T, sep='\\t'), FNS=read.table('%s/FNS.tsv', h=T, sep='\\t'), NEG=read.table('%s/NEG.tsv', h=T, sep='\\t'))\n" % (case_r, case_r, case_r, case_r, case_r, case_r)) o.write( "res.null = list(BOOT=read.table('%s/null_BOOT.tsv', h=T, sep='\\t'), CI=read.table('%s/null_CI.tsv', h=T, sep='\\t'), DO=read.table('%s/null_DO.tsv', h=T, sep='\\t'), LME=read.table('%s/null_LME.tsv', h=T, sep='\\t'), FNS=read.table('%s/null_FNS.tsv', h=T, sep='\\t'), NEG=read.table('%s/null_NEG.tsv', h=T, sep='\\t'))\n" % (case_r, case_r, case_r, case_r, case_r, case_r)) o.write( "colnames(res$NEG) <- c('Neg.Slope', 'Data')\n" ) o.write( "colnames(res.null$NEG) <- c('Neg.Slope', 'Data')\n" ) o.write( "res$DO <- res$DO[which(res$DO$Overlap <= 1),]\n" ) o.write( "res.null$DO <- res.null$DO[which(res.null$DO$Overlap <= 1),]\n" ) o.write("pdf('%s')\n" % pdf) o.write( "merged <- DOC.merge(list(s_%s = res, s_%s=res.null))\n" % (plot, plot)) o.write("plot(merged)\n") o.write("dev.off()\n") main_written += 1 written += 1 if written: if chunkit and len(shs) >= chunkit: chunks = [ list(x) for x in np.array_split(np.array(shs), chunkit) ] if split and len(shs) >= 3: chunks = [ list(x) for x in np.array_split(np.array(shs), 3) ] else: chunks = [shs] for cdx, chunk in enumerate(chunks): out_sh = '%s/run_R_doc_%s%s_%s.sh' % (job_folder2, dat, filt_raref, cdx) out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as o: for c in chunk: o.write('echo "%s"\n\n' % c) o.write('%s\n\n' % c) run_xpbs( out_sh, out_pbs, '%s.doc.R.%s%s_%s' % (prjct_nm, dat, filt_raref, cdx), 'xdoc', run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', main_o, noloc, slurm, jobs) if main_written: print_message('# DOC (R)', 'sh', main_sh, jobs)
def run_rarefy(i_datasets_folder: str, datasets: dict, datasets_read: dict, datasets_phylo: dict, datasets_filt_map: dict, datasets_rarefs: dict, p_raref_depths: str, eval_rarefs: bool, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, run_params: dict, filt_raref: str, filt_only: bool, jobs: bool, slurm: bool, chunkit: int) -> dict: """ Run rarefy: Rarefy table. https://docs.qiime2.org/2019.10/plugins/available/feature-table/rarefy/ :param i_datasets_folder: Path to the folder containing the data/metadata subfolders. :param datasets: dataset -> [tsv/biom path, meta path] :param datasets_read: dataset -> [tsv table, meta table] :param datasets_features: dataset -> list of features names in the dataset tsv / biom file. :param datasets_phylo: to be updated with ('tree_to_use', 'corrected_or_not') per dataset. :param force: Force the re-writing of scripts for all commands. :param prjct_nm: Nick name for your project. :param qiime_env: qiime2-xxxx.xx conda environment. :param chmod: whether to change permission of output files (defalt: 775). :return: deta divesity matrices. """ evaluation = '' eval_depths = {} datasets_raref_depths, datasets_raref_evals = check_rarefy_need( i_datasets_folder, datasets_read, p_raref_depths) if eval_rarefs: evaluation = '_eval' set_filt_rarefy(datasets_raref_depths, datasets_filt_map) datasets_update = {} datasets_read_update = {} datasets_phylo_update = {} datasets_append = {} main_written = 0 job_folder = get_job_folder(i_datasets_folder, 'rarefy%s' % evaluation) job_folder2 = get_job_folder(i_datasets_folder, 'rarefy%s/chunks' % evaluation) to_chunk = [] run_pbs = '%s/1_run_rarefy_%s%s%s.sh' % (job_folder, prjct_nm, evaluation, filt_raref) with open(run_pbs, 'w') as o: for dat, tsv_meta_pds_ in datasets.items(): written = 0 if dat not in datasets_raref_depths: continue if filt_only and dat not in datasets_filt_map: continue odir = get_analysis_folder(i_datasets_folder, 'rarefy%s/%s' % (evaluation, dat)) out_sh = '%s/run_rarefy_%s%s_%s.sh' % (job_folder2, prjct_nm, evaluation, dat) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: depths = datasets_raref_depths[dat][1] if eval_rarefs: depths = datasets_raref_evals[dat] tsv_pd, meta_pd = datasets_read[dat][0] tsv_sums = tsv_pd.sum() for tsv_meta_pds in tsv_meta_pds_: tsv, meta = tsv_meta_pds for depth_ in depths: depth = get_digit_depth(depth_, tsv_sums) dat_raref = '%s_raref%s%s' % (dat, evaluation, str(depth)) meta_out = '%s/meta_%s.tsv' % (odir, dat_raref) remaining_samples = tsv_sums[ tsv_sums >= depth].index.tolist() meta_raref_pd = meta_pd.loc[ meta_pd.sample_name.isin(remaining_samples), :] meta_raref_pd.to_csv(meta_out, index=False, sep='\t') qza = tsv.replace('.tsv', '.qza') qza_out = '%s/tab_%s.qza' % (odir, dat_raref) tsv_out = '%s.tsv' % splitext(qza_out)[0] if force or not os.path.isfile(tsv_out): cmd = write_rarefy(qza, qza_out, depth) cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n\n' % cmd) cmd = run_export(qza_out, tsv_out, 'FeatureTable[Frequency]') cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n\n' % cmd) main_written += 1 written += 1 if eval_rarefs: eval_depths.setdefault(dat, []).append( '%s_%s' % (dat, str(depth))) datasets_update['%s_%s' % (dat, str(depth))] = [[ tsv_out, meta_out ]] datasets_read_update['%s_%s' % (dat, str(depth))] = ( 'raref', str(depth)) datasets_phylo_update[ '%s_%s' % (dat, str(depth))] = datasets_phylo[dat] else: datasets_append.setdefault(dat, []).append( [tsv_out, meta_out]) if isfile(tsv_out) and isfile(meta_out): tab_filt_pd = pd.read_csv(tsv_out, index_col=0, header=0, sep='\t') with open(meta_out) as f: for line in f: break meta_filt_pd = pd.read_csv( meta_out, header=0, sep='\t', dtype={line.split('\t')[0]: str}, low_memory=False) datasets_read[dat].append( [tab_filt_pd, meta_filt_pd]) else: datasets_read[dat].append( ('raref', str(depth))) datasets_rarefs.setdefault(dat, []).append( '_raref%s%s' % (evaluation, str(depth))) to_chunk.append(out_sh) if not chunkit: run_xpbs( out_sh, out_pbs, '%s.bt%s.%s%s' % (prjct_nm, evaluation, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if to_chunk and chunkit: simple_chunks(run_pbs, job_folder2, to_chunk, 'rarefy%s' % evaluation, prjct_nm, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit, None) if main_written: print_message('# Get rarefied datasets', 'sh', run_pbs, jobs) if eval_rarefs: datasets.update(datasets_update) datasets_read.update(datasets_read_update) datasets_phylo.update(datasets_phylo_update) else: for dat, fps in datasets_append.items(): datasets[dat].extend(fps) return eval_depths
def run_procrustes(i_datasets_folder: str, datasets_filt: dict, p_procrustes: str, betas: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, slurm: bool, split: bool, run_params: dict, filt_raref: str, filt_only: bool, eval_depths: dict, jobs: bool, chunkit: int) -> None: """ """ evaluation = '' if eval_depths: evaluation = '_eval' procrustes_pairs = {} for dat, depths in eval_depths.items(): sorted_depths = sorted(depths, key=lambda x: int(x.split('_')[-1])) for idx, x in enumerate(sorted_depths[:-1]): y = sorted_depths[(idx + 1)] n0 = x.split('_')[-1] n1 = y.split('_')[-1] procrustes_pairs['%s_%s' % (n0, n1)] = [x, y] procrustes_subsets = {'ALL': [[]]} else: procrustes_pairs, procrustes_subsets = get_procrustes_mantel_dicts( p_procrustes) get_job_folder(i_datasets_folder, 'procrustes%s' % evaluation) dms_tab = [] all_sh_pbs = {} missing_dats = set() for pair, (dat1_, dat2_) in procrustes_pairs.items(): dat1, raref1 = get_dat_idx(dat1_, evaluation, datasets_filt, filt_only) dat2, raref2 = get_dat_idx(dat2_, evaluation, datasets_filt, filt_only) if check_dat_exists(betas, dat1, missing_dats) or check_dat_exists( betas, dat2, missing_dats): continue if evaluation: metrics_groups_metas_qzas_dms_trees1 = betas[dat1] metrics_groups_metas_qzas_dms_trees2 = betas[dat2] else: metrics_groups_metas_qzas_dms_trees1 = betas[dat1][0] metrics_groups_metas_qzas_dms_trees2 = betas[dat2][0] job_folder2 = get_job_folder( i_datasets_folder, 'procrustes%s/chunks/%s%s' % (evaluation, pair, filt_raref)) if not split: out_sh = '%s/run_procrustes_%s%s_%s%s.sh' % ( job_folder2, prjct_nm, evaluation, pair, filt_raref) for metric, groups_metas_qzas_dms_trees1 in metrics_groups_metas_qzas_dms_trees1.items( ): if split: out_sh = '%s/run_procrustes_%s%s_%s_%s%s.sh' % ( job_folder2, prjct_nm, evaluation, pair, metric, filt_raref) if metric not in metrics_groups_metas_qzas_dms_trees2: continue groups_metas_qzas_dms_trees2 = metrics_groups_metas_qzas_dms_trees2[ metric] groups1 = sorted(groups_metas_qzas_dms_trees1.keys()) groups2 = sorted(groups_metas_qzas_dms_trees2.keys()) for (group1_, group2_) in itertools.product(*[groups1, groups2]): if group1_ == '': group1 = 'full' else: group1 = group1_ if group2_ == '': group2 = 'full' else: group2 = group2_ meta1, qza1, dm1, tree1 = groups_metas_qzas_dms_trees1[ group1_][0] meta2, qza2, dm2, tree2 = groups_metas_qzas_dms_trees2[ group2_][0] skip = 0 if not evaluation: if '__raref' in dat1_: dm1, meta1 = get_dm_meta(dat1, dm1, meta1, raref1, metric, i_datasets_folder, skip) if '__raref' in dat2_: dm2, meta2 = get_dm_meta(dat2, dm2, meta2, raref2, metric, i_datasets_folder, skip) if skip: print( '[Proscustes] One desired rarefaction depth not run (pair %s)' % pair) continue meta_pd1 = read_meta_pd(meta1) meta_pd2 = read_meta_pd(meta2) common_sams = list( set(meta_pd1.sample_name) & set(meta_pd2.sample_name)) if len(common_sams) < 3: continue meta_pd = meta_pd1.loc[meta_pd1.sample_name.isin(common_sams)] cases_dict = check_metadata_cases_dict( meta1, meta_pd, dict(procrustes_subsets), 'procrustes') odir = get_analysis_folder( i_datasets_folder, 'procrustes%s/%s%s/%s_vs_%s' % (evaluation, pair, filt_raref, group1, group2)) job_folder3 = get_job_folder( i_datasets_folder, 'procrustes%s/chunks/%s%s/%s_vs_%s' % (evaluation, pair, filt_raref, group1, group2)) for case_var, case_vals_list in cases_dict.items(): for case_vals in case_vals_list: case_ = get_case(case_vals, case_var).replace(' ', '_') cur = '%s__%s' % (metric, case_) cur_sh = '%s/run_procrustes%s_%s%s.sh' % ( job_folder3, evaluation, cur, filt_raref) cur_sh = cur_sh.replace(' ', '-') all_sh_pbs.setdefault((pair, out_sh), []).append(cur_sh) dm_out1 = '%s/dm_%s__%s_DM.qza' % (odir, dat1_, cur) dm_out2 = '%s/dm_%s__%s_DM.qza' % (odir, dat2_, cur) dm_out1_tsv = '%s.tsv' % splitext(dm_out1)[0] dm_out2_tsv = '%s.tsv' % splitext(dm_out2)[0] biplot = '%s/procrustes%s_%s__%s__%s.qzv' % ( odir, evaluation, dat1_, dat2_, cur) run_single_procrustes_mantel('procrustes', odir, dm1, dm2, meta_pd, dm_out1, dm_out2, biplot, cur_sh, cur, case_var, case_vals, force) dms_tab.append([ pair, dat1_, dat2_, group1, group2, case_, metric, dm_out1_tsv, dm_out2_tsv ]) job_folder = get_job_folder(i_datasets_folder, 'procrustes%s' % evaluation) main_sh = write_main_sh( job_folder, '4_run_procrustes_%s%s%s' % (prjct_nm, evaluation, filt_raref), all_sh_pbs, '%s.prcst%s%s' % (prjct_nm, evaluation, filt_raref), run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit) if main_sh: if p_procrustes and p_procrustes != 1: if p_procrustes.startswith('/panfs'): p_procrustes = p_procrustes.replace(os.getcwd(), '') print('# Procrustes (pairs and samples subsets config in %s)' % p_procrustes) else: print('# Procrustes') print_message('', 'sh', main_sh, jobs) dms_tab_pd = pd.DataFrame(dms_tab, columns=[ 'pair', 'dat1', 'dat2', 'metric', 'group1', 'group2', 'case', 'dm_out1', 'dm_out2', ]) odir = get_analysis_folder(i_datasets_folder, 'procrustes%s/R' % evaluation) out_Rs = glob.glob('%s/pairs_proscrustes_results%s%s*.tsv' % (odir, evaluation, filt_raref)) if len(out_Rs): done_R = pd.concat([pd.read_table(x, sep=' ') for x in out_Rs]) dms_tab_pd = dms_tab_pd.loc[~dms_tab_pd[['dm_out1', 'dm_out2']].sum(1). isin(done_R[['f1', 'f2']].sum(1))] if dms_tab_pd.shape[0]: fp_num = 0 if len(out_Rs): last = sorted( out_Rs, key=lambda fp: int(fp.split('.tsv')[0].split('_')[-1])) fp_num = int(last[-1].split('.tsv')[0].split('_')[-1]) + 1 dms_tab_fp = '%s/pairs%s%s_%s.tsv' % (odir, evaluation, filt_raref, fp_num) dms_tab_pd.to_csv(dms_tab_fp, index=False, sep='\t') out_R = '%s/pairs_proscrustes_results%s%s_%s.tsv' % ( odir, evaluation, filt_raref, fp_num) job_folder = get_job_folder(i_datasets_folder, 'procrustes/R') R_script = '%s/4_run_procrustes_%s%s.R' % (job_folder, prjct_nm, filt_raref) with open(R_script, 'w') as o: o.write("library(vegan)\n") o.write("dms_files <- read.table('%s', h=T)\n" % dms_tab_fp) o.write( "cols <- c('pair', 'd1', 'd2', 'g1', 'g2', 'case', 'metric', 'f1', 'f2', 'samples', 'M2', 'p-value')\n" ) o.write( "res <- setNames(data.frame(matrix(ncol = 12, nrow = 0)), cols)\n" ) o.write("for (i in seq(1, dim(dms_files)[1])) {\n") o.write(" row <- as.vector(unlist(dms_files[i,]))\n") o.write(" pair <- row[1]\n") o.write(" d1 <- row[2]\n") o.write(" d2 <- row[3]\n") o.write(" group1 <- row[4]\n") o.write(" group2 <- row[5]\n") o.write(" case <- row[6]\n") o.write(" metric <- row[7]\n") o.write(" f1 <- row[8]\n") o.write(" f2 <- row[9]\n") o.write(" if (sum(file.exists(f1, f2)) == 2) {\n") o.write( " filin_tsv_pd1 <- read.csv(f1, header = TRUE, check.names=FALSE,\n" ) o.write( " row.names = 1, colClasses = 'character', sep = '\\t')\n" ) o.write( " filin_tsv_pd2 <- read.csv(f2, header = TRUE, check.names=FALSE,\n" ) o.write( " row.names = 1, colClasses = 'character', sep = '\\t')\n" ) o.write(" filin_tsv_pd1 <- data.matrix(filin_tsv_pd1)\n") o.write(" filin_tsv_pd2 <- data.matrix(filin_tsv_pd2)\n") o.write( " filin_tsv_pd1 <- filin_tsv_pd1[rownames(filin_tsv_pd2), rownames(filin_tsv_pd2)]\n" ) o.write( " # procrustes12 <- procrustes(filin_tsv_pd1, filin_tsv_pd2, kind=2, permutations=999)\n" ) o.write( " prtst <- protest(filin_tsv_pd1, filin_tsv_pd2, permutations = 999)\n" ) o.write(" n <- dim(filin_tsv_pd1)[1]\n") o.write( " res[i,] <- c(pair, d1, d2, group1, group2, case, metric, f1, f2, n, prtst$ss, prtst$signif)\n" ) o.write(" }\n") o.write("}\n") o.write("write.table(x = res, file = '%s')\n" % out_R) out_sh = '%s/4_run_procrustes_%s%s_R%s.sh' % (job_folder, prjct_nm, evaluation, filt_raref) out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as o: o.write('R -f %s --vanilla\n' % R_script) run_xpbs( out_sh, out_pbs, '%s.prcrt%s.R%s' % (prjct_nm, evaluation, filt_raref), 'renv', run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, 1, '# Procrustes for stats in R (pairs and samples subsets config in %s)' % p_procrustes, None, False, jobs)
def merge_meta_alpha(i_datasets_folder: str, datasets: dict, datasets_rarefs: dict, diversities: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, slurm: bool, dropout: bool, run_params: dict, filt_raref: str, eval_depths: dict, jobs: bool, chunkit: int) -> dict: """ Computes the alpha diversity vectors for each dataset. :param i_datasets_folder: Path to the folder containing the data/metadata subfolders. :param datasets: list of datasets. :param datasets_rarefs: list of rarefied datasets. :param diversities: paths to [alpha_divs] :param force: Force the re-writing of scripts for all commands. :param prjct_nm: Short nick name for your project. :param qiime_env: name of your qiime2 conda environment (e.g. qiime2-2019.10). :param chmod: whether to change permission of output files (defalt: 775). :return: """ evaluation = '' if len(eval_depths): evaluation = '_eval' job_folder = get_job_folder(i_datasets_folder, 'tabulate%s' % evaluation) job_folder2 = get_job_folder(i_datasets_folder, 'tabulate%s/chunks' % evaluation) to_export = {} to_chunk = [] main_written = 0 run_pbs = '%s/2_run_merge_alphas_%s%s%s.sh' % (job_folder, prjct_nm, evaluation, filt_raref) with open(run_pbs, 'w') as o: for dat, group_divs_list in diversities.items(): written = 0 to_export[dat] = [] out_sh = '%s/run_merge_alpha_%s%s_%s%s.sh' % ( job_folder2, prjct_nm, evaluation, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: for idx, group_divs in enumerate(group_divs_list): tsv, meta = datasets[dat][idx] cur_raref = datasets_rarefs[dat][idx] base = basename(splitext(tsv)[0]).lstrip('tab_') to_export_groups = [] for group, divs in group_divs.items(): if group: output_folder = get_analysis_folder( i_datasets_folder, 'tabulate%s/%s%s/%s' % (evaluation, dat, cur_raref, group)) else: output_folder = get_analysis_folder( i_datasets_folder, 'tabulate%s/%s%s' % (evaluation, dat, cur_raref)) if dropout: out_fp = '%s/%s_alphas__%s.qzv' % (output_folder, base, group) else: out_fp = '%s/%s_alphas_noDropout__%s.qzv' % ( output_folder, base, group) out_fp_tsv = '%s.tsv' % splitext(out_fp)[0] if isfile(out_fp_tsv): with open(out_fp_tsv) as f: for line in f: indices = line.strip().split('\t')[1:] break divs_alphas = [x[1] for x in divs] if len(indices) < len(divs_alphas): force = True to_export_groups.append(out_fp_tsv) if force or not isfile(out_fp): write_metadata_tabulate(out_fp, divs, meta, cur_sh) cmd = run_export(out_fp, out_fp_tsv, '') cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n\n' % cmd) main_written += 1 written += 1 to_export[dat].append(to_export_groups) to_chunk.append(out_sh) if not chunkit: run_xpbs( out_sh, out_pbs, '%s.mrg.lph%s.%s%s' % (prjct_nm, evaluation, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if to_chunk and chunkit: simple_chunks(run_pbs, job_folder2, to_chunk, 'tabulate', prjct_nm, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit, None) if main_written: print_message('# Merge and export alpha diversity indices', 'sh', run_pbs, jobs) return to_export
def run_qemistree(i_datasets_folder: str, datasets: dict, prjct_nm: str, i_qemistree: str, taxonomies: dict, force: bool, qiime_env: str, chmod: str, noloc: bool, slurm: bool, run_params: dict, filt_raref: str, jobs: bool, chunkit: int) -> None: """ :param i_datasets_folder: Path to the folder containing the data/metadata subfolders. :param datasets_read: dataset -> [tsv table, meta table] :param prjct_nm: Short nick name for your project. :param i_qemistree: path to qemistree folder (feature-data and tree). :param taxonomies: dataset -> [method, assignment qza] :param force: Force the re-writing of scripts for all commands. :param qiime_env: name of your qiime2 conda environment (e.g. qiime2-2019.10). :param chmod: whether to change permission of output files (defalt: 775). """ job_folder = get_job_folder(i_datasets_folder, 'qemistree') job_folder2 = get_job_folder(i_datasets_folder, 'qemistree/chunks') written = 0 to_chunk = [] run_pbs = '%s/1_run_qemistree_%s%s.sh' % (job_folder, prjct_nm, filt_raref) with open(run_pbs, 'w') as o: for dat, tsv_meta_pds in datasets.items(): feature_data = '%s/feature-data_%s.qza' % (i_qemistree, dat) qemistree = '%s/qemistree_%s.qza' % (i_qemistree, dat) if not isfile(feature_data) or not isfile(qemistree): continue out_sh = '%s/run_qemistree_%s_%s%s.sh' % (job_folder2, prjct_nm, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] odir = get_analysis_folder(i_datasets_folder, 'qemistree/%s' % dat) classyfire_qza = '%s/%s-classyfire.qza' % (odir, dat) classyfire_tsv = '%s.tsv' % splitext(classyfire_qza)[0] with open(out_sh, 'w') as cur_sh: if force or not isfile(classyfire_tsv): write_qemistree(feature_data, classyfire_qza, classyfire_tsv, qemistree, cur_sh) written += 1 if isfile(classyfire_tsv): odir = get_analysis_folder(i_datasets_folder, 'taxonomy/%s' % dat) out_rad = '%s/tax_%s' % (odir, dat) tax_qza = '%s.qza' % out_rad tax_tsv = '%s.tsv' % out_rad classyfire_pd = pd.read_csv(classyfire_tsv, header=0, sep='\t') with open(tax_tsv, 'w') as o: cols = ['id', 'kingdom', 'superclass', 'class', 'subclass', 'direct_parent'] o.write('Feature ID\tTaxon\n') for row in classyfire_pd[cols].values: o.write('%s\t%s\n' % (row[0], '; '.join(row[1:]))) run_export(tax_tsv, tax_qza, 'FeatureData[Taxonomy]') taxonomies[dat] = ['direct_parent', tax_qza] written += 1 else: print('[Warning] Maybe run qemistree first and then re-run pipeline to ' 'have the classyfire taxonomy include in the barplots!') to_chunk.append(out_sh) if not chunkit: run_xpbs(out_sh, out_pbs, '%s.qmstr.%s%s' % (prjct_nm, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if to_chunk and chunkit: simple_chunks(run_pbs, job_folder2, to_chunk, 'qemistree', prjct_nm, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit, None) if written: print_message('# Make qemistree classyfire classifications', 'sh', run_pbs, jobs)
def filter_rare_samples(i_datasets_folder: str, datasets: dict, datasets_read: dict, datasets_features: dict, datasets_rarefs: dict, datasets_filt: dict, datasets_filt_map: dict, datasets_phylo: dict, prjct_nm: str, qiime_env: str, p_filt_threshs: str, chmod: str, noloc: bool, run_params: dict, filt_raref: str, jobs: bool, slurm: bool, chunkit: int) -> None: """ Filter the rare features, keep samples with enough reads/features and import to Qiime2. :param i_datasets_folder: Path to the folder containing the data/metadata subfolders. :param datasets: dataset -> [tsv/biom path, meta path] :param datasets_read: dataset -> [tsv table, meta table] :param datasets_features: dataset -> list of features names in the dataset tsv / biom file. :param datasets_phylo: to be updated with ('tree_to_use', 'corrected_or_not') per dataset. :param prjct_nm: Short nick name for your project. :param qiime_env: name of your qiime2 conda environment (e.g. qiime2-2019.10). :param thresh: min number of reads per sample to keep it. :param chmod: whether to change permission of output files (defalt: 775). """ threshs_dats = read_yaml_file(p_filt_threshs) written = 0 datasets_update = {} datasets_read_update = {} datasets_features_update = {} datasets_phylo_update = {} job_folder = get_job_folder(i_datasets_folder, 'import_filtered') out_sh = '%s/1_run_import_filtered_%s%s.sh' % (job_folder, prjct_nm, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] to_chunk = [] with open(out_sh, 'w') as sh: for dat, tab_meta_pds_ in datasets_read.items(): if dat not in threshs_dats: continue names, thresh_sam, thresh_feat = get_thresholds(threshs_dats[dat]) if no_filtering(dat, thresh_sam, thresh_feat): continue dat_filt = get_dat_filt(dat, names, thresh_sam, thresh_feat) datasets_filt[dat] = dat_filt datasets_filt_map[dat_filt] = dat datasets_rarefs[dat_filt] = [''] tsv_filt, qza_filt, meta_filt = get_fps(i_datasets_folder, dat_filt) if isfile(qza_filt) and isfile(meta_filt): datasets_update[dat_filt] = [[tsv_filt, meta_filt]] tab_filt_pd = pd.read_csv(tsv_filt, index_col=0, header=0, sep='\t') with open(meta_filt) as f: for line in f: break meta_filt_pd = pd.read_csv(meta_filt, header=0, sep='\t', dtype={line.split('\t')[0]: str}, low_memory=False) # datasets_read_update[dat_filt] = [tab_filt_pd, meta_filt_pd] datasets_read_update[dat_filt] = [[tab_filt_pd, meta_filt_pd]] datasets_phylo_update[dat_filt] = datasets_phylo[dat] datasets_features_update[dat_filt] = dict( gid_feat for gid_feat in datasets_features[dat].items() if gid_feat[1] in tab_filt_pd.index) continue for (tab_pd, meta_pd) in tab_meta_pds_: tab_filt_pd = filtering_thresholds(names, thresh_sam, thresh_feat, tab_pd) if harsh_filtering(dat_filt, tab_filt_pd): continue meta_filt_pd = meta_pd.loc[meta_pd.sample_name.isin( tab_filt_pd.columns.tolist())].copy() tab_filt_pd.reset_index().to_csv(tsv_filt, index=False, sep='\t') meta_filt_pd.to_csv(meta_filt, index=False, sep='\t') datasets_update[dat_filt] = [[tsv_filt, meta_filt]] datasets_read_update[dat_filt] = [[tab_filt_pd, meta_filt_pd]] datasets_phylo_update[dat_filt] = datasets_phylo[dat] datasets_features_update[dat_filt] = dict( gid_feat for gid_feat in datasets_features[dat].items() if gid_feat[1] in tab_filt_pd.index) cmd = run_import(tsv_filt, qza_filt, "FeatureTable[Frequency]") sh.write('echo "%s"\n' % cmd) sh.write('%s\n' % cmd) written += 1 if written: run_xpbs( out_sh, out_pbs, '%s.fltr%s' % (prjct_nm, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, '# Filter samples for a min number of %s reads' % p_filt_threshs, None, noloc, slurm, jobs) # after this update, the raw dataset remain included datasets.update(datasets_update) datasets_read.update(datasets_read_update) datasets_features.update(datasets_features_update) datasets_phylo.update(datasets_phylo_update)
def import_datasets(i_datasets_folder: str, datasets: dict, datasets_phylo: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, run_params: dict, filt_raref: str, jobs: bool, slurm: bool, chunkit: int) -> None: """Initial imports of the .tsv datasets in to Qiime2 Artefacts Parameters ---------- i_datasets_folder : str Names identifying the datasets in the input folder datasets : dict Mapping dataset name -> [data file path, metadata file path] datasets_phylo : dict Mapping dataset name -> ('tree_to_use', 'corrected_or_not') force : bool Force the re-writing of scripts for all commands prjct_nm : str Nick name for the project. qiime_env : str Name of a qiime2 conda environment where analysis tools to be run are installed chmod : str noloc : bool run_params : dict filt_raref : str jobs : bool chunkit : int Returns ------- """ job_folder = get_job_folder(i_datasets_folder, 'import_tables') job_folder2 = get_job_folder(i_datasets_folder, 'import_tables/chunks') to_chunk = [] main_written = 0 run_pbs = '%s/0_run_import_%s%s.sh' % (job_folder, prjct_nm, filt_raref) with open(run_pbs, 'w') as o: for dat, tsv_meta_pds_ in datasets.items(): written = 0 out_sh = '%s/0_run_import_%s_%s%s.sh' % (job_folder2, prjct_nm, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: for tsv_meta_pds in tsv_meta_pds_: # REMOVE IF FIXED NOT KEPT tsv, meta = tsv_meta_pds qza = '%s.qza' % splitext(tsv)[0] if datasets_phylo[dat][1]: cmd = run_import(tsv, qza, 'FeatureTable[Frequency]') cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n' % cmd) written += 1 elif force or not isfile(qza): cmd = run_import(tsv, qza, 'FeatureTable[Frequency]') cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n' % cmd) written += 1 if written: main_written += 1 to_chunk.append(out_sh) if not chunkit: job_name = '%s.mprt.%s%s' % (prjct_nm, dat, filt_raref) run_xpbs(out_sh, out_pbs, job_name, qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if to_chunk and chunkit: simple_chunks(run_pbs, job_folder2, to_chunk, 'imports', prjct_nm, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit, None) if main_written: print_message('# Import tables to qiime2', 'sh', run_pbs, jobs)
def run_volatility(i_datasets_folder: str, datasets: dict, p_longi_column: str, datasets_rarefs: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, slurm: bool, run_params: dict, filt_raref: str, jobs: bool, chunkit: int) -> None: """ Run volatility: Generate interactive volatility plot. https://docs.qiime2.org/2019.10/plugins/available/longitudinal/volatility/ :param i_datasets_folder: Path to the folder containing the data/metadata subfolders. :param datasets: list of datasets. :param p_longi_column: metadata column that is the time stratification. :param force: Force the re-writing of scripts for all commands. :param prjct_nm: Nick name for your project. :param qiime_env: qiime2-xxxx.xx conda environment. :param chmod: whether to change permission of output files (defalt: 775). """ job_folder = get_job_folder(i_datasets_folder, 'longitudinal') job_folder2 = get_job_folder(i_datasets_folder, 'longitudinal/chunks') main_written = 0 first_print = 0 first_print2 = 0 to_chunk = [] run_pbs = '%s/5_run_volatility_%s%s.sh' % (job_folder, prjct_nm, filt_raref) with open(run_pbs, 'w') as o: for dat, tsv_meta_pds_ in datasets.items(): written = 0 out_sh = '%s/run_volatility_%s_%s%s.sh' % (job_folder2, prjct_nm, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: for idx, tsv_meta_pds in enumerate(tsv_meta_pds_): tsv, meta = tsv_meta_pds cur_raref = datasets_rarefs[dat][idx] meta_alphas = '%s_alphas.tsv' % splitext(meta)[0] if not isfile(meta_alphas): if not first_print: print( '\nWarning: First make sure you run alpha -> alpha merge/export (2_run_merge_alphas.sh) ' ' before running volatility\n\t(if you need the alpha as a response variable)!' ) first_print += 1 continue with open(meta) as f: for line in f: break time_point = [ x for x in line.strip().split('\t') if p_longi_column in x ][0] if not time_point: if not first_print2: print('Variable %s not in metadata %s\n' % (p_longi_column, meta_alphas)) first_print2 += 1 continue odir = get_analysis_folder( i_datasets_folder, 'longitudinal/%s%s' % (dat, cur_raref)) out_fp = '%s/%s_volatility.qzv' % (odir, dat) if force or not isfile(out_fp): write_longitudinal_volatility(out_fp, meta_alphas, time_point, cur_sh) written += 1 main_written += 1 to_chunk.append(out_sh) if not chunkit: run_xpbs(out_sh, out_pbs, '%s.vltlt.%s%s' % (prjct_nm, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if to_chunk and chunkit: simple_chunks(run_pbs, job_folder2, to_chunk, 'volatility', prjct_nm, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit, None) if main_written: print_message('# Longitudinal change in alpha diversity indices', 'sh', run_pbs, jobs)
def run_alpha_rarefaction(i_datasets_folder: str, datasets: dict, datasets_rarefs: dict, datasets_phylo: dict, trees: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, slurm: bool, As: tuple, run_params: dict, filt_raref: str, jobs: bool, chunkit: int) -> None: """ Run alpha-rarefaction: Alpha rarefaction https://docs.qiime2.org/2019.10/plugins/available/diversity/alpha-rarefaction/ """ alpha_metrics = get_metrics('alpha_metrics', As) job_folder = get_job_folder(i_datasets_folder, 'alpha_rarefaction') job_folder2 = get_job_folder(i_datasets_folder, 'alpha_rarefaction/chunks') main_written = 0 run_pbs = '%s/4_run_alpha_rarefaction_%s%s.sh' % (job_folder, prjct_nm, filt_raref) to_chunk = [] with open(run_pbs, 'w') as o: for dat, tsv_meta_pds_ in datasets.items(): written = 0 out_sh = '%s/run_alpha_rarefaction_%s_%s%s.sh' % ( job_folder2, prjct_nm, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: for idx, tsv_meta_pds in enumerate(tsv_meta_pds_): tsv, meta = tsv_meta_pds qza = '%s.qza' % splitext(tsv)[0] cur_raref = datasets_rarefs[dat][idx] odir = get_analysis_folder( i_datasets_folder, 'alpha_rarefaction/%s%s' % (dat, cur_raref)) for metric in alpha_metrics: out_fp = '%s/rarefcurve_%s%s_%s.qzv' % ( odir, dat, cur_raref, metric) if force or not isfile(out_fp): if write_diversity_alpha_rarefaction( out_fp, qza, metric, datasets_phylo, trees, dat, meta, cur_sh): continue written += 1 main_written += 1 to_chunk.append(out_sh) if not chunkit: run_xpbs(out_sh, out_pbs, '%s.lphrrf.%s%s' % (prjct_nm, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if to_chunk and chunkit: simple_chunks(run_pbs, job_folder2, to_chunk, 'alpha_rarefaction', prjct_nm, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit, None) if main_written: print_message('# Compute rarefaction curve on alpha diversity indices', 'sh', run_pbs, jobs)
def run_correlations(i_datasets_folder: str, datasets: dict, diversities: dict, datasets_rarefs: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, slurm: bool, run_params: dict, filt_raref: str, jobs: bool, chunkit: int) -> None: """ Run alpha-correlation: Alpha diversity correlation https://docs.qiime2.org/2019.10/plugins/available/diversity/alpha-correlation/ :param i_datasets_folder: Path to the folder containing the data/metadata subfolders. :param datasets: list of datasets. :param diversities: alpha diversity qiime2 Arfetact per dataset. :param force: Force the re-writing of scripts for all commands. :param prjct_nm: Nick name for your project. :param qiime_env: qiime2-xxxx.xx conda environment. :param chmod: whether to change permission of output files (defalt: 775). """ job_folder = get_job_folder(i_datasets_folder, 'alpha_correlations') job_folder2 = get_job_folder(i_datasets_folder, 'alpha_correlations/chunks') main_written = 0 run_pbs = '%s/4_run_alpha_correlation_%s%s.sh' % (job_folder, prjct_nm, filt_raref) to_chunk = [] with open(run_pbs, 'w') as o: for dat, tsv_meta_pds_ in datasets.items(): if dat not in diversities: continue written = 0 out_sh = '%s/run_alpha_correlation_%s_%s%s.sh' % ( job_folder2, prjct_nm, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: for idx, tsv_meta_pds in enumerate(tsv_meta_pds_): tsv, meta = tsv_meta_pds cur_raref = datasets_rarefs[dat][idx] for method in ['spearman', 'pearson']: for group, divs in diversities[dat][idx].items(): if group: odir = get_analysis_folder( i_datasets_folder, 'alpha_correlations/%s%s/%s' % (dat, cur_raref, group)) else: odir = get_analysis_folder( i_datasets_folder, 'alpha_correlations/%s%s' % (dat, cur_raref)) for qza in [x[0] for x in divs]: out_fp = '%s/alpha_corr_%s' % ( odir, basename(qza).replace( '.qza', '_%s.qzv' % method)) if force or not isfile(out_fp): write_diversity_alpha_correlation( out_fp, qza, method, meta, cur_sh) written += 1 main_written += 1 to_chunk.append(out_sh) if not chunkit: run_xpbs(out_sh, out_pbs, '%s.lphcrr.%s%s' % (prjct_nm, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if to_chunk and chunkit: simple_chunks(run_pbs, job_folder2, to_chunk, 'alpha_correlations', prjct_nm, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit, None) if main_written: print_message( '# Correlate numeric metadata variables with alpha diversity indices', 'sh', run_pbs, jobs)
def run_alpha(i_datasets_folder: str, datasets: dict, datasets_read: dict, datasets_phylo: dict, datasets_rarefs: dict, p_alpha_subsets: str, trees: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, slurm: bool, As: tuple, dropout: bool, run_params: dict, filt_raref: str, eval_depths: dict, jobs: bool, chunkit: int) -> dict: """ Computes the alpha diversity vectors for each dataset. :param i_datasets_folder: Path to the folder containing the data/metadata subfolders. :param datasets: dataset -> [tsv, meta] :param datasets_read: dataset -> [tsv table, meta table] :param datasets_phylo: to be updated with ('tree_to_use', 'corrected_or_not') per dataset. :param p_alpha_subsets: Subsets for alpha diversity. :param trees: to be update with tree to use for a dataset phylogenetic analyses. :param force: Force the re-writing of scripts for all commands. :param prjct_nm: Short nick name for your project. :param qiime_env: name of your qiime2 conda environment (e.g. qiime2-2019.10). :param chmod: whether to change permission of output files (defalt: 775). :return: {'dataset1': [ 'meta', {'div_index1': '.qza', 'div_index2': '.qza', ... }], 'dataset2': [ 'meta', {'div_index1': '.qza', 'div_index2': '.qza', ... }], '...'} """ evaluation = '' if len(eval_depths): evaluation = '_eval' alpha_metrics = get_metrics('alpha_metrics', As) alpha_subsets = read_yaml_file(p_alpha_subsets) job_folder = get_job_folder(i_datasets_folder, 'alpha%s' % evaluation) job_folder2 = get_job_folder(i_datasets_folder, 'alpha%s/chunks' % evaluation) diversities = {} run_pbs = '%s/1_run_alpha_%s%s%s.sh' % (job_folder, prjct_nm, evaluation, filt_raref) main_written = 0 to_chunk = [] with open(run_pbs, 'w') as o: for dat, tsv_meta_pds_ in datasets.items(): written = 0 diversities[dat] = [] out_sh = '%s/run_alpha_%s%s_%s%s.sh' % ( job_folder2, prjct_nm, evaluation, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: for idx, tsv_meta_pds in enumerate(tsv_meta_pds_): tsv, meta = tsv_meta_pds if not isinstance( datasets_read[dat][idx][0], pd.DataFrame ) and datasets_read[dat][idx][0] == 'raref': if not isfile(tsv): print( 'Must have run rarefaction to use it further...\nExiting' ) sys.exit(0) tsv_pd, meta_pd = get_raref_tab_meta_pds(meta, tsv) datasets_read[dat][idx] = [tsv_pd, meta_pd] else: tsv_pd, meta_pd = datasets_read[dat][idx] cur_raref = datasets_rarefs[dat][idx] qza = '%s.qza' % splitext(tsv)[0] divs = {} for metric in alpha_metrics: odir = get_analysis_folder( i_datasets_folder, 'alpha/%s%s' % (dat, cur_raref)) out_fp = '%s/%s_%s.qza' % ( odir, basename(splitext(qza)[0]), metric) out_tsv = '%s.tsv' % splitext(out_fp)[0] if force or not isfile(out_fp): ret_continue = write_diversity_alpha( out_fp, datasets_phylo, trees, dat, qza, metric, cur_sh, qiime_env) if ret_continue: continue cmd = run_export(out_fp, out_tsv, '') cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n\n' % cmd) written += 1 main_written += 1 divs.setdefault('', []).append((out_fp, metric)) if alpha_subsets and dat in alpha_subsets: for subset, subset_regex in alpha_subsets[dat].items(): odir = get_analysis_folder( i_datasets_folder, 'alpha/%s%s/%s' % (dat, cur_raref, subset)) if dropout: qza_subset_ = '%s/%s_%s.qza' % ( odir, basename(splitext(qza)[0]), subset) else: qza_subset_ = '%s/%s_%s_noDropout.qza' % ( odir, basename(splitext(qza)[0]), subset) feats_subset = '%s.meta' % splitext(qza_subset_)[0] feats = get_subset(tsv_pd, subset_regex) if not len(feats): continue subset_pd = pd.DataFrame({ 'Feature ID': feats, 'Subset': [subset] * len(feats) }) subset_pd.to_csv(feats_subset, index=False, sep='\t') write_filter_features(tsv_pd, feats, qza, qza_subset_, feats_subset, cur_sh, dropout) for metric in alpha_metrics: if metric in [ 'faith_pd' ] and datasets_phylo[dat][1] and dat in trees: tree_in_qza = trees[dat][0] tree_in_tsv = '%s.tsv' % splitext( tree_in_qza)[0] if dropout: qza_subset = '%s/%s_%s.qza' % ( odir, basename(splitext(tree_in_qza)[0]), subset) else: qza_subset = '%s/%s_%s_noDropout.qza' % ( odir, basename(splitext(tree_in_qza)[0]), subset) write_filter_features( pd.read_csv(tree_in_tsv, header=0, index_col=0, sep='\t'), feats, tree_in_qza, qza_subset, feats_subset, cur_sh, dropout) else: qza_subset = qza_subset_ out_fp = '%s/%s__%s.qza' % ( odir, basename( splitext(qza_subset)[0]), metric) out_tsv = '%s.tsv' % splitext(out_fp)[0] if force or not isfile(out_fp): ret_continue = write_diversity_alpha( out_fp, {dat: [1, 0]}, trees, dat, qza_subset, metric, cur_sh, qiime_env) if ret_continue: continue cmd = run_export(out_fp, out_tsv, '') cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n\n' % cmd) written += 1 main_written += 1 divs.setdefault(subset, []).append( (out_fp, metric)) diversities[dat].append(divs) to_chunk.append(out_sh) if not chunkit: run_xpbs( out_sh, out_pbs, '%s.mg.lph%s.%s%s' % (prjct_nm, evaluation, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if to_chunk and chunkit: simple_chunks(run_pbs, job_folder2, to_chunk, 'alpha', prjct_nm, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit, None) if main_written: print_message('# Calculate alpha diversity indices', 'sh', run_pbs, jobs) return diversities
def run_taxonomy(method: str, i_datasets_folder: str, datasets: dict, datasets_read: dict, datasets_phylo: dict, datasets_features: dict, datasets_filt_map: dict, i_classifier: str, taxonomies: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, slurm: bool, run_params: dict, filt_raref: str, jobs: bool, chunkit: int) -> None: """ Parameters ---------- method i_datasets_folder : str Path to the folder containing the data/metadata subfolders. datasets : dict Mappring dataset name -> [data file path, metadata file path]. datasets_read : dict Mapping dataset name -> [data table, metadata table] datasets_phylo : dict To be updated with ('tree_to_use', 'corrected_or_not') per dataset. datasets_features : dict Mapping dataset name -> list of features names in the dataset tsv / biom file. datasets_filt_map : dict i_classifier : str Path to the taxonomic classifier. taxonomies : dict Mapping Dataset name -> [method, assignment qza] force : bool Force the re-writing of scripts for all commands. prjct_nm : str Short nick name for your project. qiime_env : str Name of your qiime2 conda environment (e.g. qiime2-2019.10). chmod : str Whether to change permission of output files (default: 744). noloc : str run_params : dict filt_raref : str jobs : bool chunkit : int Returns ------- """ job_folder = get_job_folder(i_datasets_folder, 'taxonomy') job_folder2 = get_job_folder(i_datasets_folder, 'taxonomy/chunks') amplicon_datasets = [ dat for dat, (tree, correction) in datasets_phylo.items() if tree == 'amplicon' ] wol_datasets = [ dat for dat, (tree, correction) in datasets_phylo.items() if tree == 'wol' ] main_written = 0 to_chunk = [] run_pbs = '%s/1_run_taxonomy_%s%s.sh' % (job_folder, prjct_nm, filt_raref) with open(run_pbs, 'w') as o: for dat, tsv_meta_pds_ in datasets_read.items(): out_sh = '%s/run_taxonomy_%s_%s%s.sh' % (job_folder2, prjct_nm, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] if dat in datasets_filt_map: taxonomies[dat] = taxonomies[datasets_filt_map[dat]] continue written = 0 with open(out_sh, 'w') as cur_sh: for idx, tsv_meta_pds in enumerate(tsv_meta_pds_): if idx: continue tsv, meta = datasets[dat][idx] if not isinstance(tsv_meta_pds[0], pd.DataFrame) and \ tsv_meta_pds[0] == 'raref': if not isfile(tsv): print('Must have run rarefaction to use it ' 'further...\nExiting') sys.exit(0) tsv_pd, meta_pd = get_raref_tab_meta_pds(meta, tsv) datasets_read[dat][idx] = [tsv_pd, meta_pd] else: tsv_pd, meta_pd = tsv_meta_pds odir = get_analysis_folder(i_datasets_folder, 'taxonomy/%s' % dat) out_rad = '%s/tax_%s' % (odir, dat) if dat in amplicon_datasets: out_qza = '%s_%s.qza' % (out_rad, method) out_tsv = '%s.tsv' % splitext(out_qza)[0] taxonomies[dat] = [method, out_qza, out_tsv] if not i_classifier: print('No classifier passed for 16S ' 'data\nExiting...') continue cmd = run_taxonomy_amplicon(dat, i_datasets_folder, force, tsv_pd, out_qza, out_tsv, i_classifier) else: out_qza = '%s.qza' % out_rad out_tsv = '%s.tsv' % out_rad if dat in wol_datasets: cur_datasets_features = datasets_features[dat] taxonomies[dat] = ['wol', out_qza, out_tsv] cmd = run_taxonomy_wol(force, tsv_pd, out_qza, out_tsv, cur_datasets_features) else: if len( [x for x in tsv_pd.index if str(x).isdigit()]) == tsv_pd.shape[0]: continue taxonomies[dat] = ['feat', out_qza, out_tsv] cmd = run_taxonomy_others(force, tsv_pd, out_qza, out_tsv) if cmd: cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n\n' % cmd) main_written += 1 written += 1 if written: to_chunk.append(out_sh) if not chunkit: run_xpbs(out_sh, out_pbs, '%s.tx.sklrn.%s%s' % (prjct_nm, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if to_chunk and chunkit: simple_chunks(run_pbs, job_folder2, to_chunk, 'taxonomy', prjct_nm, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit, None) if main_written: print_message('# Classify features using classify-sklearn', 'sh', run_pbs, jobs)
def run_barplot(i_datasets_folder: str, datasets: dict, taxonomies: dict, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, slurm: bool, run_params: dict, filt_raref: str, jobs: bool, chunkit: int) -> None: """Visualize taxonomy with an interactive bar plot. Parameters ---------- i_datasets_folder : str Path to the folder containing the data/metadata subfolders datasets : dict Mappig dataset name -> [tsv file path, metadata file path] taxonomies : dict Mappig dataset name -> [classification_method, tax_qza] force : bool Force the re-writing of scripts for all commands prjct_nm : str Short nick name for your project qiime_env : str Mame of a qiime2 conda environment chmod : str Whether to change permission of output files (defalt: 744) noloc : bool run_params : dict filt_raref : str jobs : bool chunkit : int Returns ------- """ job_folder = get_job_folder(i_datasets_folder, 'barplot') job_folder2 = get_job_folder(i_datasets_folder, 'barplot/chunks') written = 0 to_chunk = [] run_pbs = '%s/1_run_barplot_%s%s.sh' % (job_folder, prjct_nm, filt_raref) with open(run_pbs, 'w') as o: for dat, tsv_meta_pds_ in datasets.items(): out_sh = '%s/run_barplot_%s_%s%s.sh' % (job_folder2, prjct_nm, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: for tsv_meta_pds in tsv_meta_pds_: tsv, meta = tsv_meta_pds if dat not in taxonomies: continue method, tax_qza, tax_tsv = taxonomies[dat] if not method: method = 'taxofromfile' qza = '%s.qza' % splitext(tsv)[0] odir = get_analysis_folder(i_datasets_folder, 'barplot/%s' % dat) out_qzv = '%s/bar_%s_%s.qzv' % (odir, dat, method) if force or not isfile(out_qzv): write_barplots(out_qzv, qza, meta, tax_qza, cur_sh) written += 1 to_chunk.append(out_sh) if not chunkit: run_xpbs(out_sh, out_pbs, '%s.brplt.%s%s' % (prjct_nm, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, slurm, jobs) if to_chunk and chunkit: simple_chunks(run_pbs, job_folder2, to_chunk, 'barplot', prjct_nm, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], qiime_env, chmod, noloc, slurm, jobs, chunkit, None) if written: print_message('# Make sample compositions barplots', 'sh', run_pbs, jobs)
def shear_tree( i_datasets_folder: str, datasets: dict, datasets_read: dict, datasets_phylo: dict, datasets_features: dict, prjct_nm: str, i_wol_tree: str, trees: dict, datasets_rarefs: dict, force: bool, qiime_env: str, chmod: str, noloc: bool, slurm: bool, run_params: dict, filt_raref: str, jobs: bool) -> None: """ Get the sub-tree from the Web of Life tree that corresponds to the gOTUs-labeled features. :param i_datasets_folder: Path to the folder containing the data/metadata subfolders. :param datasets_read: dataset -> [tsv table, meta table] :param datasets_phylo: to be updated with ('tree_to_use', 'corrected_or_not') per dataset. :param datasets_features: dataset -> list of features names in the dataset tsv / biom file. :param prjct_nm: Short nick name for your project. :param i_wol_tree: default on barnacle /projects/wol/profiling/dbs/wol/phylogeny/wol_tree.nwk. :param trees: to be update with tree to use for a dataset phylogenetic analyses. :param force: Force the re-writing of scripts for all commands. :param qiime_env: name of your qiime2 conda environment (e.g. qiime2-2019.10). :param chmod: whether to change permission of output files (defalt: 775). """ # check whether there's dataset(s) that may use # the Web of Life tree (i.e. features contain gID) wol_datasets = [dat for dat, (tree, correction) in datasets_phylo.items() if tree == 'wol'] if len(wol_datasets): job_folder = get_job_folder(i_datasets_folder, 'phylo') job_folder2 = get_job_folder(i_datasets_folder, 'phylo/chunks') i_wol_tree = get_wol_tree(i_wol_tree) wol = TreeNode.read(i_wol_tree) main_written = 0 main_sh = '%s/0_run_import_trees_%s%s.sh' % (job_folder, prjct_nm, filt_raref) with open(main_sh, 'w') as main_o: for dat, tsv_metas_fps_ in datasets.items(): written = 0 if dat not in wol_datasets: continue out_sh = '%s/run_import_tree_%s_%s%s.sh' % (job_folder2, prjct_nm, dat, filt_raref) if slurm: out_pbs = out_sh.replace('.sh', '.slm') else: out_pbs = out_sh.replace('.sh', '.pbs') with open(out_sh, 'w') as o: for idx, tsv_metas_fps in enumerate(tsv_metas_fps_): tsv, meta = tsv_metas_fps if not isinstance(datasets_read[dat][idx][0], pd.DataFrame) and datasets_read[dat][idx][0] == 'raref': if not isfile(tsv): print('Must have run rarefaction to use it further...\nExiting') sys.exit(0) tsv_pd, meta_pd = get_raref_tab_meta_pds(meta, tsv) datasets_read[dat][idx] = [tsv_pd, meta_pd] else: tsv_pd, meta_pd = datasets_read[dat][idx] cur_raref = datasets_rarefs[dat][idx] cur_datasets_features = dict( gid for gid in datasets_features[dat].items() if gid[1] in tsv_pd.index) analysis_folder = get_analysis_folder(i_datasets_folder, 'phylo/%s' % dat) wol_features_fpo = '%s/tree_%s%s.nwk' % (analysis_folder, dat, cur_raref) wol_features_qza = wol_features_fpo.replace('.nwk', '.qza') # if idx: # trees[dat].append(('', wol_features_qza)) # else: # trees[dat] = [('', wol_features_qza)] if not idx: trees[dat] = ('', wol_features_qza) if force or not isfile(wol_features_qza): wol_features = wol.shear(list(cur_datasets_features.keys())) # rename the tip per the features names associated with each gID for tip in wol_features.tips(): tip.name = cur_datasets_features[tip.name] wol_features.write(wol_features_fpo) cmd = run_import(wol_features_fpo, wol_features_qza, "Phylogeny[Rooted]") o.write("echo '%s'\n" % cmd) o.write('%s\n\n' % cmd) written += 1 main_written += 1 run_xpbs(out_sh, out_pbs, '%s.shr.%s%s' % (prjct_nm, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', main_o, noloc, slurm, jobs) if main_written: print_message("# Shear Web of Life tree to features' genome IDs (%s)" % ', '.join(wol_datasets), 'sh', main_sh, jobs)
def run_sepp(i_datasets_folder: str, datasets: dict, datasets_read: dict, datasets_phylo: dict, datasets_rarefs: dict, prjct_nm: str, i_sepp_tree: str, trees: dict, force: bool, qiime_env: str, chmod: str, noloc: bool, slurm: bool, run_params: dict, filt_raref: str, jobs: bool) -> None: """ Run SEPP on the datasets composed or 16S deblur sequences (e.g. from redbiom/Qiita). :param i_datasets_folder: Path to the folder containing the data/metadata subfolders. :param datasets: dataset -> [tsv/biom path, meta path] :param datasets_read: dataset -> [tsv table, meta table] :param datasets_phylo: to be updated with ('tree_to_use', 'corrected_or_not') per dataset. :param prjct_nm: Short nick name for your project. :param i_sepp_tree: database to use for sepp phylogeny reads placement. :param trees: to be update with tree to use for a dataset phylogenetic analyses. :param force: Force the re-writing of scripts for all commands. :param qiime_env: name of your qiime2 conda environment (e.g. qiime2-2019.10). :param chmod: whether to change permission of output files (defalt: 775). """ # check whether there's dataset(s) that may use the reference tree (i.e. features are DNA sequences) sepp_datasets = [dat for dat, (tree, correction) in datasets_phylo.items() if tree == 'amplicon'] if len(sepp_datasets): ref_tree_qza = get_sepp_tree(i_sepp_tree) job_folder = get_job_folder(i_datasets_folder, 'phylo') job_folder2 = get_job_folder(i_datasets_folder, 'phylo/chunks') main_written = 0 main_sh = '%s/1_run_sepp_%s%s.sh' % (job_folder, prjct_nm, filt_raref) with open(main_sh, 'w') as main_o: for dat, tsv_metas_fps_ in datasets.items(): written = 0 if dat not in sepp_datasets: continue out_sh = '%s/run_sepp_%s_%s%s.sh' % (job_folder2, prjct_nm, dat, filt_raref) if slurm: out_pbs = '%s.slm' % splitext(out_sh)[0] else: out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: for idx, tsv_metas_fps in enumerate(tsv_metas_fps_): tsv, meta = tsv_metas_fps if not isinstance(datasets_read[dat][idx][0], pd.DataFrame) and datasets_read[dat][idx][0] == 'raref': qza_raw_in = '%s/data/tab_%s_inTree.qza' % (i_datasets_folder, dat) if isfile(qza_raw_in) and not force: odir_sepp = get_analysis_folder(i_datasets_folder, 'phylo/%s' % dat) out_fp_sepp_tree = '%s/tree_%s.qza' % (odir_sepp, dat) # if idx: # trees[dat].append((qza_raw_in, out_fp_sepp_tree)) # else: # trees[dat] = [(qza_raw_in, out_fp_sepp_tree)] if not idx: trees[dat] = (qza_raw_in, out_fp_sepp_tree) print('Using the non rarefied tree (no need to recompute)...\nExiting') continue elif not isfile(tsv): print('Must have run rarefaction to use it further...\nExiting') sys.exit(0) tsv_pd, meta_pd = get_raref_tab_meta_pds(meta, tsv) datasets_read[dat][idx] = [tsv_pd, meta_pd] else: tsv_pd, meta_pd = datasets_read[dat][idx] qza = '%s.qza' % splitext(tsv)[0] if not isfile(qza): print('Need to first import %s to .qza to do reads placement ' '(see "# Import tables to qiime2")\nExiting...' % tsv) sys.exit(0) cur_raref = datasets_rarefs[dat][idx] qza_in = '%s_inTree%s.qza' % (splitext(tsv)[0], cur_raref) qza_in_tsv = '%s.tsv' % splitext(qza_in)[0] qza_out = '%s_notInTree%s.qza' % (splitext(tsv)[0], cur_raref) odir_seqs = get_analysis_folder(i_datasets_folder, 'seqs/%s' % dat) odir_sepp = get_analysis_folder(i_datasets_folder, 'phylo/%s' % dat) out_fp_seqs_rad = '%s/seq_%s%s' % (odir_seqs, dat, cur_raref) out_fp_seqs_fasta = '%s.fasta' % out_fp_seqs_rad out_fp_seqs_qza = '%s.qza' % out_fp_seqs_rad out_fp_sepp_tree = '%s/tree_%s%s.qza' % (odir_sepp, dat, cur_raref) # if idx: # trees[dat].append((qza_in, out_fp_sepp_tree)) # else: # trees[dat] = [(qza_in, out_fp_sepp_tree)] if not idx: trees[dat] = (qza_in, out_fp_sepp_tree) written = 0 if force or not isfile(out_fp_seqs_qza): cmd = write_seqs_fasta(out_fp_seqs_fasta, out_fp_seqs_qza, tsv_pd) cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n\n' % cmd) written += 1 main_written += 1 if force or not isfile(out_fp_sepp_tree) or not isfile(qza_in_tsv): cmd = write_fragment_insertion( out_fp_seqs_qza, ref_tree_qza, out_fp_sepp_tree, qza, qza_in, qza_in_tsv, qza_out) cur_sh.write('echo "%s"\n' % cmd) cur_sh.write('%s\n\n' % cmd) written += 1 main_written += 1 run_xpbs(out_sh, out_pbs, '%s.spp.%s%s' % (prjct_nm, dat, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', main_o, noloc, slurm, jobs) if main_written: print_message("# Fragment insertion using SEPP (%s)" % ', '.join(sepp_datasets), 'sh', main_sh, jobs)
def run_mmbird(i_datasets_folder: str, songbird_outputs: list, p_mmvec_highlights: str, p_xmmvec: str, mmvec_outputs: list, force: bool, prjct_nm: str, qiime_env: str, chmod: str, noloc: bool, filt_raref: str, run_params: dict, input_to_filtered: dict, jobs: bool, chunkit: int): if not mmvec_outputs: print('No mmvec output detected...') sys.exit(0) # if not songbird_outputs: # print('No songbird output detected...') # sys.exit(0) print('\t-> [mmbird] Get mmvec output...', end=' ') mmvec_outputs_pd = get_mmvec_outputs(mmvec_outputs) print('Done.') print('\t-> [mmbird] Get songbird output...', end=' ') if len(songbird_outputs): # songbird_outputs_pd = get_songbird_outputs(songbird_outputs) songbird_outputs_pd = get_songbird_outputs(songbird_outputs) mmvec_songbird_pd = merge_mmvec_songbird_outputs(mmvec_outputs_pd, songbird_outputs_pd) else: mmvec_songbird_pd = mmvec_outputs_pd.copy() print('Done.') omics_pairs = [tuple(x) for x in mmvec_songbird_pd[ ['omic_case_filt1', 'omic_case_filt2']].values.tolist()] print('\t-> [mmbird] Get taxonomy (feature metadata)...', end=' ') taxo_pds = get_taxo_pds( i_datasets_folder, mmvec_songbird_pd, input_to_filtered) print('Done.') print('\t-> [mmbird] Get songbird differentials + taxonomy...', end=' ') omics_pairs_metas = get_omics_songbirds_taxa( i_datasets_folder, mmvec_songbird_pd, taxo_pds) print('Done.') print('\t-> [mmbird] Get res dict...') mmvec_res = get_mmvec_res(mmvec_songbird_pd) print('Done.') print('\t-> [mmbird] Get commands...') highlights = get_highlights_mmbird(p_mmvec_highlights) xmmvecs = get_highlights_mmbird(p_xmmvec) pair_cmds, pc_sb_correlations = get_pair_cmds( mmvec_res, omics_pairs_metas, omics_pairs, force, highlights, xmmvecs) if len(pc_sb_correlations): out_folder = get_analysis_folder(i_datasets_folder, 'mmbird') out_correlations = '%s/pc_vs_songbird_correlations.tsv' % out_folder pc_sb_correlations_pd = pd.concat(pc_sb_correlations) if pc_sb_correlations_pd.shape[0]: pc_sb_correlations_pd.to_csv(out_correlations, index=False, sep='\t') print('\t\t==> Written:', out_correlations) else: print('\t\t==> No good songbird model to make correlations with mmvec PCs...') # get_log_ratios(pc_sb_correlations_pd) job_folder = get_job_folder(i_datasets_folder, 'mmbird') job_folder2 = get_job_folder(i_datasets_folder, 'mmbird/chunks') written = 0 run_pbs = '%s/run_mmbird_%s%s.sh' % (job_folder, prjct_nm, filt_raref) with open(run_pbs, 'w') as o: for pair, cmds in pair_cmds.items(): out_sh = '%s/run_mmbird_%s_%s%s.sh' % (job_folder2, prjct_nm, pair, filt_raref) out_pbs = '%s.pbs' % splitext(out_sh)[0] with open(out_sh, 'w') as cur_sh: for cmd in cmds: cur_sh.write(cmd) written += 1 run_xpbs(out_sh, out_pbs, '%s.mmbrd.%s%s' % (prjct_nm, pair, filt_raref), qiime_env, run_params["time"], run_params["n_nodes"], run_params["n_procs"], run_params["mem_num"], run_params["mem_dim"], chmod, written, 'single', o, noloc, jobs) if written: print_message('# Generate mmvec biplot with songbird models', 'sh', run_pbs, jobs)