"""Created: Monday August 13, 2018 Modified: Monday August 20, 2018 Jorge Luis Flores Generates a normalizing vector of 940 structural motifs from the collection of all mRNAs This vector is the average of all windows folded and will be used to normalize""" import sys sys.path.append('/u/floresj/Pyth_modules/') import motifs_vector_3 as mv from scipy.stats.stats import pearsonr import numpy as np import pandas as pd from datetime import datetime as dt ALL_MOTIFS = mv.list_of_all_motifs() ## keep track of execution time start_time = dt.now() freq_df = pd.DataFrame() # load all dataframes for i in ['01', '02', '03', '04', '05', '06', '07' ,'08', '09', '10', '11', '12', '13']: filename = '/u/floresj/mRNA_norm/mRNA_vectors/mrna_folded_int_subset'+i tmp_df = pd.read_parquet(filename) freq_df = freq_df.append( tmp_df.iloc[:,:101].astype('uint16') ) print(f'{dt.now() - start_time}\tLoaded dataframe {i}') # if all motifs are used, len(freq_df.columns.values) == 941 nb_motifs = len(freq_df.columns.values) - 1
all_mrna.append(rna) # initialize logging file with open('/u/floresj/mRNA_norm/normalization_fold_all_log.txt', 'w') as fp: fp.write('Version: Monday August 20, 2018\n') fp.write('Time\t\tProcessed\n') ## keep track of execution time start_time = dt.now() # multiprocessing setup compteur = TimeCounter(len(all_mrna)) multiprocessing.set_start_method('spawn') pool = multiprocessing.Pool(36) for rna in all_mrna: pool.apply_async(get_sum_signs, (rna, ), callback=compteur.print_progress) #compteur.print_progress(get_sum_signs(rna)) pool.close() pool.join() # average the sum of frequencies by number of windows used vector_avg = compteur.vector_sum / compteur.nb_windows print(f'{compteur.count}\t Sequences folded') motif_index = mv.list_of_all_motifs() vector_df = pd.DataFrame(vector_avg, index=motif_index).transpose() vector_df.to_csv('/u/floresj/mRNA_norm/mrna_avg_vector.csv')