Example #1
0
def main(
        data_df,model_df,
        start=0,end=None,err=False,coarse_graining_level=0):
    dicttype, modeltype = qc.get_model_type(model_df)
    seq_cols = qc.get_cols_from_df(data_df,'seqs')
    if not len(seq_cols)==1:
        raise SortSeqError('Dataframe has multiple seq cols: %s'%str(seq_cols))
    seq_dict,inv_dict = utils.choose_dict(dicttype,modeltype=modeltype)
    #set name of sequences column based on type of sequence
    type_name_dict = {'dna':'seq','rna':'seq_rna','protein':'seq_pro'}
    seq_col_name = type_name_dict[dicttype]
    #Cut the sequences based on start and end, and then check if it makes sense
    if (start != 0 or end):
        data_df.loc[:,seq_col_name] = data_df.loc[:,seq_col_name].str.slice(start,end)
        if modeltype=='MAT':
            if len(data_df.loc[0,seq_col_name]) != len(model_df.loc[:,'pos']):
                raise SortSeqError('model length does not match dataset length')
        elif modeltype=='NBR':
            if len(data_df.loc[0,seq_col_name]) != len(model_df.loc[:,'pos'])+1:
                raise SortSeqError('model length does not match dataset length')
    col_headers = utils.get_column_headers(data_df)
    if 'ct' not in data_df.columns:
                data_df['ct'] = data_df[col_headers].sum(axis=1)
    data_df = data_df[data_df.ct != 0]        
    if not end:
        seqL = len(data_df[seq_col_name][0]) - start
    else:
        seqL = end-start
    data_df = data_df[data_df[seq_col_name].apply(len) == (seqL)] 
    #make a numpy array out of the model data frame
    model_df_headers = ['val_' + str(inv_dict[i]) for i in range(len(seq_dict))]
    value = np.transpose(np.array(model_df[model_df_headers]))  
    #now we evaluate the expression of each sequence according to the model.
    seq_mat,wtrow = numerics.dataset2mutarray(data_df.copy(),modeltype)
    temp_df = data_df.copy()
    temp_df['val'] = numerics.eval_modelmatrix_on_mutarray(np.array(model_df[model_df_headers]),seq_mat,wtrow) 
    temp_sorted = temp_df.sort_values(by='val')
    temp_sorted.reset_index(inplace=True,drop=True)
    #we must divide by the total number of counts in each bin for the MI calculator
    #temp_sorted[col_headers] = temp_sorted[col_headers].div(temp_sorted['ct'],axis=0)     
    MI = EstimateMutualInfoforMImax.alt4(temp_sorted,coarse_graining_level=coarse_graining_level)
    if not err:
        Std = np.NaN
    else:
        data_df_for_sub = data_df.copy()
        sub_MI = np.zeros(15)
        for i in range(15):
            sub_df = data_df_for_sub.sample(int(len(data_df_for_sub.index)/2))
            sub_df.reset_index(inplace=True,drop=True)
            sub_MI[i],sub_std = main(
                sub_df,model_df,err=False)
        Std = np.std(sub_MI)/np.sqrt(2)
    return MI,Std
Example #2
0
def main(df,lm='IM',modeltype='MAT',LS_means_std=None,\
    db=None,iteration=30000,burnin=1000,thin=10,\
    runnum=0,initialize='LS',start=0,end=None,foreground=1,\
    background=0,alpha=0,pseudocounts=1,test=False,drop_library=False,\
    verbose=False):
    
    # Determine dictionary
    seq_cols = qc.get_cols_from_df(df,'seqs')
    if not len(seq_cols)==1:
        raise SortSeqError('Dataframe has multiple seq cols: %s'%str(seq_cols))
    dicttype = qc.colname_to_seqtype_dict[seq_cols[0]]

    seq_dict,inv_dict = utils.choose_dict(dicttype,modeltype=modeltype)
    
    '''Check to make sure the chosen dictionary type correctly describes
         the sequences. An issue with this test is that if you have DNA sequence
         but choose a protein dictionary, you will still pass this test bc A,C,
         G,T are also valid amino acids'''
    #set name of sequences column based on type of sequence
    type_name_dict = {'dna':'seq','rna':'seq_rna','protein':'seq_pro'}
    seq_col_name = type_name_dict[dicttype]
    lin_seq_dict,lin_inv_dict = utils.choose_dict(dicttype,modeltype='MAT')
    #wtseq = utils.profile_counts(df.copy(),dicttype,return_wtseq=True,start=start,end=end)
    #wt_seq_dict_list = [{inv_dict[np.mod(i+1+seq_dict[w],len(seq_dict))]:i for i in range(len(seq_dict)-1)} for w in wtseq]
    par_seq_dict = {v:k for v,k in seq_dict.items() if k != (len(seq_dict)-1)}
    #drop any rows with ct = 0
    df = df[df.loc[:,'ct'] != 0]
    df.reset_index(drop=True,inplace=True)
    
    #If there are sequences of different lengths, then print error but continue
    if len(set(df[seq_col_name].apply(len))) > 1:
         sys.stderr.write('Lengths of all sequences are not the same!')
    #select target sequence region
    df.loc[:,seq_col_name] = df.loc[:,seq_col_name].str.slice(start,end)
    df = utils.collapse_further(df)
    col_headers = utils.get_column_headers(df)
    #make sure all counts are ints
    df[col_headers] = df[col_headers].astype(int)
    #create vector of column names
    val_cols = ['val_' + inv_dict[i] for i in range(len(seq_dict))]
    df.reset_index(inplace=True,drop=True)
    #Drop any sequences with incorrect length
    if not end:
        '''is no value for end of sequence was supplied, assume first seq is
            correct length'''
        seqL = len(df[seq_col_name][0]) - start
    else:
        seqL = end-start
    df = df[df[seq_col_name].apply(len) == (seqL)]
    df.reset_index(inplace=True,drop=True)
    #Do something different for each type of learning method (lm)
    if lm == 'ER':
        emat = Berg_von_Hippel(
            df,dicttype,foreground=foreground,background=background,
            pseudocounts=pseudocounts)
    if lm == 'LS':
        '''First check that is we don't have a penalty for ridge regression,
            that we at least have all possible base values so that the analysis
            will not fail'''
        if LS_means_std: #If user supplied preset means and std for each bin
            means_std_df = io.load_meanstd(LS_means_std)

            #change bin number to 'ct_number' and then use as index
            labels = list(means_std_df['bin'].apply(add_label))
            std = means_std_df['std']
            std.index = labels
            #Change Weighting of each sequence by dividing counts by bin std
            df[labels] = df[labels].div(std)
            means = means_std_df['mean']
            means.index = labels
        else:
            means = None
        #drop all rows without counts
        df['ct'] = df[col_headers].sum(axis=1)
        df = df[df.ct != 0]        
        df.reset_index(inplace=True,drop=True)
        ''' For sort-seq experiments, bin_0 is library only and isn't the lowest
            expression even though it is will be calculated as such if we proceed.
            Therefore is drop_library is passed, drop this column from analysis.'''
        if drop_library:
            try:     
                df.drop('ct_0',inplace=True)
                col_headers = utils.get_column_headers(df)
                if len(col_headers) < 2:
                    raise SortSeqError(
                        '''After dropping library there are no longer enough 
                        columns to run the analysis''')
            except:
                raise SortSeqError('''drop_library option was passed, but no ct_0
                    column exists''')
        #parameterize sequences into 3xL vectors
                               
        raveledmat,batch,sw = utils.genweightandmat(
                                  df,par_seq_dict,dicttype,means=means,modeltype=modeltype)
        #Use ridge regression to find matrix.       
        emat = Compute_Least_Squares(raveledmat,batch,sw,alpha=alpha)

    if lm == 'IM':
        seq_mat,wtrow = numerics.dataset2mutarray(df.copy(),modeltype)
        #this is also an MCMC routine, do the same as above.
        if initialize == 'rand':
            if modeltype == 'MAT':
                emat_0 = utils.RandEmat(len(df[seq_col_name][0]),len(seq_dict))
            elif modeltype == 'NBR':
                emat_0 = utils.RandEmat(len(df['seq'][0])-1,len(seq_dict))
        elif initialize == 'LS':
            emat_cols = ['val_' + inv_dict[i] for i in range(len(seq_dict))]
            emat_0_df = main(df.copy(),lm='LS',modeltype=modeltype,alpha=alpha,start=0,end=None,verbose=verbose)
            emat_0 = np.transpose(np.array(emat_0_df[emat_cols]))   
            #pymc doesn't take sparse mat        
        emat = MaximizeMI_memsaver(
                seq_mat,df.copy(),emat_0,wtrow,db=db,iteration=iteration,burnin=burnin,
                thin=thin,runnum=runnum,verbose=verbose)
    #now format the energy matrices to get them ready to output
    if (lm == 'IM' or lm == 'memsaver'):       
        if modeltype == 'NBR':
             emat_typical = gauge.fix_neighbor(np.transpose(emat))
        elif modeltype == 'MAT':
             emat_typical = gauge.fix_matrix(np.transpose(emat))
    
    elif lm == 'ER': 
        '''the emat for this format is currently transposed compared to other formats
        it is also already a data frame with columns [pos,val_...]'''
        emat_cols = ['val_' + inv_dict[i] for i in range(len(seq_dict))]
        emat_typical = emat[emat_cols]
        emat_typical = (gauge.fix_matrix((np.array(emat_typical))))
        
    else: #must be Least squares
        emat_typical = utils.emat_typical_parameterization(emat,len(seq_dict))        
        if modeltype == 'NBR':
             emat_typical = gauge.fix_neighbor(np.transpose(emat_typical))
        elif modeltype == 'MAT':
             emat_typical = gauge.fix_matrix(np.transpose(emat_typical))
    
    em = pd.DataFrame(emat_typical)
    em.columns = val_cols
    #add position column
    if modeltype == 'NBR':
        pos = pd.Series(range(start,start - 1 + len(df[seq_col_name][0])),name='pos') 
    else:
        pos = pd.Series(range(start,start + len(df[seq_col_name][0])),name='pos')    
    output_df = pd.concat([pos,em],axis=1)

    # Validate model and return
    output_df = qc.validate_model(output_df,fix=True)
    return output_df
Example #3
0
        M = pymc.MCMC([pymcdf,emat],db='sqlite',dbname=dbname)
    else:
        M = pymc.MCMC([pymcdf,emat])
    M.use_step_method(GaugePreservingStepper,emat)

    if not verbose:
        M.sample = shutthefuckup(M.sample)

    #M.sample(iteration,thin=thin,tune_interval=20000)
    M.sample(iteration,thin=thin)
    emat_mean = np.mean(M.trace('emat')[burnin:],axis=0)
    return emat_mean

df = pd.io.parsers.read_csv(sys.argv[1],delim_whitespace=True)

temp_seq_mat,wtrow = numerics.dataset2mutarray(df.copy(),'MAT')
temp_seq_mat2 = temp_seq_mat.toarray()

#we need a parameter for the effect of mutation at each base pair.
#we remove 20 base pairs to remove the barcode sequence on the end of the sequence.
len_seq = len(df.loc[0,'seq'])
len_outputseq = len_seq - 20
len_barcode = 20
#We add 4 parameters for the barcode, because 
total_params = len_outputseq + len_barcode*4
seq_mat = np.zeros((temp_seq_mat2.shape[0],total_params))
for i in range(len_outputseq):
    seq_mat[:,i] = np.sum(temp_seq_mat2[:,i*4:(i*4+4)],axis=1)
seq_mat[:,len_outputseq:] = temp_seq_mat2[:,-len_barcode*4:]
seq_mat = scipy.sparse.csr_matrix(seq_mat)
emat_0 = np.zeros((4,len_seq))
def main(data_df,
         model_df,
         start=0,
         end=None,
         err=False,
         coarse_graining_level=0,
         rsquared=False,
         return_freg=False):

    #determine whether you are working with RNA, DNA, or protein.
    #this also should determine modeltype (MAT, NBR, PAIR).
    dicttype, modeltype = qc.get_model_type(model_df)

    #get column header for the sequence column.
    seq_cols = qc.get_cols_from_df(data_df, 'seqs')
    if not len(seq_cols) == 1:
        raise SortSeqError('Dataframe has multiple seq cols: %s' %
                           str(seq_cols))

    #create dictionary that goes from, for example, nucleotide to number and
    #visa versa.
    seq_dict, inv_dict = utils.choose_dict(dicttype, modeltype=modeltype)

    #set name of sequences column based on type of sequence
    type_name_dict = {'dna': 'seq', 'rna': 'seq_rna', 'protein': 'seq_pro'}
    seq_col_name = type_name_dict[dicttype]

    if not end:
        seqL = len(data_df[seq_col_name][0]) - start
    else:
        seqL = end - start
    #throw out wrong length sequences.
    #Cut the sequences based on start and end, and then check if it makes sense
    if (start != 0 or end):
        data_df.loc[:,seq_col_name] = \
            data_df.loc[:,seq_col_name].str.slice(start,end)
        right_length = data_df.loc[:, seq_col_name].apply(len) == (seqL)
        if not right_length.all():
            sys.stderr.write('''Not all sequences are the same length! 
                       Throwing out incorrect sequences!''')
            data_df = data_df.loc[right_length, :]
        data_df = data_df.reset_index(drop=True)

        if modeltype == 'MAT':
            if seqL != len(model_df.loc[:, 'pos']):
                raise SortSeqError(
                    'model length does not match dataset length')
        elif modeltype == 'NBR':
            if seqL != len(model_df.loc[:, 'pos']) + 1:
                raise SortSeqError(
                    'model length does not match dataset length')
        elif modeltype == 'PAIR':
            if int(scipy.misc.comb(seqL, 2)) != len(model_df.loc[:, 'pos']):
                raise SortSeqError(
                    'model length does not match dataset length')

    #get column names of the counts columns (excluding total counts 'ct')
    col_headers = utils.get_column_headers(data_df)
    if 'ct' not in data_df.columns:
        data_df['ct'] = data_df[col_headers].sum(axis=1)

    #remove empty rows.
    data_df = data_df[data_df.ct != 0]

    #determine sequence length.

    #make a numpy array out of the model data frame
    model_df_headers = [
        'val_' + str(inv_dict[i]) for i in range(len(seq_dict))
    ]
    value = np.array(model_df[model_df_headers])

    #now we evaluate the expression of each sequence according to the model.
    #first convert to matrix representation of sequences
    seq_mat, wtrow = numerics.dataset2mutarray(data_df.copy(), modeltype)
    temp_df = data_df.copy()

    #evaluate energy of each sequence
    temp_df['val'] = numerics.eval_modelmatrix_on_mutarray(
        value, seq_mat, wtrow)

    #sort based on value
    temp_sorted = temp_df.sort_values(by='val')
    temp_sorted.reset_index(inplace=True, drop=True)

    #freg is a regularized plot which show how sequences are distributed
    #in energy space.
    if return_freg:
        fig, ax = plt.subplots()
        MI, freg = EstimateMutualInfoforMImax.alt4(
            temp_sorted,
            coarse_graining_level=coarse_graining_level,
            return_freg=return_freg)
        plt.imshow(freg, interpolation='nearest', aspect='auto')

        plt.savefig(return_freg)
    else:
        MI = EstimateMutualInfoforMImax.alt4(
            temp_sorted,
            coarse_graining_level=coarse_graining_level,
            return_freg=return_freg)

    #if we want to calculate error then use bootstrapping.
    if not err:
        Std = np.NaN
    else:
        data_df_for_sub = data_df.copy()
        sub_MI = np.zeros(15)
        for i in range(15):
            sub_df = data_df_for_sub.sample(int(
                len(data_df_for_sub.index) / 2))
            sub_df.reset_index(inplace=True, drop=True)
            sub_MI[i], sub_std = main(sub_df, model_df, err=False)
        Std = np.std(sub_MI) / np.sqrt(2)

    #we can return linfoot corrolation (rsquared) or return MI.
    if rsquared:
        return (1 - 2**(-2 * MI)), (1 - 2**(-2 * Std))
    else:
        return MI, Std
Example #5
0
# Create sequences to test this on
wtseq = 'AAAAAAAGTGAGATGGCAATCTAATTCGGCACCCCAGGTTTTACACTTTATGCTTCCGGCTCGTATGTTGTGTGG'
L = len(wtseq)
modeltypes = ['MAT','NBR']
seqtypes = ['dna','protein']
numseqs_dict = {'dna':10000,'protein':1000}
for seqtype in seqtypes:
    for modeltype in modeltypes:
        for mutrate in [0.01,0.1,1]:
            numseqs = numseqs_dict[seqtype]
            dataset_df = simulate_library(wtseq,numseq=numseqs,mutrate=mutrate,tags=True,\
                dicttype=seqtype)
            seqarray = numerics.dataset2seqarray(dataset_df,\
                modeltype=modeltype)
            mutarray, wtrow = numerics.dataset2mutarray(dataset_df,\
                modeltype=modeltype)

            # Print compression results
            seqarray_size = numerics.nbytes(seqarray)
            mutarray_size = numerics.nbytes(mutarray)

            # Create matrix for random model
            alphabet = qc.seqtype_to_alphabet_dict[seqtype]
            C = len(alphabet)
            num_rows = (L-1) if modeltype=='NBR' else L
            num_cols = C**2 if modeltype=='NBR' else C
            modelmatrix = randn(num_rows,num_cols)

            # Create model dataframe
            val_cols = qc.model_parameters_dict[(modeltype,seqtype)]
            model_df = pd.DataFrame(modelmatrix,columns=val_cols)
Example #6
0
# Create sequences to test this on
wtseq = 'AAAAAAAGTGAGATGGCAATCTAATTCGGCACCCCAGGTTTTACACTTTATGCTTCCGGCTCGTATGTTGTGTGG'
L = len(wtseq)
modeltypes = ['MAT', 'NBR']
seqtypes = ['dna', 'protein']
numseqs_dict = {'dna': 10000, 'protein': 1000}
for seqtype in seqtypes:
    for modeltype in modeltypes:
        for mutrate in [0.01, 0.1, 1]:
            numseqs = numseqs_dict[seqtype]
            dataset_df = simulate_library(wtseq,numseq=numseqs,mutrate=mutrate,tags=True,\
                dicttype=seqtype)
            seqarray = numerics.dataset2seqarray(dataset_df,\
                modeltype=modeltype)
            mutarray, wtrow = numerics.dataset2mutarray(dataset_df,\
                modeltype=modeltype)

            # Print compression results
            seqarray_size = numerics.nbytes(seqarray)
            mutarray_size = numerics.nbytes(mutarray)

            # Create matrix for random model
            alphabet = qc.seqtype_to_alphabet_dict[seqtype]
            C = len(alphabet)
            num_rows = (L - 1) if modeltype == 'NBR' else L
            num_cols = C**2 if modeltype == 'NBR' else C
            modelmatrix = randn(num_rows, num_cols)

            # Create model dataframe
            val_cols = qc.model_parameters_dict[(modeltype, seqtype)]
            model_df = pd.DataFrame(modelmatrix, columns=val_cols)