Ejemplo n.º 1
0
def generate_DF(ref):    
    ref_arr = numpy.asarray(ref)
    qval = qvalue.estimate(ref_arr[:,[1]])
    q_val = numpy.asarray(qval)
    df_all  = numpy.append(ref_arr, q_val, 1)
    insig_ref = df_all[df_all[:,2] > 0.2]
    print df_all
    def adjust_p_values(self):
        """
        Calculates q-values for all p-values.
        """

        # Fetch all p-values
        p_values = list()
        for chrom in self.bin_dict.keys():
            for bin in self.bin_dict[chrom].keys():
                p_values.append(self.bin_dict[chrom][bin][2])

        # convert to numpy array and calculate q-values
        p_array = numpy.array(p_values)
        q_array = qvalue.estimate(p_array)

        # build p to q-value dict
        p_to_q_value_dict = dict()
        for i in range(len(q_array)):
            p_to_q_value_dict[p_array[i]] = q_array[i]

        # Adds a q-value at the end of the list for each bin
        for chrom in self.bin_dict.keys():
            for bin in self.bin_dict[chrom].keys():
                pval = self.bin_dict[chrom][bin][2]
                self.bin_dict[chrom][bin].append(p_to_q_value_dict[pval])
Ejemplo n.º 3
0
    def find_incoming_edges(self, t):
        """
        find incoming edges that build a v-structure a-->t<--b with
        a) corr(a,t)
        b) corr(b,t)
        c) ind(a,b)
        d) corr(a,b  t)

        input:
        t   :   index of the gene t
        """

        # incoming edges are associated with the gene of interest...
        pv_genes = self.genecorr_reader.getRows([t])[0]
        idx_assoc = qvalue.estimate(pv_genes) < self.thresh_corr
        idx_assoc[t] = False
        if not (idx_assoc).any():
            return None, None

        # independent of each other
        _idx_assoc = np.nonzero(idx_assoc)[0]
        pv_genes = self.genecorr_reader.getRows(_idx_assoc)[:, idx_assoc]
        idx_vstruct = np.nonzero(idx_assoc)[0]
        vstruct = pv_genes > self.thresh_ind
        idx_ind = vstruct.any(axis=1)
        idx_vstruct = idx_vstruct[idx_ind]
        vstruct = vstruct[idx_ind][:, idx_ind]
        if not (idx_vstruct).any():
            return None, None

        # becoming dependent once we condition on the gene under observation
        Yv = self.phenoreader.getRows(idx_vstruct)
        Yt = self.phenoreader.getRows([t])[0]
        _, pv_cond = pcor.pcorParallel(Yv, Yt)
        qv_cond = qvalue.estimate(pv_cond)
        vstruct *= qv_cond < self.thresh_corr
        idx_partcorr = vstruct.any(axis=0)
        if not (idx_partcorr).any():
            return None, None
        vstruct = vstruct[idx_partcorr][:, idx_partcorr]
        idx_vstruct = idx_vstruct[idx_partcorr]

        return vstruct, idx_vstruct
Ejemplo n.º 4
0
 def association_scan(self):
     print "Association scan... ",
     K = self.kernel_testing(genetics=False, confounders=True)
     pval = testing.interface(self.S_centered, self.Y, K, I = None,
                              model='LMM', parallel=False, # TODO parallelize
                              file_directory = None, jobs = 0)[0]
     print "[DONE]"
     # convert to qvalues
     qval = qvalue.estimate(pval)
     return qval, pval
Ejemplo n.º 5
0
    def find_incoming_edges(self, t):
        """
        find incoming edges that build a v-structure a-->t<--b with
        a) corr(a,t)
        b) corr(b,t)
        c) ind(a,b)
        d) corr(a,b  t)

        input:
        t   :   index of the gene t
        """

        # incoming edges are associated with the gene of interest...
        pv_genes = self.genecorr_reader.getRows([t])[0]
        idx_assoc = qvalue.estimate(pv_genes) < self.thresh_corr
        idx_assoc[t] = False
        if not (idx_assoc).any(): return None, None

        # independent of each other
        _idx_assoc = np.nonzero(idx_assoc)[0]
        pv_genes = self.genecorr_reader.getRows(_idx_assoc)[:, idx_assoc]
        idx_vstruct = np.nonzero(idx_assoc)[0]
        vstruct = pv_genes > self.thresh_ind
        idx_ind = vstruct.any(axis=1)
        idx_vstruct = idx_vstruct[idx_ind]
        vstruct = vstruct[idx_ind][:, idx_ind]
        if not (idx_vstruct).any(): return None, None

        # becoming dependent once we condition on the gene under observation
        Yv = self.phenoreader.getRows(idx_vstruct)
        Yt = self.phenoreader.getRows([t])[0]
        _, pv_cond = pcor.pcorParallel(Yv, Yt)
        qv_cond = qvalue.estimate(pv_cond)
        vstruct *= (qv_cond < self.thresh_corr)
        idx_partcorr = vstruct.any(axis=0)
        if not (idx_partcorr).any(): return None, None
        vstruct = vstruct[idx_partcorr][:, idx_partcorr]
        idx_vstruct = idx_vstruct[idx_partcorr]

        return vstruct, idx_vstruct
Ejemplo n.º 6
0
    def panama_step(self):
        X = self.get_latent()
        K = self.kernel_testing(genetics=False, confounders=False)
        K = scaleK(K)
        if len(self.candidate_associations) != 0:
            covs = self.S_centered[:, self.candidate_associations].copy()
        else:
            covs = None

        pv = testing.interface(self.S_centered, X[:, :self.Q], K, covs=covs, model = "LMM",
                               parallel = False, jobs = 0,
                               file_directory=None)[0] # TODO cleanup


        # Number of tests conducted
        num_tests = X.shape[1]*self.S.shape[1]*self.iteration
        qv = qvalue.estimate(pv, m=num_tests)

        # Set the qvalue of the current associations to 1
        qv[self.candidate_associations,:] = 1

        # Greedily construct addition set by adding the BEST (lowest qv) SNP for each factor
        # (if significant)
        new_candidates = []

        for i in xrange(qv.shape[1]):
            i_best = qv[:,i].argmin()
            qv_best = qv[:,i].min()
            # if significant, add it
            if qv_best<=self.FDR_associations:
                new_candidates.append(i_best)
                # and set the corrisponding qvalue to 1
                qv[i_best,:] = 1

        # Add candidates
        nc_old = len(self.candidate_associations)
        self.candidate_associations.extend(new_candidates)
        nc = len(self.candidate_associations)
        dl = nc-nc_old

        assert len(np.unique(self.candidate_associations)) == len(self.candidate_associations)

        return dl
Ejemplo n.º 7
0
def fix_rows(genes, pvalues):
    pvalues = numpy.array(pvalues)
    _qvalues = qvalue.estimate(pvalues)
    _qvalues = {pvalues[i]:q for i,q in enumerate(_qvalues)}
    F = Utilities.WDBIF
    keys = genes.keys()
    for gene in keys:
        rows = genes[gene]
        input = [(math.fabs(float(r[F.WEIGHT])), float(r[F.GENE_PVALUE]), _qvalues[float(r[F.GENE_PVALUE])], r[F.SNP]) for k,r in rows.iteritems()]
        w = sum(map(lambda x: x[0], input))
        if w == 0:
            logging.info("Cannot handle null weight for %s", gene)
            a_p = None
            a_q = None
        else:
            a_p = str(sum(map(lambda x: math.fabs(x[0])*x[1], input)) / w)
            a_q = str(sum(map(lambda x: math.fabs(x[0])*x[2], input)) / w)
        n = str(len(input))
        genes[gene] = [(r[F.SNP], r[F.GENE], r[F.GENE_NAME], r[F.REFERENCE_ALLELE], r[F.EFFECT_ALLELE], r[F.WEIGHT], n, r[F.GENE_R2], a_p, a_q) for k,r in rows.iteritems()]
    return genes
Ejemplo n.º 8
0
def reorder(data,
            data_headers,
            array_order,
            comp_group_list,
            probeset_db,
            include_raw_data,
            array_type,
            norm,
            fl,
            logvalues=True,
            blanksPresent=False):
    ###array_order gives the final level order sorted, followed by the original index order as a tuple
    expbuilder_value_db = {}
    group_name_db = {}
    summary_filtering_stats = {}
    pval_summary_db = {}
    replicates = 'yes'

    stat_result_names = ['avg-', 'log_fold-', 'fold-', 'rawp-', 'adjp-']
    group_summary_result_names = ['avg-']

    ### Define expression variables
    try:
        probability_statistic = fl.ProbabilityStatistic()
    except Exception:
        probability_statistic = 'unpaired t-test'
    try:
        gene_exp_threshold = math.log(fl.GeneExpThreshold(), 2)
    except Exception:
        gene_exp_threshold = 0
    try:
        gene_rpkm_threshold = float(fl.RPKMThreshold())
    except Exception:
        gene_rpkm_threshold = 0
    try:
        FDR_statistic = fl.FDRStatistic()
    except Exception:
        FDR_statistic = 'Benjamini-Hochberg'
    calculateAsNonLog = True
    if blanksPresent:
        calculateAsNonLog = False

    ### Begin processing sample expression values according to the organized groups
    for row_id in data:
        try:
            gene = probeset_db[row_id][0]
        except TypeError:
            gene = ''  #not needed if not altsplice data
        data_headers2 = {}  #reset each time
        grouped_ordered_array_list = {}
        for x in array_order:
            y = x[1]  #this is the new first index
            group = x[2]
            group_name = x[3]
            group_name_db[group] = group_name
            #for example y = 5, therefore the data[row_id][5] entry is now the first
            try:
                try:
                    new_item = data[row_id][y]
                except IndexError:
                    print row_id, data[row_id], len(
                        data[row_id]), y, len(array_order), array_order
                    kill
                if logvalues == False and calculateAsNonLog and array_type == 'RNASeq':
                    new_item = math.pow(2, new_item)
            except TypeError:
                new_item = ''  #this is for a spacer added in the above function
            try:
                grouped_ordered_array_list[group].append(new_item)
            except KeyError:
                grouped_ordered_array_list[group] = [new_item]
            try:
                data_headers2[group].append(data_headers[y])
            except KeyError:
                data_headers2[group] = [data_headers[y]]
        #perform statistics on each group comparison - comp_group_list: [(1,2),(3,4)]
        stat_results = {}
        group_summary_results = {}
        for comp in comp_group_list:
            group1 = int(comp[0])
            group2 = int(comp[1])
            group1_name = group_name_db[group1]
            group2_name = group_name_db[group2]
            groups_name = group1_name + "_vs_" + group2_name
            data_list1 = grouped_ordered_array_list[group1]
            data_list2 = grouped_ordered_array_list[
                group2]  #baseline expression
            if blanksPresent:  ### Allows for empty cells
                data_list1 = filterBlanks(data_list1)
                data_list2 = filterBlanks(data_list2)
            try:
                avg1 = statistics.avg(data_list1)
            except Exception:
                avg1 = ''
            try:
                avg2 = statistics.avg(data_list2)
            except Exception:
                avg2 = ''
            try:
                if (logvalues == False
                        and array_type != 'RNASeq') or (logvalues == False
                                                        and calculateAsNonLog):
                    fold = avg1 / avg2
                    log_fold = math.log(fold, 2)
                    if fold < 1: fold = -1.0 / fold
                else:
                    log_fold = avg1 - avg2
                    fold = statistics.log_fold_conversion(log_fold)
            except Exception:
                log_fold = ''
                fold = ''
            try:
                #t,df,tails = statistics.ttest(data_list1,data_list2,2,3) #unpaired student ttest, calls p_value function
                #t = abs(t); df = round(df); p = str(statistics.t_probability(t,df))
                p = statistics.runComparisonStatistic(data_list1, data_list2,
                                                      probability_statistic)
            except Exception:
                p = 1
                sg = 1
                N1 = 0
                N2 = 0
            comp = group1, group2
            if array_type == 'RNASeq':  ### Also non-log but treated differently
                if 'RPKM' == norm: adj = 0
                else: adj = 1
                if calculateAsNonLog == False:
                    try:
                        avg1 = math.pow(2, avg1) - adj
                        avg2 = math.pow(2, avg2) - adj
                    except Exception:
                        avg1 = ''
                        avg2 = ''
                if 'RPKM' == norm:
                    if avg1 < gene_rpkm_threshold and avg2 < gene_rpkm_threshold:
                        log_fold = 'Insufficient Expression'
                        fold = 'Insufficient Expression'
                else:
                    if avg1 < gene_exp_threshold and avg2 < gene_exp_threshold:
                        log_fold = 'Insufficient Expression'
                        fold = 'Insufficient Expression'
                    #if row_id=='ENSG00000085514':
                    #if fold=='Insufficient Expression':
                    #print [norm, avg1, avg2, fold, comp, gene_exp_threshold, gene_rpkm_threshold, row_id]
                    #5.96999111075 7.72930768675 Insufficient Expression (3, 1) 1.0 ENSG00000085514
            if gene_rpkm_threshold != 0 and calculateAsNonLog:  ### Any other data
                a1 = nonLogAvg(data_list1)
                a2 = nonLogAvg(data_list2)
                #print [a1,a2,gene_rpkm_threshold]
                if a1 < gene_rpkm_threshold and a2 < gene_rpkm_threshold:
                    log_fold = 'Insufficient Expression'
                    fold = 'Insufficient Expression'
                #print log_fold;kill
            try:
                gs = statistics.GroupStats(log_fold, fold, p)
                stat_results[comp] = groups_name, gs, group2_name
                if probability_statistic == 'moderated t-test':
                    gs.setAdditionalStats(
                        data_list1, data_list2)  ### Assuming equal variance
                if probability_statistic == 'moderated Welch-test':
                    gs.setAdditionalWelchStats(
                        data_list1, data_list2)  ### Assuming unequal variance
            except Exception:
                null = []
                replicates = 'no'  ### Occurs when not enough replicates
                #print comp, len(stat_results); kill_program
            group_summary_results[group1] = group1_name, [avg1]
            group_summary_results[group2] = group2_name, [avg2]

        ### Replaces the below method to get the largest possible comparison fold and ftest p-value
        grouped_exp_data = []
        avg_exp_data = []
        for group in grouped_ordered_array_list:
            data_list = grouped_ordered_array_list[group]
            if blanksPresent:  ### Allows for empty cells
                data_list = filterBlanks(data_list)
            if len(data_list) > 0: grouped_exp_data.append(data_list)
            try:
                avg = statistics.avg(data_list)
                avg_exp_data.append(avg)
            except Exception:
                avg = ''
                #print row_id, group, data_list;kill
        try:
            avg_exp_data.sort()
            max_fold = avg_exp_data[-1] - avg_exp_data[0]
        except Exception:
            max_fold = 'NA'
        try:
            ftestp = statistics.OneWayANOVA(grouped_exp_data)
        except Exception:
            ftestp = 1
        gs = statistics.GroupStats(max_fold, 0, ftestp)
        summary_filtering_stats[row_id] = gs

        stat_result_list = []
        for entry in stat_results:
            data_tuple = entry, stat_results[entry]
            stat_result_list.append(data_tuple)
        stat_result_list.sort()

        grouped_ordered_array_list2 = []
        for group in grouped_ordered_array_list:
            data_tuple = group, grouped_ordered_array_list[group]
            grouped_ordered_array_list2.append(data_tuple)
        grouped_ordered_array_list2.sort(
        )  #now the list is sorted by group number

        ###for each rowid, add in the reordered data, and new statistics for each group and for each comparison
        for entry in grouped_ordered_array_list2:
            group_number = entry[0]
            original_data_values = entry[1]
            if include_raw_data == 'yes':  ###optionally exclude the raw values
                for value in original_data_values:
                    if array_type == 'RNASeq':
                        if norm == 'RPKM': adj = 0
                        else: adj = 1
                        if calculateAsNonLog == False:
                            value = math.pow(2, value) - adj
                    try:
                        expbuilder_value_db[row_id].append(value)
                    except KeyError:
                        expbuilder_value_db[row_id] = [value]
            if group_number in group_summary_results:
                group_summary_data = group_summary_results[group_number][
                    1]  #the group name is listed as the first entry
                for value in group_summary_data:
                    try:
                        expbuilder_value_db[row_id].append(value)
                    except KeyError:
                        expbuilder_value_db[row_id] = [value]
            for info in stat_result_list:
                if info[0][
                        0] == group_number:  #comp,(groups_name,[avg1,log_fold,fold,ttest])
                    comp = info[0]
                    gs = info[1][1]
                    expbuilder_value_db[row_id].append(gs.LogFold())
                    expbuilder_value_db[row_id].append(gs.Fold())
                    expbuilder_value_db[row_id].append(gs.Pval())
                    ### Create a placeholder and store the position of the adjusted p-value to be calculated
                    expbuilder_value_db[row_id].append('')
                    gs.SetAdjPIndex(len(expbuilder_value_db[row_id]) - 1)
                    gs.SetPvalIndex(len(expbuilder_value_db[row_id]) - 2)
                    pval_summary_db[(row_id, comp)] = gs

    ###do the same for the headers, but at the dataset level (redundant processes)
    array_fold_headers = []
    data_headers3 = []
    try:
        for group in data_headers2:
            data_tuple = group, data_headers2[
                group]  #e.g. 1, ['X030910_25_hl.CEL', 'X030910_29R_hl.CEL', 'X030910_45_hl.CEL'])
            data_headers3.append(data_tuple)
        data_headers3.sort()
    except UnboundLocalError:
        print data_headers, '\n', array_order, '\n', comp_group_list, '\n'
        kill_program

    for entry in data_headers3:
        x = 0  #indicates the times through a loop
        y = 0  #indicates the times through a loop
        group_number = entry[0]
        original_data_values = entry[1]
        if include_raw_data == 'yes':  ###optionally exclude the raw values
            for value in original_data_values:
                array_fold_headers.append(value)
        if group_number in group_summary_results:
            group_name = group_summary_results[group_number][0]
            group_summary_data = group_summary_results[group_number][1]
            for value in group_summary_data:
                combined_name = group_summary_result_names[
                    x] + group_name  #group_summary_result_names = ['avg-']
                array_fold_headers.append(combined_name)
                x += 1  #increment the loop index

        for info in stat_result_list:
            if info[0][
                    0] == group_number:  #comp,(groups_name,[avg1,log_fold,fold,ttest],group2_name)
                groups_name = info[1][0]
                only_add_these = stat_result_names[1:]
                for value in only_add_these:
                    new_name = value + groups_name
                    array_fold_headers.append(new_name)

    ###For the raw_data only export we need the headers for the different groups (data_headers2) and group names (group_name_db)
    raw_data_comp_headers = {}
    for comp in comp_group_list:
        temp_raw = []
        group1 = int(comp[0])
        group2 = int(comp[1])
        comp = str(comp[0]), str(comp[1])
        g1_headers = data_headers2[group1]
        g2_headers = data_headers2[group2]
        g1_name = group_name_db[group1]
        g2_name = group_name_db[group2]
        for header in g2_headers:
            temp_raw.append(g2_name + ':' + header)
        for header in g1_headers:
            temp_raw.append(g1_name + ':' + header)
        raw_data_comp_headers[comp] = temp_raw

    ###Calculate adjusted ftest p-values using BH95 sorted method
    statistics.adjustPermuteStats(summary_filtering_stats)

    ### Calculate adjusted p-values for all p-values using BH95 sorted method
    round = 0
    for info in comp_group_list:
        compid = int(info[0]), int(info[1])
        pval_db = {}
        for (rowid, comp) in pval_summary_db:
            if comp == compid:
                gs = pval_summary_db[(rowid, comp)]
                pval_db[rowid] = gs

        if 'moderated' in probability_statistic and replicates == 'yes':
            ### Moderates the original reported test p-value prior to adjusting
            try:
                statistics.moderateTestStats(pval_db, probability_statistic)
            except Exception:
                if round == 0:
                    if replicates == 'yes':
                        print 'Moderated test failed due to issue with mpmpath or out-of-range values\n   ... using unmoderated unpaired test instead!'
                null = []  ### Occurs when not enough replicates
            round += 1

        if FDR_statistic == 'Benjamini-Hochberg':
            statistics.adjustPermuteStats(pval_db)
        else:
            ### Calculate a qvalue (https://github.com/nfusi/qvalue)
            import numpy
            import qvalue
            pvals = []
            keys = []
            for key in pval_db:
                pvals.append(pval_db[key].Pval())
                keys.append(key)
            pvals = numpy.array(pvals)
            pvals = qvalue.estimate(pvals)
            for i in range(len(pvals)):
                pval_db[keys[i]].SetAdjP(pvals[i])

        for rowid in pval_db:
            gs = pval_db[rowid]
            expbuilder_value_db[rowid][gs.AdjIndex()] = gs.AdjP(
            )  ### set the place holder to the calculated value
            if 'moderated' in probability_statistic:
                expbuilder_value_db[rowid][gs.RawIndex()] = gs.Pval(
                )  ### Replace the non-moderated with a moderated p-value

    pval_summary_db = []
    ###Finished re-ordering lists and adding statistics to expbuilder_value_db
    return expbuilder_value_db, array_fold_headers, summary_filtering_stats, raw_data_comp_headers
Ejemplo n.º 9
0
def reorder(data,data_headers,array_order,comp_group_list,probeset_db,include_raw_data,array_type,norm,fl,logvalues=True,blanksPresent=False):
    ###array_order gives the final level order sorted, followed by the original index order as a tuple                   
    expbuilder_value_db = {}; group_name_db = {}; summary_filtering_stats = {}; pval_summary_db= {}
    replicates = 'yes'
    
    stat_result_names = ['avg-','log_fold-','fold-','rawp-','adjp-']
    group_summary_result_names = ['avg-']
    
    ### Define expression variables
    try: probability_statistic = fl.ProbabilityStatistic()
    except Exception: probability_statistic = 'unpaired t-test'
    try: gene_exp_threshold = math.log(fl.GeneExpThreshold(),2)
    except Exception: gene_exp_threshold = 0
    try: gene_rpkm_threshold = float(fl.RPKMThreshold())
    except Exception: gene_rpkm_threshold = 0
    try: FDR_statistic = fl.FDRStatistic()
    except Exception: FDR_statistic = 'Benjamini-Hochberg'
    calculateAsNonLog=True
    if blanksPresent:
        calculateAsNonLog=False
    
    ### Begin processing sample expression values according to the organized groups
    for row_id in data:
        try: gene = probeset_db[row_id][0]
        except TypeError: gene = '' #not needed if not altsplice data
        data_headers2 = {} #reset each time
        grouped_ordered_array_list = {}
        for x in array_order:
            y = x[1]  #this is the new first index
            group = x[2]
            group_name = x[3]
            group_name_db[group] = group_name
            #for example y = 5, therefore the data[row_id][5] entry is now the first
            try:
                try: new_item = data[row_id][y]
                except IndexError: print row_id,data[row_id],len(data[row_id]),y,len(array_order),array_order;kill
                if logvalues==False and calculateAsNonLog and array_type == 'RNASeq':
                    new_item = math.pow(2,new_item)
            except TypeError: new_item = ''  #this is for a spacer added in the above function
            try: grouped_ordered_array_list[group].append(new_item)
            except KeyError: grouped_ordered_array_list[group] = [new_item]
            try: data_headers2[group].append(data_headers[y])
            except KeyError: data_headers2[group]= [data_headers[y]]
        #perform statistics on each group comparison - comp_group_list: [(1,2),(3,4)]
        stat_results = {}
        group_summary_results = {}
        for comp in comp_group_list:
            group1 = int(comp[0])
            group2 = int(comp[1])
            group1_name = group_name_db[group1]
            group2_name = group_name_db[group2]
            groups_name = group1_name + "_vs_" + group2_name
            data_list1 = grouped_ordered_array_list[group1] 
            data_list2 = grouped_ordered_array_list[group2] #baseline expression
            if blanksPresent: ### Allows for empty cells
                data_list1 = filterBlanks(data_list1)
                data_list2 = filterBlanks(data_list2)
            try: avg1 = statistics.avg(data_list1)
            except Exception: avg1 = ''
            try: avg2 = statistics.avg(data_list2)
            except Exception: avg2=''
            try:
                if (logvalues == False and array_type != 'RNASeq') or (logvalues==False and calculateAsNonLog):
                    fold = avg1/avg2
                    log_fold = math.log(fold,2)
                    if fold<1: fold = -1.0/fold
                else:
                    log_fold = avg1 - avg2
                    fold = statistics.log_fold_conversion(log_fold) 
            except Exception:
                log_fold=''; fold=''
            try:
                #t,df,tails = statistics.ttest(data_list1,data_list2,2,3) #unpaired student ttest, calls p_value function
                #t = abs(t); df = round(df); p = str(statistics.t_probability(t,df))
                p = statistics.runComparisonStatistic(data_list1,data_list2,probability_statistic)
            except Exception: p = 1; sg = 1; N1=0; N2=0
            comp = group1,group2
            if array_type == 'RNASeq': ### Also non-log but treated differently
                if 'RPKM' == norm: adj = 0
                else: adj = 1
                if calculateAsNonLog == False:
                    try: avg1 = math.pow(2,avg1)-adj; avg2 = math.pow(2,avg2)-adj
                    except Exception: avg1=''; avg2=''
                if 'RPKM' == norm:
                    if avg1 < gene_rpkm_threshold and avg2 < gene_rpkm_threshold:
                        log_fold = 'Insufficient Expression'
                        fold = 'Insufficient Expression'
                else:
                    if avg1 < gene_exp_threshold and avg2 < gene_exp_threshold:
                        log_fold = 'Insufficient Expression'
                        fold = 'Insufficient Expression'
                    #if row_id=='ENSG00000085514':
                    #if fold=='Insufficient Expression':
                    #print [norm, avg1, avg2, fold, comp, gene_exp_threshold, gene_rpkm_threshold, row_id]
                    #5.96999111075 7.72930768675 Insufficient Expression (3, 1) 1.0 ENSG00000085514
            if gene_rpkm_threshold!=0 and calculateAsNonLog: ### Any other data
                a1 = nonLogAvg(data_list1)
                a2 = nonLogAvg(data_list2)
                #print [a1,a2,gene_rpkm_threshold]
                if a1<gene_rpkm_threshold and a2<gene_rpkm_threshold:
                    log_fold = 'Insufficient Expression'
                    fold = 'Insufficient Expression'
                #print log_fold;kill
            try:
                gs = statistics.GroupStats(log_fold,fold,p)
                stat_results[comp] = groups_name,gs,group2_name
                if probability_statistic == 'moderated t-test':
                    gs.setAdditionalStats(data_list1,data_list2) ### Assuming equal variance
                if probability_statistic == 'moderated Welch-test':
                    gs.setAdditionalWelchStats(data_list1,data_list2) ### Assuming unequal variance
            except Exception:
                null=[]; replicates = 'no' ### Occurs when not enough replicates
                #print comp, len(stat_results); kill_program
            group_summary_results[group1] = group1_name,[avg1]
            group_summary_results[group2] = group2_name,[avg2]

        ### Replaces the below method to get the largest possible comparison fold and ftest p-value
        grouped_exp_data = []; avg_exp_data = []
        for group in grouped_ordered_array_list:
            data_list = grouped_ordered_array_list[group]
            if blanksPresent: ### Allows for empty cells
                data_list = filterBlanks(data_list)
            if len(data_list)>0: grouped_exp_data.append(data_list)
            try: avg = statistics.avg(data_list); avg_exp_data.append(avg)
            except Exception:
                avg = ''
                #print row_id, group, data_list;kill
        try: avg_exp_data.sort(); max_fold = avg_exp_data[-1]-avg_exp_data[0]
        except Exception: max_fold = 'NA'
        try: ftestp = statistics.OneWayANOVA(grouped_exp_data)
        except Exception: ftestp = 1
        gs = statistics.GroupStats(max_fold,0,ftestp)
        summary_filtering_stats[row_id] = gs
        
        stat_result_list = []
        for entry in stat_results:
            data_tuple = entry,stat_results[entry]
            stat_result_list.append(data_tuple)
        stat_result_list.sort()
        
        grouped_ordered_array_list2 = []
        for group in grouped_ordered_array_list:
            data_tuple = group,grouped_ordered_array_list[group]
            grouped_ordered_array_list2.append(data_tuple)
        grouped_ordered_array_list2.sort() #now the list is sorted by group number
        
        ###for each rowid, add in the reordered data, and new statistics for each group and for each comparison
        for entry in grouped_ordered_array_list2:
            group_number = entry[0]
            original_data_values = entry[1]
            if include_raw_data == 'yes': ###optionally exclude the raw values
                for value in original_data_values:
                    if array_type == 'RNASeq':
                        if norm == 'RPKM': adj = 0
                        else: adj = 1
                        if calculateAsNonLog == False:
                            value = math.pow(2,value)-adj
                    try: expbuilder_value_db[row_id].append(value)
                    except KeyError: expbuilder_value_db[row_id] = [value]
            if group_number in group_summary_results:
                group_summary_data = group_summary_results[group_number][1] #the group name is listed as the first entry
                for value in group_summary_data:
                    try: expbuilder_value_db[row_id].append(value)
                    except KeyError: expbuilder_value_db[row_id] = [value]
            for info in stat_result_list:
                if info[0][0] == group_number: #comp,(groups_name,[avg1,log_fold,fold,ttest])
                    comp = info[0]; gs = info[1][1]
                    expbuilder_value_db[row_id].append(gs.LogFold())
                    expbuilder_value_db[row_id].append(gs.Fold())
                    expbuilder_value_db[row_id].append(gs.Pval())
                    ### Create a placeholder and store the position of the adjusted p-value to be calculated
                    expbuilder_value_db[row_id].append('') 
                    gs.SetAdjPIndex(len(expbuilder_value_db[row_id])-1)
                    gs.SetPvalIndex(len(expbuilder_value_db[row_id])-2)
                    pval_summary_db[(row_id,comp)] = gs

    ###do the same for the headers, but at the dataset level (redundant processes)
    array_fold_headers = []; data_headers3 = []
    try:
        for group in data_headers2:
            data_tuple = group,data_headers2[group]  #e.g. 1, ['X030910_25_hl.CEL', 'X030910_29R_hl.CEL', 'X030910_45_hl.CEL'])
            data_headers3.append(data_tuple)
        data_headers3.sort()
    except UnboundLocalError:
        print data_headers,'\n',array_order,'\n',comp_group_list,'\n'; kill_program
    
    for entry in data_headers3:
        x = 0 #indicates the times through a loop
        y = 0 #indicates the times through a loop
        group_number = entry[0]
        original_data_values = entry[1]
        if include_raw_data == 'yes': ###optionally exclude the raw values
            for value in original_data_values:
                array_fold_headers.append(value)
        if group_number in group_summary_results:
            group_name = group_summary_results[group_number][0]
            group_summary_data = group_summary_results[group_number][1]
            for value in group_summary_data:
                combined_name = group_summary_result_names[x] + group_name  #group_summary_result_names = ['avg-']
                array_fold_headers.append(combined_name)
                x += 1 #increment the loop index

        for info in stat_result_list:
            if info[0][0] == group_number:  #comp,(groups_name,[avg1,log_fold,fold,ttest],group2_name)
                groups_name = info[1][0]
                only_add_these = stat_result_names[1:]
                for value in only_add_these:
                    new_name = value + groups_name
                    array_fold_headers.append(new_name)

    ###For the raw_data only export we need the headers for the different groups (data_headers2) and group names (group_name_db)       
    raw_data_comp_headers = {}
    for comp in comp_group_list:
        temp_raw = []
        group1 = int(comp[0]);group2 = int(comp[1])
        comp = str(comp[0]),str(comp[1])
        g1_headers = data_headers2[group1]
        g2_headers = data_headers2[group2]
        g1_name = group_name_db[group1]
        g2_name = group_name_db[group2]
        for header in g2_headers: temp_raw.append(g2_name+':'+header)
        for header in g1_headers: temp_raw.append(g1_name+':'+header)
        raw_data_comp_headers[comp] = temp_raw

    ###Calculate adjusted ftest p-values using BH95 sorted method
    statistics.adjustPermuteStats(summary_filtering_stats)
    
    ### Calculate adjusted p-values for all p-values using BH95 sorted method
    round=0
    for info in comp_group_list:
        compid = int(info[0]),int(info[1]); pval_db={}
        for (rowid,comp) in pval_summary_db:
            if comp == compid:
                gs = pval_summary_db[(rowid,comp)]
                pval_db[rowid] = gs

        if 'moderated' in probability_statistic and replicates == 'yes':
            ### Moderates the original reported test p-value prior to adjusting
            try: statistics.moderateTestStats(pval_db,probability_statistic)
            except Exception:
                if round == 0:
                    if replicates == 'yes':
                        print 'Moderated test failed due to issue with mpmpath or out-of-range values\n   ... using unmoderated unpaired test instead!'
                null=[] ### Occurs when not enough replicates
            round+=1
            
        if FDR_statistic == 'Benjamini-Hochberg':
            statistics.adjustPermuteStats(pval_db)
        else:
            ### Calculate a qvalue (https://github.com/nfusi/qvalue)
            import numpy; import qvalue; pvals = []; keys = []
            for key in pval_db: pvals.append(pval_db[key].Pval()); keys.append(key)
            pvals = numpy.array(pvals)
            pvals = qvalue.estimate(pvals)
            for i in range(len(pvals)): pval_db[keys[i]].SetAdjP(pvals[i])
            
        for rowid in pval_db:
            gs = pval_db[rowid]
            expbuilder_value_db[rowid][gs.AdjIndex()] = gs.AdjP() ### set the place holder to the calculated value
            if 'moderated' in probability_statistic:
                expbuilder_value_db[rowid][gs.RawIndex()] = gs.Pval() ### Replace the non-moderated with a moderated p-value
                
    pval_summary_db=[]            
    ###Finished re-ordering lists and adding statistics to expbuilder_value_db
    return expbuilder_value_db, array_fold_headers, summary_filtering_stats, raw_data_comp_headers
Ejemplo n.º 10
0
    def performTest(self, ontology_type, genes_of_interest, cutoff):

        if cutoff is None:
            cutoff = p_val_threshold

        total_num_genes = self._mapper.get_gene_count(ontology_type)
        term2genes = self._mapper.get_term_mapping(ontology_type)
        gene2terms = self._mapper.get_gene_mapping(ontology_type)

        filtered = self.__filter_input(genes_of_interest, gene2terms)
        
        #print(filtered)
        #print(len(filtered))

        sample_map = self.__calculateTermFrequency(filtered, gene2terms)

        n = len(filtered)

        # Number of tests performed: will be used for correction.
        num_tests = len(sample_map)

        results = [ {} ] * num_tests

        pvals = np.zeros(num_tests)

        idx = 0
        for term in sample_map:
            # Calculate p-value
            sampled = sample_map[term]
            assigned_genes = term2genes[term]
            k = len(sampled)
            m = len(assigned_genes)

            p = stats.hypergeom.pmf(k,total_num_genes,m,n)
            p_corrected = p * num_tests
            pvals[idx] = p
            
            result = {}
            result['id'] = term
            result['p-value'] = p_corrected
            result['background'] = m
            result['genes'] = list(sampled)
            results[idx] = result
            idx = idx + 1
            #print(term + " = " + str(p) + ", k = " + str(k) + ", m = " + str(m) + ", total = " + str(total_num_genes) + ", n= " + str(n))
        
        # Correct border values (library does not accept 0 & 1)
        i = 0
        for p in pvals:
            if p >= 1.0 or math.isnan(p):
                pvals[i] = 0.9999999999999999999999
            elif p <= 0:
                pvals[i] = 0.0000000000000000000001
            i += 1
        qvals = qvalue.estimate(pvals)
        filtered_results = []

        idx = 0

        for term in sample_map:
            qv = qvals[idx]
            res = results[idx]

            pv = res['p-value']
            k = len(res['genes'])
            res['q-value'] = qv
            if pv < cutoff and k >= gene_threshold:
                filtered_results.append(res)
       
            idx = idx + 1

        return {'results':filtered_results, 'total_genes':total_num_genes}
Ejemplo n.º 11
0
    print('postprocessed data size', testddirect.shape)
    t = time.time()
    r = RunBootstrapPercentile(testddirect,
                               NumPerm=100,
                               gN=G,
                               useTF=False,
                               n_cores=1,
                               fSavefile=False)
    print('Completed in %g seconds' % (time.time() - t))

    ####################### Hypothesis test ############################################
    import qvalue
    alpha = 1e-6  # significance level for hypothesis test. Can vary this here.
    dj = np.diag_indices(G, 2)  # index to diagonal of a G X G matrix
    # qvalue calculation should be similar to Bonferonni
    qvalues, pi0 = qvalue.estimate(r['pvalues'], verbose=True)
    adjMatrixBootstrapQ = np.zeros((G, G), dtype=bool)
    adjMatrixBootstrapQ[qvalues < alpha] = True
    adjMatrixBootstrapQ[dj] = False  # remove self-transitions

    adjMatrixTrue = np.zeros((G, G), dtype=bool)
    for i in range(G):
        for j in range(G):
            adjMatrixTrue[i, j] = (angularSpeed[i] == angularSpeed[j])
    adjMatrixTrue[
        dj] = False  # have to do it, because white noise diagonal not true

    trueDF = pd.DataFrame(adjMatrixTrue, columns=data.index, index=data.index)
    trueDF.to_csv('trueClusteringAdjMatrix.csv')
    edgeNetworkTrue = CreateEdgeNetwork(adjMatrixBootstrap=adjMatrixTrue,
                                        cost=np.ones((G, G)),
Ejemplo n.º 12
0
    def perform_test(self, ontology_type, genes_of_interest, cutoff):

        if cutoff is None:
            cutoff = p_val_threshold

        total_num_genes = self._mapper.get_gene_count(ontology_type)
        term2genes = self._mapper.get_term_mapping(ontology_type)
        gene2terms = self._mapper.get_gene_mapping(ontology_type)

        filtered = self.__filter_input(genes_of_interest, gene2terms)

        # print(filtered)
        # print(len(filtered))

        sample_map = self.__calculateTermFrequency(filtered, gene2terms)

        n = len(filtered)

        # Number of tests performed: will be used for correction.
        num_tests = len(sample_map)

        results = [{}] * num_tests

        pvals = np.zeros(num_tests)

        idx = 0
        for term in sample_map:
            # Calculate p-value
            sampled = sample_map[term]
            assigned_genes = term2genes[term]
            k = len(sampled)
            m = len(assigned_genes)

            p = stats.hypergeom.pmf(k, total_num_genes, m, n)
            p_corrected = p * num_tests
            pvals[idx] = p

            result = {}
            result['id'] = term
            result['p-value'] = p_corrected
            result['background'] = m
            result['genes'] = list(sampled)
            results[idx] = result
            idx = idx + 1
            # print(term + " = " + str(p) + ", k = " + str(k) + ",
            # m = " + str(m) + ", total = " + str(total_num_genes) + ", n= " + str(n))

        # Correct border values (library does not accept 0 & 1)
        i = 0
        for p in pvals:
            if p >= 1.0 or math.isnan(p):
                pvals[i] = 0.9999999999999999999999
            elif p <= 0:
                pvals[i] = 0.0000000000000000000001
            i += 1
        qvals = qvalue.estimate(pvals)
        filtered_results = []

        idx = 0

        for term in sample_map:
            qv = qvals[idx]
            res = results[idx]

            pv = res['p-value']
            k = len(res['genes'])
            res['q-value'] = qv
            if pv < cutoff and k >= gene_threshold:
                filtered_results.append(res)

            idx = idx + 1

        return {'results': filtered_results, 'total_genes': total_num_genes}
Ejemplo n.º 13
0
    final_matrix = pd.DataFrame()
    pathway_list = np.unique(asd["pathway"])

    for i in pathway_list[0:1]:
        pathway_list_each = list(asd.gene[asd.pathway == i])

        expression_overlaid_pathway = pathway_overlaid(expression_data_raw,
                                                       pathway_list_each)

        expression_overlaid_pathway.columns = [
            "gene"
        ] + ["case"] * 5 + ["control"] * 10

        pathway_matrix = activity_score_calculate(expression_overlaid_pathway,
                                                  "case", "control")
        pathway_matrix_T = pathway_matrix.T
        pathway_matrix_T["pathway"] = i

        final_matrix = final_matrix.append(pathway_matrix_T)

    print(final_matrix)
    print(final_matrix.loc["p_value"])
    final_matrix["q-value"] = qvalue.estimate(final_matrix["p_value"])

    print(final_matrix)

    final_matrix.to_csv("./output data/0926_final_matrix_" +
                        ex_list.split(".")[0] + ".csv",
                        index=False)