def main():
    motif_filename = "Gamma_collection.txt"
    genome_filename = "NC_000913.fna" # Escherichia coli str. K-12 substr. MG1655
    top_sites = ["TACTGTATAAATAAACAGTA", "TACTGTATATAAAAACAGTA", "TACTGTATATAAAAACAGTA", "AACTGTATATAAATACAGTT", "TACTGTATATAAAACCAGTT", "TACTGGATAAAAAAACAGTT", "TACTGTATAAAATCACAGTT", "AACTGGATAATCATACAGTA", "CACTGTATAAATAAACAGCT", "TACTGTACATCCATACAGTA", "ACCTGTATAAATAACCAGTA", "TACTGTATGAGCATACAGTA", "TACTGGATATTTAAACAGGT", "AACTGTATATACACCCAGGG", "TGCTGTATATACTCACAGCA", "AACTGGATAAAATTACAGGG", "TACTGTATATTCATTCAGGT", "TACTGTACACAATAACAGTA", "AGCTGAATAAATATACAGCA", "ATCTGTATATATACCCAGCT", "CACTGGATAGATAACCAGCA", "TACTGCATATACAACCAGAA", "CACTGTATACTTTACCAGTG", "AACTGTCGATACGTACAGTA", "TGCTGTACAAACGTCCAGTT", "CGCTGGATATCTATCCAGCA", "TGCTGTTTATTAAACCAGAA", "CATTGTTTATAAAAACAGCA", "TACTGGAGACAAATACAGCT", "ACCTGTATATATCATCAGTA", "TACTGTATAAACAGCCAATA", "TACTGTTTATCTTCCCAGCG", "GACTGTATAAAACCACAGCC", "TATTGTATATATTCACATTA", "AAGTGTATTTACACACAGCG", "AACTGGATAATCATACCGTT", "TGCTGTATGGATTAACAGGA", "TCCTGTATGAAAAACCATTA", "GCCTGTCTGAACAAACAGTA", "AAGTGTATATATATCCATCG", "AATTGTTTAAAAAACCAGAA", "TATTGTATTTATAAACATTA", "AACTGATTAAAAACCCAGCG", "GAGTGTATATAAAGCCAGAA", "TTCTGGATAAGCATCCAGAA", "ACCTGAATATTCAAACAGCG", "TACTGTCTACCAAAACAGAG", "TCCTGTATAAATTAACCGTT", "TAATGAATATAAAACCAGGA", "AACTGTAAATAATTACATGA", "TACTGAATAAAAAAGCAGAA", "TGCTGTACAATCAGCCAGCA", "TGCTGGATTTACGACCAGAA", "TACTGTTTATAAACCGAGCG", "TCTTGTATATCCAACCAGTT", "CACTGTATTAAAAACCATTC", "AACTGTATTACCTTCCAGCC", "TACTGTTTCCATTTACAGCC", "TAGTGGATGTAAAAACATTT", "CACTGTCTATACTTACATGT", "TACTGTTTGTGCAAATAGTA", "AACTGGTTATCAACCCAGAC", "CTCTGTATCTAATTACAGGT", "CACTGAATGCTAAAACAGCA", "TGCTGGATGTGAAACCAGCG", "TGCTGATTAAAAAACCAGCG", "AACTGGATATCTATCCGGAA", "CACTGTCTGATACAACAGTT", "AATTGTTAATATATCCAGAA", "AACTTTATGTACAGCCAGTG", "AACTGGTTATTCCACCAGAA", "GAATGGATAAAAAAACAGCC", "AGCTGTATAAAAATCCTGAG", "TCCTGGCTATTTTGCCAGTA", "AGCTGGCTATCTGAACAGTT", "CACTGGATATTCCTTCAGGT", "GGCTGGATAAAGAACCAGAA", "AACTGAATAAAAACAAAGGA", "TGCTGTACATCAGCACAGAT", "CAGTGGATTAACTTCCAGTT", "TACTGGATACAAAAACGGAT", "AACTATATAAATAAACATAA", "TGCTGCATGAAGAAACAGTA", "GTCTGAATGAATACCCAGTA", "TACTTTATTTACTCCCAGTG", "TACTGGAAACTAACACAGGC", "TACTGGATATCAAACCTGAA", "TACTGTCGGAAAAATCAGTG", "AACTGTATATGTCGCCAGGC", "TACTCTTCATTAAAACAGTG", "GACTGGCTTAATACACAGCC", "CAGTGTAAGAATAGACAGTG", "CGCTGGATGAGCGTACAGCA", "AACGGGATAATAAAACAGCC", "TACCGTTTGTATTTCCAGCT", "CTCTGGACATTAAACCAGGA", "TAGTTTATATAAATTCAGTC", "TACTGTATATTCCTCAAGCG", "AACTGGATATATAAATAATG", "AACGGTTTAAACACCCAGCG", "GCCTGGTTAAATGACCAGCA", "TACTGTTTAGCCAGCCAGTC", "TGCTGAACATTCTTCCAGCA", "TACTGGAAATAAGATCAGCC", "AACTGGATCTTAAAAAAGTA", "GGCTGGACAATTTTACAGCT", "AACTGTCTCTTATGACAGTT", "AACTGGATACAGAAACAATA", "TAGTGTTCAGTTTTACAGTA", "TAGTGTTTGTTCATCCATTA", "CAGTGTTGAAAAGTCCAGTA", "ATCTGGCTGAAATTACAGAA", "GATTGAATGAATATACAGGG", "CACTGGTTTTCCACCCAGCA", "AACTGGTTGAAAAACCATCA", "AACTGGTTTAACTCCCAGGG", "AGCTGGATAAACAAAAAGCG", "AACTGGATAAATTACCGGAT", "AACTGGATTTAAATCCTGGT", "AACTGTATTTACAATCATCT", "TACTCTCTGTTCATCCAGCA", "TACTCTATGCAATAACAGAA", "TGCTGAATGCATTAACAGCG", "CACTGGTTATCTTTACCGTA", "TGGTGGATATTTTTTCAGGA", "AACTGGATATCAATCCGGAT", "AAGTGTAAAATTCTACAGAA", "AACCGGACATAAAGCCAGTA", "CACTGTATAAAAATCCTATA", "AACTCTATATTACCCCAGTT", "ACCAGAATAAACATCCAGTA", "TACTGGATGCATTACCGGTA", "CACTCTTTATAAAACCAGGC", "CAGTGAAAATAAAAACAGGA", "AACGGTTTATCTAGCCAGTA", "TCATGTATGATCATACAGAC", "TGCTGAATATATAAAAAGAG", "ACCTGGATGTCACCACAGTT", "TACTAAATGAAAAAACAGCG", "AACTGGATGAACAACCGGCG", "TTCTTTATACATATCCAGCG", "AAGTGTTTGCTAACACAGCA", "AAGTGTAAAAAATCCCAGCG", "AACTGGATAAAGACACCGCT", "TGCTGTATGGTAAATCAGAA", "AACTGTATGATTTAAAAGAT", "TACGGTATAAAAAGACCGTA", "TGCTGGATATTATCCCATCA", "TACTGTCTGAAGAAGCAGTG", "AACTGAATAAATACCCCGGT", "TGCTGTATGAGTAACCGGTA", "CACTGGAAAAATGCGCAGTA", "AACTGGATAGCTATGCAGAA", "AATTGTAAAAAACAACAGCA", "AACTGTTTATCAACACCGCT", "ACCTGTACCTTAAACCAGGA", "TACTTTATAGTTTCCCAGTT", "ATCTGCATAAAGAACCAGTA", "CCCTCTTTATATTTCCAGTG", "TACTGTTTATTAATGTAGCA", "ATGTGAATGAATATCCAGTT", "AACTGGTTAAAATTAGAGAT", "TACTGTAAGAAAAACCCGCA", "ACCTGGAGAAAGAAACAGCG", "CACTGTTTACCCTGACAGTC", "AACTGGCTCATAACCCAGAA"]
    top_seqs = [nt2int(seq) for seq in top_sites]
    top_seqs = top_seqs[0:3] # Truncate to first three for the sake of less output spam
    
    # Load genome (validated)
    genome, __ = load_scaffolds(genome_filename)
    _genome, __ = metagenomics.load_scaffolds(genome_filename)
    print "Genomes loaded are the same:", np.array_equal(genome, _genome)
    print

    # Ensure the sites are in the genome (validated)
    print "Finding the sites in the genome:"
    positions = []
    sites = []
    strands = []
    for seq in top_seqs:
        matches = np.where(np.all(sliding_window(genome, seq.size) == seq, axis=1))
        matches_r = np.where(np.all(sliding_window(genome, seq.size) == wc(seq), axis=1))
        
        # forward strand
        for match in matches:
            if match.size > 0 and match[0] not in positions:
                pos = match[0]
                seq = genome[pos:pos + 20]
                print "Found:", int2nt(seq), "->", match[0] 
                positions.append(pos)
                sites.append(seq)
                strands.append(0) # forward
        
        # reverse strand
        for match in matches_r:
            if match.size > 0 and match[0] not in positions:
                pos = match[0]
                seq = wc(genome[match[0]:match[0] + 20])
                print "Found:", int2nt(seq), "<-", match[0]
                positions.append(pos)
                sites.append(seq)
                strands.append(1) # reverse
    print
    
    # Load PSSM (validated)
    genome_frequencies = np.bincount(genome).astype(np.float) / genome.size
    _pssm = create_pssm(motif_filename, genome_frequencies=genome_frequencies)
    _pssm2 = gpu_pssm.create_pssm(motif_filename, genome_frequencies=genome_frequencies)
    print "PSSMs are the same:", np.array_equal(_pssm, _pssm2)
    print
    
    # Score sites (validated!!)
    print "Scoring sites (method score seq strand pos):"
    for n, site in enumerate(sites):
        print "True:", score_site(site, _pssm), int2nt(site), strands[n], positions[n]
        print "CPU: ", cpu_score_site(site, _pssm), int2nt(site), strands[n], positions[n]
        print "GPU: ", gpu_score_site(site, _pssm), int2nt(site), strands[n], positions[n]
        print
def main():
    
    ##### Parameters #####
    #Want to run with promoter regions
    promoter = True
    
    # Path to the MetaHit database.
    # The patient files can be downloaded from:
    #   http://www.bork.embl.de/~arumugam/Qin_et_al_2010/
    # Make sure these are extracted from their packages! (i.e.: "gunzip *.gz")
    # Note: After extraction, the 84 patient files occupy a total of 6.56 GB on
    # disk!
    # The original paper can be found at:
    #   http://www.nature.com/nature/journal/v464/n7285/full/nature08821.html
    if promoter:
        metahit_path = "./MetaHit/Pruned"
    else:
        metahit_path = "./MetaHit/Data"
    
    # The collection of binding sites to generate the PSSM from.
    # LexA.seq.fa is a collection of 115 experimentally-determined binding
    # sites reported in literature. See Table S1 in Cornish et al. (2013).
    # binding_sites_path = "./LexA.seq.fa"
    #binding_sites_path= "./LexA_Gamma_collection.fas"
    binding_sites_path = "./LexA_Grampos_collection.fas"
    
    # Number of permutations to run. Note that the COLUMNS of the PSSM
    permutations = 50
    
    # Scores below this number of bits will not be reported.
    # Lower values will give more (false-positive) results and also slow down
    # the execution of the program since more memory needs to be allocated to
    # store the score values.
    score_threshold = -50.0
    
    ### Parameters below this line should *probably* not be changed. ###
    
    # The background frequency of the bases. An equiprobable frequency
    # distribution assumes that each base has an equal probability of occuring,
    # that is: P(A) = P(C) = P(G) = P(T) = 0.25 => GC-content = 0.5
    # If set to False, the background frequency will be calculated based on the
    # nucleotide composition of each patient.
    # For better comparison across patients, this should be set to True.
    equiprobable_nuc_freqs = True
    
    # The ranges that the scores should be binned into.
    # Since no scores will be saved below the score_threshold, it serves as a
    # lower bound to the bins. A good upper bound is around ~30 bits since the
    # maximum theoretical score a sequence can have from a PSSM is 32.
    # Under equiprobable frequencies, the LexA consensus sequence has a score
    # of ~24 bits, so no sequence should score higher than that.
    bins = range(int(score_threshold), 32, 1)
    
    # If you'd like to compare the output of this program by scoring the
    # E. coli genome, set this to True.
    # Make sure NC_000913.fna is in the parent directory.
    # The genome sequence can be download from:
    #  ftp://ftp.ncbi.nlm.nih.gov/genomes/Bacteria/Escherichia_coli_K_12_substr__MG1655_uid57779/
    # Use this for comparison/debugging.
    score_ecoli_instead = False			
    
    #Calculate the total number of sites, scaffolds scanned
    total_num_sites = 0
    total_size = 0
    total_scaffold = 0
    ##########

    # Find patient files on disk
    if promoter:
        metahit_db = glob.glob(metahit_path + "/Pruned_MH[0-9]*.seq.fa")
    else:
        metahit_db = glob.glob(metahit_path + "/MH[0-9]*.seq.fa")
       #  metahit_db = ["Eco_300_1_50_P.txt"]
         
    if score_ecoli_instead:
        metahit_db = ["../NC_000913.fna"] # E. coli genome (for debugging)
    
    # gather total time
    start = time.time()
				
    # For debugging, truncate to just first patient
    # An alert just incase I am clumsy and forget these lines are uncommented.
    #print "USING ONLY ONE PATIENT!!"
    #metahit_db = ["Eco_300_1_50_P.txt"]
    #metahit_db = [metahit_db[0]]
    # Assume equiprobable mononucleotide frequencies
    mg_frequencies = [0.25] * 4
        
    if not equiprobable_nuc_freqs:
        # Calculate the background nucleotide frequency for the metagenome
        mg_frequencies = np.bincount(metagenome).astype(np.float) / metagenome.size
        
    # Calculate the original PSSM from binding sites
    original_pssm = gpu_pssm.create_pssm(binding_sites_path, genome_frequencies = mg_frequencies)
        
    # Print the unpermuted PSSM        
    print "Unpermuted PSSM:"
    mg.print_pssm(original_pssm)

    #preallocate the array to speed up process
    permute_pssm = np.empty(shape=(permutations+1,len(original_pssm)), dtype=object)
    patient_scores = np.zeros(shape=(permutations+1,len(bins)-1))
	
    #calculate predetermined pssm
    for permutation in range(permutations):	
       
        # Permute the PSSM
        permute_pssm[permutation] = mg.permute_pssm(original_pssm)

    #Cycle through every patient file
    for patient_file in metahit_db:

        #Status Update
        print "File: ", patient_file

        # Load the sequence into memory
        metagenome, scaffolds = mg.load_scaffolds(patient_file)
        total_size += metagenome.size
        total_scaffold += scaffolds.size
        print "Genome size:", metagenome.size, "| Scaffolds:", scaffolds.size
           
        # Score the metagenome using the original PSSM
        print "Scoring without permuting..."
        original_scores,partial_num_sites = score_patient(metagenome, scaffolds, original_pssm, score_threshold, bins)

        # Keep the distributions of the scores
        patient_scores[0]+= original_scores
        total_num_sites += partial_num_sites

        #For each permutated pssm
        for permutation in range(permutations):

            #which permutation it is on
            print "Permutation %d/%d..." % (permutation + 1, permutations)            
            
            # Re-score using permuted PSSM
            perm_scores,partial_num_sites = score_patient(metagenome, scaffolds, permute_pssm[permutation], score_threshold, bins)            
            
            # Save the distribution of the scores
            patient_scores[permutation+1] += perm_scores

    # Plot results
    plt.figure()
    for score in patient_scores[1:]:
        cdf = np.cumsum(score)
        plt.plot(bins[1:], cdf, "D-r", alpha=0.5, label="Permutation")
    
    cdf = np.cumsum(patient_scores[0])
    plt.plot(bins[1:], cdf, "D-b", lw=3, label="Original")
    plt.xlabel("Site score (bits)")
    plt.ylabel("# of Sites Found")
    plt.title("Cumulative Density Function")
    handles, labels = plt.gca().get_legend_handles_labels()
    plt.legend(handles[-2:], labels[-2:], loc="best")
    plt.grid()
    
    plt.figure()
    
    for score in patient_scores[1:]:
        plt.plot(bins[1:], score, "D-r", alpha=0.5, label="Permutation")
    plt.plot(bins[1:], patient_scores[0], 'D-b', lw=3, label="Original")
    plt.xlabel("Site score (bits)")
    plt.ylabel("# of Sites Found ")
    plt.title("Probability Density Function")
    handles, labels = plt.gca().get_legend_handles_labels()
    plt.legend(handles[-2:], labels[-2:], loc="best")
    plt.grid()

    #Calculate p-values
    total_fake_patient_scores = patient_scores[1]
    for score in patient_scores[2:]:
	   total_fake_patient_scores += score
    total_fake_patient_scores = np.vectorize(lambda x: x if x > 0 else .000001)(np.float64(total_fake_patient_scores))

    print
    real_prob = np.float64(patient_scores[0])/total_num_sites
    fake_prob = total_fake_patient_scores/(total_num_sites * permutations)

    #print out the p-values
    print "Probability (True Matrix | Score):"
    matrix_prob = (real_prob/(fake_prob+real_prob)) * 100
    print "\n".join("%d:%.2f" % (s, p) for s, p in zip(bins,matrix_prob))

    #Plot the Probability Values
    plt.figure()
    plt.bar(bins[1:],matrix_prob)
    plt.xlabel("Site score (bits)")
    plt.ylabel("Confidence Level (%)")
    plt.grid()
    
    #final diagnostics
    end = time.time()
    print "Total time: %.2f seconds" % (end-start)	
    print "Total Metagenome Size: %d bp" % (total_size)
    print "Total Scaffolds Scanned: %d" % (total_scaffold)