Esempio n. 1
0
def make_mean_hist(hists, debug=False):
    """ return the hist with bin contents the mean over <hists> of each bin """
    binvals = {}
    for hist in hists:
        if debug:
            print '    sub',
        for ib in range(0, hist.n_bins + 2):
            low_edge = hist.low_edges[ib]
            if low_edge not in binvals:
                binvals[low_edge] = 0.
            binvals[low_edge] += hist.bin_contents[ib]
            if debug:
                print '   ', low_edge, hist.bin_contents[ib],
        if debug:
            print ''
    binlist = sorted(binvals.keys())
    meanhist = Hist(len(binlist) - 2, binlist[1], binlist[-1], binlist[1 : -1])
    if debug:
        print '   mean',
    for ib in range(len(binlist)):
        meanhist.set_ibin(ib, binvals[binlist[ib]])
        if debug:
            print '   ', meanhist.low_edges[ib], meanhist.bin_contents[ib],
    if debug:
        print ''

    meanhist.normalize()
    return meanhist
Esempio n. 2
0
    def read_mute_freqs(self, mute_freq_dir):
        # NOTE these are mute freqs, not branch lengths, but it's ok for now
        for mtype in ['all',] + utils.regions:
            infname = mute_freq_dir + '/' + mtype + '-mean-mute-freqs.csv'
            self.branch_lengths[mtype] = {}
            self.branch_lengths[mtype]['lengths'], self.branch_lengths[mtype]['probs'] = [], []
            mutehist = Hist(fname=infname)
            self.branch_lengths[mtype]['mean'] = mutehist.get_mean()

            # if mutehist.GetBinContent(0) > 0.0 or mutehist.GetBinContent(mutehist.GetNbinsX()+1) > 0.0:
            #     print 'WARNING nonzero under/overflow bins read from %s' % infname
            mutehist.normalize(include_overflows=False, overflow_eps_to_ignore=1e-2)  # if it was written with overflows included, it'll need to be renormalized
            check_sum = 0.0
            for ibin in range(1, mutehist.n_bins + 1):  # ignore under/overflow bins
                freq = mutehist.get_bin_centers()[ibin]
                branch_length = self.convert_observed_changes_to_branch_length(float(freq))
                prob = mutehist.bin_contents[ibin]
                self.branch_lengths[mtype]['lengths'].append(branch_length)
                self.branch_lengths[mtype]['probs'].append(prob)
                check_sum += self.branch_lengths[mtype]['probs'][-1]
            if not utils.is_normed(check_sum):
                raise Exception('not normalized %f' % check_sum)

        if self.args.debug:
            print '  mean branch lengths'
            for mtype in ['all',] + utils.regions:
                print '     %4s %7.3f (ratio %7.3f)' % (mtype, self.branch_lengths[mtype]['mean'], self.branch_lengths[mtype]['mean'] / self.branch_lengths['all']['mean'])
Esempio n. 3
0
    def __init__(self, base_indir, outdir, gene_name, naivety, glfo, args):
        self.region = utils.get_region(gene_name)
        self.raw_name = gene_name  # i.e. unsanitized
        self.germline_seqs = glfo['seqs']  # all germline alleles
        self.germline_seq = self.germline_seqs[self.region][gene_name]  # germline sequence for this hmm
        self.indir = base_indir
        self.args = args
        self.cyst_positions = glfo['cyst-positions']
        self.tryp_positions = glfo['tryp-positions']

        # parameters with values that I more or less made up
        self.precision = '16'  # number of digits after the decimal for probabilities
        self.eps = 1e-6  # NOTE I also have an eps defined in utils, and they should in principle be combined
        self.n_max_to_interpolate = 20
        self.min_mean_unphysical_insertion_length = {'fv' : 1.5, 'jf' : 25}  # jf has to be quite a bit bigger, since besides account for the variation in J length from the tryp position to the end, it has to account for the difference in cdr3 lengths

        self.erosion_pseudocount_length = 10  # if we're closer to the end of the gene than this, make sure erosion probability isn't zero

        # self.insert_mute_prob = 0.0
        # self.mean_mute_freq = 0.0

        self.outdir = outdir
        self.naivety = naivety
        self.smallest_entry_index = -1  # keeps track of the first state that has a chance of being entered from init -- we want to start writing (with add_internal_state) from there

        # self.insertions = [ insert for insert in utils.index_keys if re.match(self.region + '._insertion', insert) or re.match('.' + self.region + '_insertion', insert)]  OOPS that's not what I want to do
        self.insertions = []
        if self.region == 'v':
            self.insertions.append('fv')
        elif self.region == 'd':
            self.insertions.append('vd')
        elif self.region == 'j':
            self.insertions.append('dj')
            self.insertions.append('jf')

        self.erosion_probs = {}
        self.insertion_probs = {}
        self.insertion_content_probs = {}

        self.n_occurences = utils.read_overall_gene_probs(self.indir, only_gene=gene_name, normalize=False)  # how many times did we observe this gene in data?
        replacement_genes = None
        if self.n_occurences < self.args.min_observations_to_write:  # if we didn't see it enough, average over all the genes that find_replacement_genes() gives us
            if self.args.debug:
                print '    only saw it %d times, use info from other genes' % self.n_occurences
            replacement_genes = utils.find_replacement_genes(self.indir, self.args.min_observations_to_write, gene_name, single_gene=False, debug=self.args.debug)

        self.read_erosion_info(gene_name, replacement_genes)  # try this exact gene, but...

        self.read_insertion_info(gene_name, replacement_genes)

        if self.naivety == 'M':  # mutate if not naive
            self.mute_freqs, self.mute_obs = paramutils.read_mute_info(self.indir, this_gene=gene_name, approved_genes=replacement_genes)

        self.track = Track('nukes', utils.nukes)
        self.saniname = utils.sanitize_name(gene_name)
        self.hmm = HMM(self.saniname, self.track.getdict())  # pass the track as a dict rather than a Track object to keep the yaml file a bit more readable
        self.hmm.extras['gene_prob'] = max(self.eps, utils.read_overall_gene_probs(self.indir, only_gene=gene_name))  # if we really didn't see this gene at all, take pity on it and kick it an eps
        mean_freq_hist = Hist(fname=self.indir + '/all-mean-mute-freqs.csv')
        self.hmm.extras['overall_mute_freq'] = mean_freq_hist.get_mean()
Esempio n. 4
0
def get_cluster_size_hist(partition, rebin=None):
    sizes = [len(c) for c in partition]
    nbins = max(sizes)
    # if nbins > 30:
    #     rebin = 2
    if rebin is not None:
        nbins = int(float(nbins) / rebin)
    hist = Hist(nbins, 0.5, max(sizes) + 0.5)
    for sz in sizes:
        hist.fill(sz)
    return hist
Esempio n. 5
0
    def __init__(self, base_indir, outdir, gene_name, glfo, args, debug=False):
        self.region = utils.get_region(gene_name)
        self.raw_name = gene_name  # i.e. unsanitized
        self.germline_seqs = glfo['seqs']  # all germline alleles
        self.germline_seq = self.germline_seqs[self.region][gene_name]  # germline sequence for this hmm
        self.indir = base_indir
        self.args = args
        self.debug = debug
        self.codon_positions = {r : glfo[c + '-positions'] for r, c in utils.conserved_codons[args.chain].items()}

        # parameters with values that I more or less made up
        self.precision = '16'  # number of digits after the decimal for probabilities
        self.eps = 1e-6  # NOTE I also have an eps defined in utils, and they should in principle be combined
        self.n_max_to_interpolate = args.min_observations_to_write
        self.min_mean_unphysical_insertion_length = {'fv' : 1.5, 'jf' : 25}  # jf has to be quite a bit bigger, since besides account for the variation in J length from the tryp position to the end, it has to account for the difference in cdr3 lengths

        self.erosion_pseudocount_length = 10  # if we're closer to the end of the gene than this, make sure erosion probability isn't zero

        self.outdir = outdir
        self.smallest_entry_index = -1  # keeps track of the first state that has a chance of being entered from init -- we want to start writing (with add_internal_state) from there

        self.insertions = []
        if self.region == 'v':
            self.insertions.append('fv')
        elif self.region == 'd':
            self.insertions.append('vd')
        elif self.region == 'j':
            self.insertions.append('dj')
            self.insertions.append('jf')

        assert len(utils.ambiguous_bases) == 1 and utils.ambiguous_bases[0] == 'N'  # maybe need to update some stuff below if this changes

        if self.debug:
            print '%s' % utils.color_gene(gene_name)

        self.n_occurences = utils.read_single_gene_count(self.indir, gene_name, debug=self.debug)  # how many times did we observe this gene in data?
        replacement_genes = None
        if self.n_occurences < self.args.min_observations_to_write:  # if we didn't see it enough, average over all the genes that find_replacement_genes() gives us
            if self.debug:
                print '      only saw it %d times (wanted %d), so use info from all other genes' % (self.n_occurences, self.args.min_observations_to_write)
            replacement_genes = utils.find_replacement_genes(self.indir, self.args.min_observations_to_write, gene_name, debug=self.debug)

        self.erosion_probs = self.read_erosion_info(gene_name, replacement_genes)
        self.insertion_probs, self.insertion_content_probs = self.read_insertion_info(gene_name, replacement_genes)
        self.mute_freqs, self.mute_obs = paramutils.read_mute_info(self.indir, this_gene=gene_name, chain=self.args.chain, approved_genes=replacement_genes)  # actual info in <self.mute_obs> isn't actually used a.t.m.

        self.track = Track('nukes', utils.nukes)
        self.saniname = utils.sanitize_name(gene_name)
        self.hmm = HMM(self.saniname, self.track.getdict())  # pass the track as a dict rather than a Track object to keep the yaml file a bit more readable
        self.hmm.extras['gene_prob'] = max(self.eps, utils.read_overall_gene_probs(self.indir, only_gene=gene_name))  # if we really didn't see this gene at all, take pity on it and kick it an eps
        tmp_mean_freq_hist = Hist(fname=self.indir + '/all-mean-mute-freqs.csv')
        self.hmm.extras['overall_mute_freq'] = tmp_mean_freq_hist.get_mean()
Esempio n. 6
0
def make_hist_from_dict_of_counts(values, var_type, hist_label, log='', xmin_force=0.0, xmax_force=0.0, normalize=False, sort=False):
    """ Fill a histogram with values from a dictionary (each key will correspond to one bin) """
    assert var_type == 'int' or var_type == 'string'  # floats should be handled by Hist class in hist.py

    if len(values) == 0:
        print 'WARNING no values for %s in make_hist' % hist_label
        return TH1D(hist_label, '', 1, 0, 1)

    bin_labels = sorted(values)
    if not sort and var_type == 'string':  # for strings, sort so most common value is to left side
        bin_labels = sorted(values, key=values.get, reverse=True)

    if var_type == 'string':
        n_bins = len(values)
    else:
        n_bins = bin_labels[-1] - bin_labels[0] + 1

    hist = None
    xbins = [0. for _ in range(n_bins+1)]  # NOTE the +1 is 'cause you need the lower edge of the overflow bin
    if xmin_force == xmax_force:  # if boundaries aren't set explicitly, work out what they should be
        if var_type == 'string':
            set_bins(bin_labels, n_bins, 'x' in log, xbins, var_type)
            hist = Hist(n_bins, xbins[0], xbins[-1], xbins=xbins)
        else:
            hist = Hist(n_bins, bin_labels[0] - 0.5, bin_labels[-1] + 0.5)  # for integers, just go from the first to the last bin label (they're sorted)
    else:
      hist = Hist(n_bins, xmin_force, xmax_force)

    for ival in range(len(values)):
        if var_type == 'string':
            label = bin_labels[ival]
            ibin = ival + 1
        else:
            label = ''
            ibin = hist.find_bin(bin_labels[ival])
        hist.set_ibin(ibin, values[bin_labels[ival]], error=math.sqrt(values[bin_labels[ival]]), label=label)
  
    # make sure there's no overflows
    if hist.bin_contents[0] != 0.0 or hist.bin_contents[-1] != 0.0:
        for ibin in range(hist.n_bins + 2):
            print '%d %f %f' % (ibin, hist.low_edges[ibin], hist.bin_contents[ibin])
        raise Exception('overflows in ' + hist_label)

    if normalize:
        hist.normalize()
        hist.ytitle = 'freq'
    else:
        hist.ytitle = 'counts'
    
    roothist = make_hist_from_my_hist_class(hist, hist_label)
    return roothist
Esempio n. 7
0
    def plot(self, plotdir, only_csv=False, only_overall=False):
        if not self.finalized:
            self.finalize()

        overall_plotdir = plotdir + '/overall'

        for gene in self.freqs:
            if only_overall:
                continue
            freqs = self.freqs[gene]
            if len(freqs) == 0:
                if gene not in glutils.dummy_d_genes.values():
                    print '    %s no mutefreqer obs for %s' % (utils.color('red', 'warning'), utils.color_gene(gene))
                continue
            sorted_positions = sorted(freqs.keys())
            genehist = Hist(sorted_positions[-1] - sorted_positions[0] + 1, sorted_positions[0] - 0.5, sorted_positions[-1] + 0.5, xtitle='position', ytitle='mut freq', title=gene)
            for position in sorted_positions:
                hi_diff = abs(freqs[position]['freq'] - freqs[position]['freq_hi_err'])
                lo_diff = abs(freqs[position]['freq'] - freqs[position]['freq_lo_err'])
                err = 0.5*(hi_diff + lo_diff)
                genehist.set_ibin(genehist.find_bin(position), freqs[position]['freq'], error=err)
            xline = None
            figsize = [7, 4]
            if utils.get_region(gene) in utils.conserved_codons[self.glfo['chain']]:
                codon = utils.conserved_codons[self.glfo['chain']][utils.get_region(gene)]
                xline = self.glfo[codon + '-positions'][gene]
            if utils.get_region(gene) == 'v':
                figsize[0] *= 3.5
            elif utils.get_region(gene) == 'j':
                figsize[0] *= 2
            plotting.draw_no_root(self.per_gene_mean_rates[gene], plotdir=plotdir + '/per-gene/' + utils.get_region(gene), plotname=utils.sanitize_name(gene), errors=True, write_csv=True, only_csv=only_csv, shift_overflows=True)
            # per-position plots:
            plotting.draw_no_root(genehist, plotdir=plotdir + '/per-gene-per-position/' + utils.get_region(gene), plotname=utils.sanitize_name(gene), errors=True, write_csv=True, xline=xline, figsize=figsize, only_csv=only_csv, shift_overflows=True)
            # # per-position, per-base plots:
            # paramutils.make_mutefreq_plot(plotdir + '/' + utils.get_region(gene) + '-per-base', utils.sanitize_name(gene), plotting_info)  # needs translation to mpl UPDATE fcn is fixed, but I can't be bothered uncommenting this at the moment

        # make mean mute freq hists
        for rstr in ['all', 'cdr3'] + utils.regions:
            if rstr == 'all':
                bounds = (0.0, 0.4)
            else:
                bounds = (0.0, 0.6 if rstr == 'd' else 0.4)
            plotting.draw_no_root(self.mean_rates[rstr], plotname=rstr+'_mean-freq', plotdir=overall_plotdir, stats='mean', bounds=bounds, write_csv=True, only_csv=only_csv, shift_overflows=True)
            plotting.draw_no_root(self.mean_n_muted[rstr], plotname=rstr+'_mean-n-muted', plotdir=overall_plotdir, stats='mean', write_csv=True, only_csv=only_csv, shift_overflows=True)

        if not only_csv:  # write html file and fix permissiions
            for substr in self.subplotdirs:
                plotting.make_html(plotdir + '/' + substr)
Esempio n. 8
0
def peruse_naive_seqs():
    from hist import Hist
    # hall = Hist(n_set_list[-1], n_set_list[0] - 0.5, n_set_list[-1] + 0.5)
    means = []
    for n_set in n_set_list:
        plotdir = baseplotdir + '/' + str(n_set)
        hist = Hist(fname=plotdir + '/hmm/hamming_to_true_naive.csv')
        print '%2d   %.2f' % (n_set, hist.get_mean()),
        # hall.set_ibin(hall.find_bin(n_set), hist.get_mean())
        means.append(hist.get_mean())
    
    import plotting
    fig, ax = plotting.mpl_init()
    # hall.mpl_plot(ax)
    ax.plot(n_set_list, means, marker='.')
    plotting.mpl_finish(ax, baseplotdir, 'means', xlabel='N simultaneous seqs', ylabel='mean hamming to true naive', ybounds=(0, None))
Esempio n. 9
0
def make_mean_hist(hists):
    """ return the hist with bin contents the mean over <hists> of each bin """
    binvals = {}
    for hist in hists:  # I could probably do this with list comprehensions or something, but this way handles different bin bounds
        for ib in range(0, hist.n_bins + 2):
            low_edge = hist.low_edges[ib]
            if low_edge not in binvals:
                binvals[low_edge] = []
            binvals[low_edge].append(hist.bin_contents[ib])
    binlist = sorted(binvals.keys())
    meanhist = Hist(len(binlist) - 2, binlist[1], binlist[-1], xbins=binlist[1 :])
    for ib in range(len(binlist)):
        vlist = binvals[binlist[ib]]
        meanhist.set_ibin(ib, numpy.mean(vlist), error=(numpy.std(vlist, ddof=1) / math.sqrt(len(vlist))))
    # meanhist.normalize()
    return meanhist
Esempio n. 10
0
    def plot(self, base_plotdir, cyst_positions=None, tryp_positions=None, only_csv=False):
        if not self.finalized:
            self.finalize()

        plotdir = base_plotdir + '/mute-freqs'
        overall_plotdir = plotdir + '/overall'
        utils.prep_dir(overall_plotdir, multilings=('*.csv', '*.svg'))
        for region in utils.regions:
            utils.prep_dir(plotdir + '/' + region, multilings=('*.csv', '*.svg'))
            # utils.prep_dir(plotdir + '/' + region + '-per-base/plots', multilings=('*.csv', '*.png'))
        if self.tigger:
            utils.prep_dir(plotdir + '/tigger', multilings=('*.csv', '*.svg'))

        for gene in self.freqs:
            freqs = self.freqs[gene]
            sorted_positions = sorted(freqs.keys())
            genehist = Hist(sorted_positions[-1] - sorted_positions[0] + 1, sorted_positions[0] - 0.5, sorted_positions[-1] + 0.5, xtitle='fixme', ytitle='fixme')  #, title=utils.sanitize_name(gene))
            for position in sorted_positions:
                hi_diff = abs(freqs[position]['freq'] - freqs[position]['freq_hi_err'])
                lo_diff = abs(freqs[position]['freq'] - freqs[position]['freq_lo_err'])
                err = 0.5*(hi_diff + lo_diff)
                genehist.set_ibin(genehist.find_bin(position), freqs[position]['freq'], error=err)
            xline = None
            figsize = [3, 3]
            if utils.get_region(gene) == 'v' and cyst_positions is not None:
                xline = cyst_positions[gene]
                figsize[0] *= 3.5
            elif utils.get_region(gene) == 'j' and tryp_positions is not None:
                xline = tryp_positions[gene]
                figsize[0] *= 2
            plotting.draw_no_root(genehist, plotdir=plotdir + '/' + utils.get_region(gene), plotname=utils.sanitize_name(gene), errors=True, write_csv=True, xline=xline, figsize=figsize, only_csv=only_csv)
            # paramutils.make_mutefreq_plot(plotdir + '/' + utils.get_region(gene) + '-per-base', utils.sanitize_name(gene), plotting_info)  # needs translation to mpl

        # make mean mute freq hists
        plotting.draw_no_root(self.mean_rates['all'], plotname='all-mean-freq', plotdir=overall_plotdir, stats='mean', bounds=(0.0, 0.4), write_csv=True, only_csv=only_csv)
        for region in utils.regions:
            plotting.draw_no_root(self.mean_rates[region], plotname=region+'-mean-freq', plotdir=overall_plotdir, stats='mean', bounds=(0.0, 0.4), write_csv=True, only_csv=only_csv)

        if self.tigger:
            self.tigger_plot(only_csv)

        if not only_csv:  # write html file and fix permissiions
            plotting.make_html(overall_plotdir)
            for region in utils.regions:
                plotting.make_html(plotdir + '/' + region, n_columns=1)
Esempio n. 11
0
    def read_annotation_performance(self, version_stype, input_stype, debug=False):
        """ version_stype is the code version, while input_stype is the input data version, i.e. 'ref', 'new' is the reference code version (last commit) run on the then-new simulation and parameters"""
        ptest = "annotate-" + input_stype + "-simu"
        if args.quick and ptest not in self.quick_tests:
            return
        if debug:
            print "  version %s input %s annotation" % (version_stype, input_stype)

        def read_performance_file(fname, column, only_ibin=None):
            values = []
            with open(fname) as csvfile:
                reader = csv.DictReader(csvfile)
                ibin = 0
                for line in reader:
                    if only_ibin is not None and ibin != only_ibin:
                        ibin += 1
                        continue
                    values.append(float(line[column]))
                    ibin += 1
            if len(values) == 1:
                return values[0]
            else:
                return values

        perfdir = self.dirs[version_stype] + "/" + self.perfdirs[input_stype]
        for method in ["sw", "hmm"]:
            if debug:
                print "   ", method

            # fraction of genes correct
            for region in utils.regions:
                fraction_correct = read_performance_file(
                    perfdir + "/" + method + "/plots/" + region + "_gene.csv", "contents", only_ibin=1
                )
                if debug:
                    print "      %s %.3f" % (region, fraction_correct)
                self.perf_info[version_stype][
                    input_stype + "-" + method + "-" + region + "_gene_correct"
                ] = fraction_correct

            # hamming fraction
            hamming_hist = Hist(fname=perfdir + "/" + method + "/plots/hamming_to_true_naive.csv")
            if debug:
                print "      mean hamming %.2f" % hamming_hist.get_mean()
            self.perf_info[version_stype][input_stype + "-" + method + "-mean_hamming"] = hamming_hist.get_mean()
Esempio n. 12
0
def make_fraction_plot(hright, hwrong, plotdir, plotname, xlabel, ylabel, xbounds, only_csv=False, write_csv=False):
    if 'fraction_uncertainty' not in sys.modules:
        import fraction_uncertainty

    # NOTE should really merge this with draw_no_root()
    xvals = hright.get_bin_centers() #ignore_overflows=True)
    right = hright.bin_contents
    wrong = hwrong.bin_contents
    yvals = [float(r) / (r + w) if r + w > 0. else 0. for r, w in zip(right, wrong)]

    # remove values corresponding to bins with no entries
    while yvals.count(0.) > 0:
        iv = yvals.index(0.)
        xvals.pop(iv)
        right.pop(iv)
        wrong.pop(iv)
        yvals.pop(iv)

    tmphilos = [sys.modules['fraction_uncertainty'].err(r, r + w) for r, w in zip(right, wrong)]
    yerrs = [err[1] - err[0] for err in tmphilos]
    # print '%s' % region
    # for iv in range(len(xvals)):
    #     print '   %5.2f     %5.0f / %5.0f  =  %5.2f   +/-  %.3f' % (xvals[iv], right[iv], right[iv] + wrong[iv], yvals[iv], yerrs[iv])

    if write_csv:
        hist_for_csv = Hist(hright.n_bins, hright.xmin, hright.xmax)
        bincenters = hright.get_bin_centers()
        for ibin in range(hright.n_bins):
            bcenter = bincenters[ibin]
            if bcenter in xvals:  # if we didn't remove it
                iy = xvals.index(bcenter)
                hist_for_csv.set_ibin(ibin, yvals[iy], error=yerrs[iy])

        hist_for_csv.write(plotdir + '/' + plotname + '.csv')

    if not only_csv:
        fig, ax = mpl_init()
        ax.errorbar(xvals, yvals, yerr=yerrs, markersize=10, linewidth=1, marker='.')
        if xlabel == 'support':
            ax.plot((0, 1), (0, 1), color='black', linestyle='--', linewidth=3)  # line with slope 1 and intercept 0
        mpl_finish(ax, plotdir, plotname, xlabel=xlabel, ylabel=ylabel, title=plotconfig.plot_titles.get(plotname, plotname), xbounds=xbounds, ybounds=(-0.1, 1.1))

    plt.close()
Esempio n. 13
0
    def plot(self, base_plotdir, cyst_positions=None, tryp_positions=None, only_csv=False):
        if not self.finalized:
            self.finalize()

        plotdir = base_plotdir + '/mute-freqs'
        utils.prep_dir(plotdir + '/plots', multilings=('*.csv', '*.svg'))
        for region in utils.regions:
            utils.prep_dir(plotdir + '/' + region + '/plots', multilings=('*.csv', '*.svg'))
            # utils.prep_dir(plotdir + '/' + region + '-per-base/plots', multilings=('*.csv', '*.png'))

        for gene in self.counts:
            counts, plotting_info = self.counts[gene], self.plotting_info[gene]
            sorted_positions = sorted(counts)
            genehist = Hist(sorted_positions[-1] - sorted_positions[0] + 1, sorted_positions[0] - 0.5, sorted_positions[-1] + 0.5, xtitle='fixme', ytitle='fixme')  #, title=utils.sanitize_name(gene))
            for position in sorted_positions:
                hi_diff = abs(counts[position]['freq'] - counts[position]['freq_hi_err'])
                lo_diff = abs(counts[position]['freq'] - counts[position]['freq_lo_err'])
                err = 0.5*(hi_diff + lo_diff)
                genehist.set_ibin(genehist.find_bin(position), counts[position]['freq'], error=err)
            xline = None
            figsize = [3, 3]
            if utils.get_region(gene) == 'v' and cyst_positions is not None:
                xline = cyst_positions[gene]['cysteine-position']
                figsize[0] *= 3.5
            elif utils.get_region(gene) == 'j' and tryp_positions is not None:
                xline = int(tryp_positions[gene])
                figsize[0] *= 2
            plotting.draw_no_root(genehist, plotdir=plotdir + '/' + utils.get_region(gene), plotname=utils.sanitize_name(gene), errors=True, write_csv=True, xline=xline, figsize=figsize, only_csv=only_csv)
            # paramutils.make_mutefreq_plot(plotdir + '/' + utils.get_region(gene) + '-per-base', utils.sanitize_name(gene), plotting_info)  # needs translation to mpl

        # make mean mute freq hists
        plotting.draw_no_root(self.mean_rates['all'], plotname='all-mean-freq', plotdir=plotdir, stats='mean', bounds=(0.0, 0.4), write_csv=True, only_csv=only_csv)
        for region in utils.regions:
            plotting.draw_no_root(self.mean_rates[region], plotname=region+'-mean-freq', plotdir=plotdir, stats='mean', bounds=(0.0, 0.4), write_csv=True, only_csv=only_csv)

        if not only_csv:  # write html file and fix permissiions
            check_call(['./bin/makeHtml', plotdir, '3', 'null', 'svg'])
            for region in utils.regions:
                check_call(['./bin/makeHtml', plotdir + '/' + region, '1', 'null', 'svg'])
                # check_call(['./bin/makeHtml', plotdir + '/' + region + '-per-base', '1', 'null', 'png'])
            check_call(['./bin/permissify-www', plotdir])  # NOTE this should really permissify starting a few directories higher up
Esempio n. 14
0
    def read_annotation_performance(self, version_stype, input_stype, debug=False):
        """ version_stype is the code version, while input_stype is the input data version, i.e. 'ref', 'new' is the reference code version (last commit) run on the then-new simulation and parameters"""
        ptest = 'annotate-' + input_stype + '-simu'
        if args.quick and ptest not in self.quick_tests:
            return
        if debug:
            print '  version %s input %s annotation' % (version_stype, input_stype)

        def read_performance_file(fname, column, only_ibin=None):
            values = []
            with open(fname) as csvfile:
                reader = csv.DictReader(csvfile)
                ibin = 0
                for line in reader:
                    if only_ibin is not None and ibin != only_ibin:
                        ibin += 1
                        continue
                    values.append(float(line[column]))
                    ibin += 1
            if len(values) == 1:
                return values[0]
            else:
                return values

        perfdir = self.dirs[version_stype] + '/' + self.perfdirs[input_stype]
        for method in ['sw', 'hmm']:
            if debug:
                print '   ', method

            # fraction of genes correct
            for region in utils.regions:
                fraction_correct = read_performance_file(perfdir + '/' + method + '/gene-call/' + region + '_gene.csv', 'contents', only_ibin=1)
                if debug:
                    print '      %s %.3f' % (region, fraction_correct)
                self.perf_info[version_stype][input_stype + '-' + method + '-' + region + '_gene_correct'] = fraction_correct

            # hamming fraction
            hamming_hist = Hist(fname=perfdir + '/' + method + '/mutation/hamming_to_true_naive.csv')
            if debug:
                print '      mean hamming %.2f' % hamming_hist.get_mean()
            self.perf_info[version_stype][input_stype + '-' + method + '-mean_hamming'] = hamming_hist.get_mean()
Esempio n. 15
0
def make_mean_hist(hists, debug=False):
    """ return the hist with bin contents the mean over <hists> of each bin """
    binvals = {}
    all_data = None
    for hist in hists:
        if debug:
            print '    sub',
        for ib in range(0, hist.n_bins + 2):
            low_edge = hist.low_edges[ib]
            if low_edge not in binvals:
                binvals[low_edge] = 0.
            binvals[low_edge] += hist.bin_contents[ib]
            if debug:
                print '   ', low_edge, hist.bin_contents[ib],
        if all_data is not None and hist.all_data is None:
            raise Exception('tried to average hists with and without all_data set')
        if hist.all_data is not None:
            if all_data is None:
                all_data = []
            all_data += hist.all_data
        if debug:
            print ''
    binlist = sorted(binvals.keys())
    meanhist = Hist(len(binlist) - 2, binlist[1], binlist[-1], binlist[1 : -1])
    meanhist.all_data = all_data
    if debug:
        print '   mean',
    for ib in range(len(binlist)):
        meanhist.set_ibin(ib, binvals[binlist[ib]])
        if debug:
            print '   ', meanhist.low_edges[ib], meanhist.bin_contents[ib],
    if debug:
        print ''

    meanhist.normalize()
    return meanhist
Esempio n. 16
0
    def get_mute_hist(self, mtype):
        if self.args.mutate_from_scratch:
            mean_mute_val = self.args.scratch_mute_freq
            if self.args.same_mute_freq_for_all_seqs:
                hist = Hist(1, mean_mute_val - utils.eps,
                            mean_mute_val + utils.eps)
                hist.fill(mean_mute_val)
            else:
                n_entries = 500
                length_vals = [
                    v
                    for v in numpy.random.exponential(mean_mute_val, n_entries)
                ]  # count doesn't work on numpy.ndarray objects
                max_val = 0.8  # this is arbitrary, but you shouldn't be calling this with anything that gets a significant number anywhere near there, anyway
                if length_vals.count(max_val):
                    print '%s lots of really high mutation rates treegenerator::get_mute_hist()' % utils.color(
                        'yellow', 'warning')
                length_vals = [min(v, max_val) for v in length_vals]
                hist = Hist(30, 0., max_val)
                for val in length_vals:
                    hist.fill(val)
                hist.normalize()
        else:
            hist = Hist(fname=self.parameter_dir + '/' + mtype +
                        '-mean-mute-freqs.csv')

        return hist
Esempio n. 17
0
def test_general_project():
    """
    Test general project -- whether Hist can be projected properly.
    """
    h = Hist(
        axis.Regular(50,
                     -5,
                     5,
                     name="A",
                     label="a [units]",
                     underflow=False,
                     overflow=False),
        axis.Boolean(name="B", label="b [units]"),
        axis.Variable(range(11), name="C", label="c [units]"),
        axis.Integer(0, 10, name="D", label="d [units]"),
        axis.IntCategory(range(10), name="E", label="e [units]"),
        axis.StrCategory("FT", name="F", label="f [units]"),
    )

    # via indices
    assert h.project()
    assert h.project(0, 1)
    assert h.project(0, 1, 2, 3, 4, 5)

    # via names
    assert h.project()
    assert h.project("A", "B")
    assert h.project("A", "B", "C", "D", "E", "F")

    h = Hist(
        axis.Regular(50,
                     -5,
                     5,
                     name="A",
                     label="a [units]",
                     underflow=False,
                     overflow=False),
        axis.Boolean(name="B", label="b [units]"),
        axis.Variable(range(11), name="C", label="c [units]"),
        axis.Integer(0, 10, name="D", label="d [units]"),
        axis.IntCategory(range(10), name="E", label="e [units]"),
        axis.StrCategory("FT", name="F", label="f [units]"),
    )

    # duplicated
    with pytest.raises(Exception):
        h.project(0, 0)

    with pytest.raises(Exception):
        h.project("A", "A")

    with pytest.raises(Exception):
        h.project(0, "A")

    # mixed types
    assert h.project(2, "A")

    # cannot found
    with pytest.raises(Exception):
        h.project(-1, 9)

    with pytest.raises(Exception):
        h.project("G", "H")
Esempio n. 18
0
def test_general_transform_proxy():
    """
    Test general transform proxy -- whether Hist transform proxy works properly.
    """

    h0 = Hist.new.Sqrt(3, 4, 25).Sqrt(4, 25, 81).Double()
    h0.fill([5, 10, 17, 17], [26, 37, 50, 65])
    assert h0[0, 0] == 1
    assert h0[1, 1] == 1
    assert h0[2, 2] == 1
    assert h0[2, 3] == 1

    # wrong value
    with pytest.raises(Exception):
        Hist.new.Sqrt(3, -4, 25)

    h1 = Hist.new.Log(4, 1, 10_000).Log(3, 1 / 1_000, 1).Double()
    h1.fill([2, 11, 101, 1_001], [1 / 999, 1 / 99, 1 / 9, 1 / 9])
    assert h1[0, 0] == 1
    assert h1[1, 1] == 1
    assert h1[2, 2] == 1
    assert h1[3, 2] == 1

    # Missing arguments
    with pytest.raises(Exception):
        Hist.new.Regular(4, 1, 10_000).Log()

    # wrong value
    with pytest.raises(Exception):
        Hist.new.Log(3, -1, 10_000)

    h2 = Hist.new.Pow(24, 1, 5, power=2).Pow(124, 1, 5, power=3).Int64()
    h2.fill([1, 2, 3, 4], [1, 2, 3, 4])
    assert h2[0, 0] == 1
    assert h2[3, 7] == 1
    assert h2[8, 26] == 1
    assert h2[15, 63] == 1

    # wrong value
    with pytest.raises(Exception):
        Hist.new.Regular(24, 1, 5).Pow(2)

    # wrong value
    with pytest.raises(Exception):
        Hist.new.Pow(24, -1, 5, power=0.5)

    # lack args
    with pytest.raises(Exception):
        Hist.new.Pow(24, 1, 5)

    ftype = ctypes.CFUNCTYPE(ctypes.c_double, ctypes.c_double)
    h3 = (Hist.new.Func(4,
                        1,
                        5,
                        forward=ftype(math.log),
                        inverse=ftype(math.exp)).Func(4,
                                                      1,
                                                      5,
                                                      forward=ftype(np.log),
                                                      inverse=ftype(
                                                          np.exp))).Int64()
    h3.fill([1, 2, 3, 4], [1, 2, 3, 4])
    assert h3[0, 0] == 1
    assert h3[1, 1] == 1
    assert h3[2, 2] == 1
    assert h3[3, 3] == 1

    # wrong value
    with pytest.raises(Exception):
        Hist().Regular(24, 1, 5).Func(ftype(math.log), ftype(math.exp))

    # wrong value
    assert Hist.new.Func(4,
                         -1,
                         5,
                         forward=ftype(math.log),
                         inverse=ftype(math.log)).Double()
    with pytest.raises(Exception):
        Hist.new.Func(4, -1, 5, forward=ftype(np.log), inverse=ftype(np.log))

    # lack args
    with pytest.raises(Exception):
        Hist.new.Func(4, 1, 5)
Esempio n. 19
0
def test_general_fill():
    """
    Test general fill -- whether Hist can be properly filled.
    """

    # Regular
    h = Hist(
        axis.Regular(10, 0, 1, name="x"),
        axis.Regular(10, 0, 1, name="y"),
        axis.Regular(2, 0, 2, name="z"),
    ).fill(
        x=[0.35, 0.35, 0.35, 0.45, 0.55, 0.55, 0.55],
        y=[0.35, 0.35, 0.45, 0.45, 0.45, 0.45, 0.45],
        z=[0, 0, 1, 1, 1, 1, 1],
    )

    z_one_only = h[{2: bh.loc(1)}]
    for idx_x in range(0, 10):
        for idx_y in range(0, 10):
            if idx_x == 3 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 1
            elif idx_x == 4 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 1
            elif idx_x == 5 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 3
            else:
                assert z_one_only[idx_x, idx_y] == 0

    # Boolean
    h = Hist(
        axis.Boolean(name="x"),
        axis.Boolean(name="y"),
        axis.Boolean(name="z"),
    ).fill(
        [True, True, True, True, True, False, True],
        [False, True, True, False, False, True, False],
        [False, False, True, True, True, True, True],
    )

    z_one_only = h[{2: bh.loc(True)}]
    assert z_one_only[False, False] == 0
    assert z_one_only[False, True] == 1
    assert z_one_only[True, False] == 3
    assert z_one_only[True, True] == 1

    # Variable
    h = Hist(
        axis.Variable(range(11), name="x"),
        axis.Variable(range(11), name="y"),
        axis.Variable(range(3), name="z"),
    ).fill(
        x=[3.5, 3.5, 3.5, 4.5, 5.5, 5.5, 5.5],
        y=[3.5, 3.5, 4.5, 4.5, 4.5, 4.5, 4.5],
        z=[0, 0, 1, 1, 1, 1, 1],
    )

    z_one_only = h[{2: bh.loc(1)}]
    for idx_x in range(0, 10):
        for idx_y in range(0, 10):
            if idx_x == 3 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 1
            elif idx_x == 4 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 1
            elif idx_x == 5 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 3
            else:
                assert z_one_only[idx_x, idx_y] == 0

    # Integer
    h = Hist(
        axis.Integer(0, 10, name="x"),
        axis.Integer(0, 10, name="y"),
        axis.Integer(0, 2, name="z"),
    ).fill(
        [3.5, 3.5, 3.5, 4.5, 5.5, 5.5, 5.5],
        [3.5, 3.5, 4.5, 4.5, 4.5, 4.5, 4.5],
        [0, 0, 1, 1, 1, 1, 1],
    )

    z_one_only = h[{2: bh.loc(1)}]
    for idx_x in range(0, 10):
        for idx_y in range(0, 10):
            if idx_x == 3 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 1
            elif idx_x == 4 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 1
            elif idx_x == 5 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 3
            else:
                assert z_one_only[idx_x, idx_y] == 0

    # IntCategory
    h = Hist(
        axis.IntCategory(range(10), name="x"),
        axis.IntCategory(range(10), name="y"),
        axis.IntCategory(range(2), name="z"),
    ).fill(
        x=[3.5, 3.5, 3.5, 4.5, 5.5, 5.5, 5.5],
        y=[3.5, 3.5, 4.5, 4.5, 4.5, 4.5, 4.5],
        z=[0, 0, 1, 1, 1, 1, 1],
    )

    z_one_only = h[{2: bh.loc(1)}]
    for idx_x in range(0, 10):
        for idx_y in range(0, 10):
            if idx_x == 3 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 1
            elif idx_x == 4 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 1
            elif idx_x == 5 and idx_y == 4:
                assert z_one_only[idx_x, idx_y] == 3
            else:
                assert z_one_only[idx_x, idx_y] == 0

    # StrCategory
    h = Hist(
        axis.StrCategory("FT", name="x"),
        axis.StrCategory(list("FT"), name="y"),
        axis.StrCategory(["F", "T"], name="z"),
    ).fill(
        ["T", "T", "T", "T", "T", "F", "T"],
        ["F", "T", "T", "F", "F", "T", "F"],
        ["F", "F", "T", "T", "T", "T", "T"],
    )

    z_one_only = h[{2: bh.loc("T")}]
    assert z_one_only[bh.loc("F"), bh.loc("F")] == 0
    assert z_one_only[bh.loc("F"), bh.loc("T")] == 1
    assert z_one_only[bh.loc("T"), bh.loc("F")] == 3
    assert z_one_only[bh.loc("T"), bh.loc("T")] == 1

    # with names
    assert Hist(axis.Regular(50, -3, 3, name="x"),
                axis.Regular(50, -3, 3, name="y")).fill(x=np.random.randn(10),
                                                        y=np.random.randn(10))

    assert Hist(axis.Boolean(name="x"),
                axis.Boolean(name="y")).fill(x=[True, False, True],
                                             y=[True, False, True])

    assert Hist(axis.Variable(range(-3, 3), name="x"),
                axis.Variable(range(-3, 3),
                              name="y")).fill(x=np.random.randn(10),
                                              y=np.random.randn(10))

    assert Hist(axis.Integer(-3, 3, name="x"),
                axis.Integer(-3, 3, name="y")).fill(x=np.random.randn(10),
                                                    y=np.random.randn(10))

    assert Hist(
        axis.IntCategory(range(-3, 3), name="x"),
        axis.IntCategory(range(-3, 3), name="y"),
    ).fill(x=np.random.randn(10), y=np.random.randn(10))

    assert Hist(axis.StrCategory(["F", "T"], name="x"),
                axis.StrCategory("FT", name="y")).fill(x=["T", "F", "T"],
                                                       y=["T", "F", "T"])

    h = Hist(
        axis.Regular(50,
                     -4,
                     4,
                     name="X",
                     label="s [units]",
                     underflow=False,
                     overflow=False)).fill(np.random.normal(size=10))
Esempio n. 20
0
def test_hist_proxy():
    """
    Test general hist proxy -- whether Hist hist proxy works properly.
    """

    h = Hist.new.Reg(10, 0, 1, name="x").Double().fill([0.5, 0.5])
    assert h[0.5j] == 2

    assert type(h) == Hist

    with pytest.raises(AttributeError):
        Hist().new

    h = (Hist.new.Reg(10, 0, 1,
                      name="x").Reg(10, 0, 1,
                                    name="y").Double().fill([0.5, 0.5],
                                                            [0.2, 0.6]))

    assert h[0.5j, 0.2j] == 1
    assert h[bh.loc(0.5), bh.loc(0.6)] == 1

    h = Hist.new.Bool(name="x").Double().fill([True, True])
    assert h[bh.loc(True)] == 2

    h = Hist.new.Bool(name="x").Bool(name="y").Int64().fill([True, True],
                                                            [True, False])

    assert h[True, True] == 1
    assert h[True, False] == 1

    h = Hist.new.Var(range(10), name="x").Double().fill([5, 5])
    assert h[5j] == 2

    h = (Hist.new.Var(range(10), name="x").Var(range(10),
                                               name="y").Double().fill([5, 5],
                                                                       [2, 6]))

    assert h[5j, 2j] == 1
    assert h[bh.loc(5), bh.loc(6)] == 1

    h = Hist.new.Int(0, 10, name="x").Int64().fill([5, 5])
    assert h[5j] == 2

    h = Hist.new.Int(0, 10, name="x").Int(0, 10, name="y").Int64().fill([5, 5],
                                                                        [2, 6])

    assert h[5j, 2j] == 1
    assert h[bh.loc(5), bh.loc(6)] == 1

    h = Hist.new.IntCat(range(10), name="x").Double().fill([5, 5])
    assert h[5j] == 2

    h = (Hist.new.IntCat(range(10),
                         name="x").IntCat(range(10),
                                          name="y").Double().fill([5, 5],
                                                                  [2, 6]))

    assert h[5j, 2j] == 1
    assert h[bh.loc(5), bh.loc(6)] == 1

    h = Hist.new.StrCat("TF", name="x").Int64().fill(["T", "T"])
    assert h["T"] == 2

    h = (Hist.new.StrCat("TF",
                         name="x").StrCat("TF",
                                          name="y").Int64().fill(["T", "T"],
                                                                 ["T", "F"]))

    assert h["T", "T"] == 1
    assert h["T", "F"] == 1
Esempio n. 21
0
    def elaborate(self, platform):
        # VGA constants
        pixel_f           = self.timing.pixel_freq
        hsync_front_porch = self.timing.h_front_porch
        hsync_pulse_width = self.timing.h_sync_pulse
        hsync_back_porch  = self.timing.h_back_porch
        vsync_front_porch = self.timing.v_front_porch
        vsync_pulse_width = self.timing.v_sync_pulse
        vsync_back_porch  = self.timing.v_back_porch

        # Pins
        clk25   = platform.request("clk25")
        ov7670  = platform.request("ov7670")
        led     = [platform.request("led", i) for i in range(8)]
        leds    = Cat([i.o for i in led])
        led8_2  = platform.request("led8_2")
        leds8_2 = Cat([led8_2.leds[i] for i in range(8)])
        led8_3  = platform.request("led8_3")
        leds8_3 = Cat([led8_3.leds[i] for i in range(8)])
        leds16  = Cat(leds8_3, leds8_2)
        btn1    = platform.request("button_fire", 0)
        btn2    = platform.request("button_fire", 1)
        up      = platform.request("button_up", 0)
        down    = platform.request("button_down", 0)
        pwr     = platform.request("button_pwr", 0)
        left    = platform.request("button_left", 0)
        right   = platform.request("button_right", 0)
        sw      =  Cat([platform.request("switch",i) for i in range(4)])
        uart    = platform.request("uart")
        divisor = int(platform.default_clk_frequency // 460800)
        esp32   = platform.request("esp32_spi")

        csn     = esp32.csn
        sclk    = esp32.sclk
        copi    = esp32.copi
        cipo    = esp32.cipo

        m = Module()
        
        # Clock generator.
        m.domains.sync  = cd_sync  = ClockDomain("sync")
        m.domains.pixel = cd_pixel = ClockDomain("pixel")
        m.domains.shift = cd_shift = ClockDomain("shift")

        m.submodules.ecp5pll = pll = ECP5PLL()
        pll.register_clkin(clk25,  platform.default_clk_frequency)
        pll.create_clkout(cd_sync,  platform.default_clk_frequency)
        pll.create_clkout(cd_pixel, pixel_f)
        pll.create_clkout(cd_shift, pixel_f * 5.0 * (1.0 if self.ddr else 2.0))

        # Add CamRead submodule
        camread = CamRead()
        m.submodules.camread = camread

        # Camera config
        cam_x_res = 640
        cam_y_res = 480

        camconfig = CamConfig()
        m.submodules.camconfig = camconfig

        # Connect the camera pins and config and read modules
        m.d.comb += [
            ov7670.cam_RESET.eq(1),
            ov7670.cam_PWON.eq(0),
            ov7670.cam_XCLK.eq(clk25.i),
            ov7670.cam_SIOC.eq(camconfig.sioc),
            ov7670.cam_SIOD.eq(camconfig.siod),
            camconfig.start.eq(btn1),
            camread.p_data.eq(Cat([ov7670.cam_data[i] for i in range(8)])),
            camread.href.eq(ov7670.cam_HREF),
            camread.vsync.eq(ov7670.cam_VSYNC),
            camread.p_clock.eq(ov7670.cam_PCLK)
        ]

        # Create the uart
        m.submodules.serial = serial = AsyncSerial(divisor=divisor, pins=uart)

        # Input fifo
        fifo_depth=1024

        m.submodules.fifo = fifo = SyncFIFOBuffered(width=16,depth=fifo_depth)

        # Frame buffer
        x_res= cam_x_res // 2
        y_res= cam_y_res

        buffer = Memory(width=16, depth=x_res * y_res)
        m.submodules.r = r = buffer.read_port()
        m.submodules.w = w = buffer.write_port()

        # Button debouncers
        m.submodules.debup   = debup = Debouncer()
        m.submodules.debdown = debdown = Debouncer()
        m.submodules.debosd  = debosd = Debouncer()
        m.submodules.debsel  = debsel = Debouncer()
        m.submodules.debsnap = debsnap = Debouncer()
        m.submodules.debhist = debhist = Debouncer()

        # Connect the buttons to debouncers
        m.d.comb += [
            debup.btn.eq(up),
            debdown.btn.eq(down),
            debosd.btn.eq(pwr),
            debsel.btn.eq(right),
            debsnap.btn.eq(left),
            debhist.btn.eq(btn2)
        ]

        # Image processing options
        flip        = Signal(2, reset=1)
        mono        = Signal(reset=0)
        invert      = Signal(reset=0)
        gamma       = Signal(reset=0)
        border      = Signal(reset=0)
        filt        = Signal(reset=0)
        grid        = Signal(reset=0)
        histo       = Signal(reset=1)
        hbin        = Signal(6, reset=0)
        bin_cnt     = Signal(5, reset=0)
        thresh      = Signal(reset=0)
        threshold   = Signal(8, reset=0)
        hist_chan   = Signal(2, reset=0)

        ccc         = CC(reset=(0,18,12,16))
        sharpness   = Signal(unsigned(4), reset=0)

        osd_val     = Signal(4, reset=0) # Account for spurious start-up button pushes
        osd_on      = Signal(reset=1)
        osd_sel     = Signal(reset=1)
        snap        = Signal(reset=0)
        frozen      = Signal(reset=1)
        writing     = Signal(reset=0)
        written     = Signal(reset=0)
        byte        = Signal(reset=0)
        w_addr      = Signal(18)

        # Color filter
        l           = Rgb565(reset=(18,12,6)) # Initialised to red LEGO filter
        h           = Rgb565(reset=(21,22,14))

        # Region of interest
        roi         = Roi()

        # VGA signals
        vga_r       = Signal(8)
        vga_g       = Signal(8)
        vga_b       = Signal(8)
        vga_hsync   = Signal()
        vga_vsync   = Signal()
        vga_blank   = Signal()

        # Fifo co-ordinates
        f_x          = Signal(9)
        f_y          = Signal(9)
        f_frame_done = Signal()

        # Pixel from fifo
        pix         = Rgb565()

        # SPI memory for remote configuration
        m.submodules.spimem = spimem = SpiMem(addr_bits=32)

        # Color Control
        m.submodules.cc = cc = ColorControl()

        # Image convolution
        m.submodules.imc = imc = ImageConv()

        # Statistics
        m.submodules.stats = stats = Stats()

        # Histogram
        m.submodules.hist = hist = Hist()

        # Filter
        m.submodules.fil = fil = Filt()

        # Monochrome
        m.submodules.mon = mon = Mono()

        # Sync the fifo with the camera
        sync_fifo = Signal(reset=0)
        with m.If(~sync_fifo & ~fifo.r_rdy & (camread.col == cam_x_res - 1) & (camread.row == cam_y_res -1)):
            m.d.sync += [
                sync_fifo.eq(1),
                f_x.eq(0),
                f_y.eq(0)
            ]

        with m.If(btn1):
            m.d.sync += sync_fifo.eq(0)

        # Connect the fifo
        m.d.comb += [
            fifo.w_en.eq(camread.pixel_valid & camread.col[0] & sync_fifo), # Only write every other pixel
            fifo.w_data.eq(camread.pixel_data), 
            fifo.r_en.eq(fifo.r_rdy & ~imc.o_stall)
        ]

        # Calculate fifo co-ordinates
        m.d.sync += f_frame_done.eq(0)

        with m.If(fifo.r_en & sync_fifo):
            m.d.sync += f_x.eq(f_x + 1)
            with m.If(f_x == x_res - 1):
                m.d.sync += [
                    f_x.eq(0),
                    f_y.eq(f_y + 1)
                ]
                with m.If(f_y == y_res - 1):
                    m.d.sync += [
                        f_y.eq(0),
                        f_frame_done.eq(1)
                    ]
        
        # Extract pixel from fifo data
        m.d.comb += [
            pix.r.eq(fifo.r_data[11:]),
            pix.g.eq(fifo.r_data[5:11]),
            pix.b.eq(fifo.r_data[:5])
        ]

        # Connect color control
        m.d.comb += [
            cc.i.eq(pix),
            cc.i_cc.eq(ccc)
        ]

        # Calculate per-frame statistics, after applying color correction
        m.d.comb += [
            stats.i.eq(cc.o),
            stats.i_valid.eq(fifo.r_rdy),
            # This is not valid when a region of interest is active
            stats.i_avg_valid.eq((f_x >= 32) & (f_x < 288) &
                                 (f_y >= 112) & (f_y < 368)),
            stats.i_frame_done.eq(f_frame_done),
            stats.i_x.eq(f_x),
            stats.i_y.eq(f_y),
            stats.i_roi.eq(roi)
        ]
        
        # Produce histogram, after applying color correction, and after monochrome, for monochrome histogram
        with m.Switch(hist_chan):
            with m.Case(0):
                m.d.comb += hist.i_p.eq(cc.o.r)
            with m.Case(1):
                m.d.comb += hist.i_p.eq(cc.o.g)
            with m.Case(2):
                m.d.comb += hist.i_p.eq(cc.o.b)
            with m.Case(3):
                m.d.comb += hist.i_p.eq(mon.o_m)

        m.d.comb += [
            hist.i_valid.eq(fifo.r_rdy),
            hist.i_clear.eq(f_frame_done),
            hist.i_x.eq(f_x),
            hist.i_y.eq(f_y),
            hist.i_roi.eq(roi),
            hist.i_bin.eq(hbin) # Used when displaying histogram
        ]

        # Apply filter, after color correction
        m.d.comb += [
            fil.i.eq(cc.o),
            fil.i_valid.eq(fifo.r_en),
            fil.i_en.eq(filt),
            fil.i_frame_done.eq(f_frame_done),
            fil.i_l.eq(l),
            fil.i_h.eq(h)
        ]

        # Apply mono, after color correction and filter
        m.d.comb += [
            mon.i.eq(fil.o),
            mon.i_en.eq(mono),
            mon.i_invert.eq(invert),
            mon.i_thresh.eq(thresh),
            mon.i_threshold.eq(threshold)
        ]

        # Apply image convolution, after other transformations
        m.d.comb += [
            imc.i.eq(mon.o),
            imc.i_valid.eq(fifo.r_rdy),
            imc.i_reset.eq(~sync_fifo),
            # Select image convolution
            imc.i_sel.eq(sharpness)
        ]

        # Take a snapshot, freeze the camera, and write the framebuffer to the uart
        # Note that this suspends video output
        with m.If(debsnap.btn_down | (spimem.wr & (spimem.addr == 22))):
            with m.If(frozen):
                m.d.sync += frozen.eq(0)
            with m.Else():
                m.d.sync += [
                    snap.eq(1),
                    frozen.eq(0),
                    w_addr.eq(0),
                    written.eq(0),
                    byte.eq(0)
                ]

        # Wait to end of frame after requesting snapshot, before start of writing to uart
        with m.If(imc.o_frame_done & snap):
            m.d.sync += [
                frozen.eq(1),
                snap.eq(0)
            ]
            with m.If(~written):
                m.d.sync += writing.eq(1)

        # Connect the uart
        m.d.comb += [
            serial.tx.data.eq(Mux(byte, r.data[8:], r.data[:8])),
            serial.tx.ack.eq(writing)
        ]

        # Write to the uart from frame buffer (affects video output)
        with m.If(writing):
            with m.If(w_addr == x_res * y_res):
                m.d.sync += [
                    writing.eq(0),
                    written.eq(1)
                ]
            with m.Elif(serial.tx.ack & serial.tx.rdy):
                m.d.sync += byte.eq(~byte)
                with m.If(byte):
                    m.d.sync += w_addr.eq(w_addr+1)

        # Connect spimem
        m.d.comb += [
            spimem.csn.eq(~csn),
            spimem.sclk.eq(sclk),
            spimem.copi.eq(copi),
            cipo.eq(spimem.cipo),
        ]

        # Writable configuration registers
        spi_wr_vals = Array([ccc.brightness, ccc.redness, ccc.greenness, ccc.blueness, l.r, h.r, l.g, h.g, l.b, h.b,
                             sharpness, filt, border, mono, invert, grid, histo,
                             roi.x[1:], roi.y[1:], roi.w[1:], roi.h[1:], roi.en, None, None, None,
                             threshold, thresh, hist_chan, flip,
                             None, None, None, None, None, None, None, None, None,
                             frozen])

        with m.If(spimem.wr):
            with m.Switch(spimem.addr):
                for i in range(len(spi_wr_vals)):
                    if spi_wr_vals[i] is not None:
                        with m.Case(i):
                            m.d.sync += spi_wr_vals[i].eq(spimem.dout)

        # Readable configuration registers
        spi_rd_vals = Array([ccc.brightness, ccc.redness, ccc.greenness, ccc.blueness, l.r, h.r, l.g, h.g, l.b, h.b,
                             sharpness, filt, border, mono, invert, grid, histo,
                             roi.x[1:], roi.y[1:], roi.w[1:], roi.h[1:], roi.en, fil.o_nz[16:], fil.o_nz[8:16], fil.o_nz[:8],
                             threshold, thresh, hist_chan, flip,
                             stats.o_min.r, stats.o_min.g, stats.o_min.b,
                             stats.o_max.r, stats.o_max.g, stats.o_max.b,
                             stats.o_avg.r, stats.o_avg.g, stats.o_avg.b,
                             frozen, writing, written])

        with m.If(spimem.rd):
            with m.Switch(spimem.addr):
                for i in range(len(spi_rd_vals)):
                    with m.Case(i):
                        m.d.sync += spimem.din.eq(spi_rd_vals[i])

        # Add VGA generator
        m.submodules.vga = vga = VGA(
           resolution_x      = self.timing.x,
           hsync_front_porch = hsync_front_porch,
           hsync_pulse       = hsync_pulse_width,
           hsync_back_porch  = hsync_back_porch,
           resolution_y      = self.timing.y,
           vsync_front_porch = vsync_front_porch,
           vsync_pulse       = vsync_pulse_width,
           vsync_back_porch  = vsync_back_porch,
           bits_x            = 16, # Play around with the sizes because sometimes
           bits_y            = 16  # a smaller/larger value will make it pass timing.
        )

        # Fetch histogram for display
        old_x = Signal(10)
        m.d.sync += old_x.eq(vga.o_beam_x)

        with m.If(vga.o_beam_x == 0):
            m.d.sync += [
                hbin.eq(0),
                bin_cnt.eq(0)
            ]
        with m.Elif(vga.o_beam_x != old_x):
            m.d.sync += bin_cnt.eq(bin_cnt+1)
            with m.If(bin_cnt == 19):
                m.d.sync += [
                    bin_cnt.eq(0),
                    hbin.eq(hbin+1)
                ]

        # Switch between camera and histogram view
        with m.If(debhist.btn_down):
            m.d.sync += histo.eq(~histo)
       
        # Connect frame buffer, with optional x and y flip
        x = Signal(10)
        y = Signal(9)
        
        m.d.comb += [
            w.en.eq(imc.o_valid & ~frozen),
            w.addr.eq(imc.o_y * x_res + imc.o_x),
            w.data.eq(Cat(imc.o.b, imc.o.g, imc.o.r)),
            y.eq(Mux(flip[1], y_res - 1 - vga.o_beam_y, vga.o_beam_y)),
            x.eq(Mux(flip[0], x_res - 1 - vga.o_beam_x[1:], vga.o_beam_x[1:])),
            r.addr.eq(Mux(writing, w_addr, y * x_res + x))
        ]

        # Apply the On-Screen Display (OSD)
        m.submodules.osd = osd = OSD()

        hist_col = Signal(8)

        m.d.comb += [
            osd.x.eq(vga.o_beam_x),
            osd.y.eq(vga.o_beam_y),
            hist_col.eq(Mux((479 - osd.y) < hist.o_val[8:], 0xff, 0x00)),
            osd.i_r.eq(Mux(histo, Mux((hist_chan == 0) | (hist_chan == 3), hist_col, 0), Cat(Const(0, unsigned(3)), r.data[11:16]))),
            osd.i_g.eq(Mux(histo, Mux((hist_chan == 1) | (hist_chan == 3), hist_col, 0), Cat(Const(0, unsigned(2)), r.data[5:11]))),
            osd.i_b.eq(Mux(histo, Mux((hist_chan == 2) | (hist_chan == 3), hist_col, 0), Cat(Const(0, unsigned(3)), r.data[0:5]))),
            osd.on.eq(osd_on),
            osd.osd_val.eq(osd_val),
            osd.sel.eq(osd_sel),
            osd.grid.eq(grid),
            osd.border.eq(border),
            osd.roi.eq(roi.en & ~histo),
            osd.roi_x.eq(roi.x),
            osd.roi_y.eq(roi.y),
            osd.roi_w.eq(roi.w),
            osd.roi_h.eq(roi.h)
        ]

        # OSD control
        osd_vals = Array([ccc.brightness, ccc.redness, ccc.greenness, ccc.blueness, mono, flip[0], flip[1],
                          border, sharpness, invert, grid, filt])

        with m.If(debosd.btn_down):
            m.d.sync += osd_on.eq(~osd_on)

        with m.If(osd_on):
            with m.If(debsel.btn_down):
                m.d.sync += osd_sel.eq(~osd_sel)

            with m.If(debup.btn_down):
                with m.If(~osd_sel):
                    m.d.sync += osd_val.eq(Mux(osd_val == 0, 11, osd_val-1))
                with m.Else():
                    with m.Switch(osd_val):
                        for i in range(len(osd_vals)):
                            with m.Case(i):
                                if (len(osd_vals[i]) == 1):
                                    m.d.sync += osd_vals[i].eq(1)
                                else:
                                    m.d.sync += osd_vals[i].eq(osd_vals[i]+1)

            with m.If(debdown.btn_down):
                with m.If(~osd_sel):
                    m.d.sync += osd_val.eq(Mux(osd_val == 11, 0, osd_val+1))
                with m.Else():
                    with m.Switch(osd_val):
                        for i in range(len(osd_vals)):
                            with m.Case(i):
                                if (len(osd_vals[i]) == 1):
                                    m.d.sync += osd_vals[i].eq(0)
                                else:
                                    m.d.sync += osd_vals[i].eq(osd_vals[i]-1)

        # Show configuration values on leds
        with m.Switch(osd_val):
            for i in range(len(osd_vals)):
                with m.Case(i):
                    m.d.comb += leds.eq(osd_vals[i])

        # Generate VGA signals
        m.d.comb += [
            vga.i_clk_en.eq(1),
            vga.i_test_picture.eq(0),
            vga.i_r.eq(osd.o_r), 
            vga.i_g.eq(osd.o_g), 
            vga.i_b.eq(osd.o_b), 
            vga_r.eq(vga.o_vga_r),
            vga_g.eq(vga.o_vga_g),
            vga_b.eq(vga.o_vga_b),
            vga_hsync.eq(vga.o_vga_hsync),
            vga_vsync.eq(vga.o_vga_vsync),
            vga_blank.eq(vga.o_vga_blank),
        ]

        # VGA to digital video converter.
        tmds = [Signal(2) for i in range(4)]
        m.submodules.vga2dvid = vga2dvid = VGA2DVID(ddr=self.ddr, shift_clock_synchronizer=False)
        m.d.comb += [
            vga2dvid.i_red.eq(vga_r),
            vga2dvid.i_green.eq(vga_g),
            vga2dvid.i_blue.eq(vga_b),
            vga2dvid.i_hsync.eq(vga_hsync),
            vga2dvid.i_vsync.eq(vga_vsync),
            vga2dvid.i_blank.eq(vga_blank),
            tmds[3].eq(vga2dvid.o_clk),
            tmds[2].eq(vga2dvid.o_red),
            tmds[1].eq(vga2dvid.o_green),
            tmds[0].eq(vga2dvid.o_blue),
        ]

        # GPDI pins
        if (self.ddr):
            # Vendor specific DDR modules.
            # Convert SDR 2-bit input to DDR clocked 1-bit output (single-ended)
            # onboard GPDI.
            m.submodules.ddr0_clock = Instance("ODDRX1F",
                i_SCLK = ClockSignal("shift"),
                i_RST  = 0b0,
                i_D0   = tmds[3][0],
                i_D1   = tmds[3][1],
                o_Q    = self.o_gpdi_dp[3])
            m.submodules.ddr0_red   = Instance("ODDRX1F",
                i_SCLK = ClockSignal("shift"),
                i_RST  = 0b0,
                i_D0   = tmds[2][0],
                i_D1   = tmds[2][1],
                o_Q    = self.o_gpdi_dp[2])
            m.submodules.ddr0_green = Instance("ODDRX1F",
                i_SCLK = ClockSignal("shift"),
                i_RST  = 0b0,
                i_D0   = tmds[1][0],
                i_D1   = tmds[1][1],
                o_Q    = self.o_gpdi_dp[1])
            m.submodules.ddr0_blue  = Instance("ODDRX1F",
                i_SCLK = ClockSignal("shift"),
                i_RST  = 0b0,
                i_D0   = tmds[0][0],
                i_D1   = tmds[0][1],
                o_Q    = self.o_gpdi_dp[0])
        else:
            m.d.comb += [
                self.o_gpdi_dp[3].eq(tmds[3][0]),
                self.o_gpdi_dp[2].eq(tmds[2][0]),
                self.o_gpdi_dp[1].eq(tmds[1][0]),
                self.o_gpdi_dp[0].eq(tmds[0][0]),
            ]

        return m
Esempio n. 22
0
def get_cluster_size_hist(partition):
    sizes = [len(c) for c in partition]
    hist = Hist(max(sizes), 0.5, max(sizes) + 0.5)
    for sz in sizes:
        hist.fill(sz)
    return hist