def make_mean_hist(hists, debug=False):
""" return the hist with bin contents the mean over <hists> of each bin """
binvals = {}
for hist in hists:
if debug:
print ' sub',
for ib in range(0, hist.n_bins + 2):
low_edge = hist.low_edges[ib]
if low_edge not in binvals:
binvals[low_edge] = 0.
binvals[low_edge] += hist.bin_contents[ib]
if debug:
print ' ', low_edge, hist.bin_contents[ib],
if debug:
print ''
binlist = sorted(binvals.keys())
meanhist = Hist(len(binlist) - 2, binlist[1], binlist[-1], binlist[1 : -1])
if debug:
print ' mean',
for ib in range(len(binlist)):
meanhist.set_ibin(ib, binvals[binlist[ib]])
if debug:
print ' ', meanhist.low_edges[ib], meanhist.bin_contents[ib],
if debug:
print ''
meanhist.normalize()
return meanhist
def read_mute_freqs(self, mute_freq_dir):
# NOTE these are mute freqs, not branch lengths, but it's ok for now
for mtype in ['all',] + utils.regions:
infname = mute_freq_dir + '/' + mtype + '-mean-mute-freqs.csv'
self.branch_lengths[mtype] = {}
self.branch_lengths[mtype]['lengths'], self.branch_lengths[mtype]['probs'] = [], []
mutehist = Hist(fname=infname)
self.branch_lengths[mtype]['mean'] = mutehist.get_mean()
# if mutehist.GetBinContent(0) > 0.0 or mutehist.GetBinContent(mutehist.GetNbinsX()+1) > 0.0:
# print 'WARNING nonzero under/overflow bins read from %s' % infname
mutehist.normalize(include_overflows=False, overflow_eps_to_ignore=1e-2) # if it was written with overflows included, it'll need to be renormalized
check_sum = 0.0
for ibin in range(1, mutehist.n_bins + 1): # ignore under/overflow bins
freq = mutehist.get_bin_centers()[ibin]
branch_length = self.convert_observed_changes_to_branch_length(float(freq))
prob = mutehist.bin_contents[ibin]
self.branch_lengths[mtype]['lengths'].append(branch_length)
self.branch_lengths[mtype]['probs'].append(prob)
check_sum += self.branch_lengths[mtype]['probs'][-1]
if not utils.is_normed(check_sum):
raise Exception('not normalized %f' % check_sum)
if self.args.debug:
print ' mean branch lengths'
for mtype in ['all',] + utils.regions:
print ' %4s %7.3f (ratio %7.3f)' % (mtype, self.branch_lengths[mtype]['mean'], self.branch_lengths[mtype]['mean'] / self.branch_lengths['all']['mean'])
def __init__(self, base_indir, outdir, gene_name, naivety, glfo, args):
self.region = utils.get_region(gene_name)
self.raw_name = gene_name # i.e. unsanitized
self.germline_seqs = glfo['seqs'] # all germline alleles
self.germline_seq = self.germline_seqs[self.region][gene_name] # germline sequence for this hmm
self.indir = base_indir
self.args = args
self.cyst_positions = glfo['cyst-positions']
self.tryp_positions = glfo['tryp-positions']
# parameters with values that I more or less made up
self.precision = '16' # number of digits after the decimal for probabilities
self.eps = 1e-6 # NOTE I also have an eps defined in utils, and they should in principle be combined
self.n_max_to_interpolate = 20
self.min_mean_unphysical_insertion_length = {'fv' : 1.5, 'jf' : 25} # jf has to be quite a bit bigger, since besides account for the variation in J length from the tryp position to the end, it has to account for the difference in cdr3 lengths
self.erosion_pseudocount_length = 10 # if we're closer to the end of the gene than this, make sure erosion probability isn't zero
# self.insert_mute_prob = 0.0
# self.mean_mute_freq = 0.0
self.outdir = outdir
self.naivety = naivety
self.smallest_entry_index = -1 # keeps track of the first state that has a chance of being entered from init -- we want to start writing (with add_internal_state) from there
# self.insertions = [ insert for insert in utils.index_keys if re.match(self.region + '._insertion', insert) or re.match('.' + self.region + '_insertion', insert)] OOPS that's not what I want to do
self.insertions = []
if self.region == 'v':
self.insertions.append('fv')
elif self.region == 'd':
self.insertions.append('vd')
elif self.region == 'j':
self.insertions.append('dj')
self.insertions.append('jf')
self.erosion_probs = {}
self.insertion_probs = {}
self.insertion_content_probs = {}
self.n_occurences = utils.read_overall_gene_probs(self.indir, only_gene=gene_name, normalize=False) # how many times did we observe this gene in data?
replacement_genes = None
if self.n_occurences < self.args.min_observations_to_write: # if we didn't see it enough, average over all the genes that find_replacement_genes() gives us
if self.args.debug:
print ' only saw it %d times, use info from other genes' % self.n_occurences
replacement_genes = utils.find_replacement_genes(self.indir, self.args.min_observations_to_write, gene_name, single_gene=False, debug=self.args.debug)
self.read_erosion_info(gene_name, replacement_genes) # try this exact gene, but...
self.read_insertion_info(gene_name, replacement_genes)
if self.naivety == 'M': # mutate if not naive
self.mute_freqs, self.mute_obs = paramutils.read_mute_info(self.indir, this_gene=gene_name, approved_genes=replacement_genes)
self.track = Track('nukes', utils.nukes)
self.saniname = utils.sanitize_name(gene_name)
self.hmm = HMM(self.saniname, self.track.getdict()) # pass the track as a dict rather than a Track object to keep the yaml file a bit more readable
self.hmm.extras['gene_prob'] = max(self.eps, utils.read_overall_gene_probs(self.indir, only_gene=gene_name)) # if we really didn't see this gene at all, take pity on it and kick it an eps
mean_freq_hist = Hist(fname=self.indir + '/all-mean-mute-freqs.csv')
self.hmm.extras['overall_mute_freq'] = mean_freq_hist.get_mean()
def get_cluster_size_hist(partition, rebin=None):
sizes = [len(c) for c in partition]
nbins = max(sizes)
# if nbins > 30:
# rebin = 2
if rebin is not None:
nbins = int(float(nbins) / rebin)
hist = Hist(nbins, 0.5, max(sizes) + 0.5)
for sz in sizes:
hist.fill(sz)
return hist
def __init__(self, base_indir, outdir, gene_name, glfo, args, debug=False):
self.region = utils.get_region(gene_name)
self.raw_name = gene_name # i.e. unsanitized
self.germline_seqs = glfo['seqs'] # all germline alleles
self.germline_seq = self.germline_seqs[self.region][gene_name] # germline sequence for this hmm
self.indir = base_indir
self.args = args
self.debug = debug
self.codon_positions = {r : glfo[c + '-positions'] for r, c in utils.conserved_codons[args.chain].items()}
# parameters with values that I more or less made up
self.precision = '16' # number of digits after the decimal for probabilities
self.eps = 1e-6 # NOTE I also have an eps defined in utils, and they should in principle be combined
self.n_max_to_interpolate = args.min_observations_to_write
self.min_mean_unphysical_insertion_length = {'fv' : 1.5, 'jf' : 25} # jf has to be quite a bit bigger, since besides account for the variation in J length from the tryp position to the end, it has to account for the difference in cdr3 lengths
self.erosion_pseudocount_length = 10 # if we're closer to the end of the gene than this, make sure erosion probability isn't zero
self.outdir = outdir
self.smallest_entry_index = -1 # keeps track of the first state that has a chance of being entered from init -- we want to start writing (with add_internal_state) from there
self.insertions = []
if self.region == 'v':
self.insertions.append('fv')
elif self.region == 'd':
self.insertions.append('vd')
elif self.region == 'j':
self.insertions.append('dj')
self.insertions.append('jf')
assert len(utils.ambiguous_bases) == 1 and utils.ambiguous_bases[0] == 'N' # maybe need to update some stuff below if this changes
if self.debug:
print '%s' % utils.color_gene(gene_name)
self.n_occurences = utils.read_single_gene_count(self.indir, gene_name, debug=self.debug) # how many times did we observe this gene in data?
replacement_genes = None
if self.n_occurences < self.args.min_observations_to_write: # if we didn't see it enough, average over all the genes that find_replacement_genes() gives us
if self.debug:
print ' only saw it %d times (wanted %d), so use info from all other genes' % (self.n_occurences, self.args.min_observations_to_write)
replacement_genes = utils.find_replacement_genes(self.indir, self.args.min_observations_to_write, gene_name, debug=self.debug)
self.erosion_probs = self.read_erosion_info(gene_name, replacement_genes)
self.insertion_probs, self.insertion_content_probs = self.read_insertion_info(gene_name, replacement_genes)
self.mute_freqs, self.mute_obs = paramutils.read_mute_info(self.indir, this_gene=gene_name, chain=self.args.chain, approved_genes=replacement_genes) # actual info in <self.mute_obs> isn't actually used a.t.m.
self.track = Track('nukes', utils.nukes)
self.saniname = utils.sanitize_name(gene_name)
self.hmm = HMM(self.saniname, self.track.getdict()) # pass the track as a dict rather than a Track object to keep the yaml file a bit more readable
self.hmm.extras['gene_prob'] = max(self.eps, utils.read_overall_gene_probs(self.indir, only_gene=gene_name)) # if we really didn't see this gene at all, take pity on it and kick it an eps
tmp_mean_freq_hist = Hist(fname=self.indir + '/all-mean-mute-freqs.csv')
self.hmm.extras['overall_mute_freq'] = tmp_mean_freq_hist.get_mean()
def make_hist_from_dict_of_counts(values, var_type, hist_label, log='', xmin_force=0.0, xmax_force=0.0, normalize=False, sort=False):
""" Fill a histogram with values from a dictionary (each key will correspond to one bin) """
assert var_type == 'int' or var_type == 'string' # floats should be handled by Hist class in hist.py
if len(values) == 0:
print 'WARNING no values for %s in make_hist' % hist_label
return TH1D(hist_label, '', 1, 0, 1)
bin_labels = sorted(values)
if not sort and var_type == 'string': # for strings, sort so most common value is to left side
bin_labels = sorted(values, key=values.get, reverse=True)
if var_type == 'string':
n_bins = len(values)
else:
n_bins = bin_labels[-1] - bin_labels[0] + 1
hist = None
xbins = [0. for _ in range(n_bins+1)] # NOTE the +1 is 'cause you need the lower edge of the overflow bin
if xmin_force == xmax_force: # if boundaries aren't set explicitly, work out what they should be
if var_type == 'string':
set_bins(bin_labels, n_bins, 'x' in log, xbins, var_type)
hist = Hist(n_bins, xbins[0], xbins[-1], xbins=xbins)
else:
hist = Hist(n_bins, bin_labels[0] - 0.5, bin_labels[-1] + 0.5) # for integers, just go from the first to the last bin label (they're sorted)
else:
hist = Hist(n_bins, xmin_force, xmax_force)
for ival in range(len(values)):
if var_type == 'string':
label = bin_labels[ival]
ibin = ival + 1
else:
label = ''
ibin = hist.find_bin(bin_labels[ival])
hist.set_ibin(ibin, values[bin_labels[ival]], error=math.sqrt(values[bin_labels[ival]]), label=label)
# make sure there's no overflows
if hist.bin_contents[0] != 0.0 or hist.bin_contents[-1] != 0.0:
for ibin in range(hist.n_bins + 2):
print '%d %f %f' % (ibin, hist.low_edges[ibin], hist.bin_contents[ibin])
raise Exception('overflows in ' + hist_label)
if normalize:
hist.normalize()
hist.ytitle = 'freq'
else:
hist.ytitle = 'counts'
roothist = make_hist_from_my_hist_class(hist, hist_label)
return roothist
def plot(self, plotdir, only_csv=False, only_overall=False):
if not self.finalized:
self.finalize()
overall_plotdir = plotdir + '/overall'
for gene in self.freqs:
if only_overall:
continue
freqs = self.freqs[gene]
if len(freqs) == 0:
if gene not in glutils.dummy_d_genes.values():
print ' %s no mutefreqer obs for %s' % (utils.color('red', 'warning'), utils.color_gene(gene))
continue
sorted_positions = sorted(freqs.keys())
genehist = Hist(sorted_positions[-1] - sorted_positions[0] + 1, sorted_positions[0] - 0.5, sorted_positions[-1] + 0.5, xtitle='position', ytitle='mut freq', title=gene)
for position in sorted_positions:
hi_diff = abs(freqs[position]['freq'] - freqs[position]['freq_hi_err'])
lo_diff = abs(freqs[position]['freq'] - freqs[position]['freq_lo_err'])
err = 0.5*(hi_diff + lo_diff)
genehist.set_ibin(genehist.find_bin(position), freqs[position]['freq'], error=err)
xline = None
figsize = [7, 4]
if utils.get_region(gene) in utils.conserved_codons[self.glfo['chain']]:
codon = utils.conserved_codons[self.glfo['chain']][utils.get_region(gene)]
xline = self.glfo[codon + '-positions'][gene]
if utils.get_region(gene) == 'v':
figsize[0] *= 3.5
elif utils.get_region(gene) == 'j':
figsize[0] *= 2
plotting.draw_no_root(self.per_gene_mean_rates[gene], plotdir=plotdir + '/per-gene/' + utils.get_region(gene), plotname=utils.sanitize_name(gene), errors=True, write_csv=True, only_csv=only_csv, shift_overflows=True)
# per-position plots:
plotting.draw_no_root(genehist, plotdir=plotdir + '/per-gene-per-position/' + utils.get_region(gene), plotname=utils.sanitize_name(gene), errors=True, write_csv=True, xline=xline, figsize=figsize, only_csv=only_csv, shift_overflows=True)
# # per-position, per-base plots:
# paramutils.make_mutefreq_plot(plotdir + '/' + utils.get_region(gene) + '-per-base', utils.sanitize_name(gene), plotting_info) # needs translation to mpl UPDATE fcn is fixed, but I can't be bothered uncommenting this at the moment
# make mean mute freq hists
for rstr in ['all', 'cdr3'] + utils.regions:
if rstr == 'all':
bounds = (0.0, 0.4)
else:
bounds = (0.0, 0.6 if rstr == 'd' else 0.4)
plotting.draw_no_root(self.mean_rates[rstr], plotname=rstr+'_mean-freq', plotdir=overall_plotdir, stats='mean', bounds=bounds, write_csv=True, only_csv=only_csv, shift_overflows=True)
plotting.draw_no_root(self.mean_n_muted[rstr], plotname=rstr+'_mean-n-muted', plotdir=overall_plotdir, stats='mean', write_csv=True, only_csv=only_csv, shift_overflows=True)
if not only_csv: # write html file and fix permissiions
for substr in self.subplotdirs:
plotting.make_html(plotdir + '/' + substr)
def peruse_naive_seqs():
from hist import Hist
# hall = Hist(n_set_list[-1], n_set_list[0] - 0.5, n_set_list[-1] + 0.5)
means = []
for n_set in n_set_list:
plotdir = baseplotdir + '/' + str(n_set)
hist = Hist(fname=plotdir + '/hmm/hamming_to_true_naive.csv')
print '%2d %.2f' % (n_set, hist.get_mean()),
# hall.set_ibin(hall.find_bin(n_set), hist.get_mean())
means.append(hist.get_mean())
import plotting
fig, ax = plotting.mpl_init()
# hall.mpl_plot(ax)
ax.plot(n_set_list, means, marker='.')
plotting.mpl_finish(ax, baseplotdir, 'means', xlabel='N simultaneous seqs', ylabel='mean hamming to true naive', ybounds=(0, None))
def make_mean_hist(hists):
""" return the hist with bin contents the mean over <hists> of each bin """
binvals = {}
for hist in hists: # I could probably do this with list comprehensions or something, but this way handles different bin bounds
for ib in range(0, hist.n_bins + 2):
low_edge = hist.low_edges[ib]
if low_edge not in binvals:
binvals[low_edge] = []
binvals[low_edge].append(hist.bin_contents[ib])
binlist = sorted(binvals.keys())
meanhist = Hist(len(binlist) - 2, binlist[1], binlist[-1], xbins=binlist[1 :])
for ib in range(len(binlist)):
vlist = binvals[binlist[ib]]
meanhist.set_ibin(ib, numpy.mean(vlist), error=(numpy.std(vlist, ddof=1) / math.sqrt(len(vlist))))
# meanhist.normalize()
return meanhist
def plot(self, base_plotdir, cyst_positions=None, tryp_positions=None, only_csv=False):
if not self.finalized:
self.finalize()
plotdir = base_plotdir + '/mute-freqs'
overall_plotdir = plotdir + '/overall'
utils.prep_dir(overall_plotdir, multilings=('*.csv', '*.svg'))
for region in utils.regions:
utils.prep_dir(plotdir + '/' + region, multilings=('*.csv', '*.svg'))
# utils.prep_dir(plotdir + '/' + region + '-per-base/plots', multilings=('*.csv', '*.png'))
if self.tigger:
utils.prep_dir(plotdir + '/tigger', multilings=('*.csv', '*.svg'))
for gene in self.freqs:
freqs = self.freqs[gene]
sorted_positions = sorted(freqs.keys())
genehist = Hist(sorted_positions[-1] - sorted_positions[0] + 1, sorted_positions[0] - 0.5, sorted_positions[-1] + 0.5, xtitle='fixme', ytitle='fixme') #, title=utils.sanitize_name(gene))
for position in sorted_positions:
hi_diff = abs(freqs[position]['freq'] - freqs[position]['freq_hi_err'])
lo_diff = abs(freqs[position]['freq'] - freqs[position]['freq_lo_err'])
err = 0.5*(hi_diff + lo_diff)
genehist.set_ibin(genehist.find_bin(position), freqs[position]['freq'], error=err)
xline = None
figsize = [3, 3]
if utils.get_region(gene) == 'v' and cyst_positions is not None:
xline = cyst_positions[gene]
figsize[0] *= 3.5
elif utils.get_region(gene) == 'j' and tryp_positions is not None:
xline = tryp_positions[gene]
figsize[0] *= 2
plotting.draw_no_root(genehist, plotdir=plotdir + '/' + utils.get_region(gene), plotname=utils.sanitize_name(gene), errors=True, write_csv=True, xline=xline, figsize=figsize, only_csv=only_csv)
# paramutils.make_mutefreq_plot(plotdir + '/' + utils.get_region(gene) + '-per-base', utils.sanitize_name(gene), plotting_info) # needs translation to mpl
# make mean mute freq hists
plotting.draw_no_root(self.mean_rates['all'], plotname='all-mean-freq', plotdir=overall_plotdir, stats='mean', bounds=(0.0, 0.4), write_csv=True, only_csv=only_csv)
for region in utils.regions:
plotting.draw_no_root(self.mean_rates[region], plotname=region+'-mean-freq', plotdir=overall_plotdir, stats='mean', bounds=(0.0, 0.4), write_csv=True, only_csv=only_csv)
if self.tigger:
self.tigger_plot(only_csv)
if not only_csv: # write html file and fix permissiions
plotting.make_html(overall_plotdir)
for region in utils.regions:
plotting.make_html(plotdir + '/' + region, n_columns=1)
def read_annotation_performance(self, version_stype, input_stype, debug=False):
""" version_stype is the code version, while input_stype is the input data version, i.e. 'ref', 'new' is the reference code version (last commit) run on the then-new simulation and parameters"""
ptest = "annotate-" + input_stype + "-simu"
if args.quick and ptest not in self.quick_tests:
return
if debug:
print " version %s input %s annotation" % (version_stype, input_stype)
def read_performance_file(fname, column, only_ibin=None):
values = []
with open(fname) as csvfile:
reader = csv.DictReader(csvfile)
ibin = 0
for line in reader:
if only_ibin is not None and ibin != only_ibin:
ibin += 1
continue
values.append(float(line[column]))
ibin += 1
if len(values) == 1:
return values[0]
else:
return values
perfdir = self.dirs[version_stype] + "/" + self.perfdirs[input_stype]
for method in ["sw", "hmm"]:
if debug:
print " ", method
# fraction of genes correct
for region in utils.regions:
fraction_correct = read_performance_file(
perfdir + "/" + method + "/plots/" + region + "_gene.csv", "contents", only_ibin=1
)
if debug:
print " %s %.3f" % (region, fraction_correct)
self.perf_info[version_stype][
input_stype + "-" + method + "-" + region + "_gene_correct"
] = fraction_correct
# hamming fraction
hamming_hist = Hist(fname=perfdir + "/" + method + "/plots/hamming_to_true_naive.csv")
if debug:
print " mean hamming %.2f" % hamming_hist.get_mean()
self.perf_info[version_stype][input_stype + "-" + method + "-mean_hamming"] = hamming_hist.get_mean()
def make_fraction_plot(hright, hwrong, plotdir, plotname, xlabel, ylabel, xbounds, only_csv=False, write_csv=False):
if 'fraction_uncertainty' not in sys.modules:
import fraction_uncertainty
# NOTE should really merge this with draw_no_root()
xvals = hright.get_bin_centers() #ignore_overflows=True)
right = hright.bin_contents
wrong = hwrong.bin_contents
yvals = [float(r) / (r + w) if r + w > 0. else 0. for r, w in zip(right, wrong)]
# remove values corresponding to bins with no entries
while yvals.count(0.) > 0:
iv = yvals.index(0.)
xvals.pop(iv)
right.pop(iv)
wrong.pop(iv)
yvals.pop(iv)
tmphilos = [sys.modules['fraction_uncertainty'].err(r, r + w) for r, w in zip(right, wrong)]
yerrs = [err[1] - err[0] for err in tmphilos]
# print '%s' % region
# for iv in range(len(xvals)):
# print ' %5.2f %5.0f / %5.0f = %5.2f +/- %.3f' % (xvals[iv], right[iv], right[iv] + wrong[iv], yvals[iv], yerrs[iv])
if write_csv:
hist_for_csv = Hist(hright.n_bins, hright.xmin, hright.xmax)
bincenters = hright.get_bin_centers()
for ibin in range(hright.n_bins):
bcenter = bincenters[ibin]
if bcenter in xvals: # if we didn't remove it
iy = xvals.index(bcenter)
hist_for_csv.set_ibin(ibin, yvals[iy], error=yerrs[iy])
hist_for_csv.write(plotdir + '/' + plotname + '.csv')
if not only_csv:
fig, ax = mpl_init()
ax.errorbar(xvals, yvals, yerr=yerrs, markersize=10, linewidth=1, marker='.')
if xlabel == 'support':
ax.plot((0, 1), (0, 1), color='black', linestyle='--', linewidth=3) # line with slope 1 and intercept 0
mpl_finish(ax, plotdir, plotname, xlabel=xlabel, ylabel=ylabel, title=plotconfig.plot_titles.get(plotname, plotname), xbounds=xbounds, ybounds=(-0.1, 1.1))
plt.close()
def plot(self, base_plotdir, cyst_positions=None, tryp_positions=None, only_csv=False):
if not self.finalized:
self.finalize()
plotdir = base_plotdir + '/mute-freqs'
utils.prep_dir(plotdir + '/plots', multilings=('*.csv', '*.svg'))
for region in utils.regions:
utils.prep_dir(plotdir + '/' + region + '/plots', multilings=('*.csv', '*.svg'))
# utils.prep_dir(plotdir + '/' + region + '-per-base/plots', multilings=('*.csv', '*.png'))
for gene in self.counts:
counts, plotting_info = self.counts[gene], self.plotting_info[gene]
sorted_positions = sorted(counts)
genehist = Hist(sorted_positions[-1] - sorted_positions[0] + 1, sorted_positions[0] - 0.5, sorted_positions[-1] + 0.5, xtitle='fixme', ytitle='fixme') #, title=utils.sanitize_name(gene))
for position in sorted_positions:
hi_diff = abs(counts[position]['freq'] - counts[position]['freq_hi_err'])
lo_diff = abs(counts[position]['freq'] - counts[position]['freq_lo_err'])
err = 0.5*(hi_diff + lo_diff)
genehist.set_ibin(genehist.find_bin(position), counts[position]['freq'], error=err)
xline = None
figsize = [3, 3]
if utils.get_region(gene) == 'v' and cyst_positions is not None:
xline = cyst_positions[gene]['cysteine-position']
figsize[0] *= 3.5
elif utils.get_region(gene) == 'j' and tryp_positions is not None:
xline = int(tryp_positions[gene])
figsize[0] *= 2
plotting.draw_no_root(genehist, plotdir=plotdir + '/' + utils.get_region(gene), plotname=utils.sanitize_name(gene), errors=True, write_csv=True, xline=xline, figsize=figsize, only_csv=only_csv)
# paramutils.make_mutefreq_plot(plotdir + '/' + utils.get_region(gene) + '-per-base', utils.sanitize_name(gene), plotting_info) # needs translation to mpl
# make mean mute freq hists
plotting.draw_no_root(self.mean_rates['all'], plotname='all-mean-freq', plotdir=plotdir, stats='mean', bounds=(0.0, 0.4), write_csv=True, only_csv=only_csv)
for region in utils.regions:
plotting.draw_no_root(self.mean_rates[region], plotname=region+'-mean-freq', plotdir=plotdir, stats='mean', bounds=(0.0, 0.4), write_csv=True, only_csv=only_csv)
if not only_csv: # write html file and fix permissiions
check_call(['./bin/makeHtml', plotdir, '3', 'null', 'svg'])
for region in utils.regions:
check_call(['./bin/makeHtml', plotdir + '/' + region, '1', 'null', 'svg'])
# check_call(['./bin/makeHtml', plotdir + '/' + region + '-per-base', '1', 'null', 'png'])
check_call(['./bin/permissify-www', plotdir]) # NOTE this should really permissify starting a few directories higher up
def read_annotation_performance(self, version_stype, input_stype, debug=False):
""" version_stype is the code version, while input_stype is the input data version, i.e. 'ref', 'new' is the reference code version (last commit) run on the then-new simulation and parameters"""
ptest = 'annotate-' + input_stype + '-simu'
if args.quick and ptest not in self.quick_tests:
return
if debug:
print ' version %s input %s annotation' % (version_stype, input_stype)
def read_performance_file(fname, column, only_ibin=None):
values = []
with open(fname) as csvfile:
reader = csv.DictReader(csvfile)
ibin = 0
for line in reader:
if only_ibin is not None and ibin != only_ibin:
ibin += 1
continue
values.append(float(line[column]))
ibin += 1
if len(values) == 1:
return values[0]
else:
return values
perfdir = self.dirs[version_stype] + '/' + self.perfdirs[input_stype]
for method in ['sw', 'hmm']:
if debug:
print ' ', method
# fraction of genes correct
for region in utils.regions:
fraction_correct = read_performance_file(perfdir + '/' + method + '/gene-call/' + region + '_gene.csv', 'contents', only_ibin=1)
if debug:
print ' %s %.3f' % (region, fraction_correct)
self.perf_info[version_stype][input_stype + '-' + method + '-' + region + '_gene_correct'] = fraction_correct
# hamming fraction
hamming_hist = Hist(fname=perfdir + '/' + method + '/mutation/hamming_to_true_naive.csv')
if debug:
print ' mean hamming %.2f' % hamming_hist.get_mean()
self.perf_info[version_stype][input_stype + '-' + method + '-mean_hamming'] = hamming_hist.get_mean()
def make_mean_hist(hists, debug=False):
""" return the hist with bin contents the mean over <hists> of each bin """
binvals = {}
all_data = None
for hist in hists:
if debug:
print ' sub',
for ib in range(0, hist.n_bins + 2):
low_edge = hist.low_edges[ib]
if low_edge not in binvals:
binvals[low_edge] = 0.
binvals[low_edge] += hist.bin_contents[ib]
if debug:
print ' ', low_edge, hist.bin_contents[ib],
if all_data is not None and hist.all_data is None:
raise Exception('tried to average hists with and without all_data set')
if hist.all_data is not None:
if all_data is None:
all_data = []
all_data += hist.all_data
if debug:
print ''
binlist = sorted(binvals.keys())
meanhist = Hist(len(binlist) - 2, binlist[1], binlist[-1], binlist[1 : -1])
meanhist.all_data = all_data
if debug:
print ' mean',
for ib in range(len(binlist)):
meanhist.set_ibin(ib, binvals[binlist[ib]])
if debug:
print ' ', meanhist.low_edges[ib], meanhist.bin_contents[ib],
if debug:
print ''
meanhist.normalize()
return meanhist
def get_mute_hist(self, mtype):
if self.args.mutate_from_scratch:
mean_mute_val = self.args.scratch_mute_freq
if self.args.same_mute_freq_for_all_seqs:
hist = Hist(1, mean_mute_val - utils.eps,
mean_mute_val + utils.eps)
hist.fill(mean_mute_val)
else:
n_entries = 500
length_vals = [
v
for v in numpy.random.exponential(mean_mute_val, n_entries)
] # count doesn't work on numpy.ndarray objects
max_val = 0.8 # this is arbitrary, but you shouldn't be calling this with anything that gets a significant number anywhere near there, anyway
if length_vals.count(max_val):
print '%s lots of really high mutation rates treegenerator::get_mute_hist()' % utils.color(
'yellow', 'warning')
length_vals = [min(v, max_val) for v in length_vals]
hist = Hist(30, 0., max_val)
for val in length_vals:
hist.fill(val)
hist.normalize()
else:
hist = Hist(fname=self.parameter_dir + '/' + mtype +
'-mean-mute-freqs.csv')
return hist
def test_general_project():
"""
Test general project -- whether Hist can be projected properly.
"""
h = Hist(
axis.Regular(50,
-5,
5,
name="A",
label="a [units]",
underflow=False,
overflow=False),
axis.Boolean(name="B", label="b [units]"),
axis.Variable(range(11), name="C", label="c [units]"),
axis.Integer(0, 10, name="D", label="d [units]"),
axis.IntCategory(range(10), name="E", label="e [units]"),
axis.StrCategory("FT", name="F", label="f [units]"),
)
# via indices
assert h.project()
assert h.project(0, 1)
assert h.project(0, 1, 2, 3, 4, 5)
# via names
assert h.project()
assert h.project("A", "B")
assert h.project("A", "B", "C", "D", "E", "F")
h = Hist(
axis.Regular(50,
-5,
5,
name="A",
label="a [units]",
underflow=False,
overflow=False),
axis.Boolean(name="B", label="b [units]"),
axis.Variable(range(11), name="C", label="c [units]"),
axis.Integer(0, 10, name="D", label="d [units]"),
axis.IntCategory(range(10), name="E", label="e [units]"),
axis.StrCategory("FT", name="F", label="f [units]"),
)
# duplicated
with pytest.raises(Exception):
h.project(0, 0)
with pytest.raises(Exception):
h.project("A", "A")
with pytest.raises(Exception):
h.project(0, "A")
# mixed types
assert h.project(2, "A")
# cannot found
with pytest.raises(Exception):
h.project(-1, 9)
with pytest.raises(Exception):
h.project("G", "H")
def test_general_transform_proxy():
"""
Test general transform proxy -- whether Hist transform proxy works properly.
"""
h0 = Hist.new.Sqrt(3, 4, 25).Sqrt(4, 25, 81).Double()
h0.fill([5, 10, 17, 17], [26, 37, 50, 65])
assert h0[0, 0] == 1
assert h0[1, 1] == 1
assert h0[2, 2] == 1
assert h0[2, 3] == 1
# wrong value
with pytest.raises(Exception):
Hist.new.Sqrt(3, -4, 25)
h1 = Hist.new.Log(4, 1, 10_000).Log(3, 1 / 1_000, 1).Double()
h1.fill([2, 11, 101, 1_001], [1 / 999, 1 / 99, 1 / 9, 1 / 9])
assert h1[0, 0] == 1
assert h1[1, 1] == 1
assert h1[2, 2] == 1
assert h1[3, 2] == 1
# Missing arguments
with pytest.raises(Exception):
Hist.new.Regular(4, 1, 10_000).Log()
# wrong value
with pytest.raises(Exception):
Hist.new.Log(3, -1, 10_000)
h2 = Hist.new.Pow(24, 1, 5, power=2).Pow(124, 1, 5, power=3).Int64()
h2.fill([1, 2, 3, 4], [1, 2, 3, 4])
assert h2[0, 0] == 1
assert h2[3, 7] == 1
assert h2[8, 26] == 1
assert h2[15, 63] == 1
# wrong value
with pytest.raises(Exception):
Hist.new.Regular(24, 1, 5).Pow(2)
# wrong value
with pytest.raises(Exception):
Hist.new.Pow(24, -1, 5, power=0.5)
# lack args
with pytest.raises(Exception):
Hist.new.Pow(24, 1, 5)
ftype = ctypes.CFUNCTYPE(ctypes.c_double, ctypes.c_double)
h3 = (Hist.new.Func(4,
1,
5,
forward=ftype(math.log),
inverse=ftype(math.exp)).Func(4,
1,
5,
forward=ftype(np.log),
inverse=ftype(
np.exp))).Int64()
h3.fill([1, 2, 3, 4], [1, 2, 3, 4])
assert h3[0, 0] == 1
assert h3[1, 1] == 1
assert h3[2, 2] == 1
assert h3[3, 3] == 1
# wrong value
with pytest.raises(Exception):
Hist().Regular(24, 1, 5).Func(ftype(math.log), ftype(math.exp))
# wrong value
assert Hist.new.Func(4,
-1,
5,
forward=ftype(math.log),
inverse=ftype(math.log)).Double()
with pytest.raises(Exception):
Hist.new.Func(4, -1, 5, forward=ftype(np.log), inverse=ftype(np.log))
# lack args
with pytest.raises(Exception):
Hist.new.Func(4, 1, 5)
def test_general_fill():
"""
Test general fill -- whether Hist can be properly filled.
"""
# Regular
h = Hist(
axis.Regular(10, 0, 1, name="x"),
axis.Regular(10, 0, 1, name="y"),
axis.Regular(2, 0, 2, name="z"),
).fill(
x=[0.35, 0.35, 0.35, 0.45, 0.55, 0.55, 0.55],
y=[0.35, 0.35, 0.45, 0.45, 0.45, 0.45, 0.45],
z=[0, 0, 1, 1, 1, 1, 1],
)
z_one_only = h[{2: bh.loc(1)}]
for idx_x in range(0, 10):
for idx_y in range(0, 10):
if idx_x == 3 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 1
elif idx_x == 4 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 1
elif idx_x == 5 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 3
else:
assert z_one_only[idx_x, idx_y] == 0
# Boolean
h = Hist(
axis.Boolean(name="x"),
axis.Boolean(name="y"),
axis.Boolean(name="z"),
).fill(
[True, True, True, True, True, False, True],
[False, True, True, False, False, True, False],
[False, False, True, True, True, True, True],
)
z_one_only = h[{2: bh.loc(True)}]
assert z_one_only[False, False] == 0
assert z_one_only[False, True] == 1
assert z_one_only[True, False] == 3
assert z_one_only[True, True] == 1
# Variable
h = Hist(
axis.Variable(range(11), name="x"),
axis.Variable(range(11), name="y"),
axis.Variable(range(3), name="z"),
).fill(
x=[3.5, 3.5, 3.5, 4.5, 5.5, 5.5, 5.5],
y=[3.5, 3.5, 4.5, 4.5, 4.5, 4.5, 4.5],
z=[0, 0, 1, 1, 1, 1, 1],
)
z_one_only = h[{2: bh.loc(1)}]
for idx_x in range(0, 10):
for idx_y in range(0, 10):
if idx_x == 3 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 1
elif idx_x == 4 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 1
elif idx_x == 5 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 3
else:
assert z_one_only[idx_x, idx_y] == 0
# Integer
h = Hist(
axis.Integer(0, 10, name="x"),
axis.Integer(0, 10, name="y"),
axis.Integer(0, 2, name="z"),
).fill(
[3.5, 3.5, 3.5, 4.5, 5.5, 5.5, 5.5],
[3.5, 3.5, 4.5, 4.5, 4.5, 4.5, 4.5],
[0, 0, 1, 1, 1, 1, 1],
)
z_one_only = h[{2: bh.loc(1)}]
for idx_x in range(0, 10):
for idx_y in range(0, 10):
if idx_x == 3 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 1
elif idx_x == 4 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 1
elif idx_x == 5 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 3
else:
assert z_one_only[idx_x, idx_y] == 0
# IntCategory
h = Hist(
axis.IntCategory(range(10), name="x"),
axis.IntCategory(range(10), name="y"),
axis.IntCategory(range(2), name="z"),
).fill(
x=[3.5, 3.5, 3.5, 4.5, 5.5, 5.5, 5.5],
y=[3.5, 3.5, 4.5, 4.5, 4.5, 4.5, 4.5],
z=[0, 0, 1, 1, 1, 1, 1],
)
z_one_only = h[{2: bh.loc(1)}]
for idx_x in range(0, 10):
for idx_y in range(0, 10):
if idx_x == 3 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 1
elif idx_x == 4 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 1
elif idx_x == 5 and idx_y == 4:
assert z_one_only[idx_x, idx_y] == 3
else:
assert z_one_only[idx_x, idx_y] == 0
# StrCategory
h = Hist(
axis.StrCategory("FT", name="x"),
axis.StrCategory(list("FT"), name="y"),
axis.StrCategory(["F", "T"], name="z"),
).fill(
["T", "T", "T", "T", "T", "F", "T"],
["F", "T", "T", "F", "F", "T", "F"],
["F", "F", "T", "T", "T", "T", "T"],
)
z_one_only = h[{2: bh.loc("T")}]
assert z_one_only[bh.loc("F"), bh.loc("F")] == 0
assert z_one_only[bh.loc("F"), bh.loc("T")] == 1
assert z_one_only[bh.loc("T"), bh.loc("F")] == 3
assert z_one_only[bh.loc("T"), bh.loc("T")] == 1
# with names
assert Hist(axis.Regular(50, -3, 3, name="x"),
axis.Regular(50, -3, 3, name="y")).fill(x=np.random.randn(10),
y=np.random.randn(10))
assert Hist(axis.Boolean(name="x"),
axis.Boolean(name="y")).fill(x=[True, False, True],
y=[True, False, True])
assert Hist(axis.Variable(range(-3, 3), name="x"),
axis.Variable(range(-3, 3),
name="y")).fill(x=np.random.randn(10),
y=np.random.randn(10))
assert Hist(axis.Integer(-3, 3, name="x"),
axis.Integer(-3, 3, name="y")).fill(x=np.random.randn(10),
y=np.random.randn(10))
assert Hist(
axis.IntCategory(range(-3, 3), name="x"),
axis.IntCategory(range(-3, 3), name="y"),
).fill(x=np.random.randn(10), y=np.random.randn(10))
assert Hist(axis.StrCategory(["F", "T"], name="x"),
axis.StrCategory("FT", name="y")).fill(x=["T", "F", "T"],
y=["T", "F", "T"])
h = Hist(
axis.Regular(50,
-4,
4,
name="X",
label="s [units]",
underflow=False,
overflow=False)).fill(np.random.normal(size=10))
def test_hist_proxy():
"""
Test general hist proxy -- whether Hist hist proxy works properly.
"""
h = Hist.new.Reg(10, 0, 1, name="x").Double().fill([0.5, 0.5])
assert h[0.5j] == 2
assert type(h) == Hist
with pytest.raises(AttributeError):
Hist().new
h = (Hist.new.Reg(10, 0, 1,
name="x").Reg(10, 0, 1,
name="y").Double().fill([0.5, 0.5],
[0.2, 0.6]))
assert h[0.5j, 0.2j] == 1
assert h[bh.loc(0.5), bh.loc(0.6)] == 1
h = Hist.new.Bool(name="x").Double().fill([True, True])
assert h[bh.loc(True)] == 2
h = Hist.new.Bool(name="x").Bool(name="y").Int64().fill([True, True],
[True, False])
assert h[True, True] == 1
assert h[True, False] == 1
h = Hist.new.Var(range(10), name="x").Double().fill([5, 5])
assert h[5j] == 2
h = (Hist.new.Var(range(10), name="x").Var(range(10),
name="y").Double().fill([5, 5],
[2, 6]))
assert h[5j, 2j] == 1
assert h[bh.loc(5), bh.loc(6)] == 1
h = Hist.new.Int(0, 10, name="x").Int64().fill([5, 5])
assert h[5j] == 2
h = Hist.new.Int(0, 10, name="x").Int(0, 10, name="y").Int64().fill([5, 5],
[2, 6])
assert h[5j, 2j] == 1
assert h[bh.loc(5), bh.loc(6)] == 1
h = Hist.new.IntCat(range(10), name="x").Double().fill([5, 5])
assert h[5j] == 2
h = (Hist.new.IntCat(range(10),
name="x").IntCat(range(10),
name="y").Double().fill([5, 5],
[2, 6]))
assert h[5j, 2j] == 1
assert h[bh.loc(5), bh.loc(6)] == 1
h = Hist.new.StrCat("TF", name="x").Int64().fill(["T", "T"])
assert h["T"] == 2
h = (Hist.new.StrCat("TF",
name="x").StrCat("TF",
name="y").Int64().fill(["T", "T"],
["T", "F"]))
assert h["T", "T"] == 1
assert h["T", "F"] == 1
def elaborate(self, platform):
# VGA constants
pixel_f = self.timing.pixel_freq
hsync_front_porch = self.timing.h_front_porch
hsync_pulse_width = self.timing.h_sync_pulse
hsync_back_porch = self.timing.h_back_porch
vsync_front_porch = self.timing.v_front_porch
vsync_pulse_width = self.timing.v_sync_pulse
vsync_back_porch = self.timing.v_back_porch
# Pins
clk25 = platform.request("clk25")
ov7670 = platform.request("ov7670")
led = [platform.request("led", i) for i in range(8)]
leds = Cat([i.o for i in led])
led8_2 = platform.request("led8_2")
leds8_2 = Cat([led8_2.leds[i] for i in range(8)])
led8_3 = platform.request("led8_3")
leds8_3 = Cat([led8_3.leds[i] for i in range(8)])
leds16 = Cat(leds8_3, leds8_2)
btn1 = platform.request("button_fire", 0)
btn2 = platform.request("button_fire", 1)
up = platform.request("button_up", 0)
down = platform.request("button_down", 0)
pwr = platform.request("button_pwr", 0)
left = platform.request("button_left", 0)
right = platform.request("button_right", 0)
sw = Cat([platform.request("switch",i) for i in range(4)])
uart = platform.request("uart")
divisor = int(platform.default_clk_frequency // 460800)
esp32 = platform.request("esp32_spi")
csn = esp32.csn
sclk = esp32.sclk
copi = esp32.copi
cipo = esp32.cipo
m = Module()
# Clock generator.
m.domains.sync = cd_sync = ClockDomain("sync")
m.domains.pixel = cd_pixel = ClockDomain("pixel")
m.domains.shift = cd_shift = ClockDomain("shift")
m.submodules.ecp5pll = pll = ECP5PLL()
pll.register_clkin(clk25, platform.default_clk_frequency)
pll.create_clkout(cd_sync, platform.default_clk_frequency)
pll.create_clkout(cd_pixel, pixel_f)
pll.create_clkout(cd_shift, pixel_f * 5.0 * (1.0 if self.ddr else 2.0))
# Add CamRead submodule
camread = CamRead()
m.submodules.camread = camread
# Camera config
cam_x_res = 640
cam_y_res = 480
camconfig = CamConfig()
m.submodules.camconfig = camconfig
# Connect the camera pins and config and read modules
m.d.comb += [
ov7670.cam_RESET.eq(1),
ov7670.cam_PWON.eq(0),
ov7670.cam_XCLK.eq(clk25.i),
ov7670.cam_SIOC.eq(camconfig.sioc),
ov7670.cam_SIOD.eq(camconfig.siod),
camconfig.start.eq(btn1),
camread.p_data.eq(Cat([ov7670.cam_data[i] for i in range(8)])),
camread.href.eq(ov7670.cam_HREF),
camread.vsync.eq(ov7670.cam_VSYNC),
camread.p_clock.eq(ov7670.cam_PCLK)
]
# Create the uart
m.submodules.serial = serial = AsyncSerial(divisor=divisor, pins=uart)
# Input fifo
fifo_depth=1024
m.submodules.fifo = fifo = SyncFIFOBuffered(width=16,depth=fifo_depth)
# Frame buffer
x_res= cam_x_res // 2
y_res= cam_y_res
buffer = Memory(width=16, depth=x_res * y_res)
m.submodules.r = r = buffer.read_port()
m.submodules.w = w = buffer.write_port()
# Button debouncers
m.submodules.debup = debup = Debouncer()
m.submodules.debdown = debdown = Debouncer()
m.submodules.debosd = debosd = Debouncer()
m.submodules.debsel = debsel = Debouncer()
m.submodules.debsnap = debsnap = Debouncer()
m.submodules.debhist = debhist = Debouncer()
# Connect the buttons to debouncers
m.d.comb += [
debup.btn.eq(up),
debdown.btn.eq(down),
debosd.btn.eq(pwr),
debsel.btn.eq(right),
debsnap.btn.eq(left),
debhist.btn.eq(btn2)
]
# Image processing options
flip = Signal(2, reset=1)
mono = Signal(reset=0)
invert = Signal(reset=0)
gamma = Signal(reset=0)
border = Signal(reset=0)
filt = Signal(reset=0)
grid = Signal(reset=0)
histo = Signal(reset=1)
hbin = Signal(6, reset=0)
bin_cnt = Signal(5, reset=0)
thresh = Signal(reset=0)
threshold = Signal(8, reset=0)
hist_chan = Signal(2, reset=0)
ccc = CC(reset=(0,18,12,16))
sharpness = Signal(unsigned(4), reset=0)
osd_val = Signal(4, reset=0) # Account for spurious start-up button pushes
osd_on = Signal(reset=1)
osd_sel = Signal(reset=1)
snap = Signal(reset=0)
frozen = Signal(reset=1)
writing = Signal(reset=0)
written = Signal(reset=0)
byte = Signal(reset=0)
w_addr = Signal(18)
# Color filter
l = Rgb565(reset=(18,12,6)) # Initialised to red LEGO filter
h = Rgb565(reset=(21,22,14))
# Region of interest
roi = Roi()
# VGA signals
vga_r = Signal(8)
vga_g = Signal(8)
vga_b = Signal(8)
vga_hsync = Signal()
vga_vsync = Signal()
vga_blank = Signal()
# Fifo co-ordinates
f_x = Signal(9)
f_y = Signal(9)
f_frame_done = Signal()
# Pixel from fifo
pix = Rgb565()
# SPI memory for remote configuration
m.submodules.spimem = spimem = SpiMem(addr_bits=32)
# Color Control
m.submodules.cc = cc = ColorControl()
# Image convolution
m.submodules.imc = imc = ImageConv()
# Statistics
m.submodules.stats = stats = Stats()
# Histogram
m.submodules.hist = hist = Hist()
# Filter
m.submodules.fil = fil = Filt()
# Monochrome
m.submodules.mon = mon = Mono()
# Sync the fifo with the camera
sync_fifo = Signal(reset=0)
with m.If(~sync_fifo & ~fifo.r_rdy & (camread.col == cam_x_res - 1) & (camread.row == cam_y_res -1)):
m.d.sync += [
sync_fifo.eq(1),
f_x.eq(0),
f_y.eq(0)
]
with m.If(btn1):
m.d.sync += sync_fifo.eq(0)
# Connect the fifo
m.d.comb += [
fifo.w_en.eq(camread.pixel_valid & camread.col[0] & sync_fifo), # Only write every other pixel
fifo.w_data.eq(camread.pixel_data),
fifo.r_en.eq(fifo.r_rdy & ~imc.o_stall)
]
# Calculate fifo co-ordinates
m.d.sync += f_frame_done.eq(0)
with m.If(fifo.r_en & sync_fifo):
m.d.sync += f_x.eq(f_x + 1)
with m.If(f_x == x_res - 1):
m.d.sync += [
f_x.eq(0),
f_y.eq(f_y + 1)
]
with m.If(f_y == y_res - 1):
m.d.sync += [
f_y.eq(0),
f_frame_done.eq(1)
]
# Extract pixel from fifo data
m.d.comb += [
pix.r.eq(fifo.r_data[11:]),
pix.g.eq(fifo.r_data[5:11]),
pix.b.eq(fifo.r_data[:5])
]
# Connect color control
m.d.comb += [
cc.i.eq(pix),
cc.i_cc.eq(ccc)
]
# Calculate per-frame statistics, after applying color correction
m.d.comb += [
stats.i.eq(cc.o),
stats.i_valid.eq(fifo.r_rdy),
# This is not valid when a region of interest is active
stats.i_avg_valid.eq((f_x >= 32) & (f_x < 288) &
(f_y >= 112) & (f_y < 368)),
stats.i_frame_done.eq(f_frame_done),
stats.i_x.eq(f_x),
stats.i_y.eq(f_y),
stats.i_roi.eq(roi)
]
# Produce histogram, after applying color correction, and after monochrome, for monochrome histogram
with m.Switch(hist_chan):
with m.Case(0):
m.d.comb += hist.i_p.eq(cc.o.r)
with m.Case(1):
m.d.comb += hist.i_p.eq(cc.o.g)
with m.Case(2):
m.d.comb += hist.i_p.eq(cc.o.b)
with m.Case(3):
m.d.comb += hist.i_p.eq(mon.o_m)
m.d.comb += [
hist.i_valid.eq(fifo.r_rdy),
hist.i_clear.eq(f_frame_done),
hist.i_x.eq(f_x),
hist.i_y.eq(f_y),
hist.i_roi.eq(roi),
hist.i_bin.eq(hbin) # Used when displaying histogram
]
# Apply filter, after color correction
m.d.comb += [
fil.i.eq(cc.o),
fil.i_valid.eq(fifo.r_en),
fil.i_en.eq(filt),
fil.i_frame_done.eq(f_frame_done),
fil.i_l.eq(l),
fil.i_h.eq(h)
]
# Apply mono, after color correction and filter
m.d.comb += [
mon.i.eq(fil.o),
mon.i_en.eq(mono),
mon.i_invert.eq(invert),
mon.i_thresh.eq(thresh),
mon.i_threshold.eq(threshold)
]
# Apply image convolution, after other transformations
m.d.comb += [
imc.i.eq(mon.o),
imc.i_valid.eq(fifo.r_rdy),
imc.i_reset.eq(~sync_fifo),
# Select image convolution
imc.i_sel.eq(sharpness)
]
# Take a snapshot, freeze the camera, and write the framebuffer to the uart
# Note that this suspends video output
with m.If(debsnap.btn_down | (spimem.wr & (spimem.addr == 22))):
with m.If(frozen):
m.d.sync += frozen.eq(0)
with m.Else():
m.d.sync += [
snap.eq(1),
frozen.eq(0),
w_addr.eq(0),
written.eq(0),
byte.eq(0)
]
# Wait to end of frame after requesting snapshot, before start of writing to uart
with m.If(imc.o_frame_done & snap):
m.d.sync += [
frozen.eq(1),
snap.eq(0)
]
with m.If(~written):
m.d.sync += writing.eq(1)
# Connect the uart
m.d.comb += [
serial.tx.data.eq(Mux(byte, r.data[8:], r.data[:8])),
serial.tx.ack.eq(writing)
]
# Write to the uart from frame buffer (affects video output)
with m.If(writing):
with m.If(w_addr == x_res * y_res):
m.d.sync += [
writing.eq(0),
written.eq(1)
]
with m.Elif(serial.tx.ack & serial.tx.rdy):
m.d.sync += byte.eq(~byte)
with m.If(byte):
m.d.sync += w_addr.eq(w_addr+1)
# Connect spimem
m.d.comb += [
spimem.csn.eq(~csn),
spimem.sclk.eq(sclk),
spimem.copi.eq(copi),
cipo.eq(spimem.cipo),
]
# Writable configuration registers
spi_wr_vals = Array([ccc.brightness, ccc.redness, ccc.greenness, ccc.blueness, l.r, h.r, l.g, h.g, l.b, h.b,
sharpness, filt, border, mono, invert, grid, histo,
roi.x[1:], roi.y[1:], roi.w[1:], roi.h[1:], roi.en, None, None, None,
threshold, thresh, hist_chan, flip,
None, None, None, None, None, None, None, None, None,
frozen])
with m.If(spimem.wr):
with m.Switch(spimem.addr):
for i in range(len(spi_wr_vals)):
if spi_wr_vals[i] is not None:
with m.Case(i):
m.d.sync += spi_wr_vals[i].eq(spimem.dout)
# Readable configuration registers
spi_rd_vals = Array([ccc.brightness, ccc.redness, ccc.greenness, ccc.blueness, l.r, h.r, l.g, h.g, l.b, h.b,
sharpness, filt, border, mono, invert, grid, histo,
roi.x[1:], roi.y[1:], roi.w[1:], roi.h[1:], roi.en, fil.o_nz[16:], fil.o_nz[8:16], fil.o_nz[:8],
threshold, thresh, hist_chan, flip,
stats.o_min.r, stats.o_min.g, stats.o_min.b,
stats.o_max.r, stats.o_max.g, stats.o_max.b,
stats.o_avg.r, stats.o_avg.g, stats.o_avg.b,
frozen, writing, written])
with m.If(spimem.rd):
with m.Switch(spimem.addr):
for i in range(len(spi_rd_vals)):
with m.Case(i):
m.d.sync += spimem.din.eq(spi_rd_vals[i])
# Add VGA generator
m.submodules.vga = vga = VGA(
resolution_x = self.timing.x,
hsync_front_porch = hsync_front_porch,
hsync_pulse = hsync_pulse_width,
hsync_back_porch = hsync_back_porch,
resolution_y = self.timing.y,
vsync_front_porch = vsync_front_porch,
vsync_pulse = vsync_pulse_width,
vsync_back_porch = vsync_back_porch,
bits_x = 16, # Play around with the sizes because sometimes
bits_y = 16 # a smaller/larger value will make it pass timing.
)
# Fetch histogram for display
old_x = Signal(10)
m.d.sync += old_x.eq(vga.o_beam_x)
with m.If(vga.o_beam_x == 0):
m.d.sync += [
hbin.eq(0),
bin_cnt.eq(0)
]
with m.Elif(vga.o_beam_x != old_x):
m.d.sync += bin_cnt.eq(bin_cnt+1)
with m.If(bin_cnt == 19):
m.d.sync += [
bin_cnt.eq(0),
hbin.eq(hbin+1)
]
# Switch between camera and histogram view
with m.If(debhist.btn_down):
m.d.sync += histo.eq(~histo)
# Connect frame buffer, with optional x and y flip
x = Signal(10)
y = Signal(9)
m.d.comb += [
w.en.eq(imc.o_valid & ~frozen),
w.addr.eq(imc.o_y * x_res + imc.o_x),
w.data.eq(Cat(imc.o.b, imc.o.g, imc.o.r)),
y.eq(Mux(flip[1], y_res - 1 - vga.o_beam_y, vga.o_beam_y)),
x.eq(Mux(flip[0], x_res - 1 - vga.o_beam_x[1:], vga.o_beam_x[1:])),
r.addr.eq(Mux(writing, w_addr, y * x_res + x))
]
# Apply the On-Screen Display (OSD)
m.submodules.osd = osd = OSD()
hist_col = Signal(8)
m.d.comb += [
osd.x.eq(vga.o_beam_x),
osd.y.eq(vga.o_beam_y),
hist_col.eq(Mux((479 - osd.y) < hist.o_val[8:], 0xff, 0x00)),
osd.i_r.eq(Mux(histo, Mux((hist_chan == 0) | (hist_chan == 3), hist_col, 0), Cat(Const(0, unsigned(3)), r.data[11:16]))),
osd.i_g.eq(Mux(histo, Mux((hist_chan == 1) | (hist_chan == 3), hist_col, 0), Cat(Const(0, unsigned(2)), r.data[5:11]))),
osd.i_b.eq(Mux(histo, Mux((hist_chan == 2) | (hist_chan == 3), hist_col, 0), Cat(Const(0, unsigned(3)), r.data[0:5]))),
osd.on.eq(osd_on),
osd.osd_val.eq(osd_val),
osd.sel.eq(osd_sel),
osd.grid.eq(grid),
osd.border.eq(border),
osd.roi.eq(roi.en & ~histo),
osd.roi_x.eq(roi.x),
osd.roi_y.eq(roi.y),
osd.roi_w.eq(roi.w),
osd.roi_h.eq(roi.h)
]
# OSD control
osd_vals = Array([ccc.brightness, ccc.redness, ccc.greenness, ccc.blueness, mono, flip[0], flip[1],
border, sharpness, invert, grid, filt])
with m.If(debosd.btn_down):
m.d.sync += osd_on.eq(~osd_on)
with m.If(osd_on):
with m.If(debsel.btn_down):
m.d.sync += osd_sel.eq(~osd_sel)
with m.If(debup.btn_down):
with m.If(~osd_sel):
m.d.sync += osd_val.eq(Mux(osd_val == 0, 11, osd_val-1))
with m.Else():
with m.Switch(osd_val):
for i in range(len(osd_vals)):
with m.Case(i):
if (len(osd_vals[i]) == 1):
m.d.sync += osd_vals[i].eq(1)
else:
m.d.sync += osd_vals[i].eq(osd_vals[i]+1)
with m.If(debdown.btn_down):
with m.If(~osd_sel):
m.d.sync += osd_val.eq(Mux(osd_val == 11, 0, osd_val+1))
with m.Else():
with m.Switch(osd_val):
for i in range(len(osd_vals)):
with m.Case(i):
if (len(osd_vals[i]) == 1):
m.d.sync += osd_vals[i].eq(0)
else:
m.d.sync += osd_vals[i].eq(osd_vals[i]-1)
# Show configuration values on leds
with m.Switch(osd_val):
for i in range(len(osd_vals)):
with m.Case(i):
m.d.comb += leds.eq(osd_vals[i])
# Generate VGA signals
m.d.comb += [
vga.i_clk_en.eq(1),
vga.i_test_picture.eq(0),
vga.i_r.eq(osd.o_r),
vga.i_g.eq(osd.o_g),
vga.i_b.eq(osd.o_b),
vga_r.eq(vga.o_vga_r),
vga_g.eq(vga.o_vga_g),
vga_b.eq(vga.o_vga_b),
vga_hsync.eq(vga.o_vga_hsync),
vga_vsync.eq(vga.o_vga_vsync),
vga_blank.eq(vga.o_vga_blank),
]
# VGA to digital video converter.
tmds = [Signal(2) for i in range(4)]
m.submodules.vga2dvid = vga2dvid = VGA2DVID(ddr=self.ddr, shift_clock_synchronizer=False)
m.d.comb += [
vga2dvid.i_red.eq(vga_r),
vga2dvid.i_green.eq(vga_g),
vga2dvid.i_blue.eq(vga_b),
vga2dvid.i_hsync.eq(vga_hsync),
vga2dvid.i_vsync.eq(vga_vsync),
vga2dvid.i_blank.eq(vga_blank),
tmds[3].eq(vga2dvid.o_clk),
tmds[2].eq(vga2dvid.o_red),
tmds[1].eq(vga2dvid.o_green),
tmds[0].eq(vga2dvid.o_blue),
]
# GPDI pins
if (self.ddr):
# Vendor specific DDR modules.
# Convert SDR 2-bit input to DDR clocked 1-bit output (single-ended)
# onboard GPDI.
m.submodules.ddr0_clock = Instance("ODDRX1F",
i_SCLK = ClockSignal("shift"),
i_RST = 0b0,
i_D0 = tmds[3][0],
i_D1 = tmds[3][1],
o_Q = self.o_gpdi_dp[3])
m.submodules.ddr0_red = Instance("ODDRX1F",
i_SCLK = ClockSignal("shift"),
i_RST = 0b0,
i_D0 = tmds[2][0],
i_D1 = tmds[2][1],
o_Q = self.o_gpdi_dp[2])
m.submodules.ddr0_green = Instance("ODDRX1F",
i_SCLK = ClockSignal("shift"),
i_RST = 0b0,
i_D0 = tmds[1][0],
i_D1 = tmds[1][1],
o_Q = self.o_gpdi_dp[1])
m.submodules.ddr0_blue = Instance("ODDRX1F",
i_SCLK = ClockSignal("shift"),
i_RST = 0b0,
i_D0 = tmds[0][0],
i_D1 = tmds[0][1],
o_Q = self.o_gpdi_dp[0])
else:
m.d.comb += [
self.o_gpdi_dp[3].eq(tmds[3][0]),
self.o_gpdi_dp[2].eq(tmds[2][0]),
self.o_gpdi_dp[1].eq(tmds[1][0]),
self.o_gpdi_dp[0].eq(tmds[0][0]),
]
return m
def get_cluster_size_hist(partition):
sizes = [len(c) for c in partition]
hist = Hist(max(sizes), 0.5, max(sizes) + 0.5)
for sz in sizes:
hist.fill(sz)
return hist