def histogram(data, range=None, bins=50, legend=None, title=None):
"""Create a histogram.
Parameters
----------
data : :class:`.Struct` or :class:`.Float64Expression`
Sequence of data to plot.
range : Tuple[float]
Range of x values in the histogram.
bins : int
Number of bins in the histogram.
legend : str
Label of data on the x-axis.
title : str
Title of the histogram.
Returns
-------
:class:`bokeh.plotting.figure.Figure`
"""
if isinstance(data, Expression):
if data._indices.source is not None:
agg_f = data._aggregation_method()
if range is not None:
start = range[0]
end = range[1]
else:
finite_data = hail.bind(lambda x: hail.case().when(hail.is_finite(x), x).or_missing(), data)
start, end = agg_f((aggregators.min(finite_data),
aggregators.max(finite_data)))
if start is None and end is None:
raise ValueError(f"'data' contains no values that are defined and finite")
data = agg_f(aggregators.hist(data, start, end, bins))
else:
return ValueError('Invalid input')
p = figure(title=title, x_axis_label=legend, y_axis_label='Frequency', background_fill_color='#EEEEEE')
p.quad(
bottom=0, top=data.bin_freq,
left=data.bin_edges[:-1], right=data.bin_edges[1:],
legend=legend, line_color='black')
if data.n_larger > 0:
p.quad(
bottom=0, top=data.n_larger,
left=data.bin_edges[-1], right=(data.bin_edges[-1] + (data.bin_edges[1] - data.bin_edges[0])),
line_color='black', fill_color='green', legend='Outliers Above')
if data.n_smaller > 0:
p.quad(
bottom=0, top=data.n_smaller,
left=data.bin_edges[0] - (data.bin_edges[1] - data.bin_edges[0]), right=data.bin_edges[0],
line_color='black', fill_color='red', legend='Outliers Below')
return p
def cumulative_histogram(data, range=None, bins=50, legend=None, title=None, normalize=True, log=False):
"""Create a cumulative histogram.
Parameters
----------
data : :class:`.Struct` or :class:`.Float64Expression`
Sequence of data to plot.
range : Tuple[float]
Range of x values in the histogram.
bins : int
Number of bins in the histogram.
legend : str
Label of data on the x-axis.
title : str
Title of the histogram.
normalize: bool
Whether or not the cumulative data should be normalized.
log: bool
Whether or not the y-axis should be of type log.
Returns
-------
:class:`bokeh.plotting.figure.Figure`
"""
if isinstance(data, Expression):
if data._indices.source is not None:
agg_f = data._aggregation_method()
if range is not None:
start = range[0]
end = range[1]
else:
start, end = agg_f((aggregators.min(data), aggregators.max(data)))
data = agg_f(aggregators.hist(data, start, end, bins))
else:
return ValueError('Invalid input')
cumulative_data = np.cumsum(data.bin_freq) + data.n_smaller
np.append(cumulative_data, [cumulative_data[-1] + data.n_larger])
num_data_points = max(cumulative_data)
if normalize:
cumulative_data = cumulative_data / num_data_points
if title is not None:
title = f'{title} ({num_data_points:,} data points)'
if log:
p = figure(title=title, x_axis_label=legend, y_axis_label='Frequency',
background_fill_color='#EEEEEE', y_axis_type='log')
else:
p = figure(title=title, x_axis_label=legend, y_axis_label='Frequency', background_fill_color='#EEEEEE')
p.line(data.bin_edges[:-1], cumulative_data, line_color='#036564', line_width=3)
return p
def cumulative_histogram(data, range=None, bins=50, legend=None, title=None, normalize=True, log=False):
"""Create a cumulative histogram.
Parameters
----------
data : :class:`.Struct` or :class:`.Float64Expression`
Sequence of data to plot.
range : Tuple[float]
Range of x values in the histogram.
bins : int
Number of bins in the histogram.
legend : str
Label of data on the x-axis.
title : str
Title of the histogram.
normalize: bool
Whether or not the cumulative data should be normalized.
log: bool
Whether or not the y-axis should be of type log.
Returns
-------
:class:`bokeh.plotting.figure.Figure`
"""
if isinstance(data, Expression):
if data._indices.source is not None:
agg_f = data._aggregation_method()
if range is not None:
start = range[0]
end = range[1]
else:
start, end = agg_f((aggregators.min(data), aggregators.max(data)))
data = agg_f(aggregators.hist(data, start, end, bins))
else:
return ValueError('Invalid input')
cumulative_data = np.cumsum(data.bin_freq) + data.n_smaller
np.append(cumulative_data, [cumulative_data[-1] + data.n_larger])
num_data_points = max(cumulative_data)
if normalize:
cumulative_data = cumulative_data / num_data_points
if title is not None:
title = f'{title} ({num_data_points:,} data points)'
if log:
p = figure(title=title, x_axis_label=legend, y_axis_label='Frequency',
background_fill_color='#EEEEEE', y_axis_type='log')
else:
p = figure(title=title, x_axis_label=legend, y_axis_label='Frequency', background_fill_color='#EEEEEE')
p.line(data.bin_edges[:-1], cumulative_data, line_color='#036564', line_width=3)
return p
def test_aggregate2(self):
schema = hl.tstruct(status=hl.tint32, GT=hl.tcall, qPheno=hl.tint32)
rows = [{'status': 0, 'GT': hl.Call([0, 0]), 'qPheno': 3},
{'status': 0, 'GT': hl.Call([0, 1]), 'qPheno': 13}]
kt = hl.Table.parallelize(rows, schema)
result = convert_struct_to_dict(
kt.group_by(status=kt.status)
.aggregate(
x1=agg.collect(kt.qPheno * 2),
x2=agg.explode(lambda elt: agg.collect(elt), [kt.qPheno, kt.qPheno + 1]),
x3=agg.min(kt.qPheno),
x4=agg.max(kt.qPheno),
x5=agg.sum(kt.qPheno),
x6=agg.product(hl.int64(kt.qPheno)),
x7=agg.count(),
x8=agg.count_where(kt.qPheno == 3),
x9=agg.fraction(kt.qPheno == 1),
x10=agg.stats(hl.float64(kt.qPheno)),
x11=agg.hardy_weinberg_test(kt.GT),
x13=agg.inbreeding(kt.GT, 0.1),
x14=agg.call_stats(kt.GT, ["A", "T"]),
x15=agg.collect(hl.Struct(a=5, b="foo", c=hl.Struct(banana='apple')))[0],
x16=agg.collect(hl.Struct(a=5, b="foo", c=hl.Struct(banana='apple')).c.banana)[0],
x17=agg.explode(lambda elt: agg.collect(elt), hl.null(hl.tarray(hl.tint32))),
x18=agg.explode(lambda elt: agg.collect(elt), hl.null(hl.tset(hl.tint32))),
x19=agg.take(kt.GT, 1, ordering=-kt.qPheno)
).take(1)[0])
expected = {u'status': 0,
u'x13': {u'n_called': 2, u'expected_homs': 1.64, u'f_stat': -1.777777777777777,
u'observed_homs': 1},
u'x14': {u'AC': [3, 1], u'AF': [0.75, 0.25], u'AN': 4, u'homozygote_count': [1, 0]},
u'x15': {u'a': 5, u'c': {u'banana': u'apple'}, u'b': u'foo'},
u'x10': {u'min': 3.0, u'max': 13.0, u'sum': 16.0, u'stdev': 5.0, u'n': 2, u'mean': 8.0},
u'x8': 1, u'x9': 0.0, u'x16': u'apple',
u'x11': {u'het_freq_hwe': 0.5, u'p_value': 0.5},
u'x2': [3, 4, 13, 14], u'x3': 3, u'x1': [6, 26], u'x6': 39, u'x7': 2, u'x4': 13, u'x5': 16,
u'x17': [],
u'x18': [],
u'x19': [hl.Call([0, 1])]}
self.maxDiff = None
self.assertDictEqual(result, expected)
def test_aggregate2(self):
schema = hl.tstruct(status=hl.tint32, GT=hl.tcall, qPheno=hl.tint32)
rows = [{'status': 0, 'GT': hl.Call([0, 0]), 'qPheno': 3},
{'status': 0, 'GT': hl.Call([0, 1]), 'qPheno': 13}]
kt = hl.Table.parallelize(rows, schema)
result = convert_struct_to_dict(
kt.group_by(status=kt.status)
.aggregate(
x1=agg.collect(kt.qPheno * 2),
x2=agg.explode(lambda elt: agg.collect(elt), [kt.qPheno, kt.qPheno + 1]),
x3=agg.min(kt.qPheno),
x4=agg.max(kt.qPheno),
x5=agg.sum(kt.qPheno),
x6=agg.product(hl.int64(kt.qPheno)),
x7=agg.count(),
x8=agg.count_where(kt.qPheno == 3),
x9=agg.fraction(kt.qPheno == 1),
x10=agg.stats(hl.float64(kt.qPheno)),
x11=agg.hardy_weinberg_test(kt.GT),
x13=agg.inbreeding(kt.GT, 0.1),
x14=agg.call_stats(kt.GT, ["A", "T"]),
x15=agg.collect(hl.Struct(a=5, b="foo", c=hl.Struct(banana='apple')))[0],
x16=agg.collect(hl.Struct(a=5, b="foo", c=hl.Struct(banana='apple')).c.banana)[0],
x17=agg.explode(lambda elt: agg.collect(elt), hl.null(hl.tarray(hl.tint32))),
x18=agg.explode(lambda elt: agg.collect(elt), hl.null(hl.tset(hl.tint32))),
x19=agg.take(kt.GT, 1, ordering=-kt.qPheno)
).take(1)[0])
expected = {u'status': 0,
u'x13': {u'n_called': 2, u'expected_homs': 1.64, u'f_stat': -1.777777777777777,
u'observed_homs': 1},
u'x14': {u'AC': [3, 1], u'AF': [0.75, 0.25], u'AN': 4, u'homozygote_count': [1, 0]},
u'x15': {u'a': 5, u'c': {u'banana': u'apple'}, u'b': u'foo'},
u'x10': {u'min': 3.0, u'max': 13.0, u'sum': 16.0, u'stdev': 5.0, u'n': 2, u'mean': 8.0},
u'x8': 1, u'x9': 0.0, u'x16': u'apple',
u'x11': {u'het_freq_hwe': 0.5, u'p_value': 0.5},
u'x2': [3, 4, 13, 14], u'x3': 3, u'x1': [6, 26], u'x6': 39, u'x7': 2, u'x4': 13, u'x5': 16,
u'x17': [],
u'x18': [],
u'x19': [hl.Call([0, 1])]}
self.maxDiff = None
self.assertDictEqual(result, expected)
def histogram(data, range=None, bins=50, legend=None, title=None, log=False, interactive=False):
"""Create a histogram.
Notes
-----
`data` can be a :class:`.Float64Expression`, or the result of the :func:`.agg.hist`
or :func:`.agg.approx_cdf` aggregators.
Parameters
----------
data : :class:`.Struct` or :class:`.Float64Expression`
Sequence of data to plot.
range : Tuple[float]
Range of x values in the histogram.
bins : int
Number of bins in the histogram.
legend : str
Label of data on the x-axis.
title : str
Title of the histogram.
log : bool
Plot the log10 of the bin counts.
Returns
-------
:class:`bokeh.plotting.figure.Figure`
"""
if isinstance(data, Expression):
if data._indices.source is not None:
if interactive:
raise ValueError("'interactive' flag can only be used on data from 'approx_cdf'.")
agg_f = data._aggregation_method()
if range is not None:
start = range[0]
end = range[1]
else:
finite_data = hail.bind(lambda x: hail.case().when(hail.is_finite(x), x).or_missing(), data)
start, end = agg_f((aggregators.min(finite_data),
aggregators.max(finite_data)))
if start is None and end is None:
raise ValueError(f"'data' contains no values that are defined and finite")
data = agg_f(aggregators.hist(data, start, end, bins))
else:
return ValueError('Invalid input')
elif 'values' in data:
cdf = data
hist, edges = np.histogram(cdf.values, bins=bins, weights=np.diff(cdf.ranks), density=True)
data = Struct(bin_freq=hist, bin_edges=edges, n_larger=0, n_smaller=0)
if log:
data.bin_freq = [log10(x) for x in data.bin_freq]
data.n_larger = log10(data.n_larger)
data.n_smaller = log10(data.n_smaller)
y_axis_label = 'log10 Frequency'
else:
y_axis_label = 'Frequency'
x_span = data.bin_edges[-1] - data.bin_edges[0]
x_start = data.bin_edges[0] - .05 * x_span
x_end = data.bin_edges[-1] + .05 * x_span
p = figure(
title=title,
x_axis_label=legend,
y_axis_label=y_axis_label,
background_fill_color='#EEEEEE',
x_range=(x_start, x_end))
q = p.quad(
bottom=0, top=data.bin_freq,
left=data.bin_edges[:-1], right=data.bin_edges[1:],
legend=legend, line_color='black')
if data.n_larger > 0:
p.quad(
bottom=0, top=data.n_larger,
left=data.bin_edges[-1], right=(data.bin_edges[-1] + (data.bin_edges[1] - data.bin_edges[0])),
line_color='black', fill_color='green', legend='Outliers Above')
if data.n_smaller > 0:
p.quad(
bottom=0, top=data.n_smaller,
left=data.bin_edges[0] - (data.bin_edges[1] - data.bin_edges[0]), right=data.bin_edges[0],
line_color='black', fill_color='red', legend='Outliers Below')
if interactive:
def mk_interact(handle):
def update(bins=bins, phase=0):
if phase > 0 and phase < 1:
bins = bins + 1
delta = (cdf.values[-1] - cdf.values[0]) / bins
edges = np.linspace(cdf.values[0] - (1 - phase) * delta, cdf.values[-1] + phase * delta, bins)
else:
edges = np.linspace(cdf.values[0], cdf.values[-1], bins)
hist, edges = np.histogram(cdf.values, bins=edges, weights=np.diff(cdf.ranks), density=True)
new_data = {'top': hist, 'left': edges[:-1], 'right': edges[1:], 'bottom': np.full(len(hist), 0)}
q.data_source.data = new_data
bokeh.io.push_notebook(handle)
from ipywidgets import interact
interact(update, bins=(0, 5*bins), phase=(0, 1, .01))
return p, mk_interact
else:
return p
def histogram(data, range=None, bins=50, legend=None, title=None, log=False, interactive=False):
"""Create a histogram.
Notes
-----
`data` can be a :class:`.Float64Expression`, or the result of the :func:`.agg.hist`
or :func:`.agg.approx_cdf` aggregators.
Parameters
----------
data : :class:`.Struct` or :class:`.Float64Expression`
Sequence of data to plot.
range : Tuple[float]
Range of x values in the histogram.
bins : int
Number of bins in the histogram.
legend : str
Label of data on the x-axis.
title : str
Title of the histogram.
log : bool
Plot the log10 of the bin counts.
Returns
-------
:class:`bokeh.plotting.figure.Figure`
"""
if isinstance(data, Expression):
if data._indices.source is not None:
if interactive:
raise ValueError("'interactive' flag can only be used on data from 'approx_cdf'.")
agg_f = data._aggregation_method()
if range is not None:
start = range[0]
end = range[1]
else:
finite_data = hail.bind(lambda x: hail.case().when(hail.is_finite(x), x).or_missing(), data)
start, end = agg_f((aggregators.min(finite_data),
aggregators.max(finite_data)))
if start is None and end is None:
raise ValueError(f"'data' contains no values that are defined and finite")
data = agg_f(aggregators.hist(data, start, end, bins))
else:
return ValueError('Invalid input')
elif 'values' in data:
cdf = data
hist, edges = np.histogram(cdf.values, bins=bins, weights=np.diff(cdf.ranks), density=True)
data = Struct(bin_freq=hist, bin_edges=edges, n_larger=0, n_smaller=0)
if log:
data.bin_freq = [log10(x) for x in data.bin_freq]
data.n_larger = log10(data.n_larger)
data.n_smaller = log10(data.n_smaller)
y_axis_label = 'log10 Frequency'
else:
y_axis_label = 'Frequency'
x_span = data.bin_edges[-1] - data.bin_edges[0]
x_start = data.bin_edges[0] - .05 * x_span
x_end = data.bin_edges[-1] + .05 * x_span
p = figure(
title=title,
x_axis_label=legend,
y_axis_label=y_axis_label,
background_fill_color='#EEEEEE',
x_range=(x_start, x_end))
q = p.quad(
bottom=0, top=data.bin_freq,
left=data.bin_edges[:-1], right=data.bin_edges[1:],
legend=legend, line_color='black')
if data.n_larger > 0:
p.quad(
bottom=0, top=data.n_larger,
left=data.bin_edges[-1], right=(data.bin_edges[-1] + (data.bin_edges[1] - data.bin_edges[0])),
line_color='black', fill_color='green', legend='Outliers Above')
if data.n_smaller > 0:
p.quad(
bottom=0, top=data.n_smaller,
left=data.bin_edges[0] - (data.bin_edges[1] - data.bin_edges[0]), right=data.bin_edges[0],
line_color='black', fill_color='red', legend='Outliers Below')
if interactive:
def mk_interact(handle):
def update(bins=bins, phase=0):
if phase > 0 and phase < 1:
bins = bins + 1
delta = (cdf.values[-1] - cdf.values[0]) / bins
edges = np.linspace(cdf.values[0] - (1 - phase) * delta, cdf.values[-1] + phase * delta, bins)
else:
edges = np.linspace(cdf.values[0], cdf.values[-1], bins)
hist, edges = np.histogram(cdf.values, bins=edges, weights=np.diff(cdf.ranks), density=True)
new_data = {'top': hist, 'left': edges[:-1], 'right': edges[1:], 'bottom': np.full(len(hist), 0)}
q.data_source.data = new_data
bokeh.io.push_notebook(handle)
from ipywidgets import interact
interact(update, bins=(0, 5*bins), phase=(0, 1, .01))
return p, mk_interact
else:
return p
