def sortMatrix(self, sort_using="mean", sort_method="no"):
# sort the matrix using the average of values per row
self.matrixAvgsDict = OrderedDict()
self.lengthDict = OrderedDict()
for label in self.matrixDict.keys():
if sort_method != "no":
if sort_using == "region_length":
matrixAvgs = np.array([x["end"] - x["start"] for x in self.regionsDict[label]])
b = self.parameters["upstream"] / self.parameters["bin size"]
# for plotting I add the upstream
# distance
self.lengthDict[label] = b + (matrixAvgs / self.parameters["bin size"])
else:
matrixAvgs = np.__getattribute__(sort_using)(self.matrixDict[label], axis=1)
SS = matrixAvgs.argsort()
if sort_method == "descend":
SS = SS[::-1]
self.matrixDict[label] = self.matrixDict[label][SS, :]
self.regionsDict[label] = self.regionsDict[label][SS]
self.matrixAvgsDict[label] = matrixAvgs[SS]
try:
self.lengthDict[label] = self.lengthDict[label][SS]
except:
self.lengthDict[label] = None
def sort_groups(self, sort_using='mean', sort_method='no'):
"""
Sorts and rearranges the submatrices according to the
sorting method given.
"""
if sort_method == 'no':
return
# compute the row average:
if sort_using == 'region_length':
matrix_avgs = np.array([x['end'] - x['start']
for x in self.regions])
else:
matrix_avgs = np.__getattribute__(sort_using)(
self.matrix, axis=1)
# order per group
_sorted_regions = []
_sorted_matrix = []
for idx in range(len(self.group_labels)):
start = self.group_boundaries[idx]
end = self.group_boundaries[idx + 1]
order = matrix_avgs[start:end].argsort()
if sort_method == 'descend':
order = order[::-1]
_sorted_matrix.append(self.matrix[start:end, :][order, :])
# sort the regions
_reg = self.regions[start:end]
for idx in order:
_sorted_regions.append(_reg[idx])
self.matrix = np.vstack(_sorted_matrix)
self.regions = _sorted_regions
self.set_sorting_method(sort_method, sort_using)
def add_sniff_events(h5, overwrite=True, plot=False, centering_fn='mean', *args, **kwargs):
"""
:param h5:
:param overwrite:
:param plot:
:param centering_fn: (optional) Str, ('mean', 'median'. This function name will be used to compute the offset of the
sniff stream. Other usual value would be 'median'.
:param args:
:param kwargs:
:return:
"""
assert isinstance(h5, tb.File)
events_group = h5.root.events
try:
sniff_stream_tbarray = h5.root.streams.sniff
except tb.NoSuchNodeError:
print('Warning, no sniff stream found.')
return
try:
sniff_events_group = h5.create_group('/events', 'sniff')
except tb.NodeError as e:
if overwrite:
h5.remove_node('/events/sniff', recursive=True)
h5.flush()
sniff_events_group = h5.create_group('/events', 'sniff')
else:
raise SniffExistExemption
logging.info('Creating sniff events.')
sniff = sniff_stream_tbarray[:, 0]
sniff -= np.__getattribute__(centering_fn)(sniff) # subtract the mean or median from the sniff signal.
fs_stream = sniff_stream_tbarray._v_attrs['sample_rate_Hz']
fs_events = events_group._v_attrs['sample_rate_Hz']
sniff_events_array = find_inhalations_simple(sniff, fs_stream)
sniff_events_array *= (fs_events/fs_stream)
sniff_events_array = sniff_events_array.astype(np.int)
sniff_events = h5.create_carray(sniff_events_group, 'inh_events', obj=sniff_events_array)
sniff_stats = basic_sniff_stats(sniff_events_array, sniff, h5, fs_stream, fs_events)
sniff_log = np.zeros(sniff.size, dtype=np.bool)
for ev in sniff_events:
ev2 = ev / 16
sniff_log[ev2[0]:ev2[1]] = True
m = np.max(sniff[:10000])
plt.plot(sniff[:10000])
plt.plot(sniff_log[:10000] * m)
plt.grid()
# plt.plot(plt.xlim(), [sniff.mean()]*2)
plt.show()
return
def __getattr__(self,attr):
""" Allow numpy overloading, called if something else is broken
A bit hackish, but avoids poluting the namespace for the Chebfun """
if attr in NP_OVERLOAD:
ufunc = np.__getattribute__(attr)
def func():
return self._new_func( lambda x: ufunc( self._eval(x) ) )
func.__name__ = attr
func.__doc__ = "wraps numpy ufunc: {}".format(attr)
return func
return self.__getattribute__(attr)
def myAverage(valuesArray, avgType="mean"):
"""
computes the mean, median, etc but only for those values
that are not Nan
"""
valuesArray = np.ma.masked_invalid(valuesArray)
avg = np.__getattribute__(avgType)(valuesArray)
if isinstance(avg, np.ma.core.MaskedConstant):
return np.nan
else:
return avg
def register_numpy_types():
"""Register the AsIs adapter for following types from numpy:
- numpy.int8
- numpy.int16
- numpy.int32
- numpy.int64
- numpy.float16
- numpy.float32
- numpy.float64
- numpy.float128
"""
for typ in ['int8', 'int16', 'int32', 'int64',
'float16', 'float32', 'float64', 'float128']:
register_adapter(np.__getattribute__(typ), AsIs)
def reduce_axis(self, op_name, arr, axis, keepdims, transposed):
op_func = np.__getattribute__(op_name)
if transposed:
arr = arr.T
if (axis is not None and arr.shape[axis] > 1 and op_name == "sum"
and not keepdims):
# We can optimize this with matmul or tensordot.
if len(arr.shape) == 2:
assert axis in (0, 1)
if axis == 0:
return np.matmul(np.ones(arr.shape[0]), arr)
else:
return np.matmul(arr, np.ones(arr.shape[1]))
else:
return np.tensordot(np.ones(arr.shape[axis]),
arr,
axes=(0, axis))
return op_func(arr, axis=axis, keepdims=keepdims)
def bop(self, op, a1, a2, a1_shape, a2_shape, a1_T, a2_T, axes):
if a1_T:
a1 = a1.T
if a2_T:
a2 = a2.T
if a1.shape != a1_shape:
a1 = a1.reshape(a1_shape)
if a2.shape != a2_shape:
a2 = a2.reshape(a2_shape)
if op == "tensordot":
return np.tensordot(a1, a2, axes=axes)
op = np_ufunc_map.get(op, op)
try:
ufunc = np.__getattribute__(op)
except Exception as _:
ufunc = scipy.special.__getattribute__(op)
return ufunc(a1, a2)
def compute_statistic(statistic, array):
"""Compute the statistic used in a Dakota response function.
Parameters
----------
statistic : str
A string with the name of the statistic to compute ('mean',
'median', etc.).
array : array_like
An array data structure, such as a numpy array.
Returns
-------
float
The value of the computed statistic.
"""
import numpy as np
return np.__getattribute__(statistic)(array)
def arange(self, start_in, shape, block_shape, step=1, dtype=None) -> BlockArray:
assert step == 1
if dtype is None:
dtype = np.__getattribute__(
str(np.result_type(start_in, shape[0] + start_in))
)
# Generate ranges per block.
grid = ArrayGrid(shape, block_shape, dtype.__name__)
rarr = BlockArray(grid, self.cm)
for _, grid_entry in enumerate(grid.get_entry_iterator()):
syskwargs = {"grid_entry": grid_entry, "grid_shape": grid.grid_shape}
start = start_in + block_shape[0] * grid_entry[0]
entry_shape = grid.get_block_shape(grid_entry)
stop = start + entry_shape[0]
rarr.blocks[grid_entry].oid = self.cm.arange(
start, stop, step, dtype, syskwargs=syskwargs
)
return rarr
def wrapped(*args, **kwargs):
warnings.warn(
"Operation "
+ func.__name__
+ " not implemented, falling back to NumPy. "
+ "If this is too slow or failing, please open an issue on GitHub.",
RuntimeWarning,
)
new_args = [arg.get() if isinstance(arg, BlockArray) else arg for arg in args]
new_kwargs = {
k: v.get() if isinstance(v, BlockArray) else v
for k, v in zip(kwargs.keys(), kwargs.values())
}
res = np.__getattribute__(func.__name__)(*new_args, **new_kwargs)
if isinstance(res, tuple):
nps_res = tuple(array(x) for x in res)
else:
nps_res = array(res)
return nps_res
def check_basic_broadcast_correctness(
_np_a, _np_b, _a_blockshape=None, _b_blockshape=None
):
ns_a = nps.array(_np_a)
ns_b = nps.array(_np_b)
if _a_blockshape:
ns_a = ns_a.reshape(block_shape=_a_blockshape)
if _b_blockshape:
ns_b = ns_b.reshape(block_shape=_b_blockshape)
for _op in _ops:
np_op = np.__getattribute__(_op)
ns_op = nps.__getattribute__(_op)
_np_result = np_op(_np_a, _np_b)
_ns_result = ns_op(ns_a, ns_b)
assert np.allclose(_np_result, _ns_result.get())
def test_doctest_fallback(nps_app_inst):
import nums.numpy as nps
assert nps_app_inst is not None
passed = []
failed = []
excepted = []
plot_funcs = {
"kaiser", "bartlett", "hanning", "blackman", "histogram2d", "interp",
"sinc"
}
for func in settings.doctest_fallbacks:
nps_func = nps.__getattribute__(func)
nps_func.__doc__ = update_doc_string(np.__getattribute__(func).__doc__)
f = io.StringIO()
with redirect_stdout(f):
nps_func = nps.__getattribute__(func)
if func in plot_funcs:
# Skip plot functions and add them to the failed list
print("Failure")
else:
optionflags = doctest.NORMALIZE_WHITESPACE | doctest.FAIL_FAST
doctest.run_docstring_examples(nps_func,
locals(),
optionflags=optionflags)
if f.getvalue() == "":
passed.append(func)
else:
failed.append(func)
print("***DOCTESTS***")
print("PASSED:")
print(sorted(passed))
print("FAILED:")
print(sorted(failed))
print("EXCEPTED:")
print(sorted(excepted))
def print_values_for_attributes_for_keys(dataset, attributes, keys, index=0):
if type(attributes) is not list:
attributes = [attributes]
if type(index) is not list:
index = [index]
for key in keys:
trajec = dataset.trajecs[key]
print_str = key
for a, attr in enumerate(attributes):
attribute = trajec.__getattribute__(attr)
if type(attribute) in [str, float, int, long, np.float64]:
print_str += ' -- ' + attr + ': ' + str(attribute)
else:
if type(index[a]) is int:
print_str += ' -- ' + attr + ': ' + str(attribute[index[a]])
elif type(index[a]) is str:
print_str += ' -- ' + attr + ': ' + str(np.__getattribute__(index[a])(attribute))
else:
print_str += ' -- ' + attr + ': ' + str(attribute)
print print_str
def bop_reduce(self, op, a1, a2, a1_T, a2_T):
if a1_T:
a1 = a1.T
if a2_T:
a2 = a2.T
# These are faster.
if op == "sum":
r = a1 + a2
elif op == "prod":
r = a1 * a2
else:
reduce_op = np.__getattribute__(op)
a = np.stack([a1, a2], axis=0)
r = reduce_op(a, axis=0, keepdims=False)
if a1 is np.nan or a2 is np.nan or r is np.nan:
assert np.isscalar(a1) and np.isscalar(a2) and np.isscalar(r)
else:
assert a1.shape == a2.shape == r.shape
return r
def check_bitwise_error(_name, error):
np_ufunc = np.__getattribute__(_name)
ns_ufunc = nps.__getattribute__(_name)
_np_a = np.random.randn(2, 2)
_np_b = np.random.randn(2, 2)
_ns_a = nps.array(_np_a).touch()
_ns_b = nps.array(_np_b).touch()
message = ""
# Catching expected error message to ensure same error message is raised below.
try:
_np_result = np_ufunc(_np_a, _np_b)
except TypeError as err:
message = err.args[0]
assert message
with pytest.raises(error, match=message):
_ns_result = ns_ufunc(_ns_a, _ns_b)
def array(object, dtype=None, copy=True, order="K", ndmin=0, subok=False) -> BlockArray:
if order is not None and order != "K":
raise NotImplementedError("Only order='K' is supported.")
if ndmin != 0:
raise NotImplementedError("Only ndmin=0 is currently supported.")
if subok:
raise ValueError("subok must be False.")
if isinstance(object, BlockArray):
if copy:
object = object.copy()
if dtype is not None:
if dtype is not object.dtype:
object = object.astype(dtype)
return object
result = np.array(
object, dtype=dtype, copy=copy, order=order, ndmin=ndmin, subok=subok
)
dtype = np.__getattribute__(str(result.dtype))
shape = result.shape
app = _instance()
block_shape = app.compute_block_shape(shape, dtype)
return app.array(result, block_shape)
def test_add_sub_overload(ccd_data, operand, expect_failure, with_uncertainty,
operation, affects_uncertainty):
if with_uncertainty:
ccd_data.uncertainty = StdDevUncertainty(np.ones_like(ccd_data))
method = ccd_data.__getattribute__(operation)
np_method = np.__getattribute__(operation)
if expect_failure:
with pytest.raises(expect_failure):
result = method(operand)
return
else:
result = method(operand)
assert result is not ccd_data
assert isinstance(result, CCDData)
assert (result.uncertainty is None or
isinstance(result.uncertainty, StdDevUncertainty))
try:
op_value = operand.value
except AttributeError:
op_value = operand
np.testing.assert_array_equal(result.data,
np_method(ccd_data.data, op_value))
if with_uncertainty:
if affects_uncertainty:
np.testing.assert_array_equal(result.uncertainty.array,
np_method(ccd_data.uncertainty.array,
op_value))
else:
np.testing.assert_array_equal(result.uncertainty.array,
ccd_data.uncertainty.array)
else:
assert result.uncertainty is None
if isinstance(operand, u.Quantity):
assert (result.unit == ccd_data.unit and result.unit == operand.unit)
else:
assert result.unit == ccd_data.unit
def test_add_sub_overload(ccd_data, operand, expect_failure, with_uncertainty,
operation, affects_uncertainty):
if with_uncertainty:
ccd_data.uncertainty = StdDevUncertainty(np.ones_like(ccd_data))
method = ccd_data.__getattribute__(operation)
np_method = np.__getattribute__(operation)
if expect_failure:
with pytest.raises(expect_failure):
result = method(operand)
return
else:
result = method(operand)
assert result is not ccd_data
assert isinstance(result, CCDData)
assert (result.uncertainty is None
or isinstance(result.uncertainty, StdDevUncertainty))
try:
op_value = operand.value
except AttributeError:
op_value = operand
np.testing.assert_array_equal(result.data,
np_method(ccd_data.data, op_value))
if with_uncertainty:
if affects_uncertainty:
np.testing.assert_array_equal(
result.uncertainty.array,
np_method(ccd_data.uncertainty.array, op_value))
else:
np.testing.assert_array_equal(result.uncertainty.array,
ccd_data.uncertainty.array)
else:
assert result.uncertainty is None
if isinstance(operand, u.Quantity):
assert (result.unit == ccd_data.unit and result.unit == operand.unit)
else:
assert result.unit == ccd_data.unit
def read_dicom_by_path(case_dir):
"""read the dicom files in the directory"""
file_names = os.listdir(case_dir)
slices = []
for name in file_names:
try:
dcm = pydicom.read_file(case_dir + '/' + name)
except:
continue
rows = dcm.Rows
cols = dcm.Columns
padding = dcm.PixelPaddingValue
dtype = np.__getattribute__('int' + str(dcm.BitsStored))
slc = np.frombuffer(dcm.PixelData, dtype=dtype)
slc = np.reshape(slc, [rows, cols])
slices.append(slc)
if slices == []:
return None
else:
slices = np.array(slices)
return slices
def test_mult_div_overload(ccd_data, operand, with_uncertainty,
operation, affects_uncertainty):
if with_uncertainty:
ccd_data.uncertainty = StdDevUncertainty(np.ones_like(ccd_data))
method = ccd_data.__getattribute__(operation)
np_method = np.__getattribute__(operation)
result = method(operand)
assert result is not ccd_data
assert isinstance(result, CCDData)
assert (result.uncertainty is None or
isinstance(result.uncertainty, StdDevUncertainty))
try:
op_value = operand.value
except AttributeError:
op_value = operand
np.testing.assert_array_equal(result.data,
np_method(ccd_data.data, op_value))
if with_uncertainty:
if affects_uncertainty:
np.testing.assert_array_equal(result.uncertainty.array,
np_method(ccd_data.uncertainty.array,
op_value))
else:
np.testing.assert_array_equal(result.uncertainty.array,
ccd_data.uncertainty.array)
else:
assert result.uncertainty is None
if isinstance(operand, u.Quantity):
# Need the "1 *" below to force arguments to be Quantity to work around
# astropy/astropy#2377
expected_unit = np_method(1 * ccd_data.unit, 1 * operand.unit).unit
assert result.unit == expected_unit
else:
assert result.unit == ccd_data.unit
def test_mult_div_overload(ccd_data, operand, with_uncertainty, operation,
affects_uncertainty):
if with_uncertainty:
ccd_data.uncertainty = StdDevUncertainty(np.ones_like(ccd_data))
method = ccd_data.__getattribute__(operation)
np_method = np.__getattribute__(operation)
result = method(operand)
assert result is not ccd_data
assert isinstance(result, CCDData)
assert (result.uncertainty is None
or isinstance(result.uncertainty, StdDevUncertainty))
try:
op_value = operand.value
except AttributeError:
op_value = operand
np.testing.assert_array_equal(result.data,
np_method(ccd_data.data, op_value))
if with_uncertainty:
if affects_uncertainty:
np.testing.assert_array_equal(
result.uncertainty.array,
np_method(ccd_data.uncertainty.array, op_value))
else:
np.testing.assert_array_equal(result.uncertainty.array,
ccd_data.uncertainty.array)
else:
assert result.uncertainty is None
if isinstance(operand, u.Quantity):
# Need the "1 *" below to force arguments to be Quantity to work around
# astropy/astropy#2377
expected_unit = np_method(1 * ccd_data.unit, 1 * operand.unit).unit
assert result.unit == expected_unit
else:
assert result.unit == ccd_data.unit
def test_reductions(nps_app_inst):
from nums import numpy as nps
from nums.numpy import BlockArray
assert nps_app_inst is not None
ba: BlockArray = nps.array([[5, -2, 4, 8], [3, 6, 1, 7]])
block_shapes = [(1, 1), (1, 2), (1, 4), (2, 1), (2, 4)]
for block_shape in block_shapes:
ba = ba.reshape(block_shape=block_shape)
np_arr = ba.get()
op_params = ["amin", "min", "amax", "max", "sum", "mean", "var", "std"]
axis_params = [None, 0, 1]
keepdims_params = [True, False]
for op, axis, keepdims in itertools.product(op_params, axis_params,
keepdims_params):
ns_op = nps.__getattribute__(op)
np_op = np.__getattribute__(op)
np_result = np_op(np_arr, axis=axis, keepdims=keepdims)
ba_result: BlockArray = ns_op(ba, axis=axis, keepdims=keepdims)
assert ba_result.grid.grid_shape == ba_result.blocks.shape
assert ba_result.shape == np_result.shape
assert np.allclose(ba_result.get(), np_result)
def to_dtype_cls(dtype):
if hasattr(dtype, "__name__"):
return dtype
return np.__getattribute__(str(dtype))
def map_uop(self, op_name, arr, args, kwargs):
ufunc = np.__getattribute__(op_name)
return ufunc(arr, *args, **kwargs)
def reduce_axis(self, op_name, arr, axis, keepdims, transposed):
op_func = np.__getattribute__(op_name)
if transposed:
arr = arr.T
return op_func(arr, axis=axis, keepdims=keepdims)
def test_average(nps_app_inst):
from nums import numpy as nps
from nums.numpy import BlockArray
assert nps_app_inst is not None
bas = [
nps.array([
[[5, -2, 4, 8], [1, 2, 3, 4], [3, 2, 1, 0], [-1, -2, -3, 0]],
[[6, -4, 2, 7], [4, 3, 2, 1], [1, 2, 0, 3], [0, -2, -3, -1]],
]),
nps.array([
[[5, -2, 4, 8], [1, 2, 3, 4], [3, 2, 1, 0], [-1, -2, -3, 0]],
[[6, -4, 2, 7], [4, 3, 2, 1], [1, 2, 0, 3], [0, -2, -3, -1]],
]),
]
ba_wts = [
None,
nps.array([
[[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]],
[
[-2, -3, -4, -5],
[-2, -3, -4, -5],
[-2, -3, -4, -5],
[-2, -3, -4, -5],
],
]),
]
ba_shapes = [(1, 1, 1), (1, 2, 2), (2, 1, 4), (2, 4, 2), (2, 4, 4)]
for ba, ba_wt in zip(bas, ba_wts):
for ba_shape in ba_shapes:
ba = ba.reshape(block_shape=ba_shape)
if ba_wt:
ba_wt = ba_wt.reshape(block_shape=ba_shape)
np_arr = ba.get()
np_wt = ba_wt.get() if ba_wt else None
op_params = ["average"]
axis_params = [None, 0, 1, 2]
for op, axis in itertools.product(op_params, axis_params):
ns_op = nps.__getattribute__(op)
np_op = np.__getattribute__(op)
np_result = np_op(np_arr,
axis=axis,
weights=np_wt,
returned=True)
ba_result: BlockArray = ns_op(ba,
axis=axis,
weights=ba_wt,
returned=True)
np_avg, np_ws = np_result[0], np_result[1]
ba_avg, ba_ws = ba_result[0], ba_result[1]
assert np.allclose(ba_ws.get(), np_ws)
assert ba_avg.grid.grid_shape == ba_avg.blocks.shape
assert ba_avg.shape == np_avg.shape
assert np.allclose(ba_avg.get(), np_avg)
# Test zero division error
ba = nps.array([
[[5, -2, 4, 8], [1, 2, 3, 4], [3, 2, 1, 0], [-1, -2, -3, 0]],
[[6, -4, 2, 7], [4, 3, 2, 1], [1, 2, 0, 3], [0, -2, -3, -1]],
])
ba_wt = nps.array([
[[1, 2, 3, 4], [1, 2, 3, -12], [1, 2, 3, 4], [1, 2, 3, 4]], # axis 1
[
[-2, -3, 10, -5], # axis 2
[-2, -3, -4, -5],
[-2, -2, -4, -5], # axis 0
[-2, -3, -4, -5],
],
])
for ax in range(3):
err_match = True
try:
np.average(ba.get(), axis=ax, weights=ba_wt.get(), returned=False)
err_match = False
except ZeroDivisionError:
pass
try:
nps.average(ba, axis=ax, weights=ba_wt, returned=False)
err_match = False
except ZeroDivisionError:
pass
assert err_match
from nums.core.array.sparseblockarray import SparseBlockArray
from nums.core.array.blockarray import BlockArray
w = h = 400
sparsity = int(w * h / 3)
arr = np.zeros((w, h))
ind = random.sample(range(w * h), sparsity)
ind = [(i % w, i // w) for i in ind]
for i in ind:
arr[i] = np.random.randint(0, 100)
dtype = np.__getattribute__(str(arr.dtype))
shape = arr.shape
app = _instance()
block_shape = app.compute_block_shape(shape, dtype)
sparse_result = SparseBlockArray.from_np(arr,
block_shape=block_shape,
copy=False,
system=app.system)
dense_result = BlockArray.from_np(arr,
block_shape=block_shape,
copy=False,
system=app.system)
funcs = [
lambda x: x @ x,
def plot_heatmap(self):
matrix_flatten = None
if self.y_min is None:
matrix_flatten = self.hm.matrix.flatten()
# try to avoid outliers by using np.percentile
self.y_min = np.percentile(matrix_flatten, 1.0)
if np.isnan(self.y_min):
self.y_min = None
if self.y_max is None:
if matrix_flatten is None:
matrix_flatten = self.hm.matrix.flatten()
# try to avoid outliers by using np.percentile
self.y_max = np.percentile(matrix_flatten, 98.0)
if np.isnan(self.y_max):
self.y_max = None
ax_list = []
# turn off y ticks
for plot in range(self.numplots):
labels = []
col = plot % self.plots_per_row
row = int(plot / self.plots_per_row)
# split the ax to make room for the colorbar
sub_grid = gridspec.GridSpecFromSubplotSpec(1, 2, subplot_spec=self.grids[row, col],
width_ratios=[0.92, 0.08], wspace=0.05)
ax = self.fig.add_subplot(sub_grid[0])
cax = self.fig.add_subplot(sub_grid[1])
ax.tick_params(
axis='y',
which='both',
left='off',
right='off',
labelleft='on')
if self.per_group:
title = self.hm.matrix.group_labels[plot]
else:
title = self.hm.matrix.sample_labels[plot]
ax.set_title(title)
mat = [] # when drawing a heatmap (in contrast to drawing lines)
for data_idx in range(self.numlines):
if self.per_group:
row, col = plot, data_idx
else:
row, col = data_idx, plot
sub_matrix = self.hm.matrix.get_matrix(row, col)
if self.per_group:
label = sub_matrix['sample']
else:
label = sub_matrix['group']
labels.append(label)
mat.append(np.__getattribute__(self.averagetype)(sub_matrix['matrix'], axis=0))
img = ax.imshow(np.vstack(mat), interpolation='nearest',
cmap='RdYlBu_r', aspect='auto', vmin=self.y_min, vmax=self.y_max)
self.fig.colorbar(img, cax=cax)
ax.axes.set_xticks(self.xticks)
ax.axes.set_xticklabels(self.xtickslabel)
# align the first and last label
# such that they don't fall off
# the heatmap sides
ticks = ax.xaxis.get_major_ticks()
ticks[0].label1.set_horizontalalignment('left')
ticks[-1].label1.set_horizontalalignment('right')
# add labels as y ticks labels
ymin, ymax = ax.axes.get_ylim()
pos, distance = np.linspace(ymin, ymax, len(labels), retstep=True, endpoint=False)
d_half = float(distance) / 2
yticks = [x + d_half for x in pos]
ax.axes.set_yticks(yticks)
ax.axes.set_yticklabels(labels[::-1], rotation='vertical')
ax_list.append(ax)
plt.subplots_adjust(wspace=0.05, hspace=0.3)
plt.tight_layout()
plt.savefig(self.out_file_name, dpi=200, format=self.image_format)
plt.close()
def matrixAvg(matrix, avgType="mean"):
matrix = np.ma.masked_invalid(matrix)
return np.__getattribute__(avgType)(matrix, axis=0)
def plot_single(ax, ma, average_type, color, label, plot_type='simple'):
"""
Adds a line to the plot in the given ax using the specified method
Parameters
----------
ax : matplotlib axis
matplotlib axis
ma : numpy array
numpy array The data on this matrix is summarized according
to the `average_type` argument.
average_type : str
string values are sum mean median min max std
color : str
a valid color: either a html color name, hex
(e.g #002233), RGB + alpha tuple or list or RGB tuple or list
label : str
label
plot_type: str
type of plot. Either 'se' for standard error, 'std' for
standard deviation, 'overlapped_lines' to plot each line of the matrix,
fill to plot the area between the x axis and the value or None, just to
plot the average line.
Returns
-------
ax
matplotlib axis
Examples
--------
>>> import matplotlib.pyplot as plt
>>> import os
>>> fig = plt.figure()
>>> ax = fig.add_subplot(111)
>>> matrix = np.array([[1,2,3],
... [4,5,6],
... [7,8,9]])
>>> ax = plot_single(ax, matrix -2, 'mean', color=[0.6, 0.8, 0.9], label='fill light blue', plot_type='fill')
>>> ax = plot_single(ax, matrix, 'mean', color='blue', label='red')
>>> ax = plot_single(ax, matrix + 5, 'mean', color='red', label='red', plot_type='std')
>>> ax = plot_single(ax, matrix + 10, 'mean', color='#cccccc', label='gray se', plot_type='se')
>>> ax = plot_single(ax, matrix + 20, 'mean', color=(0.9, 0.5, 0.9), label='violet', plot_type='std')
>>> ax = plot_single(ax, matrix + 30, 'mean', color=(0.9, 0.5, 0.9, 0.5), label='violet with alpha', plot_type='std')
>>> leg = ax.legend()
>>> plt.savefig("/tmp/test.pdf")
>>> plt.close()
>>> fig = plt.figure()
>>> os.remove("/tmp/test.pdf")
"""
summary = np.__getattribute__(average_type)(ma, axis=0)
# only plot the average profiles without error regions
x = np.arange(len(summary))
ax.plot(x, summary, color=color, label=label, alpha=0.9)
if plot_type == 'fill':
pass
ax.fill_between(x,
summary,
facecolor=color,
alpha=0.6,
edgecolor='none')
if plot_type in ['se', 'std']:
if plot_type == 'se': # standard error
std = np.std(ma, axis=0) / np.sqrt(ma.shape[0])
else:
std = np.std(ma, axis=0)
alpha = 0.2
# an alpha channel has to be added to the color to fill the area
# between the mean (or median etc.) and the std or se
f_color = pltcolors.colorConverter.to_rgba(color, alpha)
ax.fill_between(x,
summary,
summary + std,
facecolor=f_color,
edgecolor='none')
ax.fill_between(x,
summary,
summary - std,
facecolor=f_color,
edgecolor='none')
ax.set_xlim(0, max(x))
return ax
qlist.append(build_sql_regex(f[0],f[1]))
#prepend the repo_name and path for key information
qlist = ['repo_name','path'] + qlist
return '\n'.join(['SELECT',',\n'.join(qlist),f'FROM {content_table}'])
def build_countAPIs(api_list,api_table='None'):
return 'SELECT\n'+',\n'.join([f'count(CASE WHEN {build_cname(*f)} THEN 1 END) AS {build_cname(*f)}_count' for f in api_list]) + f'\nFROM {api_table}'
if __name__ == '__main__':
import os
os.environ['GOOGLE_APPLICATION_CREDENTIALS']='/Users/mpeaton/Downloads/apt-footing-235018-aeb185ac9e31.json'
api = [(x, type(np.__getattribute__(x))) for x in dir(np) if not x.startswith('__')]
apq = API_QUERY_FACTORY(api)
with open('numpy_api_search.sql', 'w') as f:
f.write('#legacySQL\n')
f.write(apq.query)
with open('numpy_api_count.sql','w') as f:
f.write('#legacySQL\n')
f.write(apq.count_query)
def build_import_query():
"""
Construct a first pass query to detect all of the numpy imports
"""
def plot_single(ax, ma, average_type, color, label, plot_type='simple'):
"""
Adds a line to the plot in the given ax using the specified method
Parameters
----------
ax : matplotlib axis
matplotlib axis
ma : numpy array
numpy array The data on this matrix is summarized according
to the `average_type` argument.
average_type : str
string values are sum mean median min max std
color : str
a valid color: either a html color name, hex
(e.g #002233), RGB + alpha tuple or list or RGB tuple or list
label : str
label
plot_type: str
type of plot. Either 'se' for standard error, 'std' for
standard deviation, 'overlapped_lines' to plot each line of the matrix,
fill to plot the area between the x axis and the value or None, just to
plot the average line.
Returns
-------
ax
matplotlib axis
Examples
--------
>>> import matplotlib.pyplot as plt
>>> fig = plt.figure()
>>> ax = fig.add_subplot(111)
>>> matrix = np.array([[1,2,3],
... [4,5,6],
... [7,8,9]])
>>> ax = plot_single(ax, matrix -2, 'mean', color=[0.6, 0.8, 0.9], label='fill light blue', plot_type='fill')
>>> ax = plot_single(ax, matrix, 'mean', color='blue', label='red')
>>> ax = plot_single(ax, matrix + 5, 'mean', color='red', label='red', plot_type='std')
>>> ax =plot_single(ax, matrix + 10, 'mean', color='#cccccc', label='gray se', plot_type='se')
>>> ax = plot_single(ax, matrix + 20, 'mean', color=(0.9, 0.5, 0.9), label='violet', plot_type='std')
>>> ax = plot_single(ax, matrix + 30, 'mean', color=(0.9, 0.5, 0.9, 0.5), label='violet with alpha', plot_type='std')
>>> leg = ax.legend()
>>> plt.savefig("/tmp/test.pdf")
>>> fig = plt.figure()
"""
summary = np.__getattribute__(average_type)(ma, axis=0)
# only plot the average profiles without error regions
x = np.arange(len(summary))
ax.plot(x, summary, color=color, label=label, alpha=0.9)
if plot_type == 'fill':
pass
ax.fill_between(x, summary, facecolor=color, alpha=0.6, edgecolor='none')
if plot_type in ['se', 'std']:
if plot_type == 'se': # standard error
std = np.std(ma, axis=0) / np.sqrt(ma.shape[0])
else:
std = np.std(ma, axis=0)
alpha = 0.2
# an alpha channel has to be added to the color to fill the area
# between the mean (or median etc.) and the std or se
f_color = pltcolors.colorConverter.to_rgba(color, alpha)
ax.fill_between(x, summary, summary + std, facecolor=f_color, edgecolor='none')
ax.fill_between(x, summary, summary - std, facecolor=f_color, edgecolor='none')
ax.set_xlim(0, max(x))
return ax
def from_modin(df):
# pylint: disable = import-outside-toplevel, protected-access, unidiomatic-typecheck
try:
from modin.pandas.dataframe import DataFrame
from modin.engines.ray.pandas_on_ray.frame.data import PandasOnRayFrame
from modin.engines.ray.pandas_on_ray.frame.partition import (
PandasOnRayFramePartition, )
except Exception as e:
raise Exception(
"Unable to import modin. Install modin with command 'pip install modin'"
) from e
assert isinstance(
df, DataFrame), "Unexpected dataframe type %s" % str(type(df))
assert isinstance(df._query_compiler._modin_frame,
PandasOnRayFrame), "Unexpected dataframe type %s" % str(
type(df._query_compiler._modin_frame))
frame: PandasOnRayFrame = df._query_compiler._modin_frame
app: ArrayApplication = _instance()
system = app.cm
# Make sure the partitions are numeric.
dtype = frame.dtypes[0]
assert dtype in (
float,
np.float,
np.float32,
np.float64,
int,
np.int,
np.int32,
np.int64,
)
# Make sure dtypes are equal.
for dt in frame.dtypes:
if type(frame.dtypes.dtype) == np.dtype:
continue
assert dt == frame.dtypes
dtype = np.__getattribute__(str(dtype))
# Convert from Pandas to NumPy.
pd_parts = frame._partition_mgr_cls.map_partitions(frame._partitions,
lambda df: np.array(df))
grid_shape = len(frame._row_lengths), len(frame._column_widths)
shape = (np.sum(frame._row_lengths), np.sum(frame._column_widths))
block_shape = app.get_block_shape(shape, dtype)
rows = []
for i in range(grid_shape[0]):
cols = []
for j in range(grid_shape[1]):
curr_block_shape = (frame._row_lengths[i], frame._column_widths[j])
part: PandasOnRayFramePartition = pd_parts[(i, j)]
part.drain_call_queue()
ba: BlockArray = BlockArray.from_oid(part.oid, curr_block_shape,
dtype, system)
cols.append(ba)
if grid_shape[1] == 1:
row_ba: BlockArray = cols[0]
else:
row_ba: BlockArray = app.concatenate(
cols, axis=1, axis_block_size=block_shape[1])
rows.append(row_ba)
result = app.concatenate(rows, axis=0, axis_block_size=block_shape[0])
return result
def ufunc(self, op_name, arr):
ufunc = np.__getattribute__(op_name)
return ufunc(arr)
import pytest
import numpy as np
from numpy_api_stats.numpy_stats import build_numpyAPI_query, build_api_list
_typelist = [
'module', 'function', 'float', 'int', 'ufunc',
'builtin_function_or_method', 'type', 'CClass', 'NoneType', 'PytestTester',
'RClass', 'bool', 'IndexExpression', '_typedict', 'str', 'nd_grid',
'_Feature', 'float', 'dict'
]
_api = [(x, type(np.__getattribute__(x))) for x in dir(np)
if not x.startswith('__')]
@pytest.mark.parametrize('l', [([tt]) for tt in _typelist])
def test_build_query(l):
api_list = build_api_list(_api, l)
q = build_numpyAPI_query(api_list)
assert (q)
def check_dim(_np_a, _ns_a):
np_ndim = np.__getattribute__("ndim")
assert np_ndim(_np_a) == np_ndim(_ns_a)
def check_dim(_np_a, _ns_a):
np_ndim = np.__getattribute__('ndim')
ns_ndim = nps.__getattribute__('ndim')
assert np_ndim(_np_a) == ns_ndim(_ns_a)
def get_reduce_output_type(op_name, dtype):
a = np.array([0, 1], dtype=dtype)
dtype = np.__getattribute__(op_name)(a).dtype
return np.__getattribute__(str(dtype))
def reduce_axis(self, op_name, axis, keepdims=False):
if not (axis is None or isinstance(axis, (int, np.int32, np.int64))):
raise NotImplementedError("Only integer axis is currently supported.")
result_blocks = np.empty_like(self.blocks, dtype=Block)
for grid_entry in self.grid.get_entry_iterator():
result_blocks[grid_entry] = self.blocks[grid_entry].reduce_axis(op_name,
axis,
keepdims=keepdims)
result_shape = []
result_block_shape = []
for curr_axis in range(len(self.shape)):
axis_size, axis_block_size = self.shape[curr_axis], self.block_shape[curr_axis]
if curr_axis == axis or axis is None:
if keepdims:
axis_size, axis_block_size = 1, 1
else:
continue
result_shape.append(axis_size)
result_block_shape.append(axis_block_size)
result_shape = tuple(result_shape)
result_block_shape = tuple(result_block_shape)
result_dtype = array_utils.get_reduce_output_type(op_name, self.dtype)
result_grid = ArrayGrid(shape=result_shape,
block_shape=result_block_shape,
dtype=result_dtype.__name__)
result = BlockArray(result_grid, self.system)
if op_name in settings.np_pairwise_reduction_map:
# Do a pairwise reduction with the pairwise reduction op.
pairwise_op_name = settings.np_pairwise_reduction_map.get(op_name, op_name)
if axis is None:
reduced_block: Block = None
for grid_entry in self.grid.get_entry_iterator():
if reduced_block is None:
reduced_block = result_blocks[grid_entry]
continue
next_block = result_blocks[grid_entry]
reduced_block = reduced_block.bop(pairwise_op_name, next_block, {})
if result.shape == ():
result.blocks[()] = reduced_block
else:
result.blocks[:] = reduced_block
else:
for result_grid_entry in result_grid.get_entry_iterator():
reduced_block: Block = None
for sum_dim in range(self.grid.grid_shape[axis]):
grid_entry = list(result_grid_entry)
if keepdims:
grid_entry[axis] = sum_dim
else:
grid_entry = grid_entry[:axis] + [sum_dim] + grid_entry[axis:]
grid_entry = tuple(grid_entry)
next_block: Block = result_blocks[grid_entry]
if reduced_block is None:
reduced_block = next_block
else:
reduced_block = reduced_block.bop(pairwise_op_name, next_block, {})
result.blocks[result_grid_entry] = reduced_block
else:
op_func = np.__getattribute__(op_name)
if result.shape == ():
result.blocks[()] = op_func(result_blocks, axis=axis, keepdims=keepdims)
else:
result.blocks = op_func(result_blocks, axis=axis, keepdims=keepdims)
return result
def get_uop_output_type(op_name, dtype):
a = np.array(1, dtype=dtype)
result_dtype = np.__getattribute__(op_name)(a).dtype
return to_dtype_cls(result_dtype)
def plot_heatmap(self):
matrix_flatten = None
if self.y_min is None:
matrix_flatten = self.hm.matrix.flatten()
# try to avoid outliers by using np.percentile
self.y_min = np.percentile(matrix_flatten, 1.0)
if np.isnan(self.y_min):
self.y_min = None
if self.y_max is None:
if matrix_flatten is None:
matrix_flatten = self.hm.matrix.flatten()
# try to avoid outliers by using np.percentile
self.y_max = np.percentile(matrix_flatten, 98.0)
if np.isnan(self.y_max):
self.y_max = None
ax_list = []
# turn off y ticks
for plot in range(self.numplots):
labels = []
col = plot % self.plots_per_row
row = int(plot / self.plots_per_row)
# split the ax to make room for the colorbar
sub_grid = gridspec.GridSpecFromSubplotSpec(
1,
2,
subplot_spec=self.grids[row, col],
width_ratios=[0.92, 0.08],
wspace=0.05)
ax = self.fig.add_subplot(sub_grid[0])
cax = self.fig.add_subplot(sub_grid[1])
ax.tick_params(axis='y',
which='both',
left='off',
right='off',
labelleft='on')
if self.per_group:
title = self.hm.matrix.group_labels[plot]
else:
title = self.hm.matrix.sample_labels[plot]
ax.set_title(title)
mat = [] # when drawing a heatmap (in contrast to drawing lines)
for data_idx in range(self.numlines):
if self.per_group:
row, col = plot, data_idx
else:
row, col = data_idx, plot
sub_matrix = self.hm.matrix.get_matrix(row, col)
if self.per_group:
label = sub_matrix['sample']
else:
label = sub_matrix['group']
labels.append(label)
mat.append(
np.__getattribute__(self.averagetype)(sub_matrix['matrix'],
axis=0))
img = ax.imshow(np.vstack(mat),
interpolation='nearest',
cmap='jet',
aspect='auto',
vmin=self.y_min,
vmax=self.y_max)
self.fig.colorbar(img, cax=cax)
ax.axes.set_xticks(self.xticks)
ax.axes.set_xticklabels(self.xtickslabel)
# align the first and last label
# such that they don't fall off
# the heatmap sides
ticks = ax.xaxis.get_major_ticks()
ticks[0].label1.set_horizontalalignment('left')
ticks[-1].label1.set_horizontalalignment('right')
# add labels as y ticks labels
ymin, ymax = ax.axes.get_ylim()
pos, distance = np.linspace(ymin,
ymax,
len(labels),
retstep=True,
endpoint=False)
d_half = float(distance) / 2
yticks = [x + d_half for x in pos]
ax.axes.set_yticks(yticks)
ax.axes.set_yticklabels(labels)
ax_list.append(ax)
plt.subplots_adjust(wspace=0.05, hspace=0.3)
plt.tight_layout()
plt.savefig(self.out_file_name, dpi=200, format=self.image_format)
def _arithmetic(lhs, rhs, op):
return transform(np.__getattribute__(op), [lhs, rhs])
def save_tabulated_values(self, file_handle, reference_point_label='TSS', start_label='TSS', end_label='TES', averagetype='mean'):
"""
Saves the values averaged by col using the avg_type
given
Args:
file_handle: file name to save the file
reference_point_label: Name of the reference point label
start_label: Name of the star label
end_label: Name of the end label
averagetype: average type (e.g. mean, median, std)
"""
# get X labels
w = self.parameters['bin size']
b = self.parameters['upstream']
a = self.parameters['downstream']
c = self.parameters.get('unscaled 5 prime', 0)
d = self.parameters.get('unscaled 3 prime', 0)
m = self.parameters['body']
if b < 1e5:
quotient = 1000
symbol = 'Kb'
else:
quotient = 1e6
symbol = 'Mb'
if m == 0:
xticks = [(k / w) for k in [w, b, b + a]]
xtickslabel = ['{0:.1f}{1}'.format(-(float(b) / quotient), symbol), reference_point_label,
'{0:.1f}{1}'.format(float(a) / quotient, symbol)]
else:
xticks_values = [w]
xtickslabel = []
# only if upstream region is set, add a x tick
if b > 0:
xticks_values.append(b)
xtickslabel.append('{0:.1f}{1}'.format(-(float(b) / quotient), symbol))
xtickslabel.append(start_label)
if c > 0:
xticks_values.append(b + c)
xtickslabel.append("")
if d > 0:
xticks_values.append(b + c + m)
xtickslabel.append("")
xticks_values.append(b + c + m + d)
xtickslabel.append(end_label)
if a > 0:
xticks_values.append(b + c + m + d + a)
xtickslabel.append('{0:.1f}{1}'.format(float(a) / quotient, symbol))
xticks = [(k / w) for k in xticks_values]
x_axis = np.arange(xticks[-1]) + 1
labs = []
for x_value in x_axis:
if x_value in xticks:
labs.append(xtickslabel[xticks.index(x_value)])
else:
labs.append("")
with open(file_handle, 'w') as fh:
# write labels
fh.write("bin labels\t\t{}\n".format("\t".join(labs)))
fh.write('bins\t\t{}\n'.format("\t".join([str(x) for x in x_axis])))
for sample_idx in range(self.matrix.get_num_samples()):
for group_idx in range(self.matrix.get_num_groups()):
sub_matrix = self.matrix.get_matrix(group_idx, sample_idx)
values = [str(x) for x in np.__getattribute__(averagetype)(sub_matrix['matrix'], axis=0)]
fh.write("{}\t{}\t{}\n".format(sub_matrix['sample'], sub_matrix['group'], "\t".join(values)))
