def t_histogram2d_impl(self, **kw: Any) -> None:
s = self.scheduler()
random = RandomTable(3, rows=100000, scheduler=s)
stirrer = Stirrer(update_column="_2", fixed_step_size=1000, scheduler=s, **kw)
stirrer.input[0] = random.output.result
min_ = Min(scheduler=s)
min_.input[0] = stirrer.output.result
max_ = Max(scheduler=s)
max_.input[0] = stirrer.output.result
histogram2d = Histogram2D(
0, 1, xbins=100, ybins=100, scheduler=s
) # columns are called 1..30
histogram2d.input[0] = stirrer.output.result
histogram2d.input.min = min_.output.result
histogram2d.input.max = max_.output.result
heatmap = Heatmap(filename="histo_%03d.png", scheduler=s)
heatmap.input.array = histogram2d.output.result
pr = Every(proc=self.terse, scheduler=s)
pr.input[0] = heatmap.output.result
aio.run(s.start())
last = notNone(histogram2d.table.last()).to_dict()
h1 = last["array"]
bounds = [[last["ymin"], last["ymax"]], [last["xmin"], last["xmax"]]]
t = stirrer.table.loc[:, ["_1", "_2"]]
assert t is not None
v = t.to_array()
bins = [histogram2d.params.ybins, histogram2d.params.xbins]
h2 = fh.histogram2d(v[:, 1], v[:, 0], bins=bins, range=bounds)
h2 = np.flip(h2, axis=0) # type: ignore
self.assertEqual(np.sum(h1), np.sum(h2))
self.assertListEqual(h1.reshape(-1).tolist(), h2.reshape(-1).tolist())
def test_histogram2d1(self) -> None:
s = self.scheduler()
csv = CSVLoader(
get_dataset("bigfile"), index_col=False, header=None, scheduler=s
)
min_ = Min(scheduler=s)
min_.input[0] = csv.output.result
max_ = Max(scheduler=s)
max_.input[0] = csv.output.result
histogram2d = Histogram2D(
1, 2, xbins=100, ybins=100, scheduler=s
) # columns are called 1..30
histogram2d.input[0] = csv.output.result
histogram2d.input.min = min_.output.result
histogram2d.input.max = max_.output.result
heatmap = Heatmap(filename="histo_%03d.png", scheduler=s)
heatmap.input.array = histogram2d.output.result
pr = Every(proc=self.terse, scheduler=s)
pr.input[0] = heatmap.output.result
aio.run(csv.scheduler().start())
last = notNone(histogram2d.table.last()).to_dict()
h1 = last["array"]
bounds = [[last["ymin"], last["ymax"]], [last["xmin"], last["xmax"]]]
df = pd.read_csv(
get_dataset("bigfile"), header=None, usecols=[1, 2] # type: ignore
)
v = df.to_numpy() # .reshape(-1, 2)
bins = [histogram2d.params.ybins, histogram2d.params.xbins]
h2 = fh.histogram2d(v[:, 1], v[:, 0], bins=bins, range=bounds)
h2 = np.flip(h2, axis=0) # type: ignore
self.assertTrue(np.allclose(h1, h2))
def inject_stars(self, x, y, signal, xy_dim, mask=None):
"""
Inject sources into image.
Parameters
----------
x : `~numpy.ndarray`
Source x coordinates.
y : `~numpy.ndarray`
Source y coordinates.
signal : `~numpy.ndarray`
The signal to inject into image (e.g., flux, counts, etc...).
xy_dim : int or list-like
The dimensions of mock image in xy units. If `int` is given, it is
assumed to be both the x and y dimensions: (xy_dim, xy_dim).
mask : `~numpy.ndarray`, optional
Boolean mask that is set to True for stars you want to inject.
If None, all stars will be injected.
Returns
-------
image : `~numpy.ndarray`
The mock image *before* psf convolution.
"""
bins = tuple(np.asarray(xy_dim).astype(int))
hist_range = [[0, bins[0] - 1], [0, bins[1] - 1]]
if mask is not None:
_x = x[mask]
_y = y[mask]
_s = signal[mask]
else:
_x, _y, _s = x, y, signal
image = histogram2d(_x, _y, bins=bins, weights=_s, range=hist_range).T
return image
def get_density(lats, lons, nlevels, confobj):
bins=(confobj.dy, confobj.dx)
ranges=((confobj.ymin,confobj.ymax), (confobj.xmin,confobj.xmax))
bins = np.asarray(bins).astype(np.int64)
ranges = np.asarray(ranges).astype(np.float64)
edges = (np.linspace(*ranges[0,:], bins[0]+1),np.linspace(*ranges[1,:], bins[1]+1))
print(np.shape(lats))
density = fast_histogram.histogram2d(lats, lons, bins=bins, range=ranges)
print("Done with density")
H2, *ledges = np.histogram2d(lats, lons, bins=bins, range=ranges)
print("Done with numpy")
# density = histogram2d(lats, lons, range=[[confobj.ymin,confobj.ymax],[confobj.xmin,confobj.xmax]], bins=[confobj.dy,confobj.dx])
print(np.sum(density-H2))
density=H2
total = np.sum(density)
density=(density/total)*100.
print("Total number of points {} percentage sum {}".format(total,np.sum(density)))
density = ma.masked_where(density == 0, density)
levels = MaxNLocator(nbins=nlevels).tick_values(0.1,2)
#norm .tick_values(density.min(),density.max())
norm = BoundaryNorm(levels, ncolors=confobj.cmap.N, clip=True)
# Turn the lon/lat of the bins into 2 dimensional arrays
lon_bins_2d, lat_bins_2d = np.meshgrid(confobj.lon_bins, confobj.lat_bins)
return lon_bins_2d,lat_bins_2d,density,norm
def inject(self, x, y, signal, xy_dim):
"""
Inject sources into image.
Parameters
----------
x : `~numpy.ndarray`
Source x coordinates.
y : `~numpy.ndarray`
Source y coordinates.
signal : `~numpy.ndarray`
The signal to inject into image (e.g., flux, counts, etc...).
xy_dim : int or list-like
The dimensions of mock image in xy units. If `int` is given, it is
assumed to be both the x and y dimensions: (xy_dim, xy_dim).
Returns
-------
image : `~numpy.ndarray`
The mock image *before* psf convolution.
"""
bins = tuple(np.asarray(xy_dim).astype(int))
hist_range = [[0, bins[0] - 1], [0, bins[1] - 1]]
image = histogram2d(x, y, bins=bins, weights=signal,
range=hist_range).T
return image
def density_fluctuations1(x, y, average_density, range_, n_bins_):
"""
This function calculates giant density fluctuations using provided particle positions and bin edges.
*NOTE*: this particular version of the algorithm is sub-optimal i.e., it is possible to make it faster
by finding the square of a histogram first and then finding its mean. However, due to usage of a less precise
alternative (some particles can be missing in the histogram) to the numpy histogram2d that might result
in negative values of the variance.
Thus, we subtract the mean value first and find the square next potentially having slightly incorrect results.
However, large datasets should not suffer from this significantly.
FIXME try to use the number of particles in all bins instead od system population. Check how fast it is.
:param x: x coordinate of particles
:param y: y coordinate of particles
:param average_density: average number of particles per bin
:param range_: the range of x, y coordinates. By default the lower bound is 0, thus only the upper bound is provided
:param n_bins_: number of bins in x,y directions
:return: the normalized value of density fluctuations
"""
h = histogram2d(x,
y,
range=[[0, range_[0]], [0, range_[1]]],
bins=[n_bins_[0], n_bins_[1]])
h -= average_density
h *= h # squared std is marked as variance
normed_std = sqrt(mean(h) / average_density)
return normed_std
def pixelize(R):
r"""
Pixelize an area of 2R x 2R on the projection plane; bin particles
into the pixels according to their 2D coordinates.
Syntax:
pixelize(R)
where
R: spatial scale of interest [kpc] (scalar)
Note that, for imshow to work porperly, return the transposed image!
Note that the coordinates of the pixel edges range from -R to R,
and the number of pixels is defined by cfg.Npixel (assuming a
square image).
Note that this function uses the 2D coordinates in global 2D array
cfg.RR, as well as the weights in global array cfg.W.
"""
im = histogram2d(cfg.RR[:,0],cfg.RR[:,1],
range=[[-R, R], [-R, R]],
bins=[cfg.Npixel,cfg.Npixel],
weights=cfg.weight)
PixelArea = ( R / float(cfg.Npixel) )**2.
im = im / PixelArea
#im[im==0.] = cfg.Sigma_sky # <<< play with
im[im==0.] = 1e-6 * im.max()
return im.T
def count_and_convert_interfaces_to_matrix(pcg, n_minerals):
"""Counts frequencies of interfaces via fasthistogram module and
subsequently converts them to matrix format.
"""
return histogram2d(pcg[:-1],
pcg[1:],
range=[[0, n_minerals], [0, n_minerals]],
bins=n_minerals).astype(np.int32)
def fhist2d(x1,y1,xmin,xmax,xbinsize,ymin,ymax,ybinsize):
xbins=int((xmax-xmin)/xbinsize)
ybins=int((ymax-ymin)/ybinsize)
x=np.linspace(xmin,xmax,xbins+1)
y=np.linspace(ymin,ymax,ybins+1)
centersx=(x[:-1]+x[1:])/2
centersy=(y[:-1]+y[1:])/2
test_z=histogram2d(x1,y1,bins=[xbins,ybins],range=[[xmin,xmax],[ymin,ymax]])
return centersx,centersy,test_z
def calc_rg_hist(x, y, bin_):
"""Calculate the two-dimensional rg histogram with 'bin_' bin from the
given 'x' and 'y' arrays.
Typical use:
img = cv2.imread("image.png")
b, g, r = calc_bgr(img)
calc_2d_rg_hist(r, g, bin_=256)
"""
return histogram2d(y.ravel(), x.ravel(),
bins=[bin_, bin_], range=[[0, 1], [0, 1]])
def calc_mutual_information(ndfx: np.ndarray,
ndfy: np.ndarray,
*args,
bins: int = 100,
base_max: int = 1) -> (np.ndarray, np.ndarray):
"""
相互情報量を計算する
Params::
ndfx, ndfy: 入力は0~base_maxに正規化されているとする
"""
""" DEBUG
import numpy as np
from fast_histogram import histogram1d, histogram2d
x = np.random.rand(1000, 20)
y = np.random.rand(1000, 10)
ndfx, ndfy, bins, base_max = x, y, 100, 1
"""
from fast_histogram import histogram1d, histogram2d
logger.info("START")
list_ndf = []
for x in ndfx.T:
for y in ndfy.T:
ndf = histogram2d(x,
y,
range=[[0, base_max], [0, base_max]],
bins=bins)
ndf = (ndf / ndf.sum()).astype(np.float16)
list_ndf.append(ndf.reshape(1, *ndf.shape))
ndf_xy = np.concatenate(list_ndf, axis=0)
ndf_x = np.array(
[histogram1d(x, range=[0, base_max], bins=bins)
for x in ndfx.T]) / ndfx.shape[0]
ndf_y = np.array(
[histogram1d(x, range=[0, base_max], bins=bins)
for x in ndfy.T]) / ndfy.shape[0]
ndf_x = np.tile(np.tile(ndf_x.reshape(-1, bins, 1), bins),
(1, ndfy.shape[1], 1)).reshape(-1, bins, bins)
ndf_y = np.tile(
np.tile(ndf_y, bins).reshape(-1, bins, bins), (ndfx.shape[1], 1, 1))
ndf_x_y = ndf_x * ndf_y
elem: np.ma.core.MaskedArray = ndf_xy * np.ma.log(ndf_xy / ndf_x_y)
val: np.ma.core.MaskedArray = np.sum(elem * base_max / bins * base_max /
bins,
axis=(1, 2))
val = np.ma.filled(val, 0)
index_x = np.tile(np.arange(ndfx.shape[1]).reshape(-1, 1),
ndfy.shape[1]).reshape(-1)
index_y = np.tile(np.arange(ndfy.shape[1]), ndfx.shape[1]).reshape(-1)
index_list = np.concatenate([[index_x], [index_y]], axis=0).T
logger.info("END")
return (index_list, val, *args)
def make_image(self, *args, **kwargs):
xmin, xmax, ymin, ymax = self.get_extent()
if self._dpi is None:
dpi = self.axes.figure.get_dpi()
else:
dpi = self._dpi
width = (self._ax.get_position().width *
self._ax.figure.get_figwidth())
height = (self._ax.get_position().height *
self._ax.figure.get_figheight())
nx = int(round(width * dpi))
ny = int(round(height * dpi))
if self._downres:
nx_sub = nx // self._downres_factor
ny_sub = ny // self._downres_factor
array = histogram2d(self._y_sub,
self._x_sub,
bins=(ny_sub, nx_sub),
range=((ymin, ymax), (xmin, xmax)))
else:
array = histogram2d(self._y,
self._x,
bins=(ny, nx),
range=((ymin, ymax), (xmin, xmax)))
if self.origin == 'upper':
array = np.flipud(array)
self.set_clim(np.nanmin(array), np.nanmax(array))
self.set_data(array)
return super(ScatterDensityArtist, self).make_image(*args, **kwargs)
def __init__(self, xmin, xmax, ymin, ymax, nxbins=10, nybins=10):
self.nxbins = nxbins
self.nybins = nybins
self.xedges = np.linspace(xmin, xmax, nxbins + 1)
self.xcenters = (self.xedges[:-1] + self.xedges[1:]) / 2.0
self.yedges = np.linspace(ymin, ymax, nybins + 1)
self.ycenters = (self.yedges[:-1] + self.yedges[1:]) / 2.0
self.delta = 0.0
self.xrange = (xmin, xmax + self.delta)
self.yrange = (ymin, ymax + self.delta)
self.hists = histogram2d([], [], [self.nxbins, self.nybins],
[self.xrange, self.yrange])
def density_fluctuations(x, y, av_density, range_, edges):
"""
This function calculates giant density fluctuations using provided particle positions and bin edges
:param x: x coordinate of particles
:param y: y coordinate of particles
:param av_density: average bin density
:param range_: the range of x, y coordinates. By default the lower bound is 0, thus only the upper bound is provided
:param edges: bin edges
:return: the normalized value of density fluctuations
"""
h = histogram2d(x,
y,
range=[[0, range_[0]], [0, range_[1]]],
bins=[edges[0], edges[1]])
e_n_squared = mean(h * h)
e_squared_n = av_density * av_density
deviation = sqrt(e_n_squared - e_squared_n)
normed_deviation = deviation / sqrt(av_density)
return normed_deviation
def MI(x, y, bins=32, range=((0, 1), (0, 1))):
r"""Computes mutual information between time-series x and y.
The mutual information between two distributions is a measure of
correlation between them. If the distributions are independent, the
mutual information will be 0. Mathematically, it is equivalent to the
KL-divergence between the joint distribution and the product of the marginal
distributions:
.. math::
I(x, y) = D_{KL}\( p(x, y) || p(x)p(y) \)
Args:
x (torch.tensor): a 1d tensor representing a time series of x values
y (torch.tensor): a 1d tensor representing a time series of y values
bins (int): the number of bins to discretize x and y values into
range (array-like; 2x2): upper and lower values which bins can take for x and y
Returns:
float: the mutual information of the joint and marginal distributions
inferred from the time series.
TODO: implement custom version in pure pytorch without relying on sklearn
"""
# def normalize(x):
# x = x / np.sum(x)
# x[x != x] = 0
# return x
# def H(x):
# r = x / np.sum(x)
# r[r != r] = 0
# r = -r * np.log2(r)
# r[r != r] = 0
# return np.sum(r)
assert len(x) == len(y), "time series are of unequal length"
x = x.detach().numpy()
y = y.detach().numpy()
cm = histogram2d(x, y, bins=bins, range=hack_range(range))
# return H(np.sum(cm, axis=1)) + H(np.sum(cm, axis=0)) - H(cm)
return nats_to_bits(mutual_info_score(None, None, contingency=cm))
def run_step(self, run_number, step_size, howlong):
dfslot = self.get_input_slot('table')
dfslot.update(run_number)
min_slot = self.get_input_slot('min')
min_slot.update(run_number)
max_slot = self.get_input_slot('max')
max_slot.update(run_number)
if dfslot.updated.any():
logger.debug('reseting histogram')
self.reset()
dfslot.update(run_number)
if not (dfslot.created.any() or min_slot.created.any()
or max_slot.created.any()):
logger.info('Input buffers empty')
return self._return_run_step(self.state_blocked, steps_run=0)
bounds = self.get_bounds(min_slot, max_slot)
if bounds is None:
logger.debug('No bounds yet at run %d', run_number)
return self._return_run_step(self.state_blocked, steps_run=0)
xmin, xmax, ymin, ymax = bounds
if self._bounds is None:
(xdelta, ydelta) = self.get_delta(*bounds)
self._bounds = (xmin - xdelta, xmax + xdelta, ymin - ydelta,
ymax + ydelta)
logger.info("New bounds at run %d: %s", run_number, self._bounds)
else:
(dxmin, dxmax, dymin, dymax) = self._bounds
(xdelta, ydelta) = self.get_delta(*bounds)
assert xdelta >= 0 and ydelta >= 0
# Either the min/max has extended, or it has shrunk beyond the deltas
if ((xmin < dxmin or xmax > dxmax or ymin < dymin or ymax > dymax)
or (xmin > (dxmin + xdelta) or xmax <
(dxmax - xdelta) or ymin > (dymin + ydelta) or ymax <
(dymax - ydelta))):
#print('Old bounds: %s,%s,%s,%s'%(dxmin,dxmax,dymin,dymax))
self._bounds = (xmin - xdelta, xmax + xdelta, ymin - ydelta,
ymax + ydelta)
#print('Updated bounds at run %d: %s old %s deltas %s, %s'%(run_number,self._bounds, bounds, xdelta, ydelta))
logger.info('Updated bounds at run %s: %s', run_number,
self._bounds)
self.reset()
dfslot.update(run_number)
xmin, xmax, ymin, ymax = self._bounds
if xmin >= xmax or ymin >= ymax:
logger.error('Invalid bounds: %s', self._bounds)
return self._return_run_step(self.state_blocked, steps_run=0)
# Now, we know we have data and bounds, proceed to create a new histogram
# or to update the previous if is still exists (i.e. no reset)
p = self.params
steps = 0
# if there are new deletions, build the histogram of the deleted pairs
# then subtract it from the main histogram
if dfslot.deleted.any() and self._histo is not None:
input_df = get_physical_base(dfslot.data())
indices = dfslot.deleted.next(step_size)
steps += indices_len(indices)
#print('Histogram2D steps :%d'% steps)
logger.info('Read %d rows', steps)
x = input_df[self.x_column]
y = input_df[self.y_column]
idx = input_df.id_to_index(fix_loc(indices))
#print(idx)
x = x[idx]
y = y[idx]
bins = [p.ybins, p.xbins]
if len(x) > 0:
histo = histogram2d(y,
x,
bins=bins,
range=[[ymin, ymax], [xmin, xmax]])
self._histo -= histo
# if there are new creations, build a partial histogram with them then
# add it to the main histogram
input_df = dfslot.data()
indices = dfslot.created.next(step_size)
steps += indices_len(indices)
#print('Histogram2D steps :%d'% steps)
logger.info('Read %d rows', steps)
self.total_read += steps
x = input_df[self.x_column]
y = input_df[self.y_column]
idx = input_df.id_to_index(fix_loc(indices))
#print(idx)
x = x[idx]
y = y[idx]
if self._xedges is not None:
bins = [self._xedges, self._yedges]
else:
bins = [p.ybins, p.xbins]
if len(x) > 0:
#t = default_timer()
# using fast_histogram
histo = histogram2d(y,
x,
bins=bins,
range=[[ymin, ymax], [xmin, xmax]])
# using numpy histogram
#histo, xedges, yedges = np.histogram2d(y, x,
# bins=bins,
# range=[[ymin, ymax], [xmin, xmax]],
# normed=False)
#t = default_timer()-t
#print('Time for histogram2d: %f'%t)
#self._xedges = xedges
#self._yedges = yedges
else:
histo = None
cmax = 0
if self._histo is None:
self._histo = histo
elif histo is not None:
self._histo += histo
if self._histo is not None:
cmax = self._histo.max()
values = {
'array': np.flip(self._histo, axis=0),
'cmin': 0,
'cmax': cmax,
'xmin': xmin,
'xmax': xmax,
'ymin': ymin,
'ymax': ymax,
'time': run_number
}
if self._with_output:
with self.lock:
table = self._table
table['array'].set_shape([p.ybins, p.xbins])
l = len(table)
last = table.last()
if l == 0 or last['time'] != run_number:
table.add(values)
else:
table.iloc[last.row] = values
self.build_heatmap(values)
return self._return_run_step(self.next_state(dfslot), steps_run=steps)
def _EI_of_layer_auto_samples(layer, batch_size, in_shape, in_range, in_bins, \
out_shape, out_range, out_bins, activation, device, threshold):
"""Helper function of EI_of_layer that computes the EI of layer `layer`
using enough samples to be within `threshold`% of the true value.
"""
MULTIPLIER = 2
INTERVAL = 10000
SAMPLES_SO_FAR = INTERVAL
EIs = []
def has_converged(EIs):
if len(EIs) < 2:
return False
slope = (EIs[-2] - EIs[-1]) / INTERVAL
error = slope * SAMPLES_SO_FAR * (MULTIPLIER - 1)
if error / EIs[-1] > threshold:
return False
return True
in_l, in_u = in_range
num_inputs = reduce(lambda x, y: x * y, in_shape)
num_outputs = reduce(lambda x, y: x * y, out_shape)
#################################################
# Create histograms for each A -> B pair #
#################################################
in_bin_width = (in_u - in_l) / in_bins
if out_bins != 'dynamic':
CMs = np.zeros((num_inputs, num_outputs, in_bins,
out_bins)) # histograms for each input/output pair
else:
CMs = [[None for B in range(num_outputs)] for A in range(num_inputs)]
if out_range == 'dynamic':
dyn_out_bins = [None for B in range(num_outputs)]
dyn_out_bins_set = False
if out_range == 'dynamic':
dyn_out_ranges = np.zeros((num_outputs, 2))
dyn_ranges_set = False
while True:
for chunk_size in _chunk_sizes(INTERVAL, num_inputs, num_outputs,
MEMORY_LIMIT):
#################################################
# Create buffers for layer input and output #
#################################################
inputs = torch.zeros((chunk_size, *in_shape), device=device)
outputs = torch.zeros((chunk_size, *out_shape), device=device)
#################################################
# Evaluate module on noise #
#################################################
for (i0, i1), bsize in _indices_and_batch_sizes(
chunk_size, batch_size):
sample = (in_u - in_l) * torch.rand(
(bsize, *in_shape), device=device) + in_l
inputs[i0:i1] = sample
with torch.no_grad():
result = activation(layer(sample))
outputs[i0:i1] = result
inputs = torch.flatten(inputs, start_dim=1)
outputs = torch.flatten(outputs, start_dim=1)
#################################################
# If specified to be dynamic, #
# and first time in the loop, #
# determine out_range for output neurons #
#################################################
if out_range == 'dynamic' and not dyn_ranges_set:
for B in range(num_outputs):
out_l = torch.min(outputs[:, B]).item()
out_u = torch.max(outputs[:, B]).item()
dyn_out_ranges[B][0] = out_l
dyn_out_ranges[B][1] = out_u
dyn_ranges_set = True
#################################################
# If specified to be dynamic, #
# and first time in the loop, #
# determine out_bins for output neurons #
#################################################
if out_bins == 'dynamic' and not dyn_out_bins_set:
if out_range == 'dynamic':
for B in range(num_outputs):
out_l, out_u = dyn_out_ranges[B]
bins = int((out_u - out_l) / in_bin_width) + 1
out_u = out_l + (bins * in_bin_width)
dyn_out_bins[B] = bins
dyn_out_ranges[B][1] = out_u
else:
out_l, out_u = out_range
bins = int((out_u - out_l) / in_bin_width) + 1
out_u = out_l + (bins * in_bin_width)
dyn_out_bins = bins
out_range = (out_l, out_u)
for A in range(num_inputs):
for B in range(num_outputs):
if out_range == 'dynamic':
out_b = dyn_out_bins[B]
else:
out_b = dyn_out_bins
CMs[A][B] = np.zeros((in_bins, out_b))
dyn_out_bins_set = True
#################################################
# Update Histograms for each A -> B pair #
#################################################
for A in range(num_inputs):
for B in range(num_outputs):
if out_range == 'dynamic':
out_r = tuple(dyn_out_ranges[B])
else:
out_r = out_range
if out_bins == 'dynamic':
if out_range == 'dynamic':
out_b = dyn_out_bins[B]
else:
out_b = dyn_out_bins
else:
out_b = out_bins
CMs[A][B] += histogram2d(
inputs[:, A].to('cpu').detach().numpy(),
outputs[:, B].to('cpu').detach().numpy(),
bins=(in_bins, out_b),
range=hack_range((in_range, out_r)))
#################################################
# Compute mutual information #
#################################################
EI = 0.0
for A in range(num_inputs):
for B in range(num_outputs):
A_B_EI = nats_to_bits(
mutual_info_score(None, None, contingency=CMs[A][B]))
EI += A_B_EI
EIs.append(EI)
#################################################
# Determine whether more samples #
# are needed and update how many #
#################################################
if has_converged(EIs):
return EIs[-1]
INTERVAL = int(SAMPLES_SO_FAR * (MULTIPLIER - 1))
SAMPLES_SO_FAR += INTERVAL
def _EI_of_layer_manual_samples(layer, samples, batch_size, in_shape, in_range, in_bins, \
out_shape, out_range, out_bins, activation, device):
"""Helper function for EI_of_layer that computes the EI of layer `layer`
with a set number of samples."""
in_l, in_u = in_range
num_inputs = reduce(lambda x, y: x * y, in_shape)
num_outputs = reduce(lambda x, y: x * y, out_shape)
#################################################
# Create histograms for each A -> B pair #
#################################################
in_bin_width = (in_u - in_l) / in_bins
if out_bins != 'dynamic':
CMs = np.zeros((num_inputs, num_outputs, in_bins,
out_bins)) # histograms for each input/output pair
else:
CMs = [[None for B in range(num_outputs)] for A in range(num_inputs)]
if out_range == 'dynamic':
dyn_out_bins = [None for B in range(num_outputs)]
dyn_out_bins_set = False
if out_range == 'dynamic':
dyn_out_ranges = np.zeros((num_outputs, 2))
dyn_ranges_set = False
for chunk_size in _chunk_sizes(samples, num_inputs, num_outputs,
MEMORY_LIMIT):
#################################################
# Create buffers for layer input and output #
#################################################
inputs = torch.zeros((chunk_size, *in_shape), device=device)
outputs = torch.zeros((chunk_size, *out_shape), device=device)
#################################################
# Evaluate module on noise #
#################################################
for (i0,
i1), bsize in _indices_and_batch_sizes(chunk_size, batch_size):
sample = (in_u - in_l) * torch.rand(
(bsize, *in_shape), device=device) + in_l
inputs[i0:i1] = sample
with torch.no_grad():
result = activation(layer(sample))
outputs[i0:i1] = result
inputs = torch.flatten(inputs, start_dim=1)
outputs = torch.flatten(outputs, start_dim=1)
#################################################
# If specified to be dynamic, #
# and first time in the loop, #
# determine out_range for output neurons #
#################################################
if out_range == 'dynamic' and not dyn_ranges_set:
for B in range(num_outputs):
out_l = torch.min(outputs[:, B]).item()
out_u = torch.max(outputs[:, B]).item()
dyn_out_ranges[B][0] = out_l
dyn_out_ranges[B][1] = out_u
dyn_ranges_set = True
#################################################
# If specified to be dynamic, #
# and first time in the loop, #
# determine out_bins for output neurons #
#################################################
if out_bins == 'dynamic' and not dyn_out_bins_set:
if out_range == 'dynamic':
for B in range(num_outputs):
out_l, out_u = dyn_out_ranges[B]
bins = int((out_u - out_l) / in_bin_width) + 1
out_u = out_l + (bins * in_bin_width)
dyn_out_bins[B] = bins
dyn_out_ranges[B][1] = out_u
else:
out_l, out_u = out_range
bins = int((out_u - out_l) / in_bin_width) + 1
out_u = out_l + (bins * in_bin_width)
dyn_out_bins = bins
out_range = (out_l, out_u)
for A in range(num_inputs):
for B in range(num_outputs):
if out_range == 'dynamic':
out_b = dyn_out_bins[B]
else:
out_b = dyn_out_bins
CMs[A][B] = np.zeros((in_bins, out_b))
dyn_out_bins_set = True
#################################################
# Update Histograms for each A -> B pair #
#################################################
for A in range(num_inputs):
for B in range(num_outputs):
if out_range == 'dynamic':
out_r = tuple(dyn_out_ranges[B])
else:
out_r = out_range
if out_bins == 'dynamic':
if out_range == 'dynamic':
out_b = dyn_out_bins[B]
else:
out_b = dyn_out_bins
else:
out_b = out_bins
# print("in_range: {}".format(in_range))
# print("in_bins: {}".format(in_bins))
# print("out_range: {}".format(out_r))
# print("out_bins: {}".format(out_b))
CMs[A][B] += histogram2d(inputs[:,
A].to('cpu').detach().numpy(),
outputs[:,
B].to('cpu').detach().numpy(),
bins=(in_bins, out_b),
range=hack_range((in_range, out_r)))
#################################################
# Compute mutual information #
#################################################
EI = 0.0
for A in range(num_inputs):
for B in range(num_outputs):
A_B_EI = nats_to_bits(
mutual_info_score(None, None, contingency=CMs[A][B]))
EI += A_B_EI
return EI
def get_observation(self, dm, my_orientation, their_orientation, feat,
vels):
evader_dists = dm[-self.n_evaders:]
evader_bearings = my_orientation[-self.n_evaders:]
pursuer_dists = dm[:-self.n_evaders]
pursuer_bearings = my_orientation[:-self.n_evaders]
if self.obs_mode == 'fix':
local_obs = self.get_local_obs()
if self.obs_radius > 100:
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings),
np.sin(evader_bearings)
]
local_obs = np.zeros(self.dim_local_o)
else:
if evader_dists < self.obs_radius:
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings),
np.sin(evader_bearings)
]
else:
dist_to_evader = 1.
angle_to_evader = [0, 0]
see_evader = 1 if dist_to_evader < 1 else 0
self.see_evader = see_evader
shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\
if self.graph_feature < (5 * self.comm_radius) else 1.
local_obs[-1] = shortest_path_to_evader
evader_obs = np.zeros(self.dim_evader_o)
evader_obs[:, 0] = dist_to_evader
evader_obs[:, 1] = angle_to_evader[0]
evader_obs[:, 2] = angle_to_evader[1]
ind = np.where(dm == -1)[0][0]
fix_obs = np.zeros(self.dim_rec_o)
if self.obs_radius > 100:
fix_obs[:, 0] = np.concatenate([
pursuer_dists[0:ind], pursuer_dists[ind + 1:]
]) / self.comm_radius
fix_obs[:, 1] = np.cos(
np.concatenate(
[pursuer_bearings[0:ind], pursuer_bearings[ind + 1:]]))
fix_obs[:, 2] = np.sin(
np.concatenate(
[pursuer_bearings[0:ind], pursuer_bearings[ind + 1:]]))
fix_obs[:, 3] = np.cos(
np.concatenate([
their_orientation[0:ind], their_orientation[ind + 1:]
]))
fix_obs[:, 4] = np.sin(
np.concatenate([
their_orientation[0:ind], their_orientation[ind + 1:]
]))
else:
in_range = (evader_dists < self.comm_radius) & (0 <
evader_dists)
dists_in_range = np.array(feat)[in_range]
dists_in_range_capped = np.where(
dists_in_range <= 5 * self.comm_radius,
dists_in_range / (5 * self.comm_radius), 1.)
fix_obs[:, 0] = np.concatenate([
pursuer_dists[0:ind], pursuer_dists[ind + 1:]
]) / self.comm_radius
fix_obs[:, 1] = np.cos(
np.concatenate(
[pursuer_bearings[0:ind], pursuer_bearings[ind + 1:]]))
fix_obs[:, 2] = np.sin(
np.concatenate(
[pursuer_bearings[0:ind], pursuer_bearings[ind + 1:]]))
fix_obs[:, 3] = np.cos(
np.concatenate([
their_orientation[0:ind], their_orientation[ind + 1:]
]))
fix_obs[:, 4] = np.sin(
np.concatenate([
their_orientation[0:ind], their_orientation[ind + 1:]
]))
fix_obs[:, 5] = dists_in_range_capped
obs = np.hstack(
[fix_obs.flatten(),
evader_obs.flatten(),
local_obs.flatten()])
elif self.obs_mode == 'sum_obs':
# local obs
if evader_dists < self.obs_radius:
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings),
np.sin(evader_bearings)
]
else:
dist_to_evader = 1.
angle_to_evader = [0, 0]
local_obs = np.zeros(self.dim_local_o)
if self.torus is False:
if np.any(self.state.p_pos <= 1) or np.any(
self.state.p_pos >= 99):
wall = 1
else:
wall = 0
local_obs[0] = wall
evader_obs = np.zeros(self.dim_evader_o)
evader_obs[:, 0] = dist_to_evader
evader_obs[:, 1] = angle_to_evader[0]
evader_obs[:, 2] = angle_to_evader[1]
# neighbor obs
pursuers_in_range = (pursuer_dists <
self.comm_radius) & (0 < pursuer_dists)
nr_neighbors = np.sum(pursuers_in_range)
sum_obs = np.zeros(self.dim_rec_o)
nr_agents = dm.size - 1
sum_obs[0:nr_neighbors,
0] = pursuer_dists[pursuers_in_range] / self.comm_radius
sum_obs[0:nr_neighbors,
1] = np.cos(pursuer_bearings[pursuers_in_range])
sum_obs[0:nr_neighbors,
2] = np.sin(pursuer_bearings[pursuers_in_range])
sum_obs[0:nr_neighbors,
3] = np.cos(their_orientation[pursuers_in_range])
sum_obs[0:nr_neighbors,
4] = np.sin(their_orientation[pursuers_in_range])
sum_obs[0:nr_neighbors, 5] = 1
sum_obs[0:nr_agents, 6] = 1
obs = np.hstack(
[sum_obs.flatten(),
local_obs.flatten(),
evader_obs.flatten()])
elif self.obs_mode == 'sum_obs_no_ori':
# local obs
if evader_dists < self.obs_radius:
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings),
np.sin(evader_bearings)
]
else:
dist_to_evader = 1.
angle_to_evader = [0, 0]
local_obs = np.zeros(self.dim_local_o)
if self.torus is False:
if np.any(self.state.p_pos <= 1) or np.any(
self.state.p_pos >= 99):
wall = 1
else:
wall = 0
local_obs[0] = wall
evader_obs = np.zeros(self.dim_evader_o)
evader_obs[:, 0] = dist_to_evader
evader_obs[:, 1] = angle_to_evader[0]
evader_obs[:, 2] = angle_to_evader[1]
# neighbor obs
pursuers_in_range = (pursuer_dists <
self.comm_radius) & (0 < pursuer_dists)
nr_neighbors = np.sum(pursuers_in_range)
sum_obs = np.zeros(self.dim_rec_o)
nr_agents = dm.size - 1
sum_obs[0:nr_neighbors,
0] = pursuer_dists[pursuers_in_range] / self.comm_radius
sum_obs[0:nr_neighbors,
1] = np.cos(pursuer_bearings[pursuers_in_range])
sum_obs[0:nr_neighbors,
2] = np.sin(pursuer_bearings[pursuers_in_range])
sum_obs[0:nr_neighbors, 3] = 1
sum_obs[0:nr_agents, 4] = 1
obs = np.hstack(
[sum_obs.flatten(),
local_obs.flatten(),
evader_obs.flatten()])
elif self.obs_mode == 'sum_obs_limited':
# local obs
if evader_dists < self.obs_radius:
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings),
np.sin(evader_bearings)
]
else:
dist_to_evader = 1.
angle_to_evader = [0, 0]
see_evader = 1 if dist_to_evader < 1 else 0
self.see_evader = see_evader
shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\
if self.graph_feature < (5 * self.comm_radius) else 1.
local_obs = np.zeros(self.dim_local_o)
local_obs[0] = shortest_path_to_evader
if self.torus is False:
if np.any(self.state.p_pos <= 1) or np.any(
self.state.p_pos >= 99):
wall = 1
else:
wall = 0
local_obs[1] = wall
evader_obs = np.zeros(self.dim_evader_o)
evader_obs[:, 0] = dist_to_evader
evader_obs[:, 1] = angle_to_evader[0]
evader_obs[:, 2] = angle_to_evader[1]
# neighbor obs
evaders_in_range = (evader_dists <
self.obs_radius) & (0 < evader_dists)
pursuers_in_range = (pursuer_dists <
self.comm_radius) & (0 < pursuer_dists)
nr_neighbors = np.sum(pursuers_in_range)
dists_in_range = np.array(feat)[pursuers_in_range]
dists_in_range_capped = np.where(
dists_in_range <= 5 * self.comm_radius,
dists_in_range / (5 * self.comm_radius), 1.)
sum_obs = np.zeros(self.dim_rec_o)
nr_agents = dm.size - 1
sum_obs[0:nr_neighbors,
0] = pursuer_dists[pursuers_in_range] / self.comm_radius
sum_obs[0:nr_neighbors,
1] = np.cos(pursuer_bearings[pursuers_in_range])
sum_obs[0:nr_neighbors,
2] = np.sin(pursuer_bearings[pursuers_in_range])
sum_obs[0:nr_neighbors,
3] = np.cos(their_orientation[pursuers_in_range])
sum_obs[0:nr_neighbors,
4] = np.sin(their_orientation[pursuers_in_range])
sum_obs[0:nr_neighbors, 5] = dists_in_range_capped
sum_obs[0:nr_neighbors, 6] = 1
sum_obs[0:nr_agents, 7] = 1
# obs = np.hstack([sum_obs.flatten(), local_obs])
obs = np.hstack(
[sum_obs.flatten(),
local_obs.flatten(),
evader_obs.flatten()])
elif self.obs_mode == 'sum_obs_multi':
in_range = (evader_dists <= self.obs_radius) & (0 <= evader_dists)
nr_neighboring_evaders = np.sum(in_range)
dist_to_evader = evader_dists[in_range] / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings[in_range]),
np.sin(evader_bearings[in_range])
]
if self.obs_radius > 100:
local_obs = np.zeros(self.dim_local_o)
if self.torus is False:
if np.any(self.state.p_pos <= 1) or np.any(
self.state.p_pos >= 99):
wall = 1
else:
wall = 0
local_obs[0] = wall
else:
shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\
if self.graph_feature < (5 * self.comm_radius) else 1.
local_obs = np.zeros(self.dim_local_o)
local_obs[0] = shortest_path_to_evader
if self.torus is False:
if np.any(self.state.p_pos <= 1) or np.any(
self.state.p_pos >= 99):
wall = 1
else:
wall = 0
local_obs[1] = wall
sum_evader_obs = np.zeros(self.dim_evader_o)
sum_evader_obs[:nr_neighboring_evaders, 0] = dist_to_evader
sum_evader_obs[:nr_neighboring_evaders, 1] = angle_to_evader[0]
sum_evader_obs[:nr_neighboring_evaders, 2] = angle_to_evader[1]
sum_evader_obs[:nr_neighboring_evaders, 3] = 1
sum_evader_obs[:self.n_evaders, 4] = 1
pursuers_in_range = (pursuer_dists <=
self.comm_radius) & (0 <= pursuer_dists)
nr_neighbors = np.sum(pursuers_in_range)
if self.obs_radius > 100:
sum_obs = np.zeros(self.dim_rec_o)
nr_agents = dm.size - self.n_evaders
sum_obs[
0:nr_neighbors,
0] = pursuer_dists[pursuers_in_range] / self.comm_radius
sum_obs[0:nr_neighbors,
1] = np.cos(pursuer_bearings[pursuers_in_range])
sum_obs[0:nr_neighbors,
2] = np.sin(pursuer_bearings[pursuers_in_range])
sum_obs[0:nr_neighbors,
3] = np.cos(their_orientation[pursuers_in_range])
sum_obs[0:nr_neighbors,
4] = np.sin(their_orientation[pursuers_in_range])
sum_obs[0:nr_neighbors, 5] = 1
sum_obs[0:nr_agents, 6] = 1
else:
dists_in_range = np.array(feat)[pursuers_in_range]
dists_in_range_capped = np.where(
dists_in_range <= 5 * self.comm_radius,
dists_in_range / (5 * self.comm_radius), 1.)
sum_obs = np.zeros(self.dim_rec_o)
nr_agents = dm.size - self.n_evaders
sum_obs[
0:nr_neighbors,
0] = pursuer_dists[pursuers_in_range] / self.comm_radius
sum_obs[0:nr_neighbors,
1] = np.cos(pursuer_bearings[pursuers_in_range])
sum_obs[0:nr_neighbors,
2] = np.sin(pursuer_bearings[pursuers_in_range])
sum_obs[0:nr_neighbors,
3] = np.cos(their_orientation[pursuers_in_range])
sum_obs[0:nr_neighbors,
4] = np.sin(their_orientation[pursuers_in_range])
sum_obs[0:nr_neighbors, 5] = dists_in_range_capped
sum_obs[0:nr_neighbors, 6] = 1
sum_obs[0:nr_agents, 7] = 1
obs = np.hstack(
[sum_obs.flatten(),
sum_evader_obs.flatten(), local_obs])
elif self.obs_mode == '2d_hist':
if self.obs_radius > 100:
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings),
np.sin(evader_bearings)
]
local_obs = np.zeros(self.dim_flat_o)
if self.torus is False:
if np.any(self.state.p_pos <= 1) or np.any(
self.state.p_pos >= 99):
wall = 1
else:
wall = 0
local_obs[0] = wall
else:
if evader_dists < self.obs_radius:
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings),
np.sin(evader_bearings)
]
else:
dist_to_evader = 1.
angle_to_evader = [0, 0]
see_evader = 1 if dist_to_evader < 1 else 0
self.see_evader = see_evader
shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\
if self.graph_feature < (5 * self.comm_radius) else 1.
local_obs = np.zeros(self.dim_flat_o)
local_obs[0] = shortest_path_to_evader
if self.torus is False:
if np.any(self.state.p_pos <= 1) or np.any(
self.state.p_pos >= 99):
wall = 1
else:
wall = 0
local_obs[1] = wall
evader_obs = np.zeros(self.dim_evader_o)
evader_obs[:, 0] = dist_to_evader
evader_obs[:, 1] = angle_to_evader[0]
evader_obs[:, 2] = angle_to_evader[1]
# neighbor obs
pursuers_in_range = (pursuer_dists <
self.comm_radius) & (0 < pursuer_bearings)
evader_in_range = (evader_dists < self.obs_radius) & (0 <
evader_dists)
nr_agents = dm.size - 2 # exclude self and evader
hist_2d_agents = fh.histogram2d(
pursuer_bearings[pursuers_in_range],
pursuer_dists[pursuers_in_range],
bins=(self.bearing_bins, self.distance_bins),
range=[[-np.pi, np.pi], [0, self.world_size * np.sqrt(2) / 2]])
hist_2d_evader = fh.histogram2d(
evader_bearings[evader_in_range],
evader_dists[evader_in_range],
bins=(self.bearing_bins, self.distance_bins),
range=[[-np.pi, np.pi], [0, self.world_size * np.sqrt(2) / 2]])
histogram = np.hstack([
hist_2d_agents.flatten() / (nr_agents),
hist_2d_evader.flatten()
])
obs = np.hstack([histogram, local_obs])
elif self.obs_mode == '2d_hist_short':
if self.obs_radius > 100:
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings),
np.sin(evader_bearings)
]
else:
if evader_dists < self.obs_radius:
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings),
np.sin(evader_bearings)
]
else:
dist_to_evader = 1.
angle_to_evader = [0, 0]
see_evader = 1 if dist_to_evader < 1 else 0
self.see_evader = see_evader
local_obs = self.get_local_obs()
evader_obs = np.zeros(self.dim_evader_o)
evader_obs[0] = dist_to_evader
evader_obs[1] = angle_to_evader[0]
evader_obs[2] = angle_to_evader[1]
# neighbor obs
in_range = (pursuer_dists < self.comm_radius) & (0 < pursuer_dists)
nr_agents = dm.size - 2 # exclude self and evader
hist_2d_agents = fh.histogram2d(
pursuer_bearings[in_range],
pursuer_dists[in_range],
bins=(self.bearing_bins, self.distance_bins),
range=[[-np.pi, np.pi], [0, self.world_size * np.sqrt(2) / 2]])
histogram = hist_2d_agents.flatten() / (nr_agents - 1)
obs = np.hstack([histogram, evader_obs.flatten(), local_obs])
elif self.obs_mode == '2d_rbf':
local_obs = np.zeros(self.dim_local_o)
if self.torus is False:
if np.any(self.state.p_pos <= 1) or np.any(
self.state.p_pos >= 99):
wall = 1
else:
wall = 0
local_obs[0] = wall
evader_in_range = evader_dists < self.obs_radius
if np.any(evader_in_range):
dbn = np.stack([
evader_dists[evader_in_range],
evader_bearings[evader_in_range] + np.pi
],
axis=1)
evader_rbf_hist = U.get_weights_2d(
dbn, self.mu_e, self.s_e,
[self.bearing_bins, self.distance_bins])
else:
evader_rbf_hist = np.zeros(
[self.bearing_bins, self.distance_bins])
in_range = (0 < pursuer_dists) & (pursuer_dists < self.comm_radius)
if np.any(in_range):
dbn = np.stack([
pursuer_dists[in_range], pursuer_bearings[in_range] + np.pi
],
axis=1)
rbf_hist = U.get_weights_2d(dbn, self.mu_n, self.s_n, [
self.bearing_bins, self.distance_bins
]) / (self.n_agents - 1)
else:
rbf_hist = np.zeros([self.bearing_bins, self.distance_bins])
rbf_hist_flat = rbf_hist.flatten()
obs = np.hstack(
[rbf_hist_flat,
evader_rbf_hist.flatten(), local_obs])
elif self.obs_mode == '2d_rbf_short':
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings),
np.sin(evader_bearings)
]
see_evader = 1 if dist_to_evader < 1 else 0
self.see_evader = see_evader
evader_obs = np.zeros(self.dim_evader_o)
evader_obs[0] = dist_to_evader
evader_obs[1] = angle_to_evader[0]
evader_obs[2] = angle_to_evader[1]
local_obs = self.get_local_obs()
in_range = (0 < pursuer_dists) & (pursuer_dists < self.comm_radius)
if np.any(in_range):
dbn = np.stack([
pursuer_dists[in_range], pursuer_bearings[in_range] + np.pi
],
axis=1)
rbf_hist = U.get_weights_2d(dbn, self.mu_n, self.s_n, [
self.bearing_bins, self.distance_bins
]) / (self.n_agents - 1)
else:
rbf_hist = np.zeros([self.bearing_bins, self.distance_bins])
rbf_hist_flat = rbf_hist.flatten()
obs = np.hstack([rbf_hist_flat, evader_obs, local_obs])
elif self.obs_mode == '2d_rbf_limited':
if evader_dists < self.obs_radius:
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = [
np.cos(evader_bearings),
np.sin(evader_bearings)
]
else:
dist_to_evader = 1.
angle_to_evader = [0, 0]
see_evader = 1 if dist_to_evader < 1 else 0
self.see_evader = see_evader
shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\
if self.graph_feature < (5 * self.comm_radius) else 1.
local_obs = np.zeros(self.dim_flat_o)
local_obs[0] = shortest_path_to_evader
if self.torus is False:
if np.any(self.state.p_pos <= 1) or np.any(
self.state.p_pos >= 99):
wall = 1
else:
wall = 0
local_obs[1] = wall
evader_in_range = evader_dists < self.obs_radius
sub = []
if evader_in_range:
dbn = np.hstack([evader_dists, evader_bearings + np.pi])
evader_rbf_hist = U.get_weights_2d(
dbn, self.mu_e, self.s_e,
[self.bearing_bins, self.distance_bins])
else:
evader_rbf_hist = np.zeros(
[self.bearing_bins, self.distance_bins])
in_range = (0 < pursuer_dists) & (pursuer_dists < self.comm_radius)
if np.any(in_range):
dbn = np.stack([
pursuer_dists[in_range], pursuer_bearings[in_range] + np.pi
],
axis=1)
rbf_hist = U.get_weights_2d(dbn, self.mu_n, self.s_n, [
self.bearing_bins, self.distance_bins
]) / (self.n_agents - 1)
else:
rbf_hist = np.zeros([self.bearing_bins, self.distance_bins])
rbf_hist_flat = rbf_hist.flatten()
obs = np.hstack(
[rbf_hist_flat,
evader_rbf_hist.flatten(), local_obs])
elif self.obs_mode == '2d_rbf_limited_short':
if evader_dists < self.obs_radius:
dist_to_evader = evader_dists / self.obs_radius
angle_to_evader = np.array(
[np.cos(evader_bearings),
np.sin(evader_bearings)])
else:
dist_to_evader = 1.
angle_to_evader = [0, 0]
see_evader = 1 if dist_to_evader < 1 else 0
self.see_evader = see_evader
shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\
if self.graph_feature < (5 * self.comm_radius) else 1.
evader_obs = np.zeros(self.dim_evader_o)
evader_obs[0] = dist_to_evader
evader_obs[1] = angle_to_evader[0]
evader_obs[2] = angle_to_evader[1]
local_obs = np.zeros(self.dim_local_o)
local_obs[0] = shortest_path_to_evader
if self.torus is False:
if np.any(self.state.p_pos <= 1) or np.any(
self.state.p_pos >= 99):
wall = 1
else:
wall = 0
local_obs[1] = wall
in_range = (0 < pursuer_dists) & (pursuer_dists < self.comm_radius)
if np.any(in_range):
dbn = np.stack([
pursuer_dists[in_range], pursuer_bearings[in_range] + np.pi
],
axis=1)
rbf_hist = U.get_weights_2d(dbn, self.mu_n, self.s_n, [
self.bearing_bins, self.distance_bins
]) / (self.n_agents - 1)
else:
rbf_hist = np.zeros([self.bearing_bins, self.distance_bins])
rbf_hist_flat = rbf_hist.flatten()
obs = np.hstack([rbf_hist_flat, evader_obs, local_obs])
return obs
def _block_fast_hist2d(x, y, bins, range=None, weights=None) -> np.ndarray:
return histogram2d(x, y, bins, range, weights)[np.newaxis, ...]
continue
data = dis_correct(data, asp, dis_map, ya_corr, q_corr, cut)
sky_data = SkyCoord(data[:,1:3], unit='deg', frame='fk5', equinox='J2000.0')
#gal = sky_data.transform_to('galactic')
idxc, idxcatalog, d2d_p, d3d_p = sky_data.search_around_sky(c, 0.0016*u.deg)
star_mask = np.zeros(data.shape[0], dtype=bool)
star_mask[idxcatalog] = True
data_f = data[star_mask, 3:5]
qmask = data[:,5]>5
data_c = data[qmask&star_mask, 3:5]
print data.shape
print data_f.shape
print data_c.shape
pos = ((data_f/36000.)/(1.25/2.)*(1.25/(800* 0.001666))+1.)/2.*size
#print np.max(pos)
#print np.min(pos)
H=histogram2d(pos[:,0], pos[:,1],\
bins=[size,size], range=([ [0,size],[0,size] ]))
detector += H
pos = ((data_c/36000.)/(1.25/2.)*(1.25/(800* 0.001666))+1.)/2.*size
#print np.max(pos)
#print np.min(pos)
H=histogram2d(pos[:,0], pos[:,1],\
bins=[size,size], range=([ [0,size],[0,size] ]))
detector_c += H
np.save('/scratch/dw1519/galex/data/star_photon/star/'+name+'_full.npy', detector)
np.save('/scratch/dw1519/galex/data/star_photon/star/'+name+'_cut.npy', detector_c)
i0 = np.min(pi) j1 = np.max(pj) j0 = np.min(pj) print(i, j, i * n + j) ax1=pl.subplot(n,n,i * n + j + 1) col = 'b' if(i != j): xedges = np.arange(j0, j1, (j1 - j0) / 21.0) yedges = np.arange(i0, i1, (i1 - i0) / 21.0) #H, xedges, yedges = np.histogram2d(pj, pi, bins=(xedges, yedges)) H = histogram2d(pj, pi, range=[[j0,j1],[i0,i1]],bins=[21,21]) extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]] HT1 = [] HTot = 0.0 for ii in range(len(xedges)-1): for jj in range(len(yedges)-1): HT1.append(H[ii][jj]) HTot += H[ii][jj] HT2 = np.sort(HT1) nH = len(HT2)
def CE(period, data, xbins=10, ybins=5):
"""
Returns the conditional entropy of *data* rephased with *period*.
**Parameters**
period : number
The period to rephase *data* by.
data : array-like, shape = [n_samples, 2] or [n_samples, 3]
Array containing columns *time*, *mag*, and (optional) *error*.
xbins : int, optional
Number of phase bins (default 10).
ybins : int, optional
Number of magnitude bins (default 5).
"""
if period <= 0:
return np.PINF
import fast_histogram
#r = rephase(data, period)
r = np.ma.array(data, copy=True)
r[:, 0] = np.mod(r[:, 0], period) / period
#bins, xedges, yedges = np.histogram2d(r[:,0], r[:,1], bins=[xbins, ybins], range=[[0,1], [0,1]])
bins = fast_histogram.histogram2d(r[:, 0],
r[:, 1],
range=[[0, 1], [0, 1]],
bins=[xbins, ybins])
size = r.shape[0]
if size > 0:
# bins[i,j] / size
divided_bins = bins / size
# indices where that is positive
# to avoid division by zero
arg_positive = divided_bins > 0
# array containing the sums of each column in the bins array
column_sums = np.sum(divided_bins, axis=1) #changed 0 by 1
# array is repeated row-wise, so that it can be sliced by arg_positive
#column_sums = np.repeat(np.reshape(column_sums, (1,-1)), xbins, axis=0)
column_sums = np.repeat(np.atleast_2d(column_sums).T, ybins, axis=1)
# select only the elements in both arrays which correspond to a
# positive bin
select_divided_bins = divided_bins[arg_positive]
select_column_sums = column_sums[arg_positive]
# initialize the result array
A = np.empty((xbins, ybins), dtype=float)
# store at every index [i,j] in A which corresponds to a positive bin:
# bins[i,j]/size * log(bins[i,:] / size / (bins[i,j]/size))
A[ arg_positive] = select_divided_bins \
* np.log(select_column_sums / select_divided_bins)
# store 0 at every index in A which corresponds to a non-positive bin
A[~arg_positive] = 0
# return the summation
return np.sum(A)
else:
return np.PINF
def __call__(self, bins=None, range=None):
ny, nx = bins
(ymin, ymax), (xmin, xmax) = range
xscale = self._ax.get_xscale()
yscale = self._ax.get_yscale()
if xscale == 'log':
xmin, xmax = log10(xmin), log10(xmax)
if self._x_log is None:
# We do this here insead of in set_xy to save time since in
# set_xy we don't know yet if the axes will be log or not.
self._update_x_log()
if self._downres:
x = self._x_log_sub
else:
x = self._x_log
elif xscale == 'linear':
if self._downres:
x = self._x_sub
else:
x = self._x
else: # pragma: nocover
raise ValueError('Unexpected xscale: {0}'.format(xscale))
if yscale == 'log':
ymin, ymax = log10(ymin), log10(ymax)
if self._y_log is None:
# We do this here insead of in set_xy to save time since in
# set_xy we don't know yet if the axes will be log or not.
self._update_y_log()
if self._downres:
y = self._y_log_sub
else:
y = self._y_log
elif yscale == 'linear':
if self._downres:
y = self._y_sub
else:
y = self._y
else: # pragma: nocover
raise ValueError('Unexpected xscale: {0}'.format(xscale))
if self._downres:
nx_sub = nx // self._downres_factor
ny_sub = ny // self._downres_factor
bins = (ny_sub, nx_sub)
weights = self._c_sub
else:
bins = (ny, nx)
weights = self._c
if weights is None:
array = histogram2d(y,
x,
bins=bins,
range=((ymin, ymax), (xmin, xmax)))
else:
array = histogram2d(y,
x,
bins=bins,
weights=weights,
range=((ymin, ymax), (xmin, xmax)))
count = histogram2d(y,
x,
bins=bins,
range=((ymin, ymax), (xmin, xmax)))
with np.errstate(invalid='ignore'):
array /= count
return array
def hist_fast(data, wcs, imsz):
foc = wcs.all_world2pix(data,1)
H=histogram2d(foc[:,1]-0.5, foc[:,0]-0.5,\
bins=imsz, range=([ [0,imsz[0]],[0,imsz[1]] ]))
return H
def get_observation(self, dm, my_orientation, their_orientation, vels,
nh_size):
if self.obs_mode == 'fix_acc':
ind = np.where(dm == -1)[0][0]
rel_vels = self.state.w_vel - vels
local_obs = self.get_local_obs_acc()
fix_obs = np.zeros(self.dim_rec_o)
fix_obs[:, 0] = np.concatenate([dm[0:ind], dm[ind + 1:]
]) / self.comm_radius
fix_obs[:, 1] = np.cos(
np.concatenate(
[my_orientation[0:ind], my_orientation[ind + 1:]]))
fix_obs[:, 2] = np.sin(
np.concatenate(
[my_orientation[0:ind], my_orientation[ind + 1:]]))
fix_obs[:, 3] = np.concatenate([
rel_vels[0:ind, 0], rel_vels[ind + 1:, 0]
]) / (2 * self.max_lin_velocity)
fix_obs[:, 4] = np.concatenate([
rel_vels[0:ind, 1], rel_vels[ind + 1:, 1]
]) / (2 * self.max_lin_velocity)
obs = np.hstack([fix_obs.flatten(), local_obs.flatten()])
elif self.obs_mode == '2d_hist_acc':
local_obs = self.get_local_obs_acc()
in_range = (0 < dm) & (dm < self.comm_radius)
hist_2d = fh.histogram2d(my_orientation[in_range],
dm[in_range],
bins=(self.bearing_bins,
self.distance_bins),
range=[[-np.pi, np.pi],
[0, self.world_size * np.sqrt(2)]])
histogram = hist_2d.flatten() / (self.n_agents - 1)
obs = np.hstack([histogram, local_obs])
elif self.obs_mode == '2d_rbf_acc':
in_range = (dm < self.comm_radius) & (0 < dm)
local_obs = self.get_local_obs_acc()
if np.any(in_range):
dbn = np.stack(
[dm[in_range], my_orientation[in_range] + np.pi], axis=1)
rbf_hist = U.get_weights_2d(dbn, self.mu, self.s, [
self.bearing_bins, self.distance_bins
]) / (self.n_agents - 1)
else:
rbf_hist = np.zeros([self.bearing_bins, self.distance_bins])
rbf_hist_flat = rbf_hist
obs = np.hstack([rbf_hist_flat, local_obs])
elif self.obs_mode == '3d_rbf':
in_range = (dm < self.comm_radius) & (0 < dm)
local_obs = self.get_local_obs_acc()
if np.any(in_range):
dbn = np.stack([
dm[in_range], my_orientation[in_range] + np.pi,
their_orientation[in_range] + np.pi
],
axis=1)
rbf_hist = U.get_weights_3d(dbn, self.mu, self.s, [
self.bearing_bins, self.distance_bins, self.bearing_bins
]) / (self.n_agents - 1)
else:
rbf_hist = np.zeros(
[self.bearing_bins, self.distance_bins, self.bearing_bins])
rbf_hist_flat = rbf_hist.flatten()
obs = np.hstack([rbf_hist_flat, local_obs])
elif self.obs_mode == '2d_rbf_acc_limited':
in_range = (dm < self.comm_radius) & (0 < dm)
nr_neighbors = np.sum(in_range)
local_obs = self.get_local_obs_acc()
local_obs[-1] = nr_neighbors / (self.n_agents - 1)
if np.any(in_range):
dbn = np.stack(
[dm[in_range], my_orientation[in_range] + np.pi], axis=1)
rbf_hist = U.get_weights_2d(dbn, self.mu, self.s, [
self.bearing_bins, self.distance_bins
]) / (self.n_agents - 1)
else:
rbf_hist = np.zeros([self.bearing_bins, self.distance_bins])
rbf_hist_flat = rbf_hist.flatten()
obs = np.hstack([rbf_hist_flat, local_obs])
elif self.obs_mode == '2d_rbf_limited':
in_range = (dm < self.comm_radius) & (0 < dm)
nr_neighbors = np.sum(in_range)
local_obs = self.get_local_obs()
local_obs[-1] = nr_neighbors / (self.n_agents - 1)
if np.any(in_range):
dbn = np.stack(
[dm[in_range], my_orientation[in_range] + np.pi], axis=1)
rbf_hist = U.get_weights_2d(dbn, self.mu, self.s, [
self.bearing_bins, self.distance_bins
]) / (self.n_agents - 1)
else:
rbf_hist = np.zeros([self.bearing_bins, self.distance_bins])
rbf_hist_flat = rbf_hist.flatten()
obs = np.hstack([rbf_hist_flat, local_obs])
elif self.obs_mode == 'sum_obs_acc':
in_range = (dm < self.comm_radius) & (0 < dm)
nr_neighbors = np.sum(in_range)
rel_vels = self.state.w_vel - vels
local_obs = self.get_local_obs_acc()
sum_obs = np.zeros(self.dim_rec_o)
sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size
sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range])
sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range])
sum_obs[:nr_neighbors,
3] = rel_vels[:, 0][in_range] / (2 * self.max_lin_velocity)
sum_obs[:nr_neighbors,
4] = rel_vels[:, 1][in_range] / (2 * self.max_lin_velocity)
sum_obs[0:nr_neighbors, 5] = 1
sum_obs[0:self.n_agents - 1, 6] = 1
obs = np.hstack([sum_obs.flatten(), local_obs])
elif self.obs_mode == 'sum_obs_acc_full':
in_range = (dm < self.comm_radius) & (0 < dm)
nr_neighbors = np.sum(in_range)
rel_vels = self.state.w_vel - vels
local_obs = self.get_local_obs_acc()
sum_obs = np.zeros(self.dim_rec_o)
sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size
sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range])
sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range])
sum_obs[0:nr_neighbors, 3] = np.cos(their_orientation[in_range])
sum_obs[0:nr_neighbors, 4] = np.sin(their_orientation[in_range])
sum_obs[:nr_neighbors,
5] = rel_vels[:, 0][in_range] / (2 * self.max_lin_velocity)
sum_obs[:nr_neighbors,
6] = rel_vels[:, 1][in_range] / (2 * self.max_lin_velocity)
sum_obs[0:nr_neighbors, 7] = 1
sum_obs[0:self.n_agents - 1, 8] = 1
obs = np.hstack([sum_obs.flatten(), local_obs])
elif self.obs_mode == 'sum_obs_acc_no_vel':
in_range = (dm < self.comm_radius) & (0 < dm)
nr_neighbors = np.sum(in_range)
local_obs = self.get_local_obs_acc()
sum_obs = np.zeros(self.dim_rec_o)
sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size
sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range])
sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range])
sum_obs[0:nr_neighbors, 3] = 1
sum_obs[0:self.n_agents - 1, 4] = 1
obs = np.hstack([sum_obs.flatten(), local_obs])
elif self.obs_mode == 'sum_obs_acc_limited':
in_range = (dm < self.comm_radius) & (0 < dm)
nr_neighbors = np.sum(in_range)
rel_vels = self.state.w_vel - vels
local_obs = self.get_local_obs_acc()
local_obs[-1] = nr_neighbors / (self.n_agents - 1)
sum_obs = np.zeros(self.dim_rec_o)
sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size
sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range])
sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range])
sum_obs[0:nr_neighbors, 3] = (nh_size[in_range] - nr_neighbors) / (self.n_agents - 2) if self.n_agents > 2\
else np.zeros(nr_neighbors)
sum_obs[:nr_neighbors,
4] = rel_vels[:, 0][in_range] / (2 * self.max_lin_velocity)
sum_obs[:nr_neighbors,
5] = rel_vels[:, 1][in_range] / (2 * self.max_lin_velocity)
sum_obs[0:nr_neighbors, 6] = 1
sum_obs[0:self.n_agents - 1, 7] = 1
obs = np.hstack([sum_obs.flatten(), local_obs])
elif self.obs_mode == 'sum_obs':
in_range = (dm < self.comm_radius) & (0 < dm)
nr_neighbors = np.sum(in_range)
local_obs = self.get_local_obs()
local_obs[-1] = nr_neighbors / (self.n_agents - 1)
sum_obs = np.zeros(self.dim_rec_o)
sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size
sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range])
sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range])
sum_obs[0:nr_neighbors, 3] = np.cos(their_orientation[in_range])
sum_obs[0:nr_neighbors, 4] = np.sin(their_orientation[in_range])
sum_obs[0:nr_neighbors, 5] = 1
sum_obs[0:self.n_agents - 1, 6] = 1
obs = np.hstack([sum_obs.flatten(), local_obs])
elif self.obs_mode == 'sum_obs_limited':
in_range = (dm < self.comm_radius) & (0 < dm)
nr_neighbors = np.sum(in_range)
local_obs = self.get_local_obs()
local_obs[-1] = nr_neighbors / (self.n_agents - 1)
sum_obs = np.zeros(self.dim_rec_o)
sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size
sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range])
sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range])
sum_obs[0:nr_neighbors, 3] = (nh_size[in_range] - nr_neighbors) / (self.n_agents - 2) if self.n_agents > 2\
else np.zeros(nr_neighbors)
sum_obs[0:nr_neighbors, 4] = np.cos(their_orientation[in_range])
sum_obs[0:nr_neighbors, 5] = np.sin(their_orientation[in_range])
sum_obs[0:nr_neighbors, 6] = 1
sum_obs[0:self.n_agents - 1, 7] = 1
obs = np.hstack([sum_obs.flatten(), local_obs])
else:
raise ValueError('histogram form must be 1D or 2D')
return obs
def run_step(self, run_number: int, step_size: int,
howlong: float) -> ReturnRunStep:
assert self.context
with self.context as ctx:
dfslot = ctx.table
min_slot = ctx.min
max_slot = ctx.max
if not (dfslot.created.any() or min_slot.has_buffered()
or max_slot.has_buffered()):
logger.info("Input buffers empty")
return self._return_run_step(self.state_blocked, steps_run=0)
min_slot.clear_buffers()
max_slot.clear_buffers()
bounds = self.get_bounds(min_slot, max_slot)
if bounds is None:
logger.debug("No bounds yet at run %d", run_number)
return self._return_run_step(self.state_blocked, steps_run=0)
xmin, xmax, ymin, ymax = bounds
if self._bounds is None:
(xdelta, ydelta) = self.get_delta(*bounds)
self._bounds = (
xmin - xdelta,
xmax + xdelta,
ymin - ydelta,
ymax + ydelta,
)
logger.info("New bounds at run %d: %s", run_number,
self._bounds)
else:
(dxmin, dxmax, dymin, dymax) = self._bounds
(xdelta, ydelta) = self.get_delta(*bounds)
assert xdelta >= 0 and ydelta >= 0
# Either the min/max has extended or shrunk beyond the deltas
if (xmin < dxmin or xmax > dxmax or ymin < dymin
or ymax > dymax) or (xmin > (dxmin + xdelta) or xmax <
(dxmax - xdelta) or ymin >
(dymin + ydelta) or ymax <
(dymax - ydelta)):
self._bounds = (
xmin - xdelta,
xmax + xdelta,
ymin - ydelta,
ymax + ydelta,
)
logger.info("Updated bounds at run %s: %s", run_number,
self._bounds)
self.reset()
dfslot.update(run_number)
xmin, xmax, ymin, ymax = self._bounds
if xmin >= xmax or ymin >= ymax:
logger.error("Invalid bounds: %s", self._bounds)
return self._return_run_step(self.state_blocked, steps_run=0)
# Now, we know we have data and bounds, proceed to create a
# new histogram or to update the previous if is still exists
# (i.e. no reset)
p = self.params
steps = 0
# if there are new deletions, build the histogram of the del. pairs
# then subtract it from the main histogram
if dfslot.base.deleted.any():
self.reset()
dfslot.update(run_number)
elif (dfslot.selection.deleted.any()
and self._histo is not None): # i.e. TableSelectedView
input_df = dfslot.data().base # the original table
# we assume that deletions are only local to the view
# and the related records still exist in the original table ...
# TODO : test this hypothesis and reset if false
raw_indices = dfslot.selection.deleted.next(length=step_size)
indices = fix_loc(raw_indices)
steps += indices_len(indices)
x = input_df.to_array(locs=indices,
columns=[self.x_column]).reshape(-1)
y = input_df.to_array(locs=indices,
columns=[self.y_column]).reshape(-1)
bins = [p.ybins, p.xbins]
if len(x) > 0:
histo = histogram2d(y,
x,
bins=bins,
range=[[ymin, ymax], [xmin, xmax]])
self._histo -= histo
# if there are new creations, build a partial histogram with them
# add it to the main histogram
if not dfslot.created.any():
return self._return_run_step(self.state_blocked, steps_run=0)
input_df = dfslot.data()
raw_indices = dfslot.created.next(length=step_size)
indices = fix_loc(raw_indices)
steps += indices_len(indices)
logger.info("Read %d rows", steps)
self.total_read += steps
x = input_df.to_array(locs=indices,
columns=[self.x_column]).reshape(-1)
y = input_df.to_array(locs=indices,
columns=[self.y_column]).reshape(-1)
if self._xedges is not None:
bins = [self._xedges, self._yedges]
else:
bins = [p.ybins, p.xbins]
if len(x) > 0:
histo = histogram2d(y,
x,
bins=bins,
range=[[ymin, ymax], [xmin, xmax]])
else:
return self._return_run_step(self.state_blocked, steps_run=0)
if self._histo is None:
self._histo = histo
elif histo is not None:
self._histo += histo
if self._histo is not None:
cmax = self._histo.max()
values = {
"array": np.flip(self._histo, axis=0), # type: ignore
"cmin": 0,
"cmax": cmax,
"xmin": xmin,
"xmax": xmax,
"ymin": ymin,
"ymax": ymax,
"time": run_number,
}
if self._with_output:
table = self.table
table["array"].set_shape([p.ybins, p.xbins])
last = table.last()
if last is None or last["time"] != run_number:
table.add(values)
else:
table.loc[last.row] = values
self.build_heatmap(values)
return self._return_run_step(self.next_state(dfslot),
steps_run=steps)
def mutual_information(i1, i2, bins=256):
r"""
Computes the mutual information (MI) (a measure of entropy) between two images.
MI is not real metric, but a symmetric and nonnegative similarity measures that
takes high values for similar images. Negative values are also possible.
Intuitively, mutual information measures the information that ``i1`` and ``i2`` share: it
measures how much knowing one of these variables reduces uncertainty about the other.
The Entropy is defined as:
.. math::
H(X) = - \sum_i p(g_i) * ln(p(g_i)
with :math:`p(g_i)` being the intensity probability of the images grey value :math:`g_i`.
Assuming two images :math:`R` and :math:`T`, the mutual information is then computed by comparing the
images entropy values (i.e. a measure how well-structured the common histogram is).
The distance metric is then calculated as follows:
.. math::
MI(R,T) = H(R) + H(T) - H(R,T) = H(R) - H(R|T) = H(T) - H(T|R)
A maximization of the mutual information is equal to a minimization of the joint
entropy.
Parameters
----------
i1 : array_like
The first image.
i2 : array_like
The second image.
bins : integer
The number of histogram bins (squared for the joined histogram).
Returns
-------
mutual_information : float
The mutual information distance value between the supplied images.
Raises
------
ArgumentError
If the supplied arrays are of different shape.
"""
# pre-process function arguments
i1 = numpy.asarray(i1)
i2 = numpy.asarray(i2)
# # validate function arguments
# if not i1.shape == i2.shape:
# raise ArgumentError('the two supplied array-like sequences i1 and i2 must be of the same shape')
# compute i1 and i2 histogram range
i1_range = __range(i1, bins)
i2_range = __range(i2, bins)
# compute joined and separated normed histograms
i1i2_hist = histogram2d(i1.flatten(), i2.flatten(), bins=bins, range=[i1_range, i2_range]) # Note: histogram2d does not flatten array on its own
i1_hist = histogram1d(i1, bins=bins, range=i1_range)
i2_hist = histogram1d(i2, bins=bins, range=i2_range)
# compute joined and separated entropy
i1i2_entropy = __entropy(i1i2_hist)
i1_entropy = __entropy(i1_hist)
i2_entropy = __entropy(i2_hist)
# compute and return the mutual information distance
return i1_entropy + i2_entropy - i1i2_entropy
def __call__(self, bins=None, range=None, return_density=False):
ny, nx = bins
(ymin, ymax), (xmin, xmax) = range
xscale = self._ax.get_xscale()
yscale = self._ax.get_yscale()
if xscale == 'log':
xmin, xmax = log10(xmin), log10(xmax)
if self._x_log is None:
# We do this here insead of in set_xy to save time since in
# set_xy we don't know yet if the axes will be log or not.
self._update_x_log()
if self._downres:
x = self._x_log_sub
else:
x = self._x_log
elif xscale == 'linear':
if self._downres:
x = self._x_sub
else:
x = self._x
else: # pragma: nocover
raise ValueError('Unexpected xscale: {0}'.format(xscale))
if yscale == 'log':
ymin, ymax = log10(ymin), log10(ymax)
if self._y_log is None:
# We do this here insead of in set_xy to save time since in
# set_xy we don't know yet if the axes will be log or not.
self._update_y_log()
if self._downres:
y = self._y_log_sub
else:
y = self._y_log
elif yscale == 'linear':
if self._downres:
y = self._y_sub
else:
y = self._y
else: # pragma: nocover
raise ValueError('Unexpected xscale: {0}'.format(xscale))
if self._downres:
nx_sub = nx // self._downres_factor
ny_sub = ny // self._downres_factor
bins = (ny_sub, nx_sub)
weights = self._c_sub
else:
bins = (ny, nx)
weights = self._c
if weights is None:
array = histogram2d(y, x, bins=bins,
range=((ymin, ymax), (xmin, xmax)))
density = array / np.nanmax(array)
else:
array = histogram2d(y, x, bins=bins, weights=weights,
range=((ymin, ymax), (xmin, xmax)))
count = histogram2d(y, x, bins=bins,
range=((ymin, ymax), (xmin, xmax)))
density = count / np.nanmax(count)
with np.errstate(invalid='ignore'):
array /= count
if return_density: return array, density
else: return array
if len(data)==0:
print('skip')
continue
data = dis_correct(data, asp, dis_map, ya_corr, q_corr, cut)
sky_data = SkyCoord(data[:,1:3], unit='deg', frame='fk5', equinox='J2000.0')
#gal = sky_data.transform_to('galactic')
idxc, idxcatalog, d2d_p, d3d_p = sky_data.search_around_sky(c, 0.01*u.deg)
star_mask = np.ones(data.shape[0], dtype=bool)
star_mask[idxcatalog] = False
data_f = data[star_mask, 3:5]
qmask = data_f[:,5]>5
data_c = data_f[qmask]
pos = ((data_f/36000.)/(1.25/2.)*(1.25/(800* 0.001666))+1.)/2.*size
#print np.max(pos)
#print np.min(pos)
H=histogram2d(pos[:,0], pos[:,1],\
bins=[size,size], range=([ [0,size],[0,size] ]))
detector += H
data_c = dis_correct_c(data, dis_map, ya_corr, q_corr, cut)
pos = ((data_c/36000.)/(1.25/2.)*(1.25/(800* 0.001666))+1.)/2.*size
#print np.max(pos)
#print np.min(pos)
H=histogram2d(pos[:,0], pos[:,1],\
bins=[size,size], range=([ [0,size],[0,size] ]))
detector_c += H
np.save('/scratch/dw1519/galex/data/star_photon/back/'+name+'_full.npy', detector)
np.save('/scratch/dw1519/galex/data/star_photon/back/'+name+'_cut.npy', detector_c)
def make_image(self, *args, **kwargs):
xmin, xmax = self._ax.get_xlim()
ymin, ymax = self._ax.get_ylim()
xscale = self._ax.get_xscale()
yscale = self._ax.get_yscale()
if self._dpi is None:
dpi = self.axes.figure.get_dpi()
else:
dpi = self._dpi
width = (self._ax.get_position().width *
self._ax.figure.get_figwidth())
height = (self._ax.get_position().height *
self._ax.figure.get_figheight())
nx = int(round(width * dpi))
ny = int(round(height * dpi))
flip_x = xmin > xmax
flip_y = ymin > ymax
if flip_x:
xmin, xmax = xmax, xmin
if flip_y:
ymin, ymax = ymax, ymin
if xscale == 'log':
xmin, xmax = log10(xmin), log10(xmax)
if self._x_log is None:
# We do this here insead of in set_xy to save time since in
# set_xy we don't know yet if the axes will be log or not.
self._update_x_log()
if self._downres:
x = self._x_log_sub
else:
x = self._x_log
elif xscale == 'linear':
if self._downres:
x = self._x_sub
else:
x = self._x
else: # pragma: nocover
raise ValueError('Unexpected xscale: {0}'.format(xscale))
if yscale == 'log':
ymin, ymax = log10(ymin), log10(ymax)
if self._y_log is None:
# We do this here insead of in set_xy to save time since in
# set_xy we don't know yet if the axes will be log or not.
self._update_y_log()
if self._downres:
y = self._y_log_sub
else:
y = self._y_log
elif yscale == 'linear':
if self._downres:
y = self._y_sub
else:
y = self._y
else: # pragma: nocover
raise ValueError('Unexpected xscale: {0}'.format(xscale))
if self._downres:
nx_sub = nx // self._downres_factor
ny_sub = ny // self._downres_factor
bins = (ny_sub, nx_sub)
weights = self._c_sub
else:
bins = (ny, nx)
weights = self._c
if weights is None:
array = histogram2d(y,
x,
bins=bins,
weights=weights,
range=((ymin, ymax), (xmin, xmax)))
else:
array = histogram2d(y,
x,
bins=bins,
weights=weights,
range=((ymin, ymax), (xmin, xmax)))
count = histogram2d(y,
x,
bins=bins,
range=((ymin, ymax), (xmin, xmax)))
with np.errstate(invalid='ignore'):
array /= count
if flip_x or flip_y:
if flip_x and flip_y:
array = array[::-1, ::-1]
elif flip_x:
array = array[:, ::-1]
else:
array = array[::-1, :]
if self.origin == 'upper':
array = np.flipud(array)
if callable(self._density_vmin):
vmin = self._density_vmin(array)
else:
vmin = self._density_vmin
if callable(self._density_vmax):
vmax = self._density_vmax(array)
else:
vmax = self._density_vmax
self.set_data(array)
super(ScatterDensityArtist, self).set_clim(vmin, vmax)
return super(ScatterDensityArtist, self).make_image(*args, **kwargs)
def hist_fast(data, wcs, imsz):
foc = wcs.all_world2pix(data, 1)
H=histogram2d(foc[:,1]-0.5, foc[:,0]-0.5,\
bins=imsz, range=([ [0,imsz[0]],[0,imsz[1]] ]))
return H
def fill(self, xarr, yarr):
hists = histogram2d(xarr, yarr, [self.nxbins, self.nybins],
[self.xrange, self.yrange])
self.hists += hists
