def t_histogram2d_impl(self, **kw: Any) -> None: s = self.scheduler() random = RandomTable(3, rows=100000, scheduler=s) stirrer = Stirrer(update_column="_2", fixed_step_size=1000, scheduler=s, **kw) stirrer.input[0] = random.output.result min_ = Min(scheduler=s) min_.input[0] = stirrer.output.result max_ = Max(scheduler=s) max_.input[0] = stirrer.output.result histogram2d = Histogram2D( 0, 1, xbins=100, ybins=100, scheduler=s ) # columns are called 1..30 histogram2d.input[0] = stirrer.output.result histogram2d.input.min = min_.output.result histogram2d.input.max = max_.output.result heatmap = Heatmap(filename="histo_%03d.png", scheduler=s) heatmap.input.array = histogram2d.output.result pr = Every(proc=self.terse, scheduler=s) pr.input[0] = heatmap.output.result aio.run(s.start()) last = notNone(histogram2d.table.last()).to_dict() h1 = last["array"] bounds = [[last["ymin"], last["ymax"]], [last["xmin"], last["xmax"]]] t = stirrer.table.loc[:, ["_1", "_2"]] assert t is not None v = t.to_array() bins = [histogram2d.params.ybins, histogram2d.params.xbins] h2 = fh.histogram2d(v[:, 1], v[:, 0], bins=bins, range=bounds) h2 = np.flip(h2, axis=0) # type: ignore self.assertEqual(np.sum(h1), np.sum(h2)) self.assertListEqual(h1.reshape(-1).tolist(), h2.reshape(-1).tolist())
def test_histogram2d1(self) -> None: s = self.scheduler() csv = CSVLoader( get_dataset("bigfile"), index_col=False, header=None, scheduler=s ) min_ = Min(scheduler=s) min_.input[0] = csv.output.result max_ = Max(scheduler=s) max_.input[0] = csv.output.result histogram2d = Histogram2D( 1, 2, xbins=100, ybins=100, scheduler=s ) # columns are called 1..30 histogram2d.input[0] = csv.output.result histogram2d.input.min = min_.output.result histogram2d.input.max = max_.output.result heatmap = Heatmap(filename="histo_%03d.png", scheduler=s) heatmap.input.array = histogram2d.output.result pr = Every(proc=self.terse, scheduler=s) pr.input[0] = heatmap.output.result aio.run(csv.scheduler().start()) last = notNone(histogram2d.table.last()).to_dict() h1 = last["array"] bounds = [[last["ymin"], last["ymax"]], [last["xmin"], last["xmax"]]] df = pd.read_csv( get_dataset("bigfile"), header=None, usecols=[1, 2] # type: ignore ) v = df.to_numpy() # .reshape(-1, 2) bins = [histogram2d.params.ybins, histogram2d.params.xbins] h2 = fh.histogram2d(v[:, 1], v[:, 0], bins=bins, range=bounds) h2 = np.flip(h2, axis=0) # type: ignore self.assertTrue(np.allclose(h1, h2))
def inject_stars(self, x, y, signal, xy_dim, mask=None): """ Inject sources into image. Parameters ---------- x : `~numpy.ndarray` Source x coordinates. y : `~numpy.ndarray` Source y coordinates. signal : `~numpy.ndarray` The signal to inject into image (e.g., flux, counts, etc...). xy_dim : int or list-like The dimensions of mock image in xy units. If `int` is given, it is assumed to be both the x and y dimensions: (xy_dim, xy_dim). mask : `~numpy.ndarray`, optional Boolean mask that is set to True for stars you want to inject. If None, all stars will be injected. Returns ------- image : `~numpy.ndarray` The mock image *before* psf convolution. """ bins = tuple(np.asarray(xy_dim).astype(int)) hist_range = [[0, bins[0] - 1], [0, bins[1] - 1]] if mask is not None: _x = x[mask] _y = y[mask] _s = signal[mask] else: _x, _y, _s = x, y, signal image = histogram2d(_x, _y, bins=bins, weights=_s, range=hist_range).T return image
def get_density(lats, lons, nlevels, confobj): bins=(confobj.dy, confobj.dx) ranges=((confobj.ymin,confobj.ymax), (confobj.xmin,confobj.xmax)) bins = np.asarray(bins).astype(np.int64) ranges = np.asarray(ranges).astype(np.float64) edges = (np.linspace(*ranges[0,:], bins[0]+1),np.linspace(*ranges[1,:], bins[1]+1)) print(np.shape(lats)) density = fast_histogram.histogram2d(lats, lons, bins=bins, range=ranges) print("Done with density") H2, *ledges = np.histogram2d(lats, lons, bins=bins, range=ranges) print("Done with numpy") # density = histogram2d(lats, lons, range=[[confobj.ymin,confobj.ymax],[confobj.xmin,confobj.xmax]], bins=[confobj.dy,confobj.dx]) print(np.sum(density-H2)) density=H2 total = np.sum(density) density=(density/total)*100. print("Total number of points {} percentage sum {}".format(total,np.sum(density))) density = ma.masked_where(density == 0, density) levels = MaxNLocator(nbins=nlevels).tick_values(0.1,2) #norm .tick_values(density.min(),density.max()) norm = BoundaryNorm(levels, ncolors=confobj.cmap.N, clip=True) # Turn the lon/lat of the bins into 2 dimensional arrays lon_bins_2d, lat_bins_2d = np.meshgrid(confobj.lon_bins, confobj.lat_bins) return lon_bins_2d,lat_bins_2d,density,norm
def inject(self, x, y, signal, xy_dim): """ Inject sources into image. Parameters ---------- x : `~numpy.ndarray` Source x coordinates. y : `~numpy.ndarray` Source y coordinates. signal : `~numpy.ndarray` The signal to inject into image (e.g., flux, counts, etc...). xy_dim : int or list-like The dimensions of mock image in xy units. If `int` is given, it is assumed to be both the x and y dimensions: (xy_dim, xy_dim). Returns ------- image : `~numpy.ndarray` The mock image *before* psf convolution. """ bins = tuple(np.asarray(xy_dim).astype(int)) hist_range = [[0, bins[0] - 1], [0, bins[1] - 1]] image = histogram2d(x, y, bins=bins, weights=signal, range=hist_range).T return image
def density_fluctuations1(x, y, average_density, range_, n_bins_): """ This function calculates giant density fluctuations using provided particle positions and bin edges. *NOTE*: this particular version of the algorithm is sub-optimal i.e., it is possible to make it faster by finding the square of a histogram first and then finding its mean. However, due to usage of a less precise alternative (some particles can be missing in the histogram) to the numpy histogram2d that might result in negative values of the variance. Thus, we subtract the mean value first and find the square next potentially having slightly incorrect results. However, large datasets should not suffer from this significantly. FIXME try to use the number of particles in all bins instead od system population. Check how fast it is. :param x: x coordinate of particles :param y: y coordinate of particles :param average_density: average number of particles per bin :param range_: the range of x, y coordinates. By default the lower bound is 0, thus only the upper bound is provided :param n_bins_: number of bins in x,y directions :return: the normalized value of density fluctuations """ h = histogram2d(x, y, range=[[0, range_[0]], [0, range_[1]]], bins=[n_bins_[0], n_bins_[1]]) h -= average_density h *= h # squared std is marked as variance normed_std = sqrt(mean(h) / average_density) return normed_std
def pixelize(R): r""" Pixelize an area of 2R x 2R on the projection plane; bin particles into the pixels according to their 2D coordinates. Syntax: pixelize(R) where R: spatial scale of interest [kpc] (scalar) Note that, for imshow to work porperly, return the transposed image! Note that the coordinates of the pixel edges range from -R to R, and the number of pixels is defined by cfg.Npixel (assuming a square image). Note that this function uses the 2D coordinates in global 2D array cfg.RR, as well as the weights in global array cfg.W. """ im = histogram2d(cfg.RR[:,0],cfg.RR[:,1], range=[[-R, R], [-R, R]], bins=[cfg.Npixel,cfg.Npixel], weights=cfg.weight) PixelArea = ( R / float(cfg.Npixel) )**2. im = im / PixelArea #im[im==0.] = cfg.Sigma_sky # <<< play with im[im==0.] = 1e-6 * im.max() return im.T
def count_and_convert_interfaces_to_matrix(pcg, n_minerals): """Counts frequencies of interfaces via fasthistogram module and subsequently converts them to matrix format. """ return histogram2d(pcg[:-1], pcg[1:], range=[[0, n_minerals], [0, n_minerals]], bins=n_minerals).astype(np.int32)
def fhist2d(x1,y1,xmin,xmax,xbinsize,ymin,ymax,ybinsize): xbins=int((xmax-xmin)/xbinsize) ybins=int((ymax-ymin)/ybinsize) x=np.linspace(xmin,xmax,xbins+1) y=np.linspace(ymin,ymax,ybins+1) centersx=(x[:-1]+x[1:])/2 centersy=(y[:-1]+y[1:])/2 test_z=histogram2d(x1,y1,bins=[xbins,ybins],range=[[xmin,xmax],[ymin,ymax]]) return centersx,centersy,test_z
def calc_rg_hist(x, y, bin_): """Calculate the two-dimensional rg histogram with 'bin_' bin from the given 'x' and 'y' arrays. Typical use: img = cv2.imread("image.png") b, g, r = calc_bgr(img) calc_2d_rg_hist(r, g, bin_=256) """ return histogram2d(y.ravel(), x.ravel(), bins=[bin_, bin_], range=[[0, 1], [0, 1]])
def calc_mutual_information(ndfx: np.ndarray, ndfy: np.ndarray, *args, bins: int = 100, base_max: int = 1) -> (np.ndarray, np.ndarray): """ 相互情報量を計算する Params:: ndfx, ndfy: 入力は0~base_maxに正規化されているとする """ """ DEBUG import numpy as np from fast_histogram import histogram1d, histogram2d x = np.random.rand(1000, 20) y = np.random.rand(1000, 10) ndfx, ndfy, bins, base_max = x, y, 100, 1 """ from fast_histogram import histogram1d, histogram2d logger.info("START") list_ndf = [] for x in ndfx.T: for y in ndfy.T: ndf = histogram2d(x, y, range=[[0, base_max], [0, base_max]], bins=bins) ndf = (ndf / ndf.sum()).astype(np.float16) list_ndf.append(ndf.reshape(1, *ndf.shape)) ndf_xy = np.concatenate(list_ndf, axis=0) ndf_x = np.array( [histogram1d(x, range=[0, base_max], bins=bins) for x in ndfx.T]) / ndfx.shape[0] ndf_y = np.array( [histogram1d(x, range=[0, base_max], bins=bins) for x in ndfy.T]) / ndfy.shape[0] ndf_x = np.tile(np.tile(ndf_x.reshape(-1, bins, 1), bins), (1, ndfy.shape[1], 1)).reshape(-1, bins, bins) ndf_y = np.tile( np.tile(ndf_y, bins).reshape(-1, bins, bins), (ndfx.shape[1], 1, 1)) ndf_x_y = ndf_x * ndf_y elem: np.ma.core.MaskedArray = ndf_xy * np.ma.log(ndf_xy / ndf_x_y) val: np.ma.core.MaskedArray = np.sum(elem * base_max / bins * base_max / bins, axis=(1, 2)) val = np.ma.filled(val, 0) index_x = np.tile(np.arange(ndfx.shape[1]).reshape(-1, 1), ndfy.shape[1]).reshape(-1) index_y = np.tile(np.arange(ndfy.shape[1]), ndfx.shape[1]).reshape(-1) index_list = np.concatenate([[index_x], [index_y]], axis=0).T logger.info("END") return (index_list, val, *args)
def make_image(self, *args, **kwargs): xmin, xmax, ymin, ymax = self.get_extent() if self._dpi is None: dpi = self.axes.figure.get_dpi() else: dpi = self._dpi width = (self._ax.get_position().width * self._ax.figure.get_figwidth()) height = (self._ax.get_position().height * self._ax.figure.get_figheight()) nx = int(round(width * dpi)) ny = int(round(height * dpi)) if self._downres: nx_sub = nx // self._downres_factor ny_sub = ny // self._downres_factor array = histogram2d(self._y_sub, self._x_sub, bins=(ny_sub, nx_sub), range=((ymin, ymax), (xmin, xmax))) else: array = histogram2d(self._y, self._x, bins=(ny, nx), range=((ymin, ymax), (xmin, xmax))) if self.origin == 'upper': array = np.flipud(array) self.set_clim(np.nanmin(array), np.nanmax(array)) self.set_data(array) return super(ScatterDensityArtist, self).make_image(*args, **kwargs)
def __init__(self, xmin, xmax, ymin, ymax, nxbins=10, nybins=10): self.nxbins = nxbins self.nybins = nybins self.xedges = np.linspace(xmin, xmax, nxbins + 1) self.xcenters = (self.xedges[:-1] + self.xedges[1:]) / 2.0 self.yedges = np.linspace(ymin, ymax, nybins + 1) self.ycenters = (self.yedges[:-1] + self.yedges[1:]) / 2.0 self.delta = 0.0 self.xrange = (xmin, xmax + self.delta) self.yrange = (ymin, ymax + self.delta) self.hists = histogram2d([], [], [self.nxbins, self.nybins], [self.xrange, self.yrange])
def density_fluctuations(x, y, av_density, range_, edges): """ This function calculates giant density fluctuations using provided particle positions and bin edges :param x: x coordinate of particles :param y: y coordinate of particles :param av_density: average bin density :param range_: the range of x, y coordinates. By default the lower bound is 0, thus only the upper bound is provided :param edges: bin edges :return: the normalized value of density fluctuations """ h = histogram2d(x, y, range=[[0, range_[0]], [0, range_[1]]], bins=[edges[0], edges[1]]) e_n_squared = mean(h * h) e_squared_n = av_density * av_density deviation = sqrt(e_n_squared - e_squared_n) normed_deviation = deviation / sqrt(av_density) return normed_deviation
def MI(x, y, bins=32, range=((0, 1), (0, 1))): r"""Computes mutual information between time-series x and y. The mutual information between two distributions is a measure of correlation between them. If the distributions are independent, the mutual information will be 0. Mathematically, it is equivalent to the KL-divergence between the joint distribution and the product of the marginal distributions: .. math:: I(x, y) = D_{KL}\( p(x, y) || p(x)p(y) \) Args: x (torch.tensor): a 1d tensor representing a time series of x values y (torch.tensor): a 1d tensor representing a time series of y values bins (int): the number of bins to discretize x and y values into range (array-like; 2x2): upper and lower values which bins can take for x and y Returns: float: the mutual information of the joint and marginal distributions inferred from the time series. TODO: implement custom version in pure pytorch without relying on sklearn """ # def normalize(x): # x = x / np.sum(x) # x[x != x] = 0 # return x # def H(x): # r = x / np.sum(x) # r[r != r] = 0 # r = -r * np.log2(r) # r[r != r] = 0 # return np.sum(r) assert len(x) == len(y), "time series are of unequal length" x = x.detach().numpy() y = y.detach().numpy() cm = histogram2d(x, y, bins=bins, range=hack_range(range)) # return H(np.sum(cm, axis=1)) + H(np.sum(cm, axis=0)) - H(cm) return nats_to_bits(mutual_info_score(None, None, contingency=cm))
def run_step(self, run_number, step_size, howlong): dfslot = self.get_input_slot('table') dfslot.update(run_number) min_slot = self.get_input_slot('min') min_slot.update(run_number) max_slot = self.get_input_slot('max') max_slot.update(run_number) if dfslot.updated.any(): logger.debug('reseting histogram') self.reset() dfslot.update(run_number) if not (dfslot.created.any() or min_slot.created.any() or max_slot.created.any()): logger.info('Input buffers empty') return self._return_run_step(self.state_blocked, steps_run=0) bounds = self.get_bounds(min_slot, max_slot) if bounds is None: logger.debug('No bounds yet at run %d', run_number) return self._return_run_step(self.state_blocked, steps_run=0) xmin, xmax, ymin, ymax = bounds if self._bounds is None: (xdelta, ydelta) = self.get_delta(*bounds) self._bounds = (xmin - xdelta, xmax + xdelta, ymin - ydelta, ymax + ydelta) logger.info("New bounds at run %d: %s", run_number, self._bounds) else: (dxmin, dxmax, dymin, dymax) = self._bounds (xdelta, ydelta) = self.get_delta(*bounds) assert xdelta >= 0 and ydelta >= 0 # Either the min/max has extended, or it has shrunk beyond the deltas if ((xmin < dxmin or xmax > dxmax or ymin < dymin or ymax > dymax) or (xmin > (dxmin + xdelta) or xmax < (dxmax - xdelta) or ymin > (dymin + ydelta) or ymax < (dymax - ydelta))): #print('Old bounds: %s,%s,%s,%s'%(dxmin,dxmax,dymin,dymax)) self._bounds = (xmin - xdelta, xmax + xdelta, ymin - ydelta, ymax + ydelta) #print('Updated bounds at run %d: %s old %s deltas %s, %s'%(run_number,self._bounds, bounds, xdelta, ydelta)) logger.info('Updated bounds at run %s: %s', run_number, self._bounds) self.reset() dfslot.update(run_number) xmin, xmax, ymin, ymax = self._bounds if xmin >= xmax or ymin >= ymax: logger.error('Invalid bounds: %s', self._bounds) return self._return_run_step(self.state_blocked, steps_run=0) # Now, we know we have data and bounds, proceed to create a new histogram # or to update the previous if is still exists (i.e. no reset) p = self.params steps = 0 # if there are new deletions, build the histogram of the deleted pairs # then subtract it from the main histogram if dfslot.deleted.any() and self._histo is not None: input_df = get_physical_base(dfslot.data()) indices = dfslot.deleted.next(step_size) steps += indices_len(indices) #print('Histogram2D steps :%d'% steps) logger.info('Read %d rows', steps) x = input_df[self.x_column] y = input_df[self.y_column] idx = input_df.id_to_index(fix_loc(indices)) #print(idx) x = x[idx] y = y[idx] bins = [p.ybins, p.xbins] if len(x) > 0: histo = histogram2d(y, x, bins=bins, range=[[ymin, ymax], [xmin, xmax]]) self._histo -= histo # if there are new creations, build a partial histogram with them then # add it to the main histogram input_df = dfslot.data() indices = dfslot.created.next(step_size) steps += indices_len(indices) #print('Histogram2D steps :%d'% steps) logger.info('Read %d rows', steps) self.total_read += steps x = input_df[self.x_column] y = input_df[self.y_column] idx = input_df.id_to_index(fix_loc(indices)) #print(idx) x = x[idx] y = y[idx] if self._xedges is not None: bins = [self._xedges, self._yedges] else: bins = [p.ybins, p.xbins] if len(x) > 0: #t = default_timer() # using fast_histogram histo = histogram2d(y, x, bins=bins, range=[[ymin, ymax], [xmin, xmax]]) # using numpy histogram #histo, xedges, yedges = np.histogram2d(y, x, # bins=bins, # range=[[ymin, ymax], [xmin, xmax]], # normed=False) #t = default_timer()-t #print('Time for histogram2d: %f'%t) #self._xedges = xedges #self._yedges = yedges else: histo = None cmax = 0 if self._histo is None: self._histo = histo elif histo is not None: self._histo += histo if self._histo is not None: cmax = self._histo.max() values = { 'array': np.flip(self._histo, axis=0), 'cmin': 0, 'cmax': cmax, 'xmin': xmin, 'xmax': xmax, 'ymin': ymin, 'ymax': ymax, 'time': run_number } if self._with_output: with self.lock: table = self._table table['array'].set_shape([p.ybins, p.xbins]) l = len(table) last = table.last() if l == 0 or last['time'] != run_number: table.add(values) else: table.iloc[last.row] = values self.build_heatmap(values) return self._return_run_step(self.next_state(dfslot), steps_run=steps)
def _EI_of_layer_auto_samples(layer, batch_size, in_shape, in_range, in_bins, \ out_shape, out_range, out_bins, activation, device, threshold): """Helper function of EI_of_layer that computes the EI of layer `layer` using enough samples to be within `threshold`% of the true value. """ MULTIPLIER = 2 INTERVAL = 10000 SAMPLES_SO_FAR = INTERVAL EIs = [] def has_converged(EIs): if len(EIs) < 2: return False slope = (EIs[-2] - EIs[-1]) / INTERVAL error = slope * SAMPLES_SO_FAR * (MULTIPLIER - 1) if error / EIs[-1] > threshold: return False return True in_l, in_u = in_range num_inputs = reduce(lambda x, y: x * y, in_shape) num_outputs = reduce(lambda x, y: x * y, out_shape) ################################################# # Create histograms for each A -> B pair # ################################################# in_bin_width = (in_u - in_l) / in_bins if out_bins != 'dynamic': CMs = np.zeros((num_inputs, num_outputs, in_bins, out_bins)) # histograms for each input/output pair else: CMs = [[None for B in range(num_outputs)] for A in range(num_inputs)] if out_range == 'dynamic': dyn_out_bins = [None for B in range(num_outputs)] dyn_out_bins_set = False if out_range == 'dynamic': dyn_out_ranges = np.zeros((num_outputs, 2)) dyn_ranges_set = False while True: for chunk_size in _chunk_sizes(INTERVAL, num_inputs, num_outputs, MEMORY_LIMIT): ################################################# # Create buffers for layer input and output # ################################################# inputs = torch.zeros((chunk_size, *in_shape), device=device) outputs = torch.zeros((chunk_size, *out_shape), device=device) ################################################# # Evaluate module on noise # ################################################# for (i0, i1), bsize in _indices_and_batch_sizes( chunk_size, batch_size): sample = (in_u - in_l) * torch.rand( (bsize, *in_shape), device=device) + in_l inputs[i0:i1] = sample with torch.no_grad(): result = activation(layer(sample)) outputs[i0:i1] = result inputs = torch.flatten(inputs, start_dim=1) outputs = torch.flatten(outputs, start_dim=1) ################################################# # If specified to be dynamic, # # and first time in the loop, # # determine out_range for output neurons # ################################################# if out_range == 'dynamic' and not dyn_ranges_set: for B in range(num_outputs): out_l = torch.min(outputs[:, B]).item() out_u = torch.max(outputs[:, B]).item() dyn_out_ranges[B][0] = out_l dyn_out_ranges[B][1] = out_u dyn_ranges_set = True ################################################# # If specified to be dynamic, # # and first time in the loop, # # determine out_bins for output neurons # ################################################# if out_bins == 'dynamic' and not dyn_out_bins_set: if out_range == 'dynamic': for B in range(num_outputs): out_l, out_u = dyn_out_ranges[B] bins = int((out_u - out_l) / in_bin_width) + 1 out_u = out_l + (bins * in_bin_width) dyn_out_bins[B] = bins dyn_out_ranges[B][1] = out_u else: out_l, out_u = out_range bins = int((out_u - out_l) / in_bin_width) + 1 out_u = out_l + (bins * in_bin_width) dyn_out_bins = bins out_range = (out_l, out_u) for A in range(num_inputs): for B in range(num_outputs): if out_range == 'dynamic': out_b = dyn_out_bins[B] else: out_b = dyn_out_bins CMs[A][B] = np.zeros((in_bins, out_b)) dyn_out_bins_set = True ################################################# # Update Histograms for each A -> B pair # ################################################# for A in range(num_inputs): for B in range(num_outputs): if out_range == 'dynamic': out_r = tuple(dyn_out_ranges[B]) else: out_r = out_range if out_bins == 'dynamic': if out_range == 'dynamic': out_b = dyn_out_bins[B] else: out_b = dyn_out_bins else: out_b = out_bins CMs[A][B] += histogram2d( inputs[:, A].to('cpu').detach().numpy(), outputs[:, B].to('cpu').detach().numpy(), bins=(in_bins, out_b), range=hack_range((in_range, out_r))) ################################################# # Compute mutual information # ################################################# EI = 0.0 for A in range(num_inputs): for B in range(num_outputs): A_B_EI = nats_to_bits( mutual_info_score(None, None, contingency=CMs[A][B])) EI += A_B_EI EIs.append(EI) ################################################# # Determine whether more samples # # are needed and update how many # ################################################# if has_converged(EIs): return EIs[-1] INTERVAL = int(SAMPLES_SO_FAR * (MULTIPLIER - 1)) SAMPLES_SO_FAR += INTERVAL
def _EI_of_layer_manual_samples(layer, samples, batch_size, in_shape, in_range, in_bins, \ out_shape, out_range, out_bins, activation, device): """Helper function for EI_of_layer that computes the EI of layer `layer` with a set number of samples.""" in_l, in_u = in_range num_inputs = reduce(lambda x, y: x * y, in_shape) num_outputs = reduce(lambda x, y: x * y, out_shape) ################################################# # Create histograms for each A -> B pair # ################################################# in_bin_width = (in_u - in_l) / in_bins if out_bins != 'dynamic': CMs = np.zeros((num_inputs, num_outputs, in_bins, out_bins)) # histograms for each input/output pair else: CMs = [[None for B in range(num_outputs)] for A in range(num_inputs)] if out_range == 'dynamic': dyn_out_bins = [None for B in range(num_outputs)] dyn_out_bins_set = False if out_range == 'dynamic': dyn_out_ranges = np.zeros((num_outputs, 2)) dyn_ranges_set = False for chunk_size in _chunk_sizes(samples, num_inputs, num_outputs, MEMORY_LIMIT): ################################################# # Create buffers for layer input and output # ################################################# inputs = torch.zeros((chunk_size, *in_shape), device=device) outputs = torch.zeros((chunk_size, *out_shape), device=device) ################################################# # Evaluate module on noise # ################################################# for (i0, i1), bsize in _indices_and_batch_sizes(chunk_size, batch_size): sample = (in_u - in_l) * torch.rand( (bsize, *in_shape), device=device) + in_l inputs[i0:i1] = sample with torch.no_grad(): result = activation(layer(sample)) outputs[i0:i1] = result inputs = torch.flatten(inputs, start_dim=1) outputs = torch.flatten(outputs, start_dim=1) ################################################# # If specified to be dynamic, # # and first time in the loop, # # determine out_range for output neurons # ################################################# if out_range == 'dynamic' and not dyn_ranges_set: for B in range(num_outputs): out_l = torch.min(outputs[:, B]).item() out_u = torch.max(outputs[:, B]).item() dyn_out_ranges[B][0] = out_l dyn_out_ranges[B][1] = out_u dyn_ranges_set = True ################################################# # If specified to be dynamic, # # and first time in the loop, # # determine out_bins for output neurons # ################################################# if out_bins == 'dynamic' and not dyn_out_bins_set: if out_range == 'dynamic': for B in range(num_outputs): out_l, out_u = dyn_out_ranges[B] bins = int((out_u - out_l) / in_bin_width) + 1 out_u = out_l + (bins * in_bin_width) dyn_out_bins[B] = bins dyn_out_ranges[B][1] = out_u else: out_l, out_u = out_range bins = int((out_u - out_l) / in_bin_width) + 1 out_u = out_l + (bins * in_bin_width) dyn_out_bins = bins out_range = (out_l, out_u) for A in range(num_inputs): for B in range(num_outputs): if out_range == 'dynamic': out_b = dyn_out_bins[B] else: out_b = dyn_out_bins CMs[A][B] = np.zeros((in_bins, out_b)) dyn_out_bins_set = True ################################################# # Update Histograms for each A -> B pair # ################################################# for A in range(num_inputs): for B in range(num_outputs): if out_range == 'dynamic': out_r = tuple(dyn_out_ranges[B]) else: out_r = out_range if out_bins == 'dynamic': if out_range == 'dynamic': out_b = dyn_out_bins[B] else: out_b = dyn_out_bins else: out_b = out_bins # print("in_range: {}".format(in_range)) # print("in_bins: {}".format(in_bins)) # print("out_range: {}".format(out_r)) # print("out_bins: {}".format(out_b)) CMs[A][B] += histogram2d(inputs[:, A].to('cpu').detach().numpy(), outputs[:, B].to('cpu').detach().numpy(), bins=(in_bins, out_b), range=hack_range((in_range, out_r))) ################################################# # Compute mutual information # ################################################# EI = 0.0 for A in range(num_inputs): for B in range(num_outputs): A_B_EI = nats_to_bits( mutual_info_score(None, None, contingency=CMs[A][B])) EI += A_B_EI return EI
def get_observation(self, dm, my_orientation, their_orientation, feat, vels): evader_dists = dm[-self.n_evaders:] evader_bearings = my_orientation[-self.n_evaders:] pursuer_dists = dm[:-self.n_evaders] pursuer_bearings = my_orientation[:-self.n_evaders] if self.obs_mode == 'fix': local_obs = self.get_local_obs() if self.obs_radius > 100: dist_to_evader = evader_dists / self.obs_radius angle_to_evader = [ np.cos(evader_bearings), np.sin(evader_bearings) ] local_obs = np.zeros(self.dim_local_o) else: if evader_dists < self.obs_radius: dist_to_evader = evader_dists / self.obs_radius angle_to_evader = [ np.cos(evader_bearings), np.sin(evader_bearings) ] else: dist_to_evader = 1. angle_to_evader = [0, 0] see_evader = 1 if dist_to_evader < 1 else 0 self.see_evader = see_evader shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\ if self.graph_feature < (5 * self.comm_radius) else 1. local_obs[-1] = shortest_path_to_evader evader_obs = np.zeros(self.dim_evader_o) evader_obs[:, 0] = dist_to_evader evader_obs[:, 1] = angle_to_evader[0] evader_obs[:, 2] = angle_to_evader[1] ind = np.where(dm == -1)[0][0] fix_obs = np.zeros(self.dim_rec_o) if self.obs_radius > 100: fix_obs[:, 0] = np.concatenate([ pursuer_dists[0:ind], pursuer_dists[ind + 1:] ]) / self.comm_radius fix_obs[:, 1] = np.cos( np.concatenate( [pursuer_bearings[0:ind], pursuer_bearings[ind + 1:]])) fix_obs[:, 2] = np.sin( np.concatenate( [pursuer_bearings[0:ind], pursuer_bearings[ind + 1:]])) fix_obs[:, 3] = np.cos( np.concatenate([ their_orientation[0:ind], their_orientation[ind + 1:] ])) fix_obs[:, 4] = np.sin( np.concatenate([ their_orientation[0:ind], their_orientation[ind + 1:] ])) else: in_range = (evader_dists < self.comm_radius) & (0 < evader_dists) dists_in_range = np.array(feat)[in_range] dists_in_range_capped = np.where( dists_in_range <= 5 * self.comm_radius, dists_in_range / (5 * self.comm_radius), 1.) fix_obs[:, 0] = np.concatenate([ pursuer_dists[0:ind], pursuer_dists[ind + 1:] ]) / self.comm_radius fix_obs[:, 1] = np.cos( np.concatenate( [pursuer_bearings[0:ind], pursuer_bearings[ind + 1:]])) fix_obs[:, 2] = np.sin( np.concatenate( [pursuer_bearings[0:ind], pursuer_bearings[ind + 1:]])) fix_obs[:, 3] = np.cos( np.concatenate([ their_orientation[0:ind], their_orientation[ind + 1:] ])) fix_obs[:, 4] = np.sin( np.concatenate([ their_orientation[0:ind], their_orientation[ind + 1:] ])) fix_obs[:, 5] = dists_in_range_capped obs = np.hstack( [fix_obs.flatten(), evader_obs.flatten(), local_obs.flatten()]) elif self.obs_mode == 'sum_obs': # local obs if evader_dists < self.obs_radius: dist_to_evader = evader_dists / self.obs_radius angle_to_evader = [ np.cos(evader_bearings), np.sin(evader_bearings) ] else: dist_to_evader = 1. angle_to_evader = [0, 0] local_obs = np.zeros(self.dim_local_o) if self.torus is False: if np.any(self.state.p_pos <= 1) or np.any( self.state.p_pos >= 99): wall = 1 else: wall = 0 local_obs[0] = wall evader_obs = np.zeros(self.dim_evader_o) evader_obs[:, 0] = dist_to_evader evader_obs[:, 1] = angle_to_evader[0] evader_obs[:, 2] = angle_to_evader[1] # neighbor obs pursuers_in_range = (pursuer_dists < self.comm_radius) & (0 < pursuer_dists) nr_neighbors = np.sum(pursuers_in_range) sum_obs = np.zeros(self.dim_rec_o) nr_agents = dm.size - 1 sum_obs[0:nr_neighbors, 0] = pursuer_dists[pursuers_in_range] / self.comm_radius sum_obs[0:nr_neighbors, 1] = np.cos(pursuer_bearings[pursuers_in_range]) sum_obs[0:nr_neighbors, 2] = np.sin(pursuer_bearings[pursuers_in_range]) sum_obs[0:nr_neighbors, 3] = np.cos(their_orientation[pursuers_in_range]) sum_obs[0:nr_neighbors, 4] = np.sin(their_orientation[pursuers_in_range]) sum_obs[0:nr_neighbors, 5] = 1 sum_obs[0:nr_agents, 6] = 1 obs = np.hstack( [sum_obs.flatten(), local_obs.flatten(), evader_obs.flatten()]) elif self.obs_mode == 'sum_obs_no_ori': # local obs if evader_dists < self.obs_radius: dist_to_evader = evader_dists / self.obs_radius angle_to_evader = [ np.cos(evader_bearings), np.sin(evader_bearings) ] else: dist_to_evader = 1. angle_to_evader = [0, 0] local_obs = np.zeros(self.dim_local_o) if self.torus is False: if np.any(self.state.p_pos <= 1) or np.any( self.state.p_pos >= 99): wall = 1 else: wall = 0 local_obs[0] = wall evader_obs = np.zeros(self.dim_evader_o) evader_obs[:, 0] = dist_to_evader evader_obs[:, 1] = angle_to_evader[0] evader_obs[:, 2] = angle_to_evader[1] # neighbor obs pursuers_in_range = (pursuer_dists < self.comm_radius) & (0 < pursuer_dists) nr_neighbors = np.sum(pursuers_in_range) sum_obs = np.zeros(self.dim_rec_o) nr_agents = dm.size - 1 sum_obs[0:nr_neighbors, 0] = pursuer_dists[pursuers_in_range] / self.comm_radius sum_obs[0:nr_neighbors, 1] = np.cos(pursuer_bearings[pursuers_in_range]) sum_obs[0:nr_neighbors, 2] = np.sin(pursuer_bearings[pursuers_in_range]) sum_obs[0:nr_neighbors, 3] = 1 sum_obs[0:nr_agents, 4] = 1 obs = np.hstack( [sum_obs.flatten(), local_obs.flatten(), evader_obs.flatten()]) elif self.obs_mode == 'sum_obs_limited': # local obs if evader_dists < self.obs_radius: dist_to_evader = evader_dists / self.obs_radius angle_to_evader = [ np.cos(evader_bearings), np.sin(evader_bearings) ] else: dist_to_evader = 1. angle_to_evader = [0, 0] see_evader = 1 if dist_to_evader < 1 else 0 self.see_evader = see_evader shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\ if self.graph_feature < (5 * self.comm_radius) else 1. local_obs = np.zeros(self.dim_local_o) local_obs[0] = shortest_path_to_evader if self.torus is False: if np.any(self.state.p_pos <= 1) or np.any( self.state.p_pos >= 99): wall = 1 else: wall = 0 local_obs[1] = wall evader_obs = np.zeros(self.dim_evader_o) evader_obs[:, 0] = dist_to_evader evader_obs[:, 1] = angle_to_evader[0] evader_obs[:, 2] = angle_to_evader[1] # neighbor obs evaders_in_range = (evader_dists < self.obs_radius) & (0 < evader_dists) pursuers_in_range = (pursuer_dists < self.comm_radius) & (0 < pursuer_dists) nr_neighbors = np.sum(pursuers_in_range) dists_in_range = np.array(feat)[pursuers_in_range] dists_in_range_capped = np.where( dists_in_range <= 5 * self.comm_radius, dists_in_range / (5 * self.comm_radius), 1.) sum_obs = np.zeros(self.dim_rec_o) nr_agents = dm.size - 1 sum_obs[0:nr_neighbors, 0] = pursuer_dists[pursuers_in_range] / self.comm_radius sum_obs[0:nr_neighbors, 1] = np.cos(pursuer_bearings[pursuers_in_range]) sum_obs[0:nr_neighbors, 2] = np.sin(pursuer_bearings[pursuers_in_range]) sum_obs[0:nr_neighbors, 3] = np.cos(their_orientation[pursuers_in_range]) sum_obs[0:nr_neighbors, 4] = np.sin(their_orientation[pursuers_in_range]) sum_obs[0:nr_neighbors, 5] = dists_in_range_capped sum_obs[0:nr_neighbors, 6] = 1 sum_obs[0:nr_agents, 7] = 1 # obs = np.hstack([sum_obs.flatten(), local_obs]) obs = np.hstack( [sum_obs.flatten(), local_obs.flatten(), evader_obs.flatten()]) elif self.obs_mode == 'sum_obs_multi': in_range = (evader_dists <= self.obs_radius) & (0 <= evader_dists) nr_neighboring_evaders = np.sum(in_range) dist_to_evader = evader_dists[in_range] / self.obs_radius angle_to_evader = [ np.cos(evader_bearings[in_range]), np.sin(evader_bearings[in_range]) ] if self.obs_radius > 100: local_obs = np.zeros(self.dim_local_o) if self.torus is False: if np.any(self.state.p_pos <= 1) or np.any( self.state.p_pos >= 99): wall = 1 else: wall = 0 local_obs[0] = wall else: shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\ if self.graph_feature < (5 * self.comm_radius) else 1. local_obs = np.zeros(self.dim_local_o) local_obs[0] = shortest_path_to_evader if self.torus is False: if np.any(self.state.p_pos <= 1) or np.any( self.state.p_pos >= 99): wall = 1 else: wall = 0 local_obs[1] = wall sum_evader_obs = np.zeros(self.dim_evader_o) sum_evader_obs[:nr_neighboring_evaders, 0] = dist_to_evader sum_evader_obs[:nr_neighboring_evaders, 1] = angle_to_evader[0] sum_evader_obs[:nr_neighboring_evaders, 2] = angle_to_evader[1] sum_evader_obs[:nr_neighboring_evaders, 3] = 1 sum_evader_obs[:self.n_evaders, 4] = 1 pursuers_in_range = (pursuer_dists <= self.comm_radius) & (0 <= pursuer_dists) nr_neighbors = np.sum(pursuers_in_range) if self.obs_radius > 100: sum_obs = np.zeros(self.dim_rec_o) nr_agents = dm.size - self.n_evaders sum_obs[ 0:nr_neighbors, 0] = pursuer_dists[pursuers_in_range] / self.comm_radius sum_obs[0:nr_neighbors, 1] = np.cos(pursuer_bearings[pursuers_in_range]) sum_obs[0:nr_neighbors, 2] = np.sin(pursuer_bearings[pursuers_in_range]) sum_obs[0:nr_neighbors, 3] = np.cos(their_orientation[pursuers_in_range]) sum_obs[0:nr_neighbors, 4] = np.sin(their_orientation[pursuers_in_range]) sum_obs[0:nr_neighbors, 5] = 1 sum_obs[0:nr_agents, 6] = 1 else: dists_in_range = np.array(feat)[pursuers_in_range] dists_in_range_capped = np.where( dists_in_range <= 5 * self.comm_radius, dists_in_range / (5 * self.comm_radius), 1.) sum_obs = np.zeros(self.dim_rec_o) nr_agents = dm.size - self.n_evaders sum_obs[ 0:nr_neighbors, 0] = pursuer_dists[pursuers_in_range] / self.comm_radius sum_obs[0:nr_neighbors, 1] = np.cos(pursuer_bearings[pursuers_in_range]) sum_obs[0:nr_neighbors, 2] = np.sin(pursuer_bearings[pursuers_in_range]) sum_obs[0:nr_neighbors, 3] = np.cos(their_orientation[pursuers_in_range]) sum_obs[0:nr_neighbors, 4] = np.sin(their_orientation[pursuers_in_range]) sum_obs[0:nr_neighbors, 5] = dists_in_range_capped sum_obs[0:nr_neighbors, 6] = 1 sum_obs[0:nr_agents, 7] = 1 obs = np.hstack( [sum_obs.flatten(), sum_evader_obs.flatten(), local_obs]) elif self.obs_mode == '2d_hist': if self.obs_radius > 100: dist_to_evader = evader_dists / self.obs_radius angle_to_evader = [ np.cos(evader_bearings), np.sin(evader_bearings) ] local_obs = np.zeros(self.dim_flat_o) if self.torus is False: if np.any(self.state.p_pos <= 1) or np.any( self.state.p_pos >= 99): wall = 1 else: wall = 0 local_obs[0] = wall else: if evader_dists < self.obs_radius: dist_to_evader = evader_dists / self.obs_radius angle_to_evader = [ np.cos(evader_bearings), np.sin(evader_bearings) ] else: dist_to_evader = 1. angle_to_evader = [0, 0] see_evader = 1 if dist_to_evader < 1 else 0 self.see_evader = see_evader shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\ if self.graph_feature < (5 * self.comm_radius) else 1. local_obs = np.zeros(self.dim_flat_o) local_obs[0] = shortest_path_to_evader if self.torus is False: if np.any(self.state.p_pos <= 1) or np.any( self.state.p_pos >= 99): wall = 1 else: wall = 0 local_obs[1] = wall evader_obs = np.zeros(self.dim_evader_o) evader_obs[:, 0] = dist_to_evader evader_obs[:, 1] = angle_to_evader[0] evader_obs[:, 2] = angle_to_evader[1] # neighbor obs pursuers_in_range = (pursuer_dists < self.comm_radius) & (0 < pursuer_bearings) evader_in_range = (evader_dists < self.obs_radius) & (0 < evader_dists) nr_agents = dm.size - 2 # exclude self and evader hist_2d_agents = fh.histogram2d( pursuer_bearings[pursuers_in_range], pursuer_dists[pursuers_in_range], bins=(self.bearing_bins, self.distance_bins), range=[[-np.pi, np.pi], [0, self.world_size * np.sqrt(2) / 2]]) hist_2d_evader = fh.histogram2d( evader_bearings[evader_in_range], evader_dists[evader_in_range], bins=(self.bearing_bins, self.distance_bins), range=[[-np.pi, np.pi], [0, self.world_size * np.sqrt(2) / 2]]) histogram = np.hstack([ hist_2d_agents.flatten() / (nr_agents), hist_2d_evader.flatten() ]) obs = np.hstack([histogram, local_obs]) elif self.obs_mode == '2d_hist_short': if self.obs_radius > 100: dist_to_evader = evader_dists / self.obs_radius angle_to_evader = [ np.cos(evader_bearings), np.sin(evader_bearings) ] else: if evader_dists < self.obs_radius: dist_to_evader = evader_dists / self.obs_radius angle_to_evader = [ np.cos(evader_bearings), np.sin(evader_bearings) ] else: dist_to_evader = 1. angle_to_evader = [0, 0] see_evader = 1 if dist_to_evader < 1 else 0 self.see_evader = see_evader local_obs = self.get_local_obs() evader_obs = np.zeros(self.dim_evader_o) evader_obs[0] = dist_to_evader evader_obs[1] = angle_to_evader[0] evader_obs[2] = angle_to_evader[1] # neighbor obs in_range = (pursuer_dists < self.comm_radius) & (0 < pursuer_dists) nr_agents = dm.size - 2 # exclude self and evader hist_2d_agents = fh.histogram2d( pursuer_bearings[in_range], pursuer_dists[in_range], bins=(self.bearing_bins, self.distance_bins), range=[[-np.pi, np.pi], [0, self.world_size * np.sqrt(2) / 2]]) histogram = hist_2d_agents.flatten() / (nr_agents - 1) obs = np.hstack([histogram, evader_obs.flatten(), local_obs]) elif self.obs_mode == '2d_rbf': local_obs = np.zeros(self.dim_local_o) if self.torus is False: if np.any(self.state.p_pos <= 1) or np.any( self.state.p_pos >= 99): wall = 1 else: wall = 0 local_obs[0] = wall evader_in_range = evader_dists < self.obs_radius if np.any(evader_in_range): dbn = np.stack([ evader_dists[evader_in_range], evader_bearings[evader_in_range] + np.pi ], axis=1) evader_rbf_hist = U.get_weights_2d( dbn, self.mu_e, self.s_e, [self.bearing_bins, self.distance_bins]) else: evader_rbf_hist = np.zeros( [self.bearing_bins, self.distance_bins]) in_range = (0 < pursuer_dists) & (pursuer_dists < self.comm_radius) if np.any(in_range): dbn = np.stack([ pursuer_dists[in_range], pursuer_bearings[in_range] + np.pi ], axis=1) rbf_hist = U.get_weights_2d(dbn, self.mu_n, self.s_n, [ self.bearing_bins, self.distance_bins ]) / (self.n_agents - 1) else: rbf_hist = np.zeros([self.bearing_bins, self.distance_bins]) rbf_hist_flat = rbf_hist.flatten() obs = np.hstack( [rbf_hist_flat, evader_rbf_hist.flatten(), local_obs]) elif self.obs_mode == '2d_rbf_short': dist_to_evader = evader_dists / self.obs_radius angle_to_evader = [ np.cos(evader_bearings), np.sin(evader_bearings) ] see_evader = 1 if dist_to_evader < 1 else 0 self.see_evader = see_evader evader_obs = np.zeros(self.dim_evader_o) evader_obs[0] = dist_to_evader evader_obs[1] = angle_to_evader[0] evader_obs[2] = angle_to_evader[1] local_obs = self.get_local_obs() in_range = (0 < pursuer_dists) & (pursuer_dists < self.comm_radius) if np.any(in_range): dbn = np.stack([ pursuer_dists[in_range], pursuer_bearings[in_range] + np.pi ], axis=1) rbf_hist = U.get_weights_2d(dbn, self.mu_n, self.s_n, [ self.bearing_bins, self.distance_bins ]) / (self.n_agents - 1) else: rbf_hist = np.zeros([self.bearing_bins, self.distance_bins]) rbf_hist_flat = rbf_hist.flatten() obs = np.hstack([rbf_hist_flat, evader_obs, local_obs]) elif self.obs_mode == '2d_rbf_limited': if evader_dists < self.obs_radius: dist_to_evader = evader_dists / self.obs_radius angle_to_evader = [ np.cos(evader_bearings), np.sin(evader_bearings) ] else: dist_to_evader = 1. angle_to_evader = [0, 0] see_evader = 1 if dist_to_evader < 1 else 0 self.see_evader = see_evader shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\ if self.graph_feature < (5 * self.comm_radius) else 1. local_obs = np.zeros(self.dim_flat_o) local_obs[0] = shortest_path_to_evader if self.torus is False: if np.any(self.state.p_pos <= 1) or np.any( self.state.p_pos >= 99): wall = 1 else: wall = 0 local_obs[1] = wall evader_in_range = evader_dists < self.obs_radius sub = [] if evader_in_range: dbn = np.hstack([evader_dists, evader_bearings + np.pi]) evader_rbf_hist = U.get_weights_2d( dbn, self.mu_e, self.s_e, [self.bearing_bins, self.distance_bins]) else: evader_rbf_hist = np.zeros( [self.bearing_bins, self.distance_bins]) in_range = (0 < pursuer_dists) & (pursuer_dists < self.comm_radius) if np.any(in_range): dbn = np.stack([ pursuer_dists[in_range], pursuer_bearings[in_range] + np.pi ], axis=1) rbf_hist = U.get_weights_2d(dbn, self.mu_n, self.s_n, [ self.bearing_bins, self.distance_bins ]) / (self.n_agents - 1) else: rbf_hist = np.zeros([self.bearing_bins, self.distance_bins]) rbf_hist_flat = rbf_hist.flatten() obs = np.hstack( [rbf_hist_flat, evader_rbf_hist.flatten(), local_obs]) elif self.obs_mode == '2d_rbf_limited_short': if evader_dists < self.obs_radius: dist_to_evader = evader_dists / self.obs_radius angle_to_evader = np.array( [np.cos(evader_bearings), np.sin(evader_bearings)]) else: dist_to_evader = 1. angle_to_evader = [0, 0] see_evader = 1 if dist_to_evader < 1 else 0 self.see_evader = see_evader shortest_path_to_evader = self.graph_feature / (5 * self.comm_radius)\ if self.graph_feature < (5 * self.comm_radius) else 1. evader_obs = np.zeros(self.dim_evader_o) evader_obs[0] = dist_to_evader evader_obs[1] = angle_to_evader[0] evader_obs[2] = angle_to_evader[1] local_obs = np.zeros(self.dim_local_o) local_obs[0] = shortest_path_to_evader if self.torus is False: if np.any(self.state.p_pos <= 1) or np.any( self.state.p_pos >= 99): wall = 1 else: wall = 0 local_obs[1] = wall in_range = (0 < pursuer_dists) & (pursuer_dists < self.comm_radius) if np.any(in_range): dbn = np.stack([ pursuer_dists[in_range], pursuer_bearings[in_range] + np.pi ], axis=1) rbf_hist = U.get_weights_2d(dbn, self.mu_n, self.s_n, [ self.bearing_bins, self.distance_bins ]) / (self.n_agents - 1) else: rbf_hist = np.zeros([self.bearing_bins, self.distance_bins]) rbf_hist_flat = rbf_hist.flatten() obs = np.hstack([rbf_hist_flat, evader_obs, local_obs]) return obs
def _block_fast_hist2d(x, y, bins, range=None, weights=None) -> np.ndarray: return histogram2d(x, y, bins, range, weights)[np.newaxis, ...]
continue data = dis_correct(data, asp, dis_map, ya_corr, q_corr, cut) sky_data = SkyCoord(data[:,1:3], unit='deg', frame='fk5', equinox='J2000.0') #gal = sky_data.transform_to('galactic') idxc, idxcatalog, d2d_p, d3d_p = sky_data.search_around_sky(c, 0.0016*u.deg) star_mask = np.zeros(data.shape[0], dtype=bool) star_mask[idxcatalog] = True data_f = data[star_mask, 3:5] qmask = data[:,5]>5 data_c = data[qmask&star_mask, 3:5] print data.shape print data_f.shape print data_c.shape pos = ((data_f/36000.)/(1.25/2.)*(1.25/(800* 0.001666))+1.)/2.*size #print np.max(pos) #print np.min(pos) H=histogram2d(pos[:,0], pos[:,1],\ bins=[size,size], range=([ [0,size],[0,size] ])) detector += H pos = ((data_c/36000.)/(1.25/2.)*(1.25/(800* 0.001666))+1.)/2.*size #print np.max(pos) #print np.min(pos) H=histogram2d(pos[:,0], pos[:,1],\ bins=[size,size], range=([ [0,size],[0,size] ])) detector_c += H np.save('/scratch/dw1519/galex/data/star_photon/star/'+name+'_full.npy', detector) np.save('/scratch/dw1519/galex/data/star_photon/star/'+name+'_cut.npy', detector_c)
i0 = np.min(pi) j1 = np.max(pj) j0 = np.min(pj) print(i, j, i * n + j) ax1=pl.subplot(n,n,i * n + j + 1) col = 'b' if(i != j): xedges = np.arange(j0, j1, (j1 - j0) / 21.0) yedges = np.arange(i0, i1, (i1 - i0) / 21.0) #H, xedges, yedges = np.histogram2d(pj, pi, bins=(xedges, yedges)) H = histogram2d(pj, pi, range=[[j0,j1],[i0,i1]],bins=[21,21]) extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]] HT1 = [] HTot = 0.0 for ii in range(len(xedges)-1): for jj in range(len(yedges)-1): HT1.append(H[ii][jj]) HTot += H[ii][jj] HT2 = np.sort(HT1) nH = len(HT2)
def CE(period, data, xbins=10, ybins=5): """ Returns the conditional entropy of *data* rephased with *period*. **Parameters** period : number The period to rephase *data* by. data : array-like, shape = [n_samples, 2] or [n_samples, 3] Array containing columns *time*, *mag*, and (optional) *error*. xbins : int, optional Number of phase bins (default 10). ybins : int, optional Number of magnitude bins (default 5). """ if period <= 0: return np.PINF import fast_histogram #r = rephase(data, period) r = np.ma.array(data, copy=True) r[:, 0] = np.mod(r[:, 0], period) / period #bins, xedges, yedges = np.histogram2d(r[:,0], r[:,1], bins=[xbins, ybins], range=[[0,1], [0,1]]) bins = fast_histogram.histogram2d(r[:, 0], r[:, 1], range=[[0, 1], [0, 1]], bins=[xbins, ybins]) size = r.shape[0] if size > 0: # bins[i,j] / size divided_bins = bins / size # indices where that is positive # to avoid division by zero arg_positive = divided_bins > 0 # array containing the sums of each column in the bins array column_sums = np.sum(divided_bins, axis=1) #changed 0 by 1 # array is repeated row-wise, so that it can be sliced by arg_positive #column_sums = np.repeat(np.reshape(column_sums, (1,-1)), xbins, axis=0) column_sums = np.repeat(np.atleast_2d(column_sums).T, ybins, axis=1) # select only the elements in both arrays which correspond to a # positive bin select_divided_bins = divided_bins[arg_positive] select_column_sums = column_sums[arg_positive] # initialize the result array A = np.empty((xbins, ybins), dtype=float) # store at every index [i,j] in A which corresponds to a positive bin: # bins[i,j]/size * log(bins[i,:] / size / (bins[i,j]/size)) A[ arg_positive] = select_divided_bins \ * np.log(select_column_sums / select_divided_bins) # store 0 at every index in A which corresponds to a non-positive bin A[~arg_positive] = 0 # return the summation return np.sum(A) else: return np.PINF
def __call__(self, bins=None, range=None): ny, nx = bins (ymin, ymax), (xmin, xmax) = range xscale = self._ax.get_xscale() yscale = self._ax.get_yscale() if xscale == 'log': xmin, xmax = log10(xmin), log10(xmax) if self._x_log is None: # We do this here insead of in set_xy to save time since in # set_xy we don't know yet if the axes will be log or not. self._update_x_log() if self._downres: x = self._x_log_sub else: x = self._x_log elif xscale == 'linear': if self._downres: x = self._x_sub else: x = self._x else: # pragma: nocover raise ValueError('Unexpected xscale: {0}'.format(xscale)) if yscale == 'log': ymin, ymax = log10(ymin), log10(ymax) if self._y_log is None: # We do this here insead of in set_xy to save time since in # set_xy we don't know yet if the axes will be log or not. self._update_y_log() if self._downres: y = self._y_log_sub else: y = self._y_log elif yscale == 'linear': if self._downres: y = self._y_sub else: y = self._y else: # pragma: nocover raise ValueError('Unexpected xscale: {0}'.format(xscale)) if self._downres: nx_sub = nx // self._downres_factor ny_sub = ny // self._downres_factor bins = (ny_sub, nx_sub) weights = self._c_sub else: bins = (ny, nx) weights = self._c if weights is None: array = histogram2d(y, x, bins=bins, range=((ymin, ymax), (xmin, xmax))) else: array = histogram2d(y, x, bins=bins, weights=weights, range=((ymin, ymax), (xmin, xmax))) count = histogram2d(y, x, bins=bins, range=((ymin, ymax), (xmin, xmax))) with np.errstate(invalid='ignore'): array /= count return array
def hist_fast(data, wcs, imsz): foc = wcs.all_world2pix(data,1) H=histogram2d(foc[:,1]-0.5, foc[:,0]-0.5,\ bins=imsz, range=([ [0,imsz[0]],[0,imsz[1]] ])) return H
def get_observation(self, dm, my_orientation, their_orientation, vels, nh_size): if self.obs_mode == 'fix_acc': ind = np.where(dm == -1)[0][0] rel_vels = self.state.w_vel - vels local_obs = self.get_local_obs_acc() fix_obs = np.zeros(self.dim_rec_o) fix_obs[:, 0] = np.concatenate([dm[0:ind], dm[ind + 1:] ]) / self.comm_radius fix_obs[:, 1] = np.cos( np.concatenate( [my_orientation[0:ind], my_orientation[ind + 1:]])) fix_obs[:, 2] = np.sin( np.concatenate( [my_orientation[0:ind], my_orientation[ind + 1:]])) fix_obs[:, 3] = np.concatenate([ rel_vels[0:ind, 0], rel_vels[ind + 1:, 0] ]) / (2 * self.max_lin_velocity) fix_obs[:, 4] = np.concatenate([ rel_vels[0:ind, 1], rel_vels[ind + 1:, 1] ]) / (2 * self.max_lin_velocity) obs = np.hstack([fix_obs.flatten(), local_obs.flatten()]) elif self.obs_mode == '2d_hist_acc': local_obs = self.get_local_obs_acc() in_range = (0 < dm) & (dm < self.comm_radius) hist_2d = fh.histogram2d(my_orientation[in_range], dm[in_range], bins=(self.bearing_bins, self.distance_bins), range=[[-np.pi, np.pi], [0, self.world_size * np.sqrt(2)]]) histogram = hist_2d.flatten() / (self.n_agents - 1) obs = np.hstack([histogram, local_obs]) elif self.obs_mode == '2d_rbf_acc': in_range = (dm < self.comm_radius) & (0 < dm) local_obs = self.get_local_obs_acc() if np.any(in_range): dbn = np.stack( [dm[in_range], my_orientation[in_range] + np.pi], axis=1) rbf_hist = U.get_weights_2d(dbn, self.mu, self.s, [ self.bearing_bins, self.distance_bins ]) / (self.n_agents - 1) else: rbf_hist = np.zeros([self.bearing_bins, self.distance_bins]) rbf_hist_flat = rbf_hist obs = np.hstack([rbf_hist_flat, local_obs]) elif self.obs_mode == '3d_rbf': in_range = (dm < self.comm_radius) & (0 < dm) local_obs = self.get_local_obs_acc() if np.any(in_range): dbn = np.stack([ dm[in_range], my_orientation[in_range] + np.pi, their_orientation[in_range] + np.pi ], axis=1) rbf_hist = U.get_weights_3d(dbn, self.mu, self.s, [ self.bearing_bins, self.distance_bins, self.bearing_bins ]) / (self.n_agents - 1) else: rbf_hist = np.zeros( [self.bearing_bins, self.distance_bins, self.bearing_bins]) rbf_hist_flat = rbf_hist.flatten() obs = np.hstack([rbf_hist_flat, local_obs]) elif self.obs_mode == '2d_rbf_acc_limited': in_range = (dm < self.comm_radius) & (0 < dm) nr_neighbors = np.sum(in_range) local_obs = self.get_local_obs_acc() local_obs[-1] = nr_neighbors / (self.n_agents - 1) if np.any(in_range): dbn = np.stack( [dm[in_range], my_orientation[in_range] + np.pi], axis=1) rbf_hist = U.get_weights_2d(dbn, self.mu, self.s, [ self.bearing_bins, self.distance_bins ]) / (self.n_agents - 1) else: rbf_hist = np.zeros([self.bearing_bins, self.distance_bins]) rbf_hist_flat = rbf_hist.flatten() obs = np.hstack([rbf_hist_flat, local_obs]) elif self.obs_mode == '2d_rbf_limited': in_range = (dm < self.comm_radius) & (0 < dm) nr_neighbors = np.sum(in_range) local_obs = self.get_local_obs() local_obs[-1] = nr_neighbors / (self.n_agents - 1) if np.any(in_range): dbn = np.stack( [dm[in_range], my_orientation[in_range] + np.pi], axis=1) rbf_hist = U.get_weights_2d(dbn, self.mu, self.s, [ self.bearing_bins, self.distance_bins ]) / (self.n_agents - 1) else: rbf_hist = np.zeros([self.bearing_bins, self.distance_bins]) rbf_hist_flat = rbf_hist.flatten() obs = np.hstack([rbf_hist_flat, local_obs]) elif self.obs_mode == 'sum_obs_acc': in_range = (dm < self.comm_radius) & (0 < dm) nr_neighbors = np.sum(in_range) rel_vels = self.state.w_vel - vels local_obs = self.get_local_obs_acc() sum_obs = np.zeros(self.dim_rec_o) sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range]) sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range]) sum_obs[:nr_neighbors, 3] = rel_vels[:, 0][in_range] / (2 * self.max_lin_velocity) sum_obs[:nr_neighbors, 4] = rel_vels[:, 1][in_range] / (2 * self.max_lin_velocity) sum_obs[0:nr_neighbors, 5] = 1 sum_obs[0:self.n_agents - 1, 6] = 1 obs = np.hstack([sum_obs.flatten(), local_obs]) elif self.obs_mode == 'sum_obs_acc_full': in_range = (dm < self.comm_radius) & (0 < dm) nr_neighbors = np.sum(in_range) rel_vels = self.state.w_vel - vels local_obs = self.get_local_obs_acc() sum_obs = np.zeros(self.dim_rec_o) sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range]) sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range]) sum_obs[0:nr_neighbors, 3] = np.cos(their_orientation[in_range]) sum_obs[0:nr_neighbors, 4] = np.sin(their_orientation[in_range]) sum_obs[:nr_neighbors, 5] = rel_vels[:, 0][in_range] / (2 * self.max_lin_velocity) sum_obs[:nr_neighbors, 6] = rel_vels[:, 1][in_range] / (2 * self.max_lin_velocity) sum_obs[0:nr_neighbors, 7] = 1 sum_obs[0:self.n_agents - 1, 8] = 1 obs = np.hstack([sum_obs.flatten(), local_obs]) elif self.obs_mode == 'sum_obs_acc_no_vel': in_range = (dm < self.comm_radius) & (0 < dm) nr_neighbors = np.sum(in_range) local_obs = self.get_local_obs_acc() sum_obs = np.zeros(self.dim_rec_o) sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range]) sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range]) sum_obs[0:nr_neighbors, 3] = 1 sum_obs[0:self.n_agents - 1, 4] = 1 obs = np.hstack([sum_obs.flatten(), local_obs]) elif self.obs_mode == 'sum_obs_acc_limited': in_range = (dm < self.comm_radius) & (0 < dm) nr_neighbors = np.sum(in_range) rel_vels = self.state.w_vel - vels local_obs = self.get_local_obs_acc() local_obs[-1] = nr_neighbors / (self.n_agents - 1) sum_obs = np.zeros(self.dim_rec_o) sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range]) sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range]) sum_obs[0:nr_neighbors, 3] = (nh_size[in_range] - nr_neighbors) / (self.n_agents - 2) if self.n_agents > 2\ else np.zeros(nr_neighbors) sum_obs[:nr_neighbors, 4] = rel_vels[:, 0][in_range] / (2 * self.max_lin_velocity) sum_obs[:nr_neighbors, 5] = rel_vels[:, 1][in_range] / (2 * self.max_lin_velocity) sum_obs[0:nr_neighbors, 6] = 1 sum_obs[0:self.n_agents - 1, 7] = 1 obs = np.hstack([sum_obs.flatten(), local_obs]) elif self.obs_mode == 'sum_obs': in_range = (dm < self.comm_radius) & (0 < dm) nr_neighbors = np.sum(in_range) local_obs = self.get_local_obs() local_obs[-1] = nr_neighbors / (self.n_agents - 1) sum_obs = np.zeros(self.dim_rec_o) sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range]) sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range]) sum_obs[0:nr_neighbors, 3] = np.cos(their_orientation[in_range]) sum_obs[0:nr_neighbors, 4] = np.sin(their_orientation[in_range]) sum_obs[0:nr_neighbors, 5] = 1 sum_obs[0:self.n_agents - 1, 6] = 1 obs = np.hstack([sum_obs.flatten(), local_obs]) elif self.obs_mode == 'sum_obs_limited': in_range = (dm < self.comm_radius) & (0 < dm) nr_neighbors = np.sum(in_range) local_obs = self.get_local_obs() local_obs[-1] = nr_neighbors / (self.n_agents - 1) sum_obs = np.zeros(self.dim_rec_o) sum_obs[0:nr_neighbors, 0] = dm[in_range] / self.world_size sum_obs[0:nr_neighbors, 1] = np.cos(my_orientation[in_range]) sum_obs[0:nr_neighbors, 2] = np.sin(my_orientation[in_range]) sum_obs[0:nr_neighbors, 3] = (nh_size[in_range] - nr_neighbors) / (self.n_agents - 2) if self.n_agents > 2\ else np.zeros(nr_neighbors) sum_obs[0:nr_neighbors, 4] = np.cos(their_orientation[in_range]) sum_obs[0:nr_neighbors, 5] = np.sin(their_orientation[in_range]) sum_obs[0:nr_neighbors, 6] = 1 sum_obs[0:self.n_agents - 1, 7] = 1 obs = np.hstack([sum_obs.flatten(), local_obs]) else: raise ValueError('histogram form must be 1D or 2D') return obs
def run_step(self, run_number: int, step_size: int, howlong: float) -> ReturnRunStep: assert self.context with self.context as ctx: dfslot = ctx.table min_slot = ctx.min max_slot = ctx.max if not (dfslot.created.any() or min_slot.has_buffered() or max_slot.has_buffered()): logger.info("Input buffers empty") return self._return_run_step(self.state_blocked, steps_run=0) min_slot.clear_buffers() max_slot.clear_buffers() bounds = self.get_bounds(min_slot, max_slot) if bounds is None: logger.debug("No bounds yet at run %d", run_number) return self._return_run_step(self.state_blocked, steps_run=0) xmin, xmax, ymin, ymax = bounds if self._bounds is None: (xdelta, ydelta) = self.get_delta(*bounds) self._bounds = ( xmin - xdelta, xmax + xdelta, ymin - ydelta, ymax + ydelta, ) logger.info("New bounds at run %d: %s", run_number, self._bounds) else: (dxmin, dxmax, dymin, dymax) = self._bounds (xdelta, ydelta) = self.get_delta(*bounds) assert xdelta >= 0 and ydelta >= 0 # Either the min/max has extended or shrunk beyond the deltas if (xmin < dxmin or xmax > dxmax or ymin < dymin or ymax > dymax) or (xmin > (dxmin + xdelta) or xmax < (dxmax - xdelta) or ymin > (dymin + ydelta) or ymax < (dymax - ydelta)): self._bounds = ( xmin - xdelta, xmax + xdelta, ymin - ydelta, ymax + ydelta, ) logger.info("Updated bounds at run %s: %s", run_number, self._bounds) self.reset() dfslot.update(run_number) xmin, xmax, ymin, ymax = self._bounds if xmin >= xmax or ymin >= ymax: logger.error("Invalid bounds: %s", self._bounds) return self._return_run_step(self.state_blocked, steps_run=0) # Now, we know we have data and bounds, proceed to create a # new histogram or to update the previous if is still exists # (i.e. no reset) p = self.params steps = 0 # if there are new deletions, build the histogram of the del. pairs # then subtract it from the main histogram if dfslot.base.deleted.any(): self.reset() dfslot.update(run_number) elif (dfslot.selection.deleted.any() and self._histo is not None): # i.e. TableSelectedView input_df = dfslot.data().base # the original table # we assume that deletions are only local to the view # and the related records still exist in the original table ... # TODO : test this hypothesis and reset if false raw_indices = dfslot.selection.deleted.next(length=step_size) indices = fix_loc(raw_indices) steps += indices_len(indices) x = input_df.to_array(locs=indices, columns=[self.x_column]).reshape(-1) y = input_df.to_array(locs=indices, columns=[self.y_column]).reshape(-1) bins = [p.ybins, p.xbins] if len(x) > 0: histo = histogram2d(y, x, bins=bins, range=[[ymin, ymax], [xmin, xmax]]) self._histo -= histo # if there are new creations, build a partial histogram with them # add it to the main histogram if not dfslot.created.any(): return self._return_run_step(self.state_blocked, steps_run=0) input_df = dfslot.data() raw_indices = dfslot.created.next(length=step_size) indices = fix_loc(raw_indices) steps += indices_len(indices) logger.info("Read %d rows", steps) self.total_read += steps x = input_df.to_array(locs=indices, columns=[self.x_column]).reshape(-1) y = input_df.to_array(locs=indices, columns=[self.y_column]).reshape(-1) if self._xedges is not None: bins = [self._xedges, self._yedges] else: bins = [p.ybins, p.xbins] if len(x) > 0: histo = histogram2d(y, x, bins=bins, range=[[ymin, ymax], [xmin, xmax]]) else: return self._return_run_step(self.state_blocked, steps_run=0) if self._histo is None: self._histo = histo elif histo is not None: self._histo += histo if self._histo is not None: cmax = self._histo.max() values = { "array": np.flip(self._histo, axis=0), # type: ignore "cmin": 0, "cmax": cmax, "xmin": xmin, "xmax": xmax, "ymin": ymin, "ymax": ymax, "time": run_number, } if self._with_output: table = self.table table["array"].set_shape([p.ybins, p.xbins]) last = table.last() if last is None or last["time"] != run_number: table.add(values) else: table.loc[last.row] = values self.build_heatmap(values) return self._return_run_step(self.next_state(dfslot), steps_run=steps)
def mutual_information(i1, i2, bins=256): r""" Computes the mutual information (MI) (a measure of entropy) between two images. MI is not real metric, but a symmetric and nonnegative similarity measures that takes high values for similar images. Negative values are also possible. Intuitively, mutual information measures the information that ``i1`` and ``i2`` share: it measures how much knowing one of these variables reduces uncertainty about the other. The Entropy is defined as: .. math:: H(X) = - \sum_i p(g_i) * ln(p(g_i) with :math:`p(g_i)` being the intensity probability of the images grey value :math:`g_i`. Assuming two images :math:`R` and :math:`T`, the mutual information is then computed by comparing the images entropy values (i.e. a measure how well-structured the common histogram is). The distance metric is then calculated as follows: .. math:: MI(R,T) = H(R) + H(T) - H(R,T) = H(R) - H(R|T) = H(T) - H(T|R) A maximization of the mutual information is equal to a minimization of the joint entropy. Parameters ---------- i1 : array_like The first image. i2 : array_like The second image. bins : integer The number of histogram bins (squared for the joined histogram). Returns ------- mutual_information : float The mutual information distance value between the supplied images. Raises ------ ArgumentError If the supplied arrays are of different shape. """ # pre-process function arguments i1 = numpy.asarray(i1) i2 = numpy.asarray(i2) # # validate function arguments # if not i1.shape == i2.shape: # raise ArgumentError('the two supplied array-like sequences i1 and i2 must be of the same shape') # compute i1 and i2 histogram range i1_range = __range(i1, bins) i2_range = __range(i2, bins) # compute joined and separated normed histograms i1i2_hist = histogram2d(i1.flatten(), i2.flatten(), bins=bins, range=[i1_range, i2_range]) # Note: histogram2d does not flatten array on its own i1_hist = histogram1d(i1, bins=bins, range=i1_range) i2_hist = histogram1d(i2, bins=bins, range=i2_range) # compute joined and separated entropy i1i2_entropy = __entropy(i1i2_hist) i1_entropy = __entropy(i1_hist) i2_entropy = __entropy(i2_hist) # compute and return the mutual information distance return i1_entropy + i2_entropy - i1i2_entropy
def __call__(self, bins=None, range=None, return_density=False): ny, nx = bins (ymin, ymax), (xmin, xmax) = range xscale = self._ax.get_xscale() yscale = self._ax.get_yscale() if xscale == 'log': xmin, xmax = log10(xmin), log10(xmax) if self._x_log is None: # We do this here insead of in set_xy to save time since in # set_xy we don't know yet if the axes will be log or not. self._update_x_log() if self._downres: x = self._x_log_sub else: x = self._x_log elif xscale == 'linear': if self._downres: x = self._x_sub else: x = self._x else: # pragma: nocover raise ValueError('Unexpected xscale: {0}'.format(xscale)) if yscale == 'log': ymin, ymax = log10(ymin), log10(ymax) if self._y_log is None: # We do this here insead of in set_xy to save time since in # set_xy we don't know yet if the axes will be log or not. self._update_y_log() if self._downres: y = self._y_log_sub else: y = self._y_log elif yscale == 'linear': if self._downres: y = self._y_sub else: y = self._y else: # pragma: nocover raise ValueError('Unexpected xscale: {0}'.format(xscale)) if self._downres: nx_sub = nx // self._downres_factor ny_sub = ny // self._downres_factor bins = (ny_sub, nx_sub) weights = self._c_sub else: bins = (ny, nx) weights = self._c if weights is None: array = histogram2d(y, x, bins=bins, range=((ymin, ymax), (xmin, xmax))) density = array / np.nanmax(array) else: array = histogram2d(y, x, bins=bins, weights=weights, range=((ymin, ymax), (xmin, xmax))) count = histogram2d(y, x, bins=bins, range=((ymin, ymax), (xmin, xmax))) density = count / np.nanmax(count) with np.errstate(invalid='ignore'): array /= count if return_density: return array, density else: return array
if len(data)==0: print('skip') continue data = dis_correct(data, asp, dis_map, ya_corr, q_corr, cut) sky_data = SkyCoord(data[:,1:3], unit='deg', frame='fk5', equinox='J2000.0') #gal = sky_data.transform_to('galactic') idxc, idxcatalog, d2d_p, d3d_p = sky_data.search_around_sky(c, 0.01*u.deg) star_mask = np.ones(data.shape[0], dtype=bool) star_mask[idxcatalog] = False data_f = data[star_mask, 3:5] qmask = data_f[:,5]>5 data_c = data_f[qmask] pos = ((data_f/36000.)/(1.25/2.)*(1.25/(800* 0.001666))+1.)/2.*size #print np.max(pos) #print np.min(pos) H=histogram2d(pos[:,0], pos[:,1],\ bins=[size,size], range=([ [0,size],[0,size] ])) detector += H data_c = dis_correct_c(data, dis_map, ya_corr, q_corr, cut) pos = ((data_c/36000.)/(1.25/2.)*(1.25/(800* 0.001666))+1.)/2.*size #print np.max(pos) #print np.min(pos) H=histogram2d(pos[:,0], pos[:,1],\ bins=[size,size], range=([ [0,size],[0,size] ])) detector_c += H np.save('/scratch/dw1519/galex/data/star_photon/back/'+name+'_full.npy', detector) np.save('/scratch/dw1519/galex/data/star_photon/back/'+name+'_cut.npy', detector_c)
def make_image(self, *args, **kwargs): xmin, xmax = self._ax.get_xlim() ymin, ymax = self._ax.get_ylim() xscale = self._ax.get_xscale() yscale = self._ax.get_yscale() if self._dpi is None: dpi = self.axes.figure.get_dpi() else: dpi = self._dpi width = (self._ax.get_position().width * self._ax.figure.get_figwidth()) height = (self._ax.get_position().height * self._ax.figure.get_figheight()) nx = int(round(width * dpi)) ny = int(round(height * dpi)) flip_x = xmin > xmax flip_y = ymin > ymax if flip_x: xmin, xmax = xmax, xmin if flip_y: ymin, ymax = ymax, ymin if xscale == 'log': xmin, xmax = log10(xmin), log10(xmax) if self._x_log is None: # We do this here insead of in set_xy to save time since in # set_xy we don't know yet if the axes will be log or not. self._update_x_log() if self._downres: x = self._x_log_sub else: x = self._x_log elif xscale == 'linear': if self._downres: x = self._x_sub else: x = self._x else: # pragma: nocover raise ValueError('Unexpected xscale: {0}'.format(xscale)) if yscale == 'log': ymin, ymax = log10(ymin), log10(ymax) if self._y_log is None: # We do this here insead of in set_xy to save time since in # set_xy we don't know yet if the axes will be log or not. self._update_y_log() if self._downres: y = self._y_log_sub else: y = self._y_log elif yscale == 'linear': if self._downres: y = self._y_sub else: y = self._y else: # pragma: nocover raise ValueError('Unexpected xscale: {0}'.format(xscale)) if self._downres: nx_sub = nx // self._downres_factor ny_sub = ny // self._downres_factor bins = (ny_sub, nx_sub) weights = self._c_sub else: bins = (ny, nx) weights = self._c if weights is None: array = histogram2d(y, x, bins=bins, weights=weights, range=((ymin, ymax), (xmin, xmax))) else: array = histogram2d(y, x, bins=bins, weights=weights, range=((ymin, ymax), (xmin, xmax))) count = histogram2d(y, x, bins=bins, range=((ymin, ymax), (xmin, xmax))) with np.errstate(invalid='ignore'): array /= count if flip_x or flip_y: if flip_x and flip_y: array = array[::-1, ::-1] elif flip_x: array = array[:, ::-1] else: array = array[::-1, :] if self.origin == 'upper': array = np.flipud(array) if callable(self._density_vmin): vmin = self._density_vmin(array) else: vmin = self._density_vmin if callable(self._density_vmax): vmax = self._density_vmax(array) else: vmax = self._density_vmax self.set_data(array) super(ScatterDensityArtist, self).set_clim(vmin, vmax) return super(ScatterDensityArtist, self).make_image(*args, **kwargs)
def hist_fast(data, wcs, imsz): foc = wcs.all_world2pix(data, 1) H=histogram2d(foc[:,1]-0.5, foc[:,0]-0.5,\ bins=imsz, range=([ [0,imsz[0]],[0,imsz[1]] ])) return H
def fill(self, xarr, yarr): hists = histogram2d(xarr, yarr, [self.nxbins, self.nybins], [self.xrange, self.yrange]) self.hists += hists