def compute2PCF(catalog, rmax=20, dr = 1., boxsize=250.): outfile = catalog[:-8]+".2pcf" hdu = fits.open(catalog)[1].data xR = hdu['x'] yR = hdu['y'] zR = hdu['z'] insideSel=(xR>rmax)&(xR<boxsize-rmax)&(yR>rmax)&(yR<boxsize-rmax)&(zR>rmax)&(zR<boxsize-rmax) volume=(boxsize)**3 # defines the trees print "creates trees" treeRandoms=t.cKDTree(n.transpose([xR,yR,zR]),1000.0) treeData=t.cKDTree(n.transpose([xR[insideSel],yR[insideSel],zR[insideSel]]),1000.0) nD=len(treeData.data) nR=len(treeRandoms.data) print nD, nR bin_xi3D=n.arange(0, rmax, dr) # now does the pair counts : pairs=treeData.count_neighbors(treeRandoms, bin_xi3D) t3 = time.time() DR=pairs[1:]-pairs[:-1] dV= (bin_xi3D[1:]**3 - bin_xi3D[:-1]**3 )*4*n.pi/3. pairCount=nD*nR#-nD*(nD-1)/2. xis = DR*volume/(dV * pairCount) -1. f=open(outfile,'w') cPickle.dump([bin_xi3D,xis, DR, volume, dV, pairCount, pairs, nD, nR],f) f.close()
def calculate_cistrans(data, chains, chain_id=0, cutoff=5, pbc_box=False, box_size=None): """ Analysis of the territoriality of polymer chains from simulations, using the cis/trans ratio. Cis signal is computed for the marked chain ('chain_id') as amount of contacts of the chain with itself Trans signal is the total amount of trans contacts for the marked chain with other chains from 'chains' (and with all the replicas for 'pbc_box'=True) """ if data.shape[1] != 3: raise ValueError("Incorrect polymer data shape. Must be Nx3.") if np.isnan(data).any(): raise RuntimeError("Data contains NANs") N = len(data) if pbc_box == True: if box_size is None: raise ValueError("Box size is not given") else: data_scaled = np.mod(data, box_size) else: box_size = None data_scaled = np.copy(data) if chains is None: chains = [[0, N]] chain_id = 0 chain_start = chains[chain_id][0] chain_end = chains[chain_id][1] # all contact pairs available in the scaled data tree = ckdtree.cKDTree(data_scaled, boxsize=box_size) pairs = tree.query_pairs(cutoff, output_type="ndarray") # total number of contacts of the marked chain: # each contact is counted twice if both monomers belong to the marked chain and # only once if just one of the monomers in the pair belong to the marked chain all_signal = len(pairs[pairs < chain_end]) - len( pairs[pairs < chain_start]) # contact pairs of the marked chain with itself tree = ckdtree.cKDTree(data[chain_start:chain_end], boxsize=None) pairs = tree.query_pairs(cutoff, output_type="ndarray") # doubled number of contacts of the marked chain with itself (i.e. cis signal) cis_signal = 2 * len(pairs) assert all_signal >= cis_signal trans_signal = all_signal - cis_signal return cis_signal, trans_signal
def generate_example_random_choice(positions, properties, k=26, plot=False): print('choice nn') idx_list = np.arange(len(positions)) virtual_node_positions = positions[np.random.choice(idx_list, 1000, replace=False)] kdtree = cKDTree(virtual_node_positions) dist, indices = kdtree.query(positions) virtual_properties = np.zeros((len(np.bincount(indices)), len(properties[0]))) mean_sum = [lambda x: np.bincount(indices, weights=x) / np.maximum(1., np.bincount(indices)), # mean lambda x: np.bincount(indices, weights=x)] # sum mean_sum_enc = [0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1] for p, enc in zip(np.arange(len(properties[0])), mean_sum_enc): virtual_properties[:, p] = mean_sum[enc](properties[:, p]) virtual_positions = virtual_properties[:, :3] graph = nx.DiGraph() kdtree = cKDTree(virtual_positions) dist, idx = kdtree.query(virtual_positions, k=k + 1) receivers = idx[:, 1:] # N,k senders = np.arange(virtual_positions.shape[0]) # N senders = np.tile(senders[:, None], [1, k]) # N,k receivers = receivers.flatten() senders = senders.flatten() n_nodes = virtual_positions.shape[0] pos = dict() # for plotting node positions. edgelist = [] for node, feature, position in zip(np.arange(n_nodes), virtual_properties, virtual_positions): graph.add_node(node, features=feature) pos[node] = position[:2] # edges = np.stack([senders, receivers], axis=-1) + sibling_node_offset for u, v in zip(senders, receivers): graph.add_edge(u, v, features=np.array([1., 0.])) graph.add_edge(v, u, features=np.array([1., 0.])) edgelist.append((u, v)) edgelist.append((v, u)) graph.graph["features"] = np.array([0.]) # plotting print('len(pos) = {}\nlen(edgelist) = {}'.format(len(pos), len(edgelist))) if plot: fig, ax = plt.subplots(1, 1, figsize=(20, 20)) draw(graph, ax=ax, pos=pos, node_color='blue', edge_color='red', node_size=10, width=0.1) image_dir = '/data2/hendrix/images/' graph_image_idx = len(glob.glob(os.path.join(image_dir, 'graph_image_*'))) plt.savefig(os.path.join(image_dir, 'graph_image_{}'.format(graph_image_idx))) return networkxs_to_graphs_tuple([graph], node_shape_hint=[virtual_positions.shape[1] + virtual_properties.shape[1]], edge_shape_hint=[2])
async def update(self): self.update_minerals() self.tag_cache.clear() self.own_unit_cache.clear() self.enemy_unit_cache.clear() self.force_fields.clear() self.own_numpy_vectors = [] self.enemy_numpy_vectors = [] self.all_own = self.knowledge.all_own for unit in self.all_own: if unit.is_memory: self.tag_cache[unit.tag] = unit units = self.own_unit_cache.get(unit.type_id, Units([], self.ai)) if units.amount == 0: self.own_unit_cache[unit.type_id] = units units.append(unit) self.own_numpy_vectors.append( np.array([unit.position.x, unit.position.y])) for unit in self.knowledge.known_enemy_units: if unit.is_memory: self.tag_cache[unit.tag] = unit units = self.enemy_unit_cache.get(unit.type_id, Units([], self.ai)) if units.amount == 0: self.enemy_unit_cache[unit.type_id] = units units.append(unit) self.enemy_numpy_vectors.append( np.array([unit.position.x, unit.position.y])) for unit in self.ai.all_units: # Add all non-memory units to unit tag cache self.tag_cache[unit.tag] = unit if len(self.own_numpy_vectors) > 0: self.own_tree = cKDTree(self.own_numpy_vectors) else: self.own_tree = None if len(self.enemy_numpy_vectors) > 0: self.enemy_tree = cKDTree(self.enemy_numpy_vectors) else: self.enemy_tree = None for effect in self.ai.state.effects: if effect.id == FakeEffectID.get(UnitTypeId.FORCEFIELD.value): self.force_fields.append(effect)
def _derive_nearest_neighbours( latitudes_stations: np.array, longitudes_stations: np.array, coordinates: Coordinates, num_stations_nearby: int = 1, ) -> Tuple[Union[float, np.ndarray], np.ndarray]: """ A function that uses a k-d tree algorithm to obtain the nearest neighbours to coordinate pairs Args: latitudes_stations (np.array): latitude values of stations being compared to the coordinates longitudes_stations (np.array): longitude values of stations being compared to the coordinates coordinates (Coordinates): the coordinates for which the nearest neighbour is searched num_stations_nearby: Number of stations that should be nearby Returns: Tuple of distances and ranks of nearest to most distant stations """ points = np.c_[np.radians(latitudes_stations), np.radians(longitudes_stations)] distance_tree = cKDTree(points) return distance_tree.query(coordinates.get_coordinates_in_radians(), k=num_stations_nearby)
def _init_fields(self, nc_dataset): nc_vars = nc_dataset.variables lons = nc_vars["lon"][:] lats = nc_vars["lat"][:] if lons.ndim == 1: lats2d, lons2d = np.meshgrid(lats, lons) elif lons.ndim == 2: lats2d, lons2d = lats, lons else: raise NotImplementedError( "Cannot handle {}-dimensional coordinates".format(lons.ndim)) self.lons2d, self.lats2d = lons2d, lats2d self.times_var = nc_vars["time"] self.times_num = nc_vars["time"][:] if hasattr(self.times_var, "calendar"): self.times = num2date(self.times_num, self.times_var.units, self.times_var.calendar) else: self.times = num2date(self.times_num, self.times_var.units) if not self.lazy: self.var_data = nc_vars[self.var_name][:] if nc_vars[self.var_name].shape[1:] != self.lons2d.shape: print("nc_vars[self.var_name].shape = {}".format( nc_vars[self.var_name].shape)) self.var_data = np.transpose(self.var_data, axes=[0, 2, 1]) x_in, y_in, z_in = lat_lon.lon_lat_to_cartesian( self.lons2d.flatten(), self.lats2d.flatten()) self.kdtree = cKDTree(list(zip(x_in, y_in, z_in)))
def __call__(self, x): num_points = x.shape[-1] tree = cKDTree(np.transpose(x)) to_query = np.random.randint(low=0, high=num_points) _, indices = tree.query(x[:, to_query], self.num_samples) mask = np.sort(indices) if self.mode.startswith('aniso'): matrix = np.random.uniform( low=self.low, high=self.high, size=(self.out_features, self.num_samples) ) else: matrix = np.random.uniform( low=self.low, high=self.high, size=(self.out_features, 1) ) matrix = np.repeat(matrix, self.num_samples, axis=1) y = x.copy() if self.mode.startswith('aniso'): for index, transform_id in enumerate(mask): rotation_mat = self._build_rotation_matrix(matrix[:, index]) y[:, transform_id] = np.dot(rotation_mat, y[:, transform_id]) else: rotation_mat = self._build_rotation_matrix(matrix[:, 0]) y[:, mask] = np.dot(rotation_mat, y[:, mask]) mask = np.repeat( np.expand_dims(mask, axis=0), self.out_features, axis=0 ) return y, matrix, mask
def __init__(self, startImage, x,y, sx, imageBounds, pixelSize): self.im = startImage self.x = x self.y = y self.sx = sx self.imageBounds = imageBounds self.pixelSize = pixelSize self.fuzz = 3*np.median(sx) self.roiSize = int(self.fuzz/pixelSize) self.fuzz = pixelSize*(self.roiSize) self.mask = np.ones(self.im.shape) #print imageBounds.x0 #print imageBounds.x1 #print fuzz #print roiSize #print pixelSize self.X = np.arange(imageBounds.x0,imageBounds.x1, pixelSize) self.Y = np.arange(imageBounds.y0,imageBounds.y1, pixelSize) #print X self.ctval = 1e-4 #record our image resolution so we can plot pts with a minimum size equal to res (to avoid missing small pts) self.delX = np.absolute(self.X[1] - self.X[0]) self.kdt = ckdtree.cKDTree(np.array([self.x,self.y]).T)
def __init__(self, startImage, x, y, sx, imageBounds, pixelSize): self.im = startImage self.x = x self.y = y self.sx = sx self.imageBounds = imageBounds self.pixelSize = pixelSize self.fuzz = 3 * np.median(sx) self.roiSize = int(self.fuzz / pixelSize) self.fuzz = pixelSize * (self.roiSize) self.mask = np.ones(self.im.shape) #print imageBounds.x0 #print imageBounds.x1 #print fuzz #print roiSize #print pixelSize self.X = np.arange(imageBounds.x0, imageBounds.x1, pixelSize) self.Y = np.arange(imageBounds.y0, imageBounds.y1, pixelSize) #print X self.ctval = 1e-4 #record our image resolution so we can plot pts with a minimum size equal to res (to avoid missing small pts) self.delX = np.absolute(self.X[1] - self.X[0]) self.kdt = ckdtree.cKDTree(np.array([self.x, self.y]).T)
def segments_adding(M: int, active_vertices, active_segments, segments_vertices, segments, d, epsilon, l, N, shapeAndDistribution): recalibrate = False # shuffleB = False for m in range(M): for i in range(len(active_segments) - 1, -1, -1): recalibrate, shuffleB, modified_vertices, i = segment_adding( i, active_vertices, active_segments, segments_vertices, segments, d, recalibrate) if shuffleB or recalibrate: tree = ckdtree.cKDTree(segments) if recalibrate: active_segments, segments_vertices, N = find_segments_additive( tree, active_vertices, modified_vertices, segments, d, epsilon, l, N, active_segments, segments_vertices, shapeAndDistribution) recalibrate = False break if shuffleB: active_segments, segments_vertices = shuffle( tree, i, active_vertices, segments, d, epsilon, l, N, active_segments, segments_vertices) shuffleB = False break return active_segments, segments_vertices, N
def Ptree(vals, errs, truevals, treeunit): out = [] lencheck = 0 #build truth tree truetree = ckdtree.cKDTree(truevals / treeunit) for val, err in zip(vals, errs): #get nearest neighbors within query_radius to target inear = truetree.query_ball_point(val / treeunit, r=query_radius) factor = 1. if len(inear) > 20000: lencheck += 1 factor = len(inear) / 20000. inear = random.sample(inear, 20000) #data of nearest neighbors truearr = truetree.data[inear] * treeunit Ls = likelihoods(val, err, truearr) #sum likelihoods out.append(np.sum(Ls) * factor) if lencheck > 0: print "Warning: ball query returned >20k points for {} targets ({}%).".format( lencheck, float(lencheck) / len(vals) * 100.) return np.array(out)
def brute_force(data, r_step): tree = kd.cKDTree(data) groups_list = [] for particle in data: if particle is not None: group = [] indexes = tree.query_ball_point(particle, r_step) # returns the indexes in data of particles within r_step from the initial particle if indexes.__len__() == 1: group.append(data[indexes[0]]) data[indexes[0]] = None else: del indexes[0] for x in indexes: extra_indexes = tree.query_ball_point(data[x], r_step) if extra_indexes.__len__() > 1: for k in extra_indexes: if not indexes.__contains__(k): indexes.append(k) for x in indexes: # excluding the particle counting itself group.append(data[x]) # adding each particle to this group data[x] = None # removing particle from initial data list groups_list.append(group) count = 0 for x in groups_list: if x.__len__() >= 10: count += 1 print "There are", count, "halos with 10 or more particles" return groups_list
def get_novelty(self, vector_space, genome_vector): """Return the novelty for a single behavior vector. :param vector_space: :type list: A list of lists representing the whole behavior space. :param genome_vector: :type list: The behavior vector of the genome to be evaluated. :return: Novelty score """ novelty = 0 # Get space of generation + novelty archive vector_space = np.concatenate((vector_space, self.novelty_archive)) # Get distances to k-nearest points in behavior space tree = kd.cKDTree(vector_space) indexes = tree.query(genome_vector, 20) distances = indexes[0] for i in distances: novelty += i/self.dsize novelty /= 20 # Add to novelty archive with 1% probability rand_n = random.random() if rand_n >= .99: k = 0 for element in genome_vector: self.novelty_archive[self.archive_size][k] = element k += 1 self.archive_size += 1 return novelty
def __Debounce(self, xs, ys, radius=4): if len(xs) < 2: return xs, ys kdt = ckdtree.cKDTree(numpy.array([xs,ys]).T) xsd = [] ysd = [] for xi, yi in zip(xs, ys): #neigh = kdt.query_ball_point([xi,yi], radius) dn, neigh = kdt.query(numpy.array([xi,yi]), 5) neigh = neigh[dn < radius] if len(neigh) > 1: Ii = self.filteredData[xi,yi] In = self.filteredData[xs[neigh].astype('i'),ys[neigh].astype('i')].max() if not Ii < In: xsd.append(xi) ysd.append(yi) else: xsd.append(xi) ysd.append(yi) return xsd, ysd
def tag_sites(sitesym, scaled_positions, symprec=1e-3): scaled = np.around(np.array(scaled_positions, ndmin=2), 8) scaled %= 1.0 scaled %= 1.0 np.set_printoptions(suppress=True) tags = -np.ones((len(scaled), ), dtype=int) tagdis = 100 * np.ones((len(scaled), ), dtype=float) rot, trans = spg.spacegroup.parse_sitesym(sitesym) siteskdTree = cKDTree(scaled) for i in range(len(scaled)): if tags[i] == -1: curpos = scaled[i] sympos = np.dot(rot, curpos) + trans sympos %= 1.0 sympos %= 1.0 sympos = np.unique(np.around(sympos, 8), axis=0) min_dis, min_ids = siteskdTree.query(sympos, k=1) # print(i,len(min_dis)) # print(min_dis) select = min_dis <= symprec select_ids = min_ids[select] tags[select_ids] = i tagdis[select_ids] = min_dis[select] return tags, tagdis
def nearest_neighbours_connected_graph(virtual_positions, k): kdtree = cKDTree(virtual_positions) dist, idx = kdtree.query(virtual_positions, k=k + 1) receivers = idx[:, 1:] # N,k senders = np.arange(virtual_positions.shape[0]) # N senders = np.tile(senders[:, None], [1, k]) # N,k receivers = receivers.flatten() senders = senders.flatten() graph_nodes = tf.convert_to_tensor(virtual_positions, tf.float32) graph_nodes.set_shape([None, 3]) receivers = tf.convert_to_tensor(receivers, tf.int32) receivers.set_shape([None]) senders = tf.convert_to_tensor(senders, tf.int32) senders.set_shape([None]) n_node = tf.shape(graph_nodes)[0:1] n_edge = tf.shape(senders)[0:1] graph_data_dict = dict(nodes=graph_nodes, edges=tf.zeros((n_edge[0], 1)), globals=tf.zeros([1]), receivers=receivers, senders=senders, n_node=n_node, n_edge=n_edge) return GraphsTuple(**graph_data_dict)
def run(self): if not self.copyData(): return group_name = self.get_property('Group Name') distance_name = self.get_property('Distance Name') if not group_name or not distance_name: return search_position = self.get_property('Search Position') max_count = self.get_property('Search Count') max_dist = self.get_property('Search Radius') if max_dist <= 0: max_dist = None points = self.geo.getVertexes() tree = ckdtree.cKDTree(points) dist, idx = tree.query(search_position, max_count, distance_upper_bound=max_dist) if len(idx) == 0: return npoints = self.geo.getNumVertexes() self.geo.createGroup('vertex', group_name) self.geo.createAttribute('vertex', distance_name, "float", -1.0, True) if not self.geo.hasAttribute('vertex', 'color'): self.geo.createAttribute('vertex', 'color', "vector3", [1.0, 1.0, 1.0], True) for i, d in zip(idx, dist): if i >= npoints: continue self.geo.setVertexAttrib(distance_name, i, d) self.geo.setGroup('vertex', group_name, i, True) self.geo.setVertexAttrib("color", i, [1.0, 0, 0])
def __init__(self, file_path = "", var_name = "", bathymetry_path = "/skynet3_rech1/huziy/NEMO_OFFICIAL/dev_v3_4_STABLE_2012/NEMOGCM/CONFIG/GLK_LIM3_Michigan/EXP00/bathy_meter.nc"): """ :param file_path: :param var_name: :param bathymetry_path: used to mask land points """ self.current_time_frame = -1 self.var_name = var_name self.cube = iris.load_cube(file_path, constraint=iris.Constraint(cube_func=lambda c: c.var_name == var_name)) self.lons, self.lats = cartography.get_xy_grids(self.cube) lons2d_gl, lats2d_gl = nemo_commons.get_2d_lons_lats_from_nemo(path=bathymetry_path) mask_gl = nemo_commons.get_mask(path=bathymetry_path) xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons2d_gl.flatten(), lats2d_gl.flatten()) xt, yt, zt = lat_lon.lon_lat_to_cartesian(self.lons.flatten(), self.lats.flatten()) tree = cKDTree(list(zip(xs, ys, zs))) dists, indices = tree.query(list(zip(xt, yt, zt))) self.mask = mask_gl.flatten()[indices].reshape(self.lons.shape) self.nt = self.cube.shape[0] assert isinstance(self.cube, Cube) print(self.nt)
def __Debounce(self, xs, ys, radius=4): if len(xs) < 2: return xs, ys kdt = ckdtree.cKDTree(numpy.array([xs,ys]).T) xsd = [] ysd = [] visited = 0*xs for i in xrange(len(xs)): if not visited[i]: xi = xs[i] yi = ys[i] #neigh = kdt.query_ball_point([xi,yi], radius) dn, neigh = kdt.query(numpy.array([xi,yi]), 5) neigh = neigh[dn < radius] if len(neigh) > 1: In = self.filteredData[xs[neigh].astype('i'),ys[neigh].astype('i')] mi = In.argmax() xsd.append(xs[neigh[mi]]) ysd.append(ys[neigh[mi]]) visited[neigh] = 1 else: xsd.append(xi) ysd.append(yi) return xsd, ysd
def nearest_neighbours(points, coords, points_required=1, max_distance=250.): """ An implementation of nearest neaighbour for spatial data that uses kdtrees :param points: array of points to find the nearest neighbour for :param coords: coordinates of points :param points_required: number of points to return :param max_distance: maximum search radius :return: """ if len(np.array(points).shape) == 1: points = np.array([points]) # Initialise tree instance kdtree = cKDTree(data=coords) # iterate throught the points and find the nearest neighbour distances, indices = kdtree.query(points, k=points_required, distance_upper_bound=max_distance) # Mask out infitnite distances in indices to avoid confusion mask = np.isfinite(distances) if not np.all(mask): distances[~mask] = np.nan return distances, indices
def dilate_mask(mask, mask_shape, vox_size, radius): """ Dilate the foreground in a binary mask according to a radius (in mm) """ is_to_dilate = mask == 1 is_background = mask == 0 # Get the list of indices background_pos = np.argwhere(is_background) * vox_size label_pos = np.argwhere(is_to_dilate) * vox_size ckd_tree = cKDTree(label_pos) # Compute the nearest labels for each voxel of the background dist, indices = ckd_tree.query(background_pos, k=1, distance_upper_bound=radius, n_jobs=-1) # Associate indices to the nearest label (in distance) valid_nearest = np.squeeze(np.isfinite(dist)) id_background = np.flatnonzero(is_background)[valid_nearest] id_label = np.flatnonzero(is_to_dilate)[indices[valid_nearest]] # Change values of those background mask = mask.flatten() mask[id_background.T] = mask[id_label.T] mask = mask.reshape(mask_shape) return mask
def find_potential_neighbours(x, k=100, distance_upper_bound=np.inf): tree = cKDTree(x) d, idx = tree.query(x, k + 1, distance_upper_bound=distance_upper_bound, n_jobs=-1) return d[:, 1:], idx[:, 1:]
def plot_difference(basemap, lons1, lats1, data1, label1, lons2, lats2, data2, label2, base_folder="/skynet3_rech1/huziy/veg_fractions/" ): xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons2.flatten(), lats2.flatten()) xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons1.flatten(), lats1.flatten()) ktree = cKDTree(list(zip(xs, ys, zs))) dists, inds = ktree.query(list(zip(xt, yt, zt))) # Calculate differences diff_dict = {} for key, the_field in data2.items(): diff_dict[key] = the_field.flatten()[inds].reshape(data1[key].shape) - data1[key] x, y = basemap(lons1, lats1) imname = "sand_clay_diff_{0}-{1}.jpeg".format(label2, label1) impath = os.path.join(base_folder, imname) plot_sand_and_clay_diff(x, y, basemap, diff_dict["SAND"], diff_dict["CLAY"], out_image=impath) del diff_dict["SAND"], diff_dict["CLAY"] imname = "veg_fract_diff_{0}-{1}.jpeg".format(label2, label1) impath = os.path.join(base_folder, imname) plot_veg_fractions_diff(x, y, basemap, diff_dict, out_image=impath)
def __init__(self, model): self.src_crs = ccrs.LambertCylindrical() self.tgt_crs = ccrs.Mercator(central_longitude=-87.200012, min_latitude=18.091648, max_latitude=31.960648) self.file = glob.glob(model.data_dir + '/*')[0] data = Dataset(self.file) self.lats = data['Latitude'][:] self.lons = data['Longitude'][:] self.depths = data['Depth'][:] self.lons, self.lats = np.meshgrid(self.lons, self.lats) transformed = self.tgt_crs.transform_points(self.src_crs, self.lons, self.lats) self.lons = self.lons[0] self.lats = self.lats.T[0] self.x = transformed[0, :, 0] self.y = transformed[:, 0, 1] x, y = np.meshgrid(self.x, self.y) self.points = np.array([self.x, self.y], dtype=object) # VisibleDeprecationWarning: Creating an ndarray from ragged nested # sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays # with different lengths or shapes) is deprecated. If you meant to do # this, you must specify 'dtype=object' when creating the ndarray coords = np.array([x.ravel(), y.ravel()]).T self.tree = cKDTree(coords, leafsize=model.leafsize)
def find_segments(vertices, segments, d, epsilon, l, N, shapeAndDistribution): active_segments = [] #segmenty które będą się rozrastać segments_vertices = [ ] #krawędzie do których będzie rozrastać się segment na odpowiednim miejscu powyżej tree = ckdtree.cKDTree(segments) for i in range(len(vertices) - 1, -1, -1): dist, nearest_segment = tree.query(vertices[i]) if dist < d: if not any(i in lista for lista in segments_vertices): vertices.pop(i) if N != 0: new_vertex = random_vertex(l, shapeAndDistribution) vertices.append(new_vertex) N -= 1 for i in range(len(vertices)): find_segment( i, tree, d, segments, vertices, active_segments, segments_vertices, ) return active_segments, segments_vertices, N
def __Debounce(self, xs, ys, radius=4): if len(xs) < 2: return xs, ys kdt = ckdtree.cKDTree(np.array([xs, ys]).T) xsd = [] ysd = [] visited = 0 * xs for i in xrange(len(xs)): if not visited[i]: xi = xs[i] yi = ys[i] #neigh = kdt.query_ball_point([xi,yi], radius) dn, neigh = kdt.query(np.array([xi, yi]), 5) neigh = neigh[dn < radius] if len(neigh) > 1: In = self.filteredData[xs[neigh].astype('i'), ys[neigh].astype('i')] mi = In.argmax() xsd.append(xs[neigh[mi]]) ysd.append(ys[neigh[mi]]) visited[neigh] = 1 else: xsd.append(xi) ysd.append(yi) return xsd, ysd
def gethlr(dfull, sdfull): #linear fit using just flux radii #(dfull['FLUX_RADIUS_I'] - 1.7) / 2.3 #use average star flux_radius_i around galaxies #& galaxy flux_radius from balrog to_grid = fits.open( '/Users/Christina/DES/data/balrog/sva1/balrog_tab01_avg_star_fluxradiusi_0.1deg.fits' )[1].data #make tree of dfull stars sys.stderr.write("stars in dfull: {}\n".format(len(sdfull))) sys.stderr.write("galaxies in dfull: {}\n".format(len(dfull))) sys.stderr.write('creating hlr tree...\n') star_tree = ckdtree.cKDTree(zip(sdfull['RA'], sdfull['DEC'])) gal_pos = zip(dfull['RA'], dfull['DEC']) #360 arcsec close = star_tree.query_ball_point(gal_pos, r=0.1) sys.stderr.write(' calculating average star radii...\n') start = time.time() dfull_avg_star_fr = np.array( [np.median(sdfull['FLUX_RADIUS_I'][c]) for c in close]) end = time.time() dfull_hlr = griddata( zip(to_grid['flux_radius_i'], to_grid['avg_flux_radius_i']), to_grid['hlr'], zip(dfull['FLUX_RADIUS_I'], dfull_avg_star_fr)) sys.stderr.write(' done\n') return dfull_hlr
def get_data_from_file_interpolate_if_needed(self, the_path): the_path = str(the_path) if the_path.lower()[-3:] in ["cis", "nic"]: data = self._parse_nic_cis_data_file(the_path) else: data = self.get_data_from_path(the_path) if data.shape != (self.ncols_target, self.ncols_target): # The interpolation is needed domain_descr = data.shape if domain_descr not in self.domain_descr_to_kdtree: if domain_descr == (516, 510): self.lons2d_other = np.flipud(np.loadtxt(self.path_to_other_lons)).transpose() self.lats2d_other = np.flipud(np.loadtxt(self.path_to_other_lats)).transpose() else: self.lons2d_other, self.lats2d_other = self._generate_grid_from_descriptor(domain_descr) xs, ys, zs = lat_lon.lon_lat_to_cartesian(self.lons2d_other.flatten(), self.lats2d_other.flatten()) kdtree_other = cKDTree(data=list(zip(xs, ys, zs))) self.domain_descr_to_kdtree[data.shape] = kdtree_other kdtree_other = self.domain_descr_to_kdtree[data.shape] xt, yt, zt = lat_lon.lon_lat_to_cartesian(self.lons2d_target.flatten(), self.lats2d_target.flatten()) dsts, inds = kdtree_other.query(list(zip(xt, yt, zt))) return data.flatten()[inds].reshape(self.lons2d_target.shape) else: return data
def __init__(self, edges, node_map, memory=None, algorithm=bidirectional_dijkstra): """ :param edges: [DataFrame] :param node_map: [DataFrame] :param memory [joblib.Memory] :param algorithm: [function] :return: """ self.node_map = node_map self._btree = ckdtree.cKDTree( np.array(list(zip(self.node_map.x, self.node_map.y)))) self.edges = edges self.memory = memory self.algorithm = algorithm self.routes = {} self.Graph = nx.Graph() if self.memory is not None: self.get_route = self.memory.cache(self.get_route, ignore=['self']) LOGGER.debug('loading routing graph') _ = self.edges.apply(lambda x: self.Graph.add_edge(**x), axis=1)
def __Debounce(self, xs, ys, radius=4): if len(xs) < 2: return xs, ys kdt = ckdtree.cKDTree(numpy.array([xs, ys]).T) xsd = [] ysd = [] for xi, yi in zip(xs, ys): #neigh = kdt.query_ball_point([xi,yi], radius) dn, neigh = kdt.query(numpy.array([xi, yi]), 5) neigh = neigh[dn < radius] if len(neigh) > 1: Ii = self.filteredData[xi, yi] In = self.filteredData[xs[neigh].astype('i'), ys[neigh].astype('i')].max() if not Ii < In: xsd.append(xi) ysd.append(yi) else: xsd.append(xi) ysd.append(yi) return xsd, ysd
def covAnnulus(expotab, rMin=0.005, rMax=0.02, maxErr=70.): '''Calculate the mean <dx dx>, <dx dy>, and <dy dy> for pairs within a circular annulus of separation. Exclude individual points with meas errors above maxErr. Returns cxx, cyy, cxy, npairs where last is number of object pairs used in calculation. ''' # Extract data use = expotab['measErr'] <= maxErr xy = np.vstack((expotab['u'][use], expotab['v'][use])).transpose() dx = expotab['dx'][use] dy = expotab['dy'][use] # Calculate correlations kdt = cKDTree(xy) prsmax = kdt.query_pairs(rMax, output_type='ndarray') prsmin = kdt.query_pairs(rMin, output_type='ndarray') nprs = prsmax.shape[0] - prsmin.shape[0] xx = np.sum(dx[prsmax[:,0]]*dx[prsmax[:,1]]) \ -np.sum(dx[prsmin[:,0]]*dx[prsmin[:,1]]) yy = np.sum(dy[prsmax[:,0]]*dy[prsmax[:,1]]) \ -np.sum(dy[prsmin[:,0]]*dy[prsmin[:,1]]) xy = np.sum(dx[prsmax[:,0]]*dy[prsmax[:,1]]) \ -np.sum(dx[prsmin[:,0]]*dy[prsmin[:,1]])\ +np.sum(dy[prsmax[:,0]]*dx[prsmax[:,1]])\ -np.sum(dy[prsmin[:,0]]*dx[prsmin[:,1]]) xx /= nprs yy /= nprs xy /= 2 * nprs return xx, yy, xy, nprs
def writeQSOcat(file, dmin=0.3, Lbox = 1040.): t0 = time.time() print file x, y, z, vx, vy, vz, df = readGAL(file) catalog = n.transpose([x, y, z, vx, vy, vz*0.92, df]) # identifies pairs treeD=t.cKDTree(n.transpose([x, y, z]), 1000.0) allpairs = treeD.query_ball_point(treeD.data, dmin) # removes auto-pairs pairID = [] for ii, el in enumerate(allpairs): if len(el)>1 and ii == el[0]: pairID.append(el) to_rm = [] ct = 0 for id2chooseFrom in pairID: n.random.shuffle(id2chooseFrom) #keep = id2chooseFrom[0] reject = id2chooseFrom[1:] to_rm.append(reject) ct+=1 if ct>0: to_delete = n.hstack((to_rm)) # list of ids rejected print "2del", len(to_delete) cat2save = n.delete(catalog, to_delete, axis=0) n.savetxt( join(dir, "QSO"+os.path.basename(file)[9:]), cat2save, fmt='%10.5f', header=' x y z vx vy vz df ' ) print "dt=",time.time()-t0 else: print "no del" n.savetxt( join(dir, "QSO"+os.path.basename(file)[9:]), catalog, fmt='%10.5f', header=' x y z vx vy vz df ' ) print "dt=",time.time()-t0
def _init_fields(self, nc_dataset): nc_vars = nc_dataset.variables lons = nc_vars["lon"][:] lats = nc_vars["lat"][:] if lons.ndim == 1: lats2d, lons2d = np.meshgrid(lats, lons) elif lons.ndim == 2: lats2d, lons2d = lats, lons else: raise NotImplementedError("Cannot handle {}-dimensional coordinates".format(lons.ndim)) self.lons2d, self.lats2d = lons2d, lats2d self.times_var = nc_vars["time"] self.times_num = nc_vars["time"][:] if hasattr(self.times_var, "calendar"): self.times = num2date(self.times_num, self.times_var.units, self.times_var.calendar) else: self.times = num2date(self.times_num, self.times_var.units) if not self.lazy: self.var_data = nc_vars[self.var_name][:] if nc_vars[self.var_name].shape[1:] != self.lons2d.shape: print("nc_vars[self.var_name].shape = {}".format(nc_vars[self.var_name].shape)) self.var_data = np.transpose(self.var_data, axes=[0, 2, 1]) x_in, y_in, z_in = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten()) self.kdtree = cKDTree(list(zip(x_in, y_in, z_in)))
def _fit(self, X): if isinstance(X, NeighborsBase): self._fit_X = X._fit_X self._tree = X._tree self._fit_method = X._fit_method return self elif isinstance(X, BallTree): self._fit_X = X.data self._tree = X self._fit_method = 'ball_tree' return self elif isinstance(X, cKDTree): self._fit_X = X.data self._tree = X self._fit_method = 'kd_tree' return self X = safe_asarray(X) if X.ndim != 2: raise ValueError("data type not understood") n_samples = X.shape[0] if n_samples == 0: raise ValueError("n_samples must be greater than 0") if issparse(X): if self.algorithm not in ('auto', 'brute'): warnings.warn("cannot use tree with sparse input: " "using brute force") self._fit_X = X.tocsr() self._tree = None self._fit_method = 'brute' return self self._fit_method = self.algorithm self._fit_X = X if self._fit_method == 'auto': # BallTree outperforms the others in nearly any circumstance. if self.n_neighbors is None: self._fit_method = 'ball_tree' elif self.n_neighbors < self._fit_X.shape[0] // 2: self._fit_method = 'ball_tree' else: self._fit_method = 'brute' if self._fit_method == 'kd_tree': self._tree = cKDTree(X, self.leaf_size) elif self._fit_method == 'ball_tree': self._tree = BallTree(X, self.leaf_size, p=self.p) elif self._fit_method == 'brute': self._tree = None else: raise ValueError("algorithm = '%s' not recognized" % self.algorithm) return self
def main(dfs_var_name="t2", cru_var_name="tmp", dfs_folder="/home/huziy/skynet3_rech1/NEMO_OFFICIAL/DFS5.2_interpolated", cru_file = "data/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.tmp.dat.nc"): if not os.path.isdir(NEMO_IMAGES_DIR): os.mkdir(NEMO_IMAGES_DIR) #year range is inclusive [start_year, end_year] start_year = 1981 end_year = 2009 season_name_to_months = OrderedDict([ ("Winter", (1, 2, 12)), ("Spring", list(range(3, 6))), ("Summer", list(range(6, 9))), ("Fall", list(range(9, 12)))]) cru_t_manager = CRUDataManager(var_name=cru_var_name, path=cru_file) cru_lons, cru_lats = cru_t_manager.lons2d, cru_t_manager.lats2d #get seasonal means (CRU) season_to_mean_cru = cru_t_manager.get_seasonal_means(season_name_to_months=season_name_to_months, start_year=start_year, end_year=end_year) #get seasonal means Drakkar dfs_manager = DFSDataManager(folder_path=dfs_folder, var_name=dfs_var_name) season_to_mean_dfs = dfs_manager.get_seasonal_means(season_name_to_months=season_name_to_months, start_year=start_year, end_year=end_year) dfs_lons, dfs_lats = dfs_manager.get_lons_and_lats_2d() xt, yt, zt = lat_lon.lon_lat_to_cartesian(dfs_lons.flatten(), dfs_lats.flatten()) xs, ys, zs = lat_lon.lon_lat_to_cartesian(cru_lons.flatten(), cru_lats.flatten()) ktree = cKDTree(data=list(zip(xs, ys, zs))) dists, inds = ktree.query(list(zip(xt, yt, zt))) season_to_err = OrderedDict() for k in season_to_mean_dfs: interpolated_cru = season_to_mean_cru[k].flatten()[inds].reshape(dfs_lons.shape) if dfs_var_name.lower() == "t2": #interpolated_cru += 273.15 season_to_mean_dfs[k] -= 273.15 elif dfs_var_name.lower() == "precip": # precipitation in mm/day season_to_mean_dfs[k] *= 24 * 60 * 60 season_to_err[k] = season_to_mean_dfs[k] #- interpolated_cru season_indicator = "-".join(sorted(season_to_err.keys())) fig_path = os.path.join(NEMO_IMAGES_DIR, "{3}_errors_{0}-{1}_{2}_dfs.jpeg".format(start_year, end_year, season_indicator, dfs_var_name)) basemap = nemo_commons.get_default_basemap_for_glk(dfs_lons, dfs_lats, resolution="l") x, y = basemap(dfs_lons, dfs_lats) coords_and_basemap = { "basemap": basemap, "x": x, "y": y } plot_errors_in_one_figure(season_to_err, fig_path=fig_path, **coords_and_basemap)
def read_and_interpolate_homa_data(self, path="", start_year=None, end_year=None, season_to_months=None): """ :param path: :param target_cube: """ import pandas as pd ds = Dataset(path) sst = ds.variables["sst"][:] # read longitudes and latitudes from a file lons_source = ds.variables["lon"][:] lats_source = ds.variables["lat"][:] month_to_season = defaultdict(lambda: "no-season") for seas, mths in season_to_months.items(): for m in mths: month_to_season[m] = seas # time variable time_var = ds.variables["time"] dates = num2date(time_var[:], time_var.units) if hasattr(sst, "mask"): sst[sst.mask] = np.nan panel = pd.Panel(data=sst, items=dates, major_axis=range(sst.shape[1]), minor_axis=range(sst.shape[2])) seasonal_sst = panel.groupby( lambda d: (d.year, month_to_season[d.month]), axis="items").mean() # source grid xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons_source.flatten(), lats_source.flatten()) kdtree = cKDTree(data=list(zip(xs, ys, zs))) # target grid xt, yt, zt = lat_lon.lon_lat_to_cartesian(self.lons.flatten(), self.lats.flatten()) dists, inds = kdtree.query(list(zip(xt, yt, zt))) assert isinstance(seasonal_sst, pd.Panel) result = {} for the_year in range(start_year, end_year + 1): result[the_year] = {} for the_season in list(season_to_months.keys()): the_mean = seasonal_sst.select(lambda item: item == (the_year, the_season), axis="items") result[the_year][the_season] = the_mean.values.flatten()[inds].reshape(self.lons.shape) return result
def _graph_to_kdtree(self) -> None: # add node_ids to coordinates to support overlapping nodes in cKDTree data = [ tuple(node["location"]) + (node_id,) for node_id, node in self.g.nodes.items() ] # place nodes in the kdtree self.data = cKDTree(np.array(list(data)))
def waypoints_cb(self, waypoints): self.waypoints = waypoints # setup k-d tree in order to find the closest path waypoint to the given position [get_closest_waypoint(position)] # the code is copied from waypoints_cb(pose) in ros/src/waypoint_updater/waypoint_updater.py written by https://github.com/ysonggit if not self.waypoints_2d: self.waypoints_2d = [[waypoint.pose.pose.position.x, waypoint.pose.pose.position.y] for waypoint in waypoints.waypoints] self.waypoint_tree = cKDTree(self.waypoints_2d)
def get_n_neighbours(df, cls, ds, r): from scipy.spatial.ckdtree import cKDTree global points, tree points = np.array([df.x, df.y]).T points -= np.min(points, axis=0, keepdims=True) points /= np.max(points, axis=0, keepdims=True) tree = cKDTree(points) return pd.Series([*map(len, tree.query_ball_point(points, r))], index=df.index)
def computerecondensity(d3d, label, leafs=16, PIX=10, IMGMAX=40000): """ Compute Local Effective Resolution :param d3d: 3D points :param label: :param leafs: Number of leafs (SNR = np.sqrt(leafs/2)) :param PIX: nm to pixel (e.g. 10nm per pixel = 100nm^2 :param IMGMAX: Upper limit of the image ROI :return: The CKDTree of the points (with leaf size), the image array, the points in leafs, and the image array where im[pix_x, pix_y] = LER """ points = d3d[:, :2].copy() mx, MX = np.min(points[:, 0]), np.max(points[:, 0]) my, MY = np.min(points[:, 1]), np.max(points[:, 1]) if mx < 0: points[:, 0] += np.abs(mx) print("Neg pos for {}".format(label)) if my < 0: points[:, 1] += np.abs(my) print("Neg pos for {}".format(label)) tr = cKDTree(points, leafsize=leafs) lfs = [] root = tr.tree leaves(root, lfs) assert (MX < IMGMAX) assert (MY < IMGMAX) pixels = np.empty((len(lfs), 2), dtype=np.float64) imgsize = int(np.round(IMGMAX / PIX)) imarray = np.zeros((imgsize, imgsize), dtype=np.float32) for ip, p in enumerate(lfs): ps = points[p] # assert(ps.shape[0] <= leafs) assert (ps.shape[1] == 2) _mx = min(ps[:, 0]) _Mx = max(ps[:, 0]) _my = min(ps[:, 1]) _My = max(ps[:, 1]) _x, _y = topix(_mx, _my, PIX) # Lower left _X, _Y = topix(_Mx, _My, PIX) # Upper right # ELR = nr of points over area xy - XY # imarray[_x:_X, _y:_Y] += ps.shape[0] pixels[ip, 0] = ((_X - _x) + 1) * ((_Y - _y) + 1) # Superpixel area pixels[ip, 1] = ps.shape[0] / pixels[ip, 0] # Locs / px^2 for pri in range(ps.shape[0]): pti = ps[pri, :2] xp, yp = topix(pti[0], pti[1], PIX) assert (xp < IMGMAX) assert ( yp < IMGMAX) # You'd assume this check is not nec. as an index error would follow, but you'd be wrong in interesting cases try: imarray[xp, yp] += pixels[ip, 1] except OverflowError as e: print("OE {} , {} <- {} {}".format(xp, yp, pixels[ip, 1], pixels[ip, 0])) print("OE {} , {} <- {:.2f} {:.2f}".format(xp, yp, pti[0], pti[1])) raise return tr, imarray, pixels
def _graph_to_kdtree(self) -> None: # add node_ids to coordinates to support overlapping nodes in cKDTree ids, attrs = zip(*self.g.nodes.items()) assert all([a == b for a, b in zip(ids, range(len(ids))) ]), "ids are not properly sorted" data = [point_attrs["location"].tolist() for point_attrs in attrs] logger.debug("placing {} nodes in the kdtree".format(len(data))) self.data = cKDTree(np.array(list(data))) logger.debug("kdtree initialized".format(len(data)))
def get_dataless_model_points_for_stations(station_list, accumulation_area_km2_2d, model_lons2d, model_lats2d, i_array, j_array): """ returns a map {station => modelpoint} for comparison modeled streamflows with observed this uses exactly the same method for searching model points as one in diagnose_point (nc-version) """ lons = model_lons2d[i_array, j_array] lats = model_lats2d[i_array, j_array] model_acc_area_1d = accumulation_area_km2_2d[i_array, j_array] npoints = 1 result = {} x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lons, lats) kdtree = cKDTree(list(zip(x0, y0, z0))) for s in station_list: # list of model points which could represent the station assert isinstance(s, Station) x, y, z = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude) dists, inds = kdtree.query((x, y, z), k=5) if npoints == 1: deltaDaMin = np.min( np.abs(model_acc_area_1d[inds] - s.drainage_km2)) # this returns a list of numpy arrays imin = np.where( np.abs(model_acc_area_1d[inds] - s.drainage_km2) == deltaDaMin)[0][0] selected_cell_index = inds[imin] # check if difference in drainage areas is not too big less than 10 % print(s.river_name, deltaDaMin / s.drainage_km2) # if deltaDaMin / s.drainage_km2 > 0.2: # continue mp = ModelPoint() mp.accumulation_area = model_acc_area_1d[selected_cell_index] mp.longitude = lons[selected_cell_index] mp.latitude = lats[selected_cell_index] mp.cell_index = selected_cell_index mp.distance_to_station = dists[imin] print("Distance to station: ", dists[imin]) print("Model accumulation area: ", mp.accumulation_area) print("Obs accumulation area: ", s.drainage_km2) result[s] = mp else: raise Exception("npoints = {0}, is not yet implemented ...") return result
def get_dataless_model_points_for_stations(station_list, accumulation_area_km2_2d, model_lons2d, model_lats2d, i_array, j_array): """ returns a map {station => modelpoint} for comparison modeled streamflows with observed this uses exactly the same method for searching model points as one in diagnose_point (nc-version) """ lons = model_lons2d[i_array, j_array] lats = model_lats2d[i_array, j_array] model_acc_area_1d = accumulation_area_km2_2d[i_array, j_array] npoints = 1 result = {} x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lons, lats) kdtree = cKDTree(list(zip(x0, y0, z0))) for s in station_list: # list of model points which could represent the station assert isinstance(s, Station) x, y, z = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude) dists, inds = kdtree.query((x, y, z), k=5) if npoints == 1: deltaDaMin = np.min(np.abs(model_acc_area_1d[inds] - s.drainage_km2)) # this returns a list of numpy arrays imin = np.where(np.abs(model_acc_area_1d[inds] - s.drainage_km2) == deltaDaMin)[0][0] selected_cell_index = inds[imin] # check if difference in drainage areas is not too big less than 10 % print(s.river_name, deltaDaMin / s.drainage_km2) # if deltaDaMin / s.drainage_km2 > 0.2: # continue mp = ModelPoint() mp.accumulation_area = model_acc_area_1d[selected_cell_index] mp.longitude = lons[selected_cell_index] mp.latitude = lats[selected_cell_index] mp.cell_index = selected_cell_index mp.distance_to_station = dists[imin] print("Distance to station: ", dists[imin]) print("Model accumulation area: ", mp.accumulation_area) print("Obs accumulation area: ", s.drainage_km2) result[s] = mp else: raise Exception("npoints = {0}, is not yet implemented ...") return result
def generate_kdtree(folder_with_ini_files, filename_prefix): lons, lats = [], [] paths = [] sel_files = [ f for f in os.listdir(folder_with_ini_files) if f.startswith(filename_prefix) and f.lower().endswith(".ini") ] xll, yll = None, None xur, yur = None, None for fname in sel_files: fpath = os.path.join(folder_with_ini_files, fname) lon, lat = read_lon_lat_from_ini_file(fpath) lons.append(lon) lats.append(lat) paths.append(fpath) x0, y0, z0 = lat_lon.lon_lat_to_cartesian(np.asarray(lons), np.asarray(lats)) return cKDTree(list(zip(x0, y0, z0))), paths
def nearest_match(cat0, cat1, tol=1.0, **kwargs): """Find nearest neighbors in two lists. Parameters ---------- cat0, cat1 : arrays Each catalog is a 2xN array of (y, x) positions. tol : float, optional The radial match tolerance in pixels. **kwargs Keyword arguments for `meanclip` when computing mean offsets. Returns ------- matches : dictionary The best match for star `i` of `cat0` is `matches[i]` in `cat1`. Stars that are matched multiple times, or not matched at all, are not returned. dyx : ndarray Sigma-clipped mean offset. Clipping is done in x and y separately, and only the union of the two is returned. The order for `dxy` is undefined. """ from scipy.spatial.ckdtree import cKDTree from .util import takefrom, meanclip assert len(cat0) == 2 assert len(cat1) == 2 tree = cKDTree(cat0.T) d, i = tree.query(cat1.T) # 0 of cat1 -> i[0] of cat0 matches = dict() for k, j in enumerate(i): if d[k] < tol: matches[j] = k dyx = cat0[:, list(matches.keys())] - cat1[:, list(matches.values())] mcx = meanclip(dyx[1], full_output=True) mcy = meanclip(dyx[0], full_output=True) j = list(set(np.concatenate((mcx[2], mcy[2])))) return matches, dyx[:, j]
def __init__(self, flow_dirs, nx=None, ny=None, lons2d=None, lats2d=None, accumulation_area_km2=None): self.cells = [] self.lons2d = lons2d self.lats2d = lats2d self.flow_directions = flow_dirs self.accumulation_area_km2 = accumulation_area_km2 # calculate characteristic distance if not any([None is arr for arr in [self.lats2d, self.lons2d]]): v1 = lat_lon.lon_lat_to_cartesian(self.lons2d[0, 0], self.lats2d[0, 0]) v2 = lat_lon.lon_lat_to_cartesian(self.lons2d[1, 1], self.lats2d[1, 1]) dv = np.array(v2) - np.array(v1) self.characteristic_distance = np.sqrt(np.dot(dv, dv)) x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten()) self.kdtree = cKDTree(list(zip(x, y, z))) if None not in [nx, ny]: self.nx = nx self.ny = ny else: nx, ny = flow_dirs.shape self.nx, self.ny = flow_dirs.shape for i in range(nx): self.cells.append(list([Cell(i=i, j=j, flow_dir_value=flow_dirs[i, j]) for j in range(ny)])) self._without_next_mask = np.zeros((nx, ny), dtype=np.int) self._wo_next_wo_prev_mask = np.zeros((nx, ny), dtype=np.int) # mask of the potential outlets for i in range(nx): if i % 100 == 0: print("Created {}/{}".format(i, nx)) for j in range(ny): i_next, j_next = direction_and_value.to_indices(i, j, flow_dirs[i][j]) next_cell = None if 0 <= i_next < nx: if 0 <= j_next < ny: next_cell = self.cells[i_next][j_next] self._without_next_mask[i, j] = int(next_cell is None) self.cells[i][j].set_next(next_cell)
def ntree(vals, errs, truevals, knear=k_near): truetree = ckdtree.cKDTree(truevals) out = [] for val, err in zip(vals, errs): dnear, inear = truetree.query(val, k=knear) truearr = truetree.data[inear] covI = 1./err**2. A = 1./np.sqrt( (2.*np.pi)**len(val) * np.prod(err**2.) ) diff = val-truearr B = -0.5 * np.sum(diff**2.*covI, axis=1) C = A * np.exp(B) out.append(np.sum(C)) return np.array(out)
def _read_lon_lats(self): """ Read the lons and lats, 2d from the first file """ the_date = datetime(1980, 1, 1) fpath = os.path.join(self.folder_path, the_date.strftime(self.fname_format)) # Check if file exists before trying to read it if not os.path.isfile(fpath): raise IOError("No such file: {}".format(fpath)) ds = Dataset(fpath) self.lons2d = ds.variables["lon"][:].transpose() self.lats2d = ds.variables["lat"][:].transpose() x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten()) self.kdtree = cKDTree(list(zip(x, y, z))) ds.close()
def main(): name_constraint = iris.Constraint(cube_func=lambda c: c.var_name == "sosstsst") data_cube = iris.load_cube(T_FILE_PATH, constraint=name_constraint) assert isinstance(data_cube, iris.cube.Cube) lons = data_cube.coord("longitude").points[:] lats = data_cube.coord("latitude").points[:] print(lons.shape, lats.shape) x, y, z = lat_lon.lon_lat_to_cartesian(lons.flatten(), lats.flatten()) ktree = cKDTree(data=list(zip(x, y, z))) for st in get_obs_data(data_folder="/home/huziy/skynet3_rech1/nemo_obs_for_validation/temperature_at_points_ts"): st.compare_with_modelled(data_cube, img_folder=NEMO_IMAGES_DIR, ktree=ktree) import matplotlib.pyplot as plt plt.show()
def ptree(vals, errs, truevals, knear=k_near): sigma = np.median(errs.T, axis=1) truetree = ckdtree.cKDTree(truevals/sigma) out = [] for val, err in zip(vals, errs): dnear, inear = truetree.query(val/sigma, k=knear) truearr = truetree.data[inear] * sigma covI = 1./err**2. A = 1./np.sqrt( (2.*np.pi)**len(val) * np.prod(err**2.) ) diff = val-truearr B = -0.5 * np.sum(diff**2.*covI, axis=1) C = A * np.exp(B) out.append(np.sum(C) * float(len(truevals))/len(truearr)) return np.array(out)
def plot_depth_to_bedrock(basemap, lons1, lats1, field1, label1, lons2, lats2, field2, label2, base_folder="/skynet3_rech1/huziy/veg_fractions/" ): xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons2.flatten(), lats2.flatten()) xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons1.flatten(), lats1.flatten()) ktree = cKDTree(list(zip(xs, ys, zs))) dists, inds = ktree.query(list(zip(xt, yt, zt))) levels = np.arange(0, 5.5, 0.5) field2_interp = field2.flatten()[inds].reshape(field1.shape) field1 = np.ma.masked_where(field1 < 0, field1) field2_interp = np.ma.masked_where(field2_interp < 0, field2_interp) vmin = min(field1.min(), field2_interp.min()) vmax = max(field1.max(), field2_interp.max()) cmap = cm.get_cmap("BuPu", len(levels) - 1) bn = BoundaryNorm(levels, len(levels) - 1) x, y = basemap(lons1, lats1) imname = "depth_to_bedrock_{0}-{1}.jpeg".format(label2, label1) impath = os.path.join(base_folder, imname) fig = plt.figure(figsize=(6, 2.5)) gs = GridSpec(1, 3, width_ratios=[1, 1, 0.05]) ax = fig.add_subplot(gs[0, 0]) basemap.pcolormesh(x, y, field1, vmin=vmin, vmax=vmax, cmap=cmap, norm=bn) basemap.drawcoastlines(linewidth=common_plot_params.COASTLINE_WIDTH) ax.set_title(label1) ax = fig.add_subplot(gs[0, 1]) im = basemap.pcolormesh(x, y, field2_interp, vmin=vmin, vmax=vmax, cmap=cmap, norm=bn) basemap.drawcoastlines(linewidth=common_plot_params.COASTLINE_WIDTH) ax.set_title(label2) plt.colorbar(im, cax=fig.add_subplot(gs[0, 2])) fig.tight_layout() fig.savefig(impath, dpi=common_plot_params.FIG_SAVE_DPI)
def _init_fields(self, nc_dataset): nc_vars = nc_dataset.variables lons = nc_vars["lon"][:] lats = nc_vars["lat"][:] lats2d, lons2d = np.meshgrid(lats, lons) self.lons2d, self.lats2d = lons2d, lats2d self.times_var = nc_vars["time"] self.times_num = nc_vars["time"][:] if hasattr(self.times_var, "calendar"): self.times = num2date(self.times_num, self.times_var.units, self.times_var.calendar) else: self.times = num2date(self.times_num, self.times_var.units) if not self.lazy: self.var_data = np.transpose(nc_vars[self.var_name][:], axes=[0, 2, 1]) x_in, y_in, z_in = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten()) self.kdtree = cKDTree(list(zip(x_in, y_in, z_in)))
def __discardClumped(self, xs, ys, radius=4): if len(xs) < 2: return xs, ys kdt = ckdtree.cKDTree(numpy.array([xs,ys]).T) xsd = [] ysd = [] for i in xrange(len(xs)): xi = xs[i] yi = ys[i] #neigh = kdt.query_ball_point([xi,yi], radius) dn, neigh = kdt.query(numpy.array([xi,yi]), 2) print dn if (dn[1] > radius): xsd.append(xi) ysd.append(yi) print len(xsd) return numpy.array(xsd), numpy.array(ysd)
def spatial_match(cat0, cat1, tol=0.01, min_score=0, full_output=False, verbose=True): """Find spatially matching sourse between two lists. Parameters ---------- cat0, cat1 : arrays Each catalog is an 2xN array of (y, x) positions. tol : float, optional The match tolerance. min_score : float, optional Only return matches with scores greater than `min_score`. full_output : bool, optional Set to `True` to also return `match_matrix`. verbose : bool, optional Print some feedback for the user. Returns ------- matches : dictionary The best match for star `i` of `cat0` is `matches[i]` in `cat1`. Stars that are matched multiple times, not matched at all, or with scores less than `min_score` are not returned. score : dictionary Fraction of times star `i` matched star `matches[i]` out of all times stars `i` and `matches[i]` were matched to any star. match_matrix : ndarray, optional If `full_output` is `True`, also return the matrix of all star matches. Notes ----- Based on the description of DAOPHOT's catalog matching via triangles at http://ned.ipac.caltech.edu/level5/Stetson/Stetson5_2.html """ from scipy.spatial.ckdtree import cKDTree v0, s0 = triangles(*cat0) v1, s1 = triangles(*cat1) N0 = len(cat0[0]) N1 = len(cat1[0]) tree = cKDTree(s0) d, i = tree.query(s1) # nearest matches between triangles if verbose: print ("""[spatial_match] cat0 = {} triangles, cat1 = {} triangles [spatial_match] Best match score = {:.2g}, worst match sorce = {:.2g} [spatial_match] {} triangle pairs at or below given tolerance ({})""".format( len(v0), len(v1), min(d), max(d), sum(d <= tol), tol)) match_matrix = np.zeros((N0, N1), int) for k, j in enumerate(i): if d[k] <= tol: match_matrix[v0[j][0], v1[k][0]] += 1 match_matrix[v0[j][1], v1[k][1]] += 1 match_matrix[v0[j][2], v1[k][2]] += 1 m0 = match_matrix.argmax(1) m1 = match_matrix.argmax(0) matches = dict() scores = dict() for i in range(len(m0)): if i == m1[m0[i]]: matches[i] = m0[i] peak = match_matrix[i, m0[i]] * 2 total = match_matrix[i, :].sum() + match_matrix[:, m0[i]].sum() scores[i] = peak / float(total) for k in matches.keys(): if scores[k] < min_score: del matches[k], scores[k] if full_output: return matches, scores, match_matrix else: return matches, scores
SSR = n.ones_like(speccat['ALPHA'])*-1. SSR_ERR = n.ones_like(speccat['ALPHA'])*-1. redshiftBest = n.empty(Nspec_total) goodZ = (speccat['ZFLAGS']==2)|(speccat['ZFLAGS']==3)|(speccat['ZFLAGS']==4)|(speccat['ZFLAGS']==9) badZ = (goodZ==False) redshiftBest[goodZ] = speccat['Z'][goodZ] raBad, decBad = speccat['ALPHA'][badZ], speccat['DELTA'][badZ] # photometric redshift catalog zphcatall = fits.open(join(survey.vvds_photo_dir, photozCat) )[1].data zphcat = zphcatall[ (zphcatall['zphot_T07']>-0.1)&(zphcatall['zphot_T07']<2.) ] treePhZ = t.cKDTree( n.transpose([zphcat['alpha_T07'], zphcat['delta_T07']]) ,1000.0) indexes = treePhZ.query(n.transpose([raBad,decBad]),1) dist = indexes[0] ids = indexes[1] ok = (dist < 1.5/3600.) nok = (dist >= 1.5/3600.) #redshiftBest[badZ] = zphcat['zphot_T07'][ids] redshiftBest[badZ[ok]] = zphcat['zphot_T07'][ids[ok]] redshiftBest[badZ[nok]] = n.ones_like(zphcat['zphot_T07'][ids[nok]])*-1. print float(len(zphcat['zphot_T07'][ids[ok]]))/len(raBad), "% of photoz used" print len(zphcat['zphot_T07'][ids[ok]]) print len(raBad) print len(redshiftBest[badZ[ok]]) print len(redshiftBest[badZ[nok]])
def validate_max_ice_cover_with_glerl(): """ For validations of maximum annual ice concentrations with GLERL obs """ nemo_manager = NemoYearlyFilesManager(folder="/home/huziy/skynet3_rech1/offline_glk_output_daily_1979-2012", suffix="icemod.nc") nemo_manager_nosnow = NemoYearlyFilesManager( folder="/home/huziy/skynet3_rech1/output_NEMO_offline_1979-2012_nosnow", suffix="icemod.nc") # Study period start_year = 2003 end_year = 2012 lon2d, lat2d, bmp = nemo_manager.get_coords_and_basemap() model_yearmax_ice_conc, model_lake_avg_ts = nemo_manager.get_max_yearly_ice_fraction( start_year=start_year, end_year=end_year) model_yearmax_ice_conc = np.ma.masked_where(~nemo_manager.lake_mask, model_yearmax_ice_conc) model_yearmax_ice_conc_nosnow, model_lake_avg_ts_no_snow = nemo_manager_nosnow.get_max_yearly_ice_fraction( start_year=start_year, end_year=end_year) model_yearmax_ice_conc_nosnow = np.ma.masked_where(~nemo_manager.lake_mask, model_yearmax_ice_conc_nosnow) # plt.figure() xx, yy = bmp(lon2d.copy(), lat2d.copy()) # im = bmp.pcolormesh(xx, yy, model_yearmax_ice_conc) # bmp.colorbar(im) # Read and interpolate obs path_to_obs = "/RESCUE/skynet3_rech1/huziy/nemo_obs_for_validation/glerl_icecov.nc" obs_varname = "ice_cover" obs_lake_avg_ts = [] with Dataset(path_to_obs) as ds: time_var = ds.variables["time"] lons_obs = ds.variables["lon"][:] lats_obs = ds.variables["lat"][:] dates = num2date(time_var[:], time_var.units) nx, ny = lons_obs.shape data = ds.variables[obs_varname][:] data = np.ma.masked_where((data > 100) | (data < 0), data) print(data.min(), data.max()) panel = pd.Panel(data=data, items=dates, major_axis=range(nx), minor_axis=range(ny)) panel = panel.select(lambda d: start_year <= d.year <= end_year) the_max_list = [] for key, g in panel.groupby(lambda d: d.year, axis="items"): the_max_field = np.ma.max(np.ma.masked_where((g.values > 100) | (g.values < 0), g.values), axis=0) obs_lake_avg_ts.append(the_max_field.mean()) the_max_list.append(the_max_field) obs_yearmax_ice_conc = np.ma.mean(the_max_list, axis=0) / 100.0 xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons_obs.flatten(), lats_obs.flatten()) ktree = cKDTree(list(zip(xs, ys, zs))) xt, yt, zt = lat_lon.lon_lat_to_cartesian(lon2d.flatten(), lat2d.flatten()) dists, inds = ktree.query(list(zip(xt, yt, zt))) obs_yearmax_ice_conc_interp = obs_yearmax_ice_conc.flatten()[inds].reshape(lon2d.shape) obs_yearmax_ice_conc_interp = np.ma.masked_where(~nemo_manager.lake_mask, obs_yearmax_ice_conc_interp) # plt.figure() # b = Basemap() # xx, yy = b(lons_obs, lats_obs) # im = b.pcolormesh(xx, yy, obs_yearmax_ice_conc) # b.colorbar(im) # b.drawcoastlines() # Plot as usual: model, obs, model - obs img_folder = Path("nemo") if not img_folder.is_dir(): img_folder.mkdir() img_file = img_folder.joinpath("validate_yearmax_icecov_glerl_{}-{}.png".format(start_year, end_year)) plot_utils.apply_plot_params(height_cm=9, width_cm=45, font_size=12) fig = plt.figure() gs = GridSpec(1, 5, width_ratios=[1, 1, 0.05, 1, 0.05]) all_axes = [] cmap = cm.get_cmap("jet", 10) diff_cmap = cm.get_cmap("RdBu_r", 10) # Model ax = fig.add_subplot(gs[0, 0]) ax.set_title("NEMO-offline") bmp.pcolormesh(xx, yy, model_yearmax_ice_conc, cmap=cmap, vmin=0, vmax=1) all_axes.append(ax) # Model # ax = fig.add_subplot(gs[0, 1]) # ax.set_title("NEMO-offline-nosnow") # bmp.pcolormesh(xx, yy, model_yearmax_ice_conc_nosnow, cmap=cmap, vmin=0, vmax=1) # all_axes.append(ax) # Obs ax = fig.add_subplot(gs[0, 1]) ax.set_title("NIC") im = bmp.pcolormesh(xx, yy, obs_yearmax_ice_conc_interp, cmap=cmap, vmin=0, vmax=1) all_axes.append(ax) plt.colorbar(im, cax=fig.add_subplot(gs[0, 2])) # Biases ax = fig.add_subplot(gs[0, 3]) ax.set_title("NEMO - NIC") im = bmp.pcolormesh(xx, yy, model_yearmax_ice_conc - obs_yearmax_ice_conc_interp, cmap=diff_cmap, vmin=-1, vmax=1) plt.colorbar(im, cax=fig.add_subplot(gs[0, -1])) all_axes.append(ax) for the_ax in all_axes: bmp.drawcoastlines(ax=the_ax) the_ax.set_aspect("auto") fig.savefig(str(img_file), bbox_inches="tight") plt.close(fig) # Plot lake aversged ice concentrations fig = plt.figure() plt.gca().get_xaxis().get_major_formatter().set_useOffset(False) plt.plot(range(start_year, end_year + 1), model_lake_avg_ts, "b", lw=2, label="NEMO") plt.plot(range(start_year, end_year + 1), model_lake_avg_ts_no_snow, "g", lw=2, label="NEMO-nosnow") plt.plot(range(start_year, end_year + 1), np.asarray(obs_lake_avg_ts) / 100.0, "r", lw=2, label="NIC") plt.grid() plt.legend(ncol=2) fig.savefig(str(img_folder.joinpath("lake_avg_iceconc_nemo_offline_vs_NIC.pdf")), bbox_inches="tight")
def interpolate_file(self, inpath, outpath, skip_feb_29=True): """ Interpolate data in the file, save the result to a new file in the same folder, with the name of the interpolated variable and time variable unchanged. """ #check if the output file already exists if os.path.isfile(outpath): print("{0} already exists, remove to recreate ...".format(outpath)) return print("working on {0}".format(inpath)) ds_in = Dataset(inpath) lon_ncatts = {} lat_ncatts = {} #read in and calculate coordinates of source grid if "lon" in ds_in.variables: in_lon_var_name = "lon" in_lat_var_name = "lat" elif "nav_lon" in ds_in.variables: in_lon_var_name = "nav_lon" in_lat_var_name = "nav_lat" elif "lon0" in ds_in.variables: in_lon_var_name = "lon0" in_lat_var_name = "lat0" else: raise Exception("The file does not contain conventional lat/lon information: {0}".format(inpath)) in_lon_var = ds_in.variables[in_lon_var_name] in_lat_var = ds_in.variables[in_lat_var_name] source_lons, source_lats = in_lon_var[:], in_lat_var[:] if in_lon_var.ndim == 1: source_lons, source_lats = np.meshgrid(source_lons, source_lats) for attname in in_lon_var.ncattrs(): lon_ncatts[attname] = in_lon_var.getncattr(attname) for attname in in_lat_var.ncattrs(): lat_ncatts[attname] = in_lat_var.getncattr(attname) #find the name of the field to be interpolated, and read it into memory varnames = list(ds_in.variables.keys()) #write interpolated data ds_out = Dataset(outpath, "w", format="NETCDF3_CLASSIC") #copy and create dimensions ds_out.createDimension("time", None) ds_out.createDimension("x", self.target_lons.shape[1]) ds_out.createDimension("y", self.target_lons.shape[0]) #copy and interpolate variables lonVar = ds_out.createVariable(in_lon_var_name, "f4", ("y", "x")) latVar = ds_out.createVariable(in_lat_var_name, "f4", ("y", "x")) #set the attributes if len(lon_ncatts): lonVar.setncatts(lon_ncatts) latVar.setncatts(lat_ncatts) good_points = np.abs(source_lons.flatten()) < 360 xs, ys, zs = lon_lat_to_cartesian(source_lons.flatten(), source_lats.flatten()) xt, yt, zt = lon_lat_to_cartesian(self.target_lons.flatten(), self.target_lats.flatten()) ktree = cKDTree(data=list(zip(xs[good_points], ys[good_points], zs[good_points]))) dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1) # Handle time variable first timename = None time_data = None for v in varnames: if v.startswith("time"): timename = v break ## if timename is not None: time_var_in = ds_in.variables[timename] time_var_out = ds_out.createVariable(timename, "f4", ("time",)) time_vals = time_var_in[:] if time_var_in.shape[0] > 365 and skip_feb_29: if hasattr(time_var_in, "units") and time_var_in.units.strip().lower() != "unknown": time_data = num2date(time_vals, time_var_in.units) df = pd.DataFrame(data=time_data, index=time_data) time_vals = df.select(lambda d: not (d.day == 29 and d.month == 2)).values else: ntimes = time_vals.shape[0] if ntimes % 366 == 0 and ntimes % 365 != 0: nperday = ntimes // 366 dtseconds = 24 * 60 * 60 // nperday dt = timedelta(seconds=dtseconds) # dt0 = timedelta(seconds=int(time_vals[0] * 60 * 60)) # usually in seconds start_date = datetime(2008, 1, 1) time_data = [start_date + dt * i for i in range(ntimes)] assert time_data[0].year == time_data[-1].year df = pd.DataFrame(data=time_vals, index=time_data) time_vals = df.select(lambda d: not (d.day == 29 and d.month == 2)).values time_var_out[:] = time_vals if hasattr(time_var_in, "units"): time_var_out.units = time_var_in.units for varname in varnames: out_var = None in_var = ds_in.variables[varname] # Interpolate only 3d variables (time, lat, lon) and some 2d variables if in_var.ndim == 3: out_var = ds_out.createVariable(varname, "f4", ("time", "y", "x")) if hasattr(out_var, "units"): out_var.units = in_var.units in_data = in_var[:] if time_data is not None: p = pd.Panel(data=in_data, items=time_data, major_axis=list(range(in_data.shape[1])), minor_axis=list(range(in_data.shape[2]))) if in_data.shape[0] > 365 and skip_feb_29: p = p.select(lambda d: not (d.day == 29 and d.month == 2)) in_data = p.values # reshape to 2d print(in_data.shape, self.target_lons.shape) in_data.shape = (in_data.shape[0], -1) out_data = in_data[:, inds] out_data.shape = (out_data.shape[0], ) + self.target_lons.shape # print out_data.shape out_var[:] = out_data elif varname.lower() == "bathymetry": out_var = ds_out.createVariable(varname, "f4", ("y", "x")) out_var[:] = interpolate_bathymetry(in_var[:], ktree, source_coords=(xs, ys, zs), target_coords=(xt, yt, zt), out_data_shape=self.target_lons.shape) elif in_var.ndim == 2 and varname.lower() in ["socoefr"]: out_var = ds_out.createVariable(varname, "f4", ("y", "x")) in_data = in_var[:] # reshape to 2d in_data = in_data.flatten() out_data = in_data[inds] out_data.shape = self.target_lons.shape # print out_data.shape out_var[:] = out_data if out_var is not None: # Set attributes of the interpolated fields if hasattr(out_var, "long_name"): out_var.long_name = in_var.long_name if hasattr(in_var, "units"): out_var.units = in_var.units if hasattr(in_var, "missing_value"): out_var.missing_value = in_var.missing_value if hasattr(in_var, "coordinates"): out_var.coordinates = in_var.coordinates lonVar[:] = self.target_lons latVar[:] = self.target_lats # close netcdf files ds_out.close() ds_in.close()
def evolve(self, n_gen): """Perform evolution of the discriminators on the data provided in the initializer. Saves the discriminator networks to /networks. :param n_gen: :type int: Number of generations to run the evolution. """ try: for generation in range(n_gen): # run for n_gen generations if len(self.feature_list_genomes) < self.max_features: start = time.clock() outputs = np.zeros((self.params.PopulationSize, self.dsize)) # retrieve a list of all genomes in the population genome_list = NEAT.GetGenomeList(self.pop) # Get output space vectors for all genomes j = 0 for genome in genome_list: print "Computing behavior vector for genome " + str(j) output_array = self.get_output_vector(genome) k = 0 for element in output_array: outputs[j][k] = element k += 1 j += 1 # Evaluate novelty for all genomes and add to feature list i = 0 for genome in genome_list: print "Computing novelty for genome " + str(i) genome_vector = outputs[i] fitness = self.get_novelty(outputs, genome_vector) # Add to feature list if len(self.feature_list_genomes) == 0: m = 0 for element in genome_vector: self.feature_list_vectors[len(self.feature_list_genomes)][m] = element m += 1 self.feature_list_genomes.append(genome) else: tree = kd.cKDTree(self.feature_list_vectors) indexes = tree.query(genome_vector, 20) dist = indexes[0][0] print "Distance from closest in feature list: " + str(dist) if dist > 200: m = 0 for element in genome_vector: self.feature_list_vectors[len(self.feature_list_genomes)][m] = element m += 1 self.feature_list_genomes.append(genome) genome.SetFitness(fitness) i += 1 n = 0 for genome in self.feature_list_genomes: net = NEAT.NeuralNetwork() genome.BuildPhenotype(net) net.Save("path/to/dir/net" + str(n) + ".nnet") n += 1 # datafile = open("discriminator_genomes.pkl", "wb") # pickle.dump(self.feature_list_genomes, datafile) # datafile.close() # datafile = open("discriminator_genomes_backup.pkl", "wb") # pickle.dump(self.feature_list_genomes, datafile) # datafile.close() end = time.clock() elapsed = end - start print str(generation) + " generations evaluated, " + str(elapsed) + " for last gen." print str(len(self.feature_list_genomes)) + " discriminators evolved." print "----------" # advance to the next generation self.pop.Epoch() else: break except KeyboardInterrupt: print "Training stopped."
def validate_yearmax_ice_cover_from_hostetler_with_glerl(path="/home/huziy/skynet3_rech1/CRCM_GL_simulation/all_files", start_year=2003, end_year=2009): model_manager = Crcm5ModelDataManager(samples_folder_path=path, all_files_in_samples_folder=True) varname = "LC" hl_icecov_yearly_max = model_manager.get_yearmax_fields(start_year=start_year, end_year=end_year, var_name=varname) hl_icecov_yearly_max_clim = np.mean(list(hl_icecov_yearly_max.values()), axis=0) # Get Nemo manager here only for coordinates and mask nemo_manager = NemoYearlyFilesManager(folder="/home/huziy/skynet3_rech1/offline_glk_output_daily_1979-2012", suffix="icemod.nc") lon2d, lat2d, bmp = nemo_manager.get_coords_and_basemap() # Interpolate hostetler's lake fraction to the model's grid hl_icecov_yearly_max_clim = model_manager.interpolate_data_to(hl_icecov_yearly_max_clim, lon2d, lat2d) hl_icecov_yearly_max_clim = np.ma.masked_where(~nemo_manager.lake_mask, hl_icecov_yearly_max_clim) model_lake_avg_ts = [] for the_year in range(start_year, end_year + 1): model_lake_avg_ts.append(hl_icecov_yearly_max[the_year][nemo_manager.lake_mask].mean()) # plt.figure() xx, yy = bmp(lon2d.copy(), lat2d.copy()) # im = bmp.pcolormesh(xx, yy, model_yearmax_ice_conc) # bmp.colorbar(im) # Read and interpolate obs path_to_obs = "/RESCUE/skynet3_rech1/huziy/nemo_obs_for_validation/glerl_icecov.nc" obs_varname = "ice_cover" obs_lake_avg_ts = [] with Dataset(path_to_obs) as ds: time_var = ds.variables["time"] lons_obs = ds.variables["lon"][:] lats_obs = ds.variables["lat"][:] dates = num2date(time_var[:], time_var.units) nx, ny = lons_obs.shape data = ds.variables[obs_varname][:] data = np.ma.masked_where((data > 100) | (data < 0), data) print(data.min(), data.max()) panel = pd.Panel(data=data, items=dates, major_axis=range(nx), minor_axis=range(ny)) panel = panel.select(lambda d: start_year <= d.year <= end_year) the_max_list = [] for key, g in panel.groupby(lambda d: d.year, axis="items"): the_max_field = np.ma.max(np.ma.masked_where((g.values > 100) | (g.values < 0), g.values), axis=0) obs_lake_avg_ts.append(the_max_field.mean()) the_max_list.append(the_max_field) obs_yearmax_ice_conc = np.ma.mean(the_max_list, axis=0) / 100.0 xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons_obs.flatten(), lats_obs.flatten()) ktree = cKDTree(list(zip(xs, ys, zs))) xt, yt, zt = lat_lon.lon_lat_to_cartesian(lon2d.flatten(), lat2d.flatten()) dists, inds = ktree.query(list(zip(xt, yt, zt))) obs_yearmax_ice_conc_interp = obs_yearmax_ice_conc.flatten()[inds].reshape(lon2d.shape) obs_yearmax_ice_conc_interp = np.ma.masked_where(~nemo_manager.lake_mask, obs_yearmax_ice_conc_interp) # plt.figure() # b = Basemap() # xx, yy = b(lons_obs, lats_obs) # im = b.pcolormesh(xx, yy, obs_yearmax_ice_conc) # b.colorbar(im) # b.drawcoastlines() # Plot as usual: model, obs, model - obs img_folder = Path("nemo/hostetler") if not img_folder.is_dir(): img_folder.mkdir() img_file = img_folder.joinpath("validate_yearmax_icecov_hl_vs_glerl_{}-{}.pdf".format(start_year, end_year)) fig = plt.figure() gs = GridSpec(2, 4, width_ratios=[1, 1, 1, 0.05]) all_axes = [] cmap = cm.get_cmap("jet", 10) diff_cmap = cm.get_cmap("RdBu_r", 10) # Model ax = fig.add_subplot(gs[0, 0]) ax.set_title("Hostetler") bmp.pcolormesh(xx, yy, hl_icecov_yearly_max_clim, cmap=cmap, vmin=0, vmax=1) all_axes.append(ax) # Obs ax = fig.add_subplot(gs[0, 2]) ax.set_title("GLERL") im = bmp.pcolormesh(xx, yy, obs_yearmax_ice_conc_interp, cmap=cmap, vmin=0, vmax=1) all_axes.append(ax) plt.colorbar(im, cax=fig.add_subplot(gs[0, -1])) # Biases ax = fig.add_subplot(gs[1, :]) ax.set_title("Hostetler - GLERL") im = bmp.pcolormesh(xx, yy, hl_icecov_yearly_max_clim - obs_yearmax_ice_conc_interp, cmap=diff_cmap, vmin=-1, vmax=1) bmp.colorbar(im, ax=ax) all_axes.append(ax) for the_ax in all_axes: bmp.drawcoastlines(ax=the_ax) fig.savefig(str(img_file), bbox_inches="tight") plt.close(fig) # Plot lake aversged ice concentrations fig = plt.figure() plt.gca().get_xaxis().get_major_formatter().set_useOffset(False) plt.plot(range(start_year, end_year + 1), model_lake_avg_ts, "b", lw=2, label="Hostetler") plt.plot(range(start_year, end_year + 1), np.asarray(obs_lake_avg_ts) / 100.0, "r", lw=2, label="GLERL") plt.grid() plt.legend(loc=3) fig.savefig(str(img_folder.joinpath("lake_avg_iceconc_hostetler_offline_vs_GLERL.pdf")), bbox_inches="tight")