def compute2PCF(catalog, rmax=20, dr = 1., boxsize=250.): outfile = catalog[:-8]+".2pcf" hdu = fits.open(catalog)[1].data xR = hdu['x'] yR = hdu['y'] zR = hdu['z'] insideSel=(xR>rmax)&(xR<boxsize-rmax)&(yR>rmax)&(yR<boxsize-rmax)&(zR>rmax)&(zR<boxsize-rmax) volume=(boxsize)**3 # defines the trees print "creates trees" treeRandoms=t.cKDTree(n.transpose([xR,yR,zR]),1000.0) treeData=t.cKDTree(n.transpose([xR[insideSel],yR[insideSel],zR[insideSel]]),1000.0) nD=len(treeData.data) nR=len(treeRandoms.data) print nD, nR bin_xi3D=n.arange(0, rmax, dr) # now does the pair counts : pairs=treeData.count_neighbors(treeRandoms, bin_xi3D) t3 = time.time() DR=pairs[1:]-pairs[:-1] dV= (bin_xi3D[1:]**3 - bin_xi3D[:-1]**3 )*4*n.pi/3. pairCount=nD*nR#-nD*(nD-1)/2. xis = DR*volume/(dV * pairCount) -1. f=open(outfile,'w') cPickle.dump([bin_xi3D,xis, DR, volume, dV, pairCount, pairs, nD, nR],f) f.close()
def calculate_cistrans(data,
chains,
chain_id=0,
cutoff=5,
pbc_box=False,
box_size=None):
"""
Analysis of the territoriality of polymer chains from simulations, using the cis/trans ratio.
Cis signal is computed for the marked chain ('chain_id') as amount of contacts of the chain with itself
Trans signal is the total amount of trans contacts for the marked chain with other chains from 'chains'
(and with all the replicas for 'pbc_box'=True)
"""
if data.shape[1] != 3:
raise ValueError("Incorrect polymer data shape. Must be Nx3.")
if np.isnan(data).any():
raise RuntimeError("Data contains NANs")
N = len(data)
if pbc_box == True:
if box_size is None:
raise ValueError("Box size is not given")
else:
data_scaled = np.mod(data, box_size)
else:
box_size = None
data_scaled = np.copy(data)
if chains is None:
chains = [[0, N]]
chain_id = 0
chain_start = chains[chain_id][0]
chain_end = chains[chain_id][1]
# all contact pairs available in the scaled data
tree = ckdtree.cKDTree(data_scaled, boxsize=box_size)
pairs = tree.query_pairs(cutoff, output_type="ndarray")
# total number of contacts of the marked chain:
# each contact is counted twice if both monomers belong to the marked chain and
# only once if just one of the monomers in the pair belong to the marked chain
all_signal = len(pairs[pairs < chain_end]) - len(
pairs[pairs < chain_start])
# contact pairs of the marked chain with itself
tree = ckdtree.cKDTree(data[chain_start:chain_end], boxsize=None)
pairs = tree.query_pairs(cutoff, output_type="ndarray")
# doubled number of contacts of the marked chain with itself (i.e. cis signal)
cis_signal = 2 * len(pairs)
assert all_signal >= cis_signal
trans_signal = all_signal - cis_signal
return cis_signal, trans_signal
def generate_example_random_choice(positions, properties, k=26, plot=False):
print('choice nn')
idx_list = np.arange(len(positions))
virtual_node_positions = positions[np.random.choice(idx_list, 1000, replace=False)]
kdtree = cKDTree(virtual_node_positions)
dist, indices = kdtree.query(positions)
virtual_properties = np.zeros((len(np.bincount(indices)), len(properties[0])))
mean_sum = [lambda x: np.bincount(indices, weights=x) / np.maximum(1., np.bincount(indices)), # mean
lambda x: np.bincount(indices, weights=x)] # sum
mean_sum_enc = [0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1]
for p, enc in zip(np.arange(len(properties[0])), mean_sum_enc):
virtual_properties[:, p] = mean_sum[enc](properties[:, p])
virtual_positions = virtual_properties[:, :3]
graph = nx.DiGraph()
kdtree = cKDTree(virtual_positions)
dist, idx = kdtree.query(virtual_positions, k=k + 1)
receivers = idx[:, 1:] # N,k
senders = np.arange(virtual_positions.shape[0]) # N
senders = np.tile(senders[:, None], [1, k]) # N,k
receivers = receivers.flatten()
senders = senders.flatten()
n_nodes = virtual_positions.shape[0]
pos = dict() # for plotting node positions.
edgelist = []
for node, feature, position in zip(np.arange(n_nodes), virtual_properties, virtual_positions):
graph.add_node(node, features=feature)
pos[node] = position[:2]
# edges = np.stack([senders, receivers], axis=-1) + sibling_node_offset
for u, v in zip(senders, receivers):
graph.add_edge(u, v, features=np.array([1., 0.]))
graph.add_edge(v, u, features=np.array([1., 0.]))
edgelist.append((u, v))
edgelist.append((v, u))
graph.graph["features"] = np.array([0.])
# plotting
print('len(pos) = {}\nlen(edgelist) = {}'.format(len(pos), len(edgelist)))
if plot:
fig, ax = plt.subplots(1, 1, figsize=(20, 20))
draw(graph, ax=ax, pos=pos, node_color='blue', edge_color='red', node_size=10, width=0.1)
image_dir = '/data2/hendrix/images/'
graph_image_idx = len(glob.glob(os.path.join(image_dir, 'graph_image_*')))
plt.savefig(os.path.join(image_dir, 'graph_image_{}'.format(graph_image_idx)))
return networkxs_to_graphs_tuple([graph],
node_shape_hint=[virtual_positions.shape[1] + virtual_properties.shape[1]],
edge_shape_hint=[2])
async def update(self):
self.update_minerals()
self.tag_cache.clear()
self.own_unit_cache.clear()
self.enemy_unit_cache.clear()
self.force_fields.clear()
self.own_numpy_vectors = []
self.enemy_numpy_vectors = []
self.all_own = self.knowledge.all_own
for unit in self.all_own:
if unit.is_memory:
self.tag_cache[unit.tag] = unit
units = self.own_unit_cache.get(unit.type_id, Units([], self.ai))
if units.amount == 0:
self.own_unit_cache[unit.type_id] = units
units.append(unit)
self.own_numpy_vectors.append(
np.array([unit.position.x, unit.position.y]))
for unit in self.knowledge.known_enemy_units:
if unit.is_memory:
self.tag_cache[unit.tag] = unit
units = self.enemy_unit_cache.get(unit.type_id, Units([], self.ai))
if units.amount == 0:
self.enemy_unit_cache[unit.type_id] = units
units.append(unit)
self.enemy_numpy_vectors.append(
np.array([unit.position.x, unit.position.y]))
for unit in self.ai.all_units:
# Add all non-memory units to unit tag cache
self.tag_cache[unit.tag] = unit
if len(self.own_numpy_vectors) > 0:
self.own_tree = cKDTree(self.own_numpy_vectors)
else:
self.own_tree = None
if len(self.enemy_numpy_vectors) > 0:
self.enemy_tree = cKDTree(self.enemy_numpy_vectors)
else:
self.enemy_tree = None
for effect in self.ai.state.effects:
if effect.id == FakeEffectID.get(UnitTypeId.FORCEFIELD.value):
self.force_fields.append(effect)
def _derive_nearest_neighbours(
latitudes_stations: np.array,
longitudes_stations: np.array,
coordinates: Coordinates,
num_stations_nearby: int = 1,
) -> Tuple[Union[float, np.ndarray], np.ndarray]:
"""
A function that uses a k-d tree algorithm to obtain the nearest
neighbours to coordinate pairs
Args:
latitudes_stations (np.array): latitude values of stations being compared to
the coordinates
longitudes_stations (np.array): longitude values of stations being compared to
the coordinates
coordinates (Coordinates): the coordinates for which the nearest neighbour
is searched
num_stations_nearby: Number of stations that should be nearby
Returns:
Tuple of distances and ranks of nearest to most distant stations
"""
points = np.c_[np.radians(latitudes_stations),
np.radians(longitudes_stations)]
distance_tree = cKDTree(points)
return distance_tree.query(coordinates.get_coordinates_in_radians(),
k=num_stations_nearby)
def _init_fields(self, nc_dataset):
nc_vars = nc_dataset.variables
lons = nc_vars["lon"][:]
lats = nc_vars["lat"][:]
if lons.ndim == 1:
lats2d, lons2d = np.meshgrid(lats, lons)
elif lons.ndim == 2:
lats2d, lons2d = lats, lons
else:
raise NotImplementedError(
"Cannot handle {}-dimensional coordinates".format(lons.ndim))
self.lons2d, self.lats2d = lons2d, lats2d
self.times_var = nc_vars["time"]
self.times_num = nc_vars["time"][:]
if hasattr(self.times_var, "calendar"):
self.times = num2date(self.times_num, self.times_var.units,
self.times_var.calendar)
else:
self.times = num2date(self.times_num, self.times_var.units)
if not self.lazy:
self.var_data = nc_vars[self.var_name][:]
if nc_vars[self.var_name].shape[1:] != self.lons2d.shape:
print("nc_vars[self.var_name].shape = {}".format(
nc_vars[self.var_name].shape))
self.var_data = np.transpose(self.var_data, axes=[0, 2, 1])
x_in, y_in, z_in = lat_lon.lon_lat_to_cartesian(
self.lons2d.flatten(), self.lats2d.flatten())
self.kdtree = cKDTree(list(zip(x_in, y_in, z_in)))
def __call__(self, x):
num_points = x.shape[-1]
tree = cKDTree(np.transpose(x))
to_query = np.random.randint(low=0, high=num_points)
_, indices = tree.query(x[:, to_query], self.num_samples)
mask = np.sort(indices)
if self.mode.startswith('aniso'):
matrix = np.random.uniform(
low=self.low, high=self.high,
size=(self.out_features, self.num_samples)
)
else:
matrix = np.random.uniform(
low=self.low, high=self.high, size=(self.out_features, 1)
)
matrix = np.repeat(matrix, self.num_samples, axis=1)
y = x.copy()
if self.mode.startswith('aniso'):
for index, transform_id in enumerate(mask):
rotation_mat = self._build_rotation_matrix(matrix[:, index])
y[:, transform_id] = np.dot(rotation_mat, y[:, transform_id])
else:
rotation_mat = self._build_rotation_matrix(matrix[:, 0])
y[:, mask] = np.dot(rotation_mat, y[:, mask])
mask = np.repeat(
np.expand_dims(mask, axis=0), self.out_features, axis=0
)
return y, matrix, mask
def __init__(self, startImage, x,y, sx, imageBounds, pixelSize):
self.im = startImage
self.x = x
self.y = y
self.sx = sx
self.imageBounds = imageBounds
self.pixelSize = pixelSize
self.fuzz = 3*np.median(sx)
self.roiSize = int(self.fuzz/pixelSize)
self.fuzz = pixelSize*(self.roiSize)
self.mask = np.ones(self.im.shape)
#print imageBounds.x0
#print imageBounds.x1
#print fuzz
#print roiSize
#print pixelSize
self.X = np.arange(imageBounds.x0,imageBounds.x1, pixelSize)
self.Y = np.arange(imageBounds.y0,imageBounds.y1, pixelSize)
#print X
self.ctval = 1e-4
#record our image resolution so we can plot pts with a minimum size equal to res (to avoid missing small pts)
self.delX = np.absolute(self.X[1] - self.X[0])
self.kdt = ckdtree.cKDTree(np.array([self.x,self.y]).T)
def __init__(self, startImage, x, y, sx, imageBounds, pixelSize):
self.im = startImage
self.x = x
self.y = y
self.sx = sx
self.imageBounds = imageBounds
self.pixelSize = pixelSize
self.fuzz = 3 * np.median(sx)
self.roiSize = int(self.fuzz / pixelSize)
self.fuzz = pixelSize * (self.roiSize)
self.mask = np.ones(self.im.shape)
#print imageBounds.x0
#print imageBounds.x1
#print fuzz
#print roiSize
#print pixelSize
self.X = np.arange(imageBounds.x0, imageBounds.x1, pixelSize)
self.Y = np.arange(imageBounds.y0, imageBounds.y1, pixelSize)
#print X
self.ctval = 1e-4
#record our image resolution so we can plot pts with a minimum size equal to res (to avoid missing small pts)
self.delX = np.absolute(self.X[1] - self.X[0])
self.kdt = ckdtree.cKDTree(np.array([self.x, self.y]).T)
def segments_adding(M: int, active_vertices, active_segments,
segments_vertices, segments, d, epsilon, l, N,
shapeAndDistribution):
recalibrate = False
# shuffleB = False
for m in range(M):
for i in range(len(active_segments) - 1, -1, -1):
recalibrate, shuffleB, modified_vertices, i = segment_adding(
i, active_vertices, active_segments, segments_vertices,
segments, d, recalibrate)
if shuffleB or recalibrate:
tree = ckdtree.cKDTree(segments)
if recalibrate:
active_segments, segments_vertices, N = find_segments_additive(
tree, active_vertices, modified_vertices, segments, d,
epsilon, l, N, active_segments, segments_vertices,
shapeAndDistribution)
recalibrate = False
break
if shuffleB:
active_segments, segments_vertices = shuffle(
tree, i, active_vertices, segments, d, epsilon, l, N,
active_segments, segments_vertices)
shuffleB = False
break
return active_segments, segments_vertices, N
def Ptree(vals, errs, truevals, treeunit):
out = []
lencheck = 0
#build truth tree
truetree = ckdtree.cKDTree(truevals / treeunit)
for val, err in zip(vals, errs):
#get nearest neighbors within query_radius to target
inear = truetree.query_ball_point(val / treeunit, r=query_radius)
factor = 1.
if len(inear) > 20000:
lencheck += 1
factor = len(inear) / 20000.
inear = random.sample(inear, 20000)
#data of nearest neighbors
truearr = truetree.data[inear] * treeunit
Ls = likelihoods(val, err, truearr)
#sum likelihoods
out.append(np.sum(Ls) * factor)
if lencheck > 0:
print "Warning: ball query returned >20k points for {} targets ({}%).".format(
lencheck,
float(lencheck) / len(vals) * 100.)
return np.array(out)
def brute_force(data, r_step):
tree = kd.cKDTree(data)
groups_list = []
for particle in data:
if particle is not None:
group = []
indexes = tree.query_ball_point(particle, r_step) # returns the indexes in data of particles within r_step from the initial particle
if indexes.__len__() == 1:
group.append(data[indexes[0]])
data[indexes[0]] = None
else:
del indexes[0]
for x in indexes:
extra_indexes = tree.query_ball_point(data[x], r_step)
if extra_indexes.__len__() > 1:
for k in extra_indexes:
if not indexes.__contains__(k):
indexes.append(k)
for x in indexes: # excluding the particle counting itself
group.append(data[x]) # adding each particle to this group
data[x] = None # removing particle from initial data list
groups_list.append(group)
count = 0
for x in groups_list:
if x.__len__() >= 10:
count += 1
print "There are", count, "halos with 10 or more particles"
return groups_list
def get_novelty(self, vector_space, genome_vector):
"""Return the novelty for a single behavior vector.
:param vector_space: :type list: A list of lists representing the whole behavior space.
:param genome_vector: :type list: The behavior vector of the genome to be evaluated.
:return: Novelty score
"""
novelty = 0
# Get space of generation + novelty archive
vector_space = np.concatenate((vector_space, self.novelty_archive))
# Get distances to k-nearest points in behavior space
tree = kd.cKDTree(vector_space)
indexes = tree.query(genome_vector, 20)
distances = indexes[0]
for i in distances:
novelty += i/self.dsize
novelty /= 20
# Add to novelty archive with 1% probability
rand_n = random.random()
if rand_n >= .99:
k = 0
for element in genome_vector:
self.novelty_archive[self.archive_size][k] = element
k += 1
self.archive_size += 1
return novelty
def __Debounce(self, xs, ys, radius=4):
if len(xs) < 2:
return xs, ys
kdt = ckdtree.cKDTree(numpy.array([xs,ys]).T)
xsd = []
ysd = []
for xi, yi in zip(xs, ys):
#neigh = kdt.query_ball_point([xi,yi], radius)
dn, neigh = kdt.query(numpy.array([xi,yi]), 5)
neigh = neigh[dn < radius]
if len(neigh) > 1:
Ii = self.filteredData[xi,yi]
In = self.filteredData[xs[neigh].astype('i'),ys[neigh].astype('i')].max()
if not Ii < In:
xsd.append(xi)
ysd.append(yi)
else:
xsd.append(xi)
ysd.append(yi)
return xsd, ysd
def tag_sites(sitesym, scaled_positions, symprec=1e-3):
scaled = np.around(np.array(scaled_positions, ndmin=2), 8)
scaled %= 1.0
scaled %= 1.0
np.set_printoptions(suppress=True)
tags = -np.ones((len(scaled), ), dtype=int)
tagdis = 100 * np.ones((len(scaled), ), dtype=float)
rot, trans = spg.spacegroup.parse_sitesym(sitesym)
siteskdTree = cKDTree(scaled)
for i in range(len(scaled)):
if tags[i] == -1:
curpos = scaled[i]
sympos = np.dot(rot, curpos) + trans
sympos %= 1.0
sympos %= 1.0
sympos = np.unique(np.around(sympos, 8), axis=0)
min_dis, min_ids = siteskdTree.query(sympos, k=1)
# print(i,len(min_dis))
# print(min_dis)
select = min_dis <= symprec
select_ids = min_ids[select]
tags[select_ids] = i
tagdis[select_ids] = min_dis[select]
return tags, tagdis
def nearest_neighbours_connected_graph(virtual_positions, k):
kdtree = cKDTree(virtual_positions)
dist, idx = kdtree.query(virtual_positions, k=k + 1)
receivers = idx[:, 1:] # N,k
senders = np.arange(virtual_positions.shape[0]) # N
senders = np.tile(senders[:, None], [1, k]) # N,k
receivers = receivers.flatten()
senders = senders.flatten()
graph_nodes = tf.convert_to_tensor(virtual_positions, tf.float32)
graph_nodes.set_shape([None, 3])
receivers = tf.convert_to_tensor(receivers, tf.int32)
receivers.set_shape([None])
senders = tf.convert_to_tensor(senders, tf.int32)
senders.set_shape([None])
n_node = tf.shape(graph_nodes)[0:1]
n_edge = tf.shape(senders)[0:1]
graph_data_dict = dict(nodes=graph_nodes,
edges=tf.zeros((n_edge[0], 1)),
globals=tf.zeros([1]),
receivers=receivers,
senders=senders,
n_node=n_node,
n_edge=n_edge)
return GraphsTuple(**graph_data_dict)
def run(self):
if not self.copyData():
return
group_name = self.get_property('Group Name')
distance_name = self.get_property('Distance Name')
if not group_name or not distance_name:
return
search_position = self.get_property('Search Position')
max_count = self.get_property('Search Count')
max_dist = self.get_property('Search Radius')
if max_dist <= 0:
max_dist = None
points = self.geo.getVertexes()
tree = ckdtree.cKDTree(points)
dist, idx = tree.query(search_position,
max_count,
distance_upper_bound=max_dist)
if len(idx) == 0:
return
npoints = self.geo.getNumVertexes()
self.geo.createGroup('vertex', group_name)
self.geo.createAttribute('vertex', distance_name, "float", -1.0, True)
if not self.geo.hasAttribute('vertex', 'color'):
self.geo.createAttribute('vertex', 'color', "vector3",
[1.0, 1.0, 1.0], True)
for i, d in zip(idx, dist):
if i >= npoints:
continue
self.geo.setVertexAttrib(distance_name, i, d)
self.geo.setGroup('vertex', group_name, i, True)
self.geo.setVertexAttrib("color", i, [1.0, 0, 0])
def __init__(self, file_path = "", var_name = "",
bathymetry_path = "/skynet3_rech1/huziy/NEMO_OFFICIAL/dev_v3_4_STABLE_2012/NEMOGCM/CONFIG/GLK_LIM3_Michigan/EXP00/bathy_meter.nc"):
"""
:param file_path:
:param var_name:
:param bathymetry_path: used to mask land points
"""
self.current_time_frame = -1
self.var_name = var_name
self.cube = iris.load_cube(file_path, constraint=iris.Constraint(cube_func=lambda c: c.var_name == var_name))
self.lons, self.lats = cartography.get_xy_grids(self.cube)
lons2d_gl, lats2d_gl = nemo_commons.get_2d_lons_lats_from_nemo(path=bathymetry_path)
mask_gl = nemo_commons.get_mask(path=bathymetry_path)
xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons2d_gl.flatten(), lats2d_gl.flatten())
xt, yt, zt = lat_lon.lon_lat_to_cartesian(self.lons.flatten(), self.lats.flatten())
tree = cKDTree(list(zip(xs, ys, zs)))
dists, indices = tree.query(list(zip(xt, yt, zt)))
self.mask = mask_gl.flatten()[indices].reshape(self.lons.shape)
self.nt = self.cube.shape[0]
assert isinstance(self.cube, Cube)
print(self.nt)
def __Debounce(self, xs, ys, radius=4):
if len(xs) < 2:
return xs, ys
kdt = ckdtree.cKDTree(numpy.array([xs,ys]).T)
xsd = []
ysd = []
visited = 0*xs
for i in xrange(len(xs)):
if not visited[i]:
xi = xs[i]
yi = ys[i]
#neigh = kdt.query_ball_point([xi,yi], radius)
dn, neigh = kdt.query(numpy.array([xi,yi]), 5)
neigh = neigh[dn < radius]
if len(neigh) > 1:
In = self.filteredData[xs[neigh].astype('i'),ys[neigh].astype('i')]
mi = In.argmax()
xsd.append(xs[neigh[mi]])
ysd.append(ys[neigh[mi]])
visited[neigh] = 1
else:
xsd.append(xi)
ysd.append(yi)
return xsd, ysd
def nearest_neighbours(points, coords, points_required=1, max_distance=250.):
"""
An implementation of nearest neaighbour for spatial data that uses kdtrees
:param points: array of points to find the nearest neighbour for
:param coords: coordinates of points
:param points_required: number of points to return
:param max_distance: maximum search radius
:return:
"""
if len(np.array(points).shape) == 1:
points = np.array([points])
# Initialise tree instance
kdtree = cKDTree(data=coords)
# iterate throught the points and find the nearest neighbour
distances, indices = kdtree.query(points,
k=points_required,
distance_upper_bound=max_distance)
# Mask out infitnite distances in indices to avoid confusion
mask = np.isfinite(distances)
if not np.all(mask):
distances[~mask] = np.nan
return distances, indices
def dilate_mask(mask, mask_shape, vox_size, radius):
"""
Dilate the foreground in a binary mask according to a radius (in mm)
"""
is_to_dilate = mask == 1
is_background = mask == 0
# Get the list of indices
background_pos = np.argwhere(is_background) * vox_size
label_pos = np.argwhere(is_to_dilate) * vox_size
ckd_tree = cKDTree(label_pos)
# Compute the nearest labels for each voxel of the background
dist, indices = ckd_tree.query(background_pos,
k=1,
distance_upper_bound=radius,
n_jobs=-1)
# Associate indices to the nearest label (in distance)
valid_nearest = np.squeeze(np.isfinite(dist))
id_background = np.flatnonzero(is_background)[valid_nearest]
id_label = np.flatnonzero(is_to_dilate)[indices[valid_nearest]]
# Change values of those background
mask = mask.flatten()
mask[id_background.T] = mask[id_label.T]
mask = mask.reshape(mask_shape)
return mask
def find_potential_neighbours(x, k=100, distance_upper_bound=np.inf):
tree = cKDTree(x)
d, idx = tree.query(x,
k + 1,
distance_upper_bound=distance_upper_bound,
n_jobs=-1)
return d[:, 1:], idx[:, 1:]
def plot_difference(basemap,
lons1, lats1, data1, label1,
lons2, lats2, data2, label2,
base_folder="/skynet3_rech1/huziy/veg_fractions/"
):
xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons2.flatten(), lats2.flatten())
xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons1.flatten(), lats1.flatten())
ktree = cKDTree(list(zip(xs, ys, zs)))
dists, inds = ktree.query(list(zip(xt, yt, zt)))
# Calculate differences
diff_dict = {}
for key, the_field in data2.items():
diff_dict[key] = the_field.flatten()[inds].reshape(data1[key].shape) - data1[key]
x, y = basemap(lons1, lats1)
imname = "sand_clay_diff_{0}-{1}.jpeg".format(label2, label1)
impath = os.path.join(base_folder, imname)
plot_sand_and_clay_diff(x, y, basemap, diff_dict["SAND"], diff_dict["CLAY"],
out_image=impath)
del diff_dict["SAND"], diff_dict["CLAY"]
imname = "veg_fract_diff_{0}-{1}.jpeg".format(label2, label1)
impath = os.path.join(base_folder, imname)
plot_veg_fractions_diff(x, y, basemap, diff_dict,
out_image=impath)
def __init__(self, model):
self.src_crs = ccrs.LambertCylindrical()
self.tgt_crs = ccrs.Mercator(central_longitude=-87.200012,
min_latitude=18.091648,
max_latitude=31.960648)
self.file = glob.glob(model.data_dir + '/*')[0]
data = Dataset(self.file)
self.lats = data['Latitude'][:]
self.lons = data['Longitude'][:]
self.depths = data['Depth'][:]
self.lons, self.lats = np.meshgrid(self.lons, self.lats)
transformed = self.tgt_crs.transform_points(self.src_crs, self.lons,
self.lats)
self.lons = self.lons[0]
self.lats = self.lats.T[0]
self.x = transformed[0, :, 0]
self.y = transformed[:, 0, 1]
x, y = np.meshgrid(self.x, self.y)
self.points = np.array([self.x, self.y], dtype=object)
# VisibleDeprecationWarning: Creating an ndarray from ragged nested
# sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays
# with different lengths or shapes) is deprecated. If you meant to do
# this, you must specify 'dtype=object' when creating the ndarray
coords = np.array([x.ravel(), y.ravel()]).T
self.tree = cKDTree(coords, leafsize=model.leafsize)
def find_segments(vertices, segments, d, epsilon, l, N, shapeAndDistribution):
active_segments = [] #segmenty które będą się rozrastać
segments_vertices = [
] #krawędzie do których będzie rozrastać się segment na odpowiednim miejscu powyżej
tree = ckdtree.cKDTree(segments)
for i in range(len(vertices) - 1, -1, -1):
dist, nearest_segment = tree.query(vertices[i])
if dist < d:
if not any(i in lista for lista in segments_vertices):
vertices.pop(i)
if N != 0:
new_vertex = random_vertex(l, shapeAndDistribution)
vertices.append(new_vertex)
N -= 1
for i in range(len(vertices)):
find_segment(
i,
tree,
d,
segments,
vertices,
active_segments,
segments_vertices,
)
return active_segments, segments_vertices, N
def __Debounce(self, xs, ys, radius=4):
if len(xs) < 2:
return xs, ys
kdt = ckdtree.cKDTree(np.array([xs, ys]).T)
xsd = []
ysd = []
visited = 0 * xs
for i in xrange(len(xs)):
if not visited[i]:
xi = xs[i]
yi = ys[i]
#neigh = kdt.query_ball_point([xi,yi], radius)
dn, neigh = kdt.query(np.array([xi, yi]), 5)
neigh = neigh[dn < radius]
if len(neigh) > 1:
In = self.filteredData[xs[neigh].astype('i'),
ys[neigh].astype('i')]
mi = In.argmax()
xsd.append(xs[neigh[mi]])
ysd.append(ys[neigh[mi]])
visited[neigh] = 1
else:
xsd.append(xi)
ysd.append(yi)
return xsd, ysd
def gethlr(dfull, sdfull):
#linear fit using just flux radii
#(dfull['FLUX_RADIUS_I'] - 1.7) / 2.3
#use average star flux_radius_i around galaxies
#& galaxy flux_radius from balrog
to_grid = fits.open(
'/Users/Christina/DES/data/balrog/sva1/balrog_tab01_avg_star_fluxradiusi_0.1deg.fits'
)[1].data
#make tree of dfull stars
sys.stderr.write("stars in dfull: {}\n".format(len(sdfull)))
sys.stderr.write("galaxies in dfull: {}\n".format(len(dfull)))
sys.stderr.write('creating hlr tree...\n')
star_tree = ckdtree.cKDTree(zip(sdfull['RA'], sdfull['DEC']))
gal_pos = zip(dfull['RA'], dfull['DEC'])
#360 arcsec
close = star_tree.query_ball_point(gal_pos, r=0.1)
sys.stderr.write(' calculating average star radii...\n')
start = time.time()
dfull_avg_star_fr = np.array(
[np.median(sdfull['FLUX_RADIUS_I'][c]) for c in close])
end = time.time()
dfull_hlr = griddata(
zip(to_grid['flux_radius_i'], to_grid['avg_flux_radius_i']),
to_grid['hlr'], zip(dfull['FLUX_RADIUS_I'], dfull_avg_star_fr))
sys.stderr.write(' done\n')
return dfull_hlr
def get_data_from_file_interpolate_if_needed(self, the_path):
the_path = str(the_path)
if the_path.lower()[-3:] in ["cis", "nic"]:
data = self._parse_nic_cis_data_file(the_path)
else:
data = self.get_data_from_path(the_path)
if data.shape != (self.ncols_target, self.ncols_target):
# The interpolation is needed
domain_descr = data.shape
if domain_descr not in self.domain_descr_to_kdtree:
if domain_descr == (516, 510):
self.lons2d_other = np.flipud(np.loadtxt(self.path_to_other_lons)).transpose()
self.lats2d_other = np.flipud(np.loadtxt(self.path_to_other_lats)).transpose()
else:
self.lons2d_other, self.lats2d_other = self._generate_grid_from_descriptor(domain_descr)
xs, ys, zs = lat_lon.lon_lat_to_cartesian(self.lons2d_other.flatten(), self.lats2d_other.flatten())
kdtree_other = cKDTree(data=list(zip(xs, ys, zs)))
self.domain_descr_to_kdtree[data.shape] = kdtree_other
kdtree_other = self.domain_descr_to_kdtree[data.shape]
xt, yt, zt = lat_lon.lon_lat_to_cartesian(self.lons2d_target.flatten(), self.lats2d_target.flatten())
dsts, inds = kdtree_other.query(list(zip(xt, yt, zt)))
return data.flatten()[inds].reshape(self.lons2d_target.shape)
else:
return data
def __init__(self,
edges,
node_map,
memory=None,
algorithm=bidirectional_dijkstra):
"""
:param edges: [DataFrame]
:param node_map: [DataFrame]
:param memory [joblib.Memory]
:param algorithm: [function]
:return:
"""
self.node_map = node_map
self._btree = ckdtree.cKDTree(
np.array(list(zip(self.node_map.x, self.node_map.y))))
self.edges = edges
self.memory = memory
self.algorithm = algorithm
self.routes = {}
self.Graph = nx.Graph()
if self.memory is not None:
self.get_route = self.memory.cache(self.get_route, ignore=['self'])
LOGGER.debug('loading routing graph')
_ = self.edges.apply(lambda x: self.Graph.add_edge(**x), axis=1)
def __Debounce(self, xs, ys, radius=4):
if len(xs) < 2:
return xs, ys
kdt = ckdtree.cKDTree(numpy.array([xs, ys]).T)
xsd = []
ysd = []
for xi, yi in zip(xs, ys):
#neigh = kdt.query_ball_point([xi,yi], radius)
dn, neigh = kdt.query(numpy.array([xi, yi]), 5)
neigh = neigh[dn < radius]
if len(neigh) > 1:
Ii = self.filteredData[xi, yi]
In = self.filteredData[xs[neigh].astype('i'),
ys[neigh].astype('i')].max()
if not Ii < In:
xsd.append(xi)
ysd.append(yi)
else:
xsd.append(xi)
ysd.append(yi)
return xsd, ysd
def covAnnulus(expotab, rMin=0.005, rMax=0.02, maxErr=70.):
'''Calculate the mean <dx dx>, <dx dy>, and <dy dy> for
pairs within a circular annulus of separation.
Exclude individual points with meas errors above maxErr.
Returns cxx, cyy, cxy, npairs where last is number of
object pairs used in calculation.
'''
# Extract data
use = expotab['measErr'] <= maxErr
xy = np.vstack((expotab['u'][use], expotab['v'][use])).transpose()
dx = expotab['dx'][use]
dy = expotab['dy'][use]
# Calculate correlations
kdt = cKDTree(xy)
prsmax = kdt.query_pairs(rMax, output_type='ndarray')
prsmin = kdt.query_pairs(rMin, output_type='ndarray')
nprs = prsmax.shape[0] - prsmin.shape[0]
xx = np.sum(dx[prsmax[:,0]]*dx[prsmax[:,1]]) \
-np.sum(dx[prsmin[:,0]]*dx[prsmin[:,1]])
yy = np.sum(dy[prsmax[:,0]]*dy[prsmax[:,1]]) \
-np.sum(dy[prsmin[:,0]]*dy[prsmin[:,1]])
xy = np.sum(dx[prsmax[:,0]]*dy[prsmax[:,1]]) \
-np.sum(dx[prsmin[:,0]]*dy[prsmin[:,1]])\
+np.sum(dy[prsmax[:,0]]*dx[prsmax[:,1]])\
-np.sum(dy[prsmin[:,0]]*dx[prsmin[:,1]])
xx /= nprs
yy /= nprs
xy /= 2 * nprs
return xx, yy, xy, nprs
def writeQSOcat(file, dmin=0.3, Lbox = 1040.): t0 = time.time() print file x, y, z, vx, vy, vz, df = readGAL(file) catalog = n.transpose([x, y, z, vx, vy, vz*0.92, df]) # identifies pairs treeD=t.cKDTree(n.transpose([x, y, z]), 1000.0) allpairs = treeD.query_ball_point(treeD.data, dmin) # removes auto-pairs pairID = [] for ii, el in enumerate(allpairs): if len(el)>1 and ii == el[0]: pairID.append(el) to_rm = [] ct = 0 for id2chooseFrom in pairID: n.random.shuffle(id2chooseFrom) #keep = id2chooseFrom[0] reject = id2chooseFrom[1:] to_rm.append(reject) ct+=1 if ct>0: to_delete = n.hstack((to_rm)) # list of ids rejected print "2del", len(to_delete) cat2save = n.delete(catalog, to_delete, axis=0) n.savetxt( join(dir, "QSO"+os.path.basename(file)[9:]), cat2save, fmt='%10.5f', header=' x y z vx vy vz df ' ) print "dt=",time.time()-t0 else: print "no del" n.savetxt( join(dir, "QSO"+os.path.basename(file)[9:]), catalog, fmt='%10.5f', header=' x y z vx vy vz df ' ) print "dt=",time.time()-t0
def _init_fields(self, nc_dataset):
nc_vars = nc_dataset.variables
lons = nc_vars["lon"][:]
lats = nc_vars["lat"][:]
if lons.ndim == 1:
lats2d, lons2d = np.meshgrid(lats, lons)
elif lons.ndim == 2:
lats2d, lons2d = lats, lons
else:
raise NotImplementedError("Cannot handle {}-dimensional coordinates".format(lons.ndim))
self.lons2d, self.lats2d = lons2d, lats2d
self.times_var = nc_vars["time"]
self.times_num = nc_vars["time"][:]
if hasattr(self.times_var, "calendar"):
self.times = num2date(self.times_num, self.times_var.units, self.times_var.calendar)
else:
self.times = num2date(self.times_num, self.times_var.units)
if not self.lazy:
self.var_data = nc_vars[self.var_name][:]
if nc_vars[self.var_name].shape[1:] != self.lons2d.shape:
print("nc_vars[self.var_name].shape = {}".format(nc_vars[self.var_name].shape))
self.var_data = np.transpose(self.var_data, axes=[0, 2, 1])
x_in, y_in, z_in = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten())
self.kdtree = cKDTree(list(zip(x_in, y_in, z_in)))
def _fit(self, X):
if isinstance(X, NeighborsBase):
self._fit_X = X._fit_X
self._tree = X._tree
self._fit_method = X._fit_method
return self
elif isinstance(X, BallTree):
self._fit_X = X.data
self._tree = X
self._fit_method = 'ball_tree'
return self
elif isinstance(X, cKDTree):
self._fit_X = X.data
self._tree = X
self._fit_method = 'kd_tree'
return self
X = safe_asarray(X)
if X.ndim != 2:
raise ValueError("data type not understood")
n_samples = X.shape[0]
if n_samples == 0:
raise ValueError("n_samples must be greater than 0")
if issparse(X):
if self.algorithm not in ('auto', 'brute'):
warnings.warn("cannot use tree with sparse input: "
"using brute force")
self._fit_X = X.tocsr()
self._tree = None
self._fit_method = 'brute'
return self
self._fit_method = self.algorithm
self._fit_X = X
if self._fit_method == 'auto':
# BallTree outperforms the others in nearly any circumstance.
if self.n_neighbors is None:
self._fit_method = 'ball_tree'
elif self.n_neighbors < self._fit_X.shape[0] // 2:
self._fit_method = 'ball_tree'
else:
self._fit_method = 'brute'
if self._fit_method == 'kd_tree':
self._tree = cKDTree(X, self.leaf_size)
elif self._fit_method == 'ball_tree':
self._tree = BallTree(X, self.leaf_size, p=self.p)
elif self._fit_method == 'brute':
self._tree = None
else:
raise ValueError("algorithm = '%s' not recognized"
% self.algorithm)
return self
def main(dfs_var_name="t2", cru_var_name="tmp",
dfs_folder="/home/huziy/skynet3_rech1/NEMO_OFFICIAL/DFS5.2_interpolated",
cru_file = "data/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.tmp.dat.nc"):
if not os.path.isdir(NEMO_IMAGES_DIR):
os.mkdir(NEMO_IMAGES_DIR)
#year range is inclusive [start_year, end_year]
start_year = 1981
end_year = 2009
season_name_to_months = OrderedDict([
("Winter", (1, 2, 12)),
("Spring", list(range(3, 6))),
("Summer", list(range(6, 9))),
("Fall", list(range(9, 12)))])
cru_t_manager = CRUDataManager(var_name=cru_var_name, path=cru_file)
cru_lons, cru_lats = cru_t_manager.lons2d, cru_t_manager.lats2d
#get seasonal means (CRU)
season_to_mean_cru = cru_t_manager.get_seasonal_means(season_name_to_months=season_name_to_months,
start_year=start_year,
end_year=end_year)
#get seasonal means Drakkar
dfs_manager = DFSDataManager(folder_path=dfs_folder, var_name=dfs_var_name)
season_to_mean_dfs = dfs_manager.get_seasonal_means(season_name_to_months=season_name_to_months,
start_year=start_year,
end_year=end_year)
dfs_lons, dfs_lats = dfs_manager.get_lons_and_lats_2d()
xt, yt, zt = lat_lon.lon_lat_to_cartesian(dfs_lons.flatten(), dfs_lats.flatten())
xs, ys, zs = lat_lon.lon_lat_to_cartesian(cru_lons.flatten(), cru_lats.flatten())
ktree = cKDTree(data=list(zip(xs, ys, zs)))
dists, inds = ktree.query(list(zip(xt, yt, zt)))
season_to_err = OrderedDict()
for k in season_to_mean_dfs:
interpolated_cru = season_to_mean_cru[k].flatten()[inds].reshape(dfs_lons.shape)
if dfs_var_name.lower() == "t2":
#interpolated_cru += 273.15
season_to_mean_dfs[k] -= 273.15
elif dfs_var_name.lower() == "precip": # precipitation in mm/day
season_to_mean_dfs[k] *= 24 * 60 * 60
season_to_err[k] = season_to_mean_dfs[k] #- interpolated_cru
season_indicator = "-".join(sorted(season_to_err.keys()))
fig_path = os.path.join(NEMO_IMAGES_DIR, "{3}_errors_{0}-{1}_{2}_dfs.jpeg".format(start_year,
end_year,
season_indicator,
dfs_var_name))
basemap = nemo_commons.get_default_basemap_for_glk(dfs_lons, dfs_lats, resolution="l")
x, y = basemap(dfs_lons, dfs_lats)
coords_and_basemap = {
"basemap": basemap, "x": x, "y": y
}
plot_errors_in_one_figure(season_to_err, fig_path=fig_path, **coords_and_basemap)
def read_and_interpolate_homa_data(self, path="", start_year=None, end_year=None, season_to_months=None):
"""
:param path:
:param target_cube:
"""
import pandas as pd
ds = Dataset(path)
sst = ds.variables["sst"][:]
# read longitudes and latitudes from a file
lons_source = ds.variables["lon"][:]
lats_source = ds.variables["lat"][:]
month_to_season = defaultdict(lambda: "no-season")
for seas, mths in season_to_months.items():
for m in mths:
month_to_season[m] = seas
# time variable
time_var = ds.variables["time"]
dates = num2date(time_var[:], time_var.units)
if hasattr(sst, "mask"):
sst[sst.mask] = np.nan
panel = pd.Panel(data=sst, items=dates, major_axis=range(sst.shape[1]), minor_axis=range(sst.shape[2]))
seasonal_sst = panel.groupby(
lambda d: (d.year, month_to_season[d.month]), axis="items").mean()
# source grid
xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons_source.flatten(), lats_source.flatten())
kdtree = cKDTree(data=list(zip(xs, ys, zs)))
# target grid
xt, yt, zt = lat_lon.lon_lat_to_cartesian(self.lons.flatten(), self.lats.flatten())
dists, inds = kdtree.query(list(zip(xt, yt, zt)))
assert isinstance(seasonal_sst, pd.Panel)
result = {}
for the_year in range(start_year, end_year + 1):
result[the_year] = {}
for the_season in list(season_to_months.keys()):
the_mean = seasonal_sst.select(lambda item: item == (the_year, the_season), axis="items")
result[the_year][the_season] = the_mean.values.flatten()[inds].reshape(self.lons.shape)
return result
def _fit(self, X):
if isinstance(X, NeighborsBase):
self._fit_X = X._fit_X
self._tree = X._tree
self._fit_method = X._fit_method
return self
elif isinstance(X, BallTree):
self._fit_X = X.data
self._tree = X
self._fit_method = 'ball_tree'
return self
elif isinstance(X, cKDTree):
self._fit_X = X.data
self._tree = X
self._fit_method = 'kd_tree'
return self
X = safe_asarray(X)
if X.ndim != 2:
raise ValueError("data type not understood")
n_samples = X.shape[0]
if n_samples == 0:
raise ValueError("n_samples must be greater than 0")
if issparse(X):
if self.algorithm not in ('auto', 'brute'):
warnings.warn("cannot use tree with sparse input: "
"using brute force")
self._fit_X = X.tocsr()
self._tree = None
self._fit_method = 'brute'
return self
self._fit_method = self.algorithm
self._fit_X = X
if self._fit_method == 'auto':
# BallTree outperforms the others in nearly any circumstance.
if self.n_neighbors is None:
self._fit_method = 'ball_tree'
elif self.n_neighbors < self._fit_X.shape[0] // 2:
self._fit_method = 'ball_tree'
else:
self._fit_method = 'brute'
if self._fit_method == 'kd_tree':
self._tree = cKDTree(X, self.leaf_size)
elif self._fit_method == 'ball_tree':
self._tree = BallTree(X, self.leaf_size, p=self.p)
elif self._fit_method == 'brute':
self._tree = None
else:
raise ValueError("algorithm = '%s' not recognized" %
self.algorithm)
return self
def _graph_to_kdtree(self) -> None:
# add node_ids to coordinates to support overlapping nodes in cKDTree
data = [
tuple(node["location"]) + (node_id,)
for node_id, node in self.g.nodes.items()
]
# place nodes in the kdtree
self.data = cKDTree(np.array(list(data)))
def waypoints_cb(self, waypoints):
self.waypoints = waypoints
# setup k-d tree in order to find the closest path waypoint to the given position [get_closest_waypoint(position)]
# the code is copied from waypoints_cb(pose) in ros/src/waypoint_updater/waypoint_updater.py written by https://github.com/ysonggit
if not self.waypoints_2d:
self.waypoints_2d = [[waypoint.pose.pose.position.x, waypoint.pose.pose.position.y] for waypoint in waypoints.waypoints]
self.waypoint_tree = cKDTree(self.waypoints_2d)
def get_n_neighbours(df, cls, ds, r):
from scipy.spatial.ckdtree import cKDTree
global points, tree
points = np.array([df.x, df.y]).T
points -= np.min(points, axis=0, keepdims=True)
points /= np.max(points, axis=0, keepdims=True)
tree = cKDTree(points)
return pd.Series([*map(len, tree.query_ball_point(points, r))], index=df.index)
def computerecondensity(d3d, label, leafs=16, PIX=10, IMGMAX=40000):
"""
Compute Local Effective Resolution
:param d3d: 3D points
:param label:
:param leafs: Number of leafs (SNR = np.sqrt(leafs/2))
:param PIX: nm to pixel (e.g. 10nm per pixel = 100nm^2
:param IMGMAX: Upper limit of the image ROI
:return: The CKDTree of the points (with leaf size), the image array, the points in leafs, and the image array where im[pix_x, pix_y] = LER
"""
points = d3d[:, :2].copy()
mx, MX = np.min(points[:, 0]), np.max(points[:, 0])
my, MY = np.min(points[:, 1]), np.max(points[:, 1])
if mx < 0:
points[:, 0] += np.abs(mx)
print("Neg pos for {}".format(label))
if my < 0:
points[:, 1] += np.abs(my)
print("Neg pos for {}".format(label))
tr = cKDTree(points, leafsize=leafs)
lfs = []
root = tr.tree
leaves(root, lfs)
assert (MX < IMGMAX)
assert (MY < IMGMAX)
pixels = np.empty((len(lfs), 2), dtype=np.float64)
imgsize = int(np.round(IMGMAX / PIX))
imarray = np.zeros((imgsize, imgsize), dtype=np.float32)
for ip, p in enumerate(lfs):
ps = points[p]
# assert(ps.shape[0] <= leafs)
assert (ps.shape[1] == 2)
_mx = min(ps[:, 0])
_Mx = max(ps[:, 0])
_my = min(ps[:, 1])
_My = max(ps[:, 1])
_x, _y = topix(_mx, _my, PIX) # Lower left
_X, _Y = topix(_Mx, _My, PIX) # Upper right
# ELR = nr of points over area xy - XY
# imarray[_x:_X, _y:_Y] += ps.shape[0]
pixels[ip, 0] = ((_X - _x) + 1) * ((_Y - _y) + 1) # Superpixel area
pixels[ip, 1] = ps.shape[0] / pixels[ip, 0] # Locs / px^2
for pri in range(ps.shape[0]):
pti = ps[pri, :2]
xp, yp = topix(pti[0], pti[1], PIX)
assert (xp < IMGMAX)
assert (
yp < IMGMAX) # You'd assume this check is not nec. as an index error would follow, but you'd be wrong in interesting cases
try:
imarray[xp, yp] += pixels[ip, 1]
except OverflowError as e:
print("OE {} , {} <- {} {}".format(xp, yp, pixels[ip, 1], pixels[ip, 0]))
print("OE {} , {} <- {:.2f} {:.2f}".format(xp, yp, pti[0], pti[1]))
raise
return tr, imarray, pixels
def _graph_to_kdtree(self) -> None:
# add node_ids to coordinates to support overlapping nodes in cKDTree
ids, attrs = zip(*self.g.nodes.items())
assert all([a == b for a, b in zip(ids, range(len(ids)))
]), "ids are not properly sorted"
data = [point_attrs["location"].tolist() for point_attrs in attrs]
logger.debug("placing {} nodes in the kdtree".format(len(data)))
self.data = cKDTree(np.array(list(data)))
logger.debug("kdtree initialized".format(len(data)))
def get_dataless_model_points_for_stations(station_list,
accumulation_area_km2_2d,
model_lons2d, model_lats2d, i_array,
j_array):
"""
returns a map {station => modelpoint} for comparison modeled streamflows with observed
this uses exactly the same method for searching model points as one in diagnose_point (nc-version)
"""
lons = model_lons2d[i_array, j_array]
lats = model_lats2d[i_array, j_array]
model_acc_area_1d = accumulation_area_km2_2d[i_array, j_array]
npoints = 1
result = {}
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lons, lats)
kdtree = cKDTree(list(zip(x0, y0, z0)))
for s in station_list:
# list of model points which could represent the station
assert isinstance(s, Station)
x, y, z = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude)
dists, inds = kdtree.query((x, y, z), k=5)
if npoints == 1:
deltaDaMin = np.min(
np.abs(model_acc_area_1d[inds] - s.drainage_km2))
# this returns a list of numpy arrays
imin = np.where(
np.abs(model_acc_area_1d[inds] -
s.drainage_km2) == deltaDaMin)[0][0]
selected_cell_index = inds[imin]
# check if difference in drainage areas is not too big less than 10 %
print(s.river_name, deltaDaMin / s.drainage_km2)
# if deltaDaMin / s.drainage_km2 > 0.2:
# continue
mp = ModelPoint()
mp.accumulation_area = model_acc_area_1d[selected_cell_index]
mp.longitude = lons[selected_cell_index]
mp.latitude = lats[selected_cell_index]
mp.cell_index = selected_cell_index
mp.distance_to_station = dists[imin]
print("Distance to station: ", dists[imin])
print("Model accumulation area: ", mp.accumulation_area)
print("Obs accumulation area: ", s.drainage_km2)
result[s] = mp
else:
raise Exception("npoints = {0}, is not yet implemented ...")
return result
def get_dataless_model_points_for_stations(station_list, accumulation_area_km2_2d,
model_lons2d, model_lats2d,
i_array, j_array):
"""
returns a map {station => modelpoint} for comparison modeled streamflows with observed
this uses exactly the same method for searching model points as one in diagnose_point (nc-version)
"""
lons = model_lons2d[i_array, j_array]
lats = model_lats2d[i_array, j_array]
model_acc_area_1d = accumulation_area_km2_2d[i_array, j_array]
npoints = 1
result = {}
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lons, lats)
kdtree = cKDTree(list(zip(x0, y0, z0)))
for s in station_list:
# list of model points which could represent the station
assert isinstance(s, Station)
x, y, z = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude)
dists, inds = kdtree.query((x, y, z), k=5)
if npoints == 1:
deltaDaMin = np.min(np.abs(model_acc_area_1d[inds] - s.drainage_km2))
# this returns a list of numpy arrays
imin = np.where(np.abs(model_acc_area_1d[inds] - s.drainage_km2) == deltaDaMin)[0][0]
selected_cell_index = inds[imin]
# check if difference in drainage areas is not too big less than 10 %
print(s.river_name, deltaDaMin / s.drainage_km2)
# if deltaDaMin / s.drainage_km2 > 0.2:
# continue
mp = ModelPoint()
mp.accumulation_area = model_acc_area_1d[selected_cell_index]
mp.longitude = lons[selected_cell_index]
mp.latitude = lats[selected_cell_index]
mp.cell_index = selected_cell_index
mp.distance_to_station = dists[imin]
print("Distance to station: ", dists[imin])
print("Model accumulation area: ", mp.accumulation_area)
print("Obs accumulation area: ", s.drainage_km2)
result[s] = mp
else:
raise Exception("npoints = {0}, is not yet implemented ...")
return result
def generate_kdtree(folder_with_ini_files, filename_prefix):
lons, lats = [], []
paths = []
sel_files = [
f for f in os.listdir(folder_with_ini_files) if f.startswith(filename_prefix) and f.lower().endswith(".ini")
]
xll, yll = None, None
xur, yur = None, None
for fname in sel_files:
fpath = os.path.join(folder_with_ini_files, fname)
lon, lat = read_lon_lat_from_ini_file(fpath)
lons.append(lon)
lats.append(lat)
paths.append(fpath)
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(np.asarray(lons), np.asarray(lats))
return cKDTree(list(zip(x0, y0, z0))), paths
def nearest_match(cat0, cat1, tol=1.0, **kwargs):
"""Find nearest neighbors in two lists.
Parameters
----------
cat0, cat1 : arrays
Each catalog is a 2xN array of (y, x) positions.
tol : float, optional
The radial match tolerance in pixels.
**kwargs
Keyword arguments for `meanclip` when computing mean offsets.
Returns
-------
matches : dictionary
The best match for star `i` of `cat0` is `matches[i]` in `cat1`.
Stars that are matched multiple times, or not matched at all,
are not returned.
dyx : ndarray
Sigma-clipped mean offset. Clipping is done in x and y
separately, and only the union of the two is returned. The
order for `dxy` is undefined.
"""
from scipy.spatial.ckdtree import cKDTree
from .util import takefrom, meanclip
assert len(cat0) == 2
assert len(cat1) == 2
tree = cKDTree(cat0.T)
d, i = tree.query(cat1.T) # 0 of cat1 -> i[0] of cat0
matches = dict()
for k, j in enumerate(i):
if d[k] < tol:
matches[j] = k
dyx = cat0[:, list(matches.keys())] - cat1[:, list(matches.values())]
mcx = meanclip(dyx[1], full_output=True)
mcy = meanclip(dyx[0], full_output=True)
j = list(set(np.concatenate((mcx[2], mcy[2]))))
return matches, dyx[:, j]
def __init__(self, flow_dirs, nx=None, ny=None,
lons2d=None,
lats2d=None,
accumulation_area_km2=None):
self.cells = []
self.lons2d = lons2d
self.lats2d = lats2d
self.flow_directions = flow_dirs
self.accumulation_area_km2 = accumulation_area_km2
# calculate characteristic distance
if not any([None is arr for arr in [self.lats2d, self.lons2d]]):
v1 = lat_lon.lon_lat_to_cartesian(self.lons2d[0, 0], self.lats2d[0, 0])
v2 = lat_lon.lon_lat_to_cartesian(self.lons2d[1, 1], self.lats2d[1, 1])
dv = np.array(v2) - np.array(v1)
self.characteristic_distance = np.sqrt(np.dot(dv, dv))
x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten())
self.kdtree = cKDTree(list(zip(x, y, z)))
if None not in [nx, ny]:
self.nx = nx
self.ny = ny
else:
nx, ny = flow_dirs.shape
self.nx, self.ny = flow_dirs.shape
for i in range(nx):
self.cells.append(list([Cell(i=i, j=j, flow_dir_value=flow_dirs[i, j]) for j in range(ny)]))
self._without_next_mask = np.zeros((nx, ny), dtype=np.int)
self._wo_next_wo_prev_mask = np.zeros((nx, ny), dtype=np.int) # mask of the potential outlets
for i in range(nx):
if i % 100 == 0:
print("Created {}/{}".format(i, nx))
for j in range(ny):
i_next, j_next = direction_and_value.to_indices(i, j, flow_dirs[i][j])
next_cell = None
if 0 <= i_next < nx:
if 0 <= j_next < ny:
next_cell = self.cells[i_next][j_next]
self._without_next_mask[i, j] = int(next_cell is None)
self.cells[i][j].set_next(next_cell)
def ntree(vals, errs, truevals, knear=k_near):
truetree = ckdtree.cKDTree(truevals)
out = []
for val, err in zip(vals, errs):
dnear, inear = truetree.query(val, k=knear)
truearr = truetree.data[inear]
covI = 1./err**2.
A = 1./np.sqrt( (2.*np.pi)**len(val) * np.prod(err**2.) )
diff = val-truearr
B = -0.5 * np.sum(diff**2.*covI, axis=1)
C = A * np.exp(B)
out.append(np.sum(C))
return np.array(out)
def _read_lon_lats(self):
"""
Read the lons and lats, 2d from the first file
"""
the_date = datetime(1980, 1, 1)
fpath = os.path.join(self.folder_path, the_date.strftime(self.fname_format))
# Check if file exists before trying to read it
if not os.path.isfile(fpath):
raise IOError("No such file: {}".format(fpath))
ds = Dataset(fpath)
self.lons2d = ds.variables["lon"][:].transpose()
self.lats2d = ds.variables["lat"][:].transpose()
x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten())
self.kdtree = cKDTree(list(zip(x, y, z)))
ds.close()
def main():
name_constraint = iris.Constraint(cube_func=lambda c: c.var_name == "sosstsst")
data_cube = iris.load_cube(T_FILE_PATH, constraint=name_constraint)
assert isinstance(data_cube, iris.cube.Cube)
lons = data_cube.coord("longitude").points[:]
lats = data_cube.coord("latitude").points[:]
print(lons.shape, lats.shape)
x, y, z = lat_lon.lon_lat_to_cartesian(lons.flatten(), lats.flatten())
ktree = cKDTree(data=list(zip(x, y, z)))
for st in get_obs_data(data_folder="/home/huziy/skynet3_rech1/nemo_obs_for_validation/temperature_at_points_ts"):
st.compare_with_modelled(data_cube, img_folder=NEMO_IMAGES_DIR, ktree=ktree)
import matplotlib.pyplot as plt
plt.show()
def ptree(vals, errs, truevals, knear=k_near):
sigma = np.median(errs.T, axis=1)
truetree = ckdtree.cKDTree(truevals/sigma)
out = []
for val, err in zip(vals, errs):
dnear, inear = truetree.query(val/sigma, k=knear)
truearr = truetree.data[inear] * sigma
covI = 1./err**2.
A = 1./np.sqrt( (2.*np.pi)**len(val) * np.prod(err**2.) )
diff = val-truearr
B = -0.5 * np.sum(diff**2.*covI, axis=1)
C = A * np.exp(B)
out.append(np.sum(C) * float(len(truevals))/len(truearr))
return np.array(out)
def plot_depth_to_bedrock(basemap,
lons1, lats1, field1, label1,
lons2, lats2, field2, label2,
base_folder="/skynet3_rech1/huziy/veg_fractions/"
):
xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons2.flatten(), lats2.flatten())
xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons1.flatten(), lats1.flatten())
ktree = cKDTree(list(zip(xs, ys, zs)))
dists, inds = ktree.query(list(zip(xt, yt, zt)))
levels = np.arange(0, 5.5, 0.5)
field2_interp = field2.flatten()[inds].reshape(field1.shape)
field1 = np.ma.masked_where(field1 < 0, field1)
field2_interp = np.ma.masked_where(field2_interp < 0, field2_interp)
vmin = min(field1.min(), field2_interp.min())
vmax = max(field1.max(), field2_interp.max())
cmap = cm.get_cmap("BuPu", len(levels) - 1)
bn = BoundaryNorm(levels, len(levels) - 1)
x, y = basemap(lons1, lats1)
imname = "depth_to_bedrock_{0}-{1}.jpeg".format(label2, label1)
impath = os.path.join(base_folder, imname)
fig = plt.figure(figsize=(6, 2.5))
gs = GridSpec(1, 3, width_ratios=[1, 1, 0.05])
ax = fig.add_subplot(gs[0, 0])
basemap.pcolormesh(x, y, field1, vmin=vmin, vmax=vmax, cmap=cmap, norm=bn)
basemap.drawcoastlines(linewidth=common_plot_params.COASTLINE_WIDTH)
ax.set_title(label1)
ax = fig.add_subplot(gs[0, 1])
im = basemap.pcolormesh(x, y, field2_interp, vmin=vmin, vmax=vmax, cmap=cmap, norm=bn)
basemap.drawcoastlines(linewidth=common_plot_params.COASTLINE_WIDTH)
ax.set_title(label2)
plt.colorbar(im, cax=fig.add_subplot(gs[0, 2]))
fig.tight_layout()
fig.savefig(impath, dpi=common_plot_params.FIG_SAVE_DPI)
def _init_fields(self, nc_dataset):
nc_vars = nc_dataset.variables
lons = nc_vars["lon"][:]
lats = nc_vars["lat"][:]
lats2d, lons2d = np.meshgrid(lats, lons)
self.lons2d, self.lats2d = lons2d, lats2d
self.times_var = nc_vars["time"]
self.times_num = nc_vars["time"][:]
if hasattr(self.times_var, "calendar"):
self.times = num2date(self.times_num, self.times_var.units, self.times_var.calendar)
else:
self.times = num2date(self.times_num, self.times_var.units)
if not self.lazy:
self.var_data = np.transpose(nc_vars[self.var_name][:], axes=[0, 2, 1])
x_in, y_in, z_in = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten())
self.kdtree = cKDTree(list(zip(x_in, y_in, z_in)))
def __discardClumped(self, xs, ys, radius=4):
if len(xs) < 2:
return xs, ys
kdt = ckdtree.cKDTree(numpy.array([xs,ys]).T)
xsd = []
ysd = []
for i in xrange(len(xs)):
xi = xs[i]
yi = ys[i]
#neigh = kdt.query_ball_point([xi,yi], radius)
dn, neigh = kdt.query(numpy.array([xi,yi]), 2)
print dn
if (dn[1] > radius):
xsd.append(xi)
ysd.append(yi)
print len(xsd)
return numpy.array(xsd), numpy.array(ysd)
def spatial_match(cat0, cat1, tol=0.01, min_score=0, full_output=False,
verbose=True):
"""Find spatially matching sourse between two lists.
Parameters
----------
cat0, cat1 : arrays
Each catalog is an 2xN array of (y, x) positions.
tol : float, optional
The match tolerance.
min_score : float, optional
Only return matches with scores greater than `min_score`.
full_output : bool, optional
Set to `True` to also return `match_matrix`.
verbose : bool, optional
Print some feedback for the user.
Returns
-------
matches : dictionary
The best match for star `i` of `cat0` is `matches[i]` in `cat1`.
Stars that are matched multiple times, not matched at all, or
with scores less than `min_score` are not returned.
score : dictionary
Fraction of times star `i` matched star `matches[i]` out of all
times stars `i` and `matches[i]` were matched to any star.
match_matrix : ndarray, optional
If `full_output` is `True`, also return the matrix of all star
matches.
Notes
-----
Based on the description of DAOPHOT's catalog matching via
triangles at
http://ned.ipac.caltech.edu/level5/Stetson/Stetson5_2.html
"""
from scipy.spatial.ckdtree import cKDTree
v0, s0 = triangles(*cat0)
v1, s1 = triangles(*cat1)
N0 = len(cat0[0])
N1 = len(cat1[0])
tree = cKDTree(s0)
d, i = tree.query(s1) # nearest matches between triangles
if verbose:
print ("""[spatial_match] cat0 = {} triangles, cat1 = {} triangles
[spatial_match] Best match score = {:.2g}, worst match sorce = {:.2g}
[spatial_match] {} triangle pairs at or below given tolerance ({})""".format(
len(v0), len(v1), min(d), max(d), sum(d <= tol), tol))
match_matrix = np.zeros((N0, N1), int)
for k, j in enumerate(i):
if d[k] <= tol:
match_matrix[v0[j][0], v1[k][0]] += 1
match_matrix[v0[j][1], v1[k][1]] += 1
match_matrix[v0[j][2], v1[k][2]] += 1
m0 = match_matrix.argmax(1)
m1 = match_matrix.argmax(0)
matches = dict()
scores = dict()
for i in range(len(m0)):
if i == m1[m0[i]]:
matches[i] = m0[i]
peak = match_matrix[i, m0[i]] * 2
total = match_matrix[i, :].sum() + match_matrix[:, m0[i]].sum()
scores[i] = peak / float(total)
for k in matches.keys():
if scores[k] < min_score:
del matches[k], scores[k]
if full_output:
return matches, scores, match_matrix
else:
return matches, scores
SSR = n.ones_like(speccat['ALPHA'])*-1. SSR_ERR = n.ones_like(speccat['ALPHA'])*-1. redshiftBest = n.empty(Nspec_total) goodZ = (speccat['ZFLAGS']==2)|(speccat['ZFLAGS']==3)|(speccat['ZFLAGS']==4)|(speccat['ZFLAGS']==9) badZ = (goodZ==False) redshiftBest[goodZ] = speccat['Z'][goodZ] raBad, decBad = speccat['ALPHA'][badZ], speccat['DELTA'][badZ] # photometric redshift catalog zphcatall = fits.open(join(survey.vvds_photo_dir, photozCat) )[1].data zphcat = zphcatall[ (zphcatall['zphot_T07']>-0.1)&(zphcatall['zphot_T07']<2.) ] treePhZ = t.cKDTree( n.transpose([zphcat['alpha_T07'], zphcat['delta_T07']]) ,1000.0) indexes = treePhZ.query(n.transpose([raBad,decBad]),1) dist = indexes[0] ids = indexes[1] ok = (dist < 1.5/3600.) nok = (dist >= 1.5/3600.) #redshiftBest[badZ] = zphcat['zphot_T07'][ids] redshiftBest[badZ[ok]] = zphcat['zphot_T07'][ids[ok]] redshiftBest[badZ[nok]] = n.ones_like(zphcat['zphot_T07'][ids[nok]])*-1. print float(len(zphcat['zphot_T07'][ids[ok]]))/len(raBad), "% of photoz used" print len(zphcat['zphot_T07'][ids[ok]]) print len(raBad) print len(redshiftBest[badZ[ok]]) print len(redshiftBest[badZ[nok]])
def validate_max_ice_cover_with_glerl():
"""
For validations of maximum annual ice concentrations with GLERL obs
"""
nemo_manager = NemoYearlyFilesManager(folder="/home/huziy/skynet3_rech1/offline_glk_output_daily_1979-2012",
suffix="icemod.nc")
nemo_manager_nosnow = NemoYearlyFilesManager(
folder="/home/huziy/skynet3_rech1/output_NEMO_offline_1979-2012_nosnow",
suffix="icemod.nc")
# Study period
start_year = 2003
end_year = 2012
lon2d, lat2d, bmp = nemo_manager.get_coords_and_basemap()
model_yearmax_ice_conc, model_lake_avg_ts = nemo_manager.get_max_yearly_ice_fraction(
start_year=start_year, end_year=end_year)
model_yearmax_ice_conc = np.ma.masked_where(~nemo_manager.lake_mask, model_yearmax_ice_conc)
model_yearmax_ice_conc_nosnow, model_lake_avg_ts_no_snow = nemo_manager_nosnow.get_max_yearly_ice_fraction(
start_year=start_year, end_year=end_year)
model_yearmax_ice_conc_nosnow = np.ma.masked_where(~nemo_manager.lake_mask, model_yearmax_ice_conc_nosnow)
# plt.figure()
xx, yy = bmp(lon2d.copy(), lat2d.copy())
# im = bmp.pcolormesh(xx, yy, model_yearmax_ice_conc)
# bmp.colorbar(im)
# Read and interpolate obs
path_to_obs = "/RESCUE/skynet3_rech1/huziy/nemo_obs_for_validation/glerl_icecov.nc"
obs_varname = "ice_cover"
obs_lake_avg_ts = []
with Dataset(path_to_obs) as ds:
time_var = ds.variables["time"]
lons_obs = ds.variables["lon"][:]
lats_obs = ds.variables["lat"][:]
dates = num2date(time_var[:], time_var.units)
nx, ny = lons_obs.shape
data = ds.variables[obs_varname][:]
data = np.ma.masked_where((data > 100) | (data < 0), data)
print(data.min(), data.max())
panel = pd.Panel(data=data, items=dates, major_axis=range(nx), minor_axis=range(ny))
panel = panel.select(lambda d: start_year <= d.year <= end_year)
the_max_list = []
for key, g in panel.groupby(lambda d: d.year, axis="items"):
the_max_field = np.ma.max(np.ma.masked_where((g.values > 100) | (g.values < 0), g.values), axis=0)
obs_lake_avg_ts.append(the_max_field.mean())
the_max_list.append(the_max_field)
obs_yearmax_ice_conc = np.ma.mean(the_max_list, axis=0) / 100.0
xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons_obs.flatten(), lats_obs.flatten())
ktree = cKDTree(list(zip(xs, ys, zs)))
xt, yt, zt = lat_lon.lon_lat_to_cartesian(lon2d.flatten(), lat2d.flatten())
dists, inds = ktree.query(list(zip(xt, yt, zt)))
obs_yearmax_ice_conc_interp = obs_yearmax_ice_conc.flatten()[inds].reshape(lon2d.shape)
obs_yearmax_ice_conc_interp = np.ma.masked_where(~nemo_manager.lake_mask, obs_yearmax_ice_conc_interp)
# plt.figure()
# b = Basemap()
# xx, yy = b(lons_obs, lats_obs)
# im = b.pcolormesh(xx, yy, obs_yearmax_ice_conc)
# b.colorbar(im)
# b.drawcoastlines()
# Plot as usual: model, obs, model - obs
img_folder = Path("nemo")
if not img_folder.is_dir():
img_folder.mkdir()
img_file = img_folder.joinpath("validate_yearmax_icecov_glerl_{}-{}.png".format(start_year, end_year))
plot_utils.apply_plot_params(height_cm=9, width_cm=45, font_size=12)
fig = plt.figure()
gs = GridSpec(1, 5, width_ratios=[1, 1, 0.05, 1, 0.05])
all_axes = []
cmap = cm.get_cmap("jet", 10)
diff_cmap = cm.get_cmap("RdBu_r", 10)
# Model
ax = fig.add_subplot(gs[0, 0])
ax.set_title("NEMO-offline")
bmp.pcolormesh(xx, yy, model_yearmax_ice_conc, cmap=cmap, vmin=0, vmax=1)
all_axes.append(ax)
# Model
# ax = fig.add_subplot(gs[0, 1])
# ax.set_title("NEMO-offline-nosnow")
# bmp.pcolormesh(xx, yy, model_yearmax_ice_conc_nosnow, cmap=cmap, vmin=0, vmax=1)
# all_axes.append(ax)
# Obs
ax = fig.add_subplot(gs[0, 1])
ax.set_title("NIC")
im = bmp.pcolormesh(xx, yy, obs_yearmax_ice_conc_interp, cmap=cmap, vmin=0, vmax=1)
all_axes.append(ax)
plt.colorbar(im, cax=fig.add_subplot(gs[0, 2]))
# Biases
ax = fig.add_subplot(gs[0, 3])
ax.set_title("NEMO - NIC")
im = bmp.pcolormesh(xx, yy, model_yearmax_ice_conc - obs_yearmax_ice_conc_interp, cmap=diff_cmap, vmin=-1, vmax=1)
plt.colorbar(im, cax=fig.add_subplot(gs[0, -1]))
all_axes.append(ax)
for the_ax in all_axes:
bmp.drawcoastlines(ax=the_ax)
the_ax.set_aspect("auto")
fig.savefig(str(img_file), bbox_inches="tight")
plt.close(fig)
# Plot lake aversged ice concentrations
fig = plt.figure()
plt.gca().get_xaxis().get_major_formatter().set_useOffset(False)
plt.plot(range(start_year, end_year + 1), model_lake_avg_ts, "b", lw=2, label="NEMO")
plt.plot(range(start_year, end_year + 1), model_lake_avg_ts_no_snow, "g", lw=2, label="NEMO-nosnow")
plt.plot(range(start_year, end_year + 1), np.asarray(obs_lake_avg_ts) / 100.0, "r", lw=2, label="NIC")
plt.grid()
plt.legend(ncol=2)
fig.savefig(str(img_folder.joinpath("lake_avg_iceconc_nemo_offline_vs_NIC.pdf")), bbox_inches="tight")
def interpolate_file(self, inpath, outpath, skip_feb_29=True):
"""
Interpolate data in the file, save the result to a new
file in the same folder, with the name of the interpolated variable
and time variable unchanged.
"""
#check if the output file already exists
if os.path.isfile(outpath):
print("{0} already exists, remove to recreate ...".format(outpath))
return
print("working on {0}".format(inpath))
ds_in = Dataset(inpath)
lon_ncatts = {}
lat_ncatts = {}
#read in and calculate coordinates of source grid
if "lon" in ds_in.variables:
in_lon_var_name = "lon"
in_lat_var_name = "lat"
elif "nav_lon" in ds_in.variables:
in_lon_var_name = "nav_lon"
in_lat_var_name = "nav_lat"
elif "lon0" in ds_in.variables:
in_lon_var_name = "lon0"
in_lat_var_name = "lat0"
else:
raise Exception("The file does not contain conventional lat/lon information: {0}".format(inpath))
in_lon_var = ds_in.variables[in_lon_var_name]
in_lat_var = ds_in.variables[in_lat_var_name]
source_lons, source_lats = in_lon_var[:], in_lat_var[:]
if in_lon_var.ndim == 1:
source_lons, source_lats = np.meshgrid(source_lons, source_lats)
for attname in in_lon_var.ncattrs():
lon_ncatts[attname] = in_lon_var.getncattr(attname)
for attname in in_lat_var.ncattrs():
lat_ncatts[attname] = in_lat_var.getncattr(attname)
#find the name of the field to be interpolated, and read it into memory
varnames = list(ds_in.variables.keys())
#write interpolated data
ds_out = Dataset(outpath, "w", format="NETCDF3_CLASSIC")
#copy and create dimensions
ds_out.createDimension("time", None)
ds_out.createDimension("x", self.target_lons.shape[1])
ds_out.createDimension("y", self.target_lons.shape[0])
#copy and interpolate variables
lonVar = ds_out.createVariable(in_lon_var_name, "f4", ("y", "x"))
latVar = ds_out.createVariable(in_lat_var_name, "f4", ("y", "x"))
#set the attributes
if len(lon_ncatts):
lonVar.setncatts(lon_ncatts)
latVar.setncatts(lat_ncatts)
good_points = np.abs(source_lons.flatten()) < 360
xs, ys, zs = lon_lat_to_cartesian(source_lons.flatten(), source_lats.flatten())
xt, yt, zt = lon_lat_to_cartesian(self.target_lons.flatten(), self.target_lats.flatten())
ktree = cKDTree(data=list(zip(xs[good_points], ys[good_points], zs[good_points])))
dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1)
# Handle time variable first
timename = None
time_data = None
for v in varnames:
if v.startswith("time"):
timename = v
break
##
if timename is not None:
time_var_in = ds_in.variables[timename]
time_var_out = ds_out.createVariable(timename, "f4", ("time",))
time_vals = time_var_in[:]
if time_var_in.shape[0] > 365 and skip_feb_29:
if hasattr(time_var_in, "units") and time_var_in.units.strip().lower() != "unknown":
time_data = num2date(time_vals, time_var_in.units)
df = pd.DataFrame(data=time_data, index=time_data)
time_vals = df.select(lambda d: not (d.day == 29 and d.month == 2)).values
else:
ntimes = time_vals.shape[0]
if ntimes % 366 == 0 and ntimes % 365 != 0:
nperday = ntimes // 366
dtseconds = 24 * 60 * 60 // nperday
dt = timedelta(seconds=dtseconds)
# dt0 = timedelta(seconds=int(time_vals[0] * 60 * 60)) # usually in seconds
start_date = datetime(2008, 1, 1)
time_data = [start_date + dt * i for i in range(ntimes)]
assert time_data[0].year == time_data[-1].year
df = pd.DataFrame(data=time_vals, index=time_data)
time_vals = df.select(lambda d: not (d.day == 29 and d.month == 2)).values
time_var_out[:] = time_vals
if hasattr(time_var_in, "units"):
time_var_out.units = time_var_in.units
for varname in varnames:
out_var = None
in_var = ds_in.variables[varname]
# Interpolate only 3d variables (time, lat, lon) and some 2d variables
if in_var.ndim == 3:
out_var = ds_out.createVariable(varname, "f4", ("time", "y", "x"))
if hasattr(out_var, "units"):
out_var.units = in_var.units
in_data = in_var[:]
if time_data is not None:
p = pd.Panel(data=in_data, items=time_data,
major_axis=list(range(in_data.shape[1])),
minor_axis=list(range(in_data.shape[2])))
if in_data.shape[0] > 365 and skip_feb_29:
p = p.select(lambda d: not (d.day == 29 and d.month == 2))
in_data = p.values
# reshape to 2d
print(in_data.shape, self.target_lons.shape)
in_data.shape = (in_data.shape[0], -1)
out_data = in_data[:, inds]
out_data.shape = (out_data.shape[0], ) + self.target_lons.shape
# print out_data.shape
out_var[:] = out_data
elif varname.lower() == "bathymetry":
out_var = ds_out.createVariable(varname, "f4", ("y", "x"))
out_var[:] = interpolate_bathymetry(in_var[:], ktree, source_coords=(xs, ys, zs),
target_coords=(xt, yt, zt),
out_data_shape=self.target_lons.shape)
elif in_var.ndim == 2 and varname.lower() in ["socoefr"]:
out_var = ds_out.createVariable(varname, "f4", ("y", "x"))
in_data = in_var[:]
# reshape to 2d
in_data = in_data.flatten()
out_data = in_data[inds]
out_data.shape = self.target_lons.shape
# print out_data.shape
out_var[:] = out_data
if out_var is not None:
# Set attributes of the interpolated fields
if hasattr(out_var, "long_name"):
out_var.long_name = in_var.long_name
if hasattr(in_var, "units"):
out_var.units = in_var.units
if hasattr(in_var, "missing_value"):
out_var.missing_value = in_var.missing_value
if hasattr(in_var, "coordinates"):
out_var.coordinates = in_var.coordinates
lonVar[:] = self.target_lons
latVar[:] = self.target_lats
# close netcdf files
ds_out.close()
ds_in.close()
def evolve(self, n_gen):
"""Perform evolution of the discriminators on the data provided in the initializer. Saves the discriminator
networks to /networks.
:param n_gen: :type int: Number of generations to run the evolution.
"""
try:
for generation in range(n_gen): # run for n_gen generations
if len(self.feature_list_genomes) < self.max_features:
start = time.clock()
outputs = np.zeros((self.params.PopulationSize, self.dsize))
# retrieve a list of all genomes in the population
genome_list = NEAT.GetGenomeList(self.pop)
# Get output space vectors for all genomes
j = 0
for genome in genome_list:
print "Computing behavior vector for genome " + str(j)
output_array = self.get_output_vector(genome)
k = 0
for element in output_array:
outputs[j][k] = element
k += 1
j += 1
# Evaluate novelty for all genomes and add to feature list
i = 0
for genome in genome_list:
print "Computing novelty for genome " + str(i)
genome_vector = outputs[i]
fitness = self.get_novelty(outputs, genome_vector)
# Add to feature list
if len(self.feature_list_genomes) == 0:
m = 0
for element in genome_vector:
self.feature_list_vectors[len(self.feature_list_genomes)][m] = element
m += 1
self.feature_list_genomes.append(genome)
else:
tree = kd.cKDTree(self.feature_list_vectors)
indexes = tree.query(genome_vector, 20)
dist = indexes[0][0]
print "Distance from closest in feature list: " + str(dist)
if dist > 200:
m = 0
for element in genome_vector:
self.feature_list_vectors[len(self.feature_list_genomes)][m] = element
m += 1
self.feature_list_genomes.append(genome)
genome.SetFitness(fitness)
i += 1
n = 0
for genome in self.feature_list_genomes:
net = NEAT.NeuralNetwork()
genome.BuildPhenotype(net)
net.Save("path/to/dir/net" + str(n) + ".nnet")
n += 1
# datafile = open("discriminator_genomes.pkl", "wb")
# pickle.dump(self.feature_list_genomes, datafile)
# datafile.close()
# datafile = open("discriminator_genomes_backup.pkl", "wb")
# pickle.dump(self.feature_list_genomes, datafile)
# datafile.close()
end = time.clock()
elapsed = end - start
print str(generation) + " generations evaluated, " + str(elapsed) + " for last gen."
print str(len(self.feature_list_genomes)) + " discriminators evolved."
print "----------"
# advance to the next generation
self.pop.Epoch()
else:
break
except KeyboardInterrupt:
print "Training stopped."
def validate_yearmax_ice_cover_from_hostetler_with_glerl(path="/home/huziy/skynet3_rech1/CRCM_GL_simulation/all_files",
start_year=2003, end_year=2009):
model_manager = Crcm5ModelDataManager(samples_folder_path=path, all_files_in_samples_folder=True)
varname = "LC"
hl_icecov_yearly_max = model_manager.get_yearmax_fields(start_year=start_year, end_year=end_year, var_name=varname)
hl_icecov_yearly_max_clim = np.mean(list(hl_icecov_yearly_max.values()), axis=0)
# Get Nemo manager here only for coordinates and mask
nemo_manager = NemoYearlyFilesManager(folder="/home/huziy/skynet3_rech1/offline_glk_output_daily_1979-2012",
suffix="icemod.nc")
lon2d, lat2d, bmp = nemo_manager.get_coords_and_basemap()
# Interpolate hostetler's lake fraction to the model's grid
hl_icecov_yearly_max_clim = model_manager.interpolate_data_to(hl_icecov_yearly_max_clim, lon2d, lat2d)
hl_icecov_yearly_max_clim = np.ma.masked_where(~nemo_manager.lake_mask, hl_icecov_yearly_max_clim)
model_lake_avg_ts = []
for the_year in range(start_year, end_year + 1):
model_lake_avg_ts.append(hl_icecov_yearly_max[the_year][nemo_manager.lake_mask].mean())
# plt.figure()
xx, yy = bmp(lon2d.copy(), lat2d.copy())
# im = bmp.pcolormesh(xx, yy, model_yearmax_ice_conc)
# bmp.colorbar(im)
# Read and interpolate obs
path_to_obs = "/RESCUE/skynet3_rech1/huziy/nemo_obs_for_validation/glerl_icecov.nc"
obs_varname = "ice_cover"
obs_lake_avg_ts = []
with Dataset(path_to_obs) as ds:
time_var = ds.variables["time"]
lons_obs = ds.variables["lon"][:]
lats_obs = ds.variables["lat"][:]
dates = num2date(time_var[:], time_var.units)
nx, ny = lons_obs.shape
data = ds.variables[obs_varname][:]
data = np.ma.masked_where((data > 100) | (data < 0), data)
print(data.min(), data.max())
panel = pd.Panel(data=data, items=dates, major_axis=range(nx), minor_axis=range(ny))
panel = panel.select(lambda d: start_year <= d.year <= end_year)
the_max_list = []
for key, g in panel.groupby(lambda d: d.year, axis="items"):
the_max_field = np.ma.max(np.ma.masked_where((g.values > 100) | (g.values < 0), g.values), axis=0)
obs_lake_avg_ts.append(the_max_field.mean())
the_max_list.append(the_max_field)
obs_yearmax_ice_conc = np.ma.mean(the_max_list, axis=0) / 100.0
xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons_obs.flatten(), lats_obs.flatten())
ktree = cKDTree(list(zip(xs, ys, zs)))
xt, yt, zt = lat_lon.lon_lat_to_cartesian(lon2d.flatten(), lat2d.flatten())
dists, inds = ktree.query(list(zip(xt, yt, zt)))
obs_yearmax_ice_conc_interp = obs_yearmax_ice_conc.flatten()[inds].reshape(lon2d.shape)
obs_yearmax_ice_conc_interp = np.ma.masked_where(~nemo_manager.lake_mask, obs_yearmax_ice_conc_interp)
# plt.figure()
# b = Basemap()
# xx, yy = b(lons_obs, lats_obs)
# im = b.pcolormesh(xx, yy, obs_yearmax_ice_conc)
# b.colorbar(im)
# b.drawcoastlines()
# Plot as usual: model, obs, model - obs
img_folder = Path("nemo/hostetler")
if not img_folder.is_dir():
img_folder.mkdir()
img_file = img_folder.joinpath("validate_yearmax_icecov_hl_vs_glerl_{}-{}.pdf".format(start_year, end_year))
fig = plt.figure()
gs = GridSpec(2, 4, width_ratios=[1, 1, 1, 0.05])
all_axes = []
cmap = cm.get_cmap("jet", 10)
diff_cmap = cm.get_cmap("RdBu_r", 10)
# Model
ax = fig.add_subplot(gs[0, 0])
ax.set_title("Hostetler")
bmp.pcolormesh(xx, yy, hl_icecov_yearly_max_clim, cmap=cmap, vmin=0, vmax=1)
all_axes.append(ax)
# Obs
ax = fig.add_subplot(gs[0, 2])
ax.set_title("GLERL")
im = bmp.pcolormesh(xx, yy, obs_yearmax_ice_conc_interp, cmap=cmap, vmin=0, vmax=1)
all_axes.append(ax)
plt.colorbar(im, cax=fig.add_subplot(gs[0, -1]))
# Biases
ax = fig.add_subplot(gs[1, :])
ax.set_title("Hostetler - GLERL")
im = bmp.pcolormesh(xx, yy, hl_icecov_yearly_max_clim - obs_yearmax_ice_conc_interp,
cmap=diff_cmap, vmin=-1, vmax=1)
bmp.colorbar(im, ax=ax)
all_axes.append(ax)
for the_ax in all_axes:
bmp.drawcoastlines(ax=the_ax)
fig.savefig(str(img_file), bbox_inches="tight")
plt.close(fig)
# Plot lake aversged ice concentrations
fig = plt.figure()
plt.gca().get_xaxis().get_major_formatter().set_useOffset(False)
plt.plot(range(start_year, end_year + 1), model_lake_avg_ts, "b", lw=2, label="Hostetler")
plt.plot(range(start_year, end_year + 1), np.asarray(obs_lake_avg_ts) / 100.0, "r", lw=2, label="GLERL")
plt.grid()
plt.legend(loc=3)
fig.savefig(str(img_folder.joinpath("lake_avg_iceconc_hostetler_offline_vs_GLERL.pdf")), bbox_inches="tight")
