def check_dmd_dask(D, mu, Phi, show_warning=True):
"""
Checks how close the approximation using DMD is to the original data.
Returns:
None if the difference is within the tolerance
Displays a warning otherwise.
"""
X = D[:, 0:-1]
Y = D[:, 1:]
#Y_est = da.dot(da.dot(da.dot(Phi, da.diag(mu)), pinv_SVD(Phi)), X)
Phi_inv = pinv_SVD(Phi)
PhiMu = da.dot(Phi, da.diag(mu))
#Y_est = da.dot(da.dot(PhiMu, Phi_inv), X)
Y_est = da.dot(PhiMu, da.dot(Phi_inv, X))
diff = da.real(Y - Y_est)
res = da.fabs(diff)
rtol = 1.e-8
atol = 1.e-5
if da.all(res < atol + rtol * da.fabs(da.real(Y_est))).compute():
return (None)
else:
#if not b and show_warning:
warn('dmd result does not satisfy Y=AX')
def get_angle_info(vza, sza, raa, m_pi):
"""
Gets the angle information
"""
AngleInfo = namedtuple('AngleInfo',
'vza sza raa vza_rad sza_rad raa_rad')
# View zenith angle
vza_rad = da.deg2rad(vza)
# Solar zenith angle
sza_rad = da.deg2rad(sza)
# Relative azimuth angle
raa_rad = da.deg2rad(raa)
vza_abs = da.fabs(vza_rad)
sza_abs = da.fabs(sza_rad)
raa_abs = da.where((vza_rad < 0) | (sza_rad < 0), m_pi, raa_rad)
return AngleInfo(vza=vza,
sza=sza,
raa=raa,
vza_rad=vza_abs,
sza_rad=sza_abs,
raa_rad=raa_abs)
def grid_glm_data(self, flashes):
"""
Aggregate the point flashes into a grid of flash counts occurring within each grid box.
Args:
flashes (:class:`pandas.DataFrame`): Contains the longitudes and latitudes of each flash
Returns:
:class:`numpy.ndarray` [y, x]: The number of flashes occurring at each grid point.
"""
flash_x, flash_y = self.glm_proj(flashes["flash_lon"].values,
flashes["flash_lat"].values)
flash_x /= 1000
flash_y /= 1000
valid_flashes = np.where(
(flash_x >= self.x_points.min() - self.dx_km / 2)
& (flash_x <= self.x_points.max() + self.dx_km / 2)
& (flash_y >= self.y_points.min() - self.dx_km / 2)
& (flash_y <= self.y_points.max() + self.dx_km / 2))[0]
if valid_flashes.size > 0:
if PARALLEL:
x_grid_flat = da.from_array(self.x_grid.reshape(
(self.x_grid.size, 1)),
chunks=512)
y_grid_flat = da.from_array(self.y_grid.reshape(
(self.x_grid.size, 1)),
chunks=512)
flash_x_flat = da.from_array(flash_x[valid_flashes].reshape(
1, valid_flashes.size),
chunks=512)
flash_y_flat = da.from_array(flash_y[valid_flashes].reshape(
1, valid_flashes.size),
chunks=512)
x_dist = da.fabs(x_grid_flat - flash_x_flat)
y_dist = da.fabs(y_grid_flat - flash_y_flat)
flash_grid_counts = da.sum(
(x_dist <= self.dx_km / 2) & (y_dist <= self.dx_km / 2),
axis=1)
flash_grid = flash_grid_counts.reshape(
self.lon_grid.shape).astype(np.int32).compute()
else:
x_grid_flat = self.x_grid.reshape((self.x_grid.size, 1))
y_grid_flat = self.y_grid.reshape((self.x_grid.size, 1))
flash_x_flat = flash_x[valid_flashes].reshape(
1, valid_flashes.size)
flash_y_flat = flash_y[valid_flashes].reshape(
1, valid_flashes.size)
x_dist = np.abs(x_grid_flat - flash_x_flat)
y_dist = np.abs(y_grid_flat - flash_y_flat)
flash_grid_counts = np.sum(
(x_dist <= self.dx_km / 2) & (y_dist <= self.dx_km / 2),
axis=1)
flash_grid = flash_grid_counts.reshape(
self.lon_grid.shape).astype(np.int32)
else:
flash_grid = np.zeros(self.lon_grid.shape, dtype=np.int32)
return flash_grid
def initialize_da(X, k, init='random', W=None, H=None):
n_components = k
n_samples, n_features = X.shape
if init == 'random':
avg = da.sqrt(X.mean() / n_components)
H = avg * da.random.RandomState(42).normal(
0,
1,
size=(n_components, n_features),
chunks=(n_components, X.chunks[1][0]))
W = avg * da.random.RandomState(42).normal(
0,
1,
size=(n_samples, n_components),
chunks=(n_samples, n_components))
H = da.fabs(H)
W = da.fabs(W)
return W, H
if init == 'nndsvd' or init == 'nndsvda':
# not converted to da yet
raise NotImplementedError
if init == 'custom':
return W, H
if init == 'random_vcol':
import math
#p_c = options.get('p_c', int(ceil(1. / 5 * X.shape[1])))
#p_r = options.get('p_r', int(ceil(1. / 5 * X.shape[0])))
p_c = int(math.ceil(1. / 5 * X.shape[1]))
p_r = int(math.ceil(1. / 5 * X.shape[0]))
prng = np.random.RandomState(42)
#W = da.zeros((X.shape[0], n_components), chunks = (X.shape[0],n_components))
#H = da.zeros((n_components, X.shape[1]), chunks = (n_components,X.chunks[1][0]))
W = []
H = []
for i in range(k):
W.append(X[:, prng.randint(low=0, high=X.shape[1], size=p_c)].mean(
axis=1).compute())
H.append(X[prng.randint(low=0, high=X.shape[0], size=p_r), :].mean(
axis=0).compute())
W = np.stack(W, axis=1)
H = np.stack(H, axis=0)
return W, H
def _transformlng_dask(lng, lat):
ret = 300.0 + lng + 2.0 * lat + 0.1 * lng * lng + \
0.1 * lng * lat + 0.1 * da.sqrt(da.fabs(lng))
ret += (20.0 * da.sin(6.0 * lng * pi) +
20.0 * da.sin(2.0 * lng * pi)) * 2.0 / 3.0
ret += (20.0 * da.sin(lng * pi) +
40.0 * da.sin(lng / 3.0 * pi)) * 2.0 / 3.0
ret += (150.0 * da.sin(lng / 12.0 * pi) +
300.0 * da.sin(lng / 30.0 * pi)) * 2.0 / 3.0
return ret
def _transformlat_dask(lng, lat):
ret = -100.0 + 2.0 * lng + 3.0 * lat + 0.2 * lat * lat + \
0.1 * lng * lat + 0.2 * da.sqrt(da.fabs(lng))
ret += (20.0 * da.sin(6.0 * lng * pi) +
20.0 * da.sin(2.0 * lng * pi)) * 2.0 / 3.0
ret += (20.0 * da.sin(lat * pi) +
40.0 * da.sin(lat / 3.0 * pi)) * 2.0 / 3.0
ret += (160.0 * da.sin(lat / 12.0 * pi) +
320 * da.sin(lat * pi / 30.0)) * 2.0 / 3.0
return ret
def get_li(self, kernel_type, li_recip):
# relative azimuth angle
# ensure it is in a [0,2] pi range
phi = da.fabs(
(self.angle_info.raa_rad % (2.0 * self.global_args.m_pi)))
cos_phi = da.cos(phi)
sin_phi = da.sin(phi)
tanti = da.tan(self.angle_info.sza_rad)
tantv = da.tan(self.angle_info.vza_rad)
cos1, sin1, tan1 = self.get_pangles(tantv, self.global_args.br,
self.global_args.nearly_zero)
cos2, sin2, tan2 = self.get_pangles(tanti, self.global_args.br,
self.global_args.nearly_zero)
# sets cos & sin phase angle terms
cos_phaang, phaang, sin_phaang = self.get_phaang(
cos1, cos2, sin1, sin2, cos_phi)
distance = self.get_distance(tan1, tan2, cos_phi)
overlap_info = self.get_overlap(cos1, cos2, tan1, tan2, sin_phi,
distance, self.global_args.hb,
self.global_args.m_pi)
if kernel_type.lower() == 'sparse':
if li_recip:
li = overlap_info.overlap - overlap_info.temp + 0.5 * (
1.0 + cos_phaang) / cos1 / cos2
else:
li = overlap_info.overlap - overlap_info.temp + 0.5 * (
1.0 + cos_phaang) / cos1
else:
if kernel_type.lower() == 'dense':
if li_recip:
li = (1.0 + cos_phaang) / (
cos1 * cos2 *
(overlap_info.temp - overlap_info.overlap)) - 2.0
else:
li = (1.0 + cos_phaang) / (
cos1 *
(overlap_info.temp - overlap_info.overlap)) - 2.0
return li
def _radiance_to_reflectance(self, rad):
"""Convert VIS radiance to reflectance factor.
Note: Produces huge reflectances in situations where both radiance and
solar zenith angle are small. Maybe the corresponding uncertainties
can be used to filter these cases before calculating reflectances.
Reference: [PUG], equation (6).
"""
sza = self.solar_zenith_angle.where(
da.fabs(self.solar_zenith_angle) < 90,
np.float32(np.nan)) # direct illumination only
cos_sza = np.cos(np.deg2rad(sza))
refl = ((np.pi * self.coefs['distance_sun_earth']**2) /
(self.coefs['solar_irradiance'] * cos_sza) * rad)
return self.refl_factor_to_percent(refl)
def calc_moments(self):
with h5py.File(self.infile, 'r', rdcc_nbytes=1000 * 1000 * 1000) as f:
data = da.from_array(f['data'],
chunks=(-1, 256, -1, -1)) # CNHW layout
data = da.transpose(data, (1, 2, 3, 0))
dtype = data.dtype
if dtype != np.float32:
print(
'WARNING: data will be saved as float32 but input ist float64!'
)
if self.mean is None:
arr = data
with ProgressBar():
self.mean, self.std = da.compute(arr.mean(axis=[0, 1, 2]),
arr.std(axis=[0, 1, 2]),
num_workers=8)
else:
self.mean, self.std = np.asarray(
self.mean, dtype=dtype), np.asarray(self.std, dtype=dtype)
print('mean: {}, std: {}'.format(list(self.mean), list(self.std)))
if self.log1p_norm:
data_z_norm = (data - self.mean) / self.std
data_log1p = da.sign(data_z_norm) * da.log1p(
da.fabs(data_z_norm))
if self.mean_log1p is None:
arr = data_log1p
with ProgressBar():
self.mean_log1p, self.std_log1p = da.compute(
arr.mean(axis=[0, 1, 2]),
arr.std(axis=[0, 1, 2]),
num_workers=8)
else:
self.mean_log1p, self.std_log1p = np.asarray(
self.mean_log1p,
dtype=dtype), np.asarray(self.std_log1p, dtype=dtype)
print('mean_log1p: {}, std_log1p: {}'.format(
list(self.mean_log1p), list(self.std_log1p)))
def test_arithmetic():
x = np.arange(5).astype('f4') + 2
y = np.arange(5).astype('i8') + 2
z = np.arange(5).astype('i4') + 2
a = da.from_array(x, chunks=(2,))
b = da.from_array(y, chunks=(2,))
c = da.from_array(z, chunks=(2,))
assert eq(a + b, x + y)
assert eq(a * b, x * y)
assert eq(a - b, x - y)
assert eq(a / b, x / y)
assert eq(b & b, y & y)
assert eq(b | b, y | y)
assert eq(b ^ b, y ^ y)
assert eq(a // b, x // y)
assert eq(a ** b, x ** y)
assert eq(a % b, x % y)
assert eq(a > b, x > y)
assert eq(a < b, x < y)
assert eq(a >= b, x >= y)
assert eq(a <= b, x <= y)
assert eq(a == b, x == y)
assert eq(a != b, x != y)
assert eq(a + 2, x + 2)
assert eq(a * 2, x * 2)
assert eq(a - 2, x - 2)
assert eq(a / 2, x / 2)
assert eq(b & True, y & True)
assert eq(b | True, y | True)
assert eq(b ^ True, y ^ True)
assert eq(a // 2, x // 2)
assert eq(a ** 2, x ** 2)
assert eq(a % 2, x % 2)
assert eq(a > 2, x > 2)
assert eq(a < 2, x < 2)
assert eq(a >= 2, x >= 2)
assert eq(a <= 2, x <= 2)
assert eq(a == 2, x == 2)
assert eq(a != 2, x != 2)
assert eq(2 + b, 2 + y)
assert eq(2 * b, 2 * y)
assert eq(2 - b, 2 - y)
assert eq(2 / b, 2 / y)
assert eq(True & b, True & y)
assert eq(True | b, True | y)
assert eq(True ^ b, True ^ y)
assert eq(2 // b, 2 // y)
assert eq(2 ** b, 2 ** y)
assert eq(2 % b, 2 % y)
assert eq(2 > b, 2 > y)
assert eq(2 < b, 2 < y)
assert eq(2 >= b, 2 >= y)
assert eq(2 <= b, 2 <= y)
assert eq(2 == b, 2 == y)
assert eq(2 != b, 2 != y)
assert eq(-a, -x)
assert eq(abs(a), abs(x))
assert eq(~(a == b), ~(x == y))
assert eq(~(a == b), ~(x == y))
assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
assert eq(da.exp(b), np.exp(y))
assert eq(da.log(a), np.log(x))
assert eq(da.log10(a), np.log10(x))
assert eq(da.log1p(a), np.log1p(x))
assert eq(da.expm1(b), np.expm1(y))
assert eq(da.sqrt(a), np.sqrt(x))
assert eq(da.square(a), np.square(x))
assert eq(da.sin(a), np.sin(x))
assert eq(da.cos(b), np.cos(y))
assert eq(da.tan(a), np.tan(x))
assert eq(da.arcsin(b/10), np.arcsin(y/10))
assert eq(da.arccos(b/10), np.arccos(y/10))
assert eq(da.arctan(b/10), np.arctan(y/10))
assert eq(da.arctan2(b*10, a), np.arctan2(y*10, x))
assert eq(da.hypot(b, a), np.hypot(y, x))
assert eq(da.sinh(a), np.sinh(x))
assert eq(da.cosh(b), np.cosh(y))
assert eq(da.tanh(a), np.tanh(x))
assert eq(da.arcsinh(b*10), np.arcsinh(y*10))
assert eq(da.arccosh(b*10), np.arccosh(y*10))
assert eq(da.arctanh(b/10), np.arctanh(y/10))
assert eq(da.deg2rad(a), np.deg2rad(x))
assert eq(da.rad2deg(a), np.rad2deg(x))
assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))
assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
assert eq(da.isfinite(a), np.isfinite(x))
assert eq(da.isinf(a), np.isinf(x))
assert eq(da.isnan(a), np.isnan(x))
assert eq(da.signbit(a - 3), np.signbit(x - 3))
assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
assert eq(da.ldexp(c, c), np.ldexp(z, z))
assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
assert eq(da.ceil(a), np.ceil(x))
assert eq(da.trunc(a / 2), np.trunc(x / 2))
assert eq(da.degrees(b), np.degrees(y))
assert eq(da.radians(a), np.radians(x))
assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
assert eq(da.fix(a - 2.5), np.fix(x - 2.5))
assert eq(da.angle(a + 1j), np.angle(x + 1j))
assert eq(da.real(a + 1j), np.real(x + 1j))
assert eq((a + 1j).real, np.real(x + 1j))
assert eq(da.imag(a + 1j), np.imag(x + 1j))
assert eq((a + 1j).imag, np.imag(x + 1j))
assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
assert eq((a + 1j * b).conj(), (x + 1j * y).conj())
assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
assert eq(da.fabs(b), np.fabs(y))
assert eq(da.sign(b - 2), np.sign(y - 2))
l1, l2 = da.frexp(a)
r1, r2 = np.frexp(x)
assert eq(l1, r1)
assert eq(l2, r2)
l1, l2 = da.modf(a)
r1, r2 = np.modf(x)
assert eq(l1, r1)
assert eq(l2, r2)
assert eq(da.around(a, -1), np.around(x, -1))
def test_arithmetic():
x = np.arange(5).astype('f4') + 2
y = np.arange(5).astype('i8') + 2
z = np.arange(5).astype('i4') + 2
a = da.from_array(x, chunks=(2, ))
b = da.from_array(y, chunks=(2, ))
c = da.from_array(z, chunks=(2, ))
assert eq(a + b, x + y)
assert eq(a * b, x * y)
assert eq(a - b, x - y)
assert eq(a / b, x / y)
assert eq(b & b, y & y)
assert eq(b | b, y | y)
assert eq(b ^ b, y ^ y)
assert eq(a // b, x // y)
assert eq(a**b, x**y)
assert eq(a % b, x % y)
assert eq(a > b, x > y)
assert eq(a < b, x < y)
assert eq(a >= b, x >= y)
assert eq(a <= b, x <= y)
assert eq(a == b, x == y)
assert eq(a != b, x != y)
assert eq(a + 2, x + 2)
assert eq(a * 2, x * 2)
assert eq(a - 2, x - 2)
assert eq(a / 2, x / 2)
assert eq(b & True, y & True)
assert eq(b | True, y | True)
assert eq(b ^ True, y ^ True)
assert eq(a // 2, x // 2)
assert eq(a**2, x**2)
assert eq(a % 2, x % 2)
assert eq(a > 2, x > 2)
assert eq(a < 2, x < 2)
assert eq(a >= 2, x >= 2)
assert eq(a <= 2, x <= 2)
assert eq(a == 2, x == 2)
assert eq(a != 2, x != 2)
assert eq(2 + b, 2 + y)
assert eq(2 * b, 2 * y)
assert eq(2 - b, 2 - y)
assert eq(2 / b, 2 / y)
assert eq(True & b, True & y)
assert eq(True | b, True | y)
assert eq(True ^ b, True ^ y)
assert eq(2 // b, 2 // y)
assert eq(2**b, 2**y)
assert eq(2 % b, 2 % y)
assert eq(2 > b, 2 > y)
assert eq(2 < b, 2 < y)
assert eq(2 >= b, 2 >= y)
assert eq(2 <= b, 2 <= y)
assert eq(2 == b, 2 == y)
assert eq(2 != b, 2 != y)
assert eq(-a, -x)
assert eq(abs(a), abs(x))
assert eq(~(a == b), ~(x == y))
assert eq(~(a == b), ~(x == y))
assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
assert eq(da.exp(b), np.exp(y))
assert eq(da.log(a), np.log(x))
assert eq(da.log10(a), np.log10(x))
assert eq(da.log1p(a), np.log1p(x))
assert eq(da.expm1(b), np.expm1(y))
assert eq(da.sqrt(a), np.sqrt(x))
assert eq(da.square(a), np.square(x))
assert eq(da.sin(a), np.sin(x))
assert eq(da.cos(b), np.cos(y))
assert eq(da.tan(a), np.tan(x))
assert eq(da.arcsin(b / 10), np.arcsin(y / 10))
assert eq(da.arccos(b / 10), np.arccos(y / 10))
assert eq(da.arctan(b / 10), np.arctan(y / 10))
assert eq(da.arctan2(b * 10, a), np.arctan2(y * 10, x))
assert eq(da.hypot(b, a), np.hypot(y, x))
assert eq(da.sinh(a), np.sinh(x))
assert eq(da.cosh(b), np.cosh(y))
assert eq(da.tanh(a), np.tanh(x))
assert eq(da.arcsinh(b * 10), np.arcsinh(y * 10))
assert eq(da.arccosh(b * 10), np.arccosh(y * 10))
assert eq(da.arctanh(b / 10), np.arctanh(y / 10))
assert eq(da.deg2rad(a), np.deg2rad(x))
assert eq(da.rad2deg(a), np.rad2deg(x))
assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))
assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
assert eq(da.isfinite(a), np.isfinite(x))
assert eq(da.isinf(a), np.isinf(x))
assert eq(da.isnan(a), np.isnan(x))
assert eq(da.signbit(a - 3), np.signbit(x - 3))
assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
assert eq(da.ldexp(c, c), np.ldexp(z, z))
assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
assert eq(da.ceil(a), np.ceil(x))
assert eq(da.trunc(a / 2), np.trunc(x / 2))
assert eq(da.degrees(b), np.degrees(y))
assert eq(da.radians(a), np.radians(x))
assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
assert eq(da.fix(a - 2.5), np.fix(x - 2.5))
assert eq(da.angle(a + 1j), np.angle(x + 1j))
assert eq(da.real(a + 1j), np.real(x + 1j))
assert eq((a + 1j).real, np.real(x + 1j))
assert eq(da.imag(a + 1j), np.imag(x + 1j))
assert eq((a + 1j).imag, np.imag(x + 1j))
assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
assert eq((a + 1j * b).conj(), (x + 1j * y).conj())
assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
assert eq(da.fabs(b), np.fabs(y))
assert eq(da.sign(b - 2), np.sign(y - 2))
l1, l2 = da.frexp(a)
r1, r2 = np.frexp(x)
assert eq(l1, r1)
assert eq(l2, r2)
l1, l2 = da.modf(a)
r1, r2 = np.modf(x)
assert eq(l1, r1)
assert eq(l2, r2)
assert eq(da.around(a, -1), np.around(x, -1))
def ds(self):
if self._ds is None:
file_exists = os.path.exists(self._result_file)
reprocess = not file_exists or self._reprocess
if reprocess:
if file_exists:
print('Old file exists ' + self._result_file)
#print('Removing old file ' + self._result_file)
#shutil.rmtree(self._result_file)
ds_data = OrderedDict()
to_seconds = np.vectorize(
lambda x: x.seconds + x.microseconds / 1E6)
print('Processing binary data...')
xx, yy, zz = self._loadgrid()
if xx is None:
if self._from_nc:
print('Processing existing netcdf...')
fn = self._result_file[:-5] + '_QC_raw.nc'
if os.path.exists(fn):
ds_temp = xr.open_dataset(self._result_file[:-5] +
'_QC_raw.nc',
chunks={'time': 50})
u = da.transpose(ds_temp['U'].data,
axes=[3, 0, 1, 2])
v = da.transpose(ds_temp['V'].data,
axes=[3, 0, 1, 2])
w = da.transpose(ds_temp['W'].data,
axes=[3, 0, 1, 2])
tt = ds_temp['time']
te = (tt - tt[0]) / np.timedelta64(1, 's')
xx = ds_temp['x'].values
yy = ds_temp['y'].values
zz = ds_temp['z'].values
else:
print('USING OLD ZARR DATA')
ds_temp = xr.open_zarr(self._result_file)
u = da.transpose(ds_temp['U'].data,
axes=[3, 0, 1, 2])
v = da.transpose(ds_temp['V'].data,
axes=[3, 0, 1, 2])
w = da.transpose(ds_temp['W'].data,
axes=[3, 0, 1, 2])
tt = ds_temp['time']
te = (tt - tt[0]) / np.timedelta64(1, 's')
xx = ds_temp['x'].values
yy = ds_temp['y'].values
zz = ds_temp['z'].values
print('ERROR: No NetCDF data found for ' +
self._xml_file)
#return None
# print(u.shape)
else:
tt, uvw = self._loaddata(xx, yy, zz)
if tt is None:
print('ERROR: No binary data found for ' +
self._xml_file)
return None
# calculate the elapsed time from the Timestamp objects and then convert to datetime64 datatype
te = to_seconds(tt - tt[0])
tt = pd.to_datetime(tt)
uvw = uvw.persist()
u = uvw[:, :, :, :, 0]
v = uvw[:, :, :, :, 1]
w = uvw[:, :, :, :, 2]
# u = xr.DataArray(uvw[:,:,:,:,0], coords=[tt, xx, yy, zz], dims=['time','x', 'y', 'z'],
# name='U', attrs={'standard_name': 'sea_water_x_velocity', 'units': 'm s-1'})
# v = xr.DataArray(uvw[:,:,:,:,1], coords=[tt, xx, yy, zz], dims=['time', 'x', 'y', 'z'],
# name='V', attrs={'standard_name': 'sea_water_x_velocity', 'units': 'm s-1'})
# w = xr.DataArray(uvw[:,:,:,:,2], coords=[tt, xx, yy, zz], dims=['time', 'x', 'y', 'z'],
# name='W', attrs={'standard_name': 'upward_sea_water_velocity', 'units': 'm s-1'})
if xx is None:
print('No data found')
return None
u = u.persist()
v = v.persist()
w = w.persist()
dx = float(xx[1] - xx[0])
dy = float(yy[1] - yy[0])
dz = float(zz[1] - zz[0])
if self._norm_dims:
exp = self._result_root.split('/')[4]
runSheet = pd.read_csv('~/RunSheet-%s.csv' % exp)
runSheet = runSheet.set_index('RunID')
runDetails = runSheet.ix[int(self.run_id[-2:])]
T = runDetails['T (s)']
h = runDetails['h (m)']
D = runDetails['D (m)']
ww = te / T
om = 2. * np.pi / T
d_s = (2. * 1E-6 / om)**0.5
bl = 3. * np.pi / 4. * d_s
if exp == 'Exp6':
if D == 0.1:
dy_c = (188. + 82.) / 2
dx_c = 39.25
cx = dx_c / 1000.
cy = dy_c / 1000.
else:
dy_c = (806. + 287.) / 2. * 0.22
dx_c = 113 * 0.22
cx = dx_c / 1000.
cy = dy_c / 1000.
elif exp == 'Exp8':
dy_c = 624 * 0.22
dx_c = 15
cx = dx_c / 1000.
cy = dy_c / 1000.
xn = (xx + (D / 2. - cx)) / D
yn = (yy - cy) / D
zn = zz / h
xnm, ynm = np.meshgrid(xn, yn)
rr = np.sqrt(xnm**2. + ynm**2)
cylMask = rr < 0.5
nanPlane = np.ones(cylMask.shape)
nanPlane[cylMask] = np.nan
nanPlane = nanPlane.T
nanPlane = nanPlane[np.newaxis, :, :, np.newaxis]
u = u * nanPlane
v = v * nanPlane
w = w * nanPlane
if D == 0.1:
xInds = xn > 3.
else:
xInds = xn > 2.
blInd = np.argmax(zn > bl / h)
blPlane = int(round(blInd))
Ue = u[:, xInds, :, :]
Ue_bar = da.nanmean(Ue, axis=(1, 2, 3)).compute()
Ue_bl = da.nanmean(Ue[:, :, :, blPlane],
axis=(1, 2)).compute()
inds = ~np.isnan(Ue_bl)
xv = ww[inds] % 1.
xv = xv + np.random.normal(scale=1E-6, size=xv.shape)
yv = Ue_bl[inds]
xy = np.stack([
np.concatenate([xv - 1., xv, xv + 1.]),
np.concatenate([yv, yv, yv])
]).T
xy = xy[xy[:, 0].argsort(), :]
xi = np.linspace(-0.5, 1.5, len(xv) / 8)
n = np.nanmax(xy[:, 1])
# print(n)
# fig,ax = pl.subplots()
# ax.scatter(xy[:,0],xy[:,1]/n)
# print(xy)
spl = si.LSQUnivariateSpline(xy[:, 0],
xy[:, 1] / n,
t=xi,
k=3)
roots = spl.roots()
der = spl.derivative()
slope = der(roots)
inds = np.min(np.where(slope > 0))
dt = (roots[inds] % 1.).mean() - 0.5
tpx = np.arange(0, 0.5, 0.001)
U0_bl = np.abs(spl(tpx + dt).min() * n)
ws = ww - dt
Ue_spl = spl((ws - 0.5) % 1.0 + dt) * n * -1.0
#maxima = spl.derivative().roots()
#Umax = spl(maxima)
#UminIdx = np.argmin(Umax)
#U0_bl = np.abs(Umax[UminIdx]*n)
#ww_at_min = maxima[UminIdx]
#ws = ww - ww_at_min + 0.25
inds = ~np.isnan(Ue_bar)
xv = ww[inds] % 1.
xv = xv + np.random.normal(scale=1E-6, size=xv.shape)
yv = Ue_bar[inds]
xy = np.stack([
np.concatenate([xv - 1., xv, xv + 1.]),
np.concatenate([yv, yv, yv])
]).T
xy = xy[xy[:, 0].argsort(), :]
xi = np.linspace(-0.5, 1.5, len(xv) / 8)
n = np.nanmax(xy[:, 1])
spl = si.LSQUnivariateSpline(xy[:, 0],
xy[:, 1] / n,
t=xi,
k=4)
maxima = spl.derivative().roots()
Umax = spl(maxima)
UminIdx = np.argmin(Umax)
U0_bar = np.abs(Umax[UminIdx] * n)
ww = xr.DataArray(ww, coords=[
tt,
], dims=[
'time',
])
ws = xr.DataArray(ws - 0.5, coords=[
tt,
], dims=[
'time',
])
xn = xr.DataArray(xn, coords=[
xx,
], dims=[
'x',
])
yn = xr.DataArray(yn, coords=[
yy,
], dims=[
'y',
])
zn = xr.DataArray(zn, coords=[
zz,
], dims=[
'z',
])
Ue_bar = xr.DataArray(Ue_bar,
coords=[
tt,
],
dims=[
'time',
])
Ue_bl = xr.DataArray(Ue_bl, coords=[
tt,
], dims=[
'time',
])
Ue_spl = xr.DataArray(Ue_spl,
coords=[
tt,
],
dims=[
'time',
])
ds_data['ww'] = ww
ds_data['ws'] = ws
ds_data['xn'] = xn
ds_data['yn'] = yn
ds_data['zn'] = zn
ds_data['Ue_bar'] = Ue_bar
ds_data['Ue_bl'] = Ue_bl
ds_data['Ue_spl'] = Ue_spl
te = xr.DataArray(te, coords=[
tt,
], dims=[
'time',
])
dims = ['time', 'x', 'y', 'z']
coords = [tt, xx, yy, zz]
ds_data['U'] = xr.DataArray(u,
coords=coords,
dims=dims,
name='U',
attrs={
'standard_name':
'sea_water_x_velocity',
'units': 'm s-1'
})
ds_data['V'] = xr.DataArray(v,
coords=coords,
dims=dims,
name='V',
attrs={
'standard_name':
'sea_water_x_velocity',
'units': 'm s-1'
})
ds_data['W'] = xr.DataArray(w,
coords=coords,
dims=dims,
name='W',
attrs={
'standard_name':
'sea_water_x_velocity',
'units': 'm s-1'
})
ds_data['te'] = te
# stdV = da.nanstd(v)
# stdW = da.nanstd(w)
# thres=7.
if 'U0_bl' in locals():
condition = (da.fabs(v) / U0_bl >
1.5) | (da.fabs(w) / U0_bl > 0.6)
for var in ['U', 'V', 'W']:
ds_data[var].data = da.where(condition, np.nan,
ds_data[var].data)
piv_step_frame = float(
self._xml_root.findall('piv/stepFrame')[0].text)
print('Calculating tensor')
# j = jacobianConv(ds.U, ds.V, ds.W, dx, dy, dz, sigma=1.5)
j = jacobianDask(u, v, w, piv_step_frame, dx, dy, dz)
print('Done')
#j = da.from_array(j,chunks=(20,-1,-1,-1,-1,-1))
# j = jacobianDask(uvw[:,:,:,:,0],uvw[:,:,:,:,1], uvw[:,:,:,:,2], piv_step_frame, dx, dy, dz)
jT = da.transpose(j, axes=[0, 1, 2, 3, 5, 4])
# j = j.persist()
# jT = jT.persist()
jacobianNorm = da.sqrt(
da.nansum(da.nansum(j**2., axis=-1), axis=-1))
strainTensor = (j + jT) / 2.
vorticityTensor = (j - jT) / 2.
strainTensorNorm = da.sqrt(
da.nansum(da.nansum(strainTensor**2., axis=-1), axis=-1))
vorticityTensorNorm = da.sqrt(
da.nansum(da.nansum(vorticityTensor**2., axis=-1),
axis=-1))
divergence = j[:, :, :, :, 0, 0] + j[:, :, :, :, 1,
1] + j[:, :, :, :, 2, 2]
# print(divergence)
omx = vorticityTensor[:, :, :, :, 2, 1] * 2.
omy = vorticityTensor[:, :, :, :, 0, 2] * 2.
omz = vorticityTensor[:, :, :, :, 1, 0] * 2.
divNorm = divergence / jacobianNorm
# divNorm = divNorm.persist()
# divNorm_mean = da.nanmean(divNorm)
# divNorm_std = da.nanstd(divNorm)
dims = ['x', 'y', 'z']
comp = ['u', 'v', 'w']
ds_data['jacobian'] = xr.DataArray(
j,
coords=[tt, xx, yy, zz, comp, dims],
dims=['time', 'x', 'y', 'z', 'comp', 'dims'],
name='jacobian')
ds_data['jacobianNorm'] = xr.DataArray(
jacobianNorm,
coords=[tt, xx, yy, zz],
dims=['time', 'x', 'y', 'z'],
name='jacobianNorm')
ds_data['strainTensor'] = xr.DataArray(
strainTensor,
coords=[tt, xx, yy, zz, comp, dims],
dims=['time', 'x', 'y', 'z', 'comp', 'dims'],
name='strainTensor')
ds_data['vorticityTensor'] = xr.DataArray(
vorticityTensor,
coords=[tt, xx, yy, zz, comp, dims],
dims=['time', 'x', 'y', 'z', 'comp', 'dims'],
name='vorticityTensor')
ds_data['vorticityNorm'] = xr.DataArray(
vorticityTensorNorm,
coords=[tt, xx, yy, zz],
dims=['time', 'x', 'y', 'z'],
name='vorticityNorm')
ds_data['strainNorm'] = xr.DataArray(
strainTensorNorm,
coords=[tt, xx, yy, zz],
dims=['time', 'x', 'y', 'z'],
name='strainNorm')
ds_data['divergence'] = xr.DataArray(
divergence,
coords=[tt, xx, yy, zz],
dims=['time', 'x', 'y', 'z'],
name='divergence')
ds_data['omx'] = xr.DataArray(omx,
coords=[tt, xx, yy, zz],
dims=['time', 'x', 'y', 'z'],
name='omx')
ds_data['omy'] = xr.DataArray(omy,
coords=[tt, xx, yy, zz],
dims=['time', 'x', 'y', 'z'],
name='omy')
ds_data['omz'] = xr.DataArray(omz,
coords=[tt, xx, yy, zz],
dims=['time', 'x', 'y', 'z'],
name='omz')
ds_data['divNorm'] = xr.DataArray(divNorm,
coords=[tt, xx, yy, zz],
dims=['time', 'x', 'y', 'z'],
name='divNorm')
# ds_data['divNorm_mean'] = xr.DataArray(divNorm_mean)
# ds_data['divNorm_std'] = xr.DataArray(divNorm_std)
ds = xr.Dataset(ds_data)
# if self._from_nc:
# for k,v in ds_temp.attrs.items():
# ds.attrs[k]=v
#ds = ds.chunk({'time': 20})
self._append_CF_attrs(ds)
self._append_attrs(ds)
ds.attrs['filename'] = self._result_file
if self._norm_dims:
KC = U0_bl * T / D
delta = (2. * np.pi * d_s) / h
S = delta / KC
ds.attrs['T'] = T
ds.attrs['h'] = h
ds.attrs['D'] = D
ds.attrs['U0_bl'] = U0_bl
ds.attrs['U0_bar'] = U0_bar
ds.attrs['KC'] = KC
ds.attrs['S'] = S
ds.attrs['Delta+'] = ((1E-6 * T)**0.5) / h
ds.attrs['Delta_l'] = 2 * np.pi * d_s
ds.attrs['Delta_s'] = d_s
ds.attrs['Re_D'] = U0_bl * D / 1E-6
ds.attrs['Beta'] = D**2. / (1E-6 * T)
delta = (ds.attrs['dx'] * ds.attrs['dy'] *
ds.attrs['dz'])**(1. / 3.)
dpx = (ds.attrs['pdx'] * ds.attrs['pdy'] *
ds.attrs['pdz'])**(1. / 3.)
delta_px = delta / dpx
dt = ds.attrs['piv_step_ensemble']
# divRMS = da.sqrt(da.nanmean((divergence * dt) ** 2.))
# divRMS = divRMS.persist()
# vorticityTensorNorm.persist()
# velocityError = divRMS/((3./(2.*delta_px**2.))**0.5)
# print(da.percentile(ds_new['vorticityTensorNorm'].data.ravel(),99.))
# print(ds_new['divRMS'])
# print(ds_new['divNorm_mean'])
# vorticityError = divRMS/dt/da.percentile(vorticityTensorNorm.ravel(),99.)
# divNorm_mean = da.nanmean(divNorm)
# divNorm_std = da.nanstd(divNorm)
# print("initial save")
#ds.to_zarr(self._result_file,compute=False)
#ds = xr.open_zarr(self._result_file)
# xstart = np.argmax(xx > 0.05)
# ystart = np.argmax(yy > 0.07)
divRMS = da.sqrt(da.nanmean(
(divergence * dt)**2.)) #.compute()
#divNorm = divergence / jacobianNorm
#divNorm = divNorm.compute()
#divNorm_mean = da.nanmean(divNorm).compute()
#divNorm_std = da.nanstd(divNorm).compute()
velocityError = divRMS / ((3. / (2. * delta_px**2.))**0.5)
vortNorm = vorticityTensorNorm #.compute()
vorticityError = divRMS / dt / np.percentile(
vortNorm.ravel(), 99.)
velocityError, vorticityError = da.compute(
velocityError, vorticityError)
#ds.attrs['divNorm_mean'] = divNorm_mean
#ds.attrs['divNorm_std'] = divNorm_std
ds.attrs['velocityError'] = velocityError
ds.attrs['vorticityError'] = vorticityError
if self._norm_dims:
xInds = (xn > 0.5) & (xn < 2.65)
yInds = (yn > -0.75) & (yn < 0.75)
else:
xInds = range(len(ds['x']))
yInds = range(len(ds['y']))
vrms = (ds['V'][:, xInds, yInds, :]**2.).mean(
dim=['time', 'x', 'y', 'z'])**0.5
wrms = (ds['W'][:, xInds, yInds, :]**2.).mean(
dim=['time', 'x', 'y', 'z'])**0.5
ds.attrs['Vrms'] = float(vrms.compute())
ds.attrs['Wrms'] = float(wrms.compute())
#fig,ax = pl.subplots()
#ax.plot(ds.ws,ds.Ue_spl/U0_bl,color='k')
#ax.plot(ds.ws,ds.Ue_bl/U0_bl,color='g')
#ax.set_xlabel(r'$t/T$')
#ax.set_ylabel(r'$U_{bl}/U_0$')
#fig.savefig(self._result_file[:-4] + 'png',dpi=125)
#pl.close(fig)
# print("second save")
#ds.to_netcdf(self._result_file)
ds.to_zarr(self._result_file, mode='w')
print('Cached ' + self._result_file)
#ds = xr.open_dataset(self._result_file,chunks={'time':20})
ds = xr.open_zarr(self._result_file)
ds.attrs['filename'] = self._result_file
else:
#ds = xr.open_dataset(self._result_file,chunks={'time':20})
ds = xr.open_zarr(self._result_file)
ds.attrs['filename'] = self._result_file
self._ds = ds
return self._ds
def update_H_da(M, H, W):
denominator = da.dot(W.T, da.dot(W, H))
denominator_new = da.where(
da.fabs(denominator) < EPSILON, EPSILON, denominator)
H_new = H * da.dot(W.T, M) / denominator_new
return (H_new)
def gradient(image):
return da.where(
da.fabs(image) <= huber['threshold'],
2 * image,
2 * huber['threshold'] * da.sign(image))
def update_W_da(M, H, W):
denominator = da.dot(W, da.dot(H, H.T))
denominator_new = da.where(
da.fabs(denominator) < EPSILON, EPSILON, denominator)
W_new = W * da.dot(M, H.T) / denominator_new
return (W_new)
def main():
########################################################
infile = '/data/ERA5/hdf5_hr/ds_eval_2000_to_2002.hdf5'
outfile = 'sr_eval_2000_2002.{}.tfrecords'
n_files = 8
gzip = True
shuffle = False
########################################################
SR_ratio = 4
log1p_norm = True
z_norm = False
mean, std = [2.0152406e-08, 2.1581373e-07], [2.8560082e-05, 5.0738556e-05]
mean_log1p, std_log1p = [0.008315503, 0.0028762482], [0.5266841, 0.5418187]
#########################################################
HR_reduce_latitude = 0#107 # 15 deg from each pole
patch_size = None#(96, 96)
n_patches = 50
mode = 'train'
########################################################
f = h5py.File(infile, 'r', rdcc_nbytes=1000*1000*1000)
data = da.from_array(f['data'], chunks=(-1, 16 if shuffle else 256, -1, -1)) # CNHW layout
data = da.transpose(data, (1,2,3,0))
dtype = data.dtype
if dtype != np.float32:
print('WARNING: data will be saved as float32 but input ist float64!')
if mean is None:
arr = data
with ProgressBar():
mean, std = da.compute(arr.mean(axis=[0,1,2]), arr.std(axis=[0,1,2]), num_workers=8)
else:
mean, std = np.asarray(mean, dtype=dtype), np.asarray(std, dtype=dtype)
print('mean: {}, std: {}'.format(list(mean), list(std)))
if log1p_norm:
data_z_norm = (data-mean) / std
data_log1p = da.sign(data_z_norm) * da.log1p(da.fabs(data_z_norm))
if mean_log1p is None:
arr = data_log1p
with ProgressBar():
mean_log1p, std_log1p = da.compute(arr.mean(axis=[0,1,2]), arr.std(axis=[0,1,2]), num_workers=8)
else:
mean_log1p, std_log1p = np.asarray(mean_log1p, dtype=dtype), np.asarray(std_log1p, dtype=dtype)
print('mean_log1p: {}, std_log1p: {}'.format(list(mean_log1p), list(std_log1p)))
data = data_log1p
elif z_norm:
data = (data-mean) / std
if shuffle:
block_indices = np.random.permutation(data.numblocks[0])
else:
block_indices = np.arange(data.numblocks[0])
file_blocks = np.array_split(block_indices, n_files)
i = 0
for n, indices in enumerate(file_blocks):
if n_files > 1:
name = outfile.format(n)
else:
name = outfile
with tf.io.TFRecordWriter(name, options='ZLIB' if gzip else None) as writer:
for block_idx in indices:
block = data.blocks[block_idx].compute()
if shuffle:
block = np.random.permutation(block)
if HR_reduce_latitude:
lat_start = HR_reduce_latitude//2
block = block[:, lat_start:(-lat_start),:, :]
generate_TFRecords(writer, block, SR_ratio, mode, patch_size, n_patches)
i += 1
print('{} / {}'.format(i, data.numblocks[0]), flush=True)
def _apply(func,
datasets,
chunk=CHUNK,
pad=None,
relabel=False,
stack=False,
compute=True,
out=None,
normalize=False,
**kwargs):
"""
Appplies a function to a given set of datasets. Wraps a standard
function call of the form:
func(*datasets, **kwargs)
Named parameters gives extra functionality.
Parameters
----------
func: callable
Function to be mapped across datasets.
datasets: list of numpy array-like
Input datasets.
chunk: boolean
If `True` then input datasets will be assumed tobe `Dask.Array`s and
the function will be mapped across arrays blocks.
pad: None, int or iterable
The padding to apply (only if `chunk = True`). If `pad != None` then
`dask.array.ghost.map_overlap` will be used to map the function across
overlapping blocks, otherwise `dask.array.map_blocks` will be used.
relabel: boolean
Some of the labelling functions will yield local labelling if `chunk=True`.
If `func` is a labelling function, set `relabel = True` to map the result
for global consistency. See `survos2.improc.utils.dask_relabel_chunks` for
more details.
compute: boolean
If `True` the result will be computed and returned in numpy array form,
otherwise a `dask.delayed` will be returned if `chunk = True`.
out: None or numpy array-like
if `out != None` then the result will be stored in there.
**kwargs: other keyword arguments
Arguments to be passed to `func`.
Returns
-------
result: numpy array-like
The computed result if `compute = True` or `chunk = False`, the result
of the lazy wrapping otherwise.
"""
if stack and len(datasets) > 1:
dataset = da.stack(datasets, axis=0)
dataset = da.rechunk(dataset,
chunks=(dataset.shape[0], ) + dataset.chunks[1:])
datasets = [dataset]
if chunk == True:
kwargs.setdefault('dtype', out.dtype if out else datasets[0].dtype)
kwargs.setdefault('drop_axis', 0 if stack else None)
if pad is None or pad == False:
result = da.map_blocks(func, *datasets, **kwargs)
elif len(datasets) == 1:
if np.isscalar(pad):
pad = [pad] * datasets[0].ndim
if stack:
pad[0] = 0 # don't pad feature channel
depth = {i: d for i, d in enumerate(pad)}
trim = {i: d for i, d in enumerate(pad[1:])}
else:
depth = trim = {i: d for i, d in enumerate(pad)}
g = da.ghost.ghost(datasets[0], depth=depth, boundary='reflect')
r = g.map_blocks(func, **kwargs)
result = da.ghost.trim_internal(r, trim)
else:
raise ValueError('`pad` only works with single')
rchunks = result.chunks
if not relabel and normalize:
result = result / da.nanmax(da.fabs(result))
if out is not None:
result.store(out, compute=True)
elif compute:
result = result.compute()
if relabel:
if out is not None:
result = dask_relabel_chunks(da.from_array(out,
chunks=rchunks))
result.store(out, compute=True)
else:
result = dask_relabel_chunks(
da.from_array(result, chunks=rchunks))
if compute:
result = result.compute()
else:
result = func(*datasets, **kwargs)
if out is not None:
out[...] = result
if out is None:
return result
