def assignMprops(self):
#assign static, figure out dynamic assignment later
static = True
if (static):
for units in self.mdata["UNITS"].keys():
#load up the current grid obj
print(units)
vp = da.from_array(self.gridObj.data["/points/vp"],
chunks=(self.gridObj.pointx.chunksize))
vs = da.from_array(self.gridObj.data["/points/vs"],
chunks=(self.gridObj.pointx.chunksize))
p = da.from_array(self.gridObj.data["/points/p"],
chunks=(self.gridObj.pointx.chunksize))
qp = da.from_array(self.gridObj.data["/points/qp"],
chunks=(self.gridObj.pointx.chunksize))
qs = da.from_array(self.gridObj.data["/points/qs"],
chunks=(self.gridObj.pointx.chunksize))
#where is there a valid unit that matches the current unit or one that hasnt been assigned?
unitAssignments = da.where(
da.logical_and(self.gridObj.pointunit == int(units),
vp == -666), True, False)
#note that unit assignment is NOT a list of indexes as you would normally expect, it behaves more like
# a np.where operator which is why I need to reset it before each assignment
vp[unitAssignments] = self.mdata["UNITS"][units]["VP"]
unitAssignments = da.where(
da.logical_and(self.gridObj.pointunit == int(units),
vs == -666), True, False)
vs[unitAssignments] = self.mdata["UNITS"][units]["VS"]
unitAssignments = da.where(
da.logical_and(self.gridObj.pointunit == int(units),
p == -666), True, False)
p[unitAssignments] = self.mdata["UNITS"][units]["P"]
unitAssignments = da.where(
da.logical_and(self.gridObj.pointunit == int(units),
qp == -666), True, False)
qp[unitAssignments] = self.mdata["UNITS"][units]["QP"]
unitAssignments = da.where(
da.logical_and(self.gridObj.pointunit == int(units),
qs == -666), True, False)
qs[unitAssignments] = self.mdata["UNITS"][units]["QS"]
#save all of the changes to the arrays (for this cycle)
da.to_hdf5(self.gridObj.fname, "/points/vp", vp)
da.to_hdf5(self.gridObj.fname, "/points/vs", vs)
da.to_hdf5(self.gridObj.fname, "/points/p", p)
da.to_hdf5(self.gridObj.fname, "/points/qp", qp)
da.to_hdf5(self.gridObj.fname, "/points/qs", qs)
def _do_ctp_validation(data, adef, out_size, idxs):
""" Calculate CTP validation (included in CTTH plot). """
# detected ctth mask
detected_clouds = da.logical_and(data['caliop_cma'] == 1,
data['imager_cma'] == 1)
detected_height = da.logical_and(detected_clouds,
np.isfinite(data['imager_cth']))
# find pps low and caliop low
low_clouds_c = gc.get_calipso_low_clouds(data['caliop_cflag'])
detected_low_c = np.logical_and(detected_height, low_clouds_c)
low_clouds_pps = da.where(data['imager_ctp'] > 680., 1, 0)
detected_low_pps = da.logical_and(detected_height, low_clouds_pps)
# pattern: CALIOP_SEVIRI
cld_cld_a = da.logical_and(detected_low_c == 1, detected_low_pps == 1)
clr_cld_b = da.logical_and(detected_low_c == 0, detected_low_pps == 1)
cld_clr_c = da.logical_and(detected_low_c == 1, detected_low_pps == 0)
clr_clr_d = da.logical_and(detected_low_c == 0, detected_low_pps == 0)
cld_cld_a = cld_cld_a.astype(np.int64)
clr_cld_b = clr_cld_b.astype(np.int64)
cld_clr_c = cld_clr_c.astype(np.int64)
clr_clr_d = clr_clr_d.astype(np.int64)
a, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=cld_cld_a,
density=False)
b, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=clr_cld_b,
density=False)
c, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=cld_clr_c,
density=False)
d, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=clr_clr_d,
density=False)
scu = ScoreUtils(a, b, c, d)
scores = dict()
scores['CTP low clouds POD'] = [
scu.pod_1().reshape(adef.shape), 0, 1, 'rainbow'
]
scores['CTP low clouds FAR'] = [
scu.far_1().reshape(adef.shape), 0, 1, 'rainbow'
]
scores['CTP low clouds POFD'] = [
scu.pofd_1().reshape(adef.shape), 0, 1, 'rainbow'
]
# scores['Heidke low clouds'] = [scu.heidke().reshape(adef.shape),0, 1, 'rainbow']
return scores
def do_ctp_validation(data, adef, out_size, idxs):
""" Scores: low clouds detection """
# detected ctth mask
detected_clouds = da.logical_and(data['caliop_cma'] == 1,
data['imager_cma'] == 1)
detected_height = da.logical_and(detected_clouds,
np.isfinite(data['imager_cth']))
# find pps low and caliop low
low_clouds_c = get_calipso_low_clouds(data['caliop_cflag'])
detected_low_c = np.logical_and(detected_height, low_clouds_c)
low_clouds_pps = da.where(data['imager_ctp'] > 680., 1, 0)
detected_low_pps = da.logical_and(detected_height, low_clouds_pps)
# pattern: CALIOP_SEVIRI
cld_cld_a = da.logical_and(detected_low_c == 1, detected_low_pps == 1)
clr_cld_b = da.logical_and(detected_low_c == 0, detected_low_pps == 1)
cld_clr_c = da.logical_and(detected_low_c == 1, detected_low_pps == 0)
clr_clr_d = da.logical_and(detected_low_c == 0, detected_low_pps == 0)
cld_cld_a = cld_cld_a.astype(np.int64)
clr_cld_b = clr_cld_b.astype(np.int64)
cld_clr_c = cld_clr_c.astype(np.int64)
clr_clr_d = clr_clr_d.astype(np.int64)
a, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=cld_cld_a,
density=False)
b, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=clr_cld_b,
density=False)
c, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=cld_clr_c,
density=False)
d, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=clr_clr_d,
density=False)
# n = a + b + c + d
# n2d = N.reshape(adef.shape)
# scores = [hitrate(a, d, n).reshape(adef.shape),
# 0.7, 1, 'rainbow'] # hitrate low PPS
pod_low = a / (a + c)
far_low = c / (a + c)
scores = dict()
scores['POD low clouds'] = [pod_low.reshape(adef.shape), 0.2, 1, 'rainbow']
scores['FAR low clouds'] = [far_low.reshape(adef.shape), 0.2, 1, 'rainbow']
return scores
def mask_out_bad_beams(self,
threshold,
reference_beam=None,
criteria=['sr', 'major', 'minor'],
mid_value=np.nanmedian):
"""
See `identify_bad_beams`. This function returns a masked cube
Returns
-------
newcube : VaryingResolutionSpectralCube
The cube with bad beams masked out
"""
goodbeams = self.identify_bad_beams(threshold=threshold,
reference_beam=reference_beam,
criteria=criteria,
mid_value=mid_value)
includemask = BooleanArrayMask(goodbeams[:, None, None],
self._wcs,
shape=self._data.shape)
use_dask = isinstance(self._data, da.Array)
if use_dask:
newmask = da.logical_and(self._mask_include, includemask)
elif self.mask is None:
newmask = includemask
else:
newmask = np.bitwise_and(self.mask, includemask)
return self._new_thing_with(
mask=newmask,
beam_threshold=threshold,
goodbeams_mask=np.bitwise_and(self.goodbeams_mask, goodbeams),
)
def test_arithmetic():
x = np.arange(5).astype('f4') + 2
y = np.arange(5).astype('i8') + 2
z = np.arange(5).astype('i4') + 2
a = da.from_array(x, chunks=(2,))
b = da.from_array(y, chunks=(2,))
c = da.from_array(z, chunks=(2,))
assert eq(a + b, x + y)
assert eq(a * b, x * y)
assert eq(a - b, x - y)
assert eq(a / b, x / y)
assert eq(b & b, y & y)
assert eq(b | b, y | y)
assert eq(b ^ b, y ^ y)
assert eq(a // b, x // y)
assert eq(a ** b, x ** y)
assert eq(a % b, x % y)
assert eq(a > b, x > y)
assert eq(a < b, x < y)
assert eq(a >= b, x >= y)
assert eq(a <= b, x <= y)
assert eq(a == b, x == y)
assert eq(a != b, x != y)
assert eq(a + 2, x + 2)
assert eq(a * 2, x * 2)
assert eq(a - 2, x - 2)
assert eq(a / 2, x / 2)
assert eq(b & True, y & True)
assert eq(b | True, y | True)
assert eq(b ^ True, y ^ True)
assert eq(a // 2, x // 2)
assert eq(a ** 2, x ** 2)
assert eq(a % 2, x % 2)
assert eq(a > 2, x > 2)
assert eq(a < 2, x < 2)
assert eq(a >= 2, x >= 2)
assert eq(a <= 2, x <= 2)
assert eq(a == 2, x == 2)
assert eq(a != 2, x != 2)
assert eq(2 + b, 2 + y)
assert eq(2 * b, 2 * y)
assert eq(2 - b, 2 - y)
assert eq(2 / b, 2 / y)
assert eq(True & b, True & y)
assert eq(True | b, True | y)
assert eq(True ^ b, True ^ y)
assert eq(2 // b, 2 // y)
assert eq(2 ** b, 2 ** y)
assert eq(2 % b, 2 % y)
assert eq(2 > b, 2 > y)
assert eq(2 < b, 2 < y)
assert eq(2 >= b, 2 >= y)
assert eq(2 <= b, 2 <= y)
assert eq(2 == b, 2 == y)
assert eq(2 != b, 2 != y)
assert eq(-a, -x)
assert eq(abs(a), abs(x))
assert eq(~(a == b), ~(x == y))
assert eq(~(a == b), ~(x == y))
assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
assert eq(da.exp(b), np.exp(y))
assert eq(da.log(a), np.log(x))
assert eq(da.log10(a), np.log10(x))
assert eq(da.log1p(a), np.log1p(x))
assert eq(da.expm1(b), np.expm1(y))
assert eq(da.sqrt(a), np.sqrt(x))
assert eq(da.square(a), np.square(x))
assert eq(da.sin(a), np.sin(x))
assert eq(da.cos(b), np.cos(y))
assert eq(da.tan(a), np.tan(x))
assert eq(da.arcsin(b/10), np.arcsin(y/10))
assert eq(da.arccos(b/10), np.arccos(y/10))
assert eq(da.arctan(b/10), np.arctan(y/10))
assert eq(da.arctan2(b*10, a), np.arctan2(y*10, x))
assert eq(da.hypot(b, a), np.hypot(y, x))
assert eq(da.sinh(a), np.sinh(x))
assert eq(da.cosh(b), np.cosh(y))
assert eq(da.tanh(a), np.tanh(x))
assert eq(da.arcsinh(b*10), np.arcsinh(y*10))
assert eq(da.arccosh(b*10), np.arccosh(y*10))
assert eq(da.arctanh(b/10), np.arctanh(y/10))
assert eq(da.deg2rad(a), np.deg2rad(x))
assert eq(da.rad2deg(a), np.rad2deg(x))
assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))
assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
assert eq(da.isfinite(a), np.isfinite(x))
assert eq(da.isinf(a), np.isinf(x))
assert eq(da.isnan(a), np.isnan(x))
assert eq(da.signbit(a - 3), np.signbit(x - 3))
assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
assert eq(da.ldexp(c, c), np.ldexp(z, z))
assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
assert eq(da.ceil(a), np.ceil(x))
assert eq(da.trunc(a / 2), np.trunc(x / 2))
assert eq(da.degrees(b), np.degrees(y))
assert eq(da.radians(a), np.radians(x))
assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
assert eq(da.fix(a - 2.5), np.fix(x - 2.5))
assert eq(da.angle(a + 1j), np.angle(x + 1j))
assert eq(da.real(a + 1j), np.real(x + 1j))
assert eq((a + 1j).real, np.real(x + 1j))
assert eq(da.imag(a + 1j), np.imag(x + 1j))
assert eq((a + 1j).imag, np.imag(x + 1j))
assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
assert eq((a + 1j * b).conj(), (x + 1j * y).conj())
assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
assert eq(da.fabs(b), np.fabs(y))
assert eq(da.sign(b - 2), np.sign(y - 2))
l1, l2 = da.frexp(a)
r1, r2 = np.frexp(x)
assert eq(l1, r1)
assert eq(l2, r2)
l1, l2 = da.modf(a)
r1, r2 = np.modf(x)
assert eq(l1, r1)
assert eq(l2, r2)
assert eq(da.around(a, -1), np.around(x, -1))
def do_ctth_validation(data, resampler, thrs=10):
""" thrs: threshold value for filtering boxes with small number of obs """
# mask of detected ctth
detected_clouds = da.logical_and(data['caliop_cma'] == 1,
data['imager_cma'] == 1)
detected_height = da.logical_and(detected_clouds,
np.isfinite(data['imager_cth']))
detected_temperature = np.logical_and(detected_clouds,
np.isfinite(data['imager_ctt']))
detected_height_mask = detected_height.astype(int)
# calculate bias and mea for all ctth cases
delta_h = data['imager_cth'] - data['caliop_cth'] # HEIGHT
height_bias = np.where(detected_height, delta_h, np.nan)
mae = np.abs(height_bias)
delta_t = data['imager_ctt'] - data['caliop_ctt'] # TEMPERATURE
temperature_bias = np.where(detected_temperature, delta_t, np.nan)
# clouds levels (from calipso 'cloud type')
low_clouds = get_calipso_low_clouds(data['caliop_cflag'])
detected_low = np.logical_and(detected_height, low_clouds)
bias_low = np.where(detected_low, height_bias, np.nan)
bias_temperature_low = np.where(detected_low, temperature_bias, np.nan)
mid_clouds = get_calipso_medium_clouds(data['caliop_cflag'])
detected_mid = np.logical_and(detected_height, mid_clouds)
bias_mid = np.where(detected_mid, height_bias, np.nan)
high_clouds = get_calipso_high_clouds(data['caliop_cflag'])
detected_high = np.logical_and(detected_height, high_clouds)
bias_high = np.where(detected_high, height_bias, np.nan)
# opaque/transparent clouds (from calipso 'cloud type')
# tp_clouds = get_calipso_tp(data['caliop_cflag'])
# detected_tp = np.logical_and(detected_height,tp_clouds)
# bias_tp = np.where(detected_tp, height_bias, np.nan)
# op_clouds = get_calipso_op(data['caliop_cflag'])
# detected_op = np.logical_and(detected_height,op_clouds)
# bias_op = np.where(detected_op, height_bias, np.nan)
# low+opaque, mid/high+transparent
mid_high_tp_clouds = get_calipso_medium_and_high_clouds_tp(
data['caliop_cflag'])
detected_mid_high_tp = np.logical_and(detected_height, mid_high_tp_clouds)
bias_mid_high_tp = np.where(detected_mid_high_tp, height_bias, np.nan)
low_op_clouds = get_calipso_low_clouds_op(data['caliop_cflag'])
detected_low_op = np.logical_and(detected_height, low_op_clouds)
bias_low_op = np.where(detected_low_op, height_bias, np.nan)
# resample and filter some data out
# N = resampler.get_count()
n_matched_cases = resampler.get_sum(detected_height_mask)
sev_cth_average = resampler.get_average(data['imager_cth'])
cal_cth_average = resampler.get_average(data['caliop_cth'])
bias_average = resampler.get_average(height_bias)
bias_average = np.where(n_matched_cases < thrs, np.nan, bias_average)
mae_average = resampler.get_average(mae)
mae_average = np.where(n_matched_cases < thrs, np.nan, mae_average)
bias_temperature_average = resampler.get_average(temperature_bias)
bias_temperature_average = np.where(n_matched_cases < thrs, np.nan,
bias_temperature_average)
n_matched_cases_low = resampler.get_sum(detected_low.astype(int))
bias_low_average = resampler.get_average(bias_low)
bias_low_average = np.where(n_matched_cases_low < thrs, np.nan,
bias_low_average)
bias_temperature_low_average = resampler.get_average(bias_temperature_low)
bias_temperature_low_average = np.where(n_matched_cases_low < thrs, np.nan,
bias_temperature_low_average)
n_matched_cases_mid = resampler.get_sum(detected_mid.astype(int))
bias_mid_average = resampler.get_average(bias_mid)
bias_mid_average = np.where(n_matched_cases_mid < thrs, np.nan,
bias_mid_average)
n_matched_cases_high = resampler.get_sum(detected_high.astype(int))
bias_high_average = resampler.get_average(bias_high)
bias_high_average = np.where(n_matched_cases_high < thrs, np.nan,
bias_high_average)
# n_matched_cases_tp = resampler.get_sum(detected_tp.astype(int))
# bias_tp_average = resampler.get_average(bias_tp)
# bias_tp_average = np.where(n_matched_cases_tp<thrs,
# np.nan, bias_tp_average)
# n_matched_cases_op = resampler.get_sum(detected_op.astype(int))
# bias_op_average = resampler.get_average(bias_op)
# bias_op_average = np.where(n_matched_cases_op<thrs,
# np.nan, bias_op_average)
n_matched_cases_mid_high_tp = resampler.get_sum(
detected_mid_high_tp.astype(int))
bias_mid_high_tp_average = resampler.get_average(bias_mid_high_tp)
bias_mid_high_tp_average = np.where(n_matched_cases_mid_high_tp < thrs,
np.nan, bias_mid_high_tp_average)
n_matched_cases_low_op = resampler.get_sum(detected_low_op.astype(int))
bias_low_op_average = resampler.get_average(bias_low_op)
bias_low_op_average = np.where(n_matched_cases_low_op < thrs, np.nan,
bias_low_op_average)
# calculate scores
scores = dict()
scores['Bias CTH'] = [bias_average, -4000, 4000, 'bwr']
scores['MAE CTH'] = [mae_average, 0, 2500, 'Reds']
scores['Bias low'] = [bias_low_average, -2000, 2000, 'bwr']
scores['Bias middle'] = [bias_mid_average, -2000, 2000, 'bwr']
scores['Bias high'] = [bias_high_average, -6000, 6000, 'bwr']
# scores['Bias opaque'] = [bias_op_average, -4000, 4000, 'bwr']
# scores['Bias transparent'] = [bias_tp_average, -4000, 4000, 'bwr']
scores['Bias low opaque'] = [bias_low_op_average, -2000, 2000, 'bwr']
scores['Bias mid+high transparent'] = [
bias_mid_high_tp_average, -6000, 6000, 'bwr'
]
scores['Bias temperature'] = [bias_temperature_average, -30, 30, 'bwr']
scores['Bias temperature low'] = [
bias_temperature_low_average, -10, 10, 'bwr'
]
scores['CALIOP CTH mean'] = [cal_cth_average, 1000, 14000, 'rainbow']
scores['SEVIRI CTH mean'] = [sev_cth_average, 1000, 14000, 'rainbow']
scores['Num_detected_height'] = [n_matched_cases, None, None, 'rainbow']
# addit_scores = do_ctp_validation(data, adef, out_size, idxs)
# scores.update(addit_scores)
# scores['N_matched_cases_low'] = [N_matched_cases_low,
# None, None, 'rainbow']
# scores['N_matched_cases_middle'] = [N_matched_cases_mid,
# None, None, 'rainbow']
# scores['N_matched_cases_high'] = [N_matched_cases_high,
# None, None, 'rainbow']
return scores
def do_cph_validation(data, adef, out_size, idxs):
cal_cph = data['caliop_cph']
img_cph = data['imager_cph']
# pattern: CALIOP_SEVIRI
# get counts for contigency table
ice_ice_a = da.logical_and(cal_cph == 1, img_cph == 1)
liq_ice_b = da.logical_and(cal_cph == 0, img_cph == 1)
ice_liq_c = da.logical_and(cal_cph == 1, img_cph == 0)
liq_liq_d = da.logical_and(cal_cph == 0, img_cph == 0)
ice_ice_a = ice_ice_a.astype(np.int64)
liq_ice_b = liq_ice_b.astype(np.int64)
ice_liq_c = ice_liq_c.astype(np.int64)
liq_liq_d = liq_liq_d.astype(np.int64)
# use histogram functionality to get contigency table summed up for every
# grid box in target grid
a, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=ice_ice_a,
density=False)
b, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=liq_ice_b,
density=False)
c, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=ice_liq_c,
density=False)
d, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=liq_liq_d,
density=False)
n = a + b + c + d
n2d = n.reshape(adef.shape)
# calculate scores
scores = dict()
scores['Hitrate'] = [
hitrate(a, d, n).reshape(adef.shape), 0.5, 1, 'rainbow'
]
scores['PODliq'] = [pod_clr(b, d).reshape(adef.shape), 0.5, 1, 'rainbow']
scores['PODice'] = [pod_cld(a, c).reshape(adef.shape), 0.5, 1, 'rainbow']
scores['FARliq'] = [far_clr(c, d).reshape(adef.shape), 0, 1, 'rainbow']
scores['FARice'] = [far_cld(a, b).reshape(adef.shape), 0, 1, 'rainbow']
scores['POFDliq'] = [pofd_clr(a, c).reshape(adef.shape), 0, 1, 'rainbow']
scores['POFDice'] = [pofd_cld(b, d).reshape(adef.shape), 0, 1, 'rainbow']
scores['Heidke'] = [
heidke(a, b, c, d).reshape(adef.shape), 0, 1, 'rainbow'
]
scores['Kuiper'] = [
kuiper(a, b, c, d).reshape(adef.shape), 0, 1, 'rainbow'
]
scores['Bias'] = [bias(b, c, n).reshape(adef.shape), 0, 1, 'bwr']
scores['CALIOP mean'] = [
mean(a, c, n).reshape(adef.shape), None, None, 'rainbow'
]
scores['SEVIRI mean'] = [
mean(a, b, n).reshape(adef.shape), None, None, 'rainbow'
]
scores['Nobs'] = [n2d, None, None, 'rainbow']
scores['Bias'][2] = np.nanmax(np.abs(scores['Bias'][0])) / 2
scores['Bias'][1] = scores['Bias'][2] * (-1)
return scores
def do_cma_validation(data, adef, out_size, idxs):
cal_cma = data['caliop_cma']
img_cma = data['imager_cma']
# pattern: CALIOP_SEVIRI
cld_cld_a = da.logical_and(cal_cma == 1, img_cma == 1)
clr_cld_b = da.logical_and(cal_cma == 0, img_cma == 1)
cld_clr_c = da.logical_and(cal_cma == 1, img_cma == 0)
clr_clr_d = da.logical_and(cal_cma == 0, img_cma == 0)
cld_cld_a = cld_cld_a.astype(np.int64)
clr_cld_b = clr_cld_b.astype(np.int64)
cld_clr_c = cld_clr_c.astype(np.int64)
clr_clr_d = clr_clr_d.astype(np.int64)
a, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=cld_cld_a,
density=False)
b, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=clr_cld_b,
density=False)
c, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=cld_clr_c,
density=False)
d, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=clr_clr_d,
density=False)
n = a + b + c + d
n2d = n.reshape(adef.shape)
scores = dict()
scores['Hitrate'] = [
hitrate(a, d, n).reshape(adef.shape), 0.5, 1, 'rainbow'
]
scores['PODclr'] = [pod_clr(b, d).reshape(adef.shape), 0.5, 1, 'rainbow']
scores['PODcld'] = [pod_cld(a, c).reshape(adef.shape), 0.5, 1, 'rainbow']
scores['FARclr'] = [far_clr(c, d).reshape(adef.shape), 0, 1, 'rainbow']
scores['FARcld'] = [far_cld(a, b).reshape(adef.shape), 0, 1, 'rainbow']
scores['POFDclr'] = [pofd_clr(a, c).reshape(adef.shape), 0, 1, 'rainbow']
scores['POFDcld'] = [pofd_cld(b, d).reshape(adef.shape), 0, 1, 'rainbow']
scores['Heidke'] = [
heidke(a, b, c, d).reshape(adef.shape), 0, 1, 'rainbow'
]
scores['Kuiper'] = [
kuiper(a, b, c, d).reshape(adef.shape), 0, 1, 'rainbow'
]
scores['Bias'] = [bias(b, c, n).reshape(adef.shape), 0, 1, 'bwr']
scores['CALIOP mean'] = [
mean(a, c, n).reshape(adef.shape), None, None, 'rainbow'
]
scores['SEVIRI mean'] = [
mean(a, b, n).reshape(adef.shape), None, None, 'rainbow'
]
scores['Nobs'] = [n2d, None, None, 'rainbow']
scores['Bias'][2] = np.nanmax(np.abs(scores['Bias'][0])) / 2
scores['Bias'][1] = scores['Bias'][2] * (-1)
return scores
def get_collocated_file_info(ipath,
chunksize,
dnt='ALL',
satz_lim=None,
dataset='CCI'):
file = h5py.File(ipath, 'r')
caliop = file['calipso']
if dataset == 'CCI':
imager = file['cci']
elif dataset == 'CLAAS':
imager = file['pps']
else:
raise Exception('Dataset {} not known!'.format(dataset))
# get CTH and CTT
sev_cth = da.from_array(get_imager_cth(imager), chunks=chunksize)
cal_cth = da.from_array(get_caliop_cth(caliop), chunks=chunksize)
sev_ctt = da.from_array(get_imager_ctt(imager), chunks=chunksize)
cal_ctt = da.from_array(get_caliop_ctt(caliop), chunks=chunksize)
cal_cflag = np.array(caliop['feature_classification_flags'][::, 0])
# ctp_c = np.array(caliop['layer_top_pressure'])[:,0]
# ctp_c = np.where(ctp_c == -9999, np.nan,ctp_c)
# ctp_pps = np.array(imager['ctth_pressure'])
# ctp_pps = np.where(ctp_pps==-9, np.nan, ctp_pps)
# sev_ctp = da.from_array(ctp_pps, chunks=(chunksize))
# cal_ctp = da.from_array(ctp_c, chunks=(chunksize))
# get CMA, CPH, VZA, SZA, LAT and LON
sev_cph = da.from_array(get_imager_cph(imager), chunks=chunksize)
cal_cph = da.from_array(get_caliop_cph(caliop), chunks=chunksize)
cal_cma = da.from_array(get_caliop_cma(caliop), chunks=chunksize)
sev_cma = da.from_array(get_imager_cma(imager), chunks=chunksize)
satz = da.from_array(imager['satz'], chunks=chunksize)
sunz = da.from_array(imager['sunz'], chunks=chunksize)
lat = da.from_array(imager['latitude'], chunks=chunksize)
lon = da.from_array(imager['longitude'], chunks=chunksize)
# mask satellize zenith angle
if satz_lim is not None:
mask = satz > satz_lim
cal_cma = da.where(mask, np.nan, cal_cma)
sev_cma = da.where(mask, np.nan, sev_cma)
cal_cph = da.where(mask, np.nan, cal_cph)
sev_cph = da.where(mask, np.nan, sev_cph)
cal_cth = da.where(mask, np.nan, cal_cth)
sev_cth = da.where(mask, np.nan, sev_cth)
# cal_ctp = da.where(mask, np.nan, cal_ctt)
# sev_ctp = da.where(mask, np.nan, sev_ctt)
# mask all pixels except daytime
if dnt == 'DAY':
mask = sunz >= 80
cal_cma = da.where(mask, np.nan, cal_cma)
sev_cma = da.where(mask, np.nan, sev_cma)
cal_cph = da.where(mask, np.nan, cal_cph)
sev_cph = da.where(mask, np.nan, sev_cph)
cal_cth = da.where(mask, np.nan, cal_cth)
sev_cth = da.where(mask, np.nan, sev_cth)
# cal_ctp = da.where(mask, np.nan, cal_ctt)
# sev_ctp = da.where(mask, np.nan, sev_ctt)
# mask all pixels except nighttime
elif dnt == 'NIGHT':
mask = sunz <= 95
cal_cma = da.where(mask, np.nan, cal_cma)
sev_cma = da.where(mask, np.nan, sev_cma)
cal_cph = da.where(mask, np.nan, cal_cph)
sev_cph = da.where(mask, np.nan, sev_cph)
cal_cth = da.where(mask, np.nan, cal_cth)
sev_cth = da.where(mask, np.nan, sev_cth)
# cal_ctp = da.where(mask, np.nan, cal_ctt)
# sev_ctp = da.where(mask, np.nan, sev_ctt)
# mask all pixels except twilight
elif dnt == 'TWILIGHT':
mask = ~da.logical_and(sunz > 80, sunz < 95)
cal_cma = da.where(mask, np.nan, cal_cma)
sev_cma = da.where(mask, np.nan, sev_cma)
cal_cph = da.where(mask, np.nan, cal_cph)
sev_cph = da.where(mask, np.nan, sev_cph)
cal_cth = da.where(mask, np.nan, cal_cth)
sev_cth = da.where(mask, np.nan, sev_cth)
# cal_ctp = da.where(mask, np.nan, cal_ctt)
# sev_ctp = da.where(mask, np.nan, sev_ctt)
elif dnt == 'ALL':
pass
else:
raise Exception('DNT option ', dnt, ' is invalid.')
data = {
'caliop_cma': cal_cma,
'imager_cma': sev_cma,
'caliop_cph': cal_cph,
'imager_cph': sev_cph,
'satz': satz,
'sunz': sunz,
'caliop_cth': cal_cth,
'imager_cth': sev_cth,
'caliop_ctt': cal_ctt,
'imager_ctt': sev_ctt,
'caliop_cflag': cal_cflag
}
latlon = {'lat': lat, 'lon': lon}
return data, latlon
def _get_calipso_matchup_file_content(ipath,
chunksize,
dnt='ALL',
satz_lim=None,
dataset='CCI'):
""" Obtain variables from atrain_match HDF5 matchup file. """
file = h5py.File(ipath, 'r')
caliop = file['calipso']
if dataset == 'CCI':
imager = file['cci']
elif dataset == 'CLAAS':
imager = file['pps']
else:
raise Exception('Dataset {} not known!'.format(dataset))
# get CTH, CTT, CTP
sev_cth = da.from_array(gi.get_imager_cth(imager), chunks=chunksize)
cal_cth = da.from_array(gc.get_caliop_cth(caliop), chunks=chunksize)
sev_ctt = da.from_array(gi.get_imager_ctt(imager), chunks=chunksize)
cal_ctt = da.from_array(gc.get_caliop_ctt(caliop), chunks=chunksize)
cal_cflag = np.array(caliop['feature_classification_flags'][::, 0])
if filter_stratospheric:
tropo_height = da.from_array(gc.get_tropopause_height(caliop),
chunks=chunksize)
sev_ctp = da.from_array(gi.get_imager_ctp(imager), chunks=(chunksize))
cal_ctp = da.from_array(gc.get_caliop_ctp(caliop), chunks=(chunksize))
if dataset == 'CCI':
sev_quality = da.from_array(imager['cpp_quality'][:].astype(int),
chunks=chunksize)
bad_quality_imager = np.bitwise_and(sev_quality, 3) > 0
sev_cth[bad_quality_imager] = np.nan
sev_ctt[bad_quality_imager] = np.nan
# get CMA, CPH, VZA, SZA, LAT and LON
sev_cph = da.from_array(gi.get_imager_cph(imager), chunks=chunksize)
cal_cph = da.from_array(gc.get_caliop_cph(caliop), chunks=chunksize)
cal_cma = da.from_array(gc.get_caliop_cma(caliop), chunks=chunksize)
sev_cma = da.from_array(gi.get_imager_cma(imager), chunks=chunksize)
satz = da.from_array(imager['satz'], chunks=chunksize)
sunz = da.from_array(imager['sunz'], chunks=chunksize)
lat = da.from_array(imager['latitude'], chunks=chunksize)
lon = da.from_array(imager['longitude'], chunks=chunksize)
# mask fill values for angles
satz = np.where(satz == -9, np.nan, satz)
sunz = np.where(sunz == -9, np.nan, sunz)
# mask satellize zenith angle
if satz_lim is not None:
mask = satz > satz_lim
cal_cma = da.where(mask, np.nan, cal_cma)
sev_cma = da.where(mask, np.nan, sev_cma)
cal_cph = da.where(mask, np.nan, cal_cph)
sev_cph = da.where(mask, np.nan, sev_cph)
cal_cth = da.where(mask, np.nan, cal_cth)
sev_cth = da.where(mask, np.nan, sev_cth)
cal_ctt = da.where(mask, np.nan, cal_ctt)
sev_ctt = da.where(mask, np.nan, sev_ctt)
cal_ctp = da.where(mask, np.nan, cal_ctp)
sev_ctp = da.where(mask, np.nan, sev_ctp)
# mask all pixels except daytime
if dnt == 'DAY':
mask = sunz >= 80
# mask all pixels except nighttime
elif dnt == 'NIGHT':
mask = sunz <= 95
# mask all pixels except twilight
elif dnt == 'TWILIGHT':
mask = ~da.logical_and(sunz > 80, sunz < 95)
# no masking
elif dnt == 'ALL':
mask = None
else:
raise Exception('DNT option ', dnt, ' is invalid.')
# filter stratospheric clouds in CALIPSO
if filter_stratospheric:
strato_mask = cal_cth > tropo_height
cal_cma = da.where(strato_mask, np.nan, cal_cma)
cal_cph = da.where(strato_mask, np.nan, cal_cph)
cal_cth = da.where(strato_mask, np.nan, cal_cth)
cal_ctt = da.where(strato_mask, np.nan, cal_ctt)
cal_ctp = da.where(strato_mask, np.nan, cal_ctp)
# apply DNT masking
masked = _apply_dnt_mask(cal_cma, sev_cma, cal_cph, sev_cph, cal_cth,
sev_cth, cal_ctt, sev_ctt, cal_ctp, sev_ctp, mask)
data = {
'caliop_cma': masked[0],
'imager_cma': masked[1],
'caliop_cph': masked[2],
'imager_cph': masked[3],
'satz': satz,
'sunz': sunz,
'caliop_cth': masked[4],
'imager_cth': masked[5],
'caliop_ctt': masked[6],
'imager_ctt': masked[7],
'caliop_ctp': masked[8],
'imager_ctp': masked[9],
'caliop_cflag': cal_cflag,
}
latlon = {'lat': lat, 'lon': lon}
return data, latlon
def _do_ctth_validation(data, adef, out_size, idxs, resampler, thrs=10):
"""
Calculate CTH and CTT bias. thrs: threshold value for filtering
boxes with small number of obs.
"""
# mask of detected ctth
detected_clouds = da.logical_and(data['caliop_cma'] == 1,
data['imager_cma'] == 1)
detected_height = da.logical_and(detected_clouds,
np.isfinite(data['imager_cth']))
detected_temperature = np.logical_and(detected_clouds,
np.isfinite(data['imager_ctt']))
detected_pressure = np.logical_and(detected_clouds,
np.isfinite(data['imager_ctp']))
detected_height_mask = detected_height.astype(int)
# calculate bias and mea for all ctth cases
delta_h = data['imager_cth'] - data['caliop_cth'] # HEIGHT
height_bias = np.where(detected_height, delta_h, np.nan)
delta_t = data['imager_ctt'] - data['caliop_ctt'] # TEMPERATURE
temperature_bias = np.where(detected_temperature, delta_t, np.nan)
delta_p = data['imager_ctp'] - data['caliop_ctp'] # PRESSURE
pressure_bias = np.where(detected_pressure, delta_p, np.nan)
# clouds levels (from calipso 'cloud type')
low_clouds = gc.get_calipso_low_clouds(data['caliop_cflag'])
detected_low = np.logical_and(detected_height, low_clouds)
bias_low = np.where(detected_low, height_bias, np.nan)
bias_temperature_low = np.where(detected_low, temperature_bias, np.nan)
mid_clouds = gc.get_calipso_medium_clouds(data['caliop_cflag'])
detected_mid = np.logical_and(detected_height, mid_clouds)
bias_mid = np.where(detected_mid, height_bias, np.nan)
high_clouds = gc.get_calipso_high_clouds(data['caliop_cflag'])
detected_high = np.logical_and(detected_height, high_clouds)
bias_high = np.where(detected_high, height_bias, np.nan)
# resample and filter some data out
n_matched_cases = resampler.get_sum(detected_height_mask)
sev_cth_average = resampler.get_average(data['imager_cth'])
cal_cth_average = resampler.get_average(data['caliop_cth'])
bias_average = resampler.get_average(height_bias)
bias_average = np.where(n_matched_cases < thrs, np.nan, bias_average)
bias_temperature_average = resampler.get_average(temperature_bias)
bias_temperature_average = np.where(n_matched_cases < thrs, np.nan,
bias_temperature_average)
bias_pressure_average = resampler.get_average(pressure_bias)
bias_pressure_average = np.where(n_matched_cases < thrs, np.nan,
bias_pressure_average)
n_matched_cases_low = resampler.get_sum(detected_low.astype(int))
bias_low_average = resampler.get_average(bias_low)
bias_low_average = np.where(n_matched_cases_low < thrs, np.nan,
bias_low_average)
n_matched_cases_mid = resampler.get_sum(detected_mid.astype(int))
bias_mid_average = resampler.get_average(bias_mid)
bias_mid_average = np.where(n_matched_cases_mid < thrs, np.nan,
bias_mid_average)
n_matched_cases_high = resampler.get_sum(detected_high.astype(int))
bias_high_average = resampler.get_average(bias_high)
bias_high_average = np.where(n_matched_cases_high < thrs, np.nan,
bias_high_average)
# calculate scores
scores = dict()
# [scores_on_target_grid, vmin, vmax, cmap]
scores['Bias CTH'] = [bias_average, -4000, 4000, 'bwr']
scores['Bias CTT'] = [bias_temperature_average, -30, 30, 'bwr']
scores['Bias CTP'] = [bias_pressure_average, -200, 200, 'bwr']
scores['Bias CTH low'] = [bias_low_average, -2000, 2000, 'bwr']
scores['Bias CTH middle'] = [bias_mid_average, -2000, 2000, 'bwr']
scores['Bias CTH high'] = [bias_high_average, -6000, 6000, 'bwr']
addit_scores = _do_ctp_validation(data, adef, out_size, idxs)
scores.update(addit_scores)
scores['CALIOP CTH mean'] = [cal_cth_average, 1000, 14000, 'rainbow']
scores['SEVIRI CTH mean'] = [sev_cth_average, 1000, 14000, 'rainbow']
scores['Num_detected_height'] = [n_matched_cases, None, None, 'rainbow']
return scores
def _do_cma_cph_validation(data, adef, out_size, idxs, variable):
""" Calculate scores for CMA or CPH depending on variable arg. """
# !!! for CPH '0' is water, '1' is ice !!!
if variable.lower() == 'cma':
cal = data['caliop_cma']
img = data['imager_cma']
elif variable.lower() == 'cph':
cal = data['caliop_cph']
img = data['imager_cph']
else:
msg = 'Variable {} for CMA/CPH validation unknown. [cma, cph]'
raise Exception(msg.format(variable))
# pattern: CALIOP_SEVIRI
a11 = da.logical_and(cal == 1, img == 1).astype(np.int64)
b01 = da.logical_and(cal == 0, img == 1).astype(np.int64)
c10 = da.logical_and(cal == 1, img == 0).astype(np.int64)
d00 = da.logical_and(cal == 0, img == 0).astype(np.int64)
a, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=a11,
density=False)
b, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=b01,
density=False)
c, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=c10,
density=False)
d, _ = da.histogram(idxs,
bins=out_size,
range=(0, out_size),
weights=d00,
density=False)
scu = ScoreUtils(a, b, c, d)
scores = dict()
# [scores_on_target_grid, vmin, vmax, cmap]
scores['Hitrate'] = [scu.hitrate().reshape(adef.shape), 0.5, 1, 'rainbow']
scores['PODclr'] = [scu.pod_0().reshape(adef.shape), 0.5, 1, 'rainbow']
if variable.lower() == 'cph': scores['PODwater'] = scores.pop('PODclr')
scores['PODcld'] = [scu.pod_1().reshape(adef.shape), 0.5, 1, 'rainbow']
if variable.lower() == 'cph': scores['PODice'] = scores.pop('PODcld')
scores['FARclr'] = [scu.far_0().reshape(adef.shape), 0, 1, 'rainbow']
if variable.lower() == 'cph': scores['FARwater'] = scores.pop('FARclr')
scores['FARcld'] = [scu.far_1().reshape(adef.shape), 0, 1, 'rainbow']
if variable.lower() == 'cph': scores['FARice'] = scores.pop('FARcld')
scores['POFDclr'] = [scu.pofd_0().reshape(adef.shape), 0, 1, 'rainbow']
if variable.lower() == 'cph': scores['POFDwater'] = scores.pop('POFDclr')
scores['POFDcld'] = [scu.pofd_1().reshape(adef.shape), 0, 1, 'rainbow']
if variable.lower() == 'cph': scores['POFDice'] = scores.pop('POFDcld')
scores['Heidke'] = [scu.heidke().reshape(adef.shape), 0, 1, 'rainbow']
scores['Kuiper'] = [scu.kuiper().reshape(adef.shape), 0, 1, 'rainbow']
scores['Bias'] = [scu.bias().reshape(adef.shape), 0, 1, 'bwr']
scores['CALIOP mean'] = [
scu.mean(a, c).reshape(adef.shape), None, None, 'rainbow'
]
scores['SEVIRI mean'] = [
scu.mean(a, b).reshape(adef.shape), None, None, 'rainbow'
]
scores['Nobs'] = [scu.n.reshape(adef.shape), None, None, 'rainbow']
# calculate bias limits for plotting
scores['Bias'][2] = np.nanmax(np.abs(scores['Bias'][0])) / 2
scores['Bias'][1] = scores['Bias'][2] * (-1)
# change mean values, bias
if variable.lower() == 'cph':
scores['CALIOP mean'][0] = scu.mean(d, b).reshape(adef.shape)
scores['SEVIRI mean'][0] = scu.mean(d, c).reshape(adef.shape)
scores['Bias'][0] = scores['Bias'][0] * (-1)
return scores
def test_arithmetic():
x = np.arange(5).astype('f4') + 2
y = np.arange(5).astype('i8') + 2
z = np.arange(5).astype('i4') + 2
a = da.from_array(x, chunks=(2, ))
b = da.from_array(y, chunks=(2, ))
c = da.from_array(z, chunks=(2, ))
assert eq(a + b, x + y)
assert eq(a * b, x * y)
assert eq(a - b, x - y)
assert eq(a / b, x / y)
assert eq(b & b, y & y)
assert eq(b | b, y | y)
assert eq(b ^ b, y ^ y)
assert eq(a // b, x // y)
assert eq(a**b, x**y)
assert eq(a % b, x % y)
assert eq(a > b, x > y)
assert eq(a < b, x < y)
assert eq(a >= b, x >= y)
assert eq(a <= b, x <= y)
assert eq(a == b, x == y)
assert eq(a != b, x != y)
assert eq(a + 2, x + 2)
assert eq(a * 2, x * 2)
assert eq(a - 2, x - 2)
assert eq(a / 2, x / 2)
assert eq(b & True, y & True)
assert eq(b | True, y | True)
assert eq(b ^ True, y ^ True)
assert eq(a // 2, x // 2)
assert eq(a**2, x**2)
assert eq(a % 2, x % 2)
assert eq(a > 2, x > 2)
assert eq(a < 2, x < 2)
assert eq(a >= 2, x >= 2)
assert eq(a <= 2, x <= 2)
assert eq(a == 2, x == 2)
assert eq(a != 2, x != 2)
assert eq(2 + b, 2 + y)
assert eq(2 * b, 2 * y)
assert eq(2 - b, 2 - y)
assert eq(2 / b, 2 / y)
assert eq(True & b, True & y)
assert eq(True | b, True | y)
assert eq(True ^ b, True ^ y)
assert eq(2 // b, 2 // y)
assert eq(2**b, 2**y)
assert eq(2 % b, 2 % y)
assert eq(2 > b, 2 > y)
assert eq(2 < b, 2 < y)
assert eq(2 >= b, 2 >= y)
assert eq(2 <= b, 2 <= y)
assert eq(2 == b, 2 == y)
assert eq(2 != b, 2 != y)
assert eq(-a, -x)
assert eq(abs(a), abs(x))
assert eq(~(a == b), ~(x == y))
assert eq(~(a == b), ~(x == y))
assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
assert eq(da.exp(b), np.exp(y))
assert eq(da.log(a), np.log(x))
assert eq(da.log10(a), np.log10(x))
assert eq(da.log1p(a), np.log1p(x))
assert eq(da.expm1(b), np.expm1(y))
assert eq(da.sqrt(a), np.sqrt(x))
assert eq(da.square(a), np.square(x))
assert eq(da.sin(a), np.sin(x))
assert eq(da.cos(b), np.cos(y))
assert eq(da.tan(a), np.tan(x))
assert eq(da.arcsin(b / 10), np.arcsin(y / 10))
assert eq(da.arccos(b / 10), np.arccos(y / 10))
assert eq(da.arctan(b / 10), np.arctan(y / 10))
assert eq(da.arctan2(b * 10, a), np.arctan2(y * 10, x))
assert eq(da.hypot(b, a), np.hypot(y, x))
assert eq(da.sinh(a), np.sinh(x))
assert eq(da.cosh(b), np.cosh(y))
assert eq(da.tanh(a), np.tanh(x))
assert eq(da.arcsinh(b * 10), np.arcsinh(y * 10))
assert eq(da.arccosh(b * 10), np.arccosh(y * 10))
assert eq(da.arctanh(b / 10), np.arctanh(y / 10))
assert eq(da.deg2rad(a), np.deg2rad(x))
assert eq(da.rad2deg(a), np.rad2deg(x))
assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))
assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
assert eq(da.isfinite(a), np.isfinite(x))
assert eq(da.isinf(a), np.isinf(x))
assert eq(da.isnan(a), np.isnan(x))
assert eq(da.signbit(a - 3), np.signbit(x - 3))
assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
assert eq(da.ldexp(c, c), np.ldexp(z, z))
assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
assert eq(da.ceil(a), np.ceil(x))
assert eq(da.trunc(a / 2), np.trunc(x / 2))
assert eq(da.degrees(b), np.degrees(y))
assert eq(da.radians(a), np.radians(x))
assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
assert eq(da.fix(a - 2.5), np.fix(x - 2.5))
assert eq(da.angle(a + 1j), np.angle(x + 1j))
assert eq(da.real(a + 1j), np.real(x + 1j))
assert eq((a + 1j).real, np.real(x + 1j))
assert eq(da.imag(a + 1j), np.imag(x + 1j))
assert eq((a + 1j).imag, np.imag(x + 1j))
assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
assert eq((a + 1j * b).conj(), (x + 1j * y).conj())
assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
assert eq(da.fabs(b), np.fabs(y))
assert eq(da.sign(b - 2), np.sign(y - 2))
l1, l2 = da.frexp(a)
r1, r2 = np.frexp(x)
assert eq(l1, r1)
assert eq(l2, r2)
l1, l2 = da.modf(a)
r1, r2 = np.modf(x)
assert eq(l1, r1)
assert eq(l2, r2)
assert eq(da.around(a, -1), np.around(x, -1))
def average_beams(self, threshold, mask='compute', warn=False):
"""
Average the beams. Note that this operation only makes sense in
limited contexts! Generally one would want to convolve all the beams
to a common shape, but this method is meant to handle the "simple" case
when all your beams are the same to within some small factor and can
therefore be arithmetically averaged.
Parameters
----------
threshold : float
The fractional difference between beam major, minor, and pa to
permit
mask : 'compute', None, or boolean array
The mask to apply to the beams. Useful for excluding bad channels
and edge beams.
warn : bool
Warn if successful?
Returns
-------
new_beam : radio_beam.Beam
A new radio beam object that is the average of the unmasked beams
"""
use_dask = isinstance(self._data, da.Array)
if mask == 'compute':
if use_dask:
# If we are dealing with dask arrays, we compute the beam
# mask once and for all since it is used multiple times in its
# entirety in the remainder of this method.
beam_mask = da.any(da.logical_and(
self._mask_include, self.goodbeams_mask[:, None, None]),
axis=(1, 2))
# da.any appears to return an object dtype instead of a bool
beam_mask = self._compute(beam_mask).astype('bool')
elif self.mask is not None:
beam_mask = np.any(np.logical_and(
self.mask.include(), self.goodbeams_mask[:, None, None]),
axis=(1, 2))
else:
beam_mask = self.goodbeams_mask
else:
if mask.ndim > 1:
beam_mask = np.logical_and(mask, self.goodbeams_mask[:, None,
None])
else:
beam_mask = np.logical_and(mask, self.goodbeams_mask)
# use private _beams here because the public one excludes the bad beams
# by default
new_beam = self._beams.average_beam(includemask=beam_mask)
if np.isnan(new_beam):
raise ValueError(
"Beam was not finite after averaging. "
"This either indicates that there was a problem "
"with the include mask, one of the beam's values, "
"or a bug.")
self._check_beam_areas(threshold, mean_beam=new_beam, mask=beam_mask)
if warn:
warnings.warn(
"Arithmetic beam averaging is being performed. This is "
"not a mathematically robust operation, but is being "
"permitted because the beams differ by "
"<{0}".format(threshold), BeamAverageWarning)
return new_beam
def lengths_and_angles_to_box_vectors(a_length, b_length, c_length, alpha, beta, gamma):
"""Convert from the lengths/angles of the unit cell to the box
vectors (Bravais vectors). The angles should be in degrees.
Mimics mdtraj.core.unitcell.lengths_and_angles_to_box_vectors()
Parameters
----------
a_length : scalar or ndarray
length of Bravais unit vector **a**
b_length : scalar or ndarray
length of Bravais unit vector **b**
c_length : scalar or ndarray
length of Bravais unit vector **c**
alpha : scalar or ndarray
angle between vectors **b** and **c**, in degrees.
beta : scalar or ndarray
angle between vectors **c** and **a**, in degrees.
gamma : scalar or ndarray
angle between vectors **a** and **b**, in degrees.
Returns
-------
a : dask.array
If the inputs are scalar, the vectors will one dimensional (length 3).
If the inputs are one dimension, shape=(n_frames, ), then the output
will be (n_frames, 3)
b : dask.array
If the inputs are scalar, the vectors will one dimensional (length 3).
If the inputs are one dimension, shape=(n_frames, ), then the output
will be (n_frames, 3)
c : dask.array
If the inputs are scalar, the vectors will one dimensional (length 3).
If the inputs are one dimension, shape=(n_frames, ), then the output
will be (n_frames, 3)
This code is adapted from gyroid, which is licensed under the BSD
http://pythonhosted.org/gyroid/_modules/gyroid/unitcell.html
"""
# Fix for da that requires angles and lengths to be arrays
lengths = [a_length, b_length, c_length]
for i, e in enumerate(lengths):
# Use python logic shortcutting to not compute dask Arrays
if not isinstance(e, da.core.Array) and np.isscalar(e):
lengths[i] = np.array([e])
a_length, b_length, c_length = tuple(lengths)
angles = [alpha, beta, gamma]
for i, e in enumerate(angles):
if not isinstance(e, da.core.Array) and np.isscalar(e):
angles[i] = np.array([e])
alpha, beta, gamma = tuple(angles)
if da.all(alpha < 2 * np.pi) and (
da.all(beta < 2 * np.pi) and da.all(gamma < 2 * np.pi)
):
warnings.warn(
"All your angles were less than 2*pi."
" Did you accidentally give me radians?"
)
alpha = alpha * np.pi / 180
beta = beta * np.pi / 180
gamma = gamma * np.pi / 180
a = da.stack([a_length, da.zeros_like(a_length), da.zeros_like(a_length)])
b = da.stack(
[b_length * da.cos(gamma), b_length * da.sin(gamma), da.zeros_like(b_length)]
)
cx = c_length * da.cos(beta)
cy = c_length * (da.cos(alpha) - da.cos(beta) * da.cos(gamma)) / da.sin(gamma)
cz = da.sqrt(c_length * c_length - cx * cx - cy * cy)
c = da.stack([cx, cy, cz])
if not a.shape == b.shape == c.shape:
raise TypeError("Shape is messed up.")
# Make sure that all vector components that are _almost_ 0 are set exactly
# to 0
tol = 1e-6
a[da.logical_and(a > -tol, a < tol)] = 0.0
b[da.logical_and(b > -tol, b < tol)] = 0.0
c[da.logical_and(c > -tol, c < tol)] = 0.0
return a.T, b.T, c.T
def binned_skymaps(ddf,
col_to_bins,
ra_col=None,
dec_col=None,
nside=64,
num_workers=1,
scheduler='threading',
**compute_kwargs):
""" Calculate binned skymaps for input data
Parameters
----------
ddf : dask.dataframe.DataFrame
Input data.
col_to_bins : collections.OrderedDict
Dictionary with mapping between columns in ddf and bin edges.
ra_col : str
Column name in ddf with right ascension values (in radians).
dec_col : str
Column name in ddf with declination values (in radians).
nside : int, optional
Number of sides used for healpix map (default is 64).
num_workers : int, optional
Number of processes or threads to use (default is 1).
scheduler : str, optional
Dask scheduler to use (default is 'threading').
compute_kwargs : dict
Optional keyword arguments to pass to Dask's compute() function.
Returns
-------
data : xarray.DataArray
"""
if not all([ra_col, dec_col]):
raise ValueError('Both ra_col and dec_col must not be None')
if not isinstance(ddf, dd.DataFrame):
raise TypeError('ddf must be a dask DataFrame')
if not isinstance(col_to_bins, OrderedDict):
raise TypeError(
'col_to_bins must be an instance of collections.OrderedDict')
npix = hp.nside2npix(nside)
# Get bin bin index for each column
ddf_digitized = ddf.map_partitions(digitize_columns, col_to_bins)
shape = list((len(bins) - 1 for bins in col_to_bins.values()))
bin_idxs = [np.arange(l) for l in shape]
# Compute skymaps for each unique bin combination
cols = col_to_bins.keys()
maps = []
for idx in itertools.product(*bin_idxs):
bool_masks = list(ddf_digitized['{}_digitized'.format(col)] == i
for col, i in zip(cols, idx))
if len(bool_masks) == 1:
mask = bool_masks[0]
else:
mask = da.logical_and(*bool_masks)
theta, phi = sa.equatorial_to_healpy(ddf.loc[mask, ra_col],
ddf.loc[mask, dec_col])
ipix = da.map_blocks(ang2pix, theta.values, phi.values)
m, _ = da.histogram(ipix, bins=np.arange(npix + 1))
maps.append(m)
with ProgressBar():
maps = compute(*maps,
num_workers=num_workers,
scheduler=scheduler,
**compute_kwargs)
# Format maps into an xarray.DataArray
data = np.zeros(shape + [npix])
for idx, m in zip(itertools.product(*bin_idxs), maps):
data[idx] = m
dims = col_to_bins.keys() + ['ipix']
coords = {}
for col in col_to_bins.keys():
bins = col_to_bins[col]
coords[col] = (bins[1:] + bins[:-1]) / 2
data = xr.DataArray(data, dims=dims, coords=coords)
return data