def assignMprops(self): #assign static, figure out dynamic assignment later static = True if (static): for units in self.mdata["UNITS"].keys(): #load up the current grid obj print(units) vp = da.from_array(self.gridObj.data["/points/vp"], chunks=(self.gridObj.pointx.chunksize)) vs = da.from_array(self.gridObj.data["/points/vs"], chunks=(self.gridObj.pointx.chunksize)) p = da.from_array(self.gridObj.data["/points/p"], chunks=(self.gridObj.pointx.chunksize)) qp = da.from_array(self.gridObj.data["/points/qp"], chunks=(self.gridObj.pointx.chunksize)) qs = da.from_array(self.gridObj.data["/points/qs"], chunks=(self.gridObj.pointx.chunksize)) #where is there a valid unit that matches the current unit or one that hasnt been assigned? unitAssignments = da.where( da.logical_and(self.gridObj.pointunit == int(units), vp == -666), True, False) #note that unit assignment is NOT a list of indexes as you would normally expect, it behaves more like # a np.where operator which is why I need to reset it before each assignment vp[unitAssignments] = self.mdata["UNITS"][units]["VP"] unitAssignments = da.where( da.logical_and(self.gridObj.pointunit == int(units), vs == -666), True, False) vs[unitAssignments] = self.mdata["UNITS"][units]["VS"] unitAssignments = da.where( da.logical_and(self.gridObj.pointunit == int(units), p == -666), True, False) p[unitAssignments] = self.mdata["UNITS"][units]["P"] unitAssignments = da.where( da.logical_and(self.gridObj.pointunit == int(units), qp == -666), True, False) qp[unitAssignments] = self.mdata["UNITS"][units]["QP"] unitAssignments = da.where( da.logical_and(self.gridObj.pointunit == int(units), qs == -666), True, False) qs[unitAssignments] = self.mdata["UNITS"][units]["QS"] #save all of the changes to the arrays (for this cycle) da.to_hdf5(self.gridObj.fname, "/points/vp", vp) da.to_hdf5(self.gridObj.fname, "/points/vs", vs) da.to_hdf5(self.gridObj.fname, "/points/p", p) da.to_hdf5(self.gridObj.fname, "/points/qp", qp) da.to_hdf5(self.gridObj.fname, "/points/qs", qs)
def _do_ctp_validation(data, adef, out_size, idxs): """ Calculate CTP validation (included in CTTH plot). """ # detected ctth mask detected_clouds = da.logical_and(data['caliop_cma'] == 1, data['imager_cma'] == 1) detected_height = da.logical_and(detected_clouds, np.isfinite(data['imager_cth'])) # find pps low and caliop low low_clouds_c = gc.get_calipso_low_clouds(data['caliop_cflag']) detected_low_c = np.logical_and(detected_height, low_clouds_c) low_clouds_pps = da.where(data['imager_ctp'] > 680., 1, 0) detected_low_pps = da.logical_and(detected_height, low_clouds_pps) # pattern: CALIOP_SEVIRI cld_cld_a = da.logical_and(detected_low_c == 1, detected_low_pps == 1) clr_cld_b = da.logical_and(detected_low_c == 0, detected_low_pps == 1) cld_clr_c = da.logical_and(detected_low_c == 1, detected_low_pps == 0) clr_clr_d = da.logical_and(detected_low_c == 0, detected_low_pps == 0) cld_cld_a = cld_cld_a.astype(np.int64) clr_cld_b = clr_cld_b.astype(np.int64) cld_clr_c = cld_clr_c.astype(np.int64) clr_clr_d = clr_clr_d.astype(np.int64) a, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=cld_cld_a, density=False) b, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=clr_cld_b, density=False) c, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=cld_clr_c, density=False) d, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=clr_clr_d, density=False) scu = ScoreUtils(a, b, c, d) scores = dict() scores['CTP low clouds POD'] = [ scu.pod_1().reshape(adef.shape), 0, 1, 'rainbow' ] scores['CTP low clouds FAR'] = [ scu.far_1().reshape(adef.shape), 0, 1, 'rainbow' ] scores['CTP low clouds POFD'] = [ scu.pofd_1().reshape(adef.shape), 0, 1, 'rainbow' ] # scores['Heidke low clouds'] = [scu.heidke().reshape(adef.shape),0, 1, 'rainbow'] return scores
def do_ctp_validation(data, adef, out_size, idxs): """ Scores: low clouds detection """ # detected ctth mask detected_clouds = da.logical_and(data['caliop_cma'] == 1, data['imager_cma'] == 1) detected_height = da.logical_and(detected_clouds, np.isfinite(data['imager_cth'])) # find pps low and caliop low low_clouds_c = get_calipso_low_clouds(data['caliop_cflag']) detected_low_c = np.logical_and(detected_height, low_clouds_c) low_clouds_pps = da.where(data['imager_ctp'] > 680., 1, 0) detected_low_pps = da.logical_and(detected_height, low_clouds_pps) # pattern: CALIOP_SEVIRI cld_cld_a = da.logical_and(detected_low_c == 1, detected_low_pps == 1) clr_cld_b = da.logical_and(detected_low_c == 0, detected_low_pps == 1) cld_clr_c = da.logical_and(detected_low_c == 1, detected_low_pps == 0) clr_clr_d = da.logical_and(detected_low_c == 0, detected_low_pps == 0) cld_cld_a = cld_cld_a.astype(np.int64) clr_cld_b = clr_cld_b.astype(np.int64) cld_clr_c = cld_clr_c.astype(np.int64) clr_clr_d = clr_clr_d.astype(np.int64) a, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=cld_cld_a, density=False) b, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=clr_cld_b, density=False) c, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=cld_clr_c, density=False) d, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=clr_clr_d, density=False) # n = a + b + c + d # n2d = N.reshape(adef.shape) # scores = [hitrate(a, d, n).reshape(adef.shape), # 0.7, 1, 'rainbow'] # hitrate low PPS pod_low = a / (a + c) far_low = c / (a + c) scores = dict() scores['POD low clouds'] = [pod_low.reshape(adef.shape), 0.2, 1, 'rainbow'] scores['FAR low clouds'] = [far_low.reshape(adef.shape), 0.2, 1, 'rainbow'] return scores
def mask_out_bad_beams(self, threshold, reference_beam=None, criteria=['sr', 'major', 'minor'], mid_value=np.nanmedian): """ See `identify_bad_beams`. This function returns a masked cube Returns ------- newcube : VaryingResolutionSpectralCube The cube with bad beams masked out """ goodbeams = self.identify_bad_beams(threshold=threshold, reference_beam=reference_beam, criteria=criteria, mid_value=mid_value) includemask = BooleanArrayMask(goodbeams[:, None, None], self._wcs, shape=self._data.shape) use_dask = isinstance(self._data, da.Array) if use_dask: newmask = da.logical_and(self._mask_include, includemask) elif self.mask is None: newmask = includemask else: newmask = np.bitwise_and(self.mask, includemask) return self._new_thing_with( mask=newmask, beam_threshold=threshold, goodbeams_mask=np.bitwise_and(self.goodbeams_mask, goodbeams), )
def test_arithmetic(): x = np.arange(5).astype('f4') + 2 y = np.arange(5).astype('i8') + 2 z = np.arange(5).astype('i4') + 2 a = da.from_array(x, chunks=(2,)) b = da.from_array(y, chunks=(2,)) c = da.from_array(z, chunks=(2,)) assert eq(a + b, x + y) assert eq(a * b, x * y) assert eq(a - b, x - y) assert eq(a / b, x / y) assert eq(b & b, y & y) assert eq(b | b, y | y) assert eq(b ^ b, y ^ y) assert eq(a // b, x // y) assert eq(a ** b, x ** y) assert eq(a % b, x % y) assert eq(a > b, x > y) assert eq(a < b, x < y) assert eq(a >= b, x >= y) assert eq(a <= b, x <= y) assert eq(a == b, x == y) assert eq(a != b, x != y) assert eq(a + 2, x + 2) assert eq(a * 2, x * 2) assert eq(a - 2, x - 2) assert eq(a / 2, x / 2) assert eq(b & True, y & True) assert eq(b | True, y | True) assert eq(b ^ True, y ^ True) assert eq(a // 2, x // 2) assert eq(a ** 2, x ** 2) assert eq(a % 2, x % 2) assert eq(a > 2, x > 2) assert eq(a < 2, x < 2) assert eq(a >= 2, x >= 2) assert eq(a <= 2, x <= 2) assert eq(a == 2, x == 2) assert eq(a != 2, x != 2) assert eq(2 + b, 2 + y) assert eq(2 * b, 2 * y) assert eq(2 - b, 2 - y) assert eq(2 / b, 2 / y) assert eq(True & b, True & y) assert eq(True | b, True | y) assert eq(True ^ b, True ^ y) assert eq(2 // b, 2 // y) assert eq(2 ** b, 2 ** y) assert eq(2 % b, 2 % y) assert eq(2 > b, 2 > y) assert eq(2 < b, 2 < y) assert eq(2 >= b, 2 >= y) assert eq(2 <= b, 2 <= y) assert eq(2 == b, 2 == y) assert eq(2 != b, 2 != y) assert eq(-a, -x) assert eq(abs(a), abs(x)) assert eq(~(a == b), ~(x == y)) assert eq(~(a == b), ~(x == y)) assert eq(da.logaddexp(a, b), np.logaddexp(x, y)) assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y)) assert eq(da.exp(b), np.exp(y)) assert eq(da.log(a), np.log(x)) assert eq(da.log10(a), np.log10(x)) assert eq(da.log1p(a), np.log1p(x)) assert eq(da.expm1(b), np.expm1(y)) assert eq(da.sqrt(a), np.sqrt(x)) assert eq(da.square(a), np.square(x)) assert eq(da.sin(a), np.sin(x)) assert eq(da.cos(b), np.cos(y)) assert eq(da.tan(a), np.tan(x)) assert eq(da.arcsin(b/10), np.arcsin(y/10)) assert eq(da.arccos(b/10), np.arccos(y/10)) assert eq(da.arctan(b/10), np.arctan(y/10)) assert eq(da.arctan2(b*10, a), np.arctan2(y*10, x)) assert eq(da.hypot(b, a), np.hypot(y, x)) assert eq(da.sinh(a), np.sinh(x)) assert eq(da.cosh(b), np.cosh(y)) assert eq(da.tanh(a), np.tanh(x)) assert eq(da.arcsinh(b*10), np.arcsinh(y*10)) assert eq(da.arccosh(b*10), np.arccosh(y*10)) assert eq(da.arctanh(b/10), np.arctanh(y/10)) assert eq(da.deg2rad(a), np.deg2rad(x)) assert eq(da.rad2deg(a), np.rad2deg(x)) assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4)) assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4)) assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4)) assert eq(da.logical_not(a < 1), np.logical_not(x < 1)) assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a)) assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a)) assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a)) assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a)) assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y)) assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y)) assert eq(da.isfinite(a), np.isfinite(x)) assert eq(da.isinf(a), np.isinf(x)) assert eq(da.isnan(a), np.isnan(x)) assert eq(da.signbit(a - 3), np.signbit(x - 3)) assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y)) assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y)) assert eq(da.ldexp(c, c), np.ldexp(z, z)) assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y)) assert eq(da.floor(a * 0.5), np.floor(x * 0.5)) assert eq(da.ceil(a), np.ceil(x)) assert eq(da.trunc(a / 2), np.trunc(x / 2)) assert eq(da.degrees(b), np.degrees(y)) assert eq(da.radians(a), np.radians(x)) assert eq(da.rint(a + 0.3), np.rint(x + 0.3)) assert eq(da.fix(a - 2.5), np.fix(x - 2.5)) assert eq(da.angle(a + 1j), np.angle(x + 1j)) assert eq(da.real(a + 1j), np.real(x + 1j)) assert eq((a + 1j).real, np.real(x + 1j)) assert eq(da.imag(a + 1j), np.imag(x + 1j)) assert eq((a + 1j).imag, np.imag(x + 1j)) assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y)) assert eq((a + 1j * b).conj(), (x + 1j * y).conj()) assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4)) assert eq(da.fabs(b), np.fabs(y)) assert eq(da.sign(b - 2), np.sign(y - 2)) l1, l2 = da.frexp(a) r1, r2 = np.frexp(x) assert eq(l1, r1) assert eq(l2, r2) l1, l2 = da.modf(a) r1, r2 = np.modf(x) assert eq(l1, r1) assert eq(l2, r2) assert eq(da.around(a, -1), np.around(x, -1))
def do_ctth_validation(data, resampler, thrs=10): """ thrs: threshold value for filtering boxes with small number of obs """ # mask of detected ctth detected_clouds = da.logical_and(data['caliop_cma'] == 1, data['imager_cma'] == 1) detected_height = da.logical_and(detected_clouds, np.isfinite(data['imager_cth'])) detected_temperature = np.logical_and(detected_clouds, np.isfinite(data['imager_ctt'])) detected_height_mask = detected_height.astype(int) # calculate bias and mea for all ctth cases delta_h = data['imager_cth'] - data['caliop_cth'] # HEIGHT height_bias = np.where(detected_height, delta_h, np.nan) mae = np.abs(height_bias) delta_t = data['imager_ctt'] - data['caliop_ctt'] # TEMPERATURE temperature_bias = np.where(detected_temperature, delta_t, np.nan) # clouds levels (from calipso 'cloud type') low_clouds = get_calipso_low_clouds(data['caliop_cflag']) detected_low = np.logical_and(detected_height, low_clouds) bias_low = np.where(detected_low, height_bias, np.nan) bias_temperature_low = np.where(detected_low, temperature_bias, np.nan) mid_clouds = get_calipso_medium_clouds(data['caliop_cflag']) detected_mid = np.logical_and(detected_height, mid_clouds) bias_mid = np.where(detected_mid, height_bias, np.nan) high_clouds = get_calipso_high_clouds(data['caliop_cflag']) detected_high = np.logical_and(detected_height, high_clouds) bias_high = np.where(detected_high, height_bias, np.nan) # opaque/transparent clouds (from calipso 'cloud type') # tp_clouds = get_calipso_tp(data['caliop_cflag']) # detected_tp = np.logical_and(detected_height,tp_clouds) # bias_tp = np.where(detected_tp, height_bias, np.nan) # op_clouds = get_calipso_op(data['caliop_cflag']) # detected_op = np.logical_and(detected_height,op_clouds) # bias_op = np.where(detected_op, height_bias, np.nan) # low+opaque, mid/high+transparent mid_high_tp_clouds = get_calipso_medium_and_high_clouds_tp( data['caliop_cflag']) detected_mid_high_tp = np.logical_and(detected_height, mid_high_tp_clouds) bias_mid_high_tp = np.where(detected_mid_high_tp, height_bias, np.nan) low_op_clouds = get_calipso_low_clouds_op(data['caliop_cflag']) detected_low_op = np.logical_and(detected_height, low_op_clouds) bias_low_op = np.where(detected_low_op, height_bias, np.nan) # resample and filter some data out # N = resampler.get_count() n_matched_cases = resampler.get_sum(detected_height_mask) sev_cth_average = resampler.get_average(data['imager_cth']) cal_cth_average = resampler.get_average(data['caliop_cth']) bias_average = resampler.get_average(height_bias) bias_average = np.where(n_matched_cases < thrs, np.nan, bias_average) mae_average = resampler.get_average(mae) mae_average = np.where(n_matched_cases < thrs, np.nan, mae_average) bias_temperature_average = resampler.get_average(temperature_bias) bias_temperature_average = np.where(n_matched_cases < thrs, np.nan, bias_temperature_average) n_matched_cases_low = resampler.get_sum(detected_low.astype(int)) bias_low_average = resampler.get_average(bias_low) bias_low_average = np.where(n_matched_cases_low < thrs, np.nan, bias_low_average) bias_temperature_low_average = resampler.get_average(bias_temperature_low) bias_temperature_low_average = np.where(n_matched_cases_low < thrs, np.nan, bias_temperature_low_average) n_matched_cases_mid = resampler.get_sum(detected_mid.astype(int)) bias_mid_average = resampler.get_average(bias_mid) bias_mid_average = np.where(n_matched_cases_mid < thrs, np.nan, bias_mid_average) n_matched_cases_high = resampler.get_sum(detected_high.astype(int)) bias_high_average = resampler.get_average(bias_high) bias_high_average = np.where(n_matched_cases_high < thrs, np.nan, bias_high_average) # n_matched_cases_tp = resampler.get_sum(detected_tp.astype(int)) # bias_tp_average = resampler.get_average(bias_tp) # bias_tp_average = np.where(n_matched_cases_tp<thrs, # np.nan, bias_tp_average) # n_matched_cases_op = resampler.get_sum(detected_op.astype(int)) # bias_op_average = resampler.get_average(bias_op) # bias_op_average = np.where(n_matched_cases_op<thrs, # np.nan, bias_op_average) n_matched_cases_mid_high_tp = resampler.get_sum( detected_mid_high_tp.astype(int)) bias_mid_high_tp_average = resampler.get_average(bias_mid_high_tp) bias_mid_high_tp_average = np.where(n_matched_cases_mid_high_tp < thrs, np.nan, bias_mid_high_tp_average) n_matched_cases_low_op = resampler.get_sum(detected_low_op.astype(int)) bias_low_op_average = resampler.get_average(bias_low_op) bias_low_op_average = np.where(n_matched_cases_low_op < thrs, np.nan, bias_low_op_average) # calculate scores scores = dict() scores['Bias CTH'] = [bias_average, -4000, 4000, 'bwr'] scores['MAE CTH'] = [mae_average, 0, 2500, 'Reds'] scores['Bias low'] = [bias_low_average, -2000, 2000, 'bwr'] scores['Bias middle'] = [bias_mid_average, -2000, 2000, 'bwr'] scores['Bias high'] = [bias_high_average, -6000, 6000, 'bwr'] # scores['Bias opaque'] = [bias_op_average, -4000, 4000, 'bwr'] # scores['Bias transparent'] = [bias_tp_average, -4000, 4000, 'bwr'] scores['Bias low opaque'] = [bias_low_op_average, -2000, 2000, 'bwr'] scores['Bias mid+high transparent'] = [ bias_mid_high_tp_average, -6000, 6000, 'bwr' ] scores['Bias temperature'] = [bias_temperature_average, -30, 30, 'bwr'] scores['Bias temperature low'] = [ bias_temperature_low_average, -10, 10, 'bwr' ] scores['CALIOP CTH mean'] = [cal_cth_average, 1000, 14000, 'rainbow'] scores['SEVIRI CTH mean'] = [sev_cth_average, 1000, 14000, 'rainbow'] scores['Num_detected_height'] = [n_matched_cases, None, None, 'rainbow'] # addit_scores = do_ctp_validation(data, adef, out_size, idxs) # scores.update(addit_scores) # scores['N_matched_cases_low'] = [N_matched_cases_low, # None, None, 'rainbow'] # scores['N_matched_cases_middle'] = [N_matched_cases_mid, # None, None, 'rainbow'] # scores['N_matched_cases_high'] = [N_matched_cases_high, # None, None, 'rainbow'] return scores
def do_cph_validation(data, adef, out_size, idxs): cal_cph = data['caliop_cph'] img_cph = data['imager_cph'] # pattern: CALIOP_SEVIRI # get counts for contigency table ice_ice_a = da.logical_and(cal_cph == 1, img_cph == 1) liq_ice_b = da.logical_and(cal_cph == 0, img_cph == 1) ice_liq_c = da.logical_and(cal_cph == 1, img_cph == 0) liq_liq_d = da.logical_and(cal_cph == 0, img_cph == 0) ice_ice_a = ice_ice_a.astype(np.int64) liq_ice_b = liq_ice_b.astype(np.int64) ice_liq_c = ice_liq_c.astype(np.int64) liq_liq_d = liq_liq_d.astype(np.int64) # use histogram functionality to get contigency table summed up for every # grid box in target grid a, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=ice_ice_a, density=False) b, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=liq_ice_b, density=False) c, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=ice_liq_c, density=False) d, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=liq_liq_d, density=False) n = a + b + c + d n2d = n.reshape(adef.shape) # calculate scores scores = dict() scores['Hitrate'] = [ hitrate(a, d, n).reshape(adef.shape), 0.5, 1, 'rainbow' ] scores['PODliq'] = [pod_clr(b, d).reshape(adef.shape), 0.5, 1, 'rainbow'] scores['PODice'] = [pod_cld(a, c).reshape(adef.shape), 0.5, 1, 'rainbow'] scores['FARliq'] = [far_clr(c, d).reshape(adef.shape), 0, 1, 'rainbow'] scores['FARice'] = [far_cld(a, b).reshape(adef.shape), 0, 1, 'rainbow'] scores['POFDliq'] = [pofd_clr(a, c).reshape(adef.shape), 0, 1, 'rainbow'] scores['POFDice'] = [pofd_cld(b, d).reshape(adef.shape), 0, 1, 'rainbow'] scores['Heidke'] = [ heidke(a, b, c, d).reshape(adef.shape), 0, 1, 'rainbow' ] scores['Kuiper'] = [ kuiper(a, b, c, d).reshape(adef.shape), 0, 1, 'rainbow' ] scores['Bias'] = [bias(b, c, n).reshape(adef.shape), 0, 1, 'bwr'] scores['CALIOP mean'] = [ mean(a, c, n).reshape(adef.shape), None, None, 'rainbow' ] scores['SEVIRI mean'] = [ mean(a, b, n).reshape(adef.shape), None, None, 'rainbow' ] scores['Nobs'] = [n2d, None, None, 'rainbow'] scores['Bias'][2] = np.nanmax(np.abs(scores['Bias'][0])) / 2 scores['Bias'][1] = scores['Bias'][2] * (-1) return scores
def do_cma_validation(data, adef, out_size, idxs): cal_cma = data['caliop_cma'] img_cma = data['imager_cma'] # pattern: CALIOP_SEVIRI cld_cld_a = da.logical_and(cal_cma == 1, img_cma == 1) clr_cld_b = da.logical_and(cal_cma == 0, img_cma == 1) cld_clr_c = da.logical_and(cal_cma == 1, img_cma == 0) clr_clr_d = da.logical_and(cal_cma == 0, img_cma == 0) cld_cld_a = cld_cld_a.astype(np.int64) clr_cld_b = clr_cld_b.astype(np.int64) cld_clr_c = cld_clr_c.astype(np.int64) clr_clr_d = clr_clr_d.astype(np.int64) a, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=cld_cld_a, density=False) b, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=clr_cld_b, density=False) c, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=cld_clr_c, density=False) d, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=clr_clr_d, density=False) n = a + b + c + d n2d = n.reshape(adef.shape) scores = dict() scores['Hitrate'] = [ hitrate(a, d, n).reshape(adef.shape), 0.5, 1, 'rainbow' ] scores['PODclr'] = [pod_clr(b, d).reshape(adef.shape), 0.5, 1, 'rainbow'] scores['PODcld'] = [pod_cld(a, c).reshape(adef.shape), 0.5, 1, 'rainbow'] scores['FARclr'] = [far_clr(c, d).reshape(adef.shape), 0, 1, 'rainbow'] scores['FARcld'] = [far_cld(a, b).reshape(adef.shape), 0, 1, 'rainbow'] scores['POFDclr'] = [pofd_clr(a, c).reshape(adef.shape), 0, 1, 'rainbow'] scores['POFDcld'] = [pofd_cld(b, d).reshape(adef.shape), 0, 1, 'rainbow'] scores['Heidke'] = [ heidke(a, b, c, d).reshape(adef.shape), 0, 1, 'rainbow' ] scores['Kuiper'] = [ kuiper(a, b, c, d).reshape(adef.shape), 0, 1, 'rainbow' ] scores['Bias'] = [bias(b, c, n).reshape(adef.shape), 0, 1, 'bwr'] scores['CALIOP mean'] = [ mean(a, c, n).reshape(adef.shape), None, None, 'rainbow' ] scores['SEVIRI mean'] = [ mean(a, b, n).reshape(adef.shape), None, None, 'rainbow' ] scores['Nobs'] = [n2d, None, None, 'rainbow'] scores['Bias'][2] = np.nanmax(np.abs(scores['Bias'][0])) / 2 scores['Bias'][1] = scores['Bias'][2] * (-1) return scores
def get_collocated_file_info(ipath, chunksize, dnt='ALL', satz_lim=None, dataset='CCI'): file = h5py.File(ipath, 'r') caliop = file['calipso'] if dataset == 'CCI': imager = file['cci'] elif dataset == 'CLAAS': imager = file['pps'] else: raise Exception('Dataset {} not known!'.format(dataset)) # get CTH and CTT sev_cth = da.from_array(get_imager_cth(imager), chunks=chunksize) cal_cth = da.from_array(get_caliop_cth(caliop), chunks=chunksize) sev_ctt = da.from_array(get_imager_ctt(imager), chunks=chunksize) cal_ctt = da.from_array(get_caliop_ctt(caliop), chunks=chunksize) cal_cflag = np.array(caliop['feature_classification_flags'][::, 0]) # ctp_c = np.array(caliop['layer_top_pressure'])[:,0] # ctp_c = np.where(ctp_c == -9999, np.nan,ctp_c) # ctp_pps = np.array(imager['ctth_pressure']) # ctp_pps = np.where(ctp_pps==-9, np.nan, ctp_pps) # sev_ctp = da.from_array(ctp_pps, chunks=(chunksize)) # cal_ctp = da.from_array(ctp_c, chunks=(chunksize)) # get CMA, CPH, VZA, SZA, LAT and LON sev_cph = da.from_array(get_imager_cph(imager), chunks=chunksize) cal_cph = da.from_array(get_caliop_cph(caliop), chunks=chunksize) cal_cma = da.from_array(get_caliop_cma(caliop), chunks=chunksize) sev_cma = da.from_array(get_imager_cma(imager), chunks=chunksize) satz = da.from_array(imager['satz'], chunks=chunksize) sunz = da.from_array(imager['sunz'], chunks=chunksize) lat = da.from_array(imager['latitude'], chunks=chunksize) lon = da.from_array(imager['longitude'], chunks=chunksize) # mask satellize zenith angle if satz_lim is not None: mask = satz > satz_lim cal_cma = da.where(mask, np.nan, cal_cma) sev_cma = da.where(mask, np.nan, sev_cma) cal_cph = da.where(mask, np.nan, cal_cph) sev_cph = da.where(mask, np.nan, sev_cph) cal_cth = da.where(mask, np.nan, cal_cth) sev_cth = da.where(mask, np.nan, sev_cth) # cal_ctp = da.where(mask, np.nan, cal_ctt) # sev_ctp = da.where(mask, np.nan, sev_ctt) # mask all pixels except daytime if dnt == 'DAY': mask = sunz >= 80 cal_cma = da.where(mask, np.nan, cal_cma) sev_cma = da.where(mask, np.nan, sev_cma) cal_cph = da.where(mask, np.nan, cal_cph) sev_cph = da.where(mask, np.nan, sev_cph) cal_cth = da.where(mask, np.nan, cal_cth) sev_cth = da.where(mask, np.nan, sev_cth) # cal_ctp = da.where(mask, np.nan, cal_ctt) # sev_ctp = da.where(mask, np.nan, sev_ctt) # mask all pixels except nighttime elif dnt == 'NIGHT': mask = sunz <= 95 cal_cma = da.where(mask, np.nan, cal_cma) sev_cma = da.where(mask, np.nan, sev_cma) cal_cph = da.where(mask, np.nan, cal_cph) sev_cph = da.where(mask, np.nan, sev_cph) cal_cth = da.where(mask, np.nan, cal_cth) sev_cth = da.where(mask, np.nan, sev_cth) # cal_ctp = da.where(mask, np.nan, cal_ctt) # sev_ctp = da.where(mask, np.nan, sev_ctt) # mask all pixels except twilight elif dnt == 'TWILIGHT': mask = ~da.logical_and(sunz > 80, sunz < 95) cal_cma = da.where(mask, np.nan, cal_cma) sev_cma = da.where(mask, np.nan, sev_cma) cal_cph = da.where(mask, np.nan, cal_cph) sev_cph = da.where(mask, np.nan, sev_cph) cal_cth = da.where(mask, np.nan, cal_cth) sev_cth = da.where(mask, np.nan, sev_cth) # cal_ctp = da.where(mask, np.nan, cal_ctt) # sev_ctp = da.where(mask, np.nan, sev_ctt) elif dnt == 'ALL': pass else: raise Exception('DNT option ', dnt, ' is invalid.') data = { 'caliop_cma': cal_cma, 'imager_cma': sev_cma, 'caliop_cph': cal_cph, 'imager_cph': sev_cph, 'satz': satz, 'sunz': sunz, 'caliop_cth': cal_cth, 'imager_cth': sev_cth, 'caliop_ctt': cal_ctt, 'imager_ctt': sev_ctt, 'caliop_cflag': cal_cflag } latlon = {'lat': lat, 'lon': lon} return data, latlon
def _get_calipso_matchup_file_content(ipath, chunksize, dnt='ALL', satz_lim=None, dataset='CCI'): """ Obtain variables from atrain_match HDF5 matchup file. """ file = h5py.File(ipath, 'r') caliop = file['calipso'] if dataset == 'CCI': imager = file['cci'] elif dataset == 'CLAAS': imager = file['pps'] else: raise Exception('Dataset {} not known!'.format(dataset)) # get CTH, CTT, CTP sev_cth = da.from_array(gi.get_imager_cth(imager), chunks=chunksize) cal_cth = da.from_array(gc.get_caliop_cth(caliop), chunks=chunksize) sev_ctt = da.from_array(gi.get_imager_ctt(imager), chunks=chunksize) cal_ctt = da.from_array(gc.get_caliop_ctt(caliop), chunks=chunksize) cal_cflag = np.array(caliop['feature_classification_flags'][::, 0]) if filter_stratospheric: tropo_height = da.from_array(gc.get_tropopause_height(caliop), chunks=chunksize) sev_ctp = da.from_array(gi.get_imager_ctp(imager), chunks=(chunksize)) cal_ctp = da.from_array(gc.get_caliop_ctp(caliop), chunks=(chunksize)) if dataset == 'CCI': sev_quality = da.from_array(imager['cpp_quality'][:].astype(int), chunks=chunksize) bad_quality_imager = np.bitwise_and(sev_quality, 3) > 0 sev_cth[bad_quality_imager] = np.nan sev_ctt[bad_quality_imager] = np.nan # get CMA, CPH, VZA, SZA, LAT and LON sev_cph = da.from_array(gi.get_imager_cph(imager), chunks=chunksize) cal_cph = da.from_array(gc.get_caliop_cph(caliop), chunks=chunksize) cal_cma = da.from_array(gc.get_caliop_cma(caliop), chunks=chunksize) sev_cma = da.from_array(gi.get_imager_cma(imager), chunks=chunksize) satz = da.from_array(imager['satz'], chunks=chunksize) sunz = da.from_array(imager['sunz'], chunks=chunksize) lat = da.from_array(imager['latitude'], chunks=chunksize) lon = da.from_array(imager['longitude'], chunks=chunksize) # mask fill values for angles satz = np.where(satz == -9, np.nan, satz) sunz = np.where(sunz == -9, np.nan, sunz) # mask satellize zenith angle if satz_lim is not None: mask = satz > satz_lim cal_cma = da.where(mask, np.nan, cal_cma) sev_cma = da.where(mask, np.nan, sev_cma) cal_cph = da.where(mask, np.nan, cal_cph) sev_cph = da.where(mask, np.nan, sev_cph) cal_cth = da.where(mask, np.nan, cal_cth) sev_cth = da.where(mask, np.nan, sev_cth) cal_ctt = da.where(mask, np.nan, cal_ctt) sev_ctt = da.where(mask, np.nan, sev_ctt) cal_ctp = da.where(mask, np.nan, cal_ctp) sev_ctp = da.where(mask, np.nan, sev_ctp) # mask all pixels except daytime if dnt == 'DAY': mask = sunz >= 80 # mask all pixels except nighttime elif dnt == 'NIGHT': mask = sunz <= 95 # mask all pixels except twilight elif dnt == 'TWILIGHT': mask = ~da.logical_and(sunz > 80, sunz < 95) # no masking elif dnt == 'ALL': mask = None else: raise Exception('DNT option ', dnt, ' is invalid.') # filter stratospheric clouds in CALIPSO if filter_stratospheric: strato_mask = cal_cth > tropo_height cal_cma = da.where(strato_mask, np.nan, cal_cma) cal_cph = da.where(strato_mask, np.nan, cal_cph) cal_cth = da.where(strato_mask, np.nan, cal_cth) cal_ctt = da.where(strato_mask, np.nan, cal_ctt) cal_ctp = da.where(strato_mask, np.nan, cal_ctp) # apply DNT masking masked = _apply_dnt_mask(cal_cma, sev_cma, cal_cph, sev_cph, cal_cth, sev_cth, cal_ctt, sev_ctt, cal_ctp, sev_ctp, mask) data = { 'caliop_cma': masked[0], 'imager_cma': masked[1], 'caliop_cph': masked[2], 'imager_cph': masked[3], 'satz': satz, 'sunz': sunz, 'caliop_cth': masked[4], 'imager_cth': masked[5], 'caliop_ctt': masked[6], 'imager_ctt': masked[7], 'caliop_ctp': masked[8], 'imager_ctp': masked[9], 'caliop_cflag': cal_cflag, } latlon = {'lat': lat, 'lon': lon} return data, latlon
def _do_ctth_validation(data, adef, out_size, idxs, resampler, thrs=10): """ Calculate CTH and CTT bias. thrs: threshold value for filtering boxes with small number of obs. """ # mask of detected ctth detected_clouds = da.logical_and(data['caliop_cma'] == 1, data['imager_cma'] == 1) detected_height = da.logical_and(detected_clouds, np.isfinite(data['imager_cth'])) detected_temperature = np.logical_and(detected_clouds, np.isfinite(data['imager_ctt'])) detected_pressure = np.logical_and(detected_clouds, np.isfinite(data['imager_ctp'])) detected_height_mask = detected_height.astype(int) # calculate bias and mea for all ctth cases delta_h = data['imager_cth'] - data['caliop_cth'] # HEIGHT height_bias = np.where(detected_height, delta_h, np.nan) delta_t = data['imager_ctt'] - data['caliop_ctt'] # TEMPERATURE temperature_bias = np.where(detected_temperature, delta_t, np.nan) delta_p = data['imager_ctp'] - data['caliop_ctp'] # PRESSURE pressure_bias = np.where(detected_pressure, delta_p, np.nan) # clouds levels (from calipso 'cloud type') low_clouds = gc.get_calipso_low_clouds(data['caliop_cflag']) detected_low = np.logical_and(detected_height, low_clouds) bias_low = np.where(detected_low, height_bias, np.nan) bias_temperature_low = np.where(detected_low, temperature_bias, np.nan) mid_clouds = gc.get_calipso_medium_clouds(data['caliop_cflag']) detected_mid = np.logical_and(detected_height, mid_clouds) bias_mid = np.where(detected_mid, height_bias, np.nan) high_clouds = gc.get_calipso_high_clouds(data['caliop_cflag']) detected_high = np.logical_and(detected_height, high_clouds) bias_high = np.where(detected_high, height_bias, np.nan) # resample and filter some data out n_matched_cases = resampler.get_sum(detected_height_mask) sev_cth_average = resampler.get_average(data['imager_cth']) cal_cth_average = resampler.get_average(data['caliop_cth']) bias_average = resampler.get_average(height_bias) bias_average = np.where(n_matched_cases < thrs, np.nan, bias_average) bias_temperature_average = resampler.get_average(temperature_bias) bias_temperature_average = np.where(n_matched_cases < thrs, np.nan, bias_temperature_average) bias_pressure_average = resampler.get_average(pressure_bias) bias_pressure_average = np.where(n_matched_cases < thrs, np.nan, bias_pressure_average) n_matched_cases_low = resampler.get_sum(detected_low.astype(int)) bias_low_average = resampler.get_average(bias_low) bias_low_average = np.where(n_matched_cases_low < thrs, np.nan, bias_low_average) n_matched_cases_mid = resampler.get_sum(detected_mid.astype(int)) bias_mid_average = resampler.get_average(bias_mid) bias_mid_average = np.where(n_matched_cases_mid < thrs, np.nan, bias_mid_average) n_matched_cases_high = resampler.get_sum(detected_high.astype(int)) bias_high_average = resampler.get_average(bias_high) bias_high_average = np.where(n_matched_cases_high < thrs, np.nan, bias_high_average) # calculate scores scores = dict() # [scores_on_target_grid, vmin, vmax, cmap] scores['Bias CTH'] = [bias_average, -4000, 4000, 'bwr'] scores['Bias CTT'] = [bias_temperature_average, -30, 30, 'bwr'] scores['Bias CTP'] = [bias_pressure_average, -200, 200, 'bwr'] scores['Bias CTH low'] = [bias_low_average, -2000, 2000, 'bwr'] scores['Bias CTH middle'] = [bias_mid_average, -2000, 2000, 'bwr'] scores['Bias CTH high'] = [bias_high_average, -6000, 6000, 'bwr'] addit_scores = _do_ctp_validation(data, adef, out_size, idxs) scores.update(addit_scores) scores['CALIOP CTH mean'] = [cal_cth_average, 1000, 14000, 'rainbow'] scores['SEVIRI CTH mean'] = [sev_cth_average, 1000, 14000, 'rainbow'] scores['Num_detected_height'] = [n_matched_cases, None, None, 'rainbow'] return scores
def _do_cma_cph_validation(data, adef, out_size, idxs, variable): """ Calculate scores for CMA or CPH depending on variable arg. """ # !!! for CPH '0' is water, '1' is ice !!! if variable.lower() == 'cma': cal = data['caliop_cma'] img = data['imager_cma'] elif variable.lower() == 'cph': cal = data['caliop_cph'] img = data['imager_cph'] else: msg = 'Variable {} for CMA/CPH validation unknown. [cma, cph]' raise Exception(msg.format(variable)) # pattern: CALIOP_SEVIRI a11 = da.logical_and(cal == 1, img == 1).astype(np.int64) b01 = da.logical_and(cal == 0, img == 1).astype(np.int64) c10 = da.logical_and(cal == 1, img == 0).astype(np.int64) d00 = da.logical_and(cal == 0, img == 0).astype(np.int64) a, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=a11, density=False) b, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=b01, density=False) c, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=c10, density=False) d, _ = da.histogram(idxs, bins=out_size, range=(0, out_size), weights=d00, density=False) scu = ScoreUtils(a, b, c, d) scores = dict() # [scores_on_target_grid, vmin, vmax, cmap] scores['Hitrate'] = [scu.hitrate().reshape(adef.shape), 0.5, 1, 'rainbow'] scores['PODclr'] = [scu.pod_0().reshape(adef.shape), 0.5, 1, 'rainbow'] if variable.lower() == 'cph': scores['PODwater'] = scores.pop('PODclr') scores['PODcld'] = [scu.pod_1().reshape(adef.shape), 0.5, 1, 'rainbow'] if variable.lower() == 'cph': scores['PODice'] = scores.pop('PODcld') scores['FARclr'] = [scu.far_0().reshape(adef.shape), 0, 1, 'rainbow'] if variable.lower() == 'cph': scores['FARwater'] = scores.pop('FARclr') scores['FARcld'] = [scu.far_1().reshape(adef.shape), 0, 1, 'rainbow'] if variable.lower() == 'cph': scores['FARice'] = scores.pop('FARcld') scores['POFDclr'] = [scu.pofd_0().reshape(adef.shape), 0, 1, 'rainbow'] if variable.lower() == 'cph': scores['POFDwater'] = scores.pop('POFDclr') scores['POFDcld'] = [scu.pofd_1().reshape(adef.shape), 0, 1, 'rainbow'] if variable.lower() == 'cph': scores['POFDice'] = scores.pop('POFDcld') scores['Heidke'] = [scu.heidke().reshape(adef.shape), 0, 1, 'rainbow'] scores['Kuiper'] = [scu.kuiper().reshape(adef.shape), 0, 1, 'rainbow'] scores['Bias'] = [scu.bias().reshape(adef.shape), 0, 1, 'bwr'] scores['CALIOP mean'] = [ scu.mean(a, c).reshape(adef.shape), None, None, 'rainbow' ] scores['SEVIRI mean'] = [ scu.mean(a, b).reshape(adef.shape), None, None, 'rainbow' ] scores['Nobs'] = [scu.n.reshape(adef.shape), None, None, 'rainbow'] # calculate bias limits for plotting scores['Bias'][2] = np.nanmax(np.abs(scores['Bias'][0])) / 2 scores['Bias'][1] = scores['Bias'][2] * (-1) # change mean values, bias if variable.lower() == 'cph': scores['CALIOP mean'][0] = scu.mean(d, b).reshape(adef.shape) scores['SEVIRI mean'][0] = scu.mean(d, c).reshape(adef.shape) scores['Bias'][0] = scores['Bias'][0] * (-1) return scores
def test_arithmetic(): x = np.arange(5).astype('f4') + 2 y = np.arange(5).astype('i8') + 2 z = np.arange(5).astype('i4') + 2 a = da.from_array(x, chunks=(2, )) b = da.from_array(y, chunks=(2, )) c = da.from_array(z, chunks=(2, )) assert eq(a + b, x + y) assert eq(a * b, x * y) assert eq(a - b, x - y) assert eq(a / b, x / y) assert eq(b & b, y & y) assert eq(b | b, y | y) assert eq(b ^ b, y ^ y) assert eq(a // b, x // y) assert eq(a**b, x**y) assert eq(a % b, x % y) assert eq(a > b, x > y) assert eq(a < b, x < y) assert eq(a >= b, x >= y) assert eq(a <= b, x <= y) assert eq(a == b, x == y) assert eq(a != b, x != y) assert eq(a + 2, x + 2) assert eq(a * 2, x * 2) assert eq(a - 2, x - 2) assert eq(a / 2, x / 2) assert eq(b & True, y & True) assert eq(b | True, y | True) assert eq(b ^ True, y ^ True) assert eq(a // 2, x // 2) assert eq(a**2, x**2) assert eq(a % 2, x % 2) assert eq(a > 2, x > 2) assert eq(a < 2, x < 2) assert eq(a >= 2, x >= 2) assert eq(a <= 2, x <= 2) assert eq(a == 2, x == 2) assert eq(a != 2, x != 2) assert eq(2 + b, 2 + y) assert eq(2 * b, 2 * y) assert eq(2 - b, 2 - y) assert eq(2 / b, 2 / y) assert eq(True & b, True & y) assert eq(True | b, True | y) assert eq(True ^ b, True ^ y) assert eq(2 // b, 2 // y) assert eq(2**b, 2**y) assert eq(2 % b, 2 % y) assert eq(2 > b, 2 > y) assert eq(2 < b, 2 < y) assert eq(2 >= b, 2 >= y) assert eq(2 <= b, 2 <= y) assert eq(2 == b, 2 == y) assert eq(2 != b, 2 != y) assert eq(-a, -x) assert eq(abs(a), abs(x)) assert eq(~(a == b), ~(x == y)) assert eq(~(a == b), ~(x == y)) assert eq(da.logaddexp(a, b), np.logaddexp(x, y)) assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y)) assert eq(da.exp(b), np.exp(y)) assert eq(da.log(a), np.log(x)) assert eq(da.log10(a), np.log10(x)) assert eq(da.log1p(a), np.log1p(x)) assert eq(da.expm1(b), np.expm1(y)) assert eq(da.sqrt(a), np.sqrt(x)) assert eq(da.square(a), np.square(x)) assert eq(da.sin(a), np.sin(x)) assert eq(da.cos(b), np.cos(y)) assert eq(da.tan(a), np.tan(x)) assert eq(da.arcsin(b / 10), np.arcsin(y / 10)) assert eq(da.arccos(b / 10), np.arccos(y / 10)) assert eq(da.arctan(b / 10), np.arctan(y / 10)) assert eq(da.arctan2(b * 10, a), np.arctan2(y * 10, x)) assert eq(da.hypot(b, a), np.hypot(y, x)) assert eq(da.sinh(a), np.sinh(x)) assert eq(da.cosh(b), np.cosh(y)) assert eq(da.tanh(a), np.tanh(x)) assert eq(da.arcsinh(b * 10), np.arcsinh(y * 10)) assert eq(da.arccosh(b * 10), np.arccosh(y * 10)) assert eq(da.arctanh(b / 10), np.arctanh(y / 10)) assert eq(da.deg2rad(a), np.deg2rad(x)) assert eq(da.rad2deg(a), np.rad2deg(x)) assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4)) assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4)) assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4)) assert eq(da.logical_not(a < 1), np.logical_not(x < 1)) assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a)) assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a)) assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a)) assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a)) assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y)) assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y)) assert eq(da.isfinite(a), np.isfinite(x)) assert eq(da.isinf(a), np.isinf(x)) assert eq(da.isnan(a), np.isnan(x)) assert eq(da.signbit(a - 3), np.signbit(x - 3)) assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y)) assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y)) assert eq(da.ldexp(c, c), np.ldexp(z, z)) assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y)) assert eq(da.floor(a * 0.5), np.floor(x * 0.5)) assert eq(da.ceil(a), np.ceil(x)) assert eq(da.trunc(a / 2), np.trunc(x / 2)) assert eq(da.degrees(b), np.degrees(y)) assert eq(da.radians(a), np.radians(x)) assert eq(da.rint(a + 0.3), np.rint(x + 0.3)) assert eq(da.fix(a - 2.5), np.fix(x - 2.5)) assert eq(da.angle(a + 1j), np.angle(x + 1j)) assert eq(da.real(a + 1j), np.real(x + 1j)) assert eq((a + 1j).real, np.real(x + 1j)) assert eq(da.imag(a + 1j), np.imag(x + 1j)) assert eq((a + 1j).imag, np.imag(x + 1j)) assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y)) assert eq((a + 1j * b).conj(), (x + 1j * y).conj()) assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4)) assert eq(da.fabs(b), np.fabs(y)) assert eq(da.sign(b - 2), np.sign(y - 2)) l1, l2 = da.frexp(a) r1, r2 = np.frexp(x) assert eq(l1, r1) assert eq(l2, r2) l1, l2 = da.modf(a) r1, r2 = np.modf(x) assert eq(l1, r1) assert eq(l2, r2) assert eq(da.around(a, -1), np.around(x, -1))
def average_beams(self, threshold, mask='compute', warn=False): """ Average the beams. Note that this operation only makes sense in limited contexts! Generally one would want to convolve all the beams to a common shape, but this method is meant to handle the "simple" case when all your beams are the same to within some small factor and can therefore be arithmetically averaged. Parameters ---------- threshold : float The fractional difference between beam major, minor, and pa to permit mask : 'compute', None, or boolean array The mask to apply to the beams. Useful for excluding bad channels and edge beams. warn : bool Warn if successful? Returns ------- new_beam : radio_beam.Beam A new radio beam object that is the average of the unmasked beams """ use_dask = isinstance(self._data, da.Array) if mask == 'compute': if use_dask: # If we are dealing with dask arrays, we compute the beam # mask once and for all since it is used multiple times in its # entirety in the remainder of this method. beam_mask = da.any(da.logical_and( self._mask_include, self.goodbeams_mask[:, None, None]), axis=(1, 2)) # da.any appears to return an object dtype instead of a bool beam_mask = self._compute(beam_mask).astype('bool') elif self.mask is not None: beam_mask = np.any(np.logical_and( self.mask.include(), self.goodbeams_mask[:, None, None]), axis=(1, 2)) else: beam_mask = self.goodbeams_mask else: if mask.ndim > 1: beam_mask = np.logical_and(mask, self.goodbeams_mask[:, None, None]) else: beam_mask = np.logical_and(mask, self.goodbeams_mask) # use private _beams here because the public one excludes the bad beams # by default new_beam = self._beams.average_beam(includemask=beam_mask) if np.isnan(new_beam): raise ValueError( "Beam was not finite after averaging. " "This either indicates that there was a problem " "with the include mask, one of the beam's values, " "or a bug.") self._check_beam_areas(threshold, mean_beam=new_beam, mask=beam_mask) if warn: warnings.warn( "Arithmetic beam averaging is being performed. This is " "not a mathematically robust operation, but is being " "permitted because the beams differ by " "<{0}".format(threshold), BeamAverageWarning) return new_beam
def lengths_and_angles_to_box_vectors(a_length, b_length, c_length, alpha, beta, gamma): """Convert from the lengths/angles of the unit cell to the box vectors (Bravais vectors). The angles should be in degrees. Mimics mdtraj.core.unitcell.lengths_and_angles_to_box_vectors() Parameters ---------- a_length : scalar or ndarray length of Bravais unit vector **a** b_length : scalar or ndarray length of Bravais unit vector **b** c_length : scalar or ndarray length of Bravais unit vector **c** alpha : scalar or ndarray angle between vectors **b** and **c**, in degrees. beta : scalar or ndarray angle between vectors **c** and **a**, in degrees. gamma : scalar or ndarray angle between vectors **a** and **b**, in degrees. Returns ------- a : dask.array If the inputs are scalar, the vectors will one dimensional (length 3). If the inputs are one dimension, shape=(n_frames, ), then the output will be (n_frames, 3) b : dask.array If the inputs are scalar, the vectors will one dimensional (length 3). If the inputs are one dimension, shape=(n_frames, ), then the output will be (n_frames, 3) c : dask.array If the inputs are scalar, the vectors will one dimensional (length 3). If the inputs are one dimension, shape=(n_frames, ), then the output will be (n_frames, 3) This code is adapted from gyroid, which is licensed under the BSD http://pythonhosted.org/gyroid/_modules/gyroid/unitcell.html """ # Fix for da that requires angles and lengths to be arrays lengths = [a_length, b_length, c_length] for i, e in enumerate(lengths): # Use python logic shortcutting to not compute dask Arrays if not isinstance(e, da.core.Array) and np.isscalar(e): lengths[i] = np.array([e]) a_length, b_length, c_length = tuple(lengths) angles = [alpha, beta, gamma] for i, e in enumerate(angles): if not isinstance(e, da.core.Array) and np.isscalar(e): angles[i] = np.array([e]) alpha, beta, gamma = tuple(angles) if da.all(alpha < 2 * np.pi) and ( da.all(beta < 2 * np.pi) and da.all(gamma < 2 * np.pi) ): warnings.warn( "All your angles were less than 2*pi." " Did you accidentally give me radians?" ) alpha = alpha * np.pi / 180 beta = beta * np.pi / 180 gamma = gamma * np.pi / 180 a = da.stack([a_length, da.zeros_like(a_length), da.zeros_like(a_length)]) b = da.stack( [b_length * da.cos(gamma), b_length * da.sin(gamma), da.zeros_like(b_length)] ) cx = c_length * da.cos(beta) cy = c_length * (da.cos(alpha) - da.cos(beta) * da.cos(gamma)) / da.sin(gamma) cz = da.sqrt(c_length * c_length - cx * cx - cy * cy) c = da.stack([cx, cy, cz]) if not a.shape == b.shape == c.shape: raise TypeError("Shape is messed up.") # Make sure that all vector components that are _almost_ 0 are set exactly # to 0 tol = 1e-6 a[da.logical_and(a > -tol, a < tol)] = 0.0 b[da.logical_and(b > -tol, b < tol)] = 0.0 c[da.logical_and(c > -tol, c < tol)] = 0.0 return a.T, b.T, c.T
def binned_skymaps(ddf, col_to_bins, ra_col=None, dec_col=None, nside=64, num_workers=1, scheduler='threading', **compute_kwargs): """ Calculate binned skymaps for input data Parameters ---------- ddf : dask.dataframe.DataFrame Input data. col_to_bins : collections.OrderedDict Dictionary with mapping between columns in ddf and bin edges. ra_col : str Column name in ddf with right ascension values (in radians). dec_col : str Column name in ddf with declination values (in radians). nside : int, optional Number of sides used for healpix map (default is 64). num_workers : int, optional Number of processes or threads to use (default is 1). scheduler : str, optional Dask scheduler to use (default is 'threading'). compute_kwargs : dict Optional keyword arguments to pass to Dask's compute() function. Returns ------- data : xarray.DataArray """ if not all([ra_col, dec_col]): raise ValueError('Both ra_col and dec_col must not be None') if not isinstance(ddf, dd.DataFrame): raise TypeError('ddf must be a dask DataFrame') if not isinstance(col_to_bins, OrderedDict): raise TypeError( 'col_to_bins must be an instance of collections.OrderedDict') npix = hp.nside2npix(nside) # Get bin bin index for each column ddf_digitized = ddf.map_partitions(digitize_columns, col_to_bins) shape = list((len(bins) - 1 for bins in col_to_bins.values())) bin_idxs = [np.arange(l) for l in shape] # Compute skymaps for each unique bin combination cols = col_to_bins.keys() maps = [] for idx in itertools.product(*bin_idxs): bool_masks = list(ddf_digitized['{}_digitized'.format(col)] == i for col, i in zip(cols, idx)) if len(bool_masks) == 1: mask = bool_masks[0] else: mask = da.logical_and(*bool_masks) theta, phi = sa.equatorial_to_healpy(ddf.loc[mask, ra_col], ddf.loc[mask, dec_col]) ipix = da.map_blocks(ang2pix, theta.values, phi.values) m, _ = da.histogram(ipix, bins=np.arange(npix + 1)) maps.append(m) with ProgressBar(): maps = compute(*maps, num_workers=num_workers, scheduler=scheduler, **compute_kwargs) # Format maps into an xarray.DataArray data = np.zeros(shape + [npix]) for idx, m in zip(itertools.product(*bin_idxs), maps): data[idx] = m dims = col_to_bins.keys() + ['ipix'] coords = {} for col in col_to_bins.keys(): bins = col_to_bins[col] coords[col] = (bins[1:] + bins[:-1]) / 2 data = xr.DataArray(data, dims=dims, coords=coords) return data