def destination_azimuth_distance(lat, lon, az, dist, dist_units='m'):
"""
This procedure will calculate a destination lat/lon from
an initial lat/lon and azimuth and distance.
Parameters
----------
lat : float
Initial latitude.
lon : float
Initial longitude.
az : float
Azimuth in degrees.
dist : float
Distance
dist_units : str
Units for dist
Returns
-------
lat2 : float
Latitude of new point in degrees
lon2 : float
Longitude of new point in degrees
"""
# Volumetric Mean Radius of Earth in km
R = 6378.
# Convert az to radian
brng = np.radians(az)
# Convert distance to km
d = convert_units(dist, dist_units, 'km')
# Convert lat/lon to radians
lat = np.radians(lat)
lon = np.radians(lon)
# Using great circle equations
lat2 = np.arcsin(
np.sin(lat) * np.cos(d / R) +
np.cos(lat) * np.sin(d / R) * np.cos(brng))
lon2 = lon + np.arctan2(
np.sin(brng) * np.sin(d / R) * np.cos(lat),
np.cos(d / R) - np.sin(lat) * np.sin(lat2))
return np.degrees(lat2), np.degrees(lon2)
def compare_time_series_trends(self,
var_name=None,
comp_dataset=None,
comp_var_name=None,
time_match_threshhold=60,
time_shift=60 * 60,
time_step=None,
time_qc_threshold=60 * 15):
"""
Method to perform a time series comparison test between two Xarray Datasets
to detect a shift in time based on two similar variables. This test will
compare two similar measurements and look to see if there is a time shift
forwards or backwards that makes the comparison better. If so assume the
time has shifted.
This test is not 100% accurate. It may be fooled with noisy data. Use
with your own discretion.
Parameters
----------
var_name : str
Data variable name.
comp_dataset : Xarray Dataset
Dataset containing comparison data to use in test.
comp_var_name : str
Name of variable in comp_dataset to use in test.
time_match_threshhold : int
Number of seconds to use in tolerance with reindex() method
to match time from self to comparison Dataset.
time_shift : int
Number of seconds to shift analysis window before and after
the time in self Dataset time.
time_step : int
Time step in seconds for self Dataset time. If not provided
will attempt to find the most common time step.
time_qc_threshold : int
The quality control threshold to use for setting test. If the
calculated time shift is larger than this value will set all
values in the QC variable to a tripped test value.
"""
# If no comparison variable name given assume matches variable name
if comp_var_name is None:
comp_var_name = var_name
# If no comparison Dataset given assume self Dataset
if comp_dataset is None:
comp_dataset = self
# Extract copy of DataArray for work below
self_da = copy.deepcopy(self._obj[var_name])
comp_da = copy.deepcopy(comp_dataset[comp_var_name])
# Convert comp data units to match
comp_da.values = convert_units(comp_da.values, comp_da.attrs['units'],
self_da.attrs['units'])
comp_da.attrs['units'] = self_da.attrs['units']
# Match comparison data to time of data
if time_step is None:
time_step = determine_time_delta(self._obj['time'].values)
sum_diff = np.array([], dtype=float)
time_diff = np.array([], dtype=np.int32)
for tm_shift in range(-1 * time_shift, time_shift + int(time_step),
int(time_step)):
self_da_shifted = self_da.assign_coords(
time=self_da.time.values.astype('datetime64[s]') + tm_shift)
data_matched, comp_data_matched = xr.align(self_da, comp_da)
self_da_shifted = self_da_shifted.reindex(
time=comp_da.time.values,
method='nearest',
tolerance=np.timedelta64(time_match_threshhold, 's'))
diff = np.abs(self_da_shifted.values - comp_da.values)
sum_diff = np.append(sum_diff, np.nansum(diff))
time_diff = np.append(time_diff, tm_shift)
index = np.argmin(np.abs(sum_diff))
time_diff = time_diff[index]
index = None
if np.abs(time_diff) > time_qc_threshold:
index = np.arange(0, self_da.size)
meaning = (
f"Time shift detected with Minimum Difference test. Comparison of "
f"{var_name} with {comp_var_name} off by {time_diff} seconds "
f"exceeding absolute threshold of {time_qc_threshold} seconds.")
self._obj.qcfilter.add_test(var_name,
index=index,
test_meaning=meaning,
test_assessment='Indeterminate')
def bsrn_comparison_tests(self,
test,
gbl_SW_dn_name=None,
glb_diffuse_SW_dn_name=None,
direct_normal_SW_dn_name=None,
glb_SW_up_name=None,
glb_LW_dn_name=None,
glb_LW_up_name=None,
air_temp_name=None,
test_assessment='Indeterminate',
lat_name='lat',
lon_name='lon',
LWdn_lt_LWup_component=25.,
LWdn_gt_LWup_component=300.,
use_dask=False):
"""
Method to apply BSRN comparison tests and add results to ancillary quality control variable.
Need to provided variable name for each measurement for the test to be performed. All radiation
data must be in W/m^2 units. Test will provided exception if required variable name is missing.
Parameters
----------
test : str
Type of tests to apply. Options include: 'Global over Sum SW Ratio', 'Diffuse Ratio',
'SW up', 'LW down to air temp', 'LW up to air temp', 'LW down to LW up'
gbl_SW_dn_name : str
Variable name in Dataset for global shortwave downwelling radiation
measured by unshaded pyranometer
glb_diffuse_SW_dn_name : str
Variable name in Dataset for global diffuse shortwave downwelling radiation
measured by shaded pyranometer
direct_normal_SW_dn_name : str
Variable name in Dataset for direct normal shortwave downwelling radiation
glb_SW_up_name : str
Variable name in Dataset for global shortwave upwelling radiation
glb_LW_dn_name : str
Variable name in Dataset for global longwave downwelling radiation
glb_LW_up_name : str
Variable name in Dataset for global longwave upwelling radiation
air_temp_name : str
Variable name in Dataset for atmospheric air temperature. Variable used
in longwave tests.
test_assessment : str
Test assessment string value appended to flag_assessments attribute of QC variable.
lat_name : str
Variable name in the Dataset for latitude
lon_name : str
Variable name in the Dataset for longitude
LWdn_lt_LWup_component : int or float
Value used in longwave down less than longwave up test.
LWdn_gt_LWup_component : int or float
Value used in longwave down greater than longwave up test.
use_dask : boolean
Option to use Dask for processing if data is stored in a Dask array
References
----------
Long, Charles N., and Ellsworth G. Dutton. "BSRN Global Network recommended QC tests, V2. x." (2010).
Examples
--------
.. code-block:: python
ds_object = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_BRS, cleanup_qc=True)
ds_object.qcfilter.bsrn_comparison_tests(
gbl_SW_dn_name='down_short_hemisp',
glb_diffuse_SW_dn_name='down_short_diffuse_hemisp',
direct_normal_SW_dn_name='short_direct_normal',
glb_SW_up_name='up_short_hemisp',
glb_LW_dn_name='down_long_hemisp_shaded',
glb_LW_up_name='up_long_hemisp',
use_dask=True)
"""
if isinstance(test, str):
test = [test]
test_options = [
'Global over Sum SW Ratio', 'Diffuse Ratio', 'SW up',
'LW down to air temp', 'LW up to air temp', 'LW down to LW up'
]
solar_constant = 1360.8
sza, Sa = _calculate_solar_parameters(self._obj, lat_name, lon_name,
solar_constant)
# Ratio of Global over Sum SW
if test_options[0] in test:
if gbl_SW_dn_name is None or glb_diffuse_SW_dn_name is None or direct_normal_SW_dn_name is None:
raise ValueError(
'Must set keywords gbl_SW_dn_name, glb_diffuse_SW_dn_name, '
f'direct_normal_SW_dn_name for {test_options[0]} test.')
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=RuntimeWarning)
if use_dask and isinstance(
self._obj[glb_diffuse_SW_dn_name].data, da.Array):
sum_sw_down = (self._obj[glb_diffuse_SW_dn_name].data +
self._obj[direct_normal_SW_dn_name].data *
np.cos(np.radians(sza)))
sum_sw_down[sum_sw_down < 50] = np.nan
ratio = self._obj[gbl_SW_dn_name].data / sum_sw_down
index_a = sza < 75
index_1 = da.where((ratio > 1.08) & index_a, True, False)
index_2 = da.where((ratio < 0.92) & index_a, True, False)
index_b = (sza >= 75) & (sza < 93)
index_3 = da.where((ratio > 1.15) & index_b & index_b,
True, False)
index_4 = da.where((ratio < 0.85) & index_b, True, False)
index = (index_1 | index_2 | index_3 | index_4).compute()
else:
sum_sw_down = (self._obj[glb_diffuse_SW_dn_name].values +
self._obj[direct_normal_SW_dn_name].values *
np.cos(np.radians(sza)))
sum_sw_down[sum_sw_down < 50] = np.nan
ratio = self._obj[gbl_SW_dn_name].values / sum_sw_down
index_a = sza < 75
index_1 = (ratio > 1.08) & index_a
index_2 = (ratio < 0.92) & index_a
index_b = (sza >= 75) & (sza < 93)
index_3 = (ratio > 1.15) & index_b
index_4 = (ratio < 0.85) & index_b
index = index_1 | index_2 | index_3 | index_4
test_meaning = "Ratio of Global over Sum shortwave larger than expected"
self._obj.qcfilter.add_test(gbl_SW_dn_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
self._obj.qcfilter.add_test(glb_diffuse_SW_dn_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
self._obj.qcfilter.add_test(direct_normal_SW_dn_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
# Diffuse Ratio
if test_options[1] in test:
if gbl_SW_dn_name is None or glb_diffuse_SW_dn_name is None:
raise ValueError(
'Must set keywords gbl_SW_dn_name, glb_diffuse_SW_dn_name '
f'for {test_options[1]} test.')
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=RuntimeWarning)
if use_dask and isinstance(
self._obj[glb_diffuse_SW_dn_name].data, da.Array):
ratio = self._obj[glb_diffuse_SW_dn_name].data / self._obj[
gbl_SW_dn_name].data
ratio[self._obj[gbl_SW_dn_name].data < 50] = np.nan
index_a = sza < 75
index_1 = da.where((ratio >= 1.05) & index_a, True, False)
index_b = (sza >= 75) & (sza < 93)
index_2 = da.where((ratio >= 1.10) & index_b, True, False)
index = (index_1 | index_2).compute()
else:
ratio = self._obj[
glb_diffuse_SW_dn_name].values / self._obj[
gbl_SW_dn_name].values
ratio[self._obj[gbl_SW_dn_name].values < 50] = np.nan
index_a = sza < 75
index_1 = (ratio >= 1.05) & index_a
index_b = (sza >= 75) & (sza < 93)
index_2 = (ratio >= 1.10) & index_b
index = index_1 | index_2
test_meaning = "Ratio of Diffuse Shortwave over Global Shortwave larger than expected"
self._obj.qcfilter.add_test(gbl_SW_dn_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
self._obj.qcfilter.add_test(glb_diffuse_SW_dn_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
# Shortwave up comparison
if test_options[2] in test:
if glb_SW_up_name is None or glb_diffuse_SW_dn_name is None or direct_normal_SW_dn_name is None:
raise ValueError(
'Must set keywords glb_SW_up_name, glb_diffuse_SW_dn_name, '
f'direct_normal_SW_dn_name for {test_options[2]} test.')
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=RuntimeWarning)
if use_dask and isinstance(
self._obj[glb_diffuse_SW_dn_name].data, da.Array):
sum_sw_down = (self._obj[glb_diffuse_SW_dn_name].data +
self._obj[direct_normal_SW_dn_name].data *
np.cos(np.radians(sza)))
sum_sw_down[sum_sw_down < 50] = np.nan
index = da.where(
self._obj[glb_SW_up_name].data > sum_sw_down, True,
False).compute()
else:
sum_sw_down = (self._obj[glb_diffuse_SW_dn_name].values +
self._obj[direct_normal_SW_dn_name].values *
np.cos(np.radians(sza)))
sum_sw_down[sum_sw_down < 50] = np.nan
index = self._obj[glb_SW_up_name].values > sum_sw_down
test_meaning = "Ratio of Shortwave Upwelling greater than Shortwave Sum"
self._obj.qcfilter.add_test(glb_SW_up_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
self._obj.qcfilter.add_test(glb_diffuse_SW_dn_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
self._obj.qcfilter.add_test(direct_normal_SW_dn_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
# Longwave down to air temperature comparison
if test_options[3] in test:
if glb_LW_dn_name is None or air_temp_name is None:
raise ValueError(
'Must set keywords glb_LW_dn_name, air_temp_name '
f' for {test_options[3]} test.')
air_temp = convert_units(self._obj[air_temp_name].values,
self._obj[air_temp_name].attrs['units'],
'degK')
if use_dask and isinstance(self._obj[glb_LW_dn_name].data,
da.Array):
air_temp = da.array(air_temp)
conversion = da.array(Stefan_Boltzmann * air_temp**4)
index_1 = (0.4 * conversion) > self._obj[glb_LW_dn_name].data
index_2 = (conversion + 25.) < self._obj[glb_LW_dn_name].data
index = (index_1 | index_2).compute()
else:
conversion = Stefan_Boltzmann * air_temp**4
index_1 = (0.4 * conversion) > self._obj[glb_LW_dn_name].values
index_2 = (conversion + 25.) < self._obj[glb_LW_dn_name].values
index = index_1 | index_2
test_meaning = "Longwave downwelling comparison to air temperature out side of expected range"
self._obj.qcfilter.add_test(glb_LW_dn_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
# Longwave up to air temperature comparison
if test_options[4] in test:
if glb_LW_up_name is None or air_temp_name is None:
raise ValueError(
'Must set keywords glb_LW_up_name, air_temp_name '
f'for {test_options[3]} test.')
air_temp = convert_units(self._obj[air_temp_name].values,
self._obj[air_temp_name].attrs['units'],
'degK')
if use_dask and isinstance(self._obj[glb_LW_up_name].data,
da.Array):
air_temp = da.array(air_temp)
index_1 = (Stefan_Boltzmann *
(air_temp - 15)**4) > self._obj[glb_LW_up_name].data
index_2 = (Stefan_Boltzmann *
(air_temp + 25)**4) < self._obj[glb_LW_up_name].data
index = (index_1 | index_2).compute()
else:
index_1 = (
Stefan_Boltzmann *
(air_temp - 15)**4) > self._obj[glb_LW_up_name].values
index_2 = (
Stefan_Boltzmann *
(air_temp + 25)**4) < self._obj[glb_LW_up_name].values
index = index_1 | index_2
test_meaning = "Longwave upwelling comparison to air temperature out side of expected range"
self._obj.qcfilter.add_test(glb_LW_up_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
# Lonwave down to longwave up comparison
if test_options[5] in test:
if glb_LW_dn_name is None or glb_LW_up_name is None:
raise ValueError(
'Must set keywords glb_LW_dn_name, glb_LW_up_name '
f'for {test_options[3]} test.')
if use_dask and isinstance(self._obj[glb_LW_dn_name].data,
da.Array):
index_1 = da.where(
self._obj[glb_LW_dn_name].data >
(self._obj[glb_LW_up_name].data + LWdn_lt_LWup_component),
True, False)
index_2 = da.where(
self._obj[glb_LW_dn_name].data <
(self._obj[glb_LW_up_name].data - LWdn_gt_LWup_component),
True, False)
index = (index_1 | index_2).compute()
else:
index_1 = self._obj[glb_LW_dn_name].values > (
self._obj[glb_LW_up_name].values + LWdn_lt_LWup_component)
index_2 = self._obj[glb_LW_dn_name].values < (
self._obj[glb_LW_up_name].values - LWdn_gt_LWup_component)
index = index_1 | index_2
test_meaning = "Lonwave downwelling compared to longwave upwelling outside of expected range"
self._obj.qcfilter.add_test(glb_LW_dn_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
self._obj.qcfilter.add_test(glb_LW_up_name,
index=index,
test_assessment=test_assessment,
test_meaning=test_meaning)
def run(self, data: np.ndarray, in_units: str,
out_units: str) -> np.ndarray:
return data_utils.convert_units(data, in_units, out_units)
