def _check_date_time_is_valid(self, date_time):
""" Check that the given date time lies within the range covered by the input data
Parameters
----------
date_time : Datetime
Datetime object to check
Returns
--------
: bool
Flag confirming whether the given date time is valid or not
"""
ds0 = self.dataset_reader.read_dataset(self.data_file_names[0])
data_datetime_0 = num2pydate(ds0.variables[self._time_var_name][0], units=ds0.variables[self._time_var_name].units)
ds0.close()
ds1 = self.dataset_reader.read_dataset(self.data_file_names[-1])
data_datetime_1 = num2pydate(ds1.variables[self._time_var_name][-1], units=ds1.variables[self._time_var_name].units)
ds1.close()
if data_datetime_0 <= date_time < data_datetime_1:
return True
return False
def get_datetime(self, dataset, time_index=None):
""" Get FVCOM dates/times for the given dataset
The time variable in FVCOM has the lowest precision. Instead,
we construct the time array from the Itime and Itime2 vars,
before then constructing datetime objects.
Parameters
----------
dataset : Dataset
Dataset object for an FVCOM data file.
Returns
-------
: list[datetime]
If `time_index` is None, return a full list of datetime objects.
: Datetime
"""
time_raw = dataset.variables['Itime'][:] + dataset.variables['Itime2'][:] / 1000. / 60. / 60. / 24.
units = dataset.variables['Itime'].units
# Apply rounding
# TODO - Confirm this is necessary when using Itime and Itime2?
rounding_interval = self.config.getint("OCEAN_CIRCULATION_MODEL", "rounding_interval")
if time_index is not None:
datetime_raw = num2pydate(time_raw[time_index], units=units)
return round_time([datetime_raw], rounding_interval)[0]
else:
datetime_raw = num2pydate(time_raw[:], units=units)
return round_time(datetime_raw, rounding_interval)
def ncGetTimes(ncobj, name='time'):
"""
Get the time data from a netcdf file.
:param ncobj: :class:`netCDF4.Dataset` or :class:`netCDF4.Group` instance.
:param str name: Name of the time variable.
:return times: Array of time dimension values as true Python :class:`datetime` objects.
:rtype: :class:`numpy.ndarray` of :class:`datetime` objects
"""
from datetime import datetime
from cftime import num2pydate
if hasattr(ncobj, 'Convention'):
if getattr(ncobj, 'Convention') == "COARDS":
# COARDS Compliant file - makes examining the data easier.
times = ncobj.variables['time']
elif name in ncobj.dimensions:
times = ncobj.variables[name]
else:
logger.debug(f"Unknown time variable name {name}")
if hasattr(times, 'units'):
units = times.units
if hasattr(times, 'calendar'):
calendar = times.calendar
else:
calendar = 'standard'
dates = num2pydate(times[:].data, units, calendar)
return np.array(dates, dtype=datetime)
def _parse(self):
from cftime import num2pydate
content = ReadNC(self.filename)
for attribute_name in content.attributes:
self.attribute_list.append(attribute_name)
self.info.set_attribute(attribute_name,
getattr(content, attribute_name))
for parameter_name in content.parameters:
self.parameter_list.append(parameter_name)
setattr(self, parameter_name, getattr(content, parameter_name))
self._n_records = len(self.longitude)
# Get timestamp (can be either time or timestamp in l2i files)
if hasattr(self, "time"):
time = self.time
time_parameter_name = "time"
else:
time = self.time
time_parameter_name = "timestamp"
self._time_parameter_name = time_parameter_name
dt = num2pydate(time, self.time_def.units, self.time_def.calendar)
setattr(self, "time", dt)
self.time = self.time
def __init__(self,ncfilelist,**kwargs):
self.__dict__.update(kwargs)
self.ncfilelist = ncfilelist
self.nfiles = len(self.ncfilelist)
print('Retrieving the time information from files...')
self._timelookup = {}
timeall = []
#self.time = np.zeros((0,))
for f in self.ncfilelist:
print(f)
nc = Dataset(f)
t = nc.variables[self.timevar]
time = num2pydate(t[:].ravel(),t.units)#.tolist()
nc.close()
#self.timelookup.update({f:time})
timeall.append(time)
# Create a dictionary of time and file
for ii,tt in enumerate(time):
tstr = datetime.strftime(tt,self.tformat)
if tstr in self._timelookup:
# Overwrite the old key
self._timelookup[tstr]=(f,ii)
else:
self._timelookup.update({tstr:(f,ii)})
#self.time = np.asarray(self.time)
timeall = np.array(timeall)
self.time = np.unique(timeall)
def prev_year_december_data(fname, year):
with tempfile.TemporaryDirectory() as tmpdir:
str1 = f".{year}"
str2 = f".{year-1}"
fname1 = Path(fname).name.replace(str1, str2)
fc_path = download(fname1, tmpdir)
dec_idx = []
dec_timestamp = []
with netCDF4.Dataset(fc_path) as ds:
for its, ts in enumerate(ds["time"]):
date = cftime.num2pydate(ts, ds["time"].units)
if date.month - 1 == 11:
dec_idx.append(its)
dec_timestamp.append(date)
pv = np.asarray(ds["phot_veg"][dec_idx, ...], dtype=np.float32)
npv = np.asarray(ds["nphot_veg"][dec_idx, ...], dtype=np.float32)
soil = np.asarray(ds["bare_soil"][dec_idx, ...], dtype=np.float32)
data = np.empty((len(dec_idx), 3, pv.shape[1], pv.shape[2]),
dtype=pv.dtype)
data[:, 0, :, :] = pv
data[:, 1, :, :] = npv
data[:, 2, :, :] = soil
return data, dec_timestamp
def _set_time_orbit_data_group(self):
"""
Transfer the time orbit parameter from the netcdf to l1 data object
:return: None
"""
# Transfer the timestamp
# NOTE: Here it is critical that the xarray does not automatically decodes time since it is
# difficult to work with the numpy datetime64 date format. Better to compute datetimes using
# a know num2pydate conversion
tai_datetime = num2pydate(self.nc.time_20_ku.values,
units=self.nc.time_20_ku.units)
converter = UTCTAIConverter()
utc_timestamp = converter.tai2utc(tai_datetime, check_all=False)
self.l1.time_orbit.timestamp = utc_timestamp
# Set the geolocation
self.l1.time_orbit.set_position(self.nc.lon_20_ku.values,
self.nc.lat_20_ku.values,
self.nc.alt_20_ku.values,
self.nc.orb_alt_rate_20_ku.values)
# Set antenna attitude
self.l1.time_orbit.set_antenna_attitude(
self.nc.off_nadir_pitch_angle_str_20_ku.values,
self.nc.off_nadir_roll_angle_str_20_ku.values,
self.nc.off_nadir_yaw_angle_str_20_ku.values)
def load(self, dst_folder, **kwargs):
"""
Load map diagnostic and determine map type.
"""
# get file name without extension
base_name = os.path.splitext(os.path.basename(self.path))[0]
map_type = self.cube.attributes['map_type']
map_handler = type_handling.function_mapper(map_type)
if map_handler is None:
raise exceptions.InvalidMapTypeException(map_type)
unit_text = f"{format_units(self.cube.units)}"
time_coord = self.cube.coord('time')
time_bounds = time_coord.bounds[0]
dates = cftime.num2pydate(time_bounds, time_coord.units.name)
plot_title = format_title(self.cube.long_name)
date_title = f"{dates[0].strftime('%Y')} - {dates[-1].strftime('%Y')}"
fig = map_handler(
self.cube,
title=plot_title,
dates=date_title,
units=unit_text,
**kwargs,
)
dst_file = f"./{base_name}.png"
with ChangeDirectory(dst_folder):
fig.savefig(dst_file, bbox_inches="tight")
plt.close(fig)
image_dict = {
'title': self.cube.attributes['title'],
'path': dst_file,
'comment': self.cube.attributes['comment'],
}
return image_dict
def read_content(self):
# Open the file
try:
f = Dataset(self.filename)
f.set_auto_scale(self.autoscale)
except RuntimeError:
msg = "Cannot read netCDF file: %s" % self.filename
self.error.add_error("nc-runtime-error", msg)
self.error.raise_on_error()
# Try to update the time units
# NOTE: This has become necessary with the use of
# variable epochs
try:
time_units = f.variables["time"].units
self.time_def.units = time_units
except (KeyError, AttributeError):
pass
# Get the global attributes
for attribute_name in f.ncattrs():
self.attributes.append(attribute_name)
attribute_value = getattr(f, attribute_name)
# Convert timestamps back to datetime objects
# TODO: This needs to be handled better
if attribute_name in ["start_time", "stop_time"]:
attribute_value = num2pydate(attribute_value,
self.time_def.units,
calendar=self.time_def.calendar)
setattr(self, attribute_name, attribute_value)
# Get the variables
if not self.global_attrs_only:
for key in f.variables.keys():
try:
variable = f.variables[key][:]
except ValueError:
continue
try:
is_float = variable.dtype in ["float32", "float64"]
has_mask = hasattr(variable, "mask")
except:
is_float, has_mask = False, False
if self.nan_fill_value and has_mask and is_float:
is_fill_value = np.where(variable.mask)
variable[is_fill_value] = np.nan
setattr(self, key, variable)
self.keys.append(key)
self.parameters.append(key)
if self.verbose:
print(key)
self.parameters = f.variables.keys()
f.close()
def _set_time_orbit_data_group(self):
"""
Transfer the time orbit parameter from the netcdf to l1 data object
:return: None
"""
# Transfer the timestamp
# NOTE: Here it is critical that the xarray does not automatically decodes time since it is
# difficult to work with the numpy datetime64 date format. Better to compute datetimes using
# a know num2pydate conversion
utc_timestamp = num2pydate(self.nc.time_20_ku.values,
units=self.nc.time_20_ku.units)
self.l1.time_orbit.timestamp = utc_timestamp
# Set the geolocation
self.l1.time_orbit.set_position(self.nc.lon_20_ku.values,
self.nc.lat_20_ku.values,
self.nc.alt_20_ku.values,
self.nc.orb_alt_rate_20_ku.values)
# Set antenna attitude
# NOTE: This are only available in 1Hz and need to be interpolated
time_01, time_20 = self.nc.time_01.values, self.nc.time_20_ku.values
pitch_angle_20, stat = self.interp_1Hz_to_20Hz(
self.nc.off_nadir_pitch_angle_pf_01.values, time_01, time_20)
roll_angle_20, stat = self.interp_1Hz_to_20Hz(
self.nc.off_nadir_roll_angle_pf_01.values, time_01, time_20)
yaw_angle_20, stat = self.interp_1Hz_to_20Hz(
self.nc.off_nadir_yaw_angle_pf_01.values, time_01, time_20)
self.l1.time_orbit.set_antenna_attitude(pitch_angle_20, roll_angle_20,
yaw_angle_20)
def get_month(self, time_coord):
"""
Returns the month of [0] in time_coord.points as a string
"""
dt_object = cftime.num2pydate(time_coord.points[0],
time_coord.units.name)
return dt_object.strftime('%B')
def get_masked_datetime_array(t, tvar, mask_nan=True):
# If we are passed in a scalar... return a scalar
if isinstance(t, np.ma.core.MaskedConstant):
return t
elif np.isscalar(t):
return num2date(t, tvar.units, getattr(tvar, 'calendar', 'standard'))
if mask_nan is True:
t = np.ma.masked_invalid(t)
t_cal = getattr(tvar, 'calendar', 'standard')
# Get the min value we can have and mask anything else
# This is limited by **python** datetime objects and not
# nc4 objects. The min nc4 datetime object is
# min_date = nc4.netcdftime.datetime(-4713, 1, 1, 12, 0, 0, 40)
# There is no max date for nc4.
min_nums = date2num([datetime.min, datetime.max], tvar.units, t_cal)
t = np.ma.masked_outside(t, *min_nums)
dts = num2pydate(t, tvar.units, t_cal)
if isinstance(dts, (datetime, cfdt)):
dts = np.array([dts.isoformat()], dtype='datetime64')
return dts
def get_band_info_for_date(target_date: dt.date) -> tuple[int, str]:
"""Return the band index and a descriptive text for the layer title."""
def _get_data_fp_from_cache(year: int) -> Path:
filepath = (
FETCH_DATASETS_DIR
/ f'seaice_age.{year}'
/ f'iceage_nh_12.5km_{year}0101_{year}1231_v4.1.nc'
)
if not filepath.is_file():
raise FileNotFoundError(f'Expected file {filepath} does not exist.')
return filepath
data_filepath = _get_data_fp_from_cache(target_date.year)
ds = Dataset(data_filepath)
time = ds.variables['time']
band_dates = [real_date.date() for real_date in num2pydate(time[:], units=time.units)]
for band_idx, start_date in enumerate(band_dates, start=1):
end_date = start_date + dt.timedelta(days=6)
if start_date <= target_date and target_date <= end_date:
if start_date.month == end_date.month:
week_str = f'{start_date:%B} {start_date:%-d}-{end_date:%-d}'
else:
week_str = f'{start_date:%B} {start_date:%-d}-{end_date:%B}{end_date:%-d}'
return band_idx, week_str
raise RuntimeError('Failed to find data matching target_date.')
def estimate_alpha_zhang2020(radar, band, scan_idx,
min_z=10, max_z=50, min_zdr=-4, max_zdr=4, min_rhohv=0.98,
min_r=20, max_r=120,
refl_field='reflectivity', zdr_field='corrected_differential_reflectivity', rhohv_field='corrected_cross_correlation_ratio',
isom_field='height_over_isom', temp_field='temperature', verbose=False,
z_offset=0, zdr_offset=0):
"""
WHAT: Estimate alpha by accumulating Z - ZDR pairs across scans until the pair threshold has been reaches,
and then fitting a slope to these pairs using _find_z_zdr_slope.
INPUT:
radar: pyart radar object
alpha_dict: dictionary containing Z and ZDR pairs and alpha ts
min_z: minimum reflectivity for pairs (float, dB)
max_z: maximum reflectivity for pairs (float, dB)
max_zdr: minimum differential reflectivity for pairs (float, dB)
min_zdr: maximum differential reflectivity for pairs (float, dB)
min_rhohv: minimum cross correlation for pairs (float)
min_r: minimum range (km)
max_r: maximum range (km)
OUTPUT:
alpha: alpha value (float)
"""
#get radar time
radar_starttime = cftime.num2pydate(radar.time['data'][0], radar.time['units'])
#extract data
t_data = radar.get_field(scan_idx, temp_field)
z_data = radar.get_field(scan_idx, refl_field).filled() + z_offset
zdr_data = radar.get_field(scan_idx, zdr_field).filled() + zdr_offset
rhohv_data = radar.get_field(scan_idx, rhohv_field).filled()
isom_data = radar.get_field(scan_idx, isom_field)
range_vec = radar.range['data']/1000
azi_vec = radar.get_azimuth(scan_idx)
range_data, _ = np.meshgrid(range_vec, azi_vec)
#build masks
z_mask = np.logical_and(z_data>=min_z, z_data<=max_z)
zdr_mask = np.logical_and(zdr_data>=min_zdr, zdr_data<=max_zdr)
nan_mask = np.logical_and(~np.isnan(z_data), ~np.isnan(zdr_data))
rhv_mask = rhohv_data>min_rhohv
h_mask = isom_data==0 #below melting level
r_mask = np.logical_and(range_data>=min_r, range_data<=max_r)
final_mask = z_mask & zdr_mask & rhv_mask & h_mask & r_mask & nan_mask
#get mean temperature of first tilt
try:
t_mean = np.nanmean(t_data[final_mask]) #this will crash if no valid areas
except:
t_mean = np.nanmean(t_data)
#find alpha
alpha, alpha_method = _find_alpha_zhang2020(z_data[final_mask], zdr_data[final_mask], band, t_mean, verbose=verbose)
if verbose:
print('alpha value', alpha)
return alpha, alpha_method
def get_particle_data(self):
""" Get particle data
Particle data is read in from a NetCDF file that has been created
using an object of type RestartFileCreator.
"""
logger = logging.getLogger(__name__)
logger.info('Using restart file {}'.format(self._restart_file_name))
# Open the file for reading
try:
restart = Dataset(self._restart_file_name, 'r')
logger.info('Opened data file {} for reading.'.format(
self._restart_file_name))
except Exception:
logger.error('Failed to open restart file {}.'.format(
self._restart_file_name))
raise
# Check time
datetime_start_str = self._config.get("SIMULATION", "start_datetime")
datetime_start = datetime.datetime.strptime(datetime_start_str,
"%Y-%m-%d %H:%M:%S")
datetime_restart = num2pydate(
restart.variables['time'][0],
units=restart.variables['time'].units,
calendar=restart.variables['time'].calendar)
if datetime_start != datetime_restart:
datetime_restart_str = datetime_restart.strftime(
"%Y-%m-%d %H:%M:%S")
logger.error(
"The specified start time `{}' and restart time `{}' do not match"
.format(datetime_start_str, datetime_restart_str))
raise ValueError(
"When restarting the model, the specified start time should match that given in the restart file."
)
# Extract particle data
n_particles = restart.dimensions['particles'].size
group_ids = restart.variables['group_id'][0, :]
x1_var_name = variable_library.get_coordinate_variable_name(
self.coordinate_system, 'x1')
x1_positions = restart.variables[x1_var_name][0, :]
x2_var_name = variable_library.get_coordinate_variable_name(
self.coordinate_system, 'x2')
x2_positions = restart.variables[x2_var_name][0, :]
x3_var_name = variable_library.get_coordinate_variable_name(
self.coordinate_system, 'x3')
x3_positions = restart.variables[x3_var_name][0, :]
restart.close()
return n_particles, group_ids, x1_positions, x2_positions, x3_positions
def extract_clutter(infile, clutter_mask, refl_name="total_power"):
"""
Extract the clutter and compute the RCA value.
Parameters:
-----------
infile: str
Input radar file.
clutter_mask: numpy.array(float)
Clutter mask (360 deg x 20 km)
refl_name: str
Uncorrected reflectivity field name.
Returns:
--------
dtime: np.datetime64
Datetime of infile
rca: float
95th percentile of the clutter reflectivity.
"""
# Radar data.
radar = _read_radar(infile, refl_name)
dtime = cftime.num2pydate(radar.time["data"][0], radar.time["units"])
sl = radar.get_slice(0)
r = radar.range["data"]
azi = radar.azimuth["data"][sl]
try:
refl = radar.fields[refl_name]["data"][sl][:, r < 20e3].filled(np.NaN)
except AttributeError:
refl = radar.fields[refl_name]["data"][sl][:, r < 20e3]
zclutter = np.zeros_like(refl) + np.NaN
r = r[r < 20e3]
R, A = np.meshgrid(r, azi)
R = (R // 1000).astype(int)
A = (np.round(A) % 360).astype(int)
# Mask.
RC, AC = np.meshgrid(np.arange(20), np.arange(360))
npos = np.where(clutter_mask)
for ir, ia in zip(RC[npos], AC[npos]):
pos = (R == ir) & (A == ia)
zclutter[pos] = refl[pos]
try:
zclutter = zclutter[~np.isnan(zclutter)]
rca = np.percentile(zclutter, 95)
except IndexError:
# Empty array full of NaN.
raise ValueError("All clutter is NaN.")
del radar
return dtime, rca
def get_time_from_filename(filename, date):
'''
GET_TIME_FROM_FILENAME
Capture the time string inside the filename and returns it.
Parameters:
===========
filename: str
String to parse for date.
date: str
Date (format YYYYMMDD) to look for in files.
Returns:
========
date_time: datetime
Datetime corresponding to given filename.
'''
# Looking for date followed by underscore (or not) and 6 (or 4) consecutives
# number (i.e. the time)
# There is maybe an optionnal character (like _) between date and time
try:
dtstr = re.findall(date + '.[0-9]{6}', filename)
dtime = datetime.datetime.strptime(dtstr[0], '%Y%m%d.%H%M%S')
return dtime
except Exception:
pass
if filename[-2:] == "nc" or filename[-2:] == "NC":
ds = xr.open_dataset(filename)
# I wish it was simpler in python but it's not.
dates = pd.DatetimeIndex([ds.time.values[0]])
return dates.to_pydatetime()[0]
if filename[-2:] == "gz" or '.RAW' in filename:
# SIGMET file date convention.
radar = pyart.io.read(filename)
dtime = cftime.num2pydate(radar.time['data'][0], radar.time['units'])
return dtime
else:
strlist = re.findall(date + ".?[0-9]{6}", filename)
if len(strlist) == 0:
strlist = re.findall(date + ".?[0-9]{4}", filename)
try:
date_time = parser.parse(strlist[0], fuzzy=True)
except ValueError:
date_time = datetime.datetime.strptime(strlist[0], "%Y%m%d")
except IndexError:
date_time = None
return date_time # Type: str
def get_datetime(self, dataset, time_index=None):
""" Get dates/times for the given dataset
This function searches for the basic variable `time`.
If a given source of data uses a different variable
name or approach to saving time points, support for
them can be added through subclassing (as with
FVCOM) DateTimeReader.
Parameters
----------
dataset : Dataset
Dataset object for an FVCOM data file.
time_index : int, optional
The time index at which to extract data. Default behaviour is to return
the full time array as datetime objects.
Returns
-------
: list[datetime]
If `time_index` is None, return a full list of datetime objects.
: Datetime
If `time_index` is not None, a single datetime object.
"""
time_raw = dataset.variables[self._time_var_name]
units = dataset.variables[self._time_var_name].units
# Apply rounding
rounding_interval = self.config.getint("OCEAN_CIRCULATION_MODEL", "rounding_interval")
if time_index is not None:
datetime_raw = num2pydate(time_raw[time_index], units=units)
return round_time([datetime_raw], rounding_interval)[0]
else:
datetime_raw = num2pydate(time_raw[:], units=units)
return round_time(datetime_raw, rounding_interval)
def darwin_disdro_to_radar_moments(file_list, scatterer):
#init lists
DBZ_list = []
ZDR_list = []
KDP_list = []
ATTEN_list = []
RAIN_list = []
TIME_list = []
#read DSD data
for infile in file_list:
print('processing', infile)
with netCDF4.Dataset(infile, 'r') as ncid:
time = cftime.num2pydate(ncid['time'][:], ncid['time'].units)
mean_diam_drop_class = ncid['mean_diam_drop_class'][:]
num_drop = ncid['num_drop'][:]
ndensity = ncid['nd'][:]
liq_water = ncid['liq_water'][:]
Z = ncid['Z'][:]
nclambda = ncid['lambda'][:]
n_0 = ncid['n_0'][:]
rain = ncid['rain_rate'][:]
#fir each sample, use number density
print('running scattering calcs')
cnt = 0
for nd in ndensity:
if np.sum(nd) == 0:
continue
cnt += 1
#calc radar moments
dbz, zdr, kdp, atten_spec = common.scatter_off_2dvd_packed(
mean_diam_drop_class, nd, scatterer)
DBZ_list.append(dbz)
ZDR_list.append(zdr)
KDP_list.append(kdp)
ATTEN_list.append(atten_spec)
TIME_list.append(time[cnt].date())
RAIN_list.append(rain[cnt])
dbz_array = np.array(DBZ_list)
zdr_array = np.array(ZDR_list)
kdp_array = np.array(KDP_list)
att_array = np.array(ATTEN_list)
rain_array = np.array(RAIN_list)
time_array = np.array(TIME_list)
return dbz_array, zdr_array, kdp_array, att_array, time_array, rain_array
def make_coord(fss, z, accum_dim):
# a)
accum = []
logger.debug("accumulate coords array %s", accum_dim)
times = False
for fs in fss:
zz = zarr.open_array(fs.get_mapper(accum_dim))
try:
import cftime
if not isinstance(zz, cftime.real_datetime):
# Try and get the calendar attribute from "calendar" attribute
# If it doesn't exist, assume a standard calendar
if zz.attrs.get("calendar") is not None:
calendar = zz.attrs.get("calendar", "standard")
else:
calendar = 'standard'
# Update attrs in z[accum_dim]
zattr = dict(z[accum_dim].attrs)
zattr['calendar'] = 'standard'
z[accum_dim].attrs.put(zattr)
zz = cftime.num2pydate(zz[...], units=zz.attrs["units"],
calendar=calendar)
times = True
logger.debug("converted times")
accum.append(zz)
else:
accum.append(zz)
except Exception as e:
ex = e
accum.append(zz[...].copy())
attr = dict(z[accum_dim].attrs)
if times:
accum = [np.array(a, dtype="M8") for a in accum]
attr.pop('units', None)
attr.pop('calendar', None)
acc = np.concatenate([np.atleast_1d(a) for a in accum]).squeeze()
logger.debug("write coords array")
arr = z.create_dataset(name=accum_dim,
data=acc,
overwrite=True)
arr.attrs.update(attr)
return len(acc)
def date(self):
""" Date array
"""
if self._date is not None:
return self._date
for key, var in list(self._ds.variables.items()):
if key in ['time', 'time_centered', 'time_ref']:
self._date = num2pydate(var[:], units=var.units, calendar=var.calendar)
if self._time_rounding is not None:
self._date = round_time(self._date, self._time_rounding)
return self._date
raise KeyError('Time variable not found')
def load(self, dst_folder, **kwargs):
"""
Load map diagnostic and determine map type.
"""
# get file name without extension
base_name = os.path.splitext(os.path.basename(self.path))[0]
map_type = self.cube.attributes['map_type']
map_handler = type_handling.function_mapper(map_type)
if map_handler is None:
raise exceptions.InvalidMapTypeException(map_type)
png_dir = f"{base_name}_frames"
number_of_time_steps = len(self.cube.coord('time').points)
with ChangeDirectory(dst_folder):
if not os.path.isdir(png_dir):
os.mkdir(png_dir)
number_of_pngs = len(os.listdir(png_dir))
unit_text = f"{format_units(self.cube.units)}"
dst_file = f"./{base_name}.gif"
with ChangeDirectory(f"{dst_folder}/{png_dir}"):
for time_step in range(number_of_pngs, number_of_time_steps):
time_coord = self.cube.coord('time')
time_bounds = time_coord.bounds[time_step]
dates = cftime.num2pydate(time_bounds, time_coord.units.name)
date_title = f"{dates[0].strftime('%Y')}"
plot_title = format_title(self.cube.long_name)
fig = map_handler(
self.cube[time_step],
title=plot_title,
dates=date_title,
units=unit_text,
**kwargs,
)
fig.savefig(f"./{base_name}-{time_step:03}.png",
bbox_inches="tight")
plt.close(fig)
images = []
for file_name in sorted(os.listdir(".")):
images.append(imageio.imread(file_name))
imageio.mimsave(f'.{dst_file}', images, fps=2)
image_dict = {
'title': self.cube.attributes['title'],
'path': dst_file,
'comment': self.cube.attributes['comment'],
}
return image_dict
def load(self, dst_folder, **kwargs):
"""
Load time series diagnostic and call plot creator.
"""
# get file name without extension
base_name = os.path.splitext(os.path.basename(self.path))[0]
dst_file = f"./{base_name}.png"
x_coord = self.cube.coords()[0]
if "second since" in x_coord.units.name or "hour since" in x_coord.units.name:
dates = cftime.num2pydate(x_coord.points, x_coord.units.name)
coord_points = format_dates(dates)
else:
coord_points = x_coord.points
fig = plt.figure(figsize=(6, 4), dpi=150)
ax = fig.add_subplot(1, 1, 1)
ax.plot(coord_points, self.cube.data, marker='o')
if "second since" in x_coord.units.name or "hour since" in x_coord.units.name:
fig.autofmt_xdate()
minor_step, major_step = self._determine_intervals(len(coord_points))
ax.set_xticks(coord_points[minor_step - 1::major_step])
ax.set_xticks(coord_points[minor_step - 1::minor_step], minor=True)
ax.set_xticklabels(coord_points[minor_step - 1::major_step])
ax.ticklabel_format(axis='y',
style='sci',
scilimits=(-3, 6),
useOffset=False,
useMathText=True)
ax.set_ylim(kwargs.get('value_range', [None, None]))
ax.set_title(format_title(self.cube.long_name))
ax.set_xlabel(format_title(x_coord.name()))
ax.set_ylabel(format_title(self.cube.long_name, self.cube.units))
plt.tight_layout()
with ChangeDirectory(dst_folder):
fig.savefig(dst_file, bbox_inches="tight")
plt.close(fig)
image_dict = {
'title': self.cube.attributes['title'],
'path': dst_file,
'comment': self.cube.attributes['comment'],
}
return image_dict
def get_time_bounds(self, time_coord):
"""
Get contiguous time bounds for sea ice maps
Creates new time bounds [
[01-01-current year, 01-01-next year],
]
"""
dt_object = cftime.num2pydate(time_coord.points[0],
time_coord.units.name)
start = datetime.datetime(dt_object.year, 1, 1)
end = datetime.datetime(start.year + 1, 1, 1)
start_seconds = cftime.date2num(start, time_coord.units.name)
end_seconds = cftime.date2num(end, time_coord.units.name)
new_bounds = np.array([[start_seconds, end_seconds]])
return new_bounds
def update_metadata(radar) -> dict:
"""
Update metadata of the gridded products.
Parameter:
==========
radar: pyart.core.Grid
Radar data.
Returns:
========
metadata: dict
Output metadata dictionnary.
"""
today = datetime.datetime.utcnow()
dtime = cftime.num2pydate(radar.time["data"], radar.time["units"])
longitude, latitude = radar.get_point_longitude_latitude(0)
maxlon = longitude.max()
minlon = longitude.min()
maxlat = latitude.max()
minlat = latitude.min()
metadata = {
"comment": "Gridded radar volume using Barnes et al. ROI",
"field_names": ", ".join([k for k in radar.fields.keys()]),
"geospatial_bounds":
f"POLYGON(({minlon:0.6} {minlat:0.6},{minlon:0.6} {maxlat:0.6},{maxlon:0.6} {maxlat:0.6},{maxlon:0.6} {minlat:0.6},{minlon:0.6} {minlat:0.6}))",
"geospatial_lat_max": f"{maxlat:0.6}",
"geospatial_lat_min": f"{minlat:0.6}",
"geospatial_lat_units": "degrees_north",
"geospatial_lon_max": f"{maxlon:0.6}",
"geospatial_lon_min": f"{minlon:0.6}",
"geospatial_lon_units": "degrees_east",
"geospatial_vertical_min": radar.origin_altitude["data"][0],
"geospatial_vertical_max": 20000,
"geospatial_vertical_positive": "up",
"history":
f"created by Joshua Soderholm on gadi.nci.org.au at {today.isoformat()} using Py-ART",
"processing_level": "b2",
"time_coverage_start": dtime[0].isoformat(),
"time_coverage_end": dtime[-1].isoformat(),
"uuid": str(uuid.uuid4()),
}
return metadata
def retrieve_data(self, name, path, date, save_coords=False):
# date should be datetime.datetime object
fh = Dataset(path, mode='r')
# Get index of day for requested day
times = fh.variables['time'][:].data
times = cft.num2pydate(times,
fh.variables['time'].units,
calendar='standard')
data_index = np.argwhere(times == date)[0, 0]
if save_coords:
self.lons = fh.variables['lon'][:].data
self.lats = fh.variables['lat'][:].data
return fh.variables[name][data_index].data
def temperature_profile(radar):
"""
Compute the signal-to-noise ratio as well as interpolating the radiosounding
temperature on to the radar grid. The function looks for the radiosoundings
that happened at the closest time from the radar. There is no time
difference limit.
Parameters:
===========
radar:
Py-ART radar object.
Returns:
========
z_dict: dict
Altitude in m, interpolated at each radar gates.
temp_info_dict: dict
Temperature in Celsius, interpolated at each radar gates.
"""
grlat = radar.latitude["data"][0]
grlon = radar.longitude["data"][0]
dtime = pd.Timestamp(
cftime.num2pydate(radar.time["data"][0], radar.time["units"]))
geopot_profile, temp_profile = read_era5_temperature(dtime, grlon, grlat)
# append surface data using lowest level
geopot_profile = np.append(geopot_profile, [0])
temp_profile = np.append(temp_profile, temp_profile[-1])
z_dict, temp_dict = pyart.retrieve.map_profile_to_gates(
temp_profile, geopot_profile, radar)
temp_dict["data"] = temp_dict["data"].astype(np.float32)
temp_info_dict = {
"data": temp_dict["data"], # Switch to celsius.
"long_name": "Sounding temperature at gate",
"standard_name": "temperature",
"valid_min": -100,
"valid_max": 100,
"units": "degrees Celsius",
"_Least_significant_digit": 1,
"comment": "ERA5 data date: %s" % (dtime.strftime("%Y/%m/%d")),
}
return z_dict, temp_info_dict
def _read_with_netcdf(infile, dbz_name, zdr_name, rhohv_name):
with netCDF4.Dataset(infile, "r") as ncid:
# Extract datetime
volume_date = cftime.num2pydate(ncid['time'][0], ncid['time'].units)
# Get first sweep
sweep = ncid["sweep_start_ray_index"][:]
stsw = sweep[0]
edsw = sweep[1] - 1
# Extract range and azimuth
azi = ncid["azimuth"][stsw:edsw]
r = ncid["range"][:]
# Get reflectivity and ZDR.
try:
refl = ncid[dbz_name][stsw:edsw, :].filled(np.NaN)
if zdr_name is not None:
zdr = ncid[zdr_name][stsw:edsw, :].filled(np.NaN)
else:
zdr = None
except KeyError:
print(
"Wrong RHOHV/DBZ field names provided. The field names in radar files are:"
)
print(radar.fields.keys())
raise KeyError("Wrong field name provided")
# Extract RHOHV
try:
rhohv = ncid[rhohv_name][stsw:edsw, :]
except Exception:
traceback.print_exc()
print(
"Problem with cross-correlation ratio field. Maybe missing? Continuing without it."
)
rhohv = None
try: # In case rhohv is a MaskedArray
rhohv = rhohv.filled(np.NaN)
except AttributeError:
pass
return volume_date, r, azi, refl, zdr, rhohv
def _transfer_timeorbit(self):
""" Extracts the time/orbit data group from the SGDR data """
# Transfer the orbit position
self.l1.time_orbit.set_position(self.sgdr.lon_20, self.sgdr.lat_20, self.sgdr.alt_20)
# Transfer the timestamp
sgdr_timestamp = self.sgdr.time_20
units = self.cfg.sgdr_timestamp_units
calendar = self.cfg.sgdr_timestamp_calendar
timestamp = num2pydate(sgdr_timestamp, units, calendar)
self.l1.time_orbit.timestamp = timestamp
# Mandatory antenna pointing parameter (but not available for ERS)
dummy_angle = np.full(timestamp.shape, 0.0)
self.l1.time_orbit.set_antenna_attitude(dummy_angle, dummy_angle, dummy_angle)
# Update meta data container
self.l1.update_data_limit_attributes()
def temperature_profile_access(radar):
"""
Return the 0C and -20C levels for MESH calculatations using access-g dataset on NCI for temperature information
Parameters:
===========
radar:
Py-ART radar object.
"""
grlat = radar.latitude['data'][0]
grlon = radar.longitude['data'][0]
dtime = pd.Timestamp(cftime.num2pydate(radar.time['data'][0], radar.time['units']))
#build paths
request_date = datetime.strftime(dtime, '%Y%m%d')
request_time = str(round(dtime.hour/6)*6).zfill(2) + '00'
if request_time == '2400':
request_time = '0000'
access_root = '/g/data/lb4/ops_aps2/access-g/0001' #access g
access_folder = '/'.join([access_root, request_date, request_time, 'an', 'pl'])
#build filenames
temp_ffn = access_folder + '/air_temp.nc'
geop_ffn = access_folder + '/geop_ht.nc'
if not os.path.isfile(temp_ffn):
raise FileNotFoundError(f'{temp_ffn}: no such file for temperature.')
if not os.path.isfile(geop_ffn):
raise FileNotFoundError(f'{geop_ffn}: no such file for geopotential.')
#extract data
temp_ds = xr.open_dataset(temp_ffn)
temp_profile = temp_ds.air_temp.sel(lon=grlon, method='nearest').sel(lat=grlat, method='nearest').data[0] - 273.15 #units: deg C
geop_ds = xr.open_dataset(geop_ffn)
geopot_profile = geop_ds.geop_ht.sel(lon=grlon, method='nearest').sel(lat=grlat, method='nearest').data[0] #units: m
#append surface data using lowest level
geop_profile = np.append([0], geopot_profile)
temp_profile = np.append(temp_profile[0], temp_profile)
#generate temperature levels
level_0C = _sounding_interp(temp_profile, geop_profile, 0.)
level_minus20C = _sounding_interp(temp_profile, geop_profile, -20.)
return [level_0C, level_minus20C]
