def __init__(self, filename):
self.dataset = cdf.Dataset(filename)
self.variables = self.dataset.variables # For brevity
# Get the variable of interest (e.g. mean temperature,
# precipitation etc) from the variables OrderedDict
var_name = [k for k in self.dataset.variables][3]
# The main variable of the dataset (e.g. mean temperature)
self.variable = self.variables[var_name]
# Get the start and end date of the time series (as datetime objects):
self.startdate = cdf.num2date(
self.variables['time'][0],
units=self.variables['time'].units,
calendar=self.variables['time'].calendar,
)
self.enddate = cdf.num2date(
self.variables['time'][-1],
units=self.variables['time'].units,
calendar=self.variables['time'].calendar,
)
# Calc number of days in time series
self.ndays = (
self.enddate - self.startdate + datetime.timedelta(1)).days
assert(self.ndays == len(self.variables['time']))
# Grid size in degrees
self.gridsize = abs(
(self.variables['latitude'][0] - self.variables['latitude'][1]))
# Calculate the minimum latitude and longitude of the lower left (SW)
# corner of the grid
self.minlat = self.variables['latitude'][0] - (self.gridsize / 2.0)
self.minlon = self.variables['longitude'][0] - (self.gridsize / 2.0)
def gen_data(time, times, maskedArray):
timeUnits = getUnits(time)
start = None
if timeUnits:
start = (netCDF.num2date(times[0], time.units, calendar="standard")).isoformat()
else:
start = "".join(times[0])
# =========================================================================
# if np.isnan(max) or np.isnan(min) or np.isnan(std) or np.isnan(mean) or np.isnan(median):
# output = {}
# g.graphError = "no valid data available to use"
# else:
# output['global'] = {'mean': mean, 'median': median,'std': std, 'min': min, 'max': max, 'time': start}
# =========================================================================
output = {}
output["data"] = {}
data = []
# print len(time)
for i, row in enumerate(maskedArray):
# print i
if timeUnits:
if i < len(time):
date = netCDF.num2date(time[i], time.units, calendar="standard").isoformat()
else:
date = "".join(times[i])
mean = getMean(row)
if np.isnan(mean):
pass
else:
data.append(mean)
return data
def runTest(self):
# Get the real dates
# skip this until cftime pull request #55 is in a released
# version (1.0.1?). Otherwise, fix for issue #808 breaks this
if parse_version(cftime.__version__) >= parse_version('1.0.1'):
dates = []
for file in self.files:
f = Dataset(file)
t = f.variables['time']
dates.extend(num2date(t[:], t.units, t.calendar))
f.close()
# Compare with the MF dates
f = MFDataset(self.files,check=True)
t = f.variables['time']
mfdates = num2date(t[:], t.units, t.calendar)
T = MFTime(t)
assert_equal(len(T), len(t))
assert_equal(T.shape, t.shape)
assert_equal(T.dimensions, t.dimensions)
assert_equal(T.typecode(), t.typecode())
# skip this until cftime pull request #55 is in a released
# version (1.0.1?). Otherwise, fix for issue #808 breaks this
if parse_version(cftime.__version__) >= parse_version('1.0.1'):
assert_array_equal(num2date(T[:], T.units, T.calendar), dates)
assert_equal(date2index(datetime.datetime(1980, 1, 2), T), 366)
f.close()
def getdates(f):
""" Returns the years from a filename and directory path
Parameters
----------
string : name of file including path
Returns
-------
string of start year
string of end year
"""
nc = Dataset(f, 'r')
time = nc.variables['time_bnds'][:].squeeze()
nc_time = nc.variables['time']
try:
cal = nc_time.calendar
except:
cal = 'standard'
start = nc_time[:][0]
end = nc_time[:][-1]
start = num2date(start, nc_time.units, cal)
end = num2date(end, nc_time.units, cal)
start = start.year
end = end.year
return start, end
def IsAdiacent(self,Test) :
#print stc,ttc
if self.ClimatologicalField :
import netCDF4
import datetime
#print self.TimeCells,Test.TimeCells
stc=netCDF4.num2date(self.TimeCells,units='hours since 1900-01-01 00:00:00',calendar='standard')
ttc=netCDF4.num2date(Test.TimeCells,units='hours since 1900-01-01 00:00:00',calendar='standard')
print >>sys.stderr, 'WARNING 11 : leap year for climatology...'
#print 'stc',stc
#print 'ttc',ttc
#if (stc[0].year == ttc[0].year and stc[1].year == ttc[1].year ) :
#stc[1].year=ttc[0].year
if stc[-1][1].month == 2 and stc[-1][1].day == 29 : il_day=28
else : il_day=stc[-1][1].day
nstc=datetime.datetime(ttc[0][0].year,stc[-1][1].month,il_day,stc[-1][1].hour,stc[-1][1].minute)
#print nstc,stc[0][0].year,ttc[-1][1].month,ttc[-1][1].day
if ttc[-1][1].month == 2 and ttc[-1][1].day == 29 : il_day=28
else : il_day=ttc[-1][1].day
nttc=datetime.datetime(stc[0][0].year,ttc[-1][1].month,il_day,ttc[-1][1].hour,ttc[-1][1].minute)
#print 'nstc',nstc
#print 'nttc',nttc
# if stc[0][0].year <= ttc[0][0].year and ( nstc==ttc[0][0] or nttc==stc[0][0] ) :
if ( nstc==ttc[0][0] and stc[0][0].year <= ttc[0][0].year ) or ( nttc==stc[0][0] and ttc[0][0].year <= stc[0][0].year ) :
#if self.TimeCells[0]+Test.TimeCells[1]-self.TimeCells[0]==Test.TimeCells[0] :
#print 'vero',stc[0][0].year,ttc[0][0].year
return True
else :
if self.TimeCells[-1] == Test.TimeCells[0] or self.TimeCells[0] == Test.TimeCells[1] :
return True
return False
def get_monthly_time_slices(ncvar_time):
'''
Based on an input NetCDF4 time variable returns calendar appropriate monthly slices
'''
assert 'calendar' in ncvar_time.ncattrs(), "Time variable does not have a defined calendar"
cal = ncvar_time.calendar
assert 'units' in ncvar_time.ncattrs(), "Time variable must have 'unit' attribute"
units = ncvar_time.units
assert len(ncvar_time.dimensions) == 1, "Time varaible must be single dimension"
slices = []
d_start = num2date(ncvar_time[0], units, cal)
current_month = d_start.month
t_start = 0
for i, val in enumerate(ncvar_time):
d = num2date(val, units, cal)
if d.month != current_month:
slices.append(slice(t_start, i))
t_start = i
current_month = d.month
slices.append(slice(t_start, i+1))
return slices
def get_time_from_dim(cls, time_var):
"""Get min/max from a NetCDF time variable and convert to datetime"""
ndim = len(time_var.shape)
if ndim == 0:
ret_val = time_var.item()
res = ret_val, ret_val
elif ndim == 1:
# NetCDF Users' Guide states that when time is a coordinate variable,
# it should be monotonically increasing or decreasing with no
# repeated variables. Therefore, first and last elements for a
# vector should correspond to start and end time or end and start
# time respectively. See Section 2.3.1 of the NUG
res = time_var[0], time_var[-1]
else:
# FIXME: handle multidimensional time variables. Perhaps
# take the first and last element of time variable in the first
# dimension and then take the min and max of the resulting values
return None, None
# if not > 1d, return the min and max elements found
min_elem, max_elem = np.min(res), np.max(res)
if hasattr(time_var, 'calendar'):
num2date([min_elem, max_elem], time_var.units,
time_var.calendar)
return num2date([min_elem, max_elem], time_var.units,
time_var.calendar)
else:
return num2date([min_elem, max_elem], time_var.units)
def doflow_fb(first_frame, second_frame, winSize = (5,5), filter_len = 10, sig_min = 150, n_iter = 40, levels = 1):
im0 = copy.deepcopy(first_frame.fields['IR_filt']['data'])
im0[np.where(im0 < sig_min)] = sig_min
im1 = copy.deepcopy(second_frame.fields['IR_filt']['data'])
im1[np.where(im1 < sig_min)] = sig_min
sim0 = (im0 - im0.min())*(im0.max()/(im0.max()-im0.min()))
sim1 = (im1 - im1.min())*(im1.max()/(im1.max()-im1.min()))
u, v = get_optic_flow_fb(sim0[0],
sim1[0],
winSize = winSize[0], n_iter=n_iter, levels=levels)
t1 = netCDF4.num2date(second_frame.axes['time']['data'][0], units = second_frame.axes['time']['units'])
t0 = netCDF4.num2date(first_frame.axes['time']['data'][0], units = first_frame.axes['time']['units'])
dt = (t1-t0).seconds
dx = np.expand_dims(np.gradient(second_frame.fields['x']['data'])[1], 0)
dy = np.expand_dims(np.gradient(second_frame.fields['y']['data'])[0], 0)
u_fld = {'data' : dt * ndimage.median_filter(u.reshape([1,u.shape[0], u.shape[1]]),filter_len)/dx,
'units' :'pixels',
'standard_name' : 'disp',
'long name' : 'todo'}
v_fld = {'data' : dt * ndimage.median_filter( v.reshape([1,v.shape[0], v.shape[1]]),filter_len)/dy,
'units' :'pixels',
'standard_name' : 'disp',
'long name' : 'todo'}
return u_fld, v_fld
def are_time_axis_the_same(filenames):
# print "inside get times func"
# print filenames
times = {}
for key in filenames:
# print filenames[key]
times[key] = getCoordinateVariable(netCDF.Dataset(filenames[key], "r+"), "Time")
keys = times.keys()
# if (len(times[keys[0]]) != len(times[keys[1]]) ):
# pass
# return False
# else:
time_range = len(times[keys[0]]) if len(times[keys[0]]) > len(times[keys[1]]) else len(times[keys[1]])
# print "using range %d" % time_range
# print len(times[keys[0]])
# print len(times[keys[1]])
for x in range(time_range):
time1 = datetime.datetime.strptime(
netCDF.num2date(times[keys[0]][x], times[keys[0]].units, calendar="standard").isoformat(),
"%Y-%m-%dT%H:%M:%S",
)
time2 = datetime.datetime.strptime(
netCDF.num2date(times[keys[1]][x], times[keys[1]].units, calendar="standard").isoformat(),
"%Y-%m-%dT%H:%M:%S",
)
# print time1, time2
# print times[keys[0]][x] , times[keys[1]][x]
dif = time1 - time2
# print dif
if dif > timedelta.min:
return False
return True
def _init_fields(self, nc_dataset):
nc_vars = nc_dataset.variables
lons = nc_vars["lon"][:]
lats = nc_vars["lat"][:]
if lons.ndim == 1:
lats2d, lons2d = np.meshgrid(lats, lons)
elif lons.ndim == 2:
lats2d, lons2d = lats, lons
else:
raise NotImplementedError("Cannot handle {}-dimensional coordinates".format(lons.ndim))
self.lons2d, self.lats2d = lons2d, lats2d
self.times_var = nc_vars["time"]
self.times_num = nc_vars["time"][:]
if hasattr(self.times_var, "calendar"):
self.times = num2date(self.times_num, self.times_var.units, self.times_var.calendar)
else:
self.times = num2date(self.times_num, self.times_var.units)
if not self.lazy:
self.var_data = nc_vars[self.var_name][:]
if nc_vars[self.var_name].shape[1:] != self.lons2d.shape:
print("nc_vars[self.var_name].shape = {}".format(nc_vars[self.var_name].shape))
self.var_data = np.transpose(self.var_data, axes=[0, 2, 1])
x_in, y_in, z_in = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten())
self.kdtree = cKDTree(list(zip(x_in, y_in, z_in)))
def get_pasap_plot_title(dset,
varname = 'hr24_prcp',
timestep= 0,
):
""" Given an open pydap object, and some extra information, return a nice
plot title.
"""
header = "PASAP: Dynamical Seasonal Outlooks for the Pacific."
subheader1 = "Outlook based on POAMA 1.5 CGCM adjusted for historical skill"
subheader2 = "Experimental outlook for demonstration and research only"
time_var = dset['time']
if 'units' in time_var.attributes.keys():
time_units = time_var.attributes['units']
else:
time_units = ''
if 'units' in dset[varname].attributes.keys():
units = dset[varname].attributes['units']
else:
units = ''
valid_time = datetime.datetime.strftime(
num2date(time_var[timestep],time_units),"%Y%m%d")
start_date = datetime.datetime.strftime(
num2date(dset['init_date'][0],time_units),"%Y%m%d")
period_label = str(dset['time_label'][timestep])
titlestring = header + '\n' \
+ subheader1 + '\n' \
+ subheader2 + '\n' \
+ "Variable: " + varname + ' (' + units + ')' + '\n' \
+ 'Model initialised ' + start_date + '\n' \
# + 'Forecast period: ' + period_label
return titlestring
def read_nc(infile, varname, dimension=-1, is_time=0): '''Read a variable from a netCDF file Input: input file path variable name dimension: if < 0, read in all dimensions of the variable; if >= 0, only read in the [dimension]th of the variable (index starts from 0). For example, if the first dimension of the variable is time, and if dimension=2, then only reads in the 3rd time step. is_time: if the desired variable is time (1 for time; 0 for not time). If it is time, return an array of datetime object Return: var: a numpy array of ''' from netCDF4 import Dataset from netCDF4 import num2date nc = Dataset(infile, 'r') if is_time==0: # if not time variable if dimension<0: var = nc.variables[varname][:] else: var = nc.variables[varname][dimension] if is_time==1: # if time variable time = nc.variables[varname] if hasattr(time, 'calendar'): # if time variable has 'calendar' attribute if dimension<0: var = num2date(time[:], time.units, time.calendar) else: var = num2date(time[dimension], time.units, time.calendar) else: # if time variable does not have 'calendar' attribute if dimension<0: var = num2date(time[:], time.units) else: var = num2date(time[dimension], time.units) nc.close() return var
def show_tbounds(t):
print 'Start date: ', num2date(t[0],t.units)
try:
print 'End date: ', num2date(t[-1],t.units)
except IndexError:
print num2date(t[0],t.units)
def get_event(id):
'''
Object response for the GET(id) request. This response is NOT cached.
'''
try:
#set up all the contaners.
data = {}
asset_id = ""
#create uframe instance, and fetch the data.
uframe_obj = UFrameEventsCollection()
payload = uframe_obj.to_json(id)
data = payload.json()
if payload.status_code != 200:
return jsonify({ "events" : payload.json()}), payload.status_code
try:
data['class'] = data.pop('@class')
data['startDate'] = num2date(float(data['startDate'])/1000, units='seconds since 1970-01-01 00:00:00', calendar='gregorian')
data['endDate'] = num2date(float(data['endDate'])/1000, units='seconds since 1970-01-01 00:00:00', calendar='gregorian')
except (KeyError, TypeError):
pass
return jsonify(**data)
except requests.exceptions.ConnectionError as e:
error = "Error: Cannot connect to uframe. %s" % e
print error
return make_response(error, 500)
def check_time_extents(self, ds):
"""
Check that the values of time_coverage_start/time_coverage_end approximately match the data.
"""
if not (hasattr(ds, 'time_coverage_start') and hasattr(ds, 'time_coverage_end')):
return
# Parse the ISO 8601 formatted dates
try:
t_min = dateparse(ds.time_coverage_start)
t_max = dateparse(ds.time_coverage_end)
except:
return Result(BaseCheck.MEDIUM,
False,
'time_coverage_extents_match',
['time_coverage attributes are not formatted properly. Use the ISO 8601:2004 date format, preferably the extended format.'])
timevar = cfutil.get_time_variable(ds)
if not timevar:
return Result(BaseCheck.MEDIUM,
False,
'time_coverage_extents_match',
['Could not find time variable to test extent of time_coverage_start/time_coverage_end, see CF-1.6 spec chapter 4.4'])
# Time should be monotonically increasing, so we make that assumption here so we don't have to download THE ENTIRE ARRAY
try:
# num2date returns as naive date, but with time adjusted to UTC
# we need to attach timezone information here, or the date
# subtraction from t_min/t_max will assume that a naive timestamp is
# in the same time zone and cause erroneous results.
# Pendulum uses UTC by default, but we are being explicit here
time0 = pendulum.instance(num2date(ds.variables[timevar][0],
ds.variables[timevar].units), 'UTC')
time1 = pendulum.instance(num2date(ds.variables[timevar][-1],
ds.variables[timevar].units), 'UTC')
except:
return Result(BaseCheck.MEDIUM,
False,
'time_coverage_extents_match',
['Failed to retrieve and convert times for variables %s.' % timevar])
start_dt = abs(time0 - t_min)
end_dt = abs(time1 - t_max)
score = 2
msgs = []
if start_dt > timedelta(hours=1):
msgs.append("Date time mismatch between time_coverage_start and actual "
"time values %s (time_coverage_start) != %s (time[0])" % (t_min.isoformat(), time0.isoformat()))
score -= 1
if end_dt > timedelta(hours=1):
msgs.append("Date time mismatch between time_coverage_end and actual "
"time values %s (time_coverage_end) != %s (time[N])" % (t_max.isoformat(), time1.isoformat()))
score -= 1
return Result(BaseCheck.MEDIUM,
(score, 2),
'time_coverage_extents_match',
msgs)
def _get_old_hiwrap_time(fname, ncFile, Good_Indices):
"""
Pull the time from HIWRAP file and convert to AWOT useable.
The time structure is odd here (to me) and is in
seconds since last Sunday.
The assumption that the data is the 4th 'field' in the filename
is required to make this work.
"""
# Pull out the date, convert the date to a datetime friendly string
# Adds dashes between year, month, and day
yyyymmdd = fname.split("_")[3]
# Find the date for Sunday previous and
# check this (should be = 6 for Sunday)
startday = int(yyyymmdd[6:8]) - int(divmod(
ncFile.variables['time'][0], 24 * 3600)[0])
if datetime.date(ncFile.variables['year'][:],
int(yyyymmdd[4:6]), startday).weekday() != 6:
print("Time could be incorrect, check file to see if time units "
"are 'computer time (sec from last Sunday at 12 am)'")
StartDate = yyyymmdd[0:4] + '-' + yyyymmdd[4:6] + '-' + str(startday)
# Create the time array
# Now convert the time array into a datetime instance
dtHrs = num2date(ncFile.variables['time'][
Good_Indices], 'seconds since ' + StartDate + '00:00:00+0:00')
# Now convert this datetime instance into a number of seconds since Epoch
TimeSec = date2num(dtHrs, common.EPOCH_UNITS)
# Now once again convert this data into a datetime instance
Time_unaware = num2date(TimeSec, common.EPOCH_UNITS)
Time = {'data': Time_unaware, 'units': common.EPOCH_UNITS,
'title': 'Time', 'full_name': 'Time (UTC)'}
return Time
def plot_filtered_timeseries(pressure, pressure_units, pressure_time, time_units,
pressure_filtered, delay, figdir, n1, dayrissaga):
# Very simple plot to show the evolution of the pressure
hfmt = dates.DateFormatter('%d %B')
time2plot = netCDF4.num2date(pressure_time, time_units)
time2plot_filter = netCDF4.num2date(pressure_time - delay, time_units)
fig = plt.figure(num=None, figsize=(14, 6))
ax = fig.add_subplot(111)
plt.plot(time2plot, pressure, 'k', lw=0.5, label='Raw signal')
plt.plot(time2plot_filter[n1:], pressure_filtered[n1:], 'c', linewidth=2, zorder=2, label='Filtered signal')
plt.axvline(x=dayrissaga, linewidth=3, color='r', alpha=0.5)
plt.xlabel('Time')
plt.ylabel(("Pressure\n (%s)" % (pressure_units)), ha='right', rotation=0)
plt.legend()
ax.xaxis.set_major_locator(dates.DayLocator())
ax.xaxis.set_major_formatter(hfmt)
plt.grid()
plt.savefig(os.path.join(figdir, 'SantAntoni_timeseries_' + dayrissaga.strftime('%Y%m%d')))
plt.close()
fig = plt.figure(num=None, figsize=(14, 6))
ax = fig.add_subplot(111)
plt.plot(time2plot[n1 / 2:-n1 / 2], pressure[n1 / 2:-n1 / 2] - pressure_filtered[n1:], 'k', lw=0.5)
plt.axvline(x=dayrissaga, linewidth=3, color='r', alpha=0.5)
plt.xlabel('Time')
plt.ylabel(("Pressure anomaly\n (%s)" % pressure_units), ha='right', rotation=0)
ax.set_xlim(time2plot[0], time2plot[-1])
ax.xaxis.set_major_locator(dates.DayLocator())
ax.xaxis.set_major_formatter(hfmt)
fig.autofmt_xdate()
plt.grid()
plt.savefig(os.path.join(figdir, 'SantAntoni_anomalies_' + dayrissaga.strftime('%Y%m%d')))
plt.close()
def rcs_model(winlen, modfile): from grid_tools import trim_time_jandec from netCDF4 import num2date from netCDF4 import date2num from scipy import ndimage from netcdf_tools import ncextractall from convert import mmd_mmm #Extract the model data and clip to the required start and end months modnc = ncextractall(modfile) mdata = modnc['pr'] mdata = mdata*86400. #convert to same units as obs mlon = modnc['lon'] mlat = modnc['lat'] mtime = modnc['time'] time_u = modnc['time_units'] if 'time_calendar' in modnc.keys(): cal = modnc['time_calendar'] mtime = num2date(mtime,units = time_u, calendar=cal) else: mtime = num2date(mtime,units = time_u) mdata, mtime = trim_time_jandec(mdata, mtime) mdata = mmd_mmm(mdata) mdata = ndimage.filters.uniform_filter(mdata,size=[winlen,1,1]) #Trim first or last values if required as they are unrepresentative trim = int(winlen/2) mdata = mdata[trim:,:,:] if winlen % 2 == 0: trim = trim - 1 mdata = mdata[:-trim,:,:] return, mdata, mlat, mlon
def setupROMSfiles(loc, date, ff, tout, time_units, tstride=1):
"""
setupROMSfiles()
Kristen Thyng, March 2013
Figures out necessary files to read in for track times and what
model output indices within those files to use.
Args:
loc: File location. loc can be a thredds server web address, a single
string of a file location, a list of strings of multiple file
locations to be searched through.
date: datetime format start date
ff: Time direction. ff=1 forward, ff=-1 backward
tout: Number of model outputs to use
time_units: To convert to datetime
tstride: Stride in time, in case want to use less model output than
is available. Default is 1, using all output.
Returns:
* nc - NetCDF object for relevant files
* tinds - Indices of outputs to use from fname files
"""
# For thredds server where all information is available in one place
# or for a single file
if 'http' in loc or type(loc) == str:
nc = netCDF.Dataset(loc)
# This is for the case when we have a bunch of files to sort through
else:
# the globbing should happen ahead of time so this case looks
# different than the single file case
# files in fname are in chronological order
nc = netCDF.MFDataset(loc)
# Convert date to number
# dates = netCDF.num2date(nc.variables['ocean_time'][:], time_units)
# The calendar definition extends dates to before the year 1582 for use
# with idealized simulations without meaningful dates.
if 'time' in nc.variables:
dates = netCDF.num2date(nc.variables['time'][:], time_units,
calendar='proleptic_gregorian')
elif 'ocean_time' in nc.variables:
dates = netCDF.num2date(nc.variables['ocean_time'][:], time_units,
calendar='proleptic_gregorian')
# time index with time value just below date (relative to file ifile)
istart = find(dates <= date)[-1]
# Select indices
if ff == 1:
# indices of model outputs desired
tinds = range(istart, istart+tout, tstride)
else: # backward in time
# have to shift istart since there are now new indices behind since
# going backward
tinds = range(istart, istart-tout, -tstride)
return nc, tinds
def get_date(filename):
try:
radar = pyart.io.read(filename)
t = (num2date(radar.time['data'][0], radar.time['units']), num2date(radar.time['data'][-1], radar.time['units']))
except:
#return start time and end time being the same
t = (num2date(0, 'seconds since 2000-01-01T00:00:00Z'), num2date(0, 'seconds since 2000-01-01T00:00:00Z'))
return t
def datetime_from_grid(grid, epoch=False):
""" Return a datetime for the volume start in a Grid. """
if epoch:
dtrad = num2date(grid.time['data'][0], grid.time['units'])
epnum = date2num(dtrad, EPOCH_UNITS)
return num2date(epnum, EPOCH_UNITS)
else:
return num2date(grid.time['data'][0], grid.time['units'])
def start_stop(nc,tvar):
ncv = nc.variables
time_var = ncv[tvar]
first = netCDF4.num2date(time_var[0],time_var.units)
last = netCDF4.num2date(time_var[-1],time_var.units)
print first.strftime('%Y-%b-%d %H:%M')
print last.strftime('%Y-%b-%d %H:%M')
def datetimes_from_radar(radar, epoch=False):
""" Return an array of datetimes for the rays in a Radar. """
if epoch:
dtrad = num2date(radar.time['data'][:], radar.time['units'])
epnum = date2num(dtrad, EPOCH_UNITS)
return num2date(epnum, EPOCH_UNITS)
else:
return num2date(radar.time['data'][:], radar.time['units'])
def cf_time(units): try: netCDF4.num2date(0, units) except: return False else: return True
def gettimebounds(self, var=None, **kwargs):
assert var in self.nc.variables
time = self.gettimevar(var)
if "units" in kwargs:
bounds = (netCDF4.num2date(np.min(time), units=u), netCDF4.num2date(np.max(time), units=u))
else:
bounds = (np.min(time.dates), np.max(time.dates))
return bounds
def time(self):
if self.time_units is not None:
dt = num2date(self.dataset[self.dimensions['time']][:],
self.time_units, self.calendar)
dt -= num2date(0, self.time_units, self.calendar)
return list(map(timedelta.total_seconds, dt))
else:
return self.dataset[self.dimensions['time']][:]
def show_ncfile_tbounds(filename,tvar='time'):
t = Dataset(filename).variables[tvar]
print('Start date: ', num2date(t[0],t.units))
try:
print('End date: ', num2date(t[-1],t.units))
except IndexError:
print(num2date(t[0],t.units))
def datetime_from_radar(radar, epoch=False):
""" Return a datetime for the first ray in a Radar. """
if epoch:
dtrad = num2date(radar.time['data'][0], radar.time['units'])
epnum = date2num(dtrad, EPOCH_UNITS)
return num2date(epnum, EPOCH_UNITS)
else:
return num2date(radar.time['data'][0], radar.time['units'])
def PrintMetaRecord(myin,myinm) : print myin #print myinm.StandardName #print myinm.created #print myinm.celltype #print myinm.LonCells[0],myinm.LatCells[0],'-',myinm.LonCells[-1],myinm.LatCells[-1] t_start=netCDF4.num2date(myinm.TimeCells[0],units='hours since 1900-01-01 00:00:00',calendar='standard') t_end=netCDF4.num2date(myinm.TimeCells[1],units='hours since 1900-01-01 00:00:00',calendar='standard') print t_start,'-',t_end
def time2year(self, t):
"""
convert time to year
"""
time = self.ds.variables["time"]
if type(t) == np.int:
return num2date(t, time.units).year
else:
return np.asarray([y.year for y in np.asarray(num2date(t, time.units))])
def storm_motion_deltas_algorithm(REFlev, REFlev1, big_storm, zero_z_trigger, storm_to_track, year, month, day, hour, start_min, duration, calibration, station, Bunkers_s, Bunkers_m, track_dis=10, GR_mins=1.0):
#Set vector perpendicular to FFD Z gradient
#storm_relative_dir = storm_relative_dir
#Set storm motion
Bunkers_m = Bunkers_m
#Set ZDR Threshold for outlining arcs
#zdrlev = [zdrlev]
#Set KDP Threshold for finding KDP feet
#kdplev = [kdplev]
#Set reflectivity thresholds for storm tracking algorithm
REFlev = [REFlev]
REFlev1 = [REFlev1]
#Set storm size threshold that triggers subdivision of big storms
big_storm = big_storm #km^2
Outer_r = 30 #km
Inner_r = 6 #km
#Set trigger to ignore strangely-formatted files right before 00Z
#Pre-SAILS #: 17
#SAILS #: 25
zero_z_trigger = zero_z_trigger
storm_to_track = storm_to_track
zdr_outlines = []
#Here, set the initial time of the archived radar loop you want.
#Our specified time
dt = datetime(year,month, day, hour, start_min)
station = station
end_dt = dt + timedelta(hours=duration)
#Set up nexrad interface
conn = nexradaws.NexradAwsInterface()
scans = conn.get_avail_scans_in_range(dt,end_dt,station)
results = conn.download(scans, 'RadarFolder')
#Setting counters for figures and Pandas indices
f = 27
n = 1
storm_index = 0
scan_index = 0
tracking_index = 0
#Create geod object for later distance and area calculations
g = Geod(ellps='sphere')
#Open the placefile
f = open("DELTA_dev"+station+str(dt.year)+str(dt.month)+str(dt.day)+str(dt.hour)+str(dt.minute)+"_Placefile.txt", "w+")
f.write("Title: Storm Motion Deltas Placefile \n")
f.write("Refresh: 8 \n \n")
#Load ML algorithm
# forest_loaded = pickle.load(open('BestRandomForest.pkl', 'rb'))
# forest_loaded_col = pickle.load(open('BestRandomForestColumnsLEN200.pkl', 'rb'))
#Actual algorithm code starts here
#Create a list for the lists of arc outlines
zdr_out_list = []
tracks_dataframe = []
for i,scan in enumerate(results.iter_success(),start=1):
#Local file option:
#Loop over all files in the dataset and pull out each 0.5 degree tilt for analysis
try:
radar1 = scan.open_pyart()
except:
print('bad radar file')
continue
#Local file option
print('File Reading')
#Make sure the file isn't a strange format
if radar1.nsweeps > zero_z_trigger:
continue
for i in range(radar1.nsweeps):
print('in loop')
print(radar1.nsweeps)
try:
radar4 = radar1.extract_sweeps([i])
except:
print('bad file')
#Checking to make sure the tilt in question has all needed data and is the right elevation
if ((np.mean(radar4.elevation['data']) < .65) and (np.max(np.asarray(radar4.fields['differential_reflectivity']['data'])) != np.min(np.asarray(radar4.fields['differential_reflectivity']['data'])))):
n = n+1
#Calling ungridded_section; Pulling apart radar sweeps and creating ungridded data arrays
[radar,n,range_2d,ungrid_lons,ungrid_lats] = quality_control_arcalg(radar4,n,calibration)
time_start = netCDF4.num2date(radar.time['data'][0], radar.time['units'])
object_number=0.0
month = time_start.month
if month < 10:
month = '0'+str(month)
hour = time_start.hour
if hour < 10:
hour = '0'+str(hour)
minute = time_start.minute
if minute < 10:
minute = '0'+str(minute)
day = time_start.day
if day < 10:
day = '0'+str(day)
time_beg = time_start - timedelta(minutes=0.1)
time_end = time_start + timedelta(minutes=GR_mins)
sec_beg = time_beg.second
sec_end = time_end.second
min_beg = time_beg.minute
min_end = time_end.minute
h_beg = time_beg.hour
h_end = time_end.hour
d_beg = time_beg.day
d_end = time_end.day
if sec_beg < 10:
sec_beg = '0'+str(sec_beg)
if sec_end < 10:
sec_end = '0'+str(sec_end)
if min_beg < 10:
min_beg = '0'+str(min_beg)
if min_end < 10:
min_end = '0'+str(min_end)
if h_beg < 10:
h_beg = '0'+str(h_beg)
if h_end < 10:
h_end = '0'+str(h_end)
if d_beg < 10:
d_beg = '0'+str(d_beg)
if d_end < 10:
d_end = '0'+str(d_end)
#Calling kdp_section; Using NWS method, creating ungridded, smoothed KDP field
kdp_nwsdict = kdp_genesis(radar)
#Add field to radar
radar.add_field('KDP', kdp_nwsdict)
kdp_ungridded_nws = radar.fields['KDP']['data']
#Calling grid_section; Now let's grid the data on a ~250 m x 250 m grid
[REF,KDP,CC,ZDRmasked1,REFmasked,KDPmasked,rlons,rlats,rlons_2d,rlats_2d,cenlat,cenlon] = gridding_arcalg(radar)
#Calling gradient_section; Determining gradient direction and masking some Zhh and Zdr grid fields
#[grad_mag,grad_ffd,ZDRmasked] = grad_mask_arcalg(REFmasked,REF,storm_relative_dir,ZDRmasked1,CC)
#Let's create a field for inferred hail
#Commenting out for the moment
# REF_Hail = np.copy(REFmasked)
# REF_Hail1 = ma.masked_where(ZDRmasked1 > 1.0, REF_Hail)
# REF_Hail2 = ma.masked_where(CC > 1.0, REF_Hail1)
# REF_Hail2 = ma.filled(REF_Hail2, fill_value = 1)
#Let's set up the map projection!
crs = ccrs.LambertConformal(central_longitude=-100.0, central_latitude=45.0)
#Set up our array of latitude and longitude values and transform our data to the desired projection.
tlatlons = crs.transform_points(ccrs.LambertConformal(central_longitude=265, central_latitude=25, standard_parallels=(25.,25.)),rlons[0,:,:],rlats[0,:,:])
tlons = tlatlons[:,:,0]
tlats = tlatlons[:,:,1]
#Limit the extent of the map area, must convert to proper coords.
LL = (cenlon-1.0,cenlat-1.0,ccrs.PlateCarree())
UR = (cenlon+1.0,cenlat+1.0,ccrs.PlateCarree())
print(LL)
#Get data to plot state and province boundaries
states_provinces = cfeature.NaturalEarthFeature(
category='cultural',
name='admin_1_states_provinces_lakes',
scale='50m',
facecolor='none')
#Make sure these shapefiles are in the same directory as the script
#fname = 'cb_2016_us_county_20m/cb_2016_us_county_20m.shp'
#fname2 = 'cb_2016_us_state_20m/cb_2016_us_state_20m.shp'
#counties = ShapelyFeature(Reader(fname).geometries(),ccrs.PlateCarree(), facecolor = 'none', edgecolor = 'black')
#states = ShapelyFeature(Reader(fname2).geometries(),ccrs.PlateCarree(), facecolor = 'none', edgecolor = 'black')
#Create a figure and plot up the initial data and contours for the algorithm
fig=plt.figure(n,figsize=(30.,25.))
ax = plt.subplot(111,projection=ccrs.PlateCarree())
ax.coastlines('50m',edgecolor='black',linewidth=0.75)
#ax.add_feature(counties, edgecolor = 'black', linewidth = 0.5)
#ax.add_feature(states, edgecolor = 'black', linewidth = 1.5)
ax.set_extent([LL[0],UR[0],LL[1],UR[1]])
REFlevels = np.arange(20,73,2)
#Options for Z backgrounds/contours
#refp = ax.pcolormesh(ungrid_lons, ungrid_lats, ref_c, cmap=plt.cm.gist_ncar, vmin = 10, vmax = 73)
#refp = ax.pcolormesh(ungrid_lons, ungrid_lats, ref_ungridded_base, cmap='HomeyerRainbow', vmin = 10, vmax = 73)
#refp = ax.pcolormesh(rlons_2d, rlats_2d, REFrmasked, cmap=pyart.graph.cm_colorblind.HomeyerRainbow, vmin = 10, vmax = 73)
refp2 = ax.contour(rlons_2d, rlats_2d, REFmasked, [40], colors='grey', linewidths=5, zorder=1)
#refp3 = ax.contour(rlons_2d, rlats_2d, REFmasked, [45], color='r')
#plt.contourf(rlons_2d, rlats_2d, ZDR_sum_stuff, depth_levels, cmap=plt.cm.viridis)
#Option to have a ZDR background instead of Z:
#zdrp = ax.pcolormesh(ungrid_lons, ungrid_lats, zdr_c, cmap=plt.cm.nipy_spectral, vmin = -2, vmax = 6)
#Storm tracking algorithm starts here
#Reflectivity smoothed for storm tracker
smoothed_ref = ndi.gaussian_filter(REFmasked, sigma = 3, order = 0)
#1st Z contour plotted
refc = ax.contour(rlons[0,:,:],rlats[0,:,:],smoothed_ref,REFlev, alpha=.01)
#Set up projection for area calculations
proj = partial(pyproj.transform, pyproj.Proj(init='epsg:4326'),
pyproj.Proj(init='epsg:3857'))
#Main part of storm tracking algorithm starts by looping through all contours looking for Z centroids
#This method for breaking contours into polygons based on this stack overflow tutorial:
#https://gis.stackexchange.com/questions/99917/converting-matplotlib-contour-objects-to-shapely-objects
#Calling stormid_section
[storm_ids,max_lons_c,max_lats_c,ref_areas,storm_index, alg_speeds, alg_directions] = storm_objects_new(refc,proj,REFlev,REFlev1,big_storm,smoothed_ref,ax,rlons,rlats,storm_index,tracking_index,scan_index,tracks_dataframe, track_dis, time_start)
#Setup tracking index for storm of interest
tracking_ind=np.where(np.asarray(storm_ids)==storm_to_track)[0]
max_lons_c = np.asarray(max_lons_c)
max_lats_c = np.asarray(max_lats_c)
ref_areas = np.asarray(ref_areas)
#Create the ZDR and KDP contours which will later be broken into polygons
# if np.max(ZDRmasked) > zdrlev:
# zdrc = ax.contour(rlons[0,:,:],rlats[0,:,:],ZDRmasked,zdrlev,linewidths = 2, colors='purple', alpha = .5)
# else:
# zdrc=[]
# if np.max(KDPmasked) > kdplev:
# kdpc = ax.contour(rlons[0,:,:],rlats[0,:,:],KDPmasked,kdplev,linewidths = 2, colors='green', alpha = 0.01)
# else:
# kdpc=[]
# if np.max(REF_Hail2) > 50.0:
# hailc = ax.contour(rlons[0,:,:],rlats[0,:,:],REF_Hail2,[50],linewidths = 4, colors='pink', alpha = 0.01)
# else:
# hailc=[]
# if np.max(REFmasked) > 35.0:
# zhhc = ax.contour(rlons[0,:,:],rlats[0,:,:],REFmasked,[35.0],linewidths = 3,colors='orange', alpha = 0.8)
# else:
# zhhc=[]
plt.contour(ungrid_lons, ungrid_lats, range_2d, [73000], linewidths=7, colors='r')
plt.savefig('testfig.png')
print('Testfig Saved')
if len(max_lons_c) > 0:
#Calling zdr_arc_section; Create ZDR arc objects using a similar method as employed in making the storm objects
# [zdr_storm_lon,zdr_storm_lat,zdr_dist,zdr_forw,zdr_back,zdr_areas,zdr_centroid_lon,zdr_centroid_lat,zdr_mean,zdr_cc_mean,zdr_max,zdr_masks,zdr_outlines,ax,f] = zdrarc(zdrc,ZDRmasked,CC,REF,grad_ffd,grad_mag,KDP,forest_loaded,ax,f,time_start,month,d_beg,h_beg,min_beg,sec_beg,d_end,h_end,min_end,sec_end,rlons,rlats,max_lons_c,max_lats_c,zdrlev,proj,storm_relative_dir,Outer_r,Inner_r,tracking_ind)
#Calling hail_section; Identify Hail core objects in a similar way to the ZDR arc objects
# [hail_areas,hail_centroid_lon,hail_centroid_lat,hail_storm_lon,hail_storm_lat,ax,f] = hail_objects(hailc,REF_Hail2,ax,f,time_start,month,d_beg,h_beg,min_beg,sec_beg,d_end,h_end,min_end,sec_end,rlons,rlats,max_lons_c,max_lats_c,proj)
#Calling zhh_section; Identify 35dBz storm area in a similar way to the ZDR arc objects
# [zhh_areas,zhh_centroid_lon,zhh_centroid_lat,zhh_storm_lon,zhh_storm_lat,zhh_max,zhh_core_avg] = zhh_objects(zhhc,REFmasked,rlons,rlats,max_lons_c,max_lats_c,proj)
# #Calling kdpfoot_section; Identify KDP foot objects in a similar way to the ZDR arc objects
# [kdp_areas,kdp_centroid_lon,kdp_centroid_lat,kdp_storm_lon,kdp_storm_lat,kdp_max,ax,f] = kdp_objects(kdpc,KDPmasked,ax,f,time_start,month,d_beg,h_beg,min_beg,sec_beg,d_end,h_end,min_end,sec_end,rlons,rlats,max_lons_c,max_lats_c,kdplev,proj)
#Consolidating the arc objects associated with each storm:
# zdr_areas_arr = np.zeros((len(zdr_areas)))
# zdr_max_arr = np.zeros((len(zdr_max)))
# zdr_mean_arr = np.zeros((len(zdr_mean)))
# for i in range(len(zdr_areas)):
# zdr_areas_arr[i] = zdr_areas[i].magnitude
# zdr_max_arr[i] = zdr_max[i]
# zdr_mean_arr[i] = zdr_mean[i]
# zdr_centroid_lons = np.asarray(zdr_centroid_lon)
# zdr_centroid_lats = np.asarray(zdr_centroid_lat)
# zdr_con_areas = []
# zdr_con_maxes = []
# zdr_con_means = []
# zdr_con_centroid_lon = []
# zdr_con_centroid_lat = []
# zdr_con_max_lon = []
# zdr_con_max_lat = []
# zdr_con_storm_lon = []
# zdr_con_storm_lat = []
# zdr_con_masks = []
# zdr_con_dev = []
# zdr_con_10max = []
# zdr_con_mode = []
# zdr_con_median = []
# zdr_masks = np.asarray(zdr_masks)
# #Consolidate KDP objects as well
# kdp_areas_arr = np.zeros((len(kdp_areas)))
# kdp_max_arr = np.zeros((len(kdp_max)))
# for i in range(len(kdp_areas)):
# kdp_areas_arr[i] = kdp_areas[i].magnitude
# kdp_max_arr[i] = kdp_max[i]
# kdp_centroid_lons = np.asarray(kdp_centroid_lon)
# kdp_centroid_lats = np.asarray(kdp_centroid_lat)
# kdp_con_areas = []
# kdp_con_maxes = []
# kdp_con_centroid_lon = []
# kdp_con_centroid_lat = []
# kdp_con_max_lon = []
# kdp_con_max_lat = []
# kdp_con_storm_lon = []
# kdp_con_storm_lat = []
#Consolidate Hail objects as well
# hail_areas_arr = np.zeros((len(hail_areas)))
# for i in range(len(hail_areas)):
# hail_areas_arr[i] = hail_areas[i].magnitude
# hail_centroid_lons = np.asarray(hail_centroid_lon)
# hail_centroid_lats = np.asarray(hail_centroid_lat)
# hail_con_areas = []
# hail_con_centroid_lon = []
# hail_con_centroid_lat = []
# hail_con_storm_lon = []
# hail_con_storm_lat = []
#Consolidate Zhh objects as well
# zhh_areas_arr = np.zeros((len(zhh_areas)))
# zhh_max_arr = np.zeros((len(zhh_max)))
# zhh_core_avg_arr = np.zeros((len(zhh_core_avg)))
# for i in range(len(zhh_areas)):
# zhh_areas_arr[i] = zhh_areas[i].magnitude
# zhh_max_arr[i] = zhh_max[i]
# zhh_core_avg_arr[i] = zhh_core_avg[i]
# zhh_centroid_lons = np.asarray(zhh_centroid_lon)
# zhh_centroid_lats = np.asarray(zhh_centroid_lat)
# zhh_con_areas = []
# zhh_con_maxes = []
# zhh_con_core_avg = []
# zhh_con_centroid_lon = []
# zhh_con_centroid_lat = []
# zhh_con_max_lon = []
# zhh_con_max_lat = []
# zhh_con_storm_lon = []
# zhh_con_storm_lat = []
# for i in enumerate(max_lons_c):
# try:
# #Find the arc objects associated with this storm:
# zdr_objects_lons = zdr_centroid_lons[np.where(zdr_storm_lon == max_lons_c[i[0]])]
# zdr_objects_lats = zdr_centroid_lats[np.where(zdr_storm_lon == max_lons_c[i[0]])]
# #Get the sum of their areas
# zdr_con_areas.append(np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
# #print("consolidated area", np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
# zdr_con_maxes.append(np.max(zdr_max_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
# #print("consolidated max", np.max(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
# zdr_con_means.append(np.mean(zdr_mean_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
# #print("consolidated mean", np.mean(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
# zdr_con_max_lon.append(rlons_2d[np.where(ZDRmasked==np.max(zdr_max_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))])
# zdr_con_max_lat.append(rlats_2d[np.where(ZDRmasked==np.max(zdr_max_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))])
# #Find the actual centroids
# weighted_lons = zdr_objects_lons * zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]
# zdr_con_centroid_lon.append(np.sum(weighted_lons) / np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
# weighted_lats = zdr_objects_lats * zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]
# zdr_con_centroid_lat.append(np.sum(weighted_lats) / np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
# zdr_con_storm_lon.append(max_lons_c[i[0]])
# zdr_con_storm_lat.append(max_lats_c[i[0]])
# zdr_con_masks.append(np.sum(zdr_masks[np.where(zdr_storm_lon == max_lons_c[i[0]])],axis=0, dtype=bool))
# mask_con = np.sum(zdr_masks[np.where(zdr_storm_lon == max_lons_c[i[0]])], axis=0, dtype=bool)
# zdr_con_dev.append(np.std(ZDRmasked[mask_con]))
# ZDRsorted = np.sort(ZDRmasked[mask_con])[::-1]
# zdr_con_10max.append(np.mean(ZDRsorted[0:10]))
# zdr_con_mode.append(stats.mode(ZDRmasked[mask_con]))
# zdr_con_median.append(np.median(ZDRmasked[mask_con]))
# except:
# zdr_con_maxes.append(0)
# zdr_con_means.append(0)
# zdr_con_centroid_lon.append(0)
# zdr_con_centroid_lat.append(0)
# zdr_con_max_lon.append(0)
# zdr_con_max_lat.append(0)
# zdr_con_storm_lon.append(max_lons_c[i[0]])
# zdr_con_storm_lat.append(max_lats_c[i[0]])
# zdr_con_masks.append(0)
# zdr_con_dev.append(0)
# zdr_con_10max.append(0)
# zdr_con_mode.append(0)
# zdr_con_median.append(0)
# try:
# #Find the kdp objects associated with this storm:
# kdp_objects_lons = kdp_centroid_lons[np.where(kdp_storm_lon == max_lons_c[i[0]])]
# kdp_objects_lats = kdp_centroid_lats[np.where(kdp_storm_lon == max_lons_c[i[0]])]
# #Get the sum of their areas
# kdp_con_areas.append(np.sum(kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
# kdp_con_maxes.append(np.max(kdp_max_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
# kdp_con_max_lon.append(rlons_2d[np.where(KDPmasked==np.max(kdp_max_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))])
# kdp_con_max_lat.append(rlats_2d[np.where(KDPmasked==np.max(kdp_max_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))])
# #Find the actual centroids
# weighted_lons_kdp = kdp_objects_lons * kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]
# kdp_con_centroid_lon.append(np.sum(weighted_lons_kdp) / np.sum(kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
# weighted_lats_kdp = kdp_objects_lats * kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]
# kdp_con_centroid_lat.append(np.sum(weighted_lats_kdp) / np.sum(kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
# kdp_con_storm_lon.append(max_lons_c[i[0]])
# kdp_con_storm_lat.append(max_lats_c[i[0]])
# except:
# kdp_con_maxes.append(0)
# kdp_con_max_lon.append(0)
# kdp_con_max_lat.append(0)
# kdp_con_centroid_lon.append(0)
# kdp_con_centroid_lat.append(0)
# kdp_con_storm_lon.append(0)
# kdp_con_storm_lat.append(0)
# try:
# #Find the hail core objects associated with this storm:
# hail_objects_lons = hail_centroid_lons[np.where(hail_storm_lon == max_lons_c[i[0]])]
# hail_objects_lats = hail_centroid_lats[np.where(hail_storm_lon == max_lons_c[i[0]])]
# #Get the sum of their areas
# hail_con_areas.append(np.sum(hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]))
# #Find the actual centroids
# weighted_lons_hail = hail_objects_lons * hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]
# hail_con_centroid_lon.append(np.sum(weighted_lons_hail) / np.sum(hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]))
# weighted_lats_hail = hail_objects_lats * hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]
# hail_con_centroid_lat.append(np.sum(weighted_lats_hail) / np.sum(hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]))
# hail_con_storm_lon.append(max_lons_c[i[0]])
# hail_con_storm_lat.append(max_lats_c[i[0]])
# except:
# hail_con_centroid_lon.append(0)
# hail_con_centroid_lat.append(0)
# hail_con_storm_lon.append(0)
# hail_con_storm_lat.append(0)
# try:
# #Find the zhh objects associated with this storm:
# zhh_objects_lons = zhh_centroid_lons[np.where(zhh_storm_lon == max_lons_c[i[0]])]
# zhh_objects_lats = zhh_centroid_lats[np.where(zhh_storm_lon == max_lons_c[i[0]])]
# #Get the sum of their areas
# zhh_con_areas.append(np.sum(zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
# zhh_con_maxes.append(np.max(zhh_max_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
# zhh_con_core_avg.append(np.max(zhh_core_avg_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
# zhh_con_max_lon.append(rlons_2d[np.where(REFmasked==np.max(zhh_max_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))])
# zhh_con_max_lat.append(rlats_2d[np.where(REFmasked==np.max(zhh_max_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))])
# #Find the actual centroids
# weighted_lons_zhh = zhh_objects_lons * zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]
# zhh_con_centroid_lon.append(np.sum(weighted_lons_zhh) / np.sum(zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
# weighted_lats_zhh = zhh_objects_lats * zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]
# zhh_con_centroid_lat.append(np.sum(weighted_lats_zhh) / np.sum(zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
# zhh_con_storm_lon.append(max_lons_c[i[0]])
# zhh_con_storm_lat.append(max_lats_c[i[0]])
# except:
# zhh_con_maxes.append(0)
# zhh_con_core_avg.append(0)
# zhh_con_max_lon.append(0)
# zhh_con_max_lat.append(0)
# zhh_con_centroid_lon.append(0)
# zhh_con_centroid_lat.append(0)
# zhh_con_storm_lon.append(0)
# zhh_con_storm_lat.append(0)
#Calculate KDP-ZDR separation
# kdp_con_centroid_lons1 = np.asarray(kdp_con_centroid_lon)
# kdp_con_centroid_lats1 = np.asarray(kdp_con_centroid_lat)
# zdr_con_centroid_lons1 = np.asarray(zdr_con_centroid_lon)
# zdr_con_centroid_lats1 = np.asarray(zdr_con_centroid_lat)
# #Eliminate consolidated arcs smaller than a specified area
# area = 2 #km*2
# zdr_con_areas_arr = np.asarray(zdr_con_areas)
# zdr_con_centroid_lats = zdr_con_centroid_lats1[zdr_con_areas_arr > area]
# zdr_con_centroid_lons = zdr_con_centroid_lons1[zdr_con_areas_arr > area]
# kdp_con_centroid_lats = kdp_con_centroid_lats1[zdr_con_areas_arr > area]
# kdp_con_centroid_lons = kdp_con_centroid_lons1[zdr_con_areas_arr > area]
# zdr_con_max_lons1 = np.asarray(zdr_con_max_lon)[zdr_con_areas_arr > area]
# zdr_con_max_lats1 = np.asarray(zdr_con_max_lat)[zdr_con_areas_arr > area]
# kdp_con_max_lons1 = np.asarray(kdp_con_max_lon)[zdr_con_areas_arr > area]
# kdp_con_max_lats1 = np.asarray(kdp_con_max_lat)[zdr_con_areas_arr > area]
# zdr_con_max1 = np.asarray(zdr_con_maxes)[zdr_con_areas_arr > area]
# zdr_con_areas1 = zdr_con_areas_arr[zdr_con_areas_arr > area]
# kdp_con_centroid_lat = np.asarray(kdp_con_centroid_lat)
# kdp_con_centroid_lon = np.asarray(kdp_con_centroid_lon)
# zdr_con_centroid_lat = np.asarray(zdr_con_centroid_lat)
# zdr_con_centroid_lon = np.asarray(zdr_con_centroid_lon)
# kdp_inds = np.where(kdp_con_centroid_lat*zdr_con_centroid_lat > 0)
# distance_kdp_zdr = g.inv(kdp_con_centroid_lon[kdp_inds], kdp_con_centroid_lat[kdp_inds], zdr_con_centroid_lon[kdp_inds], zdr_con_centroid_lat[kdp_inds])
# dist_kdp_zdr = distance_kdp_zdr[2] / 1000.
# #Now make an array for the distances which will have the same shape as the lats to prevent errors
# shaped_dist = np.zeros((np.shape(zdr_con_areas)))
# shaped_dist[kdp_inds] = dist_kdp_zdr
# #Get separation angle for KDP-ZDR centroids
# back_k = distance_kdp_zdr[1]
# for i in range(back_k.shape[0]):
# if distance_kdp_zdr[1][i] < 0:
# back_k[i] = distance_kdp_zdr[1][i] + 360
# forw_k = np.abs(back_k - storm_relative_dir)
# rawangle_k = back_k - storm_relative_dir
# #Account for weird angles
# for i in range(back_k.shape[0]):
# if forw_k[i] > 180:
# forw_k[i] = 360 - forw_k[i]
# rawangle_k[i] = (360-forw_k[i])*(-1)
# rawangle_k = rawangle_k*(-1)
# #Now make an array for the distances which will have the same shape as the lats to prevent errors
# shaped_ang = np.zeros((np.shape(zdr_con_areas)))
# shaped_ang[kdp_inds] = rawangle_k
# shaped_ang = (180-np.abs(shaped_ang))*(shaped_ang/np.abs(shaped_ang))
# new_angle_all = shaped_ang + storm_relative_dir
# shaped_ang = (new_angle_all - Bunkers_m)* (-1)
# shaped_ang = 180 - shaped_ang
###Now let's consolidate everything to fit the Pandas dataframe!
# p_zdr_areas = []
# p_zdr_maxes = []
# p_zdr_means = []
# p_zdr_devs = []
# p_zdr_10max = []
# p_zdr_mode = []
# p_zdr_median = []
# # p_hail_areas = []
# p_zhh_areas = []
# p_zhh_maxes = []
# p_zhh_core_avgs = []
# p_separations = []
# p_sp_angle = []
# for storm in enumerate(max_lons_c):
# matching_ind = np.flatnonzero(np.isclose(max_lons_c[storm[0]], zdr_con_storm_lon, rtol=1e-05))
# if matching_ind.shape[0] > 0:
# p_zdr_areas.append((zdr_con_areas[matching_ind[0]]))
# p_zdr_maxes.append((zdr_con_maxes[matching_ind[0]]))
# p_zdr_means.append((zdr_con_means[matching_ind[0]]))
# p_zdr_devs.append((zdr_con_dev[matching_ind[0]]))
# p_zdr_10max.append((zdr_con_10max[matching_ind[0]]))
# p_zdr_mode.append((zdr_con_mode[matching_ind[0]]))
# p_zdr_median.append((zdr_con_median[matching_ind[0]]))
# p_separations.append((shaped_dist[matching_ind[0]]))
# p_sp_angle.append((shaped_ang[matching_ind[0]]))
# else:
# p_zdr_areas.append((0))
# p_zdr_maxes.append((0))
# p_zdr_means.append((0))
# p_zdr_devs.append((0))
# p_zdr_10max.append((0))
# p_zdr_mode.append((0))
# p_zdr_median.append((0))
# p_separations.append((0))
# p_sp_angle.append((0))
# matching_ind_hail = np.flatnonzero(np.isclose(max_lons_c[storm[0]], hail_con_storm_lon, rtol=1e-05))
# if matching_ind_hail.shape[0] > 0:
# p_hail_areas.append((hail_con_areas[matching_ind_hail[0]]))
# else:
# p_hail_areas.append((0))
# matching_ind_zhh = np.flatnonzero(np.isclose(max_lons_c[storm[0]],zhh_con_storm_lon, rtol=1e-05))
# if matching_ind_zhh.shape[0] > 0:
# p_zhh_maxes.append((zhh_con_maxes[matching_ind_zhh[0]]))
# p_zhh_areas.append((zhh_con_areas[matching_ind_zhh[0]]))
# p_zhh_core_avgs.append((zhh_con_core_avg[matching_ind_zhh[0]]))
# else:
# p_zhh_areas.append((0))
# p_zhh_maxes.append((0))
# p_zhh_core_avgs.append((0))
#Now start plotting stuff!
# if np.asarray(zdr_centroid_lon).shape[0] > 0:
# ax.scatter(zdr_centroid_lon, zdr_centroid_lat, marker = '*', s = 100, color = 'black', zorder = 10, transform=ccrs.PlateCarree())
# if np.asarray(kdp_centroid_lon).shape[0] > 0:
# ax.scatter(kdp_centroid_lon, kdp_centroid_lat, marker = '^', s = 100, color = 'black', zorder = 10, transform=ccrs.PlateCarree())
#Uncomment to print all object areas
#for i in enumerate(zdr_areas):
# plt.text(zdr_centroid_lon[i[0]]+.016, zdr_centroid_lat[i[0]]+.016, "%.2f km^2" %(zdr_areas[i[0]].magnitude), size = 23)
#plt.text(zdr_centroid_lon[i[0]]+.016, zdr_centroid_lat[i[0]]+.016, "%.2f km^2 / %.2f km / %.2f dB" %(zdr_areas[i[0]].magnitude, zdr_dist[i[0]], zdr_forw[i[0]]), size = 23)
#plt.annotate(zdr_areas[i[0]], (zdr_centroid_lon[i[0]],zdr_centroid_lat[i[0]]))
#ax.contourf(rlons[0,:,:],rlats[0,:,:],KDPmasked,KDPlevels1,linewide = .01, colors ='b', alpha = .5)
#plt.tight_layout()
#plt.savefig('ZDRarcannotated.png')
storm_times = []
for l in range(len(max_lons_c)):
storm_times.append((time_start))
tracking_index = tracking_index + 1
#Get storm motion deltas:
u_B, v_B = wind_components(Bunkers_s*units('m/s'), Bunkers_m*units('degree'))
u_alg, v_alg = wind_components(alg_speeds*units('m/s'), alg_directions*units('degree'))
print(u_B, v_B, 'Bunkers motion components')
print(u_alg, v_alg, 'Observed motion components')
u_diff = u_alg-u_B
v_diff = v_alg-v_B
motion_delta = np.sqrt(u_diff**2 + v_diff**2).magnitude
#If there are no storms, set everything to empty arrays!
else:
storm_ids = []
storm_ids = []
alg_speeds = []
alg_directions = []
motion_delta = []
max_lons_c = []
max_lats_c = []
storm_times = time_start
#Now record all data in a Pandas dataframe.
new_cells = pd.DataFrame({
'scan': scan_index,
'storm_id' : storm_ids,
'storm speed' : alg_speeds,
'storm_direction' : alg_directions,
'motion_deltas' : motion_delta,
'storm_id1' : storm_ids,
'storm_lon' : max_lons_c,
'storm_lat' : max_lats_c,
'times' : storm_times
})
new_cells.set_index(['scan', 'storm_id'], inplace=True)
if scan_index == 0:
tracks_dataframe = new_cells
else:
tracks_dataframe = tracks_dataframe.append(new_cells)
n = n+1
scan_index = scan_index + 1
#Plot the consolidated stuff!
#Write some text objects for the ZDR arc attributes to add to the placefile
f.write('TimeRange: '+str(time_start.year)+'-'+str(month)+'-'+str(d_beg)+'T'+str(h_beg)+':'+str(min_beg)+':'+str(sec_beg)+'Z '+str(time_start.year)+'-'+str(month)+'-'+str(d_end)+'T'+str(h_end)+':'+str(min_end)+':'+str(sec_end)+'Z')
f.write('\n')
f.write("Color: 139 000 000 \n")
f.write('Font: 1, 30, 1,"Arial" \n')
for y in range(len(max_lats_c)):
#f.write('Text: '+str(max_lats_c[y])+','+str(max_lons_c[y])+', 1, "X"," Arc Area: '+str(p_zdr_areas[y])+'\\n Arc Mean: '+str(p_zdr_means[y])+'\\n KDP-ZDR Separation: '+str(p_separations[y])+'\\n Separation Angle: '+str(p_sp_angle[y])+'" \n')
f.write('Text: '+str(max_lats_c[y])+','+str(max_lons_c[y])+', 1, "X"," Storm Speed: %.2f m/s \\n Storm Direction: %.2f deg \\n Motion Delta: %.2f m/s \n' %(alg_speeds[y], alg_directions[y], motion_delta[y]))
title_plot = plt.title(station+' Radar Reflectivity, ZDR, and KDP '+str(time_start.year)+'-'+str(time_start.month)+'-'+str(time_start.day)+
' '+str(hour)+':'+str(minute)+' UTC', size = 25)
# try:
# plt.plot([zdr_con_centroid_lon[kdp_inds], kdp_con_centroid_lon[kdp_inds]], [zdr_con_centroid_lat[kdp_inds],kdp_con_centroid_lat[kdp_inds]], color = 'k', linewidth = 5, transform=ccrs.PlateCarree())
# except:
# print('Separation Angle Failure')
ref_centroid_lon = max_lons_c
ref_centroid_lat = max_lats_c
if len(max_lons_c) > 0:
ax.scatter(max_lons_c,max_lats_c, marker = "o", color = 'k', s = 500, alpha = .6)
for i in enumerate(ref_centroid_lon):
plt.text(ref_centroid_lon[i[0]]+.016, ref_centroid_lat[i[0]]+.016, "storm_id: %.1f" %(storm_ids[i[0]]), size = 25)
#Comment out this line if not plotting tornado tracks
#plt.plot([start_torlons, end_torlons], [start_torlats, end_torlats], color = 'purple', linewidth = 5, transform=ccrs.PlateCarree())
#Add legend stuff
zdr_outline = mlines.Line2D([], [], color='blue', linewidth = 5, linestyle = 'solid', label='ZDR Arc Outline(Area/Max)')
kdp_outline = mlines.Line2D([], [], color='green', linewidth = 5,linestyle = 'solid', label='"KDP Foot" Outline')
separation_vector = mlines.Line2D([], [], color='black', linewidth = 5,linestyle = 'solid', label='KDP/ZDR Centroid Separation Vector (Red Text=Distance)')
#tor_track = mlines.Line2D([], [], color='purple', linewidth = 5,linestyle = 'solid', label='Tornado Tracks')
elevation = mlines.Line2D([], [], color='grey', linewidth = 5,linestyle = 'solid', label='Height AGL (m)')
plt.legend(handles=[zdr_outline, kdp_outline, separation_vector, elevation], loc = 3, fontsize = 25)
alt_levs = [1000, 2000]
plt.savefig('Machine_Learning/DELTA_dev'+station+str(time_start.year)+str(time_start.month)+str(day)+str(hour)+str(minute)+'.png')
print('Figure Saved')
plt.close()
zdr_out_list.append(zdr_outlines)
#except:
# traceback.print_exc()
# continue
f.close()
plt.show()
print('Fin')
#export_csv = tracks_dataframe.to_csv(r'C:\Users\Nick\Downloads\tracksdataframe.csv',index=None,header=True)
return tracks_dataframe
def grid_displacement_pc(grid1, grid2, field, level, return_value='pixels'):
"""
Calculate the grid displacement using phase correlation.
See:
http://en.wikipedia.org/wiki/Phase_correlation
Implementation inspired by Christoph Gohlke:
http://www.lfd.uci.edu/~gohlke/code/imreg.py.html
Note that the grid must have the same dimensions in x and y and assumed to
have constant spacing in these dimensions.
Parameters
----------
grid1, grid2 : Grid
Py-ART Grid objects separated in time and square in x/y.
field : string
Field to calculate advection from. Field must be in both grid1
and grid2.
level : integer
The vertical (z) level of the grid to use in the calculation.
return_value : str, optional
'pixels', 'distance' or 'velocity'. Distance in pixels (default)
or meters or velocity vector in m/s.
Returns
-------
displacement : two-tuple
Calculated displacement in units of y and x. Value returned in
integers if pixels, otherwise floats.
"""
# create copies of the data
field_data1 = grid1.fields[field]['data'][level].copy()
field_data2 = grid2.fields[field]['data'][level].copy()
# replace fill values with valid_min or minimum value in array
if 'valid_min' in grid1.fields[field]:
min_value1 = grid1.fields[field]['valid_min']
else:
min_value1 = field_data1.min()
field_data1 = np.ma.filled(field_data1, min_value1)
if 'valid_min' in grid2.fields[field]:
min_value2 = grid2.fields[field]['valid_min']
else:
min_value2 = field_data2.min()
field_data2 = np.ma.filled(field_data2, min_value2)
# discrete fast fourier transformation and complex conjugation of field 2
image1fft = np.fft.fft2(field_data1)
image2fft = np.conjugate(np.fft.fft2(field_data2))
# inverse fourier transformation of product -> equal to cross correlation
imageccor = np.real(np.fft.ifft2((image1fft*image2fft)))
# shift the zero-frequency component to the center of the spectrum
imageccorshift = np.fft.fftshift(imageccor)
# determine the distance of the maximum from the center
# find the peak in the correlation
row, col = field_data1.shape
yshift, xshift = np.unravel_index(np.argmax(imageccorshift), (row, col))
yshift -= int(row/2)
xshift -= int(col/2)
dx = grid1.x['data'][1] - grid1.x['data'][0]
dy = grid1.y['data'][1] - grid1.y['data'][0]
x_movement = xshift * dx
y_movement = yshift * dy
if return_value == 'pixels':
displacement = (yshift, xshift)
elif return_value == 'distance':
displacement = (y_movement, x_movement)
elif return_value == 'velocity':
t1 = num2date(grid1.time['data'][0], grid1.time['units'])
t2 = num2date(grid2.time['data'][0], grid2.time['units'])
dt = (t2 - t1).total_seconds()
u = x_movement/dt
v = y_movement/dt
displacement = (v, u)
else:
displacement = (yshift, xshift)
return displacement
def in_out_water(netCDFfile, var_name=None):
out_file = []
for fn in netCDFfile:
ds = Dataset(fn, 'a')
nc_vars = ds.variables
to_add = []
if var_name:
to_add.append(var_name)
else:
for v in nc_vars:
if "TIME" in nc_vars[v].dimensions:
#print (vars[v].dimensions)
if v != 'TIME':
to_add.append(v)
# remove any anx variables from the list
for v in nc_vars:
if 'ancillary_variables' in nc_vars[v].ncattrs():
remove = nc_vars[v].getncattr('ancillary_variables').split(' ')
print("remove ", remove)
for r in remove:
to_add.remove(r)
time_var = nc_vars["TIME"]
time = num2date(time_var[:], units=time_var.units, calendar=time_var.calendar)
time_deploy = parser.parse(ds.time_deployment_start, ignoretz=True)
time_recovery = parser.parse(ds.time_deployment_end, ignoretz=True)
print('file', fn)
print('deployment time', time_deploy)
print('var to add', to_add)
# create a mask for the time range
mask = (time <= time_deploy) | (time >= time_recovery)
count = -1
for v in to_add:
if v in nc_vars:
print("var", v, ' dimensions ', nc_vars[v].dimensions)
ncVarOut = nc_vars[v + "_quality_control"]
ncVarOut[mask] = 6
# create a qc variable just for this test flags
if v + "_quality_control_io" in ds.variables:
ncVarOut = ds.variables[v + "_quality_control_io"]
ncVarOut[:] = 0
else:
ncVarOut = ds.createVariable(v + "_quality_control_io", "i1", nc_vars[v].dimensions, fill_value=99, zlib=True) # fill_value=0 otherwise defaults to max
nc_vars[v].ancillary_variables = nc_vars[v].ancillary_variables + " " + v + "_quality_control_io"
ncVarOut[:] = 0
ncVarOut.long_name = "quality flag for " + nc_vars[v].long_name
try:
ncVarOut.standard_name = nc_vars[v].standard_name + " status_flag"
except AttributeError:
pass
ncVarOut.quality_control_conventions = "IMOS standard flags"
ncVarOut.flag_values = np.array([0, 1, 2, 3, 4, 6, 7, 9], dtype=np.int8)
ncVarOut.flag_meanings = 'unknown good_data probably_good_data probably_bad_data bad_data not_deployed interpolated missing_value'
ncVarOut.comment = 'data flagged not deployed (6) when out of water'
ncVarOut[mask] = 6
# calculate the number of points marked as bad_data
marked = np.zeros_like(ncVarOut)
marked[mask] = 1
count = sum(marked)
ds.file_version = "Level 1 - Quality Controlled Data"
if count > 0:
# update the history attribute
try:
hist = ds.history + "\n"
except AttributeError:
hist = ""
ds.setncattr('history', hist + datetime.utcnow().strftime("%Y-%m-%d") + ' : ' + ' marked ' + str(int(count)))
ds.close()
out_file.append(fn)
return out_file
def test_cf_datetime(self):
import netCDF4 as nc4
for num_dates, units in [
(np.arange(10), 'days since 2000-01-01'),
(np.arange(10).reshape(2, 5), 'days since 2000-01-01'),
(12300 + np.arange(5), 'hours since 1680-01-01 00:00:00'),
# here we add a couple minor formatting errors to test
# the robustness of the parsing algorithm.
(12300 + np.arange(5), 'hour since 1680-01-01 00:00:00'),
(12300 + np.arange(5), u'Hour since 1680-01-01 00:00:00'),
(12300 + np.arange(5), ' Hour since 1680-01-01 00:00:00 '),
(10, 'days since 2000-01-01'),
([10], 'daYs since 2000-01-01'),
([[10]], 'days since 2000-01-01'),
([10, 10], 'days since 2000-01-01'),
(np.array(10), 'days since 2000-01-01'),
(0, 'days since 1000-01-01'),
([0], 'days since 1000-01-01'),
([[0]], 'days since 1000-01-01'),
(np.arange(2), 'days since 1000-01-01'),
(np.arange(0, 100000, 20000), 'days since 1900-01-01'),
(17093352.0, 'hours since 1-1-1 00:00:0.0'),
([0.5, 1.5], 'hours since 1900-01-01T00:00:00'),
(0, 'milliseconds since 2000-01-01T00:00:00'),
(0, 'microseconds since 2000-01-01T00:00:00'),
]:
for calendar in ['standard', 'gregorian', 'proleptic_gregorian']:
expected = _ensure_naive_tz(
nc4.num2date(num_dates, units, calendar))
print(num_dates, units, calendar)
with warnings.catch_warnings():
warnings.filterwarnings('ignore',
'Unable to decode time axis')
actual = conventions.decode_cf_datetime(
num_dates, units, calendar)
if (isinstance(actual, np.ndarray)
and np.issubdtype(actual.dtype, np.datetime64)):
# self.assertEqual(actual.dtype.kind, 'M')
# For some reason, numpy 1.8 does not compare ns precision
# datetime64 arrays as equal to arrays of datetime objects,
# but it works for us precision. Thus, convert to us
# precision for the actual array equal comparison...
actual_cmp = actual.astype('M8[us]')
else:
actual_cmp = actual
self.assertArrayEqual(expected, actual_cmp)
encoded, _, _ = conventions.encode_cf_datetime(
actual, units, calendar)
if '1-1-1' not in units:
# pandas parses this date very strangely, so the original
# units/encoding cannot be preserved in this case:
# (Pdb) pd.to_datetime('1-1-1 00:00:0.0')
# Timestamp('2001-01-01 00:00:00')
self.assertArrayEqual(num_dates, np.around(encoded, 1))
if (hasattr(num_dates, 'ndim') and num_dates.ndim == 1
and '1000' not in units):
# verify that wrapping with a pandas.Index works
# note that it *does not* currently work to even put
# non-datetime64 compatible dates into a pandas.Index :(
encoded, _, _ = conventions.encode_cf_datetime(
pd.Index(actual), units, calendar)
self.assertArrayEqual(num_dates, np.around(encoded, 1))
def load_history(filename,
start_time=datetime(1, 1, 1),
end_time=datetime(1, 1, 1),
grid=None,
epoch=default_epoch,
url=_url,
load_data=False):
"""
Download soda data and save into local file
Parameters
----------
filename: string
name of output file
start_time: datetime
starting date to load soda data
end_time: datetime
ending date for loading soda data
grid: seapy.model.grid, optional
if specified, only load SODA data that covers the grid
epoch: datetime, optional
reference time for new file
url: string, optional
URL to load SODA data from
load_data: bool, optional
If true (default) actually load the data. If false, it
displays the information needed to load the data using ncks
Returns
-------
None
"""
# Load the grid
grid = asgrid(grid)
# Open the soda data
soda = netCDF4.Dataset(url)
# Figure out the time records that are required
soda_time = netCDF4.num2date(soda.variables["time"][:],
soda.variables["time"].units)
time_list = np.where(
np.logical_and(soda_time >= start_time, soda_time <= end_time))
if not any(time_list):
raise Exception("Cannot find valid times")
# Get the latitude and longitude ranges
minlat = np.min(grid.lat_rho) - 0.5
maxlat = np.max(grid.lat_rho) + 0.5
minlon = np.min(grid.lon_rho) - 0.5
maxlon = np.max(grid.lon_rho) + 0.5
soda_lon = soda.variables["lon"][:]
soda_lat = soda.variables["lat"][:]
# Ensure same convention
if not grid.east():
soda_lon[soda_lon > 180] -= 360
latlist = np.where(np.logical_and(soda_lat >= minlat, soda_lat <= maxlat))
lonlist = np.where(np.logical_and(soda_lon >= minlon, soda_lon <= maxlon))
if not np.any(latlist) or not np.any(lonlist):
raise Exception("Bounds not found")
# Build the history file
if load_data:
his = ncgen.create_zlevel(filename,
len(latlist[0]),
len(lonlist[0]),
len(soda.variables["lev"][:]),
epoch,
"soda history from " + url,
dims=1)
# Write out the data
his.variables["lat"][:] = soda_lat[latlist]
his.variables["lon"][:] = soda_lon[lonlist]
his.variables["depth"][:] = soda.variables["lev"]
his.variables["time"][:] = netCDF4.date2num(
soda_time[time_list], his.variables["time"].units)
# Loop over the variables
sodavars = {"ssh": 3, "u": 4, "v": 4, "temp": 4, "salt": 4}
hisvars = {
"ssh": "zeta",
"u": "u",
"v": "v",
"temp": "temp",
"salt": "salt"
}
if not load_data:
print(
"ncks -v {:s} -d time,{:d},{:d} -d lat,{:d},{:d} -d lon,{:d},{:d} {:s} {:s}"
.format(",".join(sodavars.keys()), time_list[0][0],
time_list[0][-1], latlist[0][0], latlist[0][-1],
lonlist[0][0], lonlist[0][-1], _url, filename))
else:
for rn, recs in enumerate(chunker(time_list[0], _maxrecs)):
print("{:s}-{:s}: ".format(
soda_time[recs[0]].strftime("%m/%d/%Y"),
soda_time[recs[-1]].strftime("%m/%d/%Y")),
end='',
flush=True)
for var in sodavars:
print("{:s} ".format(var), end='', flush=True)
hisrange = np.arange(rn * _maxrecs,
(rn * _maxrecs) + len(recs))
if sodavars[var] == 3:
his.variables[hisvars[var]][hisrange, :, :] = \
np.ma.array(
soda.variables[var][recs, latlist[0], lonlist[0]]). \
filled(fill_value=9.99E10)
else:
his.variables[hisvars[var]][hisrange, :, :, :] = \
soda.variables[var][recs, :, latlist[0],
lonlist[0]].filled(fill_value=9.99E10)
his.sync()
print("", flush=True)
pass
cblabel = 'temperature 2 m [C]' #'3h acumm precipitation [mm]' #
time_units = 'seconds since 2010-01-01 00:00:00'
# map data
members = 3
treshhold = 0.01 # mm / h
# grd_data = np.where(grd_data < treshhold, np.nan, grd_data)
clevs, cmaps = make_cmap(temp()) #[0], precip1()
for date, grid, obs in zip(dates, grd_data, obs_data):
obs_dict['colors'] = obs
title = num2date(date, time_units)
print('Plotting date: {}'.format(title) )
print('--> observed: minimum value: {:.2f}, maximum value: {:.2f}'.format(obs.min(), obs.max()))
for m in range(members):
grid_m = grid[...,m]
print('--> member: {}, minimum value: {:.2f}, maximum value: {:.2f}'.format(m+1, grid_m.min(), grid_m.max()))
mapa = mu.get_mapa(lon_lat[0], lon_lat[1], proj=proj_dict)
fig = mu.graphgtmap(grd_x, grd_y, grid_m, mapa, cblabel, title,
clevs, cmaps, asp, calpha=0.9, stn_dict=obs_dict,
fig_dpi=300)
fname = 'map_' + str(title).replace(' ', '_') + '_member:'+str(m+1)
fig.savefig(os.path.join('outputs/plots', domain, var_key, fname + '.png'))
time_var_name = name
break
time_var = data.variables[time_var_name]
lat_var = data.variables['lat']
lon_var = data.variables['lon']
# Get actual data values and remove any size 1 dimensions
lat = lat_var[:].squeeze()
lon = lon_var[:].squeeze()
height = height_var[0, 0, :, :].squeeze()
u_wind = u_wind_var[0, 0, :, :].squeeze() * units('m/s')
v_wind = v_wind_var[0, 0, :, :].squeeze() * units('m/s')
# Convert number of hours since the reference time into an actual date
time = num2date(time_var[:].squeeze(), time_var.units)
# Combine 1D latitude and longitudes into a 2D grid of locations
lon_2d, lat_2d = np.meshgrid(lon, lat)
# Smooth height data
height = ndimage.gaussian_filter(height, sigma=1.5, order=0)
# Set up some constants based on our projection, including the Coriolis parameter and
# grid spacing, converting lon/lat spacing to Cartesian
f = mpcalc.coriolis_parameter(np.deg2rad(lat_2d)).to('1/s')
dx, dy = mpcalc.lat_lon_grid_deltas(lon_2d, lat_2d)
# In MetPy 0.5, geostrophic_wind() assumes the order of the dimensions is (X, Y),
# so we need to transpose from the input data, which are ordered lat (y), lon (x).
# Once we get the components,transpose again so they match our original data.
scT1fd_in = Dataset('shT1_osh_test.nc')
#%% sagehen timming
scT1time = scT1fd_in.variables['time'][:] # get values
t_unitST1 = scT1fd_in.variables[
'time'].units # get unit "days since 1950-01-01T00:00:00Z"
try:
t_cal = scT1fd_in.variables['time'].calendar
except AttributeError: # Attribute doesn't exist
t_cal = u"gregorian" # or standard
tvalueScT1 = num2date(scT1time, units=t_unitST1, calendar=t_cal)
scT1date_in = [i.strftime("%Y-%m-%d %H:%M") for i in tvalueScT1]
#%% make new nc file
new_fc_sc = Dataset("forcing_scT1_open_calib.nc",
'w',
format='NETCDF3_CLASSIC')
# define dimensions
hru = new_fc_sc.createDimension('hru', hru_num)
time = new_fc_sc.createDimension('time', None)
# define variables
hruid = new_fc_sc.createVariable('hruId', np.int32, ('hru', ))
lat = new_fc_sc.createVariable('latitude', np.float64, ('hru', ))
lon = new_fc_sc.createVariable('longitude', np.float64, ('hru', ))
ds = new_fc_sc.createVariable('data_step', np.float64)
times = new_fc_sc.createVariable('time', np.float64, ('time', ))
f = cf.read(datafile)
temperature = f.select('sea_water_temperature')
lon = temperature[0].coord('lon').array
lat = temperature[0].coord('lat').array
time = temperature[0].coord('time').array
temperatureQC = temperature[0].ancillary_variables[0].array
temperature = temperature[0].array
cf.close_files()
temperature = np.ma.masked_where(temperatureQC != 1, temperature)
temperature = np.ma.masked_where(temperature > 32., temperature)
temperature = np.ma.masked_where(temperature < 18., temperature)
print temperature.min()
with netCDF4.Dataset(datafile) as nc:
time2plot = netCDF4.num2date(nc.variables['time'][:], nc.variables['time'].units)
temperature = np.ma.masked_where(lon<coordinates[0], temperature)
lon = np.ma.masked_where(lon<coordinates[0], lon)
lat = np.ma.masked_where(lon<coordinates[0], lat)
time = np.ma.masked_where(lon<coordinates[0], time)
lon, lat = m(lon, lat)
fig = plt.figure(figsize=(15, 6))
ax = fig.add_subplot(122)
monthformat = dates.DateFormatter('%B')
monthticks = dates.MonthLocator(interval=2) # every month
dayxticks = dates.DayLocator() # every month
ax.xaxis.set_major_locator(monthticks)
ax.xaxis.set_major_formatter(monthformat)
from efa_xray.assimilation.ensrf import EnSRF
# directory where the ensemble of all times is
infile = '/home/disk/hot/stangen/Documents/GEFS/ensembles' + \
'/2017090600/2017090600_21mem_10days.nc'
# only variable I am using is 500mb height
vrbls=['Z500','T500','RH500','U500','V500','Z700','T700','RH700','U700','V700', \
'Z850','T850','RH850','U850','V850','Z925','T925','RH925','U925','V925', \
'Z1000','T1000','RH1000','U1000','V1000','T2M','RH2M','U10M','V10M', \
'PWAT','MSLP','P6HR']
# loading/accessing the netcdf data
with Dataset(infile, 'r') as ncdata:
times = ncdata.variables['time']
ftimes = num2date(times[:], times.units)
lats = ncdata.variables['lat'][:]
lons = ncdata.variables['lon'][:]
mems = ncdata.variables['ens'][:]
#print(ncdata.variables)
# storing the variable data in a dict (state?)
allvars = {}
for var in vrbls:
allvars[var] = (['validtime', 'y', 'x',
'mem'], ncdata.variables[var][:])
lonarr, latarr = np.meshgrid(lons, lats)
# Package into an EnsembleState object knowing the state and metadata
statecls = EnsembleState.from_vardict(
allvars, {
def load_time_axis(path):
with netCDF4.Dataset(path) as dataset:
var = dataset.variables["time"]
values = netCDF4.num2date(var[:], units=var.units)
return np.array(values, dtype='datetime64[s]')
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.colors
from matplotlib import ticker
import datetime as dt
import netCDF4 as nc
file_path = "/g/data/w35/mm3972/cable/EucFACE/EucFACE_run/met/"
famb = "EucFACE_met_amb.nc"
amb = nc.Dataset(os.path.join(file_path,famb), 'r')
step_2_sec = 30.*60.
df_amb = pd.DataFrame(amb.variables['Rainf'][:,0], columns=['Rainf'])
df_amb['dates'] = nc.num2date(amb.variables['time'][:],amb.variables['time'].units)
df_amb = df_amb.set_index('dates')
df_amb = df_amb*step_2_sec
df_amb = df_amb.resample("A").agg('sum')
#df_amb = df_amb.drop(df_amb.index[len(df_amb)-1])
df_amb.index = df_amb.index.strftime('%Y')
print(df_amb)
fig, ax = plt.subplots()
ax.scatter(df_amb.index, df_amb.values)
for i, txt in enumerate(df_amb.values):
ax.annotate(txt, (df_amb.index[i], df_amb.values[i]))
for style, info in style_dict.items(): #,'cpd','tas'
for scenario in ['Plus20-Artificial-v1'
]: #'Plus20-Future','Plus15-Future',
state_files = sorted(
glob.glob(working_path + scenario + '/' + style + '/' + style +
'_*_state.nc'))
for state_file in state_files:
print(state_file)
states = info['states']
nc = da.read_nc(state_file)
if 'calendar' in nc['time'].attrs.keys():
datevar = num2date(nc['time'].values,
units=nc['time'].units,
calendar=nc['time'].calendar)
else:
datevar = num2date(nc['time'].values, units=nc['time'].units)
month = np.array([date.month for date in datevar])
if 'stateCount' not in globals():
stateCount = da.Dataset({})
for state in states:
stateCount[state] = da.DimArray(
np.zeros([4, len(nc.lat), len(nc.lon)]),
axes=[seasons.keys(), nc.lat, nc.lon],
dims=['season', 'lat', 'lon'])
stateCount[state + '_possible_days'] = da.DimArray(
np.zeros([4, len(nc.lat), len(nc.lon)]),
axes=[seasons.keys(), nc.lat, nc.lon],
def plot_imd_radar(fpath, ele, R, plotpath):
# Plotting Options Set in Function
# ele = 0 #Elevation
# R = 50# #Min and Mac Range from Radar in km
# Read Files and Setup Plotting
radar = pyart.io.read(fpath)
display = pyart.graph.RadarDisplay(radar)
# set the figure title and show
instrument_name = (radar.metadata['instrument_name'])[0:7]
time_start = netCDF4.num2date(radar.time['data'][0],
radar.time['units'])
time_text = ' ' + time_start.strftime('%Y-%m-%d %H:%M:%SZ')
elevation = radar.fixed_angle['data'][ele]
ele_text = '%0.1f' % (elevation)
title = instrument_name + time_text + ' Elevation %.1f' % (elevation)
if np.round(elevation, 0) > 9.99:
ele_folder = 'ele' + ele_text[0] + ele_text[1] + '_' + ele_text[
3] + '/'
ele_name = 'ele' + ele_text[0] + ele_text[1] + '_' + ele_text[3]
else:
ele_folder = 'ele' + ele_text[0] + '_' + ele_text[2] + '/'
ele_name = 'ele' + ele_text[0] + '_' + ele_text[2]
plot_file_name = instrument_name + '_' + time_start.strftime(
'%Y-%m-%dT%H%M%SZ') + ele_name + '.png'
loc_folder = instrument_name + '/'
R_folder = str(R) + 'km' + '/'
fig_save_file_path = plotpath + loc_folder + ele_folder + R_folder
if os.path.isfile(fig_save_file_path + plot_file_name):
print(fig_save_file_path + plot_file_name + ' exists')
else:
# Figure Options
width = 30 # in inches
height = 10 # in inches
fig = plt.figure(figsize=(width, height))
nrows = 2
ncols = 5
ax1 = fig.add_subplot(nrows, ncols, 1)
display.plot('dBZ',
ele,
ax=ax1,
vmin=-32,
vmax=16.,
title='Horizontal Reflectivity',
colorbar_label=radar.fields['dBZ']['units'],
axislabels=('',
'North South distance from radar (km)'))
display.set_limits((-R, R), (-R, R), ax=ax1)
ax2 = fig.add_subplot(nrows, ncols, 2)
display.plot('ZDR',
ele,
ax=ax2,
vmin=-2,
vmax=10.,
title='Differential Reflectivity',
colorbar_label=radar.fields['ZDR']['units'],
axislabels=('', ''),
cmap='pyart_RefDiff')
display.set_limits((-R, R), (-R, R), ax=ax2)
ax3 = fig.add_subplot(nrows, ncols, 3)
display.plot('RhoHV',
ele,
ax=ax3,
vmin=0,
vmax=1.,
title='Cross Correlation Ratio',
colorbar_label=radar.fields['RhoHV']['units'],
axislabels=('', ''),
cmap='pyart_RefDiff')
display.set_limits((-R, R), (-R, R), ax=ax3)
ax4 = fig.add_subplot(nrows, ncols, 4)
display.plot('KDP',
ele,
ax=ax4,
vmin=-2,
vmax=5.,
title='Specific Differential Phase',
colorbar_label=radar.fields['KDP']['units'],
axislabels=('', ''),
cmap='pyart_Theodore16')
display.set_limits((-R, R), (-R, R), ax=ax4)
ax5 = fig.add_subplot(nrows, ncols, 5)
display.plot('PhiDP',
ele,
ax=ax5,
vmin=-180,
vmax=180.,
title='Differential Phase',
colorbar_label=radar.fields['PhiDP']['units'],
axislabels=('', ''),
cmap='pyart_Wild25')
display.set_limits((-R, R), (-R, R), ax=ax5)
ax6 = fig.add_subplot(nrows, ncols, 6)
display.plot('V',
ele,
ax=ax6,
vmin=-16,
vmax=16.,
title='Doppler Velocity',
colorbar_label=radar.fields['V']['units'],
axislabels=('East West distance from radar (km)',
'North South distance from radar (km)'),
cmap='pyart_BuDRd18')
display.set_limits((-R, R), (-R, R), ax=ax6)
ax7 = fig.add_subplot(nrows, ncols, 7)
display.plot('W',
ele,
ax=ax7,
vmin=0,
vmax=5.,
title='Spectrum Width',
colorbar_label=radar.fields['W']['units'],
axislabels=('East West distance from radar (km)', ''),
cmap='pyart_NWS_SPW')
display.set_limits((-R, R), (-R, R), ax=ax7)
ax8 = fig.add_subplot(nrows, ncols, 8)
display.plot('SNR',
ele,
ax=ax8,
title='SNR',
colorbar_label=radar.fields['SNR']['units'],
axislabels=('East West distance from radar (km)', ''),
cmap='pyart_Carbone17')
display.set_limits((-R, R), (-R, R), ax=ax8)
ax9 = fig.add_subplot(nrows, ncols, 9)
display.plot('SQI',
ele,
ax=ax9,
title='SQI',
colorbar_label=radar.fields['SQI']['units'],
axislabels=('East West distance from radar (km)', ''),
cmap='pyart_Carbone17')
display.set_limits((-R, R), (-R, R), ax=ax9)
ax10 = fig.add_subplot(nrows, ncols, 10)
display.plot('uPhiDP',
ele,
ax=ax10,
vmin=-360,
vmax=360.,
title='Unfiltered Differential Phase',
colorbar_label=radar.fields['uPhiDP']['units'],
axislabels=('East West distance from radar (km)', ''),
cmap='pyart_Wild25')
display.set_limits((-R, R), (-R, R), ax=ax10)
display.plot_cross_hair(3.)
plt.suptitle(title, fontsize=24)
if not os.path.exists(fig_save_file_path):
os.makedirs(fig_save_file_path)
plt.savefig(fig_save_file_path + plot_file_name, dpi=100)
plt.close('all')
del radar
del display
def write_pcr_timeseries(lookup_table,
srcVar,
trgFolder,
trgPrefix,
logger,
slope=1.,
offset=0.,
year=False,
remove_missings=True):
# if necessary, make trgPrefix maximum of 8 characters
if len(trgPrefix) > 8:
trgPrefix = trgPrefix[0:8]
# prepare a counter for the number of files with non-missing values
count = 0
timeList = []
# establish the points in the lookup_table where the dates of interest are located
startDate = dt.datetime(year, 1, 1, 0, 0)
endDate = dt.datetime(year, 12, 31, 0, 0)
allDates = lookup_table['time']
idxs = where(
logical_and(array(allDates) >= startDate,
array(allDates) <= endDate))[0]
# start with an empty string to compare with
srcFile = ''
for idx in idxs:
if logical_and(lookup_table[srcVar][idx] != srcFile,
lookup_table[srcVar][idx] != ''):
# read a new source netCDF file
if srcFile != '':
# if we are not reading the first time, then close the previous file
nc_src.close()
# set the new source file and open it and read x and y-axis and time and the variable of interest
srcFile = lookup_table[srcVar][idx]
nc_src = nc4.Dataset(srcFile, 'r')
# read axis, try 'lat' otherwise try 'latitude'
try:
y = nc_src.variables['lat'][:]
except:
y = nc_src.variables['latitude'][:]
try:
x = nc_src.variables['lon'][:]
except:
x = nc_src.variables['longitude'][:]
# read time axis and convert to time objects
time = nc_src.variables['time']
timeObjRaw = nc4.num2date(time[:],
units=time.units,
calendar=time.calendar)
# UUGGGHHH: this seems to be a bug. calendar other than gregorian give other objects than
# datetime.datetime objects. Here we convert to gregorian numbers, then back to date objects
timeObj = []
for t in timeObjRaw:
timeObj.append(dt.datetime.strptime(t.strftime('%Y%j'),
'%Y%j'))
# timeObj = nc4.num2date(nc4.date2num(timeObj, units=time.units, calendar='gregorian'), units=time.units, calendar='gregorian')
# now loop over all time steps, check the date and write valid dates to a list, write time series to PCRaster maps
for n in range(len(timeObj)):
# remove any hours or minutes data from curTime
timeObj[n] = timeObj[n].replace(hour=0)
timeObj[n] = timeObj[n].replace(minute=0)
# Read the variable of interest
nc_var = nc_src.variables[srcVar]
#### Correct file is read, now read in position ####
# look up the position in the current file, where the current time step is located
curTime = allDates[idx]
location = where(array(timeObj) == curTime)[0]
if len(location) == 1:
# in case of a difference in the used calendar, there may be one or more missing days in a year.
# These are accounted for by ignoring dates that are not found inside the input data files
curGrid = nc_var[location[0], :, :]
if remove_missings:
if hasattr(curGrid, 'mask'):
curMask = curGrid.mask
curGrid = curGrid.data
curGrid[curMask] = nan
# check if grid contains valid values
pixloc = where(isfinite(curGrid)) # != nc_var._FillValue)[0]
if len(pixloc) > 0:
# there are valid values, so write to a PCRaster map and add time to list
timeList.append(curTime)
count += 1
below_thousand = count % 1000
above_thousand = count / 1000
pcraster_file = str(trgPrefix + '%0' +
str(8 - len(trgPrefix)) +
'.f.%03.f') % (above_thousand,
below_thousand)
pcraster_path = os.path.join(trgFolder, pcraster_file)
logger.debug('Writing "' + srcVar + '" on date ' +
curTime.strftime('%Y-%m-%d') + ' to ' +
pcraster_path)
# prepare grid for writing to file
trgGrid = curGrid * slope + offset
trgGrid[isnan(trgGrid)] = nc_var._FillValue
# write grid to PCRaster file
if y[1] - y[0] > 0.:
# the y-axis is ascending, flip the y axis and the target grid!
writeMap(pcraster_path, 'PCRaster', x, flipud(y),
flipud(trgGrid), float(nc_var._FillValue))
else:
writeMap(pcraster_path, 'PCRaster', x, y, trgGrid,
float(nc_var._FillValue))
else:
logger.debug('"' + srcVar + '" not available on date ' +
curTime.strftime('%Y-%m-%d'))
nc_src.close()
return timeList
ds_cf = netCDF4.Dataset(data_dir + 'ecmwf_iri_' + calendar.month_abbr[target_month] + '2017_cf.nc') #Load the pf file
ds_pf = netCDF4.Dataset(data_dir + 'ecmwf_iri_' + calendar.month_abbr[target_month] + '2017_pf.nc') #Load the cf file
# You may want to uncomment the following lines
# to print out the file and variable information
#print(ds_pf.file_format)
#print(ds_pf.dimensions.keys())
#print(ds_pf.variables)
# Read in the variables from cf
prec_hd = ds_cf.variables['hdate'][:]
prec_lead = ds_cf.variables['L'][:]
prec_lat = ds_cf.variables['Y'][:]
prec_lon = ds_cf.variables['X'][:]
prec_temp = ds_cf.variables['S']
prec_start = netCDF4.num2date(prec_temp[:],prec_temp.units)
if ec_lat.all() != prec_lat.all():
print('Latitude files are not the same')
if ec_lon.all() != prec_lon.all():
print('Longitude files are not the same')
# Read in the array from pf's total precip ('tp') variable
arr_pf = ds_pf.variables['tp'][:]
# Read in the cf's total precip ('tp') variable
arr_cf = ds_cf.variables['tp'][:]
arr_shp = arr_pf.shape
# Create a new array to accommodate the 10 pf and 1 cf members
arr_comb = np.empty([arr_shp[0], arr_shp[1],arr_shp[2], arr_shp[3]+1, arr_shp[4], arr_shp[5]])
# Populate arr_comb with pf and cf data
def time_file_index(
base_dir,
time_var,
begin_date=datetime(2013, 1, 1, 0, 0),
end_date=datetime(2013, 1, 5, 0, 0),
filenames=None,
):
"""Index a set of files and generate a pandas data frame of time (index) and
filename.
Parameters
----------
base_dir : str
Index a set of files and generate a pandas data frame of time (index) and
filename.
time_var : str
The title of the time variable in the netCDF files.
begin_date : datetime.datetime, optional
The minimum date from which to include files.
end_date : datetime.datetime, optional
The maximum date from which to include files.
filenames : list, default: None
A list of filenames, if you found them some other way.
Returns
-------
pandas.Dataframe
A pandas dataframe with a DateTimeIndex constructed from the files,
and columns of filename and t_index. The latter represents the
numerical index value of the time slice in the named file.
"""
if filenames is None:
filenames = find_files(base_dir, begin_date, end_date)
nfiles = len(filenames)
# Build a tuple with dates and filenames the file contains for every file in the index
time_file = []
for i in trange(nfiles):
with netCDF4.Dataset(filenames[i]) as netcdf:
# extract the time, turn it into a date
time_dat = netcdf.variables[time_var]
times = np.array(time_dat)
# some have calendar, some don't
try:
times = netCDF4.num2date(times, time_dat.units,
time_dat.calendar)
except:
times = netCDF4.num2date(times, time_dat.units)
for j in range(len(times)):
time_file.append((times[j], filenames[i], j))
result = pd.DataFrame(time_file, columns=["date", "file", "t_index"])
result = result.set_index("date")
# check for duplicates
dupes = result.index.duplicated(keep="first")
if dupes.sum() > 0:
# print('Found duplicate times, using first one found.')
result = result[~dupes]
return result
tStart = parse(time_deployment_start, ignoretz=True)
tEnd = parse(time_deployment_end, ignoretz=True)
tStartnum = date2num(tStart,
units=ncTime[0].units,
calendar=ncTime[0].calendar)
tEndnum = date2num(tEnd,
units=ncTime[0].units,
calendar=ncTime[0].calendar)
maTime = ma.array(ncTime[0][:])
msk = (maTime < tStartnum) | (maTime > tEndnum)
maTime.mask = msk
dates = num2date(maTime.compressed(),
units=ncTime[0].units,
calendar=ncTime[0].calendar)
# create a new filename
# from:
# IMOS_<Facility-Code>_<Data-Code>_<Start-date>_<Platform-Code>_FV<File-Version>_ <Product-Type>_END-<End-date>_C-<Creation_date>_<PARTX>.nc
# to:
# OS_[PlatformCode]_[DeploymentCode]_[DataMode]_[PARTX].nc
# TODO: what to do with <Data-Code> with a reduced number of variables
fileName = os.path.basename(fn)
splitParts = fileName.split(
"_") # get the last path item (the file name), split by _
tStartMaksed = dates[0]
def fix_dataset(self, fn_out, species, fcinit):
# read vertical coordinates
hyn, hyam, hybm = load_vert_coord(CAMS_LEVELDEV_STR)
with Dataset(fn_out, 'a', format='NETCDF4_CLASSIC') as nc:
t_obj = num2date(nc.variables['time'][:],
nc.variables['time'].units)
t_unit = 'hours since {:%Y-%m-%dT%H:%M:%S}'.format(fcinit)
nc.variables['time'][:] = date2num(t_obj, t_unit)
nc.variables['time'].setncattr('units', t_unit)
nc.variables['time'].setncattr('standard_name', 'time')
nc.variables['latitude'].setncattr('standard_name', 'latitude')
nc.variables['longitude'].setncattr('standard_name', 'longitude')
if species == 'AIR_PRESSURE':
ps_ = nc.variables['lnsp'][:] # TODO use species definition
# calculate air_pressure
p_ = (hyam[np.newaxis, :, np.newaxis, np.newaxis] +
hybm[np.newaxis, :, np.newaxis, np.newaxis] *
np.exp(ps_[:, np.newaxis, :, :]))
# create 'level' variable
nc.createDimension('level', hyn.size)
v_lev = nc.createVariable('level', np.int32, ('level', ))
v_lev[:] = np.arange(1, 61, 1, dtype='int32')
# write air_pressure to file
nc.createVariable(
'P', np.float32,
('time', 'level', 'latitude', 'longitude'),
zlib=True, complevel=6, shuffle=True, fletcher32=True)
nc.variables['P'][:] = p_.astype('float32')
# set air_pressure attributes
nc.variables['P'].setncattr('standard_name', 'air_pressure')
nc.variables['P'].setncattr('units', 'Pa')
self.set_standard_name(nc, species)
v_hy = nc.createVariable('hybrid', np.float32, ('level', ))
v_hy[:] = hyn
v_hy.setncattr('standard_name',
'atmosphere_hybrid_sigma_pressure_coordinate')
v_hy.setncattr('units', 'sigma')
v_hy.setncattr('positive', 'down')
v_hy.setncattr('formula', 'p(time, level, lat, lon) = '
'ap(level) + b(level) * exp(lnsp(time, lat, lon))')
v_hy.setncattr('formula_terms', 'ap: hyam b: hybm lnsp:lnsp')
v_am = nc.createVariable('hyam', np.float32, ('level', ))
v_am[:] = hyam
v_am.setncattr('units', 'Pa')
v_am.setncattr('standard_name', 'atmosphere_pressure_coordinate')
v_bm = nc.createVariable('hybm', np.float32, ('level', ))
v_bm[:] = hybm
v_bm.setncattr('units', '1')
v_bm.setncattr('standard_name',
'atmosphere_hybrid_height_coordinate')
nc.variables['level'].setncattr('standard_name',
'model_level_number')
for var in ['time', 'latitude', 'longitude']:
nc.variables[var].setncattr('standard_name', var)
# do some finagling with the start times in the data files
for ff in os.listdir(setup.Data_Dir):
if ff.endswith("filelist.txt"):
fn = ff
f = file(os.path.join(setup.Data_Dir, fn))
flist = []
for line in f:
name = os.path.join(setup.Data_Dir, line)
flist.append(name[:-1]) # Gonzo cat version
# flist.append(name[:-1]) # laptop current version
Time_Map = []
for fn in flist:
d = nc4.Dataset(fn)
t = d['time']
file_start_time = nc4.num2date(t[0], units=t.units)
Time_Map.append((file_start_time, fn))
### WIND
# do some finagling with the start times in the data files
for ff in os.listdir(setup.Data_DirW):
if ff.endswith("filelist.txt"):
fn = ff
# fn = os.path.join(setup.Data_Dir,'SoCal_filelist.txt')
f = file(os.path.join(setup.Data_DirW, fn))
flist = []
for line in f:
name = os.path.join(setup.Data_DirW, line)
flist.append(name[:-1]) # Gonzo cat version
# flist.append(name[:-1]) # laptop current version
Time_Map_W = []
Vqc = ADCP200.variables['VCUR_quality_control'] Uqc = ADCP200.variables['UCUR_quality_control'] Dqc = ADCP200.variables['DEPTH_quality_control'] #To regrid for Depth_interp_final = (np.arange(Depth_f,Depth_i,-Depth_dz)) Depth_i = 0 Depth_f = 145 Depth_dz = 5 # In[20]: #block 4: Convert the ADCP time series dataset (Matlab to python time series: yyyy/mm/DD hh/mm/SS) units = ADCP200.variables['TIME'].units calendar = ADCP200.variables['TIME'].calendar times = num2date(ADCP200.variables['TIME'][:], units=units, calendar=calendar) times1 = mdates.date2num(times) # In[21]: #block 5: Create variables with ADCP dataset #basic data depth_ADCP = DEPTH[:, 0, 0] depth_Bin = Wheight[:] data_u = U[:, :, :, :] data_v = V[:, :, :, :] v2d = data_v[:, :, 0, 0] u2d = data_u[:, :, 0, 0] #quality controle data
def seasonal_climatology_cice(directory, out_file):
# Starting and ending months (1-based) for each season
start_month = [12, 3, 6, 9]
end_month = [2, 5, 8, 11]
# Starting and ending days of the month (1-based) for each season
# Assume no leap years, we'll fix this later if we need
start_day = [1, 1, 1, 1]
end_day = [28, 31, 31, 30]
# Find the first output file
for file in listdir(directory):
if file.startswith('iceh.') and file.endswith('.nc'):
# Read the grid
id = Dataset(directory + file, 'r')
lon = id.variables['TLON'][:, :]
lat = id.variables['TLAT'][:, :]
id.close()
break
# Set up arrays to integrate seasonal climatology of aice and hi
seasonal_aice = ma.empty([4, size(lon, 0), size(lon, 1)])
seasonal_hi = ma.empty([4, size(lon, 0), size(lon, 1)])
seasonal_aice[:, :, :] = 0.0
seasonal_hi[:, :, :] = 0.0
# Also integrate number of days in each season
ndays = zeros(4)
# Total number of days for a final check
total_days = 0
# Loop over files
for file in listdir(directory):
if file.startswith('iceh.') and file.endswith('.nc'):
print 'Processing ' + directory + file
# Read aice and hi
id = Dataset(directory + file, 'r')
aice = id.variables['aice'][0, :, :]
hi = id.variables['hi'][0, :, :]
# Read the time value and convert to Date object
time_id = id.variables['time']
time = num2date(time_id[0],
units=time_id.units,
calendar=time_id.calendar.lower())
id.close()
total_days += 5
# 5-day averages marked with next day's date
year = time.year
month = time.month
day = time.day
# Get the season of this day
if month in [12, 1, 2]:
# DJF
season = 0
elif month in [3, 4, 5]:
# MAM
season = 1
elif month in [6, 7, 8]:
# JJA
season = 2
elif month in [9, 10, 11]:
# SON
season = 3
# Check for leap years
leap_year = False
if mod(year, 4) == 0:
leap_year = True
if mod(year, 100) == 0:
leap_year = False
if mod(year, 400) == 0:
leap_year = True
# Update last day in February
if leap_year:
end_day[0] = 29
else:
end_day[0] = 28
if month == start_month[season]:
# We are in the first month of the season
if day - 5 < start_day[season]:
# Spills over into the previous season
prev_season = mod(season - 1, 4)
# How many days does it spill over by?
spill_days = start_day[season] - day + 5
# Should be between 1 and 5
if spill_days < 1 or spill_days > 5:
print 'Problem: spill_days is ' + str(spill_days)
print 'Timestep ' + str(t + 1)
print 'Year ' + str(year)
print 'Month ' + str(month + 1)
print 'Day ' + str(day)
return
# Split between previous season and this season
seasonal_aice[prev_season, :, :] += aice * spill_days
seasonal_hi[prev_season, :, :] += hi * spill_days
ndays[prev_season] += spill_days
seasonal_aice[season, :, :] += aice * (5 - spill_days)
seasonal_hi[season, :, :] += hi * (5 - spill_days)
ndays[season] += 5 - spill_days
else:
# Entirely within the season
seasonal_aice[season, :, :] += aice * 5
seasonal_hi[season, :, :] += hi * 5
ndays[season] += 5
else:
# Not in the first month of the season
# The 5 days will be entirely within the season
seasonal_aice[season, :, :] += aice * 5
seasonal_hi[season, :, :] += hi * 5
ndays[season] += 5
# Convert from sums to averages
for season in range(4):
seasonal_aice[
season, :, :] = seasonal_aice[season, :, :] / ndays[season]
seasonal_hi[season, :, :] = seasonal_hi[season, :, :] / ndays[season]
if sum(ndays) != total_days:
print 'Problem: files have ' + str(
total_days) + ' days, but climatology has ' + str(sum(ndays))
return
# Write to file
print 'Writing ' + out_file
id = Dataset(out_file, 'w')
id.createDimension('ni', size(lon, 1))
id.createDimension('nj', size(lon, 0))
id.createDimension('time', 4)
id.createVariable('TLON', 'f8', ('nj', 'ni'))
id.variables['TLON'].long_name = 'T grid center longitude'
id.variables['TLON'].units = 'degrees_east'
id.variables['TLON'][:, :] = lon
id.createVariable('TLAT', 'f8', ('nj', 'ni'))
id.variables['TLAT'].long_name = 'T grid center latitude'
id.variables['TLAT'].units = 'degrees_north'
id.variables['TLAT'][:, :] = lat
id.createVariable('time', 'f8', ('time'))
id.variables['time'].units = 'season'
id.variables['time'].long_name = 'DJF, MAM, JJA, SON'
id.variables['time'][:] = arange(1, 4 + 1)
id.createVariable('aice', 'f8', ('time', 'nj', 'ni'))
id.variables['aice'].units = '1'
id.variables['aice'][:, :, :] = seasonal_aice
id.createVariable('hi', 'f8', ('time', 'nj', 'ni'))
id.variables['hi'].units = 'm'
id.variables['hi'][:, :, :] = seasonal_hi
id.close()
timestep = 'time'
## Extracting longitude and latitude of all mesh nodes ##
lon = nc['x'][:]
lat = nc['y'][:]
## Extracting element description details (list of 3 vertices specified for each triangle denoted by index number)from the output file ##
nv = nc['element'][:, :] - 1
## Extracting time step information from the output file ##
time_var = nc['time']
## startdate specifies start time of ADCIRC computations ##
startdate = time_var.units
## Converting the time step information into datetime objects ##
dtime = netCDF4.num2date(time_var[:], startdate)
## Converting datetime objects to string format - YYMMDDHHMMSS ##
a = []
for j in dtime:
a.append(j.strftime("%Y%m%d%H%M%S"))
## RUNNING TIME ##
print 'Finished organizing input information after %d seconds' % (time.time() -
time0)
## FUNCTION TO CHECK IF VERTEX FALLS WITHIN A SPECIFIED LONG/LAT BOX##
def vertexcheck(LatN, LatS, LongE, LongW, vertex):
if lon[vertex] < LongE and lon[vertex] > LongW and lat[
vertex] > LatS and lat[vertex] < LatN:
def read_era5(domain, times, pres=True):
#Open ERA5 netcdf files and extract variables needed for a range of times
# and given spatial domain
ref = dt.datetime(1900, 1, 1, 0, 0, 0)
if len(times) > 1:
date_list = date_seq(times, "hours", 1)
else:
date_list = times
formatted_dates = [format_dates(x) for x in date_list]
unique_dates = np.unique(formatted_dates)
time_hours = np.empty(len(date_list))
for t in np.arange(0, len(date_list)):
time_hours[t] = (date_list[t] - ref).total_seconds() / (3600)
if (date_list[0].day == 1) & (date_list[0].hour < 3):
fc_unique_dates = np.insert(
unique_dates, 0, format_dates(date_list[0] - dt.timedelta(1)))
else:
fc_unique_dates = np.copy(unique_dates)
#Get time-invariant pressure and spatial info
no_p, p, p_ind = get_pressure(100)
p = p[p_ind]
lon, lat = get_lat_lon()
lon_ind = np.where((lon >= domain[2]) & (lon <= domain[3]))[0]
lat_ind = np.where((lat >= domain[0]) & (lat <= domain[1]))[0]
lon = lon[lon_ind]
lat = lat[lat_ind]
terrain = reform_terrain(lon, lat)
sfc_lon, sfc_lat = get_lat_lon_sfc()
sfc_lon_ind = np.where((sfc_lon >= domain[2]) & (sfc_lon <= domain[3]))[0]
sfc_lat_ind = np.where((sfc_lat >= domain[0]) & (sfc_lat <= domain[1]))[0]
sfc_lon = sfc_lon[sfc_lon_ind]
sfc_lat = sfc_lat[sfc_lat_ind]
#Initialise arrays for each variable
if pres:
ta = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
dp = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
hur = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
hgt = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
ua = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
va = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
wap = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
uas = np.empty((len(date_list), len(sfc_lat_ind), len(sfc_lon_ind)))
vas = np.empty((len(date_list), len(sfc_lat_ind), len(sfc_lon_ind)))
ps = np.empty((len(date_list), len(sfc_lat_ind), len(sfc_lon_ind)))
cp = np.zeros(ps.shape) * np.nan
cape = np.zeros(ps.shape) * np.nan
wg10 = np.zeros(ps.shape) * np.nan
tas = np.empty((len(date_list), len(sfc_lat_ind), len(sfc_lon_ind)))
ta2d = np.empty((len(date_list), len(sfc_lat_ind), len(sfc_lon_ind)))
for date in unique_dates:
#Load ERA-Interim reanalysis files
if pres:
ta_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/pressure/t/"+date[0:4]+\
"/t_era5_aus_"+date+"*.nc")[0])
z_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/pressure/z/"+date[0:4]+\
"/z_era5_aus_"+date+"*.nc")[0])
ua_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/pressure/u/"+date[0:4]+\
"/u_era5_aus_"+date+"*.nc")[0])
va_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/pressure/v/"+date[0:4]+\
"/v_era5_aus_"+date+"*.nc")[0])
hur_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/pressure/r/"+date[0:4]+\
"/r_era5_aus_"+date+"*.nc")[0])
uas_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/u10/"+date[0:4]+\
"/u10_era5_global_"+date+"*.nc")[0])
vas_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/v10/"+date[0:4]+\
"/v10_era5_global_"+date+"*.nc")[0])
ta2d_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/d2m/"+date[0:4]+\
"/d2m_era5_global_"+date+"*.nc")[0])
tas_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/t2m/"+date[0:4]+\
"/t2m_era5_global_"+date+"*.nc")[0])
ps_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/sp/"+date[0:4]+\
"/sp_era5_global_"+date+"*.nc")[0])
cp_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/cp/"+date[0:4]+\
"/cp_era5_global_"+date+"*.nc")[0])
wg10_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/fg10/"+date[0:4]+\
"/fg10_era5_global_"+date+"*.nc")[0])
#Get times to load in from file
if pres:
times = ta_file["time"][:]
time_ind = [
np.where(x == times)[0][0] for x in time_hours if (x in times)
]
date_ind = np.where(np.array(formatted_dates) == date)[0]
else:
times = uas_file["time"][:]
time_ind = [
np.where(x == times)[0][0] for x in time_hours if (x in times)
]
date_ind = np.where(np.array(formatted_dates) == date)[0]
#Get times to load in from forecast files (wg10 and cp)
fc_times = cp_file["time"][:]
fc_time_ind = [
np.where(x == fc_times)[0][0] for x in time_hours
if (x in fc_times)
]
#Load analysis data
if pres:
ta[date_ind, :, :, :] = ta_file["t"][time_ind, p_ind, lat_ind,
lon_ind] - 273.15
#wap[date_ind,:,:,:] = wap_file["wap"][time_ind,p_ind,lat_ind,lon_ind]
ua[date_ind, :, :, :] = ua_file["u"][time_ind, p_ind, lat_ind,
lon_ind]
va[date_ind, :, :, :] = va_file["v"][time_ind, p_ind, lat_ind,
lon_ind]
hgt[date_ind, :, :, :] = z_file["z"][time_ind, p_ind, lat_ind,
lon_ind] / 9.8
hur[date_ind, :, :, :] = hur_file["r"][time_ind, p_ind, lat_ind,
lon_ind]
hur[hur < 0] = 0
hur[hur > 100] = 100
dp[date_ind, :, :, :] = get_dp(ta[date_ind, :, :, :],
hur[date_ind, :, :, :])
uas[date_ind, :, :] = uas_file["u10"][time_ind, sfc_lat_ind,
sfc_lon_ind]
vas[date_ind, :, :] = vas_file["v10"][time_ind, sfc_lat_ind,
sfc_lon_ind]
tas[date_ind, :, :] = tas_file["t2m"][time_ind, sfc_lat_ind,
sfc_lon_ind] - 273.15
ta2d[date_ind, :, :] = ta2d_file["d2m"][time_ind, sfc_lat_ind,
sfc_lon_ind] - 273.15
ps[date_ind, :, :] = ps_file["sp"][time_ind, sfc_lat_ind,
sfc_lon_ind] / 100
fc_date_ind = np.in1d(
date_list,
nc.num2date(cp_file["time"][fc_time_ind], cp_file["time"].units))
cp[fc_date_ind, :, :] = cp_file["cp"][fc_time_ind, sfc_lat_ind,
sfc_lon_ind]
wg10[fc_date_ind, :, :] = wg10_file["fg10"][fc_time_ind, sfc_lat_ind,
sfc_lon_ind]
if pres:
ta_file.close()
z_file.close()
ua_file.close()
va_file.close()
hur_file.close()
uas_file.close()
vas_file.close()
tas_file.close()
ta2d_file.close()
ps_file.close()
if pres:
p = np.flip(p)
ta = np.flip(ta, axis=1)
dp = np.flip(dp, axis=1)
hur = np.flip(hur, axis=1)
hgt = np.flip(hgt, axis=1)
ua = np.flip(ua, axis=1)
va = np.flip(va, axis=1)
return [
ta, dp, hur, hgt, terrain, p, ps, ua, va, uas, vas, tas, ta2d, cp,
wg10, cape, lon, lat, date_list
]
else:
return [
ps, uas, vas, tas, ta2d, cp, wg10, cape, sfc_lon, sfc_lat,
date_list
]
def date_range(start=None, end=None, periods=None, freq='D', tz=None,
normalize=False, name=None, closed=None, calendar='standard',
**kwargs):
''' Return a fixed frequency datetime index, with day (calendar) as the
default frequency
Parameters
----------
start : string or datetime-like, default None
Left bound for generating dates
end : string or datetime-like, default None
Right bound for generating dates
periods : integer or None, default None
If None, must specify start and end
freq : string or DateOffset, default 'D' (calendar daily)
Frequency strings can have multiples, e.g. '5H'
tz : string or None
Time zone name for returning localized DatetimeIndex, for example
Asia/Hong_Kong
normalize : bool, default False
Normalize start/end dates to midnight before generating date range
name : str, default None
Name of the resulting index
closed : string or None, default None
Make the interval closed with respect to the given frequency to
the 'left', 'right', or both sides (None)
calendar : string
Describes the calendar used in the time calculations. Default is a the
standard calendar (with leap years)
Notes
-----
2 of start, end, or periods must be specified
To learn more about the frequency strings, please see `this link
<http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases>`__.
Returns
-------
rng : DatetimeIndex
'''
if calendar in ['standard', 'gregorian', 'propoleptic_gregorian']:
return pd.date_range(start=start, end=end, periods=periods,
freq=freq, tz=tz, normalize=normalize, name=name,
closed=closed, **kwargs)
else:
# start and end are give
if (start is not None) and (end is not None) and (periods is None):
steps, units = decode_freq(freq)
start_num, end_num = date2num(
pd.to_datetime([start, end]).to_pydatetime(),
units, calendar=calendar)
periods = int((end_num - start_num) / steps) + 1
times = num2date(
np.linspace(start_num, end_num, periods,
endpoint=True,
dtype=np.float128), units, calendar)
index = xr.CFTimeIndex(times).to_datetimeindex()
return index
else:
raise NotImplementedError(
'Specified arguments are not valid for this calendar')
def do_grid_map_gates_to_grid(radar_fname):
import pyart
from matplotlib import pyplot as plt
from netCDF4 import num2date, date2num
import numpy as np
from time import time
import os
md = '/lcrc/group/earthscience/radar/nexrad/chicago_floods/'
try:
radar = pyart.io.read(radar_fname)
rain_z = radar.fields['reflectivity']['data'].copy()
z_lin = 10.0**(radar.fields['reflectivity']['data'] / 10.)
rain_z = (z_lin / 300.0)**(1. / 1.4) #Z=300 R1.4
radar.add_field_like('reflectivity',
'rain_z',
rain_z,
replace_existing=True)
radar.fields['rain_z']['units'] = 'mm/h'
radar.fields['rain_z']['standard_name'] = 'rainfall_rate'
radar.fields['rain_z']['long_name'] = 'rainfall_rate_from_z'
radar.fields['rain_z']['valid_min'] = 0
radar.fields['rain_z']['valid_max'] = 500
grid = pyart.map.grid_from_radars(
(radar, ),
grid_shape=(1, 501, 501),
grid_limits=((0, 0), (-50000, 50000), (-50000, 50000)),
fields=radar.fields.keys(),
gridding_algo="map_gates_to_grid",
weighting_function='BARNES')
dts = num2date(grid.axes['time']['data'], grid.axes['time']['units'])
sstr = dts[0].strftime('%Y%m%d_%H%M%S')
pyart.io.write_grid(md + 'grid_250_' + sstr + '.nc', grid)
myd = pyart.graph.RadarMapDisplay(radar)
fig = plt.figure(figsize=[18, 10])
myd.plot_ppi_map('rain_z',
vmin=0,
vmax=100,
resolution='h',
max_lat=41.8,
min_lat=41.25,
min_lon=-88.3,
max_lon=-87.5)
m = myd.basemap
m.drawparallels(np.linspace(41, 42, 9), labels=[1, 0, 0, 0])
m.drawmeridians(np.linspace(-88.4, -87, 8), labels=[0, 0, 0, 1])
m.drawrivers()
m.drawcounties()
m.drawstates()
m.drawmapscale(-88.,
41.55,
-88.,
41.55,
10,
barstyle='fancy',
fontcolor='k',
fillcolor1='b',
fillcolor2='k')
myd.plot_point(-87.9706,
41.6815,
label_text='Argonne Lab',
label_offset=(0.0, 0.0))
plt.savefig(md + 'radar_' + sstr + '.png')
plt.close(fig)
fig = plt.figure(figsize=[15, 15])
max_lat = 43
min_lat = 41.5
min_lon = -88.3
max_lon = -87.5
display = pyart.graph.GridMapDisplay(grid)
display.plot_basemap(lat_lines=np.arange(min_lat, max_lat, .1),
lon_lines=np.arange(min_lon, max_lon, .1),
resolution='h')
display.plot_grid('rain_z', vmin=0, vmax=100)
xcf, ycf = display.basemap(-87.9706, 41.6815)
display.basemap.plot(xcf, ycf, 'ro')
plt.text(xcf + 2000., ycf + 2000., 'Argonne Lab')
display.basemap.drawcounties()
display.basemap.drawrivers()
display.basemap.drawmapscale(-88.,
41.55,
-88.,
41.55,
10,
barstyle='fancy',
fontcolor='k',
fillcolor1='b',
fillcolor2='k')
display.plot_colorbar()
plt.savefig(md + 'mapped_250_' + sstr + '.png')
plt.close(fig)
del (radar)
del (grid)
except:
pass
return 0
def parse(file):
hdr = True
dataLine = 0
name = []
number_samples_read = 0
nVars = 0
data = []
ts = []
filepath = file[0]
with open(filepath, 'r', errors='ignore') as fp:
line = fp.readline()
matchObj = re.match(first_line_expr, line)
if not matchObj:
print("Not a Starmon-mini DAT file !")
return None
cnt = 1
while line:
#print("Line {}: {} : {}".format(cnt, dataLine, line.strip()))
if hdr:
if line[0] != '#':
hdr = False
else:
matchObj = re.match(soft_version_expr, line)
if matchObj:
#print("soft_version_expr:matchObj.group() : ", matchObj.group())
#print("soft_version_expr:matchObj.group(1) : ", matchObj.group(1))
software = matchObj.group(1)
matchObj = re.match(first_line_expr, line)
if matchObj:
#print("first_line_expr:matchObj.group() : ", matchObj.group())
#print("first_line_expr:matchObj.group(1) : ", matchObj.group(1))
file_created = matchObj.group(1)
matchObj = re.match(recorder_expr, line)
if matchObj:
#print("recorder_expr:matchObj.group() : ", matchObj.group())
#print("recorder_expr:matchObj.group(1) : ", matchObj.group(1))
#print("recorder_expr:matchObj.group(2) : ", matchObj.group(2))
#print("recorder_expr:matchObj.group(3) : ", matchObj.group(3))
instrument_model = matchObj.group(2)
instrument_serial_number = matchObj.group(3)
matchObj = re.match(series_expr, line)
if matchObj:
#print("series_expr:matchObj.group() : ", matchObj.group())
#print("series_expr:matchObj.group(1) : ", matchObj.group(1))
#print("series_expr:matchObj.group(2) : ", matchObj.group(2))
#print("series_expr:matchObj.group(3) : ", matchObj.group(3))
nameN = matchObj.group(1)
ncVarName = matchObj.group(2)
if ncVarName in nameMap:
ncVarName = nameMap[ncVarName]
unit = matchObj.group(3)
if unit in unitMap:
unit = unitMap[unit]
name.insert(nVars, {
'col': int(nameN),
'var_name': ncVarName,
'unit': unit
})
if not hdr:
lineSplit = line.split('\t')
#print(lineSplit)
splitVarNo = 0
d = np.zeros(len(name))
d.fill(np.nan)
t = datetime.strptime(lineSplit[1], '%d/%m/%Y %H:%M:%S')
ts.append(t)
#print("timestamp %s" % ts)
for v in name:
#print("{} : {}".format(splitVarNo, v))
d[splitVarNo] = float(lineSplit[v['col'] + 2])
splitVarNo = splitVarNo + 1
data.append(d)
#print(t, d)
number_samples_read = number_samples_read + 1
dataLine = dataLine + 1
line = fp.readline()
cnt += 1
# trim data
print("samplesRead %d data shape %s" % (number_samples_read, len(name)))
#
# build the netCDF file
#
ncTimeFormat = "%Y-%m-%dT%H:%M:%SZ"
outputName = filepath + ".nc"
print("output file : %s" % outputName)
ncOut = Dataset(outputName, 'w', format='NETCDF4')
ncOut.instrument = 'StarODDI - ' + instrument_model
ncOut.instrument_model = instrument_model
ncOut.instrument_serial_number = instrument_serial_number
# TIME:axis = "T";
# TIME:calendar = "gregorian";
# TIME:long_name = "time";
# TIME:units = "days since 1950-01-01 00:00:00 UTC";
tDim = ncOut.createDimension("TIME", number_samples_read)
ncTimesOut = ncOut.createVariable("TIME", "d", ("TIME", ), zlib=True)
ncTimesOut.long_name = "time"
ncTimesOut.units = "days since 1950-01-01 00:00:00 UTC"
ncTimesOut.calendar = "gregorian"
ncTimesOut.axis = "T"
ncTimesOut[:] = date2num(ts,
units=ncTimesOut.units,
calendar=ncTimesOut.calendar)
i = 0
for v in name:
print("Variable %s : unit %s" % (v['var_name'], v['unit']))
varName = v['var_name']
ncVarOut = ncOut.createVariable(
varName, "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
if v['unit']:
ncVarOut.units = v['unit']
ncVarOut[:] = np.array([d[i] for d in data])
i += 1
ncOut.setncattr(
"time_coverage_start",
num2date(ncTimesOut[0],
units=ncTimesOut.units,
calendar=ncTimesOut.calendar).strftime(ncTimeFormat))
ncOut.setncattr(
"time_coverage_end",
num2date(ncTimesOut[-1],
units=ncTimesOut.units,
calendar=ncTimesOut.calendar).strftime(ncTimeFormat))
ncOut.setncattr("date_created", datetime.utcnow().strftime(ncTimeFormat))
ncOut.setncattr(
"history",
datetime.utcnow().strftime("%Y-%m-%d") + " created from file " +
filepath + " source file created " + file_created + " by software " +
software.replace("\t", " "))
ncOut.close()
return outputName
def main(f, t, T, y, Y, x, X, var='', z=0, Z=0, Source='usn-ncom'):
try:
url = sources[Source]
except:
url = Source
source = Source
if source == 'GLBu0.08':
Vers = get_vers(t)
url = url + Vers
elif source == 'GLBv0.08':
Vers = 'expt_93.0'
url = url + Vers
# Parse time
if t is not None:
try: # Date and time is given
startTime = datetime.datetime.strptime(t, '%Y-%m-%d %H:%M:%S')
endTime = datetime.datetime.strptime(T, '%Y-%m-%d %H:%M:%S')
except: # Only date is given
try:
startTime = datetime.datetime.strptime(t, '%Y-%m-%d')
endTime = datetime.datetime.strptime(T, '%Y-%m-%d')
except:
print('Could not parse time: ' + str(t))
# Get metadata from URL
d = Dataset(url, 'r')
if f is None:
print('==================================================')
print(d)
print('==================================================')
# Get dimensions and their vectors
for dimName in d.dimensions:
print(dimName)
dimvals = d.variables[dimName][:]
if dimName == 'time':
dimvals = num2date(dimvals, units=d.variables[dimName].units)
if t is None: # Return first time steps, if time not given
startTime = dimvals[-2]
endTime = dimvals[-1]
if startTime > dimvals[-1] or endTime < dimvals[0]:
sys.exit('Requested time span (%s - %s) not covered '
'by dataset (%s - %s)' %
(startTime, endTime, dimvals[0], dimvals[-1]))
timeIndexStart = bisect.bisect_left(dimvals, startTime)
timeIndexEnd = bisect.bisect_right(dimvals, endTime)
timeIndexStart = np.maximum(timeIndexStart, 0)
timeIndexEnd = np.minimum(timeIndexEnd, len(dimvals) - 1)
if dimName == 'depth':
print(str(dimvals))
hasDepth = True
else:
print('\t' + str(dimvals[0]) + ' (min)')
print('\t' + str(dimvals[-1]) + ' (max)')
if dimName == 'lon':
if x is None:
x = dimvals[0]
X = dimvals[1]
if dimName == 'lat':
if y is None:
y = dimvals[0]
Y = dimvals[1]
# Print available parameters
print('\nParameters (CF standard name):')
for varName in d.variables:
try:
standard_name = d.variables[varName].standard_name
print('\t' + varName + ' (' + standard_name + ')')
except:
print('\t' + varName)
if var != '':
varCommand = '-v ' + ','.join(var)
else:
varCommand = ''
d.close()
# Suggest command, if filename is not given
if f is not None:
# Subsetting in time and space
subset = varCommand+' -d lon,%.2f,%.2f -d lat,%.2f,%.2f -d time,%i,%i' % \
(x, X, y, Y, timeIndexStart, timeIndexEnd)
if 'hasDepth' in locals():
z = np.float(z)
Z = np.float(Z)
subset = subset + ' -d depth,%.2f,%.2f ' % (z, Z)
out = ' -o out.nc '
ncoCommand = 'ncks ' + subset + out + url + ' --overwrite'
ncoCommand = ncoCommand + ' -o ' + f
print(ncoCommand)
subprocess.call(ncoCommand, shell=True)
#subprocess.call('ncdump out.nc -v lon|tail -3', shell=True)
#subprocess.call('ncdump out.nc -v lat|tail -3', shell=True)
else:
templateCommand = '%s -source %s -t \'%s\' -T \'%s\' ' \
' -x %s -X %s -y %s -Y %s -f %s' % \
(sys.argv[0], source, startTime, endTime,
x, X, y, Y, 'out.nc')
templateCommand = templateCommand + varCommand
if 'hasDepth' in locals():
templateCommand = templateCommand + \
' -z %s -Z %s ' % (z, Z)
print('---------------------------------------------------------')
print('Template command for data download (cut, paste and edit):')
print(templateCommand)
print('=========================================================')
def scenarios():
"""
Handle get and push requests for list of all finished scenarios and submit
a new scenario, respectively.
"""
if request.method == 'GET':
scenarios = Scenario.objects
# this is for the first three scenarios only
if app.config['DEBUG'] and len(scenarios) < 3:
for loop_counter in range(3):
_init_dev_db(app.config['BASE_PARAMETER_NC'], loop_counter)
scenarios = Scenario.objects
return jsonify(scenarios=scenarios)
else:
BASE_PARAMETER_NC = app.config['BASE_PARAMETER_NC']
# assemble parts of a new scenario record
vegmap_json = request.json['veg_map_by_hru']
name = request.json['name']
time_received = datetime.datetime.now()
new_scenario = Scenario(
name=name,
time_received=time_received,
)
scenario_runner = new_scenario.initialize_runner(BASE_PARAMETER_NC)
# using vegmap sent by client, update coverage type variables
scenario_runner.update_cov_type(vegmap_json['bare_ground'], 0)
scenario_runner.update_cov_type(vegmap_json['grasses'], 1)
scenario_runner.update_cov_type(vegmap_json['shrubs'], 2)
scenario_runner.update_cov_type(vegmap_json['trees'], 3)
scenario_runner.update_cov_type(vegmap_json['conifers'], 4)
# commit changes to coverage type in preparation for scenario run
scenario_runner.finalize_run()
# now that changes to scenario_file are committed, we update Scenario
new_scenario.veg_map_by_hru =\
get_veg_map_by_hru(scenario_runner.scenario_file)
new_scenario.save()
modelserver_run = scenario_runner.run(
auth_host=app.config['AUTH_HOST'],
model_host=app.config['MODEL_HOST'],
app_username=app.config['APP_USERNAME'],
app_password=app.config['APP_PASSWORD'])
time_finished = datetime.datetime.now()
new_scenario.time_finished = time_finished
# extract URLs pointing to input/output resources stored on modelserver
resources = modelserver_run.resources
control =\
filter(lambda x: 'control' == x.resource_type, resources
).pop().resource_url
parameter =\
filter(lambda x: 'param' == x.resource_type, resources
).pop().resource_url
data =\
filter(lambda x: 'data' == x.resource_type, resources
).pop().resource_url
inputs = Inputs(control=control, parameter=parameter, data=data)
new_scenario.inputs = inputs
statsvar =\
filter(lambda x: 'statsvar' == x.resource_type, resources
).pop().resource_url
outputs = Outputs(statsvar=statsvar)
new_scenario.outputs = outputs
# extract hydrograph from the statsvar file and add to Scenario
if not os.path.isdir('.tmp'):
os.mkdir('.tmp')
tmp_statsvar = os.path.join('.tmp', 'statsvar-' + str(uuid4()))
urlretrieve(statsvar, tmp_statsvar)
d = netCDF4.Dataset(tmp_statsvar, 'r')
cfs = d['basin_cfs_1'][:]
t = d.variables['time']
# need to subtract 1...bug in generation of statsvar b/c t starts at 1
dates = netCDF4.num2date(t[:] - 1, t.units)
hydrograph = Hydrograph(time_array=dates, streamflow_array=cfs)
new_scenario.hydrograph = hydrograph
new_scenario.save()
# clean up temporary statsvar netCDF
d.close()
os.remove(tmp_statsvar)
return jsonify(scenario=new_scenario.to_json())
def lookup_climate_files(srcFolder,
inputVars,
startDate,
endDate,
timestep=dt.timedelta(days=1),
srcFiles=[],
lookup_table={}):
"""
Generates or appends a dictionary of dates and references to files for each date and variable of interest
The function checks for each date and each variable, which file in the srcFolder contains the variable
of interest at the date of interest
"""
if len(lookup_table) == 0:
# lookup_table is still empty. Fill lookup_table with dates and empty strings
# make list of all dates
dateList = []
curDate = startDate
while curDate <= endDate:
dateList.append(curDate)
curDate += timestep
lookup_table['time'] = dateList
# pre-fill the dictionary with lists of empty strings for each file
for inputVar in inputVars:
lookup_table[inputVar] = empty(len(dateList), 'S1000')
# make list of all available files
if len(srcFiles) == 0:
srcFiles = recursive_glob(rootdir=srcFolder, suffix='.nc')
srcFiles.sort()
for srcFile in srcFiles:
print 'Scanning ' + srcFile
nc_src = nc4.Dataset(srcFile, 'r')
time = nc_src.variables['time']
try:
timeObjRaw = nc4.num2date(time[:],
units=time.units,
calendar=time.calendar)
except:
timeObjRaw = nc4.num2date(time[:],
units=time.units,
calendar='gregorian')
# UUGGGHHH: this seems to be a bug. calendar other than gregorian give other objects than
# datetime.datetime objects. Here we convert to a year with a julian day string, then back to date objects
timeObj = []
for t in timeObjRaw:
timeObj.append(dt.datetime.strptime(t.strftime('%Y%j'), '%Y%j'))
#timeObj = nc4.num2date(nc4.date2num(timeObj, units=time.units, calendar='gregorian'), units=time.units, calendar='gregorian')
# now loop over all time steps, check the date and write valid dates to a list, write time series to PCRaster maps
for n in range(len(timeObj)):
# remove any hours or minutes data from curTime
timeObj[n] = timeObj[n].replace(hour=0)
timeObj[n] = timeObj[n].replace(minute=0)
# check for first and last date in timeObj
firstDate = timeObj[0]
lastDate = timeObj[-1]
# find locations of firstDate and lastDate in our target time periods
idxStart = where(array(lookup_table['time']) == firstDate)[0]
idxEnd = where(array(lookup_table['time']) == lastDate)[0]
if len(idxStart) == 1 or len(idxEnd == 1):
if len(idxStart) == 0:
idxStart = array([0])
if len(idxEnd) == 0:
idxEnd = len(dateList) - 1
allVars = nc_src.variables.keys()
# check which variables are present in file, check file
for inputVar in inputVars:
if inputVar in allVars:
lookup_table[inputVar][idxStart:idxEnd + 1] = srcFile
nc_src.close()
return lookup_table