def get_daterange(self):
from awrams.utils.io.netcdf_wrapper import epoch_from_nc
epoch = epoch_from_nc(self.ncd_group)
time = self.ncd_group.variables['time']
return [(epoch + dt.days(int(ts)))
for ts in (time[0], time[len(time) - 1])]
def _get_period_idx(self):
from awrams.utils.io.netcdf_wrapper import epoch_from_nc
epoch = epoch_from_nc(self.ncd)
dti = pd.DatetimeIndex([(epoch + dt.days(int(ts)))
for ts in self.ncd.variables['time']],
freq='d')
return dti.searchsorted(self.period)
def get_dates(self):
from awrams.utils.io.netcdf_wrapper import epoch_from_nc
epoch = epoch_from_nc(self.ncd_group)
dti = pd.DatetimeIndex([(epoch + dt.days(int(ts)))
for ts in self.ncd_group.variables['time']],
freq='d')
return dti
def get_coord(self, coord):
'''
Get a Coordinates object whose name matches 'coord'
'''
from awrams.utils.io.netcdf_wrapper import epoch_from_nc
from awrams.utils.helpers import aquantize
def from_epoch(epoch, ts):
return epoch + dt.days(int(ts))
if coord == 'time':
epoch = epoch_from_nc(self.ncd_group)
time_var = self.ncd_group.variables['time']
dti = pd.DatetimeIndex(
[(epoch + dt.days(int(ts)))
for ts in self.ncd_group.variables['time'][:]],
freq='d')
if 'bounds' in time_var.ncattrs():
bounds_var = self.ncd_group.variables[time_var.bounds]
boundaries = []
for b in bounds_var[:]:
boundaries.append(
[from_epoch(epoch, b[0]),
from_epoch(epoch, b[1] - 1)])
#Attempt to infer period frequency from boundaries...
p = infer_period(boundaries)
return BoundedTimeCoordinates(TimeDimension(epoch),
dti.to_period(p), boundaries)
else:
return TimeCoordinates(TimeDimension(epoch), dti)
elif coord == 'latitude':
lat = np.float64(self.ncd_group.variables['latitude'][:])
if not hasattr(lat, '__len__'):
lat = np.array([lat])
return Coordinates(latitude, aquantize(lat, 0.05))
# return Coordinates(latitude,aquantize(np.float64(self.ncd_group.variables['latitude'][:]),0.05))
elif coord == 'longitude':
lon = np.float64(self.ncd_group.variables['longitude'][:])
if not hasattr(lon, '__len__'):
lon = np.array([lon])
return Coordinates(longitude, aquantize(lon, 0.05))
# return Coordinates(longitude,aquantize(np.float64(self.ncd_group.variables['longitude'][:]),0.05))
else:
ncvar = self.ncd_group.variables[coord]
if hasattr(ncvar, 'units'):
units = Units(ncvar.units)
else:
units = Units('unknown unit')
dim = Dimension(ncvar.dimensions[0], units, ncvar.dtype)
return Coordinates(dim, ncvar[:])
def map_files(self,
path,
pattern,
variable,
ff=None,
max_open_files=MAX_FILES_PER_SFM,
map_func=None):
var_name = variable if isinstance(variable, str) else variable.name
self.var_name = var_name
search_pattern = os.path.join(path, pattern)
files = glob.glob(search_pattern)
files.sort()
if ff is None:
def ff(x):
return True
_files = []
for f in files:
if ff(f):
_files.append(f)
files = _files
if len(files) == 0:
raise Exception("No files found in %s matching %s" %
(path, pattern))
#import netCDF4 as ncd
#db_opener_TEST = ncd.Dataset
dsm_start = DatasetManager(db_opener(files[0], self.mode))
#dsm_start = DatasetManager(open_append(db_opener_TEST,files[0],self.mode))
self.ref_ds = dsm_start
coords = dsm_start.get_coords()
time = dsm_start.get_coord('time')
time_idx = time.index
self.file_time_map[files[0]] = time
self.time_file_map[time.index[0]] = files[0]
tsegs = [time.index]
if len(files) > 1:
for fn in files[1:]:
if map_func is not None:
t = map_func(fn)
else:
dsm = DatasetManager(db_opener(fn, self.mode))
t = dsm.get_coord('time')
self.file_time_map[fn] = t
self.time_file_map[t[0]] = fn
#self.time_access_map[t[0]] = dsm.variables[var_name] #+++ as below...
#self.datasetmanager_map[t[0]] = dsm #+++ deprecate? Only works if files are open...
tsegs.append(t)
#time_idx = time_idx.union(t.index)
new_segs = []
for i in range(1, len(tsegs)):
first_new, last_old = tsegs[i][0], tsegs[i - 1][-1]
tdelta = first_new - last_old
day = dt.days(1)
if tdelta > day:
new_t = dt.dates(last_old + day, first_new - day)
new_segs.append(new_t)
self.time_file_map[new_t[0]] = None
all_segs = sorted(tsegs + new_segs, key=lambda t: t[0])
self.seg_time_map = dict([(i, t[0]) for i, t in enumerate(all_segs)])
full_t = time.index.union_many(all_segs)
full_tc = TimeCoordinates(time.dimension, full_t)
self.cs = CoordinateSet((full_tc, coords.latitude, coords.longitude))
self.splitter = Splitter(full_tc, all_segs)
ncvar = dsm_start.variables[var_name]
self.fillvalue = ncvar.attrs['_FillValue'][0]
#self.fillvalue = 1.0
v = Variable.from_ncvar(ncvar)
self.mapped_var = MappedVariable(v, self.cs, ncvar.dtype)
def from_epoch(epoch, ts):
return epoch + dt.days(int(ts))
