def _resample_temporal(self, region, shapefile=None):
"""Helper method that calls temporal resampling routines.
Parameters:
region : str
Identifier of the region in the shapefile. If the default shapefile
is used, this would be the FIPS country code.
shapefile : str, optional
Path to shape file, uses "world country admin boundary shapefile"
by default.
"""
src_file = self._get_tmp_filepath('spatial', region)
if not os.path.exists(src_file):
print '[Info] No data available for this period'
return False
data = {}
variables, _, period = nc.get_properties(src_file)
dtindex = dt.get_dtindex(self.dest_temp_res, period[0], period[1])
for date in dtindex:
# skip if data for period is not complete
# if date > period[1]:
# continue
if self.dest_temp_res == 'dekad':
if date.day < 21:
begin = datetime(date.year, date.month, date.day - 10 + 1)
else:
begin = datetime(date.year, date.month, 21)
end = date
else:
begin = period[0]
end = date
data = {}
metadata = {}
for var in variables:
img, _, _, meta = \
nc.read_variable(src_file, var, begin, end)
metadata[var] = meta
data[var] = average_layers(img, self.dest_nan_value)
dest_file = self.src_file[region]
nc.save_image(data, date, region, metadata, dest_file,
self.dest_start_date, self.dest_sp_res,
self.dest_nan_value, shapefile, self.dest_temp_res)
# delete intermediate netCDF file
print ''
os.unlink(src_file)
def _resample_temporal(self, region, shapefile=None):
"""Helper method that calls temporal resampling routines.
Parameters:
region : str
Identifier of the region in the shapefile. If the default shapefile
is used, this would be the FIPS country code.
shapefile : str, optional
Path to shape file, uses "world country admin boundary shapefile"
by default.
"""
src_file = self._get_tmp_filepath('spatial', region)
if not os.path.exists(src_file):
print '[Info] No data available for this period'
return False
data = {}
variables, _, period = nc.get_properties(src_file)
dtindex = dt.get_dtindex(self.dest_temp_res, period[0], period[1])
for date in dtindex:
# skip if data for period is not complete
# if date > period[1]:
# continue
if self.dest_temp_res == 'dekad':
if date.day < 21:
begin = datetime(date.year, date.month, date.day - 10 + 1)
else:
begin = datetime(date.year, date.month, 21)
end = date
else:
begin = period[0]
end = date
data = {}
metadata = {}
for var in variables:
img, _, _, meta = \
nc.read_variable(src_file, var, begin, end)
metadata[var] = meta
data[var] = average_layers(img, self.dest_nan_value)
dest_file = self.src_file[region]
nc.save_image(data, date, region, metadata, dest_file,
self.dest_start_date, self.dest_sp_res,
self.dest_nan_value, shapefile, self.dest_temp_res)
# delete intermediate netCDF file
print ''
os.unlink(src_file)
def fill_gaps(self, begin=None, end=None):
"""
Detects gaps in data and tries to fill them by downloading and
resampling the data within these periods.
Parameters
----------
begin : datetime
Begin date of intervall to check, defaults to None.
end : datetime
End date of intervall to check, defaults to None.
"""
gaps = []
for region in self.valid_regions:
if self.regions is not None:
if region not in self.regions:
continue
_, _, period = nc.get_properties(self.src_file[region])
if begin is None:
if self.begin_date < self.dest_start_date:
begin = self.dest_start_date
else:
begin = self.begin_date
if end is None:
end = period[1]
drange = dt.get_dtindex(self.dest_temp_res, begin, end)
for date in drange:
nonans = []
for var in self.get_variables():
img, _, _, _ = self.read_img(date, region, var)
if np.nanmean(img) is not np.ma.masked:
nonans.append(1)
if len(nonans) == 0:
if date not in gaps:
gaps.append(date)
if len(gaps) == 0:
print '[INFO] No gaps found.'
else:
print('[INFO] Found ' + str(len(gaps)) +
' gap(s), attempt to fill..')
for date in gaps:
if self.dest_temp_res in ['day', 'daily']:
begin = date
end = date
else:
begin, end = dt.check_period_boundaries(self.dest_temp_res,
date)
self.download_and_resample(begin=begin, end=end)
def fill_gaps(self, begin=None, end=None):
"""
Detects gaps in data and tries to fill them by downloading and
resampling the data within these periods.
Parameters
----------
begin : datetime
Begin date of intervall to check, defaults to None.
end : datetime
End date of intervall to check, defaults to None.
"""
gaps = []
for region in self.valid_regions:
if self.regions is not None:
if region not in self.regions:
continue
_, _, period = nc.get_properties(self.src_file[region])
if begin is None:
if self.begin_date < self.dest_start_date:
begin = self.dest_start_date
else:
begin = self.begin_date
if end is None:
end = period[1]
drange = dt.get_dtindex(self.dest_temp_res, begin, end)
for date in drange:
nonans = []
for var in self.get_variables():
img, _, _, _ = self.read_img(date, region, var)
if np.nanmean(img) is not np.ma.masked:
nonans.append(1)
if len(nonans) == 0:
if date not in gaps:
gaps.append(date)
if len(gaps) == 0:
print '[INFO] No gaps found.'
else:
print('[INFO] Found ' + str(len(gaps)) +
' gap(s), attempt to fill..')
for date in gaps:
if self.dest_temp_res in ['day', 'daily']:
begin = date
end = date
else:
begin, end = dt.check_period_boundaries(
self.dest_temp_res, date)
self.download_and_resample(begin=begin, end=end)
def test_get_properties(self):
save_image(self.image, self.timestamp, self.region, self.metadata,
self.testfilename, self.start_date, self.sp_res,
temp_res=self.temp_res)
variables, dimensions, period = get_properties(self.testfilename)
timediff1 = self.timestamp - period[0]
timediff2 = self.timestamp - period[1]
assert variables[0] == self.variable
assert dimensions == ['lat', 'lon', 'time']
assert timediff1.days == 0
assert timediff2.days == 0
def test_get_properties(self):
save_image(
self.image,
self.timestamp,
self.region,
self.metadata,
self.testfilename,
self.start_date,
self.sp_res,
temp_res=self.temp_res,
)
variables, dimensions, period = get_properties(self.testfilename)
timediff1 = self.timestamp - period[0]
assert self.variable in variables
assert dimensions == ["lat", "lon", "time"]
assert timediff1.days == 0
def get_variables(self):
"""
Gets all variables given in the NetCDF file.
Returns
-------
variables : list of str
Variables from given in the NetCDF file.
"""
# nc_name = self.src_file[self.dest_regions[0]]
# nc_vars, _, _ = nc.get_properties(nc_name)
nc_vars = []
for reg in self.dest_regions:
vari, _, _ = nc.get_properties(self.src_file[reg])
if vari is None:
continue
for v in vari:
if v not in nc_vars:
nc_vars.append(v)
variables = []
if self.variables is not None:
for var in self.variables:
if var in nc_vars:
variables.append(var)
else:
if self.name + '_' + var in nc_vars:
variables.append(self.name + '_' + var)
else:
for var in nc_vars:
if self.name + '_dataset' in var:
variables.append(var)
elif self.name in var:
variables.append(var)
return variables
def get_variables(self):
"""
Gets all variables given in the NetCDF file.
Returns
-------
variables : list of str
Variables from given in the NetCDF file.
"""
nc_vars = []
for reg in self.valid_regions:
vari, _, _ = nc.get_properties(self.src_file[reg])
if vari is None:
continue
for v in vari:
if v not in nc_vars:
nc_vars.append(v)
variables = []
if self.variables is not None:
for var in self.variables:
if var in nc_vars:
variables.append(var)
else:
if self.name + '_' + var in nc_vars:
variables.append(self.name + '_' + var)
else:
for var in nc_vars:
if self.name + '_dataset' in var:
variables.append(var)
elif self.name in var:
variables.append(var)
return variables