def test_resample_to_shape(self):
# Test global with png
data, dest_lon, dest_lat, gpis, _, _ = \
resample_to_shape(self.pngfile, 'global', self.sp_res,
self.globalgrid, nan_value=self.fill_value,
dest_nan_value=self.fill_value, prefix='prefix')
assert dest_lon.shape == self.data.shape
assert dest_lat.shape == self.data.shape
assert (gpis[0], gpis[-1]) == (0, 17)
# Test local with hdf5
resimg = {
'data':
np.ma.array([[self.data[1][3].data]],
mask=self.mask[1][3],
fill_value=self.fill_value),
'data2':
np.ma.array([[self.data[1][3].data]],
mask=self.mask[1][3],
fill_value=self.fill_value)
}
data, _, _, _, _, metadata = \
resample_to_shape(self.h5file, self.region, self.sp_res,
self.grid, nan_value=self.fill_value,
dest_nan_value=self.fill_value)
nptest.assert_array_equal(data, resimg)
assert metadata == {
'data': self.metadata['data'],
'data2': self.metadata['data2']
}
# Test local with only one variable with netcdf
reslon = np.array([[30.]])
reslat = np.array([[0.]])
resimg = {
'data':
np.ma.array([[self.data[1][3].data]],
mask=self.mask[1][3],
fill_value=self.fill_value)
}
data, dest_lon, dest_lat, gpis, timestamp, metadata = \
resample_to_shape(self.ncfile, self.region, self.sp_res,
self.grid, nan_value=self.fill_value,
dest_nan_value=self.fill_value,
variables=['data'])
timediff = self.timestamp - timestamp
nptest.assert_array_equal(data, resimg)
assert dest_lon[0][0] == reslon
assert dest_lat[0][0] == reslat
assert gpis[0] == 0
assert timediff.days == 0
assert metadata == {'data': self.metadata['data']}
def test_resample_to_shape(self):
# Test global with png
datamean = 15.875
data, dest_lon, dest_lat, gpis, _, _ = \
resample_to_shape(self.pngfile, 'global', self.sp_res,
self.globalgrid, nan_value=self.fill_value,
dest_nan_value=self.fill_value, prefix='prefix')
# nptest.assert_array_equal(data, resimg)
assert datamean == data['prefix_dataset'].mean()
assert dest_lon.shape == self.data.shape
assert dest_lat.shape == self.data.shape
assert (gpis[0], gpis[-1]) == (0, 17)
# Test local with hdf5
resimg = {'data': np.ma.array([[self.data[1][3].data]],
mask=self.mask[1][3],
fill_value=self.fill_value),
'data2': np.ma.array([[self.data[1][3].data]],
mask=self.mask[1][3],
fill_value=self.fill_value)}
data, _, _, _, _, metadata = \
resample_to_shape(self.h5file, self.region, self.sp_res,
self.grid, nan_value=self.fill_value,
dest_nan_value=self.fill_value)
nptest.assert_array_equal(data, resimg)
assert metadata == {'data': self.metadata['data'],
'data2': self.metadata['data2']}
# Test local with only one variable with netcdf
reslon = np.array([[30.]])
reslat = np.array([[0.]])
resimg = {'data': np.ma.array([[self.data[1][3].data]],
mask=self.mask[1][3],
fill_value=self.fill_value)}
data, dest_lon, dest_lat, gpis, timestamp, metadata = \
resample_to_shape(self.ncfile, self.region, self.sp_res,
self.grid, nan_value=self.fill_value,
dest_nan_value=self.fill_value,
variables=['data'])
timediff = self.timestamp - timestamp
nptest.assert_array_equal(data, resimg)
assert dest_lon[0][0] == reslon
assert dest_lat[0][0] == reslat
assert gpis[0] == 0
assert timediff.days == 0
assert metadata == {'data': self.metadata['data']}
def _resample_spatial(self, region, begin, end, delete_rawdata,
shapefile=None):
"""Helper method that calls spatial resampling routines.
Parameters:
region : str
FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
begin : datetime
Start date of resampling
end : datetime
End date of resampling
delete_rawdata : bool
True if original downloaded files should be deleted after
resampling
"""
dest_file = self._get_tmp_filepath('spatial', region)
dirList = os.listdir(self.rawdata_path)
dirList.sort()
if region == 'global':
grid = gr.RegularGrid(sp_res=self.dest_sp_res)
else:
grid = gr.ShapeGrid(region, self.dest_sp_res, shapefile)
for item in dirList:
src_file = os.path.join(self.rawdata_path, item)
fdate = get_file_date(item, self.filedate)
if begin is not None:
if fdate < begin:
continue
if end is not None:
if fdate > end:
continue
if check_compressed(src_file):
dirname = os.path.splitext(item)[0]
dirpath = os.path.join(self.rawdata_path, dirname)
unpack(src_file)
src_file = select_file(os.listdir(dirpath))
src_file = os.path.join(dirpath, src_file)
if begin is not None:
if fdate < begin:
if check_compressed(item):
shutil.rmtree(os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
if end is not None:
if fdate > end:
if check_compressed(item):
shutil.rmtree(os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
print '.',
image, _, _, _, timestamp, metadata = \
resample_to_shape(src_file, region, self.dest_sp_res, grid,
self.name, self.nan_value,
self.dest_nan_value, self.variables,
shapefile)
if timestamp is None:
timestamp = get_file_date(item, self.filedate)
if self.temp_res == self.dest_temp_res:
filename = (region + '_' + str(self.dest_sp_res) + '_'
+ str(self.dest_temp_res) + '.nc')
dfile = os.path.join(self.data_path, filename)
nc.save_image(image, timestamp, region, metadata, dfile,
self.dest_start_date, self.dest_sp_res,
self.dest_nan_value, shapefile,
self.dest_temp_res)
else:
nc.write_tmp_file(image, timestamp, region, metadata,
dest_file, self.dest_start_date,
self.dest_sp_res, self.dest_nan_value,
shapefile)
# deletes unpacked files if existing
if check_compressed(item):
shutil.rmtree(os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
print ''
def _resample_spatial(self,
region,
begin,
end,
delete_rawdata,
shapefile=None):
"""Helper method that calls spatial resampling routines.
Parameters:
region : str
FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
begin : datetime
Start date of resampling
end : datetime
End date of resampling
delete_rawdata : bool
True if original downloaded files should be deleted after
resampling
"""
dest_file = self._get_tmp_filepath('spatial', region)
dirList = os.listdir(self.rawdata_path)
dirList.sort()
if region == 'global':
grid = gr.RegularGrid(sp_res=self.dest_sp_res)
else:
grid = gr.ShapeGrid(region, self.dest_sp_res, shapefile)
for item in dirList:
src_file = os.path.join(self.rawdata_path, item)
fdate = get_file_date(item, self.filedate)
if begin is not None:
if fdate < begin:
continue
if end is not None:
if fdate > end:
continue
if check_compressed(src_file):
dirname = os.path.splitext(item)[0]
dirpath = os.path.join(self.rawdata_path, dirname)
unpack(src_file)
src_file = select_file(os.listdir(dirpath))
src_file = os.path.join(dirpath, src_file)
if begin is not None:
if fdate < begin:
if check_compressed(item):
shutil.rmtree(
os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
if end is not None:
if fdate > end:
if check_compressed(item):
shutil.rmtree(
os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
print '.',
try:
image, _, _, _, timestamp, metadata = \
resample_to_shape(src_file, region, self.dest_sp_res, grid,
self.name, self.nan_value,
self.dest_nan_value, self.variables,
shapefile)
except ValueError:
print "[INFO] no data available for that region."
return "[INFO] no data available for that region."
if timestamp is None:
timestamp = get_file_date(item, self.filedate)
if self.temp_res == self.dest_temp_res:
filename = (region + '_' + str(self.dest_sp_res) + '_' +
str(self.dest_temp_res) + '.nc')
dfile = os.path.join(self.data_path, filename)
nc.save_image(image, timestamp, region, metadata, dfile,
self.dest_start_date, self.dest_sp_res,
self.dest_nan_value, shapefile,
self.dest_temp_res)
else:
nc.write_tmp_file(image, timestamp, region, metadata,
dest_file, self.dest_start_date,
self.dest_sp_res, self.dest_nan_value,
shapefile)
# deletes unpacked files if existing
if check_compressed(item):
shutil.rmtree(
os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
print ''
