def test_read_image(self):
write_tmp_file(self.image, self.timestamp, self.region, self.metadata,
self.testfilename, self.start_date, self.sp_res)
_, _, _, timestamp, metadata = read_image(self.testfilename)
timediff = self.timestamp - timestamp
assert timediff.days == 0
assert metadata == self.metadata
def test_write_tmp_file(self):
write_tmp_file(
self.image, self.timestamp, self.region, self.metadata, self.testfilename, self.start_date, self.sp_res
)
with Dataset(self.testfilename) as nc_data:
data = nc_data.variables[self.variable]
mask = np.array(self.mask, dtype=bool)
nptest.assert_array_equal(self.data, data[0])
nptest.assert_array_equal(mask, data[0].mask)
assert data.getncattr("_FillValue") == self.fill_value
assert data.getncattr("Attribute1") == self.metadata[self.variable]["Attribute1"]
def test_clip_bbox(self):
lon_min = -180
lon_max = 0
lat_min = 0
lat_max = 90
write_tmp_file(
self.image, self.timestamp, self.region, self.metadata, self.testfilename, self.start_date, self.sp_res
)
data_src, lon, lat, _, _ = read_image(self.testfilename)
data, lon_new, lat_new = clip_bbox(data_src, lon, lat, lon_min, lat_min, lon_max, lat_max)
nptest.assert_array_equal(self.bbox, data[self.variable])
nptest.assert_array_equal(self.lon_new, lon_new)
nptest.assert_array_equal(self.lat_new, lat_new)
def _resample_spatial(self, region, begin, end, delete_rawdata,
shapefile=None):
"""Helper method that calls spatial resampling routines.
Parameters:
region : str
FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
begin : datetime
Start date of resampling
end : datetime
End date of resampling
delete_rawdata : bool
True if original downloaded files should be deleted after
resampling
"""
dest_file = self._get_tmp_filepath('spatial', region)
dirList = os.listdir(self.rawdata_path)
dirList.sort()
if region == 'global':
grid = gr.RegularGrid(sp_res=self.dest_sp_res)
else:
grid = gr.ShapeGrid(region, self.dest_sp_res, shapefile)
for item in dirList:
src_file = os.path.join(self.rawdata_path, item)
fdate = get_file_date(item, self.filedate)
if begin is not None:
if fdate < begin:
continue
if end is not None:
if fdate > end:
continue
if check_compressed(src_file):
dirname = os.path.splitext(item)[0]
dirpath = os.path.join(self.rawdata_path, dirname)
unpack(src_file)
src_file = select_file(os.listdir(dirpath))
src_file = os.path.join(dirpath, src_file)
if begin is not None:
if fdate < begin:
if check_compressed(item):
shutil.rmtree(os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
if end is not None:
if fdate > end:
if check_compressed(item):
shutil.rmtree(os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
print '.',
image, _, _, _, timestamp, metadata = \
resample_to_shape(src_file, region, self.dest_sp_res, grid,
self.name, self.nan_value,
self.dest_nan_value, self.variables,
shapefile)
if timestamp is None:
timestamp = get_file_date(item, self.filedate)
if self.temp_res == self.dest_temp_res:
filename = (region + '_' + str(self.dest_sp_res) + '_'
+ str(self.dest_temp_res) + '.nc')
dfile = os.path.join(self.data_path, filename)
nc.save_image(image, timestamp, region, metadata, dfile,
self.dest_start_date, self.dest_sp_res,
self.dest_nan_value, shapefile,
self.dest_temp_res)
else:
nc.write_tmp_file(image, timestamp, region, metadata,
dest_file, self.dest_start_date,
self.dest_sp_res, self.dest_nan_value,
shapefile)
# deletes unpacked files if existing
if check_compressed(item):
shutil.rmtree(os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
print ''
def _resample_spatial(self,
region,
begin,
end,
delete_rawdata,
shapefile=None):
"""Helper method that calls spatial resampling routines.
Parameters:
region : str
FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
begin : datetime
Start date of resampling
end : datetime
End date of resampling
delete_rawdata : bool
True if original downloaded files should be deleted after
resampling
"""
dest_file = self._get_tmp_filepath('spatial', region)
dirList = os.listdir(self.rawdata_path)
dirList.sort()
if region == 'global':
grid = gr.RegularGrid(sp_res=self.dest_sp_res)
else:
grid = gr.ShapeGrid(region, self.dest_sp_res, shapefile)
for item in dirList:
src_file = os.path.join(self.rawdata_path, item)
fdate = get_file_date(item, self.filedate)
if begin is not None:
if fdate < begin:
continue
if end is not None:
if fdate > end:
continue
if check_compressed(src_file):
dirname = os.path.splitext(item)[0]
dirpath = os.path.join(self.rawdata_path, dirname)
unpack(src_file)
src_file = select_file(os.listdir(dirpath))
src_file = os.path.join(dirpath, src_file)
if begin is not None:
if fdate < begin:
if check_compressed(item):
shutil.rmtree(
os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
if end is not None:
if fdate > end:
if check_compressed(item):
shutil.rmtree(
os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
print '.',
try:
image, _, _, _, timestamp, metadata = \
resample_to_shape(src_file, region, self.dest_sp_res, grid,
self.name, self.nan_value,
self.dest_nan_value, self.variables,
shapefile)
except ValueError:
print "[INFO] no data available for that region."
return "[INFO] no data available for that region."
if timestamp is None:
timestamp = get_file_date(item, self.filedate)
if self.temp_res == self.dest_temp_res:
filename = (region + '_' + str(self.dest_sp_res) + '_' +
str(self.dest_temp_res) + '.nc')
dfile = os.path.join(self.data_path, filename)
nc.save_image(image, timestamp, region, metadata, dfile,
self.dest_start_date, self.dest_sp_res,
self.dest_nan_value, shapefile,
self.dest_temp_res)
else:
nc.write_tmp_file(image, timestamp, region, metadata,
dest_file, self.dest_start_date,
self.dest_sp_res, self.dest_nan_value,
shapefile)
# deletes unpacked files if existing
if check_compressed(item):
shutil.rmtree(
os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
print ''
