def test_unpack(self):
# create txt-file
txt1 = open(os.path.join(curpath(), 'data', 'test1.txt'), 'w')
txt1.close()
# create first zip archive
ziparc = zipfile.ZipFile(os.path.join(curpath(), 'data',
'ziparc.zip'), 'w')
# write txt-file and folder to zip archive
ziparc.write(os.path.join(curpath(), 'data', 'test1.txt'))
ziparc.close()
# 'create' second txt-file
os.rename(os.path.join(curpath(), 'data', 'test1.txt'),
os.path.join(curpath(), 'data', 'test2.txt'))
# create second zip archive
ziparc2 = zipfile.ZipFile(os.path.join(curpath(), 'data',
'ziparc2.zip'), 'w')
# write second txt-file and first zip archive to second zip archive
ziparc2.write(os.path.join(curpath(), 'data', 'test2.txt'))
ziparc2.write(os.path.join(curpath(), 'data', 'ziparc.zip'))
ziparc2.close()
dirlen = 2
up.unpack(ziparc2.filename, self.outpath)
unpack_len = len(os.listdir(self.outpath))
assert unpack_len == dirlen
shutil.rmtree(self.outpath)
os.remove(os.path.join(curpath(), 'data', ziparc.filename))
os.remove(os.path.join(curpath(), 'data', ziparc2.filename))
def test_unpack(self):
# create txt-file
txt1 = open(os.path.join(curpath(), 'data', 'test1.txt'), 'w')
txt1.close()
# create first zip archive
ziparc = zipfile.ZipFile(os.path.join(curpath(), 'data', 'ziparc.zip'),
'w')
# write txt-file and folder to zip archive
ziparc.write(os.path.join(curpath(), 'data', 'test1.txt'))
ziparc.close()
# 'create' second txt-file
os.rename(os.path.join(curpath(), 'data', 'test1.txt'),
os.path.join(curpath(), 'data', 'test2.txt'))
# create second zip archive
ziparc2 = zipfile.ZipFile(
os.path.join(curpath(), 'data', 'ziparc2.zip'), 'w')
# write second txt-file and first zip archive to second zip archive
ziparc2.write(os.path.join(curpath(), 'data', 'test2.txt'))
ziparc2.write(os.path.join(curpath(), 'data', 'ziparc.zip'))
ziparc2.close()
dirlen = 2
up.unpack(ziparc2.filename, self.outpath)
unpack_len = len(os.listdir(self.outpath))
assert unpack_len == dirlen
shutil.rmtree(self.outpath)
os.remove(os.path.join(curpath(), 'data', ziparc.filename))
os.remove(os.path.join(curpath(), 'data', ziparc2.filename))
def _resample_spatial(self, region, begin, end, delete_rawdata,
shapefile=None):
"""Helper method that calls spatial resampling routines.
Parameters:
region : str
FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
begin : datetime
Start date of resampling
end : datetime
End date of resampling
delete_rawdata : bool
True if original downloaded files should be deleted after
resampling
"""
dest_file = self._get_tmp_filepath('spatial', region)
dirList = os.listdir(self.rawdata_path)
dirList.sort()
if region == 'global':
grid = gr.RegularGrid(sp_res=self.dest_sp_res)
else:
grid = gr.ShapeGrid(region, self.dest_sp_res, shapefile)
for item in dirList:
src_file = os.path.join(self.rawdata_path, item)
fdate = get_file_date(item, self.filedate)
if begin is not None:
if fdate < begin:
continue
if end is not None:
if fdate > end:
continue
if check_compressed(src_file):
dirname = os.path.splitext(item)[0]
dirpath = os.path.join(self.rawdata_path, dirname)
unpack(src_file)
src_file = select_file(os.listdir(dirpath))
src_file = os.path.join(dirpath, src_file)
if begin is not None:
if fdate < begin:
if check_compressed(item):
shutil.rmtree(os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
if end is not None:
if fdate > end:
if check_compressed(item):
shutil.rmtree(os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
print '.',
image, _, _, _, timestamp, metadata = \
resample_to_shape(src_file, region, self.dest_sp_res, grid,
self.name, self.nan_value,
self.dest_nan_value, self.variables,
shapefile)
if timestamp is None:
timestamp = get_file_date(item, self.filedate)
if self.temp_res == self.dest_temp_res:
filename = (region + '_' + str(self.dest_sp_res) + '_'
+ str(self.dest_temp_res) + '.nc')
dfile = os.path.join(self.data_path, filename)
nc.save_image(image, timestamp, region, metadata, dfile,
self.dest_start_date, self.dest_sp_res,
self.dest_nan_value, shapefile,
self.dest_temp_res)
else:
nc.write_tmp_file(image, timestamp, region, metadata,
dest_file, self.dest_start_date,
self.dest_sp_res, self.dest_nan_value,
shapefile)
# deletes unpacked files if existing
if check_compressed(item):
shutil.rmtree(os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
print ''
def _resample_spatial(self,
region,
begin,
end,
delete_rawdata,
shapefile=None):
"""Helper method that calls spatial resampling routines.
Parameters:
region : str
FIPS country code (https://en.wikipedia.org/wiki/FIPS_country_code)
begin : datetime
Start date of resampling
end : datetime
End date of resampling
delete_rawdata : bool
True if original downloaded files should be deleted after
resampling
"""
dest_file = self._get_tmp_filepath('spatial', region)
dirList = os.listdir(self.rawdata_path)
dirList.sort()
if region == 'global':
grid = gr.RegularGrid(sp_res=self.dest_sp_res)
else:
grid = gr.ShapeGrid(region, self.dest_sp_res, shapefile)
for item in dirList:
src_file = os.path.join(self.rawdata_path, item)
fdate = get_file_date(item, self.filedate)
if begin is not None:
if fdate < begin:
continue
if end is not None:
if fdate > end:
continue
if check_compressed(src_file):
dirname = os.path.splitext(item)[0]
dirpath = os.path.join(self.rawdata_path, dirname)
unpack(src_file)
src_file = select_file(os.listdir(dirpath))
src_file = os.path.join(dirpath, src_file)
if begin is not None:
if fdate < begin:
if check_compressed(item):
shutil.rmtree(
os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
if end is not None:
if fdate > end:
if check_compressed(item):
shutil.rmtree(
os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
continue
print '.',
try:
image, _, _, _, timestamp, metadata = \
resample_to_shape(src_file, region, self.dest_sp_res, grid,
self.name, self.nan_value,
self.dest_nan_value, self.variables,
shapefile)
except ValueError:
print "[INFO] no data available for that region."
return "[INFO] no data available for that region."
if timestamp is None:
timestamp = get_file_date(item, self.filedate)
if self.temp_res == self.dest_temp_res:
filename = (region + '_' + str(self.dest_sp_res) + '_' +
str(self.dest_temp_res) + '.nc')
dfile = os.path.join(self.data_path, filename)
nc.save_image(image, timestamp, region, metadata, dfile,
self.dest_start_date, self.dest_sp_res,
self.dest_nan_value, shapefile,
self.dest_temp_res)
else:
nc.write_tmp_file(image, timestamp, region, metadata,
dest_file, self.dest_start_date,
self.dest_sp_res, self.dest_nan_value,
shapefile)
# deletes unpacked files if existing
if check_compressed(item):
shutil.rmtree(
os.path.join(self.rawdata_path,
os.path.splitext(item)[0]))
print ''