def setUpClass(self):
self.rootpath = os.path.join(curpath(), 'data', 'src_test')
self.dest_nan_value = -99
self.dest_regions = ['AU']
self.dest_sp_res = 1
self.dest_temp_res = 'daily'
self.dest_start_date = datetime(2013, 1, 7)
self.enddate = datetime(2013, 1, 13)
self.testdate = datetime(2013, 1, 10)
self.name = 'TEST'
self.filename = "test_{YYYY}_{MM}_{TT}.png"
self.filedate = {'YYYY': (5, 9), 'MM': (10, 12), 'DD': (13, 15)}
self.temp_res = 'daily'
self.protocol = 'local'
self.begin_date = datetime(2013, 1, 7)
if os.path.exists(self.rootpath):
shutil.rmtree(self.rootpath)
os.mkdir(self.rootpath)
os.mkdir(os.path.join(self.rootpath, 'RAWDATA'))
os.mkdir(os.path.join(self.rootpath, 'DATA'))
os.mkdir(os.path.join(self.rootpath, 'TMP'))
# setup test png files
self.pngdir = os.path.join(curpath(), 'data', 'testpngs')
self.host = self.pngdir
if os.path.exists(self.pngdir):
shutil.rmtree(self.pngdir)
os.mkdir(self.pngdir)
self.dtindex = get_dtindex('day', self.dest_start_date, self.enddate)
for i, dat in enumerate(self.dtindex):
if i == 3:
continue
year = str(dat.year)
month = "%02d" % (dat.month)
day = "%02d" % (dat.day)
fname = ('test_' + year + '_' + month + '_' + day + '.png')
shutil.copy(os.path.join(curpath(), 'data', 'test.png'),
os.path.join(self.pngdir, fname))
self.source = BasicSource(self.name,
self.filename,
self.filedate,
self.temp_res,
self.rootpath,
self.host,
self.protocol,
begin_date=self.begin_date,
dest_nan_value=-self.dest_nan_value,
dest_regions=self.dest_regions,
dest_sp_res=self.dest_sp_res,
dest_temp_res=self.dest_temp_res,
dest_start_date=self.dest_start_date)
def setUpClass(self):
self.rootpath = os.path.join(curpath(), 'data', 'src_test')
self.dest_nan_value = -99
self.dest_regions = ['AU']
self.dest_sp_res = 1
self.dest_temp_res = 'daily'
self.dest_start_date = datetime(2013, 1, 7)
self.enddate = datetime(2013, 1, 13)
self.testdate = datetime(2013, 1, 10)
self.name = 'TEST'
self.filename = "test_{YYYY}_{MM}_{TT}.png"
self.filedate = {'YYYY': (5, 9), 'MM': (10, 12), 'DD': (13, 15)}
self.temp_res = 'daily'
self.protocol = 'local'
self.begin_date = datetime(2013, 1, 7)
if os.path.exists(self.rootpath):
shutil.rmtree(self.rootpath)
os.mkdir(self.rootpath)
os.mkdir(os.path.join(self.rootpath, 'RAWDATA'))
os.mkdir(os.path.join(self.rootpath, 'DATA'))
os.mkdir(os.path.join(self.rootpath, 'TMP'))
# setup test png files
self.pngdir = os.path.join(curpath(), 'data', 'testpngs')
self.host = self.pngdir
if os.path.exists(self.pngdir):
shutil.rmtree(self.pngdir)
os.mkdir(self.pngdir)
self.dtindex = get_dtindex('day', self.dest_start_date,
self.enddate)
for i, dat in enumerate(self.dtindex):
if i == 3:
continue
year = str(dat.year)
month = "%02d" % (dat.month)
day = "%02d" % (dat.day)
fname = ('test_' + year + '_' + month + '_' + day + '.png')
shutil.copy(os.path.join(curpath(), 'data', 'test.png'),
os.path.join(self.pngdir, fname))
self.source = BasicSource(self.name, self.filename, self.filedate,
self.temp_res, self.rootpath,
self.host, self.protocol,
begin_date=self.begin_date,
dest_nan_value=-self.dest_nan_value,
dest_regions=self.dest_regions,
dest_sp_res=self.dest_sp_res,
dest_temp_res=self.dest_temp_res,
dest_start_date=self.dest_start_date)
class Test(unittest.TestCase):
@classmethod
def setUpClass(self):
self.rootpath = os.path.join(curpath(), 'data', 'src_test')
self.dest_nan_value = -99
self.dest_regions = ['AU']
self.dest_sp_res = 1
self.dest_temp_res = 'daily'
self.dest_start_date = datetime(2013, 1, 7)
self.enddate = datetime(2013, 1, 13)
self.testdate = datetime(2013, 1, 10)
self.name = 'TEST'
self.filename = "test_{YYYY}_{MM}_{TT}.png"
self.filedate = {'YYYY': (5, 9), 'MM': (10, 12), 'DD': (13, 15)}
self.temp_res = 'daily'
self.protocol = 'local'
self.begin_date = datetime(2013, 1, 7)
if os.path.exists(self.rootpath):
shutil.rmtree(self.rootpath)
os.mkdir(self.rootpath)
os.mkdir(os.path.join(self.rootpath, 'RAWDATA'))
os.mkdir(os.path.join(self.rootpath, 'DATA'))
os.mkdir(os.path.join(self.rootpath, 'TMP'))
# setup test png files
self.pngdir = os.path.join(curpath(), 'data', 'testpngs')
self.host = self.pngdir
if os.path.exists(self.pngdir):
shutil.rmtree(self.pngdir)
os.mkdir(self.pngdir)
self.dtindex = get_dtindex('day', self.dest_start_date, self.enddate)
for i, dat in enumerate(self.dtindex):
if i == 3:
continue
year = str(dat.year)
month = "%02d" % (dat.month)
day = "%02d" % (dat.day)
fname = ('test_' + year + '_' + month + '_' + day + '.png')
shutil.copy(os.path.join(curpath(), 'data', 'test.png'),
os.path.join(self.pngdir, fname))
self.source = BasicSource(self.name,
self.filename,
self.filedate,
self.temp_res,
self.rootpath,
self.host,
self.protocol,
begin_date=self.begin_date,
dest_nan_value=-self.dest_nan_value,
dest_regions=self.dest_regions,
dest_sp_res=self.dest_sp_res,
dest_temp_res=self.dest_temp_res,
dest_start_date=self.dest_start_date)
@classmethod
def tearDownClass(self):
if os.path.exists(self.rootpath):
shutil.rmtree(self.rootpath)
if os.path.exists(self.pngdir):
shutil.rmtree(self.pngdir)
def test_download_and_resample(self):
self.source.download_and_resample(end=self.enddate)
for date in self.dtindex:
img, _, _, _ = self.source.read_img(date)
if date == self.testdate:
assert img[2, 4] is np.ma.masked
else:
assert img[2, 4] == 177.
def test_fill_gaps(self):
shutil.copy(os.path.join(curpath(), 'data', 'test.png'),
os.path.join(self.pngdir, 'test_2013_01_10.png'))
self.source.fill_gaps(begin=self.dest_start_date, end=self.enddate)
img, _, _, _ = self.source.read_img(self.testdate)
assert img[1, 1] == 145.
def add_source(self,
name,
filename,
filedate,
temp_res,
host,
protocol,
username=None,
password=None,
port=22,
directory=None,
dirstruct=None,
begin_date=None,
regions=None,
variables=None,
nan_value=None,
valid_range=None,
unit=None,
ffilter=None,
data_range=None,
colorbar=None,
src_file=None,
labels=None,
xticks=None):
"""Creates BasicSource class and adds it to `Poet.sources`.
Parameters
----------
name : str
Name of the data source.
filename : str
Structure/convention of the file name.
filedate : dict
Position of date fields in filename, given as tuple.
temp_res : str
Temporal resolution of the source.
host : str
Link to data host.
protocol : str
Protocol for data transfer.
username : str, optional
Username for data access.
password : str, optional
Password for data access.
port : int, optional
Port to data host, defaults to 22.
directory : str, optional
Path to data on host.
dirstruct : list of strings
Structure of source directory, each list item represents a
subdirectory.
regions : list of str, optional
List of regions where data from source is available. Uses all
regions as given in Poet.regions attribute if None.
begin_date : datetime.date, optional
Date from which on data is available.
variables : string or list of strings, optional
Variables used from data source.
nan_value : int, float, optional
Nan value of the original data as given by the data provider.
valid_range : tuple of int of float, optional
Valid range of data, given as (minimum, maximum).
data_range : tuple of int of float, optional
Range of the values as data given in rawdata (minimum, maximum).
Will be scaled to valid_range.
ffilter : str, optional
Pattern that apperas in filename. Can be used to select out not
needed files if multiple files per date are provided.
colorbar : str, optional
Colorbar to use, use one from
http://matplotlib.org/examples/color/colormaps_reference.html;
defaults to jet.
labels : list, optional
Custom tick-labels for the legend in the web-app; must have same
dimension as xticks and only works if xticks is set; Defaults
to None.
xticks : list of int or float, optional
Custom tick locations for the legend in the web-app; must have same
dimension as labels and only works if labels is set; Defaults
to None.
unit : str, optional
Unit of dataset for displaying in legend. Does not have to be set
if unit is specified in input file metadata. Defaults to None.
src_file : dict of str, optional
Path to file that contains source. Uses default NetCDF file if
None.
"""
source = BasicSource(name,
filename,
filedate,
temp_res,
self.rootpath,
host,
protocol,
username=username,
password=password,
port=port,
ffilter=ffilter,
directory=directory,
dirstruct=dirstruct,
begin_date=begin_date,
variables=variables,
nan_value=nan_value,
valid_range=valid_range,
unit=unit,
regions=regions,
data_range=data_range,
colorbar=colorbar,
dest_nan_value=self.nan_value,
dest_regions=self.regions,
dest_sp_res=self.spatial_resolution,
dest_temp_res=self.temporal_resolution,
dest_start_date=self.start_date,
src_file=src_file,
labels=labels,
xticks=xticks)
self.sources[name] = source
class Test(unittest.TestCase):
@classmethod
def setUpClass(self):
self.rootpath = os.path.join(curpath(), 'data', 'src_test')
self.dest_nan_value = -99
self.dest_regions = ['AU']
self.dest_sp_res = 1
self.dest_temp_res = 'daily'
self.dest_start_date = datetime(2013, 1, 7)
self.enddate = datetime(2013, 1, 13)
self.testdate = datetime(2013, 1, 10)
self.name = 'TEST'
self.filename = "test_{YYYY}_{MM}_{TT}.png"
self.filedate = {'YYYY': (5, 9), 'MM': (10, 12), 'DD': (13, 15)}
self.temp_res = 'daily'
self.protocol = 'local'
self.begin_date = datetime(2013, 1, 7)
if os.path.exists(self.rootpath):
shutil.rmtree(self.rootpath)
os.mkdir(self.rootpath)
os.mkdir(os.path.join(self.rootpath, 'RAWDATA'))
os.mkdir(os.path.join(self.rootpath, 'DATA'))
os.mkdir(os.path.join(self.rootpath, 'TMP'))
# setup test png files
self.pngdir = os.path.join(curpath(), 'data', 'testpngs')
self.host = self.pngdir
if os.path.exists(self.pngdir):
shutil.rmtree(self.pngdir)
os.mkdir(self.pngdir)
self.dtindex = get_dtindex('day', self.dest_start_date,
self.enddate)
for i, dat in enumerate(self.dtindex):
if i == 3:
continue
year = str(dat.year)
month = "%02d" % (dat.month)
day = "%02d" % (dat.day)
fname = ('test_' + year + '_' + month + '_' + day + '.png')
shutil.copy(os.path.join(curpath(), 'data', 'test.png'),
os.path.join(self.pngdir, fname))
self.source = BasicSource(self.name, self.filename, self.filedate,
self.temp_res, self.rootpath,
self.host, self.protocol,
begin_date=self.begin_date,
dest_nan_value=-self.dest_nan_value,
dest_regions=self.dest_regions,
dest_sp_res=self.dest_sp_res,
dest_temp_res=self.dest_temp_res,
dest_start_date=self.dest_start_date)
@classmethod
def tearDownClass(self):
if os.path.exists(self.rootpath):
shutil.rmtree(self.rootpath)
if os.path.exists(self.pngdir):
shutil.rmtree(self.pngdir)
def test_download_and_resample(self):
self.source.download_and_resample(end=self.enddate)
for date in self.dtindex:
img, _, _, _ = self.source.read_img(date)
if date == self.testdate:
assert img[2, 4] is np.ma.masked
else:
assert img[2, 4] == 177.
def test_fill_gaps(self):
shutil.copy(os.path.join(curpath(), 'data', 'test.png'),
os.path.join(self.pngdir, 'test_2013_01_10.png'))
self.source.fill_gaps(begin=self.dest_start_date, end=self.enddate)
img, _, _, _ = self.source.read_img(self.testdate)
assert img[1, 1] == 145.