def setUp(self):
TestBase.setUp(self)
## download test data
self.test_data.get_rd('cancm4_rhs')
self.uri = os.path.join(ocgis.env.DIR_TEST_DATA, 'CanCM4',
'rhs_day_CanCM4_decadal2010_r2i1p1_20110101-20201231.nc')
self.variable = 'rhs'
def setUp(self):
TestBase.setUp(self)
env.DIR_DATA = os.path.join(env.DIR_TEST_DATA, 'CanCM4')
## data may need to be pulled from remote repository
self.test_data.get_rd('cancm4_tasmin_2001')
self.test_data.get_rd('cancm4_tasmax_2011')
self.test_data.get_rd('cancm4_tas')
uris = [
'tasmin_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc',
'tasmax_day_CanCM4_decadal2010_r2i1p1_20110101-20201231.nc',
'tas_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc'
]
vars = ['tasmin', 'tasmax', 'tas']
time_range = [dt(2000, 1, 1), dt(2000, 12, 31)]
level_range = [2, 2]
self.datasets = [{
'uri': uri,
'variable': var,
'time_range': time_range,
'level_range': level_range
} for uri, var in zip(uris, vars)]
self.datasets_no_range = [{
'uri': uri,
'variable': var
} for uri, var in zip(uris, vars)]
def setUp(self):
TestBase.setUp(self)
## download test data
self.test_data.get_rd('cancm4_rhs')
self.uri = os.path.join(
ocgis.env.DIR_TEST_DATA, 'CanCM4',
'rhs_day_CanCM4_decadal2010_r2i1p1_20110101-20201231.nc')
self.variable = 'rhs'
def setUp(self):
TestBase.setUp(self)
rds = [self.test_data.get_rd('cancm4_tasmin_2001'), self.test_data.get_rd('cancm4_tasmax_2011'),
self.test_data.get_rd('cancm4_tas')]
time_range = [dt(2000, 1, 1), dt(2000, 12, 31)]
level_range = [2, 2]
self.datasets = [{'uri': rd.uri, 'variable': rd.variable, 'time_range': time_range, 'level_range': level_range}
for rd in rds]
self.datasets_no_range = [{'uri': rd.uri, 'variable': rd.variable} for rd in rds]
def setUp(self):
TestBase.setUp(self)
env.DIR_DATA = os.path.join(env.DIR_TEST_DATA,'CanCM4')
uris = [
'tasmin_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc',
'tasmax_day_CanCM4_decadal2010_r2i1p1_20110101-20201231.nc',
'tas_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc'
]
vars = ['tasmin','tasmax','tas']
time_range = [dt(2000,1,1),dt(2000,12,31)]
level_range = [2,2]
self.datasets = [{'uri':uri,'variable':var,'time_range':time_range,'level_range':level_range} for uri,var in zip(uris,vars)]
self.datasets_no_range = [{'uri':uri,'variable':var} for uri,var in zip(uris,vars)]
def setUp(self):
TestBase.setUp(self)
# data may need to be pulled from remote repository
rds = [self.test_data.get_rd('cancm4_tasmin_2001'), self.test_data.get_rd('cancm4_tasmax_2011'),
self.test_data.get_rd('cancm4_tas')]
time_range = [dt(2000, 1, 1), dt(2000, 12, 31)]
level_range = [2, 2]
self.datasets = [{'uri': rd.uri, 'variable': rd.variable, 'time_range': time_range, 'level_range': level_range}
for rd in rds]
self.datasets_no_range = [{'uri': rd.uri, 'variable': rd.variable} for rd in rds]
def deploy_test_local_copy(dest):
from ocgis.test.base import TestBase
dest = os.path.expanduser(dest)
if not os.path.exists(dest):
os.path.makedirs(dest)
tdata = TestBase.get_tst_data()
tdata.copy_files(dest, verbose=True)
def deploy_test_local_copy(dest):
from ocgis.test.base import TestBase
dest = os.path.expanduser(dest)
if not os.path.exists(dest):
os.path.makedirs(dest)
tdata = TestBase.get_tst_data()
tdata.copy_files(dest, verbose=True)
def setUp(self):
TestBase.setUp(self)
env.DIR_DATA = os.path.join(env.DIR_TEST_DATA,'CanCM4')
## data may need to be pulled from remote repository
self.test_data.get_rd('cancm4_tasmin_2001')
self.test_data.get_rd('cancm4_tasmax_2011')
self.test_data.get_rd('cancm4_tas')
uris = [
'tasmin_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc',
'tasmax_day_CanCM4_decadal2010_r2i1p1_20110101-20201231.nc',
'tas_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc'
]
vars = ['tasmin','tasmax','tas']
time_range = [dt(2000,1,1),dt(2000,12,31)]
level_range = [2,2]
self.datasets = [{'uri':uri,'variable':var,'time_range':time_range,'level_range':level_range} for uri,var in zip(uris,vars)]
self.datasets_no_range = [{'uri':uri,'variable':var} for uri,var in zip(uris,vars)]
def test():
tdata = TestBase.get_tdata()
rd = tdata.get_rd('cancm4_tas')
output_formats = ['csv+', 'csv', 'nc', 'shp', 'numpy']
_with_auxiliary_files = [True, False]
for output_format, with_auxiliary_files in itertools.product(
output_formats, _with_auxiliary_files):
dir_output = tempfile.mkdtemp()
try:
ocgis.env.DIR_OUTPUT = dir_output
ops = ocgis.OcgOperations(dataset=rd,
snippet=True,
output_format=output_format,
geom='state_boundaries',
select_ugid=[23],
prefix=output_format + '_data')
ret_path = ops.execute()
fmtd_ret = format_return(ret_path,
ops,
with_auxiliary_files=with_auxiliary_files)
assert (os.path.exists(fmtd_ret))
if output_format in ['csv', 'nc'
] and with_auxiliary_files is False:
assert (fmtd_ret.endswith(output_format))
else:
assert (is_zipfile(fmtd_ret))
zipf = ZipFile(fmtd_ret, 'r')
try:
namelist = zipf.namelist()
assert (len(namelist) > 0)
if output_format in ['csv+']:
test = [
re.match('shp/.+'.format(output_format), name) !=
None for name in namelist
]
assert (any(test))
else:
test = [
re.match('shp/.+'.format(output_format),
name) == None for name in namelist
]
assert (all(test))
finally:
zipf.close()
## numpy formats are not implemented
except NotImplementedError:
assert (output_format == 'numpy')
finally:
ocgis.env.reset()
shutil.rmtree(dir_output)
def setUp(self):
TestBase.setUp(self)
env.DIR_DATA = os.path.join(env.DIR_TEST_DATA, 'CanCM4')
uris = [
'tasmin_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc',
'tasmax_day_CanCM4_decadal2010_r2i1p1_20110101-20201231.nc',
'tas_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc'
]
vars = ['tasmin', 'tasmax', 'tas']
time_range = [dt(2000, 1, 1), dt(2000, 12, 31)]
level_range = [2, 2]
self.datasets = [{
'uri': uri,
'variable': var,
'time_range': time_range,
'level_range': level_range
} for uri, var in zip(uris, vars)]
self.datasets_no_range = [{
'uri': uri,
'variable': var
} for uri, var in zip(uris, vars)]
def test_subset_years_one_year():
tdata = TestBase.get_tdata()
rd = tdata.get_rd('cancm4_tas')
f,out_nc = tempfile.mkstemp(suffix='_test_nc.nc')
try:
subset(rd.uri,out_nc,1)
ds = nc.Dataset(out_nc,'r')
try:
tvar = ds.variables['time']
dts = nc.num2date(tvar[:],tvar.units,calendar=tvar.calendar)
uyears = np.unique([dt.year for dt in dts.flat])
assert(uyears.shape[0] == 1)
finally:
ds.close()
finally:
os.remove(out_nc)
def test_subset_years_one_year():
raise(SkipTest('dev'))
tdata = TestBase.get_tdata()
rd = tdata.get_rd('cancm4_tas')
f,out_nc = tempfile.mkstemp(suffix='_test_nc.nc')
try:
subset(rd.uri,out_nc,1)
ds = nc.Dataset(out_nc,'r')
try:
tvar = ds.variables['time']
dts = nc.num2date(tvar[:],tvar.units,calendar=tvar.calendar)
uyears = np.unique([dt.year for dt in dts.flat])
assert(uyears.shape[0] == 1)
finally:
ds.close()
finally:
os.remove(out_nc)
def package(pargs):
## get destination directory
dst = pargs.d or os.getcwd()
if not os.path.exists(dst):
raise (IOError(
'Destination directory does not exist: {0}'.format(dst)))
## import ocgis using relative locations
opath = os.path.join(os.getcwd(), 'src')
sys.path.append(opath)
to_tar = []
if pargs.target in ['shp', 'all']:
import ocgis
shp_dir = ocgis.env.DIR_SHPCABINET
for dirpath, dirnames, filenames in os.walk(shp_dir):
for filename in filenames:
path = os.path.join(dirpath, filename)
arcname = os.path.join('ocgis_data', 'shp',
os.path.split(dirpath)[1], filename)
to_tar.append({'path': path, 'arcname': arcname})
if pargs.target in ['nc', 'all']:
from ocgis.test.base import TestBase
tdata = TestBase.get_tdata()
for key, value in tdata.iteritems():
path = tdata.get_uri(key)
arcname = os.path.join('ocgis_data', 'nc',
tdata.get_relative_path(key))
to_tar.append({'path': path, 'arcname': arcname})
out = os.path.join(os.path.join(dst, 'ocgis_data.tar.gz'))
if pargs.verbose: print('destination file is: {0}'.format(out))
tf = tarfile.open(out, 'w:gz')
try:
for tz in to_tar:
if pargs.verbose and any(
[tz['path'].endswith(ii) for ii in ['shp', 'nc']]):
print('adding: {0}'.format(tz['path']))
tf.add(tz['path'], arcname=tz['arcname'])
finally:
if pargs.verbose: print('closing file...')
tf.close()
if pargs.verbose: print('compression complete.')
def test():
tdata = TestBase.get_tdata()
rd = tdata.get_rd('cancm4_tas')
output_formats = [
'csv+',
'csv',
'nc',
'shp',
'numpy']
_with_auxiliary_files = [True,False]
for output_format,with_auxiliary_files in itertools.product(output_formats,_with_auxiliary_files):
dir_output = tempfile.mkdtemp()
try:
ocgis.env.DIR_OUTPUT = dir_output
ops = ocgis.OcgOperations(dataset=rd,snippet=True,output_format=output_format,
geom='state_boundaries',select_ugid=[23],
prefix=output_format+'_data')
ret_path = ops.execute()
fmtd_ret = format_return(ret_path,ops,with_auxiliary_files=with_auxiliary_files)
assert(os.path.exists(fmtd_ret))
if output_format in ['csv','nc'] and with_auxiliary_files is False:
assert(fmtd_ret.endswith(output_format))
else:
assert(is_zipfile(fmtd_ret))
zipf = ZipFile(fmtd_ret,'r')
try:
namelist = zipf.namelist()
assert(len(namelist) > 0)
if output_format in ['csv+']:
test = [re.match('shp/.+'.format(output_format),name) != None for name in namelist]
assert(any(test))
else:
test = [re.match('shp/.+'.format(output_format),name) == None for name in namelist]
assert(all(test))
finally:
zipf.close()
## numpy formats are not implemented
except NotImplementedError:
assert(output_format == 'numpy')
finally:
ocgis.env.reset()
shutil.rmtree(dir_output)
def package(pargs):
## get destination directory
dst = pargs.d or os.getcwd()
if not os.path.exists(dst):
raise(IOError('Destination directory does not exist: {0}'.format(dst)))
## import ocgis using relative locations
opath = os.path.join(os.getcwd(),'src')
sys.path.append(opath)
to_tar = []
if pargs.target in ['shp','all']:
import ocgis
shp_dir = ocgis.env.DIR_SHPCABINET
for dirpath,dirnames,filenames in os.walk(shp_dir):
for filename in filenames:
path = os.path.join(dirpath,filename)
arcname = os.path.join('ocgis_data','shp',os.path.split(dirpath)[1],filename)
to_tar.append({'path':path,'arcname':arcname})
if pargs.target in ['nc','all']:
from ocgis.test.base import TestBase
tdata = TestBase.get_tdata()
for key,value in tdata.iteritems():
path = tdata.get_uri(key)
arcname = os.path.join('ocgis_data','nc',tdata.get_relative_path(key))
to_tar.append({'path':path,'arcname':arcname})
out = os.path.join(os.path.join(dst,'ocgis_data.tar.gz'))
if pargs.verbose: print('destination file is: {0}'.format(out))
tf = tarfile.open(out,'w:gz')
try:
for tz in to_tar:
if pargs.verbose and any([tz['path'].endswith(ii) for ii in ['shp','nc']]):
print('adding: {0}'.format(tz['path']))
tf.add(tz['path'],arcname=tz['arcname'])
finally:
if pargs.verbose: print('closing file...')
tf.close()
if pargs.verbose: print('compression complete.')
def setUp(self):
TestBase.setUp(self)
self.maurer = self.test_data.get_rd('maurer_bccr_1950')
self.cancm4 = self.test_data.get_rd('cancm4_tasmax_2001')
self.tasmin = self.test_data.get_rd('cancm4_tasmin_2001')
def setUp(self):
TestBase.setUp(self)
self.nc_factory().write()
def tearDown(self):
ocgis_lh.shutdown()
TestBase.tearDown(self)
def setUp(self):
TestBase.setUp(self)
rd = self.test_data.get_rd('cancm4_rhs')
self.uri = rd.uri
self.variable = rd.variable
def setUp(self):
TestBase.setUp(self)
self.maurer = self.test_data.get_rd('maurer_bccr_1950')
self.cancm4 = self.test_data.get_rd('cancm4_tasmax_2001')
self.tasmin = self.test_data.get_rd('cancm4_tasmin_2001')
def values(self):
tdata = TestBase.get_tdata()
return([tdata.get_rd('cancm4_tasmax_2001')])
def values(self):
tdata = TestBase.get_tdata()
return ([tdata.get_rd('cancm4_tasmax_2001')])
def setUp(self):
TestBase.setUp(self)
self.uri = os.path.join(ocgis.env.DIR_TEST_DATA,'CanCM4','rhs_day_CanCM4_decadal2010_r2i1p1_20110101-20201231.nc')
self.variable = 'rhs'
def tearDown(self):
ocgis_lh.shutdown()
TestBase.tearDown(self)
from ocgis.test.base import TestBase
from datasets.base import AbstractHarvestDataset, AbstractDataPackage
import datetime
import db
import query
import os
from unittest.case import SkipTest
from sqlalchemy.orm.exc import NoResultFound
import ocgis
from db import get_or_create
from NCPP.util.data_scanner import harvest
from NCPP.util.data_scanner.query import DataQuery
tdata = TestBase.get_tdata()
class CanCM4TestDataset(AbstractHarvestDataset):
uri = [tdata.get_uri('cancm4_tas')]
variables = ['tas']
clean_units = [{'standard_name':'K','long_name':'Kelvin'}]
clean_variable = [dict(standard_name='air_temperature',long_name='Near-Surface Air Temperature',description='Fill it in!')]
dataset_category = dict(name='GCMs',description='Global Circulation Models')
dataset = dict(name='CanCM4',description='Canadian Circulation Model 4')
type = 'variable'
class AbstractMaurerDataset(AbstractHarvestDataset):
dataset = dict(name='Maurer 2010',description='Amazing dataset!')
dataset_category = dict(name='Observational',description='Some observational datasets.')
class MaurerTas(AbstractMaurerDataset):
def setUp(self):
TestBase.setUp(self)
self.uri = os.path.join(
ocgis.env.DIR_TEST_DATA, 'CanCM4',
'rhs_day_CanCM4_decadal2010_r2i1p1_20110101-20201231.nc')
self.variable = 'rhs'
def setUp(self):
TestBase.setUp(self)
self.daymet = self.test_data.get_rd('daymet_tmax').uri
def setUpClass(cls):
cls.test_data = TestBase.get_tdata()
def setUp(self):
TestBase.setUp(self)
db.build_database(in_memory=True)
def setUp(self):
TestBase.setUp(self)
self.nc_factory().write()
def setUpClass(cls):
cls.test_data = TestBase.get_tdata()
def setUp(self):
TestBase.setUp(self)
self.daymet = self.test_data.get_rd('daymet_tmax').uri
