def test():
tdata = TestBase.get_tdata()
rd = tdata.get_rd('cancm4_tas')
output_formats = ['csv+', 'csv', 'nc', 'shp', 'numpy']
_with_auxiliary_files = [True, False]
for output_format, with_auxiliary_files in itertools.product(
output_formats, _with_auxiliary_files):
dir_output = tempfile.mkdtemp()
try:
ocgis.env.DIR_OUTPUT = dir_output
ops = ocgis.OcgOperations(dataset=rd,
snippet=True,
output_format=output_format,
geom='state_boundaries',
select_ugid=[23],
prefix=output_format + '_data')
ret_path = ops.execute()
fmtd_ret = format_return(ret_path,
ops,
with_auxiliary_files=with_auxiliary_files)
assert (os.path.exists(fmtd_ret))
if output_format in ['csv', 'nc'
] and with_auxiliary_files is False:
assert (fmtd_ret.endswith(output_format))
else:
assert (is_zipfile(fmtd_ret))
zipf = ZipFile(fmtd_ret, 'r')
try:
namelist = zipf.namelist()
assert (len(namelist) > 0)
if output_format in ['csv+']:
test = [
re.match('shp/.+'.format(output_format), name) !=
None for name in namelist
]
assert (any(test))
else:
test = [
re.match('shp/.+'.format(output_format),
name) == None for name in namelist
]
assert (all(test))
finally:
zipf.close()
## numpy formats are not implemented
except NotImplementedError:
assert (output_format == 'numpy')
finally:
ocgis.env.reset()
shutil.rmtree(dir_output)
def test_subset_years_one_year():
tdata = TestBase.get_tdata()
rd = tdata.get_rd('cancm4_tas')
f,out_nc = tempfile.mkstemp(suffix='_test_nc.nc')
try:
subset(rd.uri,out_nc,1)
ds = nc.Dataset(out_nc,'r')
try:
tvar = ds.variables['time']
dts = nc.num2date(tvar[:],tvar.units,calendar=tvar.calendar)
uyears = np.unique([dt.year for dt in dts.flat])
assert(uyears.shape[0] == 1)
finally:
ds.close()
finally:
os.remove(out_nc)
def test():
tdata = TestBase.get_tdata()
rd = tdata.get_rd('cancm4_tas')
output_formats = [
'csv+',
'csv',
'nc',
'shp',
'numpy']
_with_auxiliary_files = [True,False]
for output_format,with_auxiliary_files in itertools.product(output_formats,_with_auxiliary_files):
dir_output = tempfile.mkdtemp()
try:
ocgis.env.DIR_OUTPUT = dir_output
ops = ocgis.OcgOperations(dataset=rd,snippet=True,output_format=output_format,
geom='state_boundaries',select_ugid=[23],
prefix=output_format+'_data')
ret_path = ops.execute()
fmtd_ret = format_return(ret_path,ops,with_auxiliary_files=with_auxiliary_files)
assert(os.path.exists(fmtd_ret))
if output_format in ['csv','nc'] and with_auxiliary_files is False:
assert(fmtd_ret.endswith(output_format))
else:
assert(is_zipfile(fmtd_ret))
zipf = ZipFile(fmtd_ret,'r')
try:
namelist = zipf.namelist()
assert(len(namelist) > 0)
if output_format in ['csv+']:
test = [re.match('shp/.+'.format(output_format),name) != None for name in namelist]
assert(any(test))
else:
test = [re.match('shp/.+'.format(output_format),name) == None for name in namelist]
assert(all(test))
finally:
zipf.close()
## numpy formats are not implemented
except NotImplementedError:
assert(output_format == 'numpy')
finally:
ocgis.env.reset()
shutil.rmtree(dir_output)
def package(pargs):
## get destination directory
dst = pargs.d or os.getcwd()
if not os.path.exists(dst):
raise (IOError(
'Destination directory does not exist: {0}'.format(dst)))
## import ocgis using relative locations
opath = os.path.join(os.getcwd(), 'src')
sys.path.append(opath)
to_tar = []
if pargs.target in ['shp', 'all']:
import ocgis
shp_dir = ocgis.env.DIR_SHPCABINET
for dirpath, dirnames, filenames in os.walk(shp_dir):
for filename in filenames:
path = os.path.join(dirpath, filename)
arcname = os.path.join('ocgis_data', 'shp',
os.path.split(dirpath)[1], filename)
to_tar.append({'path': path, 'arcname': arcname})
if pargs.target in ['nc', 'all']:
from ocgis.test.base import TestBase
tdata = TestBase.get_tdata()
for key, value in tdata.iteritems():
path = tdata.get_uri(key)
arcname = os.path.join('ocgis_data', 'nc',
tdata.get_relative_path(key))
to_tar.append({'path': path, 'arcname': arcname})
out = os.path.join(os.path.join(dst, 'ocgis_data.tar.gz'))
if pargs.verbose: print('destination file is: {0}'.format(out))
tf = tarfile.open(out, 'w:gz')
try:
for tz in to_tar:
if pargs.verbose and any(
[tz['path'].endswith(ii) for ii in ['shp', 'nc']]):
print('adding: {0}'.format(tz['path']))
tf.add(tz['path'], arcname=tz['arcname'])
finally:
if pargs.verbose: print('closing file...')
tf.close()
if pargs.verbose: print('compression complete.')
def test_subset_years_one_year():
raise(SkipTest('dev'))
tdata = TestBase.get_tdata()
rd = tdata.get_rd('cancm4_tas')
f,out_nc = tempfile.mkstemp(suffix='_test_nc.nc')
try:
subset(rd.uri,out_nc,1)
ds = nc.Dataset(out_nc,'r')
try:
tvar = ds.variables['time']
dts = nc.num2date(tvar[:],tvar.units,calendar=tvar.calendar)
uyears = np.unique([dt.year for dt in dts.flat])
assert(uyears.shape[0] == 1)
finally:
ds.close()
finally:
os.remove(out_nc)
def package(pargs):
## get destination directory
dst = pargs.d or os.getcwd()
if not os.path.exists(dst):
raise(IOError('Destination directory does not exist: {0}'.format(dst)))
## import ocgis using relative locations
opath = os.path.join(os.getcwd(),'src')
sys.path.append(opath)
to_tar = []
if pargs.target in ['shp','all']:
import ocgis
shp_dir = ocgis.env.DIR_SHPCABINET
for dirpath,dirnames,filenames in os.walk(shp_dir):
for filename in filenames:
path = os.path.join(dirpath,filename)
arcname = os.path.join('ocgis_data','shp',os.path.split(dirpath)[1],filename)
to_tar.append({'path':path,'arcname':arcname})
if pargs.target in ['nc','all']:
from ocgis.test.base import TestBase
tdata = TestBase.get_tdata()
for key,value in tdata.iteritems():
path = tdata.get_uri(key)
arcname = os.path.join('ocgis_data','nc',tdata.get_relative_path(key))
to_tar.append({'path':path,'arcname':arcname})
out = os.path.join(os.path.join(dst,'ocgis_data.tar.gz'))
if pargs.verbose: print('destination file is: {0}'.format(out))
tf = tarfile.open(out,'w:gz')
try:
for tz in to_tar:
if pargs.verbose and any([tz['path'].endswith(ii) for ii in ['shp','nc']]):
print('adding: {0}'.format(tz['path']))
tf.add(tz['path'],arcname=tz['arcname'])
finally:
if pargs.verbose: print('closing file...')
tf.close()
if pargs.verbose: print('compression complete.')
from ocgis.test.base import TestBase
from datasets.base import AbstractHarvestDataset, AbstractDataPackage
import datetime
import db
import query
import os
from unittest.case import SkipTest
from sqlalchemy.orm.exc import NoResultFound
import ocgis
from db import get_or_create
from NCPP.util.data_scanner import harvest
from NCPP.util.data_scanner.query import DataQuery
tdata = TestBase.get_tdata()
class CanCM4TestDataset(AbstractHarvestDataset):
uri = [tdata.get_uri('cancm4_tas')]
variables = ['tas']
clean_units = [{'standard_name':'K','long_name':'Kelvin'}]
clean_variable = [dict(standard_name='air_temperature',long_name='Near-Surface Air Temperature',description='Fill it in!')]
dataset_category = dict(name='GCMs',description='Global Circulation Models')
dataset = dict(name='CanCM4',description='Canadian Circulation Model 4')
type = 'variable'
class AbstractMaurerDataset(AbstractHarvestDataset):
dataset = dict(name='Maurer 2010',description='Amazing dataset!')
dataset_category = dict(name='Observational',description='Some observational datasets.')
class MaurerTas(AbstractMaurerDataset):
def values(self):
tdata = TestBase.get_tdata()
return([tdata.get_rd('cancm4_tasmax_2001')])
def setUpClass(cls):
cls.test_data = TestBase.get_tdata()
def setUpClass(cls):
cls.test_data = TestBase.get_tdata()
def values(self):
tdata = TestBase.get_tdata()
return ([tdata.get_rd('cancm4_tasmax_2001')])
