def test(): tdata = TestBase.get_tdata() rd = tdata.get_rd('cancm4_tas') output_formats = ['csv+', 'csv', 'nc', 'shp', 'numpy'] _with_auxiliary_files = [True, False] for output_format, with_auxiliary_files in itertools.product( output_formats, _with_auxiliary_files): dir_output = tempfile.mkdtemp() try: ocgis.env.DIR_OUTPUT = dir_output ops = ocgis.OcgOperations(dataset=rd, snippet=True, output_format=output_format, geom='state_boundaries', select_ugid=[23], prefix=output_format + '_data') ret_path = ops.execute() fmtd_ret = format_return(ret_path, ops, with_auxiliary_files=with_auxiliary_files) assert (os.path.exists(fmtd_ret)) if output_format in ['csv', 'nc' ] and with_auxiliary_files is False: assert (fmtd_ret.endswith(output_format)) else: assert (is_zipfile(fmtd_ret)) zipf = ZipFile(fmtd_ret, 'r') try: namelist = zipf.namelist() assert (len(namelist) > 0) if output_format in ['csv+']: test = [ re.match('shp/.+'.format(output_format), name) != None for name in namelist ] assert (any(test)) else: test = [ re.match('shp/.+'.format(output_format), name) == None for name in namelist ] assert (all(test)) finally: zipf.close() ## numpy formats are not implemented except NotImplementedError: assert (output_format == 'numpy') finally: ocgis.env.reset() shutil.rmtree(dir_output)
def test_subset_years_one_year(): tdata = TestBase.get_tdata() rd = tdata.get_rd('cancm4_tas') f,out_nc = tempfile.mkstemp(suffix='_test_nc.nc') try: subset(rd.uri,out_nc,1) ds = nc.Dataset(out_nc,'r') try: tvar = ds.variables['time'] dts = nc.num2date(tvar[:],tvar.units,calendar=tvar.calendar) uyears = np.unique([dt.year for dt in dts.flat]) assert(uyears.shape[0] == 1) finally: ds.close() finally: os.remove(out_nc)
def test(): tdata = TestBase.get_tdata() rd = tdata.get_rd('cancm4_tas') output_formats = [ 'csv+', 'csv', 'nc', 'shp', 'numpy'] _with_auxiliary_files = [True,False] for output_format,with_auxiliary_files in itertools.product(output_formats,_with_auxiliary_files): dir_output = tempfile.mkdtemp() try: ocgis.env.DIR_OUTPUT = dir_output ops = ocgis.OcgOperations(dataset=rd,snippet=True,output_format=output_format, geom='state_boundaries',select_ugid=[23], prefix=output_format+'_data') ret_path = ops.execute() fmtd_ret = format_return(ret_path,ops,with_auxiliary_files=with_auxiliary_files) assert(os.path.exists(fmtd_ret)) if output_format in ['csv','nc'] and with_auxiliary_files is False: assert(fmtd_ret.endswith(output_format)) else: assert(is_zipfile(fmtd_ret)) zipf = ZipFile(fmtd_ret,'r') try: namelist = zipf.namelist() assert(len(namelist) > 0) if output_format in ['csv+']: test = [re.match('shp/.+'.format(output_format),name) != None for name in namelist] assert(any(test)) else: test = [re.match('shp/.+'.format(output_format),name) == None for name in namelist] assert(all(test)) finally: zipf.close() ## numpy formats are not implemented except NotImplementedError: assert(output_format == 'numpy') finally: ocgis.env.reset() shutil.rmtree(dir_output)
def package(pargs): ## get destination directory dst = pargs.d or os.getcwd() if not os.path.exists(dst): raise (IOError( 'Destination directory does not exist: {0}'.format(dst))) ## import ocgis using relative locations opath = os.path.join(os.getcwd(), 'src') sys.path.append(opath) to_tar = [] if pargs.target in ['shp', 'all']: import ocgis shp_dir = ocgis.env.DIR_SHPCABINET for dirpath, dirnames, filenames in os.walk(shp_dir): for filename in filenames: path = os.path.join(dirpath, filename) arcname = os.path.join('ocgis_data', 'shp', os.path.split(dirpath)[1], filename) to_tar.append({'path': path, 'arcname': arcname}) if pargs.target in ['nc', 'all']: from ocgis.test.base import TestBase tdata = TestBase.get_tdata() for key, value in tdata.iteritems(): path = tdata.get_uri(key) arcname = os.path.join('ocgis_data', 'nc', tdata.get_relative_path(key)) to_tar.append({'path': path, 'arcname': arcname}) out = os.path.join(os.path.join(dst, 'ocgis_data.tar.gz')) if pargs.verbose: print('destination file is: {0}'.format(out)) tf = tarfile.open(out, 'w:gz') try: for tz in to_tar: if pargs.verbose and any( [tz['path'].endswith(ii) for ii in ['shp', 'nc']]): print('adding: {0}'.format(tz['path'])) tf.add(tz['path'], arcname=tz['arcname']) finally: if pargs.verbose: print('closing file...') tf.close() if pargs.verbose: print('compression complete.')
def test_subset_years_one_year(): raise(SkipTest('dev')) tdata = TestBase.get_tdata() rd = tdata.get_rd('cancm4_tas') f,out_nc = tempfile.mkstemp(suffix='_test_nc.nc') try: subset(rd.uri,out_nc,1) ds = nc.Dataset(out_nc,'r') try: tvar = ds.variables['time'] dts = nc.num2date(tvar[:],tvar.units,calendar=tvar.calendar) uyears = np.unique([dt.year for dt in dts.flat]) assert(uyears.shape[0] == 1) finally: ds.close() finally: os.remove(out_nc)
def package(pargs): ## get destination directory dst = pargs.d or os.getcwd() if not os.path.exists(dst): raise(IOError('Destination directory does not exist: {0}'.format(dst))) ## import ocgis using relative locations opath = os.path.join(os.getcwd(),'src') sys.path.append(opath) to_tar = [] if pargs.target in ['shp','all']: import ocgis shp_dir = ocgis.env.DIR_SHPCABINET for dirpath,dirnames,filenames in os.walk(shp_dir): for filename in filenames: path = os.path.join(dirpath,filename) arcname = os.path.join('ocgis_data','shp',os.path.split(dirpath)[1],filename) to_tar.append({'path':path,'arcname':arcname}) if pargs.target in ['nc','all']: from ocgis.test.base import TestBase tdata = TestBase.get_tdata() for key,value in tdata.iteritems(): path = tdata.get_uri(key) arcname = os.path.join('ocgis_data','nc',tdata.get_relative_path(key)) to_tar.append({'path':path,'arcname':arcname}) out = os.path.join(os.path.join(dst,'ocgis_data.tar.gz')) if pargs.verbose: print('destination file is: {0}'.format(out)) tf = tarfile.open(out,'w:gz') try: for tz in to_tar: if pargs.verbose and any([tz['path'].endswith(ii) for ii in ['shp','nc']]): print('adding: {0}'.format(tz['path'])) tf.add(tz['path'],arcname=tz['arcname']) finally: if pargs.verbose: print('closing file...') tf.close() if pargs.verbose: print('compression complete.')
from ocgis.test.base import TestBase from datasets.base import AbstractHarvestDataset, AbstractDataPackage import datetime import db import query import os from unittest.case import SkipTest from sqlalchemy.orm.exc import NoResultFound import ocgis from db import get_or_create from NCPP.util.data_scanner import harvest from NCPP.util.data_scanner.query import DataQuery tdata = TestBase.get_tdata() class CanCM4TestDataset(AbstractHarvestDataset): uri = [tdata.get_uri('cancm4_tas')] variables = ['tas'] clean_units = [{'standard_name':'K','long_name':'Kelvin'}] clean_variable = [dict(standard_name='air_temperature',long_name='Near-Surface Air Temperature',description='Fill it in!')] dataset_category = dict(name='GCMs',description='Global Circulation Models') dataset = dict(name='CanCM4',description='Canadian Circulation Model 4') type = 'variable' class AbstractMaurerDataset(AbstractHarvestDataset): dataset = dict(name='Maurer 2010',description='Amazing dataset!') dataset_category = dict(name='Observational',description='Some observational datasets.') class MaurerTas(AbstractMaurerDataset):
def values(self): tdata = TestBase.get_tdata() return([tdata.get_rd('cancm4_tasmax_2001')])
def setUpClass(cls): cls.test_data = TestBase.get_tdata()
def values(self): tdata = TestBase.get_tdata() return ([tdata.get_rd('cancm4_tasmax_2001')])