import ocgis
ocgis.env.DIR_DATA = '/usr/local/climate_data'
ocgis.env.OVERWRITE = True
filenames = [
'tasmax_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc',
'tasmax_day_CanCM4_decadal2010_r2i1p1_20110101-20201231.nc'
]
variable = 'tasmax'
select_ugid = [32] # Colorado
rd = ocgis.RequestDataset(uri=filenames, variable=variable)
ops = ocgis.OcgOperations(dataset=rd,
geom='state_boundaries',
select_ugid=select_ugid,
output_format='csv+',
snippet=False)
print(ops.execute())
for output_format in ['csv+', 'shp', 'csv']:
ops = ocgis.OcgOperations(
dataset={
'uri': uri,
'variable': variable,
'time_region': {
'year': None,
'month': [7]
}
},
output_format=output_format,
prefix=output_format,
calc=[{
'name': 'Frequency Duration',
'func': 'freq_duration',
'kwds': {
'threshold': 25.0,
'operation': 'gte'
}
}],
calc_grouping=['month', 'year'],
geom='us_counties',
select_ugid=[2778],
aggregate=True,
calc_raw=False,
spatial_operation='clip',
headers=[
'did', 'ugid', 'gid', 'year', 'month', 'day', 'variable',
'calc_name', 'value'
],
)
ret = ops.execute()
def main():
ocgis.env.OPTIMIZE_FOR_CALC = True
# 215 minutes
uri = '/tmp/gridded_obs.tasmax.OBS_125deg.daily.1950-1999.nc'
# uri = '/usr/local/climate_data/maurer/bcca/obs/tasmax/1_8deg/gridded_obs.tasmax.OBS_125deg.daily.1950.nc'
# uri = '/usr/local/climate_data/daymet/tmax.nc'
variable = 'tasmax'
# variable = 'tmax'
rd = ocgis.RequestDataset(uri, variable)
import netCDF4 as nc
ods = ocgis.api.dataset.dataset.OcgDataset(rd)
shp = ods.i.spatial.shape
print('getting schema...')
schema = tile.get_tile_schema(shp[0], shp[1], 100)
calc = [{
'func': 'mean',
'name': 'my_mean'
}, {
'func': 'freq_perc',
'name': 'perc_90',
'kwds': {
'perc': 90,
}
}, {
'func': 'freq_perc',
'name': 'perc_95',
'kwds': {
'perc': 95,
}
}, {
'func': 'freq_perc',
'name': 'perc_99',
'kwds': {
'perc': 99,
}
}]
print('getting fill file...')
fill_file = ocgis.OcgOperations(dataset=rd,
file_only=True,
calc=calc,
calc_grouping=['month'],
output_format='nc').execute()
print fill_file, len(schema)
fds = nc.Dataset(fill_file, 'a')
t1 = time.time()
for tile_id, indices in schema.iteritems():
print tile_id
row = indices['row']
col = indices['col']
ret = ocgis.OcgOperations(dataset=rd,
slice_row=row,
slice_column=col,
calc=calc,
calc_grouping=['month'],
abstraction='point').execute()
ref = ret[1].variables[variable].calc_value
for k, v in ref.iteritems():
vref = fds.variables[k]
if len(vref.shape) == 3:
vref[:, row[0]:row[1], col[0]:col[1]] = v
elif len(vref.shape) == 4:
vref[:, :, row[0]:row[1], col[0]:col[1]] = v
else:
raise (NotImplementedError)
fds.sync()
fds.close()
print((time.time() - t1) / 60.0)
# TODO:
# - change missing values to 0
# - investigate whether there are more than 2 time steps and if so why
# - use a conservative RegridMethod
# - run domain 4 (barcelona)
#
#
###############################################################################
import ocgis
emi_file = "/home/ryan/sandbox/sysveg/data/HERMES/Emisiones_CAT1_2015091_UAB_4_grid.nc"
# emi_file = "/nfs/pic.es/user/r/rokuingh/sandbox/sysveg/data/HERMES/Emisiones_CAT1_2015091_UAB_4_grid.nc"
geo_file = "/home/ryan/sandbox/sysveg/data/HERMES/geo_em.d03-2.nc"
# geo_file = "/nfs/pic.es/user/r/rokuingh/sandbox/sysveg/data/HERMES/geo_em.d03-2.nc"
# Regrid using bilinear interpolation (i.e. without corners)
rd_in = ocgis.RequestDataset(uri=emi_file, format_time=False)
rd_out = ocgis.RequestDataset(uri=geo_file, format_time=False)
regrid_options = {'regrid_method': ESMF.RegridMethod.BILINEAR, 'split': False}
# import ipdb; ipdb.set_trace()
ops = ocgis.OcgOperations(dataset=rd_in,
regrid_destination=rd_out,
output_format='nc',
regrid_options=regrid_options,
prefix='geo_output')
ret = ops.execute()
def get_operations(self):
rd = self.test_data.get_rd('cancm4_tas')
slc = [None, [0, 100], None, [0, 10], [0, 10]]
ops = ocgis.OcgOperations(dataset=rd, slice=slc)
return ops
def test_consolidating_projections(self):
def assert_projection(path, check_ugid=True):
try:
source = [fiona.open(path, 'r')]
except fiona.errors.DriverError:
shp_path = os.path.split(path)[0]
prefix = os.path.split(shp_path)[1]
shp_path = os.path.join(shp_path, 'shp')
if check_ugid:
ids = ['gid', 'ugid']
else:
ids = ['gid']
source = [
fiona.open(
os.path.join(shp_path, prefix + '_' + suffix + '.shp'))
for suffix in ids
]
try:
for src in source:
self.assertDictEqual(src.meta['crs'], {
u'no_defs': True,
u'datum': u'WGS84',
u'proj': u'longlat'
})
finally:
for src in source:
src.close()
ocgis.env.WRITE_TO_REFERENCE_PROJECTION = True
rd1 = self.test_data.get_rd('narccap_rcm3')
rd1.alias = 'rcm3'
rd2 = self.test_data.get_rd('narccap_crcm')
rd2.alias = 'crcm'
rd = [rd1, rd2]
for output_format in ['csv+', 'shp', 'nc']:
try:
ops = ocgis.OcgOperations(dataset=rd,
snippet=True,
output_format=output_format,
geom='state_boundaries',
agg_selection=False,
select_ugid=[25],
prefix='ca' + output_format)
ret = ops.execute()
## writing to a reference projection is currently not supported for
## netCDF data.
except DefinitionValidationError:
if output_format == 'nc':
continue
else:
raise
assert_projection(ret)
ops = ocgis.OcgOperations(dataset=rd,
snippet=True,
output_format=output_format,
geom='state_boundaries',
agg_selection=True,
prefix='states' + output_format)
ret = ops.execute()
assert_projection(ret)
ops = ocgis.OcgOperations(dataset=rd,
snippet=True,
output_format=output_format,
prefix='rcm3_crcm_domain' +
output_format)
ret = ops.execute()
assert_projection(ret, check_ugid=False)
import os
import ocgis
ocgis.env.DIR_OUTPUT = '/tmp'
ocgis.env.OVERWRITE = True
## write maurer grid
prefix = 'maurer_grid'
uri = '/usr/local/climate_data/maurer/bcca/obs/tasmax/1_8deg/gridded_obs.tasmax.OBS_125deg.daily.1999.nc'
variable = 'tasmax'
rdm = ocgis.RequestDataset(uri, variable)
ops = ocgis.OcgOperations(dataset=rdm,
prefix=prefix,
output_format='shp',
snippet=True)
print(ops.execute())
variable = 'perc_95'
uris = [
os.path.join('/tmp/nctest/row_{0}'.format(ii), 'row_{0}.nc'.format(ii))
for ii in [150, 151]
]
aliases = ['perc_95_{0}'.format(ii) for ii in [150, 151]]
rds = [
ocgis.RequestDataset(uri=uri, variable=variable, alias=alias)
for uri, alias in zip(uris, aliases)
]
ops = ocgis.OcgOperations(dataset=rds, output_format='shp', snippet=True)
ret = ops.execute()
from tempfile import mkdtemp import ocgis ocgis.env.DIR_OUTPUT = mkdtemp() uri = '/usr/local/climate_data/CanCM4/tas_day_CanCM4_decadal2011_r2i1p1_20120101-20211231.nc' variable = 'tas' rd = ocgis.RequestDataset(uri=uri, variable=variable) ops = ocgis.OcgOperations(dataset=rd, geom=[-73.8, 42.7], aggregate=True) ret = ops.execute() # print(ret) field = ret[1]['tas']
import os
import tempfile
import ocgis
from ocgis.test.base import create_gridxy_global, create_exact_field
# Path to the output netCDf file.
PATH = os.path.join(tempfile.gettempdir(), 'ocgis_simple_mean.nc')
# Create a test grid.
grid = create_gridxy_global()
# Create an exact field on the grid.
field = create_exact_field(grid, 'foo')
# Write the field to disk.
field.write(PATH)
# Calculate a monthly mean.
ops = ocgis.OcgOperations(dataset={'uri': PATH},
calc=[{
'func': 'mean',
'name': 'mean'
}],
calc_grouping=['month'])
# Execute the operations. This may be done in parallel with "mpirun".
ret = ops.execute()
# Inspect the data on rank 0 only.
if ocgis.vm.rank == 0:
ocgis.RequestDataset(uri=PATH).inspect()
BBOX = [-109.1, 36.9, -102.0, 41.0]
# Create synthetic data for this example.
grid = create_gridxy_global(resolution=5.0)
field = create_exact_field(grid, VAR_TAS, ntime=31)
data_path = os.path.join(ocgis.env.DIR_OUTPUT,
'ocgis_example_simple_subset.nc')
field.write(data_path)
# This object will be reused so just build it once. Variable names are typically auto-discovered.
rd = ocgis.RequestDataset(data_path, VAR_TAS)
########################################################################################################################
# Returning an OCGIS spatial collection
ret = ocgis.OcgOperations(dataset=rd, geom=BBOX, snippet=SNIPPET).execute()
########################################################################################################################
# Returning conversions
output_formats = ['shp', 'csv', 'csv-shp', 'nc']
for output_format in output_formats:
prefix = output_format + '_output'
ops = ocgis.OcgOperations(dataset=rd,
geom=BBOX,
snippet=SNIPPET,
output_format=output_format,
prefix=prefix)
ret = ops.execute()
print('simple_subset Example Finished')
import itertools
import ocgis
# ocgis.env.DIR_OUTPUT = '/home/local/WX/ben.koziol/Dropbox/nesii/project/ocg/presentation/ncpp-tech-20130611'
ocgis.env.OVERWRITE = True
ocgis.env.DIR_DATA = '/usr/local/climate_data'
ocgis.env.WRITE_TO_REFERENCE_PROJECTION = True
ocgis.env.VERBOSE = True
ocgis.env.DEBUG = True
# write geometry shapefile
geom_write = ocgis.RequestDataset(uri='tas_TMSL_gfdl_1981010100.nc',
variable='tas')
ops = ocgis.OcgOperations(dataset=geom_write,
prefix='tmsl_geometry',
snippet=True,
output_format='shp')
ops.execute()
rd = ocgis.RequestDataset(uri='tas_TMSL_gfdl_1981010100.nc',
variable='tas',
time_region={'month': [6, 7]})
# write subset files
select_ugid = [63, 65]
spatial_operation = ['intersects', 'clip']
aggregate = [False, True]
output_format = ['csv+', 'nc', 'shp', 'csv']
for so, agg, of in itertools.product(spatial_operation, aggregate,
output_format):
if so == 'intersects' and agg is True:
ocgis.env.DIR_OUTPUT = '/home/nils/data/'
ocgis.env.OVERWRITE = True
rd = ocgis.RequestDataset(PATH_NC)
# We do not want to update the coordinate system from WGS84 to Spherical. This causes issues with wrapping for unknown
# reasons. We also need to union the geometries to remove other inconsistencies requiring the unwrapping occur outside
# the operations call.
gc = ocgis.GeomCabinetIterator(path=GEOM_PATH, select_sql_where="ADMIN = 'Russia'", as_field=True)
subset_field = list(gc)[0]
shapely_geom = subset_field.geom.get_value()[0]
wrapper = ocgis.spatial.wrap.GeometryWrapper()
unwrapped = wrapper.unwrap(shapely_geom)
subset_field.geom.get_value()[0] = cascaded_union([g for g in unwrapped])
subset_field.set_crs(rd.crs)
# subset_field.set_crs(ocgis.crs.Spherical())
# Subset the netCDF file and write to disk.
ops = ocgis.OcgOperations(dataset=rd, geom=subset_field, snippet=True, output_format='nc', prefix='ocgis-russia-subset')
ret = ops.execute()
# Convert the subset netCDF file to ESRI shapefile.
ops2 = ocgis.OcgOperations(dataset={'uri': ret}, output_format='shp', vector_wrap=False, prefix='ocgis-as-shp')
ops2.execute()
# Write the original model grid to netCDF file.
ops3 = ocgis.OcgOperations(dataset=rd, output_format='shp', vector_wrap=False, snippet=True, prefix='original-grid')
ops3.execute()
ncfiles)
rds = [
ocgis.RequestDataset(ncfile, variable)
for ncfile, variable in zip(ncfiles, variables)
]
print('building operations...')
select_ugid = [2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010]
geom = 'climate_divisions'
output_format = 'shp'
snippet = True
ops_agg = ocgis.OcgOperations(dataset=rds,
aggregate=True,
geom=geom,
select_ugid=select_ugid,
spatial_operation='clip',
snippet=snippet,
output_format=output_format,
prefix='climdiv_agg')
ops_sel = ocgis.OcgOperations(dataset=rds,
aggregate=False,
geom=geom,
select_ugid=select_ugid,
spatial_operation='intersects',
snippet=snippet,
output_format=output_format,
prefix='climdiv_sel')
print('executing aggregation...')
path_agg = ops_agg.execute()
AGGREGATE = False # True
SPATIAL_OPERATION = 'intersects' # 'clip'
GEOM = 'state_boundaries'
STATES = {'CO': [32], 'CA': [25]}
OUTPUT_FORMAT = 'csv+' # 'csv' #'nc' #'shp'
PREFIX = 'ocgis_output'
TIME_REGION = {'month': [6, 7], 'year': [2011]}
################################################################################
## construct the request dataset. time subsetting is also parameterized with
## this object.
rd = ocgis.RequestDataset(uri=os.path.join(DIR_DATA, FILENAME),
variable=VARIABLE,
time_range=None,
time_region=TIME_REGION)
## construct the operations call
ops = ocgis.OcgOperations(dataset=rd,
geom=GEOM,
select_ugid=STATES['CO'],
aggregate=AGGREGATE,
spatial_operation=SPATIAL_OPERATION,
prefix=PREFIX,
output_format=OUTPUT_FORMAT)
## return the path to the folder containing the output data
path = ops.execute()
print(path)
def test():
ocgis.env.OVERWRITE = True
rd = ocgis.RequestDataset(uri=uri, variable=variable)
ops = ocgis.OcgOperations(dataset=rd, snippet=True, output_format='nc')
ops.execute()
#ocgis.env.DIR_BIN = '/home/local/WX/ben.koziol/links/ocgis/bin/QED_2013_dynamic_percentiles'
ocgis.env.DIR_DATA = '/data/maurer'
percentiles = [90,92.5,95,97.5]
operations = [
# 'gt',
'gte',
# 'lt',
# 'lte'
]
calc_groupings = [
['month'],
# ['month','year'],
# ['year']
]
uris_variables = [['Maurer02new_OBS_tasmax_daily.1971-2000.nc','tasmax'],
['Maurer02new_OBS_tasmin_daily.1971-2000.nc','tasmin']]
geoms_select_ugids = [
['qed_city_centroids',None],
# ['state_boundaries',[39]],
# ['us_counties',[2416,1335]]
]
for tup in itertools.product(percentiles,operations,calc_groupings,uris_variables,geoms_select_ugids):
print(tup)
percentile,operation,calc_grouping,uri_variable,geom_select_ugid = tup
ops = ocgis.OcgOperations(dataset={'uri':uri_variable[0],'variable':uri_variable[1],'time_region':{'year':[1995,1996]}},
geom=geom_select_ugid[0],select_ugid=geom_select_ugid[1],
calc=[{'func':'qed_dynamic_percentile','kwds':{'operation':operation,'percentile':percentile},'name':'dp'}],
calc_grouping=calc_grouping,output_format='numpy')
ret = ops.execute()
def compute(dataset,
calc,
calc_grouping,
tile_dimension,
verbose=False,
prefix=None):
'''
:type dataset: RequestDatasetCollection
'''
assert (isinstance(dataset, RequestDatasetCollection))
assert (type(calc) in (list, tuple))
tile_dimension = int(tile_dimension)
if tile_dimension <= 0:
raise (ValueError('"tile_dimension" must be greater than 0'))
orig_oc = ocgis.env.OPTIMIZE_FOR_CALC
ocgis.env.OPTIMIZE_FOR_CALC = False
try:
## load some data into the optimize store
print('loading into optimize store...')
for rd in dataset:
if verbose: print('request dataset', rd.alias)
ocgis.env._optimize_store[rd.alias] = {}
ocgis.env._optimize_store[
rd.alias]['_value_datetime'] = rd.ds.temporal.value_datetime
ocgis.env._optimize_store[
rd.alias]['_bounds_datetime'] = rd.ds.temporal.bounds_datetime
if calc_grouping is not None:
rd.ds.temporal.set_grouping(calc_grouping)
ocgis.env._optimize_store[
rd.alias]['group'] = rd.ds.temporal.group
rd._ds = None
## tell the software we are optimizing for calculations
ocgis.env.OPTIMIZE_FOR_CALC = True
ods = NcDataset(request_dataset=dataset[0])
shp = ods.spatial.grid.shape
if verbose: print('getting schema...')
schema = tile.get_tile_schema(shp[0], shp[1], tile_dimension)
if verbose: print('getting fill file...')
fill_file = ocgis.OcgOperations(dataset=dataset,
file_only=True,
calc=calc,
calc_grouping=calc_grouping,
output_format='nc',
prefix=prefix).execute()
if verbose: print('output file is: {0}'.format(fill_file))
if verbose:
lschema = len(schema)
print('tile count: {0}'.format(lschema))
fds = nc.Dataset(fill_file, 'a')
if verbose:
progress = ProgressBar('tiles progress')
for ctr, indices in enumerate(schema.itervalues(), start=1):
row = indices['row']
col = indices['col']
ret = ocgis.OcgOperations(dataset=dataset,
slice=[None, row, col],
calc=calc,
calc_grouping=calc_grouping).execute()
for vref, v in iter_variable_values(ret[1], fds):
if len(vref.shape) == 3:
vref[:, row[0]:row[1], col[0]:col[1]] = v
elif len(vref.shape) == 4:
vref[:, :, row[0]:row[1], col[0]:col[1]] = v
else:
raise (NotImplementedError(vref.shape))
fds.sync()
if verbose:
progress.progress(int((float(ctr) / lschema) * 100))
fds.close()
finally:
ocgis.env.OPTIMIZE_FOR_CALC = orig_oc
ocgis.env._optimize_store = {}
if verbose:
progress.endProgress()
print('complete.')
return (fill_file)
def test(self):
tdata = TestBase.get_tst_data()
rd = tdata.get_rd('cancm4_tas')
output_formats = [
constants.OutputFormatName.CSV_SHAPEFILE,
constants.OutputFormatName.CSV, constants.OutputFormatName.NETCDF,
constants.OutputFormatName.SHAPEFILE,
constants.OutputFormatName.OCGIS
]
_with_auxiliary_files = [True, False]
for output_format, with_auxiliary_files in itertools.product(
output_formats, _with_auxiliary_files):
dir_output = tempfile.mkdtemp()
try:
ocgis.env.DIR_OUTPUT = dir_output
ops = ocgis.OcgOperations(dataset=rd,
snippet=True,
output_format=output_format,
geom='state_boundaries',
select_ugid=[23],
prefix=output_format + '_data')
ret_path = ops.execute()
fmtd_ret = format_return(
ret_path, ops, with_auxiliary_files=with_auxiliary_files)
assert (os.path.exists(fmtd_ret))
if output_format in [
constants.OutputFormatName.CSV,
constants.OutputFormatName.NETCDF
] and with_auxiliary_files is False:
assert (fmtd_ret.endswith(output_format))
else:
assert (is_zipfile(fmtd_ret))
zipf = ZipFile(fmtd_ret, 'r')
try:
namelist = zipf.namelist()
assert (len(namelist) > 0)
if output_format in [
constants.OutputFormatName.CSV_SHAPEFILE
]:
test = [
re.match('shp/.+'.format(output_format), name)
!= None for name in namelist
]
assert (any(test))
else:
test = [
re.match('shp/.+'.format(output_format),
name) == None for name in namelist
]
assert (all(test))
finally:
zipf.close()
# numpy formats are not implemented
except NotImplementedError:
assert (output_format == constants.OutputFormatName.OCGIS)
finally:
ocgis.env.reset()
shutil.rmtree(dir_output)
def test_time_region_climatology(self):
ocgis.env.DIR_DATA = '/usr/local/climate_data'
uri = 'climatology_TNn_monthly_max.nc'
variable = 'climatology_TNn_monthly_max'
rd = ocgis.RequestDataset(uri,
variable,
time_region={
'year': [1989],
'month': [6]
})
ops = ocgis.OcgOperations(dataset=rd,
geom='state_boundaries',
select_ugid=[16])
ret = ops.execute()
ref = ret[16].variables['climatology_TNn_monthly_max']
self.assertEqual(set([6]),
set([dt.month for dt in ref.temporal.value_datetime]))
uri = 'climatology_TNn_monthly_max.nc'
variable = 'climatology_TNn_monthly_max'
rd = ocgis.RequestDataset(uri,
variable,
time_region={
'year': None,
'month': [6]
})
ops = ocgis.OcgOperations(dataset=rd,
geom='state_boundaries',
select_ugid=[16])
ret = ops.execute()
ref = ret[16].variables['climatology_TNn_monthly_max']
self.assertEqual(set([6]),
set([dt.month for dt in ref.temporal.value_datetime]))
rd = ocgis.RequestDataset('climatology_TNn_annual_min.nc',
'climatology_TNn_annual_min')
ops = ocgis.OcgOperations(dataset=rd,
geom='state_boundaries',
select_ugid=[16])
ret = ops.execute()
ref = ret[16].variables['climatology_TNn_annual_min']
rd = ocgis.RequestDataset(
'climatology_TasMin_seasonal_max_of_seasonal_means.nc',
'climatology_TasMin_seasonal_max_of_seasonal_means'
) #,time_region={'year':[1989]})
ops = ocgis.OcgOperations(dataset=rd,
geom='state_boundaries',
select_ugid=[16])
ret = ops.execute()
ref = ret[16].variables[
'climatology_TasMin_seasonal_max_of_seasonal_means']
uri = 'climatology_Tas_annual_max_of_annual_means.nc'
variable = 'climatology_Tas_annual_max_of_annual_means'
rd = ocgis.RequestDataset(uri, variable)
ops = ocgis.OcgOperations(dataset=rd,
geom='state_boundaries',
select_ugid=[16])
ret = ops.execute()
ref = ret[16].variables[variable]
def get_subset_operation(self):
geom = get_geometry_dictionaries()
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd, geom=geom, select_nearest=True)
subset = OperationsEngine(ops)
return subset
def test_seasonal_calc(self):
"""Test some calculations using a seasonal grouping."""
calc = [{
'func': 'mean',
'name': 'my_mean'
}, {
'func': 'std',
'name': 'my_std'
}]
calc_grouping = [[3, 4, 5]]
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd,
calc=calc,
calc_grouping=calc_grouping,
calc_sample_size=True,
geom='state_boundaries',
select_ugid=[23])
ret = ops.execute()
self.assertEqual(
ret.get_element(
variable_name='n_my_std').get_masked_value().mean(), 920.0)
self.assertEqual(
ret.get_element(variable_name='my_std').shape, (1, 3, 3))
calc = [{
'func': 'mean',
'name': 'my_mean'
}, {
'func': 'std',
'name': 'my_std'
}]
calc_grouping = [[12, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd,
calc=calc,
calc_grouping=calc_grouping,
calc_sample_size=True,
geom='state_boundaries',
select_ugid=[23])
ret = ops.execute()
self.assertEqual(
ret.get_element(variable_name='my_std').shape, (4, 3, 3))
temporal = ret.get_element().temporal
numtime = temporal.value_numtime.data
numtime_actual = np.array([56955.0, 56680.0, 56771.0, 56863.0])
self.assertNumpyAll(numtime, numtime_actual)
calc = [{
'func': 'mean',
'name': 'my_mean'
}, {
'func': 'std',
'name': 'my_std'
}]
calc_grouping = [[12, 1], [2, 3]]
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd,
calc=calc,
calc_grouping=calc_grouping,
calc_sample_size=True,
geom='state_boundaries',
select_ugid=[23])
ret = ops.execute()
self.assertEqual(
ret.get_element(variable_name='my_std').shape, (2, 3, 3))
temporal = ret.get_element().temporal
bounds_numtime = temporal.bounds.value_numtime.data
bounds_numtime_actual = np.array([[55115.0, 58765.0],
[55146.0, 58490.0]])
self.assertNumpyAll(bounds_numtime, bounds_numtime_actual)
rds = [ocgis.RequestDataset(ncfiles[i], variable_name)]
print('operating on '), ncfiles[i]
# get the file name (minus the '.nc'
# to later pass to the folder prefix of where the data are saved
filename = ncfiles[i]
savename = filename[0:len(filename) - 3]
#define the geometry
geom = 'climate_divisions'
#define the file output format
output_format = 'csv'
#if True, only perform operation on first time increment
snippet = False
# climate division ugid (identifyers) for MN,WI,IL,IN,MI,OH,PA,NY
for select_ugid in cdList:
#save the output file as the division ID + Filename
prefix = str(select_ugid) + str('_') + str(savename)
print 'working on division ', prefix
try:
ops_agg = ocgis.OcgOperations(dataset=rds,aggregate=True,geom=geom,\
select_ugid=[select_ugid],spatial_operation='clip',snippet=snippet,\
output_format=output_format,prefix=prefix,interpolate_spatial_bounds=True)
print('executing aggregation...')
path_agg = ops_agg.execute()
#catch extent errors
except ExtentError:
print 'error on ', select_ugid
print('success.')
crcm = ocgis.RequestDataset(uri='pr_CRCM_ccsm_1981010103.nc',variable='pr',alias='crcm')
wrfg = ocgis.RequestDataset(uri='pr_WRFG_ncep_1981010103.nc',variable='pr',alias='wrfg')
logging.info('generating destination grid')
dest_x,dest_y = np.meshgrid(np.arange(0,360),np.arange(-90,90))
#ops = ocgis.OcgOperations(dataset=rd,snippet=True,prefix='overview',output_format='shp')
#ret = ops.execute()
#ops = ocgis.OcgOperations(dataset=rd,snippet=True,prefix='output',output_format='shp',
# select_ugid=[1],geom='state_boundaries')
#ret = ops.execute()
logging.info('retrieving source grids')
ops = ocgis.OcgOperations(dataset=[crcm,wrfg],snippet=True,prefix='output',
output_format='numpy',geom='state_boundaries',agg_selection=True)
ret = ops.execute()
logging.info('projecting source grids')
for key in ['crcm','wrfg']:
ret[1].variables[key].project(WGS84())
import ipdb;ipdb.set_trace()
tas = ret[1].variables['tas']
dims = tas.spatial.grid.shape
row = tas.spatial.grid.row.value
row_bounds = tas.spatial.grid.row.bounds
col = tas.spatial.grid.column.value
col_bounds = tas.spatial.grid.column.bounds
