def make_weight_files():
n_jobs = 16
to_process = []
log.level = logbook.INFO
for ctr, l in enumerate(listdir(WD)):
# if ctr > 20: break
if l.endswith('.nc'):
path_out_weights_nc = join(ESMF_WEIGHTS_OUTPUT_DIR, 'weights_' + l)
esmf_format = join(WD, l)
log.debug(path_out_weights_nc)
log.debug(esmf_format)
search = re.search('esmf_single_element_(.+)_(.+).nc', l)
gridcode, node_count = search.group(1), search.group(2)
kwds = {'esmf_format': esmf_format,
'path_out_weights_nc': path_out_weights_nc,
'gridcode': int(gridcode),
'node_count': int(node_count)}
to_process.append(kwds)
log.debug(kwds)
rtimes = Parallel(n_jobs=n_jobs)(delayed(make_weight_file)(**k) for k in to_process)
for idx, k in enumerate(to_process):
k['time'] = rtimes[idx]
log.info(to_process)
def convert_shapefiles(pred=None, node_threshold=None):
# Run with MPI: mpirun -n 8 prep_shapefiles.py
nfie_version = 'node-threshold-10000'
name_uid = 'GRIDCODE'
storage_dir_shp = os.path.expanduser(
'/media/benkoziol/Extra Drive 1/data/nfie/linked_catchment_shapefiles')
storage_dir_esmf = os.path.expanduser(
'/media/benkoziol/Extra Drive 1/data/nfie/node-thresholded-10000')
esmf_name_template = 'esmf_format_{cid}_{nfie_version}.nc'
log.info('Starting conversion for: {}'.format(nfie_version))
for dirpath, dirnames, filenames in os.walk(
os.path.join(storage_dir_shp, storage_dir_shp)):
for fn in filenames:
if fn.endswith('.shp'):
log.debug(fn)
if pred is not None and not pred(fn):
continue
cid = re.search('linked_(.*).shp', fn).group(1)
path_catchment_shp = os.path.join(dirpath, fn)
esmf_name = esmf_name_template.format(
cid=cid, nfie_version=nfie_version)
path_esmf_format_nc = os.path.join(storage_dir_esmf, esmf_name)
log.info('Converting {}'.format(path_catchment_shp))
# log.debug((path_esmf_format_nc, path_catchment_shp, name_uid))
try:
convert_to_esmf_format(path_esmf_format_nc,
path_catchment_shp,
name_uid,
node_threshold=node_threshold)
except:
log.exception(path_catchment_shp)
def run_create_linked_shapefile():
log.level = INFO
name_uid = 'GRIDCODE'
output_variable = 'pr'
directory_shapefiles = expanduser('~/storage/catchment_shapefiles')
directory_linked_shapefiles = expanduser('~/storage/linked_catchment_shapefiles')
directory_weighted_data = expanduser('~/storage/catchment_weighted_data')
if MPI_RANK == 0:
weighted_data_files = filter(lambda x: x.startswith('pr_weighted'), listdir(directory_weighted_data))
weighted_data_files = [weighted_data_files[slc[0]: slc[1]] for slc in create_sections(len(weighted_data_files))]
else:
weighted_data_files = None
weighted_data_files = MPI_COMM.scatter(weighted_data_files, root=0)
for ii in weighted_data_files:
path_output_data = join(directory_weighted_data, ii)
# log.debug(ii)
res = re.search('pr_weighted-catchments_esmf_(.*)_', ii).groups()[0]
# log.debug(res.groups()[0])
shapefile_directory = join(directory_shapefiles, res)
path_in_shp = check_output(['find', shapefile_directory, '-name', '*Catchment.shp']).strip()
path_linked_shp = join(directory_linked_shapefiles, 'linked_{}.shp'.format(res))
log.debug((path_in_shp, path_linked_shp))
log.info('Creating linked shapefile for: {}'.format(path_output_data))
create_linked_shapefile(name_uid, output_variable, path_in_shp, path_linked_shp, path_output_data)
def run_create_linked_shapefile():
log.level = INFO
name_uid = 'GRIDCODE'
output_variable = 'pr'
directory_shapefiles = expanduser('~/storage/catchment_shapefiles')
directory_linked_shapefiles = expanduser(
'~/storage/linked_catchment_shapefiles')
directory_weighted_data = expanduser('~/storage/catchment_weighted_data')
if MPI_RANK == 0:
weighted_data_files = filter(lambda x: x.startswith('pr_weighted'),
listdir(directory_weighted_data))
weighted_data_files = [
weighted_data_files[slc[0]:slc[1]]
for slc in create_sections(len(weighted_data_files))
]
else:
weighted_data_files = None
weighted_data_files = MPI_COMM.scatter(weighted_data_files, root=0)
for ii in weighted_data_files:
path_output_data = join(directory_weighted_data, ii)
# log.debug(ii)
res = re.search('pr_weighted-catchments_esmf_(.*)_', ii).groups()[0]
# log.debug(res.groups()[0])
shapefile_directory = join(directory_shapefiles, res)
path_in_shp = check_output(
['find', shapefile_directory, '-name', '*Catchment.shp']).strip()
path_linked_shp = join(directory_linked_shapefiles,
'linked_{}.shp'.format(res))
log.debug((path_in_shp, path_linked_shp))
log.info('Creating linked shapefile for: {}'.format(path_output_data))
create_linked_shapefile(name_uid, output_variable, path_in_shp,
path_linked_shp, path_output_data)
def test(self):
log.info('test hello world')
try:
raise RuntimeError('test runtime error')
except:
log.exception('test log exception')
log.info('test passed')
def create_merged_weights(weight_files, esmf_unstructured, master_weights):
"""
Create a merged weight file containing some variables from the original ESMF mesh files.
:param weight_files: sequence of file paths to the ESMF weight files
:param esmf_unstructured: sequence of file paths to the ESMF unstructured files (This sequence must be in the same
order as ``weight_files``. The indexed weight file must have been created from the indexed ESMF unstructured.)
:param master_weights: file path to the merged, master weights file
"""
master_map = {}
current_global_index = 1
new_weight_file = Dict({'row': [], 'col': [], 'S': [], 'GRIDCODE': [], 'centerCoords': np.empty((0, 2))})
new_dimensions = Dict({'row': 'n_s', 'col': 'n_s', 'S': 'n_s', 'GRIDCODE': 'elementCount',
'centerCoords': ('elementCount', 'coordDim')})
new_dtype = {'row': np.int32, 'col': np.int32, 'S': np.float64}
for uid, (w, e) in enumerate(zip(weight_files, esmf_unstructured)):
log.info('Merge is processing weight file: {}'.format(w))
log.info('Merge is processing ESMF unstructured file: {}'.format(e))
w = RequestDataset(w).get()
e = RequestDataset(e).get()
for row_value in w['row'].value.flat:
if row_value < 0:
raise ValueError('"row" value must be greater than or equal to 1 to qualify as a Fortran index.')
if (uid, row_value) not in master_map:
master_map[(uid, row_value)] = current_global_index
current_global_index += 1
new_weight_file.row.append(master_map[(uid, row_value)])
new_weight_file.col += w['col'].value.tolist()
# new_weight_file.row += w['row'].value.tolist()
new_weight_file.S += w['S'].value.tolist()
new_weight_file.GRIDCODE += e['GRIDCODE'].value.tolist()
new_weight_file.centerCoords = np.vstack((new_weight_file.centerCoords, e['centerCoords'].value))
vc = VariableCollection()
for k, v in new_weight_file.items():
new_var = Variable(name=k, value=v, dimensions=new_dimensions[k], dtype=new_dtype.get(k))
vc.add_variable(new_var)
assert np.all(vc['row'].value >= 1)
vc.attrs['coordDim'] = "longitude latitude"
vc.attrs['description'] = "Merged ESMF weights file with auxiliary variables."
vc['GRIDCODE'].attrs['long_name'] = 'Element unique identifier.'
vc['centerCoords'].units = 'degrees'
vc['row'].attrs['long_name'] = 'ESMF index to destination array.'
vc['col'].attrs['long_name'] = 'ESMF index to source array.'
vc['S'].attrs['long_name'] = 'ESMF weight factor.'
vc.write(master_weights)
def make_weight_file(**kwargs):
esmf_format = kwargs['esmf_format']
path_out_weights_nc = kwargs['path_out_weights_nc']
gridcode = kwargs['gridcode']
node_count = kwargs['node_count']
log.info('Starting weight generation: {}, (gridcode={}, node_count={})'.format(esmf_format, gridcode, node_count))
t1 = time.time()
create_weights_file(None, ESMF_EXE_PATH, PATH_SOURCE_DATA, esmf_format, path_out_weights_nc)
t2 = time.time()
log.info('Finished weight generation: {}, (gridcode={}, node_count={})'.format(esmf_format, gridcode, node_count))
return t2 - t1
def fill_database():
s = Session()
# VectorProcessingUnit, Shapefile ##################################################################################
for d in os.listdir(SHAPEFILE_DIRECTORY):
vpu = VectorProcessingUnit(name=d)
s.add(vpu)
s.add(Shapefile.create(vpu, join(SHAPEFILE_DIRECTORY, d)))
# Catchment ########################################################################################################
to_crs = CoordinateReferenceSystem(epsg=3083)
for shapefile in s.query(Shapefile):
log.info('Loading shapefile: {}'.format(shapefile.vpu.name))
gi = GeomCabinetIterator(path=shapefile.fullpath)
for record in gi:
catchment = Catchment.create(shapefile.vpu, record, to_crs)
s.add(catchment)
s.commit()
s.commit()
def fill_database():
s = Session()
# VectorProcessingUnit, Shapefile ##################################################################################
for d in os.listdir(SHAPEFILE_DIRECTORY):
vpu = VectorProcessingUnit(name=d)
s.add(vpu)
s.add(Shapefile.create(vpu, join(SHAPEFILE_DIRECTORY, d)))
# Catchment ########################################################################################################
to_crs = CoordinateReferenceSystem(epsg=3083)
for shapefile in s.query(Shapefile):
log.info('Loading shapefile: {}'.format(shapefile.vpu.name))
gi = GeomCabinetIterator(path=shapefile.fullpath)
for record in gi:
catchment = Catchment.create(shapefile.vpu, record, to_crs)
s.add(catchment)
s.commit()
s.commit()
else:
for idx in range(var_value.shape[1]):
var_value[:, idx, :, :, :] = var_value[:, idx, :, :, :] * idx
variable = ocgis.Variable(value=var_value, name=variable_name)
field = ocgis.Field(spatial=sdim, temporal=temporal, variables=variable)
return field
def get_exact_field(lat_rad, lon_rad):
fill = 2 + np.cos(lat_rad) ** 2 + np.cos(2 * lon_rad)
return fill
if __name__ == '__main__':
ocgis.env.DIR_OUTPUT = '/tmp'
ocgis.env.OVERWRITE = True
path = '/tmp/analytic-conus_20160316-1737.nc'
create_source_netcdf_data(path, exact=True)
ops = ocgis.OcgOperations(dataset={'uri': path}, snippet=True, output_format='shp', prefix='analytic-conus')
# Create analytical data for Texas catchments.
# row = np.arange(32.0012, 32.4288 + 0.02, 0.01)
# col = np.arange(-95.0477, -94.7965 + 0.02, 0.01)
# path = '/tmp/analytic_20160316-1627.nc'
# create_source_netcdf_data(path, row=row, col=col, analytic=True)
# ops = ocgis.OcgOperations(dataset={'uri': path}, snippet=True, output_format='shp', prefix='analytic')
log.info(ops.execute())
def create_merged_weights(weight_files, esmf_unstructured, master_weights):
"""
Create a merged weight file containing some variables from the original ESMF mesh files.
:param weight_files: sequence of file paths to the ESMF weight files
:param esmf_unstructured: sequence of file paths to the ESMF unstructured files (This sequence must be in the same
order as ``weight_files``. The indexed weight file must have been created from the indexed ESMF unstructured.)
:param master_weights: file path to the merged, master weights file
"""
from ocgis import RequestDataset
from ocgis.new_interface.variable import Variable, VariableCollection
master_map = {}
current_global_index = 1
new_weight_file = Dict({
'row': [],
'col': [],
'S': [],
'GRIDCODE': [],
'centerCoords': np.empty((0, 2))
})
new_dimensions = Dict({
'row': 'n_s',
'col': 'n_s',
'S': 'n_s',
'GRIDCODE': 'elementCount',
'centerCoords': ('elementCount', 'coordDim')
})
new_dtype = {'row': np.int32, 'col': np.int32, 'S': np.float64}
for uid, (w, e) in enumerate(zip(weight_files, esmf_unstructured)):
log.info('Merge is processing weight file: {}'.format(w))
log.info('Merge is processing ESMF unstructured file: {}'.format(e))
w = RequestDataset(w).get()
e = RequestDataset(e).get()
for row_value in w['row'].value.flat:
if row_value < 0:
raise ValueError(
'"row" value must be greater than or equal to 1 to qualify as a Fortran index.'
)
if (uid, row_value) not in master_map:
master_map[(uid, row_value)] = current_global_index
current_global_index += 1
new_weight_file.row.append(master_map[(uid, row_value)])
new_weight_file.col += w['col'].value.tolist()
# new_weight_file.row += w['row'].value.tolist()
new_weight_file.S += w['S'].value.tolist()
new_weight_file.GRIDCODE += e['GRIDCODE'].value.tolist()
new_weight_file.centerCoords = np.vstack(
(new_weight_file.centerCoords, e['centerCoords'].value))
vc = VariableCollection()
for k, v in new_weight_file.items():
new_var = Variable(name=k,
value=v,
dimensions=new_dimensions[k],
dtype=new_dtype.get(k))
vc.add_variable(new_var)
assert np.all(vc['row'].value >= 1)
vc.attrs['coordDim'] = "longitude latitude"
vc.attrs[
'description'] = "Merged ESMF weights file with auxiliary variables."
vc['GRIDCODE'].attrs['long_name'] = 'Element unique identifier.'
vc['centerCoords'].units = 'degrees'
vc['row'].attrs['long_name'] = 'ESMF index to destination array.'
vc['col'].attrs['long_name'] = 'ESMF index to source array.'
vc['S'].attrs['long_name'] = 'ESMF weight factor.'
vc.write(master_weights)