def test_system_with_callback(self):
fp = get_temp_path(wd=self.current_dir_output)
def callback(message, path=fp):
with open(path, 'a') as sink:
sink.write(message)
sink.write('\n')
class FooError(Exception):
pass
ocgis_lh.configure(callback=callback)
ocgis_lh(msg='this is a test message')
ocgis_lh()
ocgis_lh(msg='this is a second test message')
ocgis_lh(msg='this should not be there', level=logging.DEBUG)
exc = FooError('foo message for value error')
try:
ocgis_lh(exc=exc)
except FooError:
pass
with open(fp, 'r') as source:
lines = source.readlines()
self.assertEqual(lines, ['this is a test message\n', 'this is a second test message\n',
'FooError: foo message for value error\n'])
def test_system_with_callback(self):
fp = get_temp_path(wd=self.current_dir_output)
def callback(message, path=fp):
with open(path, 'a') as sink:
sink.write(message)
sink.write('\n')
class FooError(Exception):
pass
ocgis_lh.configure(callback=callback)
ocgis_lh(msg='this is a test message')
ocgis_lh()
ocgis_lh(msg='this is a second test message')
ocgis_lh(msg='this should not be there', level=logging.DEBUG)
exc = FooError('foo message for value error')
try:
ocgis_lh(exc=exc)
except FooError:
pass
with open(fp, 'r') as source:
lines = source.readlines()
self.assertEqual(lines, ['this is a test message\n', 'this is a second test message\n',
'FooError: foo message for value error\n'])
def test_system_combinations(self):
_to_stream = [
True,
False
]
_to_file = [
os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log'),
None
]
_level = [logging.INFO, logging.DEBUG, logging.WARN]
for ii, (to_file, to_stream, level) in enumerate(itertools.product(_to_file, _to_stream, _level)):
ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level)
try:
ocgis_lh(ii)
ocgis_lh('a test message')
subset = ocgis_lh.get_logger('subset')
interp = ocgis_lh.get_logger('interp')
ocgis_lh('a subset message', logger=subset)
ocgis_lh('an interp message', logger=interp)
ocgis_lh('a general message', alias='foo', ugid=10)
ocgis_lh('another message', level=level)
if to_file is not None:
self.assertTrue(os.path.exists(to_file))
os.remove(to_file)
finally:
logging.shutdown()
def configure_logging(self, with_header=True):
from ocgis.util.logging_ocgis import ocgis_lh
# If file logging is enabled, check where or if the log should be written.
if self.ENABLE_FILE_LOGGING:
raise NotImplementedError
else:
to_file = None
# Flags to determine streaming to console.
if env.VERBOSE:
to_stream = True
else:
to_stream = False
# Configure the logger.
if self.DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
# This wraps the callback function with methods to capture the completion of major operations.
ocgis_lh.configure(to_file=to_file,
to_stream=to_stream,
level=level,
with_header=with_header
# callback=progress, callback_level=level,
)
def _get_progress_and_configure_logging_(self, outdir, prefix):
"""
:param str outdir: The output directory for the operations.
:param str prefix: The file prefix to use when creating the output files.
:returns: A progress object to use when executing the operations.
:rtype: :class:`ocgis.util.logging_ocgis.ProgressOcgOperations`
"""
# If file logging is enabled, check where or if the log should be written.
if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files is True:
if self.ops.output_format in self._no_directory:
to_file = None
else:
to_file = os.path.join(outdir, prefix + '.log')
else:
to_file = None
# Flags to determine streaming to console.
if env.VERBOSE:
to_stream = True
else:
to_stream = False
# Configure the logger.
if env.DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
# This wraps the callback function with methods to capture the completion of major operations.
progress = ProgressOcgOperations(callback=self.ops.callback)
ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level, callback=progress, callback_level=level)
return progress
def test_system_combinations(self):
_to_stream = [
True,
False
]
_to_file = [
os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log'),
None
]
_level = [logging.INFO, logging.DEBUG, logging.WARN]
for ii, (to_file, to_stream, level) in enumerate(itertools.product(_to_file, _to_stream, _level)):
ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level)
try:
ocgis_lh(ii)
ocgis_lh('a test message')
subset = ocgis_lh.get_logger('subset')
interp = ocgis_lh.get_logger('interp')
ocgis_lh('a subset message', logger=subset)
ocgis_lh('an interp message', logger=interp)
ocgis_lh('a general message', alias='foo', ugid=10)
ocgis_lh('another message', level=level)
if to_file is not None:
self.assertTrue(os.path.exists(to_file))
os.remove(to_file)
finally:
logging.shutdown()
def _run_():
ocgis_lh.configure()
self.assertTrue(ocgis_lh.null)
env.SUPPRESS_WARNINGS = True
ocgis_lh(level=logging.WARNING,
exc=RuntimeWarning('show me'),
force=True)
env.reset()
def test_system_exc(self):
to_file = os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log')
to_stream = False
ocgis_lh.configure(to_file=to_file, to_stream=to_stream)
try:
raise ValueError
except Exception as e:
with self.assertRaises(ValueError):
ocgis_lh('something happened', exc=e)
def test_exc(self):
to_file = os.path.join(env.DIR_OUTPUT, "test_ocgis_log.log")
to_stream = False
ocgis_lh.configure(to_file=to_file, to_stream=to_stream)
try:
raise (ValueError("some exception information"))
except Exception as e:
with self.assertRaises(ValueError):
ocgis_lh("something happened", exc=e)
def test_system_simple(self):
to_file = os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log')
to_stream = False
ocgis_lh.configure(to_file, to_stream)
ocgis_lh('a test message')
subset = ocgis_lh.get_logger('subset')
subset.info('a subset message')
def test_system_exc(self):
to_file = os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log')
to_stream = False
ocgis_lh.configure(to_file=to_file, to_stream=to_stream)
try:
raise ValueError
except Exception as e:
with self.assertRaises(ValueError):
ocgis_lh('something happened', exc=e)
def _run_():
logpath = self.get_temporary_file_path('foo.log')
ocgis_lh.configure(to_file=logpath)
ocgis_lh(msg='oh my', level=logging.WARN)
with open(logpath, 'r') as f:
lines = f.readlines()
lines = ''.join(lines)
self.assertIn('OcgWarning', lines)
self.assertIn('oh my', lines)
def test_system_simple(self):
to_file = os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log')
to_stream = False
ocgis_lh.configure(to_file, to_stream)
ocgis_lh('a test message')
subset = ocgis_lh.get_logger('subset')
subset.info('a subset message')
def test_simple(self):
to_file = os.path.join(env.DIR_OUTPUT, "test_ocgis_log.log")
to_stream = False
ocgis_lh.configure(to_file, to_stream)
ocgis_lh("a test message")
subset = ocgis_lh.get_logger("subset")
subset.info("a subset message")
def _run_():
logpath = self.get_temporary_file_path('foo.log')
ocgis_lh.configure(to_file=logpath)
ocgis_lh(msg='oh my', level=logging.WARN)
with open(logpath, 'r') as f:
lines = f.readlines()
lines = ''.join(lines)
self.assertIn('OcgWarning', lines)
self.assertIn('oh my', lines)
def _run_():
env.SUPPRESS_WARNINGS = False
logpath = self.get_temporary_file_path('ocgis.log')
ocgis_lh.configure(to_file=logpath)
exc = FutureWarning('something is about to happen')
ocgis_lh(level=logging.WARNING, exc=exc)
with open(logpath, 'r') as f:
lines = f.readlines()
lines = ''.join(lines)
self.assertIn('FutureWarning', lines)
self.assertIn('something is about to happen', lines)
env.SUPPRESS_WARNINGS = True
def _run_():
env.SUPPRESS_WARNINGS = False
logpath = self.get_temporary_file_path('ocgis.log')
ocgis_lh.configure(to_file=logpath)
exc = FutureWarning('something is about to happen')
ocgis_lh(level=logging.WARNING, exc=exc)
with open(logpath, 'r') as f:
lines = f.readlines()
lines = ''.join(lines)
self.assertIn('FutureWarning', lines)
self.assertIn('something is about to happen', lines)
env.SUPPRESS_WARNINGS = True
def _run_():
env.SUPPRESS_WARNINGS = False
ocgis_lh.configure(to_stream=True)
records = [{
'geom': Point(1, 2),
'properties': {
'a_list': [1, 2, 3]
}
}]
actual = Field.from_records(records)
self.assertNotIn("a_list", actual.keys())
env.SUPPRESS_WARNINGS = True
def test_get_dimension_map_3(self):
"""Test when bounds are found but the bounds variable is actually missing."""
_, to_file = tempfile.mkstemp(dir=self._test_dir)
ocgis_lh.configure(to_file=to_file)
try:
# remove the bounds variable from a standard metadata dictionary
rd = self.test_data.get_rd('cancm4_tas')
metadata = deepcopy(rd.source_metadata)
metadata['variables'].pop('lat_bnds')
dim_map = get_dimension_map('tas', metadata)
self.assertEqual(dim_map['Y']['bounds'], None)
self.assertTrue('lat_bnds' in list(ocgis_lh.duplicates)[0])
finally:
ocgis_lh.shutdown()
def _get_progress_and_configure_logging_(self, outdir, prefix):
"""
:param str outdir: The output directory for the operations.
:param str prefix: The file prefix to use when creating the output files.
:returns: A progress object to use when executing the operations.
:rtype: :class:`ocgis.util.logging_ocgis.ProgressOcgOperations`
"""
# TODO: This method should use ocgis.env.configure_logging().
# If file logging is enabled, check where or if the log should be written.
if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files is True:
if self.ops.output_format in self._no_directory:
to_file = None
else:
if vm.rank == 0:
os.makedirs(os.path.join(outdir, 'logs'))
vm.Barrier()
to_file = os.path.join(
outdir, 'logs',
'{prefix}-rank-{rank}.log'.format(prefix=prefix,
rank=vm.rank))
else:
to_file = None
# Flags to determine streaming to console.
if env.VERBOSE:
to_stream = True
else:
to_stream = False
# Configure the logger.
if env.DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
# This wraps the callback function with methods to capture the completion of major operations.
progress = ProgressOcgOperations(callback=self.ops.callback)
ocgis_lh.configure(to_file=to_file,
to_stream=to_stream,
level=level,
callback=progress,
callback_level=level)
return progress
def configure_logging(self, with_header=True):
from ocgis.util.logging_ocgis import ocgis_lh
# If file logging is enabled, check where or if the log should be written.
if self.ENABLE_FILE_LOGGING:
raise NotImplementedError
else:
to_file = None
# Flags to determine streaming to console.
if env.VERBOSE:
to_stream = True
else:
to_stream = False
# Configure the logger.
if self.DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
# This wraps the callback function with methods to capture the completion of major operations.
ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level, with_header=with_header
# callback=progress, callback_level=level,
)
def chunked_rwg(source, destination, weight, nchunks_dst, merge, esmf_src_type,
esmf_dst_type, genweights, esmf_regrid_method, spatial_subset,
src_resolution, dst_resolution, buffer_distance, wd, persist,
eager, ignore_degenerate, data_variables, spatial_subset_path,
verbose, loglvl, weightfilemode):
# Used for creating the history string.
the_locals = locals()
if verbose:
ocgis_lh.configure(to_stream=True, level=getattr(logging, loglvl))
ocgis_lh(msg="Starting Chunked Regrid Weight Generation",
level=logging.INFO,
logger=CRWG_LOG)
if not ocgis.env.USE_NETCDF4_MPI:
msg = (
'env.USE_NETCDF4_MPI is False. Considerable performance gains are possible if this is True. Is '
'netCDF4-python built with parallel support?')
ocgis_lh(msg, level=logging.WARN, logger=CRWG_LOG, force=True)
if data_variables is not None:
data_variables = data_variables.split(',')
if nchunks_dst is not None:
# Format the chunking decomposition from its string representation.
if ',' in nchunks_dst:
nchunks_dst = nchunks_dst.split(',')
else:
nchunks_dst = [nchunks_dst]
nchunks_dst = tuple([int(ii) for ii in nchunks_dst])
if merge:
if not spatial_subset and weight is None:
raise ValueError('"weight" must be a valid path if --merge')
if spatial_subset and genweights and weight is None:
raise ValueError('"weight" must be a valid path if --genweights')
# Make a temporary working directory is one is not provided by the client. Only do this if we are writing subsets
# and it is not a merge only operation.
should_create_wd = (nchunks_dst is None
or not all([ii == 1
for ii in nchunks_dst])) or spatial_subset
if should_create_wd:
if wd is None:
if ocgis.vm.rank == 0:
wd = tempfile.mkdtemp(prefix='ocgis_chunked_rwg_')
wd = ocgis.vm.bcast(wd)
else:
exc = None
if ocgis.vm.rank == 0:
# The working directory must not exist to proceed.
if nchunks_dst is not None:
if os.path.exists(wd):
exc = ValueError(
"Working directory {} must not exist.".format(wd))
else:
# Make the working directory nesting as needed.
os.makedirs(wd)
exc = ocgis.vm.bcast(exc)
if exc is not None:
raise exc
if merge and not spatial_subset or (spatial_subset and genweights):
if _is_subdir_(wd, weight):
raise ValueError(
'Merge weight file path must not in the working directory. It may get unintentionally deleted with the --no_persist flag.'
)
# Create the source and destination request datasets.
rd_src = _create_request_dataset_(source,
esmf_src_type,
data_variables=data_variables)
rd_dst = _create_request_dataset_(destination, esmf_dst_type)
# Execute a spatial subset if requested.
paths = None
if spatial_subset:
if spatial_subset_path is None:
spatial_subset_path = os.path.join(wd, 'spatial_subset.nc')
msg = "Executing spatial subset. Output path is: {}".format(
spatial_subset_path)
ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG)
_write_spatial_subset_(rd_src,
rd_dst,
spatial_subset_path,
src_resmax=src_resolution)
# Only split grids if a spatial subset is not requested.
else:
# Update the paths to use for the grid.
paths = {'wd': wd}
# Arguments to ESMF regridding.
esmf_kwargs = {
'regrid_method': esmf_regrid_method,
'ignore_degenerate': ignore_degenerate
}
# Create the chunked regridding object. This is used for both chunked regridding and a regrid with a spatial subset.
gs = GridChunker(rd_src,
rd_dst,
nchunks_dst=nchunks_dst,
src_grid_resolution=src_resolution,
paths=paths,
dst_grid_resolution=dst_resolution,
buffer_value=buffer_distance,
redistribute=True,
genweights=genweights,
esmf_kwargs=esmf_kwargs,
use_spatial_decomp='auto',
eager=eager)
# Write subsets and generate weights if requested in the grid splitter.
# TODO: Need a weight only option. If chunks are written, then weights are written...
if not spatial_subset and nchunks_dst is not None and not gs.is_one_chunk:
msg = "Starting main chunking loop..."
ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG)
gs.write_chunks()
else:
if spatial_subset:
source = spatial_subset_path
if genweights:
msg = "Writing ESMF weights..."
ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG)
handle_weight_file_check(weight)
gs.write_esmf_weights(source,
destination,
weight,
filemode=weightfilemode)
# Create the global weight file. This does not apply to spatial subsets because there will always be one weight
# file.
if merge and not spatial_subset and not gs.is_one_chunk:
# Weight file merge only works in serial.
exc = None
with ocgis.vm.scoped('weight file merge', [0]):
if not ocgis.vm.is_null:
msg = "Merging chunked weight files to global file. Output global weight file is: {}".format(
weight)
ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG)
handle_weight_file_check(weight)
gs.create_merged_weight_file(weight)
excs = ocgis.vm.gather(exc)
excs = ocgis.vm.bcast(excs)
for exc in excs:
if exc is not None:
raise exc
ocgis.vm.barrier()
# Append the history string if there is an output weight file.
if weight and ocgis.vm.rank == 0:
if os.path.exists(weight):
# Add some additional stuff for record keeping
import getpass
import socket
import datetime
with nc.Dataset(weight, 'a') as ds:
ds.setncattr('created_by_user', getpass.getuser())
ds.setncattr('created_on_hostname', socket.getfqdn())
ds.setncattr('history', create_history_string(the_locals))
ocgis.vm.barrier()
# Remove the working directory unless the persist flag is provided.
if not persist:
if ocgis.vm.rank == 0:
msg = "Removing working directory since persist is False."
ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG)
shutil.rmtree(wd)
ocgis.vm.barrier()
ocgis_lh(msg="Success!", level=logging.INFO, logger=CRWG_LOG)
return 0
def execute(self):
## check for a user-supplied output prefix
prefix = self.ops.prefix
# do directory management #
# flag to indicate a directory is made. mostly a precaution to make sure the appropriate directory is is removed.
made_output_directory = False
if self.ops.output_format == 'numpy':
# no output directory for numpy output
outdir = None
else:
# directories or a single output file(s) is created for the other cases
if self.ops.add_auxiliary_files:
# auxiliary files require that a directory be created
outdir = os.path.join(self.ops.dir_output,prefix)
if os.path.exists(outdir):
if env.OVERWRITE:
shutil.rmtree(outdir)
else:
raise(IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir)))
os.mkdir(outdir)
# on an exception, the output directory needs to be removed
made_output_directory = True
else:
# with no auxiliary files the output directory will do just fine
outdir = self.ops.dir_output
try:
## configure logging ###################################################
## if file logging is enable, perform some logic based on the operational
## parameters.
if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files == True:
if self.ops.output_format == 'numpy':
to_file = None
else:
to_file = os.path.join(outdir,prefix+'.log')
else:
to_file = None
## flags to determine streaming to console
if env.VERBOSE:
to_stream = True
else:
to_stream = False
## configure the logger
if env.DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
## this wraps the callback function with methods to capture the
## completion of major operations.
progress = ProgressOcgOperations(callback=self.ops.callback)
ocgis_lh.configure(to_file=to_file,to_stream=to_stream,level=level,
callback=progress,callback_level=level)
## create local logger
interpreter_log = ocgis_lh.get_logger('interpreter')
ocgis_lh('Initializing...',interpreter_log)
## set up environment ##############################################
self.check() ## run validation - doesn't do much now
## do not perform vector wrapping for NetCDF output
if self.ops.output_format == 'nc':
ocgis_lh('"vector_wrap" set to False for netCDF output',
interpreter_log,level=logging.WARN)
self.ops.vector_wrap = False
## if the requested output format is "meta" then no operations are run
## and only the operations dictionary is required to generate output.
if self.ops.output_format == 'meta':
ret = MetaConverter(self.ops).write()
## this is the standard request for other output types.
else:
## the operations object performs subsetting and calculations
ocgis_lh('initializing subset',interpreter_log,level=logging.DEBUG)
so = SubsetOperation(self.ops,progress=progress)
## if there is no grouping on the output files, a singe converter is
## is needed
if self.ops.output_grouping is None:
Conv = AbstractConverter.get_converter(self.ops.output_format)
ocgis_lh('initializing converter',interpreter_log,
level=logging.DEBUG)
conv = Conv(so,outdir,prefix,ops=self.ops,add_auxiliary_files=self.ops.add_auxiliary_files,
overwrite=env.OVERWRITE)
ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format),interpreter_log,
level=logging.DEBUG)
ret = conv.write()
else:
raise(NotImplementedError)
ocgis_lh('Operations successful.'.format(self.ops.prefix),interpreter_log)
return ret
except:
# on an exception, the output directory needs to be removed if one was created. once the output directory is
# removed, reraise.
if made_output_directory:
shutil.rmtree(outdir)
raise
finally:
## shut down logging
ocgis_lh.shutdown()
def chunked_rwg(source, destination, weight, nchunks_dst, merge, esmf_src_type, esmf_dst_type, genweights,
esmf_regrid_method, spatial_subset, src_resolution, dst_resolution, buffer_distance, wd, persist,
eager, ignore_degenerate, data_variables, spatial_subset_path, verbose, loglvl):
if verbose:
ocgis_lh.configure(to_stream=True, level=getattr(logging, loglvl))
ocgis_lh(msg="Starting Chunked Regrid Weight Generation", level=logging.INFO, logger=CRWG_LOG)
if not ocgis.env.USE_NETCDF4_MPI:
msg = ('env.USE_NETCDF4_MPI is False. Considerable performance gains are possible if this is True. Is '
'netCDF4-python built with parallel support?')
ocgis_lh(msg, level=logging.WARN, logger=CRWG_LOG, force=True)
if data_variables is not None:
data_variables = data_variables.split(',')
if nchunks_dst is not None:
# Format the chunking decomposition from its string representation.
if ',' in nchunks_dst:
nchunks_dst = nchunks_dst.split(',')
else:
nchunks_dst = [nchunks_dst]
nchunks_dst = tuple([int(ii) for ii in nchunks_dst])
if merge:
if not spatial_subset and weight is None:
raise ValueError('"weight" must be a valid path if --merge')
if spatial_subset and genweights and weight is None:
raise ValueError('"weight" must be a valid path if --genweights')
# Make a temporary working directory is one is not provided by the client. Only do this if we are writing subsets
# and it is not a merge only operation.
if wd is None:
if ocgis.vm.rank == 0:
wd = tempfile.mkdtemp(prefix='ocgis_chunked_rwg_')
wd = ocgis.vm.bcast(wd)
else:
exc = None
if ocgis.vm.rank == 0:
# The working directory must not exist to proceed.
if os.path.exists(wd):
exc = ValueError("Working directory {} must not exist.".format(wd))
else:
# Make the working directory nesting as needed.
os.makedirs(wd)
exc = ocgis.vm.bcast(exc)
if exc is not None:
raise exc
if merge and not spatial_subset or (spatial_subset and genweights):
if _is_subdir_(wd, weight):
raise ValueError(
'Merge weight file path must not in the working directory. It may get unintentionally deleted with the --no_persist flag.')
# Create the source and destination request datasets.
rd_src = _create_request_dataset_(source, esmf_src_type, data_variables=data_variables)
rd_dst = _create_request_dataset_(destination, esmf_dst_type)
# Execute a spatial subset if requested.
paths = None
if spatial_subset:
if spatial_subset_path is None:
spatial_subset_path = os.path.join(wd, 'spatial_subset.nc')
msg = "Executing spatial subset. Output path is: {}".format(spatial_subset_path)
ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG)
_write_spatial_subset_(rd_src, rd_dst, spatial_subset_path, src_resmax=src_resolution)
# Only split grids if a spatial subset is not requested.
else:
# Update the paths to use for the grid.
paths = {'wd': wd}
# Arguments to ESMF regridding.
esmf_kwargs = {'regrid_method': esmf_regrid_method,
'ignore_degenerate': ignore_degenerate}
# Create the chunked regridding object. This is used for both chunked regridding and a regrid with a spatial subset.
gs = GridChunker(rd_src, rd_dst, nchunks_dst=nchunks_dst, src_grid_resolution=src_resolution, paths=paths,
dst_grid_resolution=dst_resolution, buffer_value=buffer_distance, redistribute=True,
genweights=genweights, esmf_kwargs=esmf_kwargs, use_spatial_decomp='auto', eager=eager)
# Write subsets and generate weights if requested in the grid splitter.
# TODO: Need a weight only option. If chunks are written, then weights are written...
if not spatial_subset and nchunks_dst is not None:
msg = "Starting main chunking loop..."
ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG)
gs.write_chunks()
else:
if spatial_subset:
source = spatial_subset_path
if genweights:
msg = "Writing ESMF weights..."
ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG)
gs.write_esmf_weights(source, destination, weight)
# Create the global weight file. This does not apply to spatial subsets because there will always be one weight
# file.
if merge and not spatial_subset:
# Weight file merge only works in serial.
exc = None
with ocgis.vm.scoped('weight file merge', [0]):
if not ocgis.vm.is_null:
msg = "Merging chunked weight files to global file. Output global weight file is: {}".format(weight)
ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG)
gs.create_merged_weight_file(weight)
excs = ocgis.vm.gather(exc)
excs = ocgis.vm.bcast(excs)
for exc in excs:
if exc is not None:
raise exc
ocgis.vm.barrier()
# Remove the working directory unless the persist flag is provided.
if not persist:
if ocgis.vm.rank == 0:
msg = "Removing working directory since persist is False."
ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG)
shutil.rmtree(wd)
ocgis.vm.barrier()
ocgis_lh(msg="Success!", level=logging.INFO, logger=CRWG_LOG)
return 0
def execute(self):
## check for a user-supplied output prefix
prefix = self.ops.prefix
## do directory management.
if self.ops.output_format == 'numpy':
outdir = None
else:
outdir = os.path.join(self.ops.dir_output,prefix)
if os.path.exists(outdir):
if env.OVERWRITE:
shutil.rmtree(outdir)
else:
raise(IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir)))
os.mkdir(outdir)
try:
## configure logging ###################################################
## if file logging is enable, perform some logic based on the operational
## parameters.
if env.ENABLE_FILE_LOGGING:
if self.ops.output_format == 'numpy':
to_file = None
else:
to_file = os.path.join(outdir,prefix+'.log')
else:
to_file = None
## flags to determine streaming to console
if env.VERBOSE:
to_stream = True
else:
to_stream = False
## configure the logger
if env.DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
ocgis_lh.configure(to_file=to_file,to_stream=to_stream,level=level)
## create local logger
interpreter_log = ocgis_lh.get_logger('interpreter')
ocgis_lh('executing: {0}'.format(self.ops.prefix),interpreter_log)
## set up environment ##############################################
self.check() ## run validation - doesn't do much now
## do not perform vector wrapping for NetCDF output
if self.ops.output_format == 'nc':
ocgis_lh('"vector_wrap" set to False for netCDF output',
interpreter_log,level=logging.WARN)
self.ops.vector_wrap = False
## if the requested output format is "meta" then no operations are run
## and only the operations dictionary is required to generate output.
if self.ops.output_format == 'meta':
ret = MetaConverter(self.ops).write()
## this is the standard request for other output types.
else:
## the operations object performs subsetting and calculations
ocgis_lh('initializing subset',interpreter_log,level=logging.DEBUG)
so = SubsetOperation(self.ops,serial=env.SERIAL,nprocs=env.CORES)
## if there is no grouping on the output files, a singe converter is
## is needed
if self.ops.output_grouping is None:
Conv = OcgConverter.get_converter(self.ops.output_format)
ocgis_lh('initializing converter',interpreter_log,
level=logging.DEBUG)
conv = Conv(so,outdir,prefix,ops=self.ops)
ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format),interpreter_log,
level=logging.DEBUG)
ret = conv.write()
else:
raise(NotImplementedError)
ocgis_lh('execution complete: {0}'.format(self.ops.prefix),interpreter_log)
return(ret)
finally:
## shut down logging
ocgis_lh.shutdown()
def test_configure(self):
# test suppressing warnings in the logger
self.assertIsNone(logging._warnings_showwarning)
env.SUPPRESS_WARNINGS = False
ocgis_lh.configure()
self.assertFalse(logging._warnings_showwarning)
def test_system_parallel(self):
to_file = os.path.join(self.current_dir_output,
'rank-{}-test_ocgis_log.log'.format(vm.rank))
ocgis_lh.configure(to_file=to_file)
ocgis_lh("something happened")
self.assertEqual(len(os.listdir(self.current_dir_output)), vm.size)
def test_shutdown(self):
env.SUPPRESS_WARNINGS = False
ocgis_lh.configure(to_stream=True)
self.assertFalse(logging._warnings_showwarning)
ocgis_lh.shutdown()
self.assertIsNone(logging._warnings_showwarning)
def _run_():
env.SUPPRESS_WARNINGS = False
logpath = self.get_temporary_file_path('foo.log')
ocgis_lh.configure(to_file=logpath)
ocgis_lh(msg='hey there', level=logging.WARN)
env.SUPPRESS_WARNINGS = True
def _run_():
ocgis_lh.configure()
self.assertTrue(ocgis_lh.null)
env.SUPPRESS_WARNINGS = True
ocgis_lh(level=logging.WARNING, exc=RuntimeWarning('show me'), force=True)
env.reset()
def test_shutdown(self):
env.SUPPRESS_WARNINGS = False
ocgis_lh.configure(to_stream=True)
self.assertFalse(logging._warnings_showwarning)
ocgis_lh.shutdown()
self.assertIsNone(logging._warnings_showwarning)
def test_configure(self):
# test suppressing warnings in the logger
self.assertIsNone(logging._warnings_showwarning)
env.SUPPRESS_WARNINGS = False
ocgis_lh.configure()
self.assertFalse(logging._warnings_showwarning)
def _run_():
env.SUPPRESS_WARNINGS = False
logpath = self.get_temporary_file_path('foo.log')
ocgis_lh.configure(to_file=logpath)
ocgis_lh(msg='hey there', level=logging.WARN)
env.SUPPRESS_WARNINGS = True
def test_system_parallel(self):
to_file = os.path.join(self.current_dir_output, 'rank-{}-test_ocgis_log.log'.format(vm.rank))
ocgis_lh.configure(to_file=to_file)
ocgis_lh("something happened")
self.assertEqual(len(os.listdir(self.current_dir_output)), vm.size)
