def tearDown(self):
try:
if self.create_dir:
shutil.rmtree(self.current_dir_output)
finally:
if self.reset_env:
env.reset()
if self.shutdown_logging:
ocgis_lh.shutdown()
def test_get_dimension_map_3(self):
"""Test when bounds are found but the bounds variable is actually missing."""
_, to_file = tempfile.mkstemp(dir=self._test_dir)
ocgis_lh.configure(to_file=to_file)
try:
# remove the bounds variable from a standard metadata dictionary
rd = self.test_data.get_rd('cancm4_tas')
metadata = deepcopy(rd.source_metadata)
metadata['variables'].pop('lat_bnds')
dim_map = get_dimension_map('tas', metadata)
self.assertEqual(dim_map['Y']['bounds'], None)
self.assertTrue('lat_bnds' in list(ocgis_lh.duplicates)[0])
finally:
ocgis_lh.shutdown()
def test_call(self):
# test warning is logged to the terminal
self.assertTrue(ocgis_lh.null)
def _run_():
ocgis_lh.configure()
self.assertTrue(ocgis_lh.null)
env.SUPPRESS_WARNINGS = False
ocgis_lh(level=logging.WARNING, exc=RuntimeWarning('show me'))
env.SUPPRESS_WARNINGS = True
self.assertWarns(RuntimeWarning, _run_)
# test warning is logged to the terminal and also logged to file
ocgis_lh.shutdown()
def _run_():
env.SUPPRESS_WARNINGS = False
logpath = self.get_temporary_file_path('ocgis.log')
ocgis_lh.configure(to_file=logpath)
exc = FutureWarning('something is about to happen')
ocgis_lh(level=logging.WARNING, exc=exc)
with open(logpath, 'r') as f:
lines = f.readlines()
lines = ''.join(lines)
self.assertIn('FutureWarning', lines)
self.assertIn('something is about to happen', lines)
env.SUPPRESS_WARNINGS = True
self.assertWarns(FutureWarning, _run_)
# test a warning without an exception
ocgis_lh.shutdown()
def _run_():
env.SUPPRESS_WARNINGS = False
logpath = self.get_temporary_file_path('foo.log')
ocgis_lh.configure(to_file=logpath)
ocgis_lh(msg='hey there', level=logging.WARN)
env.SUPPRESS_WARNINGS = True
self.assertWarns(OcgWarning, _run_)
# test suppressing warnings
ocgis_lh.shutdown()
def _run_():
logpath = self.get_temporary_file_path('foo.log')
ocgis_lh.configure(to_file=logpath)
ocgis_lh(msg='oh my', level=logging.WARN)
with open(logpath, 'r') as f:
lines = f.readlines()
lines = ''.join(lines)
self.assertIn('OcgWarning', lines)
self.assertIn('oh my', lines)
with self.assertRaises(AssertionError):
self.assertWarns(OcgWarning, _run_)
def test_call(self):
# test warning is logged to the terminal
self.assertTrue(ocgis_lh.null)
def _run_():
ocgis_lh.configure()
self.assertTrue(ocgis_lh.null)
env.SUPPRESS_WARNINGS = False
ocgis_lh(level=logging.WARNING, exc=RuntimeWarning('show me'))
env.SUPPRESS_WARNINGS = True
self.assertWarns(RuntimeWarning, _run_)
# test warning is logged to the terminal and also logged to file
ocgis_lh.shutdown()
def _run_():
env.SUPPRESS_WARNINGS = False
logpath = self.get_temporary_file_path('ocgis.log')
ocgis_lh.configure(to_file=logpath)
exc = FutureWarning('something is about to happen')
ocgis_lh(level=logging.WARNING, exc=exc)
with open(logpath, 'r') as f:
lines = f.readlines()
lines = ''.join(lines)
self.assertIn('FutureWarning', lines)
self.assertIn('something is about to happen', lines)
env.SUPPRESS_WARNINGS = True
self.assertWarns(FutureWarning, _run_)
# test a warning without an exception
ocgis_lh.shutdown()
def _run_():
env.SUPPRESS_WARNINGS = False
logpath = self.get_temporary_file_path('foo.log')
ocgis_lh.configure(to_file=logpath)
ocgis_lh(msg='hey there', level=logging.WARN)
env.SUPPRESS_WARNINGS = True
self.assertWarns(OcgWarning, _run_)
# test suppressing warnings
ocgis_lh.shutdown()
def _run_():
logpath = self.get_temporary_file_path('foo.log')
ocgis_lh.configure(to_file=logpath)
ocgis_lh(msg='oh my', level=logging.WARN)
with open(logpath, 'r') as f:
lines = f.readlines()
lines = ''.join(lines)
self.assertIn('OcgWarning', lines)
self.assertIn('oh my', lines)
with self.assertRaises(AssertionError):
self.assertWarns(OcgWarning, _run_)
def test_shutdown(self):
env.SUPPRESS_WARNINGS = False
ocgis_lh.configure(to_stream=True)
self.assertFalse(logging._warnings_showwarning)
ocgis_lh.shutdown()
self.assertIsNone(logging._warnings_showwarning)
def tearDown(self):
ocgis_lh.shutdown()
TestBase.tearDown(self)
def execute(self):
# check for a user-supplied output prefix
prefix = self.ops.prefix
# do directory management #
# flag to indicate a directory is made. mostly a precaution to make sure the appropriate directory is is
# removed.
made_output_directory = False
if self.ops.output_format in self._no_directory:
# no output directory for numpy output
outdir = None
else:
# directories or a single output file(s) is created for the other cases
if self.ops.add_auxiliary_files:
# auxiliary files require that a directory be created
outdir = os.path.join(self.ops.dir_output, prefix)
if os.path.exists(outdir):
if env.OVERWRITE:
shutil.rmtree(outdir)
else:
raise IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir))
os.mkdir(outdir)
# on an exception, the output directory needs to be removed
made_output_directory = True
else:
# with no auxiliary files the output directory will do just fine
outdir = self.ops.dir_output
try:
# configure logging ########################################################################################
progress = self._get_progress_and_configure_logging_(outdir, prefix)
# create local logger
interpreter_log = ocgis_lh.get_logger('interpreter')
ocgis_lh('Initializing...', interpreter_log)
# set up environment #######################################################################################
# run validation - doesn't do much now
self.check()
# do not perform vector wrapping for NetCDF output
if self.ops.output_format == 'nc':
ocgis_lh('"vector_wrap" set to False for netCDF output',
interpreter_log, level=logging.WARN)
self.ops.vector_wrap = False
# if the requested output format is "meta" then no operations are run and only the operations dictionary is
# required to generate output.
Converter = self.ops._get_object_(OutputFormat.name).get_converter_class()
if issubclass(Converter, AbstractMetaConverter):
ret = Converter(self.ops).write()
# this is the standard request for other output types.
else:
# the operations object performs subsetting and calculations
ocgis_lh('initializing subset', interpreter_log, level=logging.DEBUG)
so = SubsetOperation(self.ops, progress=progress)
# if there is no grouping on the output files, a singe converter is is needed
if self.ops.output_grouping is None:
ocgis_lh('initializing converter', interpreter_log, level=logging.DEBUG)
conv = self._get_converter_(Converter, outdir, prefix, so)
ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format), interpreter_log,
level=logging.DEBUG)
ret = conv.write()
else:
raise NotImplementedError
ocgis_lh('Operations successful.'.format(self.ops.prefix), interpreter_log)
return ret
except:
# The output directory needs to be removed if one was created. Shutdown logging before to make sure there
# is no file lock (Windows).
ocgis_lh.shutdown()
if made_output_directory:
shutil.rmtree(outdir)
raise
finally:
ocgis_lh.shutdown()
def execute(self):
# check for a user-supplied output prefix
prefix = self.ops.prefix
# do directory management #
# flag to indicate a directory is made. mostly a precaution to make sure the appropriate directory is is
# removed.
made_output_directory = False
if self.ops.output_format in self._no_directory:
# No output directory for some formats.
outdir = None
else:
# Directories or a single output file(s) is created for the other cases.
if self.ops.add_auxiliary_files:
# Auxiliary files require that a directory be created.
outdir = os.path.join(self.ops.dir_output, prefix)
# Create and/or remove the output directory.
if vm.rank == 0:
if os.path.exists(outdir):
if env.OVERWRITE:
shutil.rmtree(outdir)
else:
raise IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir))
os.mkdir(outdir)
# Block until output directory is created. Most often the zero rank manages writing, but this is not a
# requirement.
vm.Barrier()
# On an exception, the output directory needs to be removed.
made_output_directory = True
else:
# with no auxiliary files the output directory will do just fine
outdir = self.ops.dir_output
try:
# configure logging ########################################################################################
progress = self._get_progress_and_configure_logging_(outdir, prefix)
# create local logger
interpreter_log = ocgis_lh.get_logger('interpreter')
ocgis_lh('Initializing...', interpreter_log)
# set up environment #######################################################################################
# run validation - doesn't do much now
self.check()
# do not perform vector wrapping for NetCDF output
if self.ops.output_format == 'nc':
ocgis_lh('"vector_wrap" set to False for netCDF output',
interpreter_log, level=logging.WARN)
self.ops.vector_wrap = False
# if the requested output format is "meta" then no operations are run and only the operations dictionary is
# required to generate output.
Converter = self.ops._get_object_(OutputFormat.name).get_converter_class()
if issubclass(Converter, AbstractMetaConverter):
ret = Converter(self.ops).write()
# this is the standard request for other output types.
else:
# the operations object performs subsetting and calculations
ocgis_lh('initializing subset', interpreter_log, level=logging.DEBUG)
so = OperationsEngine(self.ops, progress=progress)
# if there is no grouping on the output files, a singe converter is needed
if self.ops.output_grouping is None:
ocgis_lh('initializing converter', interpreter_log, level=logging.DEBUG)
conv = self._get_converter_(Converter, outdir, prefix, so)
ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format), interpreter_log,
level=logging.DEBUG)
ret = conv.write()
else:
raise NotImplementedError
ocgis_lh('Operations successful.'.format(self.ops.prefix), interpreter_log)
return ret
except:
# The output directory needs to be removed if one was created. Shutdown logging before to make sure there
# is no file lock (Windows).
ocgis_lh.shutdown()
if vm.rank == 0 and made_output_directory:
shutil.rmtree(outdir)
raise
finally:
ocgis_lh.shutdown()
if env.ADD_OPS_MPI_BARRIER:
vm.Barrier()
def test_shutdown(self):
env.SUPPRESS_WARNINGS = False
ocgis_lh.configure(to_stream=True)
self.assertFalse(logging._warnings_showwarning)
ocgis_lh.shutdown()
self.assertIsNone(logging._warnings_showwarning)
def tearDown(self):
ocgis_lh.shutdown()
TestBase.tearDown(self)
def execute(self):
## check for a user-supplied output prefix
prefix = self.ops.prefix
## do directory management.
if self.ops.output_format == 'numpy':
outdir = None
else:
outdir = os.path.join(self.ops.dir_output,prefix)
if os.path.exists(outdir):
if env.OVERWRITE:
shutil.rmtree(outdir)
else:
raise(IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir)))
os.mkdir(outdir)
try:
## configure logging ###################################################
## if file logging is enable, perform some logic based on the operational
## parameters.
if env.ENABLE_FILE_LOGGING:
if self.ops.output_format == 'numpy':
to_file = None
else:
to_file = os.path.join(outdir,prefix+'.log')
else:
to_file = None
## flags to determine streaming to console
if env.VERBOSE:
to_stream = True
else:
to_stream = False
## configure the logger
if env.DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
ocgis_lh.configure(to_file=to_file,to_stream=to_stream,level=level)
## create local logger
interpreter_log = ocgis_lh.get_logger('interpreter')
ocgis_lh('executing: {0}'.format(self.ops.prefix),interpreter_log)
## set up environment ##############################################
self.check() ## run validation - doesn't do much now
## do not perform vector wrapping for NetCDF output
if self.ops.output_format == 'nc':
ocgis_lh('"vector_wrap" set to False for netCDF output',
interpreter_log,level=logging.WARN)
self.ops.vector_wrap = False
## if the requested output format is "meta" then no operations are run
## and only the operations dictionary is required to generate output.
if self.ops.output_format == 'meta':
ret = MetaConverter(self.ops).write()
## this is the standard request for other output types.
else:
## the operations object performs subsetting and calculations
ocgis_lh('initializing subset',interpreter_log,level=logging.DEBUG)
so = SubsetOperation(self.ops,serial=env.SERIAL,nprocs=env.CORES)
## if there is no grouping on the output files, a singe converter is
## is needed
if self.ops.output_grouping is None:
Conv = OcgConverter.get_converter(self.ops.output_format)
ocgis_lh('initializing converter',interpreter_log,
level=logging.DEBUG)
conv = Conv(so,outdir,prefix,ops=self.ops)
ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format),interpreter_log,
level=logging.DEBUG)
ret = conv.write()
else:
raise(NotImplementedError)
ocgis_lh('execution complete: {0}'.format(self.ops.prefix),interpreter_log)
return(ret)
finally:
## shut down logging
ocgis_lh.shutdown()
def execute(self):
## check for a user-supplied output prefix
prefix = self.ops.prefix
# do directory management #
# flag to indicate a directory is made. mostly a precaution to make sure the appropriate directory is is removed.
made_output_directory = False
if self.ops.output_format == 'numpy':
# no output directory for numpy output
outdir = None
else:
# directories or a single output file(s) is created for the other cases
if self.ops.add_auxiliary_files:
# auxiliary files require that a directory be created
outdir = os.path.join(self.ops.dir_output,prefix)
if os.path.exists(outdir):
if env.OVERWRITE:
shutil.rmtree(outdir)
else:
raise(IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir)))
os.mkdir(outdir)
# on an exception, the output directory needs to be removed
made_output_directory = True
else:
# with no auxiliary files the output directory will do just fine
outdir = self.ops.dir_output
try:
## configure logging ###################################################
## if file logging is enable, perform some logic based on the operational
## parameters.
if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files == True:
if self.ops.output_format == 'numpy':
to_file = None
else:
to_file = os.path.join(outdir,prefix+'.log')
else:
to_file = None
## flags to determine streaming to console
if env.VERBOSE:
to_stream = True
else:
to_stream = False
## configure the logger
if env.DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
## this wraps the callback function with methods to capture the
## completion of major operations.
progress = ProgressOcgOperations(callback=self.ops.callback)
ocgis_lh.configure(to_file=to_file,to_stream=to_stream,level=level,
callback=progress,callback_level=level)
## create local logger
interpreter_log = ocgis_lh.get_logger('interpreter')
ocgis_lh('Initializing...',interpreter_log)
## set up environment ##############################################
self.check() ## run validation - doesn't do much now
## do not perform vector wrapping for NetCDF output
if self.ops.output_format == 'nc':
ocgis_lh('"vector_wrap" set to False for netCDF output',
interpreter_log,level=logging.WARN)
self.ops.vector_wrap = False
## if the requested output format is "meta" then no operations are run
## and only the operations dictionary is required to generate output.
if self.ops.output_format == 'meta':
ret = MetaConverter(self.ops).write()
## this is the standard request for other output types.
else:
## the operations object performs subsetting and calculations
ocgis_lh('initializing subset',interpreter_log,level=logging.DEBUG)
so = SubsetOperation(self.ops,progress=progress)
## if there is no grouping on the output files, a singe converter is
## is needed
if self.ops.output_grouping is None:
Conv = AbstractConverter.get_converter(self.ops.output_format)
ocgis_lh('initializing converter',interpreter_log,
level=logging.DEBUG)
conv = Conv(so,outdir,prefix,ops=self.ops,add_auxiliary_files=self.ops.add_auxiliary_files,
overwrite=env.OVERWRITE)
ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format),interpreter_log,
level=logging.DEBUG)
ret = conv.write()
else:
raise(NotImplementedError)
ocgis_lh('Operations successful.'.format(self.ops.prefix),interpreter_log)
return ret
except:
# on an exception, the output directory needs to be removed if one was created. once the output directory is
# removed, reraise.
if made_output_directory:
shutil.rmtree(outdir)
raise
finally:
## shut down logging
ocgis_lh.shutdown()
