def test_system_combinations(self):
_to_stream = [
True,
False
]
_to_file = [
os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log'),
None
]
_level = [logging.INFO, logging.DEBUG, logging.WARN]
for ii, (to_file, to_stream, level) in enumerate(itertools.product(_to_file, _to_stream, _level)):
ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level)
try:
ocgis_lh(ii)
ocgis_lh('a test message')
subset = ocgis_lh.get_logger('subset')
interp = ocgis_lh.get_logger('interp')
ocgis_lh('a subset message', logger=subset)
ocgis_lh('an interp message', logger=interp)
ocgis_lh('a general message', alias='foo', ugid=10)
ocgis_lh('another message', level=level)
if to_file is not None:
self.assertTrue(os.path.exists(to_file))
os.remove(to_file)
finally:
logging.shutdown()
def test_system_combinations(self):
_to_stream = [
True,
False
]
_to_file = [
os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log'),
None
]
_level = [logging.INFO, logging.DEBUG, logging.WARN]
for ii, (to_file, to_stream, level) in enumerate(itertools.product(_to_file, _to_stream, _level)):
ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level)
try:
ocgis_lh(ii)
ocgis_lh('a test message')
subset = ocgis_lh.get_logger('subset')
interp = ocgis_lh.get_logger('interp')
ocgis_lh('a subset message', logger=subset)
ocgis_lh('an interp message', logger=interp)
ocgis_lh('a general message', alias='foo', ugid=10)
ocgis_lh('another message', level=level)
if to_file is not None:
self.assertTrue(os.path.exists(to_file))
os.remove(to_file)
finally:
logging.shutdown()
def __init__(self,ops,serial=True,nprocs=1):
self.ops = ops
self.serial = serial
self.nprocs = nprocs
self._subset_log = ocgis_lh.get_logger('subset')
## create the calculation engine
if self.ops.calc is None:
self.cengine = None
else:
ocgis_lh('initializing calculation engine',self._subset_log,level=logging.DEBUG)
self.cengine = OcgCalculationEngine(self.ops.calc_grouping,
self.ops.calc,
raw=self.ops.calc_raw,
agg=self.ops.aggregate,
calc_sample_size=self.ops.calc_sample_size)
## in the case of netcdf output, geometries must be unioned. this is
## also true for the case of the selection geometry being requested as
## aggregated.
if (self.ops.output_format == 'nc' or self.ops.agg_selection is True) \
and self.ops.geom is not None:
ocgis_lh('aggregating selection geometry',self._subset_log)
build = True
for element_geom in self.ops.geom:
if build:
new_geom = element_geom['geom']
new_crs = element_geom['crs']
new_properties = {'UGID':1}
build = False
else:
new_geom = new_geom.union(element_geom['geom'])
itr = [{'geom':new_geom,'properties':new_properties,'crs':new_crs}]
self.ops.geom = itr
def test_system_simple(self):
to_file = os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log')
to_stream = False
ocgis_lh.configure(to_file, to_stream)
ocgis_lh('a test message')
subset = ocgis_lh.get_logger('subset')
subset.info('a subset message')
def test_simple(self):
to_file = os.path.join(env.DIR_OUTPUT, "test_ocgis_log.log")
to_stream = False
ocgis_lh.configure(to_file, to_stream)
ocgis_lh("a test message")
subset = ocgis_lh.get_logger("subset")
subset.info("a subset message")
def __init__(self, ops, serial=True, nprocs=1, validate=True):
self.ops = ops
self.serial = serial
self.nprocs = nprocs
subset_log = ocgis_lh.get_logger('subset')
if validate:
ocgis_lh('validating request datasets',
subset_log,
level=logging.DEBUG)
ops.dataset.validate(ops=ops)
## create the calculation engine
if self.ops.calc is None:
self.cengine = None
else:
ocgis_lh('initializing calculation engine',
subset_log,
level=logging.DEBUG)
self.cengine = OcgCalculationEngine(self.ops.calc_grouping,
self.ops.calc,
raw=self.ops.calc_raw,
agg=self.ops.aggregate)
## check for snippet request in the operations dictionary. if there is
## on, the time range should be set in the operations dictionary.
if self.ops.snippet is True:
##TODO: move snippet to iteration
ocgis_lh('getting snippet bounds', subset_log)
for rd in self.ops.dataset:
## snippet is not implemented for time regions
if rd.time_region is not None:
exc = NotImplementedError(
'snippet is not implemented for time regions')
ocgis_lh(exc=exc, logger=subset_log)
rd.level_range = [1, 1]
ods = rd.ds
## load the first time slice if there is calculation or the
## calculation does not use a temporal group.
if self.cengine is None or (self.cengine is not None
and self.cengine.grouping is None):
##TODO: improve slicing to not load all time values in a more
## elegant way.
ods._load_slice.update({'T': slice(0, 1)})
## snippet for the computation. this currently requires loading
## all the data from the time dimension into memory.
##TODO: more efficiently pull dates for monthly grouping (for
##example).
else:
ods.temporal.set_grouping(self.cengine.grouping)
tgdim = ods.temporal.group
times = ods.temporal.value[tgdim.dgroups[0]]
rd.time_range = list(
ods.temporal.get_datetime([times.min(),
times.max()]))
def test_system_simple(self):
to_file = os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log')
to_stream = False
ocgis_lh.configure(to_file, to_stream)
ocgis_lh('a test message')
subset = ocgis_lh.get_logger('subset')
subset.info('a subset message')
def __init__(self,colls,outdir,prefix,ops=None,add_meta=True):
self.colls = colls
self.ops = ops
self.prefix = prefix
self.outdir = outdir
self.add_meta = add_meta
self._log = ocgis_lh.get_logger('conv')
if self._ext is None:
self.path = self.outdir
else:
self.path = os.path.join(self.outdir,prefix+'.'+self._ext)
ocgis_lh('converter initialized',level=logging.DEBUG,logger=self._log)
def __init__(self,ops,serial=True,nprocs=1,validate=True):
self.ops = ops
self.serial = serial
self.nprocs = nprocs
subset_log = ocgis_lh.get_logger('subset')
if validate:
ocgis_lh('validating request datasets',subset_log,level=logging.DEBUG)
ops.dataset.validate(ops=ops)
## create the calculation engine
if self.ops.calc is None:
self.cengine = None
else:
ocgis_lh('initializing calculation engine',subset_log,level=logging.DEBUG)
self.cengine = OcgCalculationEngine(self.ops.calc_grouping,
self.ops.calc,
raw=self.ops.calc_raw,
agg=self.ops.aggregate)
## check for snippet request in the operations dictionary. if there is
## on, the time range should be set in the operations dictionary.
if self.ops.snippet is True:
##TODO: move snippet to iteration
ocgis_lh('getting snippet bounds',subset_log)
for rd in self.ops.dataset:
## snippet is not implemented for time regions
if rd.time_region is not None:
exc = NotImplementedError('snippet is not implemented for time regions')
ocgis_lh(exc=exc,logger=subset_log)
rd.level_range = [1,1]
ods = rd.ds
## load the first time slice if there is calculation or the
## calculation does not use a temporal group.
if self.cengine is None or (self.cengine is not None and self.cengine.grouping is None):
##TODO: improve slicing to not load all time values in a more
## elegant way.
ods._load_slice.update({'T':slice(0,1)})
## snippet for the computation. this currently requires loading
## all the data from the time dimension into memory.
##TODO: more efficiently pull dates for monthly grouping (for
##example).
else:
ods.temporal.set_grouping(self.cengine.grouping)
tgdim = ods.temporal.group
times = ods.temporal.value[tgdim.dgroups[0]]
rd.time_range = list(ods.temporal.get_datetime([times.min(),times.max()]))
def __init__(self, prefix=None, outdir=None, overwrite=False):
self.outdir = outdir
self.overwrite = overwrite
self.prefix = prefix
if self._ext is None:
self.path = self.outdir
else:
self.path = os.path.join(self.outdir, prefix + "." + self._ext)
if os.path.exists(self.path):
if not self.overwrite:
msg = messages.M3.format(self.path)
raise IOError(msg)
self._log = ocgis_lh.get_logger("conv")
def _iter_proc_args_(self):
''':rtype: tuple'''
subset_log = ocgis_lh.get_logger('subset')
## if there is no geometry, yield None.
if self.ops.geom is None:
ocgis_lh('returning entire spatial domain - no selection geometry',subset_log)
yield(self,None,subset_log)
## iterator through geometries in the ShpDataset
elif isinstance(self.ops.geom,ShpDataset):
ocgis_lh('{0} geometry(s) to process'.format(len(self.ops.geom)),subset_log)
for geom in self.ops.geom:
yield(self,geom,subset_log)
## otherwise, the data is likely a GeometryDataset with a single value.
## just return it.
else:
ocgis_lh('1 geometry to process'.format(len(self.ops.geom)),subset_log)
yield(self,self.ops.geom,subset_log)
def __init__(self, prefix=None, outdir=None, overwrite=False, options=None):
self.outdir = outdir
self.overwrite = overwrite
self.prefix = prefix
if options is None:
self.options = {}
else:
self.options = options
if self._ext is None:
self.path = self.outdir
else:
self.path = os.path.join(self.outdir, prefix + '.' + self._ext)
if os.path.exists(self.path):
if not self.overwrite:
msg = messages.M3.format(self.path)
raise IOError(msg)
self._log = ocgis_lh.get_logger('conv')
def __init__(self, ops, request_base_size_only=False, progress=None):
self.ops = ops
self._request_base_size_only = request_base_size_only
self._subset_log = ocgis_lh.get_logger('subset')
self._progress = progress or ProgressOcgOperations()
self._original_subcomm = deepcopy(vm.current_comm_name)
self._backtransform = {}
# Create the calculation engine is calculations are present.
if self.ops.calc is None or self._request_base_size_only:
self.cengine = None
self._has_multivariate_calculations = False
else:
ocgis_lh('initializing calculation engine', self._subset_log, level=logging.DEBUG)
self.cengine = CalculationEngine(self.ops.calc_grouping,
self.ops.calc,
calc_sample_size=self.ops.calc_sample_size,
progress=self._progress,
spatial_aggregation=self.ops.aggregate)
self._has_multivariate_calculations = self.cengine.has_multivariate_functions
def __init__(self,colls,outdir,prefix,ops=None,add_meta=True,add_auxiliary_files=True,
overwrite=False):
self.colls = colls
self.ops = ops
self.prefix = prefix
self.outdir = outdir
self.add_meta = add_meta
self.add_auxiliary_files = add_auxiliary_files
self.overwrite = overwrite
self._log = ocgis_lh.get_logger('conv')
if self._ext is None:
self.path = self.outdir
else:
self.path = os.path.join(self.outdir,prefix+'.'+self._ext)
if os.path.exists(self.path):
if not self.overwrite:
msg = 'Output path exists "{0}" and must be removed before proceeding. Set "overwrite" argument or env.OVERWRITE to True to overwrite.'.format(self.path)
ocgis_lh(logger=self._log,exc=IOError(msg))
ocgis_lh('converter initialized',level=logging.DEBUG,logger=self._log)
def __init__(self,
colls,
outdir,
prefix,
mode='raw',
ops=None,
add_meta=True):
self.colls = colls
self.ops = ops
self.prefix = prefix
self.outdir = outdir
self.mode = mode
self.add_meta = add_meta
self._log = ocgis_lh.get_logger('conv')
if self._ext is None:
self.path = self.outdir
else:
self.path = os.path.join(self.outdir, prefix + '.' + self._ext)
ocgis_lh('converter initialized',
level=logging.DEBUG,
logger=self._log)
def __init__(self,ops,request_base_size_only=False,progress=None):
self.ops = ops
self._request_base_size_only = request_base_size_only
self._subset_log = ocgis_lh.get_logger('subset')
self._progress = progress or ProgressOcgOperations()
## create the calculation engine
if self.ops.calc == None or self._request_base_size_only == True:
self.cengine = None
self._has_multivariate_calculations = False
else:
ocgis_lh('initializing calculation engine',self._subset_log,level=logging.DEBUG)
self.cengine = OcgCalculationEngine(self.ops.calc_grouping,
self.ops.calc,
raw=self.ops.calc_raw,
agg=self.ops.aggregate,
calc_sample_size=self.ops.calc_sample_size,
progress=self._progress)
self._has_multivariate_calculations = any([self.cengine._check_calculation_members_(self.cengine.funcs,k) \
for k in [AbstractMultivariateFunction,MultivariateEvalFunction]])
## in the case of netcdf output, geometries must be unioned. this is
## also true for the case of the selection geometry being requested as
## aggregated.
if (self.ops.output_format == 'nc' or self.ops.agg_selection is True) \
and self.ops.geom is not None:
ocgis_lh('aggregating selection geometry',self._subset_log)
build = True
for element_geom in self.ops.geom:
if build:
new_geom = element_geom['geom']
new_crs = element_geom['crs']
new_properties = {'UGID':1}
build = False
else:
new_geom = new_geom.union(element_geom['geom'])
itr = [{'geom':new_geom,'properties':new_properties,'crs':new_crs}]
self.ops.geom = itr
def _iter_proc_args_(self):
''':rtype: tuple'''
subset_log = ocgis_lh.get_logger('subset')
## if there is no geometry, yield None.
if self.ops.geom is None:
ocgis_lh('returning entire spatial domain - no selection geometry',
subset_log)
yield (self, None, subset_log)
## iterator through geometries in the ShpDataset
elif isinstance(self.ops.geom, ShpDataset):
ocgis_lh('{0} geometry(s) to process'.format(len(self.ops.geom)),
subset_log)
for geom in self.ops.geom:
yield (self, geom, subset_log)
## otherwise, the data is likely a GeometryDataset with a single value.
## just return it.
else:
ocgis_lh('1 geometry to process'.format(len(self.ops.geom)),
subset_log)
yield (self, self.ops.geom, subset_log)
def __init__(self, ops, request_base_size_only=False, progress=None):
self.ops = ops
self._request_base_size_only = request_base_size_only
self._subset_log = ocgis_lh.get_logger('subset')
self._progress = progress or ProgressOcgOperations()
self._original_subcomm = deepcopy(vm.current_comm_name)
self._backtransform = {}
# Create the calculation engine is calculations are present.
if self.ops.calc is None or self._request_base_size_only:
self.cengine = None
self._has_multivariate_calculations = False
else:
ocgis_lh('initializing calculation engine',
self._subset_log,
level=logging.DEBUG)
self.cengine = CalculationEngine(
self.ops.calc_grouping,
self.ops.calc,
calc_sample_size=self.ops.calc_sample_size,
progress=self._progress,
spatial_aggregation=self.ops.aggregate)
self._has_multivariate_calculations = self.cengine.has_multivariate_functions
def execute(self):
## check for a user-supplied output prefix
prefix = self.ops.prefix
## do directory management.
if self.ops.output_format == 'numpy':
outdir = None
else:
outdir = os.path.join(self.ops.dir_output,prefix)
if os.path.exists(outdir):
if env.OVERWRITE:
shutil.rmtree(outdir)
else:
raise(IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir)))
os.mkdir(outdir)
try:
## configure logging ###################################################
## if file logging is enable, perform some logic based on the operational
## parameters.
if env.ENABLE_FILE_LOGGING:
if self.ops.output_format == 'numpy':
to_file = None
else:
to_file = os.path.join(outdir,prefix+'.log')
else:
to_file = None
## flags to determine streaming to console
if env.VERBOSE:
to_stream = True
else:
to_stream = False
## configure the logger
if env.DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
ocgis_lh.configure(to_file=to_file,to_stream=to_stream,level=level)
## create local logger
interpreter_log = ocgis_lh.get_logger('interpreter')
ocgis_lh('executing: {0}'.format(self.ops.prefix),interpreter_log)
## set up environment ##############################################
self.check() ## run validation - doesn't do much now
## do not perform vector wrapping for NetCDF output
if self.ops.output_format == 'nc':
ocgis_lh('"vector_wrap" set to False for netCDF output',
interpreter_log,level=logging.WARN)
self.ops.vector_wrap = False
## if the requested output format is "meta" then no operations are run
## and only the operations dictionary is required to generate output.
if self.ops.output_format == 'meta':
ret = MetaConverter(self.ops).write()
## this is the standard request for other output types.
else:
## the operations object performs subsetting and calculations
ocgis_lh('initializing subset',interpreter_log,level=logging.DEBUG)
so = SubsetOperation(self.ops,serial=env.SERIAL,nprocs=env.CORES)
## if there is no grouping on the output files, a singe converter is
## is needed
if self.ops.output_grouping is None:
Conv = OcgConverter.get_converter(self.ops.output_format)
ocgis_lh('initializing converter',interpreter_log,
level=logging.DEBUG)
conv = Conv(so,outdir,prefix,ops=self.ops)
ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format),interpreter_log,
level=logging.DEBUG)
ret = conv.write()
else:
raise(NotImplementedError)
ocgis_lh('execution complete: {0}'.format(self.ops.prefix),interpreter_log)
return(ret)
finally:
## shut down logging
ocgis_lh.shutdown()
def execute(self):
# check for a user-supplied output prefix
prefix = self.ops.prefix
# do directory management #
# flag to indicate a directory is made. mostly a precaution to make sure the appropriate directory is is
# removed.
made_output_directory = False
if self.ops.output_format in self._no_directory:
# no output directory for numpy output
outdir = None
else:
# directories or a single output file(s) is created for the other cases
if self.ops.add_auxiliary_files:
# auxiliary files require that a directory be created
outdir = os.path.join(self.ops.dir_output, prefix)
if os.path.exists(outdir):
if env.OVERWRITE:
shutil.rmtree(outdir)
else:
raise IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir))
os.mkdir(outdir)
# on an exception, the output directory needs to be removed
made_output_directory = True
else:
# with no auxiliary files the output directory will do just fine
outdir = self.ops.dir_output
try:
# configure logging ########################################################################################
progress = self._get_progress_and_configure_logging_(outdir, prefix)
# create local logger
interpreter_log = ocgis_lh.get_logger('interpreter')
ocgis_lh('Initializing...', interpreter_log)
# set up environment #######################################################################################
# run validation - doesn't do much now
self.check()
# do not perform vector wrapping for NetCDF output
if self.ops.output_format == 'nc':
ocgis_lh('"vector_wrap" set to False for netCDF output',
interpreter_log, level=logging.WARN)
self.ops.vector_wrap = False
# if the requested output format is "meta" then no operations are run and only the operations dictionary is
# required to generate output.
Converter = self.ops._get_object_(OutputFormat.name).get_converter_class()
if issubclass(Converter, AbstractMetaConverter):
ret = Converter(self.ops).write()
# this is the standard request for other output types.
else:
# the operations object performs subsetting and calculations
ocgis_lh('initializing subset', interpreter_log, level=logging.DEBUG)
so = SubsetOperation(self.ops, progress=progress)
# if there is no grouping on the output files, a singe converter is is needed
if self.ops.output_grouping is None:
ocgis_lh('initializing converter', interpreter_log, level=logging.DEBUG)
conv = self._get_converter_(Converter, outdir, prefix, so)
ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format), interpreter_log,
level=logging.DEBUG)
ret = conv.write()
else:
raise NotImplementedError
ocgis_lh('Operations successful.'.format(self.ops.prefix), interpreter_log)
return ret
except:
# The output directory needs to be removed if one was created. Shutdown logging before to make sure there
# is no file lock (Windows).
ocgis_lh.shutdown()
if made_output_directory:
shutil.rmtree(outdir)
raise
finally:
ocgis_lh.shutdown()
def execute(self):
# check for a user-supplied output prefix
prefix = self.ops.prefix
# do directory management #
# flag to indicate a directory is made. mostly a precaution to make sure the appropriate directory is is
# removed.
made_output_directory = False
if self.ops.output_format in self._no_directory:
# No output directory for some formats.
outdir = None
else:
# Directories or a single output file(s) is created for the other cases.
if self.ops.add_auxiliary_files:
# Auxiliary files require that a directory be created.
outdir = os.path.join(self.ops.dir_output, prefix)
# Create and/or remove the output directory.
if vm.rank == 0:
if os.path.exists(outdir):
if env.OVERWRITE:
shutil.rmtree(outdir)
else:
raise IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir))
os.mkdir(outdir)
# Block until output directory is created. Most often the zero rank manages writing, but this is not a
# requirement.
vm.Barrier()
# On an exception, the output directory needs to be removed.
made_output_directory = True
else:
# with no auxiliary files the output directory will do just fine
outdir = self.ops.dir_output
try:
# configure logging ########################################################################################
progress = self._get_progress_and_configure_logging_(outdir, prefix)
# create local logger
interpreter_log = ocgis_lh.get_logger('interpreter')
ocgis_lh('Initializing...', interpreter_log)
# set up environment #######################################################################################
# run validation - doesn't do much now
self.check()
# do not perform vector wrapping for NetCDF output
if self.ops.output_format == 'nc':
ocgis_lh('"vector_wrap" set to False for netCDF output',
interpreter_log, level=logging.WARN)
self.ops.vector_wrap = False
# if the requested output format is "meta" then no operations are run and only the operations dictionary is
# required to generate output.
Converter = self.ops._get_object_(OutputFormat.name).get_converter_class()
if issubclass(Converter, AbstractMetaConverter):
ret = Converter(self.ops).write()
# this is the standard request for other output types.
else:
# the operations object performs subsetting and calculations
ocgis_lh('initializing subset', interpreter_log, level=logging.DEBUG)
so = OperationsEngine(self.ops, progress=progress)
# if there is no grouping on the output files, a singe converter is needed
if self.ops.output_grouping is None:
ocgis_lh('initializing converter', interpreter_log, level=logging.DEBUG)
conv = self._get_converter_(Converter, outdir, prefix, so)
ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format), interpreter_log,
level=logging.DEBUG)
ret = conv.write()
else:
raise NotImplementedError
ocgis_lh('Operations successful.'.format(self.ops.prefix), interpreter_log)
return ret
except:
# The output directory needs to be removed if one was created. Shutdown logging before to make sure there
# is no file lock (Windows).
ocgis_lh.shutdown()
if vm.rank == 0 and made_output_directory:
shutil.rmtree(outdir)
raise
finally:
ocgis_lh.shutdown()
if env.ADD_OPS_MPI_BARRIER:
vm.Barrier()
def execute(self):
## check for a user-supplied output prefix
prefix = self.ops.prefix
# do directory management #
# flag to indicate a directory is made. mostly a precaution to make sure the appropriate directory is is removed.
made_output_directory = False
if self.ops.output_format == 'numpy':
# no output directory for numpy output
outdir = None
else:
# directories or a single output file(s) is created for the other cases
if self.ops.add_auxiliary_files:
# auxiliary files require that a directory be created
outdir = os.path.join(self.ops.dir_output,prefix)
if os.path.exists(outdir):
if env.OVERWRITE:
shutil.rmtree(outdir)
else:
raise(IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir)))
os.mkdir(outdir)
# on an exception, the output directory needs to be removed
made_output_directory = True
else:
# with no auxiliary files the output directory will do just fine
outdir = self.ops.dir_output
try:
## configure logging ###################################################
## if file logging is enable, perform some logic based on the operational
## parameters.
if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files == True:
if self.ops.output_format == 'numpy':
to_file = None
else:
to_file = os.path.join(outdir,prefix+'.log')
else:
to_file = None
## flags to determine streaming to console
if env.VERBOSE:
to_stream = True
else:
to_stream = False
## configure the logger
if env.DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
## this wraps the callback function with methods to capture the
## completion of major operations.
progress = ProgressOcgOperations(callback=self.ops.callback)
ocgis_lh.configure(to_file=to_file,to_stream=to_stream,level=level,
callback=progress,callback_level=level)
## create local logger
interpreter_log = ocgis_lh.get_logger('interpreter')
ocgis_lh('Initializing...',interpreter_log)
## set up environment ##############################################
self.check() ## run validation - doesn't do much now
## do not perform vector wrapping for NetCDF output
if self.ops.output_format == 'nc':
ocgis_lh('"vector_wrap" set to False for netCDF output',
interpreter_log,level=logging.WARN)
self.ops.vector_wrap = False
## if the requested output format is "meta" then no operations are run
## and only the operations dictionary is required to generate output.
if self.ops.output_format == 'meta':
ret = MetaConverter(self.ops).write()
## this is the standard request for other output types.
else:
## the operations object performs subsetting and calculations
ocgis_lh('initializing subset',interpreter_log,level=logging.DEBUG)
so = SubsetOperation(self.ops,progress=progress)
## if there is no grouping on the output files, a singe converter is
## is needed
if self.ops.output_grouping is None:
Conv = AbstractConverter.get_converter(self.ops.output_format)
ocgis_lh('initializing converter',interpreter_log,
level=logging.DEBUG)
conv = Conv(so,outdir,prefix,ops=self.ops,add_auxiliary_files=self.ops.add_auxiliary_files,
overwrite=env.OVERWRITE)
ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format),interpreter_log,
level=logging.DEBUG)
ret = conv.write()
else:
raise(NotImplementedError)
ocgis_lh('Operations successful.'.format(self.ops.prefix),interpreter_log)
return ret
except:
# on an exception, the output directory needs to be removed if one was created. once the output directory is
# removed, reraise.
if made_output_directory:
shutil.rmtree(outdir)
raise
finally:
## shut down logging
ocgis_lh.shutdown()
