def test_hypothetical_operations_loop(self):
def callback(percent, message):
return percent, message
n = [0, 1, 2]
for n_subsettables, n_geometries, n_calculations in itertools.product(n, n, n):
try:
prog = ProgressOcgOperations(callback,
n_subsettables=n_subsettables,
n_geometries=n_geometries,
n_calculations=n_calculations)
except AssertionError:
if n_geometries == 0 or n_subsettables == 0:
continue
else:
raise
for ns in range(n_subsettables):
for ng in range(n_geometries):
for nc in range(n_calculations):
prog.mark()
if n_calculations == 0:
prog.mark()
self.assertEqual(prog(), (100.0, None))
def test_init(self):
for rb, p in itertools.product([True, False],
[None, ProgressOcgOperations()]):
sub = OperationsEngine(self.get_operations(),
request_base_size_only=rb,
progress=p)
for ii, coll in enumerate(sub):
self.assertIsInstance(coll, SpatialCollection)
self.assertEqual(ii, 0)
def test_hypothetical_operations_loop(self):
def callback(percent, message):
return percent, message
n = [0, 1, 2]
for n_subsettables, n_geometries, n_calculations in itertools.product(n, n, n):
try:
prog = ProgressOcgOperations(callback,
n_subsettables=n_subsettables,
n_geometries=n_geometries,
n_calculations=n_calculations)
except AssertionError:
if n_geometries == 0 or n_subsettables == 0:
continue
else:
raise
for ns in range(n_subsettables):
for ng in range(n_geometries):
for nc in range(n_calculations):
prog.mark()
if n_calculations == 0:
prog.mark()
self.assertEqual(prog(), (100.0, None))
def test_simple(self):
n_geometries = 3
n_calculations = 3
def callback(percent, message):
return percent, message
for cb in [callback, None]:
prog = ProgressOcgOperations(callback=cb,
n_geometries=n_geometries,
n_calculations=n_calculations)
n_operations = 9
self.assertEqual(prog.n_operations, n_operations)
prog.mark()
if cb is None:
self.assertEqual(prog.percent_complete,
100 * (1 / float(n_operations)))
self.assertEqual(prog(), None)
else:
self.assertEqual(prog(),
(100 * (1 / float(n_operations)), None))
prog.mark()
if cb is None:
self.assertEqual(prog.percent_complete,
(100 * (2 / float(n_operations))))
else:
self.assertEqual(prog(message='hi'),
(100 * (2 / float(n_operations)), 'hi'))
def test_simple(self):
n_geometries = 3
n_calculations = 3
def callback(percent, message):
return percent, message
for cb in [callback, None]:
prog = ProgressOcgOperations(callback=cb, n_geometries=n_geometries, n_calculations=n_calculations)
n_operations = 9
self.assertEqual(prog.n_operations, n_operations)
prog.mark()
if cb is None:
self.assertEqual(prog.percent_complete, 100 * (1 / float(n_operations)))
self.assertEqual(prog(), None)
else:
self.assertEqual(prog(), (100 * (1 / float(n_operations)), None))
prog.mark()
if cb is None:
self.assertEqual(prog.percent_complete, (100 * (2 / float(n_operations))))
else:
self.assertEqual(prog(message='hi'), (100 * (2 / float(n_operations)), 'hi'))
def _get_progress_and_configure_logging_(self, outdir, prefix):
"""
:param str outdir: The output directory for the operations.
:param str prefix: The file prefix to use when creating the output files.
:returns: A progress object to use when executing the operations.
:rtype: :class:`ocgis.util.logging_ocgis.ProgressOcgOperations`
"""
# TODO: This method should use ocgis.env.configure_logging().
# If file logging is enabled, check where or if the log should be written.
if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files is True:
if self.ops.output_format in self._no_directory:
to_file = None
else:
if vm.rank == 0:
os.makedirs(os.path.join(outdir, 'logs'))
vm.Barrier()
to_file = os.path.join(
outdir, 'logs',
'{prefix}-rank-{rank}.log'.format(prefix=prefix,
rank=vm.rank))
else:
to_file = None
# Flags to determine streaming to console.
if env.VERBOSE:
to_stream = True
else:
to_stream = False
# Configure the logger.
if env.DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
# This wraps the callback function with methods to capture the completion of major operations.
progress = ProgressOcgOperations(callback=self.ops.callback)
ocgis_lh.configure(to_file=to_file,
to_stream=to_stream,
level=level,
callback=progress,
callback_level=level)
return progress
def __init__(self, ops, request_base_size_only=False, progress=None):
self.ops = ops
self._request_base_size_only = request_base_size_only
self._subset_log = ocgis_lh.get_logger('subset')
self._progress = progress or ProgressOcgOperations()
self._original_subcomm = deepcopy(vm.current_comm_name)
self._backtransform = {}
# Create the calculation engine is calculations are present.
if self.ops.calc is None or self._request_base_size_only:
self.cengine = None
self._has_multivariate_calculations = False
else:
ocgis_lh('initializing calculation engine',
self._subset_log,
level=logging.DEBUG)
self.cengine = CalculationEngine(
self.ops.calc_grouping,
self.ops.calc,
calc_sample_size=self.ops.calc_sample_size,
progress=self._progress,
spatial_aggregation=self.ops.aggregate)
self._has_multivariate_calculations = self.cengine.has_multivariate_functions
def test_constructor(self):
for kwds in [None, {'progress': ProgressOcgOperations()}]:
self.get_engine(kwds=kwds)
def test_constructor(self):
prog = ProgressOcgOperations(lambda x, y: (x, y))
self.assertEqual(prog.n_operations, 1)
self.assertEqual(prog.n_completed_operations, 0)
