def test_hypothetical_operations_loop(self): def callback(percent, message): return percent, message n = [0, 1, 2] for n_subsettables, n_geometries, n_calculations in itertools.product(n, n, n): try: prog = ProgressOcgOperations(callback, n_subsettables=n_subsettables, n_geometries=n_geometries, n_calculations=n_calculations) except AssertionError: if n_geometries == 0 or n_subsettables == 0: continue else: raise for ns in range(n_subsettables): for ng in range(n_geometries): for nc in range(n_calculations): prog.mark() if n_calculations == 0: prog.mark() self.assertEqual(prog(), (100.0, None))
def test_init(self): for rb, p in itertools.product([True, False], [None, ProgressOcgOperations()]): sub = OperationsEngine(self.get_operations(), request_base_size_only=rb, progress=p) for ii, coll in enumerate(sub): self.assertIsInstance(coll, SpatialCollection) self.assertEqual(ii, 0)
def test_hypothetical_operations_loop(self): def callback(percent, message): return percent, message n = [0, 1, 2] for n_subsettables, n_geometries, n_calculations in itertools.product(n, n, n): try: prog = ProgressOcgOperations(callback, n_subsettables=n_subsettables, n_geometries=n_geometries, n_calculations=n_calculations) except AssertionError: if n_geometries == 0 or n_subsettables == 0: continue else: raise for ns in range(n_subsettables): for ng in range(n_geometries): for nc in range(n_calculations): prog.mark() if n_calculations == 0: prog.mark() self.assertEqual(prog(), (100.0, None))
def test_simple(self): n_geometries = 3 n_calculations = 3 def callback(percent, message): return percent, message for cb in [callback, None]: prog = ProgressOcgOperations(callback=cb, n_geometries=n_geometries, n_calculations=n_calculations) n_operations = 9 self.assertEqual(prog.n_operations, n_operations) prog.mark() if cb is None: self.assertEqual(prog.percent_complete, 100 * (1 / float(n_operations))) self.assertEqual(prog(), None) else: self.assertEqual(prog(), (100 * (1 / float(n_operations)), None)) prog.mark() if cb is None: self.assertEqual(prog.percent_complete, (100 * (2 / float(n_operations)))) else: self.assertEqual(prog(message='hi'), (100 * (2 / float(n_operations)), 'hi'))
def test_simple(self): n_geometries = 3 n_calculations = 3 def callback(percent, message): return percent, message for cb in [callback, None]: prog = ProgressOcgOperations(callback=cb, n_geometries=n_geometries, n_calculations=n_calculations) n_operations = 9 self.assertEqual(prog.n_operations, n_operations) prog.mark() if cb is None: self.assertEqual(prog.percent_complete, 100 * (1 / float(n_operations))) self.assertEqual(prog(), None) else: self.assertEqual(prog(), (100 * (1 / float(n_operations)), None)) prog.mark() if cb is None: self.assertEqual(prog.percent_complete, (100 * (2 / float(n_operations)))) else: self.assertEqual(prog(message='hi'), (100 * (2 / float(n_operations)), 'hi'))
def _get_progress_and_configure_logging_(self, outdir, prefix): """ :param str outdir: The output directory for the operations. :param str prefix: The file prefix to use when creating the output files. :returns: A progress object to use when executing the operations. :rtype: :class:`ocgis.util.logging_ocgis.ProgressOcgOperations` """ # TODO: This method should use ocgis.env.configure_logging(). # If file logging is enabled, check where or if the log should be written. if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files is True: if self.ops.output_format in self._no_directory: to_file = None else: if vm.rank == 0: os.makedirs(os.path.join(outdir, 'logs')) vm.Barrier() to_file = os.path.join( outdir, 'logs', '{prefix}-rank-{rank}.log'.format(prefix=prefix, rank=vm.rank)) else: to_file = None # Flags to determine streaming to console. if env.VERBOSE: to_stream = True else: to_stream = False # Configure the logger. if env.DEBUG: level = logging.DEBUG else: level = logging.INFO # This wraps the callback function with methods to capture the completion of major operations. progress = ProgressOcgOperations(callback=self.ops.callback) ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level, callback=progress, callback_level=level) return progress
def __init__(self, ops, request_base_size_only=False, progress=None): self.ops = ops self._request_base_size_only = request_base_size_only self._subset_log = ocgis_lh.get_logger('subset') self._progress = progress or ProgressOcgOperations() self._original_subcomm = deepcopy(vm.current_comm_name) self._backtransform = {} # Create the calculation engine is calculations are present. if self.ops.calc is None or self._request_base_size_only: self.cengine = None self._has_multivariate_calculations = False else: ocgis_lh('initializing calculation engine', self._subset_log, level=logging.DEBUG) self.cengine = CalculationEngine( self.ops.calc_grouping, self.ops.calc, calc_sample_size=self.ops.calc_sample_size, progress=self._progress, spatial_aggregation=self.ops.aggregate) self._has_multivariate_calculations = self.cengine.has_multivariate_functions
def test_constructor(self): for kwds in [None, {'progress': ProgressOcgOperations()}]: self.get_engine(kwds=kwds)
def test_constructor(self): prog = ProgressOcgOperations(lambda x, y: (x, y)) self.assertEqual(prog.n_operations, 1) self.assertEqual(prog.n_completed_operations, 0)