Example #1
0
    def test_hypothetical_operations_loop(self):

        def callback(percent, message):
            return percent, message

        n = [0, 1, 2]
        for n_subsettables, n_geometries, n_calculations in itertools.product(n, n, n):
            try:
                prog = ProgressOcgOperations(callback,
                                             n_subsettables=n_subsettables,
                                             n_geometries=n_geometries,
                                             n_calculations=n_calculations)
            except AssertionError:
                if n_geometries == 0 or n_subsettables == 0:
                    continue
                else:
                    raise

            for ns in range(n_subsettables):
                for ng in range(n_geometries):
                    for nc in range(n_calculations):
                        prog.mark()
                    if n_calculations == 0:
                        prog.mark()
            self.assertEqual(prog(), (100.0, None))
Example #2
0
 def test_init(self):
     for rb, p in itertools.product([True, False],
                                    [None, ProgressOcgOperations()]):
         sub = OperationsEngine(self.get_operations(),
                                request_base_size_only=rb,
                                progress=p)
         for ii, coll in enumerate(sub):
             self.assertIsInstance(coll, SpatialCollection)
     self.assertEqual(ii, 0)
Example #3
0
    def test_hypothetical_operations_loop(self):

        def callback(percent, message):
            return percent, message

        n = [0, 1, 2]
        for n_subsettables, n_geometries, n_calculations in itertools.product(n, n, n):
            try:
                prog = ProgressOcgOperations(callback,
                                             n_subsettables=n_subsettables,
                                             n_geometries=n_geometries,
                                             n_calculations=n_calculations)
            except AssertionError:
                if n_geometries == 0 or n_subsettables == 0:
                    continue
                else:
                    raise

            for ns in range(n_subsettables):
                for ng in range(n_geometries):
                    for nc in range(n_calculations):
                        prog.mark()
                    if n_calculations == 0:
                        prog.mark()
            self.assertEqual(prog(), (100.0, None))
Example #4
0
    def test_simple(self):
        n_geometries = 3
        n_calculations = 3

        def callback(percent, message):
            return percent, message

        for cb in [callback, None]:
            prog = ProgressOcgOperations(callback=cb,
                                         n_geometries=n_geometries,
                                         n_calculations=n_calculations)
            n_operations = 9
            self.assertEqual(prog.n_operations, n_operations)
            prog.mark()
            if cb is None:
                self.assertEqual(prog.percent_complete,
                                 100 * (1 / float(n_operations)))
                self.assertEqual(prog(), None)
            else:
                self.assertEqual(prog(),
                                 (100 * (1 / float(n_operations)), None))
            prog.mark()
            if cb is None:
                self.assertEqual(prog.percent_complete,
                                 (100 * (2 / float(n_operations))))
            else:
                self.assertEqual(prog(message='hi'),
                                 (100 * (2 / float(n_operations)), 'hi'))
Example #5
0
    def test_simple(self):
        n_geometries = 3
        n_calculations = 3

        def callback(percent, message):
            return percent, message

        for cb in [callback, None]:
            prog = ProgressOcgOperations(callback=cb, n_geometries=n_geometries, n_calculations=n_calculations)
            n_operations = 9
            self.assertEqual(prog.n_operations, n_operations)
            prog.mark()
            if cb is None:
                self.assertEqual(prog.percent_complete, 100 * (1 / float(n_operations)))
                self.assertEqual(prog(), None)
            else:
                self.assertEqual(prog(), (100 * (1 / float(n_operations)), None))
            prog.mark()
            if cb is None:
                self.assertEqual(prog.percent_complete, (100 * (2 / float(n_operations))))
            else:
                self.assertEqual(prog(message='hi'), (100 * (2 / float(n_operations)), 'hi'))
Example #6
0
    def _get_progress_and_configure_logging_(self, outdir, prefix):
        """
        :param str outdir: The output directory for the operations.
        :param str prefix: The file prefix to use when creating the output files.
        :returns: A progress object to use when executing the operations.
        :rtype: :class:`ocgis.util.logging_ocgis.ProgressOcgOperations`
        """
        # TODO: This method should use ocgis.env.configure_logging().
        # If file logging is enabled, check where or if the log should be written.
        if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files is True:
            if self.ops.output_format in self._no_directory:
                to_file = None
            else:
                if vm.rank == 0:
                    os.makedirs(os.path.join(outdir, 'logs'))
                vm.Barrier()
                to_file = os.path.join(
                    outdir, 'logs',
                    '{prefix}-rank-{rank}.log'.format(prefix=prefix,
                                                      rank=vm.rank))
        else:
            to_file = None

        # Flags to determine streaming to console.
        if env.VERBOSE:
            to_stream = True
        else:
            to_stream = False

        # Configure the logger.
        if env.DEBUG:
            level = logging.DEBUG
        else:
            level = logging.INFO
        # This wraps the callback function with methods to capture the completion of major operations.
        progress = ProgressOcgOperations(callback=self.ops.callback)
        ocgis_lh.configure(to_file=to_file,
                           to_stream=to_stream,
                           level=level,
                           callback=progress,
                           callback_level=level)

        return progress
Example #7
0
    def __init__(self, ops, request_base_size_only=False, progress=None):
        self.ops = ops
        self._request_base_size_only = request_base_size_only
        self._subset_log = ocgis_lh.get_logger('subset')
        self._progress = progress or ProgressOcgOperations()
        self._original_subcomm = deepcopy(vm.current_comm_name)
        self._backtransform = {}

        # Create the calculation engine is calculations are present.
        if self.ops.calc is None or self._request_base_size_only:
            self.cengine = None
            self._has_multivariate_calculations = False
        else:
            ocgis_lh('initializing calculation engine',
                     self._subset_log,
                     level=logging.DEBUG)
            self.cengine = CalculationEngine(
                self.ops.calc_grouping,
                self.ops.calc,
                calc_sample_size=self.ops.calc_sample_size,
                progress=self._progress,
                spatial_aggregation=self.ops.aggregate)
            self._has_multivariate_calculations = self.cengine.has_multivariate_functions
Example #8
0
 def test_constructor(self):
     for kwds in [None, {'progress': ProgressOcgOperations()}]:
         self.get_engine(kwds=kwds)
Example #9
0
 def test_constructor(self):
     prog = ProgressOcgOperations(lambda x, y: (x, y))
     self.assertEqual(prog.n_operations, 1)
     self.assertEqual(prog.n_completed_operations, 0)