def test_system_with_callback(self): fp = get_temp_path(wd=self.current_dir_output) def callback(message, path=fp): with open(path, 'a') as sink: sink.write(message) sink.write('\n') class FooError(Exception): pass ocgis_lh.configure(callback=callback) ocgis_lh(msg='this is a test message') ocgis_lh() ocgis_lh(msg='this is a second test message') ocgis_lh(msg='this should not be there', level=logging.DEBUG) exc = FooError('foo message for value error') try: ocgis_lh(exc=exc) except FooError: pass with open(fp, 'r') as source: lines = source.readlines() self.assertEqual(lines, ['this is a test message\n', 'this is a second test message\n', 'FooError: foo message for value error\n'])
def test_system_combinations(self): _to_stream = [ True, False ] _to_file = [ os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log'), None ] _level = [logging.INFO, logging.DEBUG, logging.WARN] for ii, (to_file, to_stream, level) in enumerate(itertools.product(_to_file, _to_stream, _level)): ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level) try: ocgis_lh(ii) ocgis_lh('a test message') subset = ocgis_lh.get_logger('subset') interp = ocgis_lh.get_logger('interp') ocgis_lh('a subset message', logger=subset) ocgis_lh('an interp message', logger=interp) ocgis_lh('a general message', alias='foo', ugid=10) ocgis_lh('another message', level=level) if to_file is not None: self.assertTrue(os.path.exists(to_file)) os.remove(to_file) finally: logging.shutdown()
def configure_logging(self, with_header=True): from ocgis.util.logging_ocgis import ocgis_lh # If file logging is enabled, check where or if the log should be written. if self.ENABLE_FILE_LOGGING: raise NotImplementedError else: to_file = None # Flags to determine streaming to console. if env.VERBOSE: to_stream = True else: to_stream = False # Configure the logger. if self.DEBUG: level = logging.DEBUG else: level = logging.INFO # This wraps the callback function with methods to capture the completion of major operations. ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level, with_header=with_header # callback=progress, callback_level=level, )
def _get_progress_and_configure_logging_(self, outdir, prefix): """ :param str outdir: The output directory for the operations. :param str prefix: The file prefix to use when creating the output files. :returns: A progress object to use when executing the operations. :rtype: :class:`ocgis.util.logging_ocgis.ProgressOcgOperations` """ # If file logging is enabled, check where or if the log should be written. if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files is True: if self.ops.output_format in self._no_directory: to_file = None else: to_file = os.path.join(outdir, prefix + '.log') else: to_file = None # Flags to determine streaming to console. if env.VERBOSE: to_stream = True else: to_stream = False # Configure the logger. if env.DEBUG: level = logging.DEBUG else: level = logging.INFO # This wraps the callback function with methods to capture the completion of major operations. progress = ProgressOcgOperations(callback=self.ops.callback) ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level, callback=progress, callback_level=level) return progress
def _run_(): ocgis_lh.configure() self.assertTrue(ocgis_lh.null) env.SUPPRESS_WARNINGS = True ocgis_lh(level=logging.WARNING, exc=RuntimeWarning('show me'), force=True) env.reset()
def test_system_exc(self): to_file = os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log') to_stream = False ocgis_lh.configure(to_file=to_file, to_stream=to_stream) try: raise ValueError except Exception as e: with self.assertRaises(ValueError): ocgis_lh('something happened', exc=e)
def test_exc(self): to_file = os.path.join(env.DIR_OUTPUT, "test_ocgis_log.log") to_stream = False ocgis_lh.configure(to_file=to_file, to_stream=to_stream) try: raise (ValueError("some exception information")) except Exception as e: with self.assertRaises(ValueError): ocgis_lh("something happened", exc=e)
def test_system_simple(self): to_file = os.path.join(env.DIR_OUTPUT, 'test_ocgis_log.log') to_stream = False ocgis_lh.configure(to_file, to_stream) ocgis_lh('a test message') subset = ocgis_lh.get_logger('subset') subset.info('a subset message')
def _run_(): logpath = self.get_temporary_file_path('foo.log') ocgis_lh.configure(to_file=logpath) ocgis_lh(msg='oh my', level=logging.WARN) with open(logpath, 'r') as f: lines = f.readlines() lines = ''.join(lines) self.assertIn('OcgWarning', lines) self.assertIn('oh my', lines)
def test_simple(self): to_file = os.path.join(env.DIR_OUTPUT, "test_ocgis_log.log") to_stream = False ocgis_lh.configure(to_file, to_stream) ocgis_lh("a test message") subset = ocgis_lh.get_logger("subset") subset.info("a subset message")
def _run_(): env.SUPPRESS_WARNINGS = False logpath = self.get_temporary_file_path('ocgis.log') ocgis_lh.configure(to_file=logpath) exc = FutureWarning('something is about to happen') ocgis_lh(level=logging.WARNING, exc=exc) with open(logpath, 'r') as f: lines = f.readlines() lines = ''.join(lines) self.assertIn('FutureWarning', lines) self.assertIn('something is about to happen', lines) env.SUPPRESS_WARNINGS = True
def _run_(): env.SUPPRESS_WARNINGS = False ocgis_lh.configure(to_stream=True) records = [{ 'geom': Point(1, 2), 'properties': { 'a_list': [1, 2, 3] } }] actual = Field.from_records(records) self.assertNotIn("a_list", actual.keys()) env.SUPPRESS_WARNINGS = True
def test_get_dimension_map_3(self): """Test when bounds are found but the bounds variable is actually missing.""" _, to_file = tempfile.mkstemp(dir=self._test_dir) ocgis_lh.configure(to_file=to_file) try: # remove the bounds variable from a standard metadata dictionary rd = self.test_data.get_rd('cancm4_tas') metadata = deepcopy(rd.source_metadata) metadata['variables'].pop('lat_bnds') dim_map = get_dimension_map('tas', metadata) self.assertEqual(dim_map['Y']['bounds'], None) self.assertTrue('lat_bnds' in list(ocgis_lh.duplicates)[0]) finally: ocgis_lh.shutdown()
def _get_progress_and_configure_logging_(self, outdir, prefix): """ :param str outdir: The output directory for the operations. :param str prefix: The file prefix to use when creating the output files. :returns: A progress object to use when executing the operations. :rtype: :class:`ocgis.util.logging_ocgis.ProgressOcgOperations` """ # TODO: This method should use ocgis.env.configure_logging(). # If file logging is enabled, check where or if the log should be written. if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files is True: if self.ops.output_format in self._no_directory: to_file = None else: if vm.rank == 0: os.makedirs(os.path.join(outdir, 'logs')) vm.Barrier() to_file = os.path.join( outdir, 'logs', '{prefix}-rank-{rank}.log'.format(prefix=prefix, rank=vm.rank)) else: to_file = None # Flags to determine streaming to console. if env.VERBOSE: to_stream = True else: to_stream = False # Configure the logger. if env.DEBUG: level = logging.DEBUG else: level = logging.INFO # This wraps the callback function with methods to capture the completion of major operations. progress = ProgressOcgOperations(callback=self.ops.callback) ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level, callback=progress, callback_level=level) return progress
def chunked_rwg(source, destination, weight, nchunks_dst, merge, esmf_src_type, esmf_dst_type, genweights, esmf_regrid_method, spatial_subset, src_resolution, dst_resolution, buffer_distance, wd, persist, eager, ignore_degenerate, data_variables, spatial_subset_path, verbose, loglvl, weightfilemode): # Used for creating the history string. the_locals = locals() if verbose: ocgis_lh.configure(to_stream=True, level=getattr(logging, loglvl)) ocgis_lh(msg="Starting Chunked Regrid Weight Generation", level=logging.INFO, logger=CRWG_LOG) if not ocgis.env.USE_NETCDF4_MPI: msg = ( 'env.USE_NETCDF4_MPI is False. Considerable performance gains are possible if this is True. Is ' 'netCDF4-python built with parallel support?') ocgis_lh(msg, level=logging.WARN, logger=CRWG_LOG, force=True) if data_variables is not None: data_variables = data_variables.split(',') if nchunks_dst is not None: # Format the chunking decomposition from its string representation. if ',' in nchunks_dst: nchunks_dst = nchunks_dst.split(',') else: nchunks_dst = [nchunks_dst] nchunks_dst = tuple([int(ii) for ii in nchunks_dst]) if merge: if not spatial_subset and weight is None: raise ValueError('"weight" must be a valid path if --merge') if spatial_subset and genweights and weight is None: raise ValueError('"weight" must be a valid path if --genweights') # Make a temporary working directory is one is not provided by the client. Only do this if we are writing subsets # and it is not a merge only operation. should_create_wd = (nchunks_dst is None or not all([ii == 1 for ii in nchunks_dst])) or spatial_subset if should_create_wd: if wd is None: if ocgis.vm.rank == 0: wd = tempfile.mkdtemp(prefix='ocgis_chunked_rwg_') wd = ocgis.vm.bcast(wd) else: exc = None if ocgis.vm.rank == 0: # The working directory must not exist to proceed. if nchunks_dst is not None: if os.path.exists(wd): exc = ValueError( "Working directory {} must not exist.".format(wd)) else: # Make the working directory nesting as needed. os.makedirs(wd) exc = ocgis.vm.bcast(exc) if exc is not None: raise exc if merge and not spatial_subset or (spatial_subset and genweights): if _is_subdir_(wd, weight): raise ValueError( 'Merge weight file path must not in the working directory. It may get unintentionally deleted with the --no_persist flag.' ) # Create the source and destination request datasets. rd_src = _create_request_dataset_(source, esmf_src_type, data_variables=data_variables) rd_dst = _create_request_dataset_(destination, esmf_dst_type) # Execute a spatial subset if requested. paths = None if spatial_subset: if spatial_subset_path is None: spatial_subset_path = os.path.join(wd, 'spatial_subset.nc') msg = "Executing spatial subset. Output path is: {}".format( spatial_subset_path) ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG) _write_spatial_subset_(rd_src, rd_dst, spatial_subset_path, src_resmax=src_resolution) # Only split grids if a spatial subset is not requested. else: # Update the paths to use for the grid. paths = {'wd': wd} # Arguments to ESMF regridding. esmf_kwargs = { 'regrid_method': esmf_regrid_method, 'ignore_degenerate': ignore_degenerate } # Create the chunked regridding object. This is used for both chunked regridding and a regrid with a spatial subset. gs = GridChunker(rd_src, rd_dst, nchunks_dst=nchunks_dst, src_grid_resolution=src_resolution, paths=paths, dst_grid_resolution=dst_resolution, buffer_value=buffer_distance, redistribute=True, genweights=genweights, esmf_kwargs=esmf_kwargs, use_spatial_decomp='auto', eager=eager) # Write subsets and generate weights if requested in the grid splitter. # TODO: Need a weight only option. If chunks are written, then weights are written... if not spatial_subset and nchunks_dst is not None and not gs.is_one_chunk: msg = "Starting main chunking loop..." ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG) gs.write_chunks() else: if spatial_subset: source = spatial_subset_path if genweights: msg = "Writing ESMF weights..." ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG) handle_weight_file_check(weight) gs.write_esmf_weights(source, destination, weight, filemode=weightfilemode) # Create the global weight file. This does not apply to spatial subsets because there will always be one weight # file. if merge and not spatial_subset and not gs.is_one_chunk: # Weight file merge only works in serial. exc = None with ocgis.vm.scoped('weight file merge', [0]): if not ocgis.vm.is_null: msg = "Merging chunked weight files to global file. Output global weight file is: {}".format( weight) ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG) handle_weight_file_check(weight) gs.create_merged_weight_file(weight) excs = ocgis.vm.gather(exc) excs = ocgis.vm.bcast(excs) for exc in excs: if exc is not None: raise exc ocgis.vm.barrier() # Append the history string if there is an output weight file. if weight and ocgis.vm.rank == 0: if os.path.exists(weight): # Add some additional stuff for record keeping import getpass import socket import datetime with nc.Dataset(weight, 'a') as ds: ds.setncattr('created_by_user', getpass.getuser()) ds.setncattr('created_on_hostname', socket.getfqdn()) ds.setncattr('history', create_history_string(the_locals)) ocgis.vm.barrier() # Remove the working directory unless the persist flag is provided. if not persist: if ocgis.vm.rank == 0: msg = "Removing working directory since persist is False." ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG) shutil.rmtree(wd) ocgis.vm.barrier() ocgis_lh(msg="Success!", level=logging.INFO, logger=CRWG_LOG) return 0
def execute(self): ## check for a user-supplied output prefix prefix = self.ops.prefix # do directory management # # flag to indicate a directory is made. mostly a precaution to make sure the appropriate directory is is removed. made_output_directory = False if self.ops.output_format == 'numpy': # no output directory for numpy output outdir = None else: # directories or a single output file(s) is created for the other cases if self.ops.add_auxiliary_files: # auxiliary files require that a directory be created outdir = os.path.join(self.ops.dir_output,prefix) if os.path.exists(outdir): if env.OVERWRITE: shutil.rmtree(outdir) else: raise(IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir))) os.mkdir(outdir) # on an exception, the output directory needs to be removed made_output_directory = True else: # with no auxiliary files the output directory will do just fine outdir = self.ops.dir_output try: ## configure logging ################################################### ## if file logging is enable, perform some logic based on the operational ## parameters. if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files == True: if self.ops.output_format == 'numpy': to_file = None else: to_file = os.path.join(outdir,prefix+'.log') else: to_file = None ## flags to determine streaming to console if env.VERBOSE: to_stream = True else: to_stream = False ## configure the logger if env.DEBUG: level = logging.DEBUG else: level = logging.INFO ## this wraps the callback function with methods to capture the ## completion of major operations. progress = ProgressOcgOperations(callback=self.ops.callback) ocgis_lh.configure(to_file=to_file,to_stream=to_stream,level=level, callback=progress,callback_level=level) ## create local logger interpreter_log = ocgis_lh.get_logger('interpreter') ocgis_lh('Initializing...',interpreter_log) ## set up environment ############################################## self.check() ## run validation - doesn't do much now ## do not perform vector wrapping for NetCDF output if self.ops.output_format == 'nc': ocgis_lh('"vector_wrap" set to False for netCDF output', interpreter_log,level=logging.WARN) self.ops.vector_wrap = False ## if the requested output format is "meta" then no operations are run ## and only the operations dictionary is required to generate output. if self.ops.output_format == 'meta': ret = MetaConverter(self.ops).write() ## this is the standard request for other output types. else: ## the operations object performs subsetting and calculations ocgis_lh('initializing subset',interpreter_log,level=logging.DEBUG) so = SubsetOperation(self.ops,progress=progress) ## if there is no grouping on the output files, a singe converter is ## is needed if self.ops.output_grouping is None: Conv = AbstractConverter.get_converter(self.ops.output_format) ocgis_lh('initializing converter',interpreter_log, level=logging.DEBUG) conv = Conv(so,outdir,prefix,ops=self.ops,add_auxiliary_files=self.ops.add_auxiliary_files, overwrite=env.OVERWRITE) ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format),interpreter_log, level=logging.DEBUG) ret = conv.write() else: raise(NotImplementedError) ocgis_lh('Operations successful.'.format(self.ops.prefix),interpreter_log) return ret except: # on an exception, the output directory needs to be removed if one was created. once the output directory is # removed, reraise. if made_output_directory: shutil.rmtree(outdir) raise finally: ## shut down logging ocgis_lh.shutdown()
def chunked_rwg(source, destination, weight, nchunks_dst, merge, esmf_src_type, esmf_dst_type, genweights, esmf_regrid_method, spatial_subset, src_resolution, dst_resolution, buffer_distance, wd, persist, eager, ignore_degenerate, data_variables, spatial_subset_path, verbose, loglvl): if verbose: ocgis_lh.configure(to_stream=True, level=getattr(logging, loglvl)) ocgis_lh(msg="Starting Chunked Regrid Weight Generation", level=logging.INFO, logger=CRWG_LOG) if not ocgis.env.USE_NETCDF4_MPI: msg = ('env.USE_NETCDF4_MPI is False. Considerable performance gains are possible if this is True. Is ' 'netCDF4-python built with parallel support?') ocgis_lh(msg, level=logging.WARN, logger=CRWG_LOG, force=True) if data_variables is not None: data_variables = data_variables.split(',') if nchunks_dst is not None: # Format the chunking decomposition from its string representation. if ',' in nchunks_dst: nchunks_dst = nchunks_dst.split(',') else: nchunks_dst = [nchunks_dst] nchunks_dst = tuple([int(ii) for ii in nchunks_dst]) if merge: if not spatial_subset and weight is None: raise ValueError('"weight" must be a valid path if --merge') if spatial_subset and genweights and weight is None: raise ValueError('"weight" must be a valid path if --genweights') # Make a temporary working directory is one is not provided by the client. Only do this if we are writing subsets # and it is not a merge only operation. if wd is None: if ocgis.vm.rank == 0: wd = tempfile.mkdtemp(prefix='ocgis_chunked_rwg_') wd = ocgis.vm.bcast(wd) else: exc = None if ocgis.vm.rank == 0: # The working directory must not exist to proceed. if os.path.exists(wd): exc = ValueError("Working directory {} must not exist.".format(wd)) else: # Make the working directory nesting as needed. os.makedirs(wd) exc = ocgis.vm.bcast(exc) if exc is not None: raise exc if merge and not spatial_subset or (spatial_subset and genweights): if _is_subdir_(wd, weight): raise ValueError( 'Merge weight file path must not in the working directory. It may get unintentionally deleted with the --no_persist flag.') # Create the source and destination request datasets. rd_src = _create_request_dataset_(source, esmf_src_type, data_variables=data_variables) rd_dst = _create_request_dataset_(destination, esmf_dst_type) # Execute a spatial subset if requested. paths = None if spatial_subset: if spatial_subset_path is None: spatial_subset_path = os.path.join(wd, 'spatial_subset.nc') msg = "Executing spatial subset. Output path is: {}".format(spatial_subset_path) ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG) _write_spatial_subset_(rd_src, rd_dst, spatial_subset_path, src_resmax=src_resolution) # Only split grids if a spatial subset is not requested. else: # Update the paths to use for the grid. paths = {'wd': wd} # Arguments to ESMF regridding. esmf_kwargs = {'regrid_method': esmf_regrid_method, 'ignore_degenerate': ignore_degenerate} # Create the chunked regridding object. This is used for both chunked regridding and a regrid with a spatial subset. gs = GridChunker(rd_src, rd_dst, nchunks_dst=nchunks_dst, src_grid_resolution=src_resolution, paths=paths, dst_grid_resolution=dst_resolution, buffer_value=buffer_distance, redistribute=True, genweights=genweights, esmf_kwargs=esmf_kwargs, use_spatial_decomp='auto', eager=eager) # Write subsets and generate weights if requested in the grid splitter. # TODO: Need a weight only option. If chunks are written, then weights are written... if not spatial_subset and nchunks_dst is not None: msg = "Starting main chunking loop..." ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG) gs.write_chunks() else: if spatial_subset: source = spatial_subset_path if genweights: msg = "Writing ESMF weights..." ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG) gs.write_esmf_weights(source, destination, weight) # Create the global weight file. This does not apply to spatial subsets because there will always be one weight # file. if merge and not spatial_subset: # Weight file merge only works in serial. exc = None with ocgis.vm.scoped('weight file merge', [0]): if not ocgis.vm.is_null: msg = "Merging chunked weight files to global file. Output global weight file is: {}".format(weight) ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG) gs.create_merged_weight_file(weight) excs = ocgis.vm.gather(exc) excs = ocgis.vm.bcast(excs) for exc in excs: if exc is not None: raise exc ocgis.vm.barrier() # Remove the working directory unless the persist flag is provided. if not persist: if ocgis.vm.rank == 0: msg = "Removing working directory since persist is False." ocgis_lh(msg=msg, level=logging.INFO, logger=CRWG_LOG) shutil.rmtree(wd) ocgis.vm.barrier() ocgis_lh(msg="Success!", level=logging.INFO, logger=CRWG_LOG) return 0
def execute(self): ## check for a user-supplied output prefix prefix = self.ops.prefix ## do directory management. if self.ops.output_format == 'numpy': outdir = None else: outdir = os.path.join(self.ops.dir_output,prefix) if os.path.exists(outdir): if env.OVERWRITE: shutil.rmtree(outdir) else: raise(IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir))) os.mkdir(outdir) try: ## configure logging ################################################### ## if file logging is enable, perform some logic based on the operational ## parameters. if env.ENABLE_FILE_LOGGING: if self.ops.output_format == 'numpy': to_file = None else: to_file = os.path.join(outdir,prefix+'.log') else: to_file = None ## flags to determine streaming to console if env.VERBOSE: to_stream = True else: to_stream = False ## configure the logger if env.DEBUG: level = logging.DEBUG else: level = logging.INFO ocgis_lh.configure(to_file=to_file,to_stream=to_stream,level=level) ## create local logger interpreter_log = ocgis_lh.get_logger('interpreter') ocgis_lh('executing: {0}'.format(self.ops.prefix),interpreter_log) ## set up environment ############################################## self.check() ## run validation - doesn't do much now ## do not perform vector wrapping for NetCDF output if self.ops.output_format == 'nc': ocgis_lh('"vector_wrap" set to False for netCDF output', interpreter_log,level=logging.WARN) self.ops.vector_wrap = False ## if the requested output format is "meta" then no operations are run ## and only the operations dictionary is required to generate output. if self.ops.output_format == 'meta': ret = MetaConverter(self.ops).write() ## this is the standard request for other output types. else: ## the operations object performs subsetting and calculations ocgis_lh('initializing subset',interpreter_log,level=logging.DEBUG) so = SubsetOperation(self.ops,serial=env.SERIAL,nprocs=env.CORES) ## if there is no grouping on the output files, a singe converter is ## is needed if self.ops.output_grouping is None: Conv = OcgConverter.get_converter(self.ops.output_format) ocgis_lh('initializing converter',interpreter_log, level=logging.DEBUG) conv = Conv(so,outdir,prefix,ops=self.ops) ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format),interpreter_log, level=logging.DEBUG) ret = conv.write() else: raise(NotImplementedError) ocgis_lh('execution complete: {0}'.format(self.ops.prefix),interpreter_log) return(ret) finally: ## shut down logging ocgis_lh.shutdown()
def test_configure(self): # test suppressing warnings in the logger self.assertIsNone(logging._warnings_showwarning) env.SUPPRESS_WARNINGS = False ocgis_lh.configure() self.assertFalse(logging._warnings_showwarning)
def test_system_parallel(self): to_file = os.path.join(self.current_dir_output, 'rank-{}-test_ocgis_log.log'.format(vm.rank)) ocgis_lh.configure(to_file=to_file) ocgis_lh("something happened") self.assertEqual(len(os.listdir(self.current_dir_output)), vm.size)
def test_shutdown(self): env.SUPPRESS_WARNINGS = False ocgis_lh.configure(to_stream=True) self.assertFalse(logging._warnings_showwarning) ocgis_lh.shutdown() self.assertIsNone(logging._warnings_showwarning)
def _run_(): env.SUPPRESS_WARNINGS = False logpath = self.get_temporary_file_path('foo.log') ocgis_lh.configure(to_file=logpath) ocgis_lh(msg='hey there', level=logging.WARN) env.SUPPRESS_WARNINGS = True