def _get_progress_and_configure_logging_(self, outdir, prefix): """ :param str outdir: The output directory for the operations. :param str prefix: The file prefix to use when creating the output files. :returns: A progress object to use when executing the operations. :rtype: :class:`ocgis.util.logging_ocgis.ProgressOcgOperations` """ # TODO: This method should use ocgis.env.configure_logging(). # If file logging is enabled, check where or if the log should be written. if env.ENABLE_FILE_LOGGING and self.ops.add_auxiliary_files is True: if self.ops.output_format in self._no_directory: to_file = None else: if vm.rank == 0: os.makedirs(os.path.join(outdir, 'logs')) vm.Barrier() to_file = os.path.join( outdir, 'logs', '{prefix}-rank-{rank}.log'.format(prefix=prefix, rank=vm.rank)) else: to_file = None # Flags to determine streaming to console. if env.VERBOSE: to_stream = True else: to_stream = False # Configure the logger. if env.DEBUG: level = logging.DEBUG else: level = logging.INFO # This wraps the callback function with methods to capture the completion of major operations. progress = ProgressOcgOperations(callback=self.ops.callback) ocgis_lh.configure(to_file=to_file, to_stream=to_stream, level=level, callback=progress, callback_level=level) return progress
def execute(self): # check for a user-supplied output prefix prefix = self.ops.prefix # do directory management # # flag to indicate a directory is made. mostly a precaution to make sure the appropriate directory is is # removed. made_output_directory = False if self.ops.output_format in self._no_directory: # No output directory for some formats. outdir = None else: # Directories or a single output file(s) is created for the other cases. if self.ops.add_auxiliary_files: # Auxiliary files require that a directory be created. outdir = os.path.join(self.ops.dir_output, prefix) # Create and/or remove the output directory. if vm.rank == 0: if os.path.exists(outdir): if env.OVERWRITE: shutil.rmtree(outdir) else: raise IOError('The output directory exists but env.OVERWRITE is False: {0}'.format(outdir)) os.mkdir(outdir) # Block until output directory is created. Most often the zero rank manages writing, but this is not a # requirement. vm.Barrier() # On an exception, the output directory needs to be removed. made_output_directory = True else: # with no auxiliary files the output directory will do just fine outdir = self.ops.dir_output try: # configure logging ######################################################################################## progress = self._get_progress_and_configure_logging_(outdir, prefix) # create local logger interpreter_log = ocgis_lh.get_logger('interpreter') ocgis_lh('Initializing...', interpreter_log) # set up environment ####################################################################################### # run validation - doesn't do much now self.check() # do not perform vector wrapping for NetCDF output if self.ops.output_format == 'nc': ocgis_lh('"vector_wrap" set to False for netCDF output', interpreter_log, level=logging.WARN) self.ops.vector_wrap = False # if the requested output format is "meta" then no operations are run and only the operations dictionary is # required to generate output. Converter = self.ops._get_object_(OutputFormat.name).get_converter_class() if issubclass(Converter, AbstractMetaConverter): ret = Converter(self.ops).write() # this is the standard request for other output types. else: # the operations object performs subsetting and calculations ocgis_lh('initializing subset', interpreter_log, level=logging.DEBUG) so = OperationsEngine(self.ops, progress=progress) # if there is no grouping on the output files, a singe converter is needed if self.ops.output_grouping is None: ocgis_lh('initializing converter', interpreter_log, level=logging.DEBUG) conv = self._get_converter_(Converter, outdir, prefix, so) ocgis_lh('starting converter write loop: {0}'.format(self.ops.output_format), interpreter_log, level=logging.DEBUG) ret = conv.write() else: raise NotImplementedError ocgis_lh('Operations successful.'.format(self.ops.prefix), interpreter_log) return ret except: # The output directory needs to be removed if one was created. Shutdown logging before to make sure there # is no file lock (Windows). ocgis_lh.shutdown() if vm.rank == 0 and made_output_directory: shutil.rmtree(outdir) raise finally: ocgis_lh.shutdown() if env.ADD_OPS_MPI_BARRIER: vm.Barrier()