def execute(targets): """Returns simulation information dervied from a set of CF files. :param list targets: File and/or directory pointers to NetCDF files, e.g. ['IPSL/IPSL-CM5B-LR']. :returns: 2 member tuple - map of simulation identifiers to dict, map of simulation identifiers to list of dates :rtype: tuple """ # Map of simulation identifiers to a list of dictionaries # (each of which describes a single file of the simulation). simulations = collections.defaultdict(list) # Map of simulation identifiers to a list of date-time objects which # collectively define the time span of the simulation. simulation_dates = collections.defaultdict(list) # ---------------------------------------------------------------- # Split the input files into groups so that all of the files in # a group belong to the same simulation # ---------------------------------------------------------------- # For each CF field in this input file ... for _, identifier, cim2_properties, dates in parser.yield_parsed(targets): cim2_properties.pop(None, None) simulation_dates[identifier].extend(dates) simulations[identifier].append(cim2_properties) # close file to prevent a proliferation of open file handles cf.close_one_file() return simulations, simulation_dates
def yield_cf_files(targets): """Yields CF files for further processing. :param str|sequence targets: Pointer(s) to file(s) and/or directorie(s). :returns: Generator yielding CF files. :rtype: generator """ for fpath in yield_files(targets): try: cf_files = cf.read(fpath, ignore_read_error=False, aggregate=False, verbose=0) except (IOError, OSError): logger.log_warning("Non netCDF file rejected: {}".format(fpath)) else: # Save the netCDF file name (from which we can extract the # dataset version) for cf_file in cf_files: cf_file.fpath = fpath yield cf_files # ... close file to prevent a proliferation of open file # handles cf.close_one_file()
def yield_parsed(targets): """Yields simulation information derived from a parse of cf files. :param str|sequence targets: Pointer(s) to file(s) and/or directorie(s). :returns: Generator yielding simulation information derived from a parse of cf files. :rtype: generator """ for cf_fields in yield_cf_files(targets): for cf_field in cf_fields: identifier, properties, dates = parse(cf_field) if identifier: yield cf_field, identifier, properties, dates # ... close file to prevent a proliferation of open file handles cf.close_one_file()
def yield_cf_files(targets): """Yields CF files for further processing. :param str|sequence targets: Pointer(s) to file(s) and/or directorie(s). :returns: Generator yielding CF files. :rtype: generator """ for fpath in yield_files(targets): try: cf_files = cf.read(fpath, ignore_read_error=False, verbose=False, aggregate=False) except (IOError, OSError): logger.log_warning("Non netCDF file rejected: {}".format(fpath)) else: # Save the netCDF file name (from which we can extract the dataset version) for cf_file in cf_files: cf_file.fpath = fpath yield cf_files # ... close file to prevent a proliferation of open file handles cf.close_one_file()