def execute(targets):
"""Returns simulation information dervied from a set of CF files.
:param list targets: File and/or directory pointers to NetCDF files, e.g. ['IPSL/IPSL-CM5B-LR'].
:returns: 2 member tuple - map of simulation identifiers to dict,
map of simulation identifiers to list of dates
:rtype: tuple
"""
# Map of simulation identifiers to a list of dictionaries
# (each of which describes a single file of the simulation).
simulations = collections.defaultdict(list)
# Map of simulation identifiers to a list of date-time objects which
# collectively define the time span of the simulation.
simulation_dates = collections.defaultdict(list)
# ----------------------------------------------------------------
# Split the input files into groups so that all of the files in
# a group belong to the same simulation
# ----------------------------------------------------------------
# For each CF field in this input file ...
for _, identifier, cim2_properties, dates in parser.yield_parsed(targets):
cim2_properties.pop(None, None)
simulation_dates[identifier].extend(dates)
simulations[identifier].append(cim2_properties)
# close file to prevent a proliferation of open file handles
cf.close_one_file()
return simulations, simulation_dates
def yield_cf_files(targets):
"""Yields CF files for further processing.
:param str|sequence targets: Pointer(s) to file(s) and/or directorie(s).
:returns: Generator yielding CF files.
:rtype: generator
"""
for fpath in yield_files(targets):
try:
cf_files = cf.read(fpath,
ignore_read_error=False,
aggregate=False,
verbose=0)
except (IOError, OSError):
logger.log_warning("Non netCDF file rejected: {}".format(fpath))
else:
# Save the netCDF file name (from which we can extract the
# dataset version)
for cf_file in cf_files:
cf_file.fpath = fpath
yield cf_files
# ... close file to prevent a proliferation of open file
# handles
cf.close_one_file()
def execute(targets):
"""Returns simulation information dervied from a set of CF files.
:param list targets: File and/or directory pointers to NetCDF files, e.g. ['IPSL/IPSL-CM5B-LR'].
:returns: 2 member tuple - map of simulation identifiers to dict,
map of simulation identifiers to list of dates
:rtype: tuple
"""
# Map of simulation identifiers to a list of dictionaries
# (each of which describes a single file of the simulation).
simulations = collections.defaultdict(list)
# Map of simulation identifiers to a list of date-time objects which
# collectively define the time span of the simulation.
simulation_dates = collections.defaultdict(list)
# ----------------------------------------------------------------
# Split the input files into groups so that all of the files in
# a group belong to the same simulation
# ----------------------------------------------------------------
# For each CF field in this input file ...
for _, identifier, cim2_properties, dates in parser.yield_parsed(targets):
cim2_properties.pop(None, None)
simulation_dates[identifier].extend(dates)
simulations[identifier].append(cim2_properties)
# close file to prevent a proliferation of open file handles
cf.close_one_file()
return simulations, simulation_dates
def yield_parsed(targets):
"""Yields simulation information derived from a parse of cf files.
:param str|sequence targets: Pointer(s) to file(s) and/or directorie(s).
:returns: Generator yielding simulation information derived from a parse of cf files.
:rtype: generator
"""
for cf_fields in yield_cf_files(targets):
for cf_field in cf_fields:
identifier, properties, dates = parse(cf_field)
if identifier:
yield cf_field, identifier, properties, dates
# ... close file to prevent a proliferation of open file handles
cf.close_one_file()
def yield_parsed(targets):
"""Yields simulation information derived from a parse of cf files.
:param str|sequence targets: Pointer(s) to file(s) and/or directorie(s).
:returns: Generator yielding simulation information derived from a parse of cf files.
:rtype: generator
"""
for cf_fields in yield_cf_files(targets):
for cf_field in cf_fields:
identifier, properties, dates = parse(cf_field)
if identifier:
yield cf_field, identifier, properties, dates
# ... close file to prevent a proliferation of open file handles
cf.close_one_file()
def yield_cf_files(targets):
"""Yields CF files for further processing.
:param str|sequence targets: Pointer(s) to file(s) and/or directorie(s).
:returns: Generator yielding CF files.
:rtype: generator
"""
for fpath in yield_files(targets):
try:
cf_files = cf.read(fpath, ignore_read_error=False, verbose=False, aggregate=False)
except (IOError, OSError):
logger.log_warning("Non netCDF file rejected: {}".format(fpath))
else:
# Save the netCDF file name (from which we can extract the dataset version)
for cf_file in cf_files:
cf_file.fpath = fpath
yield cf_files
# ... close file to prevent a proliferation of open file handles
cf.close_one_file()