def read(cfile):
"""Read `yaml` formatted config file.
Parameters
----------
cfile : str
filename of config to read
Returns
-------
dict
Dictionary with config options.
"""
log = gen.getLogger(__name__)
# read config file
cfile = pathlib.Path(cfile).resolve()
if not cfile.exists():
raise IOError('Config file does not exist: {}'.format(str(cfile)))
with open(cfile, 'r') as ymlfile:
conf = yaml.safe_load(ymlfile)
# check sections
# Note that this only really give settings of where to find files.
# Thus the key `data_folders` can be substituted with any other system
# name, as long as it corresponds to the section in the config file.
if 'data_folders' not in conf.keys():
log.critical("'data_folders' not specified in config file '%s'" %
cfile)
return (conf)
def __init__(self, cfg, time_period=0, models=NAMES_UA):
# logging
log = gen.getLogger(gen.getClassName(self))
log.info("start creating ua ensemble monthly average")
# aos
obj = aos.DataObject(('ua_ens_month_avg', time_period, models))
if obj.load():
obj.copy(self)
return (None)
# init
self._time_period = time_period
self._models = models
self.cfg = dict(cfg)
# create list of time_avg_lat_lon 2D-fields
ensemble_time_avg_lat_lon = []
if time_period == '1979-2005':
directory_prefix = pl.Path(
self.cfg['data_folders']['historical']).joinpath('ua/mon')
elif time_period == '2070-2099':
directory_prefix = pl.Path(
self.cfg['data_folders']['rcp85']).joinpath('ua/mon')
else:
raise Exception("ua_ens_avg: time period '%s' not known" %
time_period)
print("Load data from time period %s" % time_period)
log.info("ua_ens_month_avg: Load data from time period %s" %
time_period)
for name in self._models:
print("Loading %s from file" % name)
log.info("Loading %s from file" % name)
directory = directory_prefix.joinpath(name).joinpath('r1i1p1')
tall = time_avg_month_lat_lon_dir(directory,
time_period=time_period)
ensemble_time_avg_lat_lon += [tall]
# check consistancy of grids in the ensemble of time_avg_lat_lon 2D-fields and store grid
tall_0 = ensemble_time_avg_lat_lon[0]
for tall in ensemble_time_avg_lat_lon:
if not tall_0.grid_equal(tall):
raise Exception(
"ua_ens_month_avg: error, grids are not consistent")
self.points = tall_0.points
self.area = tall_0.area
# put ensemble into a 3D array with shape (n_points, 24, n_models)
tall_0 = ensemble_time_avg_lat_lon[0]
n_points = tall_0.data.shape[0]
self.n_models = len(ensemble_time_avg_lat_lon)
self.data = np.zeros((n_points, 24, self.n_models))
for tall, model in zip(ensemble_time_avg_lat_lon,
range(self.n_models)):
self.data[:, :, model] = tall.data
# aos
obj.save(self)
def __init__(self, data):
# aos
obj = aos.DataObject(('ua', data))
if obj.load():
obj.copy(self)
return (None)
# logging
log = gen.getLogger(gen.getClassName(self))
log.info("creating class")
if not data:
raise Exception("ua: Initialization error, provide data")
if data[0] == 'file':
self._init_ua_from_file(data[1])
else:
raise Exception("ua: Initialization error, unknow mode")
# aos
obj.save(self)
def _init_ua_from_file(self, ua_file):
# logging
log = gen.getLogger(gen.getClassName(self))
log.info("loading data from file '%s'" % ua_file)
if not ua_file.exists():
log.critical("Initialization error, file '%s' not found" % ua_file)
raise Exception("ua: Initialization error, file '%s' not found" %
ua_file)
self._ua_file = ua_file
g = xr.open_dataset(self._ua_file)
self.plev_to_extract = 70000
# store grid
self.x = g['lon'].values.reshape(-1)
self.y = g['lat'].values.reshape(-1)
if 'plev' in g:
self.plev = g['plev'].values.reshape(-1)
elif 'lev' in g:
self.plev = g['lev'].values.reshape(-1)
else:
log.critical("Initialization error, cannot read 'plev'")
raise Exception("ua: Initialization error, cannot read 'plev'")
log.info("plev: %s" % str(self.plev))
idc = np.where(self.plev == self.plev_to_extract)[0]
if idc.shape[0] != 1:
log.critical("ua: Initialization error: cannot find plevel %f" %
self.plev_to_extract)
raise Exception("ua: Initialization error: cannot find plevel %f" %
self.plev_to_extract)
self.plev_idx = idc[0]
self.t = time_datetime64_to_month(g['time'].values)
self.nx = self.x.shape[0]
self.ny = self.y.shape[0]
self.nt = self.t.shape[0]
# store data
self.data = g[
'ua'].values[:, self.plev_idx, :, :] # array for (time, lat, lon)
def __init__(self, x):
self.log = gen.getLogger(gen.getClassName(self))
self.X = None
self._x_ = x
self._hash_ = hsh(x)
self._file_ = "%s/data-%s" % (path, self._hash_)
self._file_exists_ = os.path.isfile(self._file_)
if verbose == 2:
print(headline)
print("auto=%d" % auto)
print("action: init")
print(format_list(self._x_))
print("file exists: %d" % self._file_exists_)
print("file=%s" % self._file_)
print(footline)
if 'PYTHONHASHSEED' not in os.environ.keys():
self.log.warning(
'`PYTHONHASHSEED` environmental variable not set.' +
' This will prevent `aos.DataObject` from identifying' +
' existing files as each run will have new seeds.')
def __init__(self, directory, time_period=0):
# logging
log = gen.getLogger(gen.getClassName(self))
log.info("starting")
# aos
obj = aos.DataObject(('time_avg_lat_lon_dir', directory, time_period))
if obj.load():
obj.copy(self)
log.info("loaded")
return (None)
ua_list = []
for f in directory.iterdir():
if '_reg.nc' in str(f):
gen.print_truncate(" Loading file %s" % f)
ua_list = ua_list + [ua(['file', directory.joinpath(f)])]
super(time_avg_lat_lon_dir, self).__init__(['ua list', ua_list],
time_period=time_period)
# aos
obj.save(self)
log.info("created")
def __init__(self, data, time_period=0):
# logging
log = gen.getLogger(gen.getClassName(self))
log.info("starting")
# input
if time_period == '1971-2000':
self._time_period = '1971-2000'
self._n_month = 360
self._month_offset = 0
elif time_period == '2071-2100':
self._time_period = '2071-2100'
self._n_month = 360
self._month_offset = 100 * 12
elif time_period == '1979-2005':
self._time_period = '1979-2005'
self._n_month = 27 * 12
self._month_offset = 8 * 12
elif time_period == '2070-2099':
self._time_period = '2070-2099'
self._n_month = 360
self._month_offset = 99 * 12
else:
log.critical("Initialization error, provide time_period")
raise Exception(
"time_avg_month_lat_lon: Initialization error, provide time_period"
)
if not data:
log.critical("Initialization error, provide data")
raise Exception(
"time_avg_month_lat_lon: Initialization error, provide data")
# initialize
log.info("initializing from %s" % data[0])
values = None
if data[0] == 'ua list':
# initialize from list
for list_item in data[1]:
if values is None:
values = array_create_unitialized(
(self._n_month, list_item.ny, list_item.nx))
self._x = list_item.x
self._y = list_item.y
if not data[1][0].grid_equal(list_item):
log.critical("initialization error, grid mismatch")
raise Exception(
"time_avg_month_lat_lon: Initialization error, grid mismatch"
)
for t_idx in range(list_item.nt):
month = list_item.t[t_idx] - self._month_offset
if (month < 0) or (month >= self._n_month):
continue
if not array_is_completely_uninitialized(values[month]):
log.error("array not completely uninitialized")
print(
"WARNING: time_avg_month_lat_lon: array not completely uninitialized"
)
values[month] = list_item.data[t_idx]
else:
log.critical("initialization mode '%s' not recognized" % data[0])
raise Exception(
"time_avg_month_lat_lon: initialization mode '%s' not recognized"
% data[0])
if not array_is_initialized(values):
log.critical(
"Initialization error, array not completely initialized from list %s"
% str(data[1]))
raise Exception(
"time_avg_lat_lon: Initialization error, array not completely initialized"
)
# create month-averages for 24 month
# first 12 month: year 0 ... n-1
# second 12 month: year 1 ... n
n_month = values.shape[0]
n_years = int(n_month / 12)
assert (12 * n_years == n_month)
values = values.reshape(n_years, 12, 180, 360)
values_1 = np.mean(values[:-1], axis=0)
values_2 = np.mean(values[1:], axis=0)
values = np.vstack((values_1, values_2))
# output array
nx = self._x.shape[0]
ny = self._y.shape[0]
self.points = np.zeros((nx * ny, 2))
self.data = np.zeros((nx * ny, 24))
self.area = np.zeros((nx * ny, 1))
n = 0
for i in range(nx):
for j in range(ny):
self.points[n, 0] = self._x[i]
self.points[n, 1] = self._y[j]
self.data[n, :] = values[:, j, i]
self.area[n, 0] = np.cos(np.pi / 180 * self._y[j]) * np.sin(
np.pi / 360) / 360
n += 1
log.info("finished loading data from list")
#!/usr/bin/env python
# libs
# own modules
import modules.aos as aos
import modules.fmt as fmt
import modules.gen as gen
import modules.data as data
import modules.config as config
import modules.parallel as parallel
import modules.evaluate as evaluate
log = gen.getLogger(__name__)
# --------------------------------------------------------------------------------------------------
# init
# --------------------------------------------------------------------------------------------------
log.info('Initializing aos.')
aos.path = 'objects'
aos.verbose = 0
aos.auto = 1
cfg = config.read('conf/2019-07-28-Fumiaki.yml')
# --------------------------------------------------------------------------------------------------
# prepare data and model names
# --------------------------------------------------------------------------------------------------
# List with names of models to exclude from calculations. Then filter.
log.info('Loading data.')
excluded_models = []
def __init__(self, e, n=360, max_length=360, min_length=30):
"""Class initialization
Parameters
----------
e : class
Evaluation class for a single evaluation to run
n : int, optional
Range of starting degrees [0, 360].
max_length : int, optional
Maximum size of averaging window [1, 360).
min_length : int, optional
Minumum size of averaging window [1, 359).
Returns
-------
TYPE
Description
"""
# logging
log = gen.getLogger(gen.getClassName(self))
log.info("start evaluating all [JJA]")
# Error files
self.minima_error_file = 'min_max_error.csv'
self.nan_error_file = 'nan_error.csv'
# aos
obj = aos.DataObject(('EvaluateAll [JJA]', n))
if obj.load():
obj.copy(self)
return (None)
# prepare evaluate
self.n = n
self.min_length = min_length
self.max_length = max_length
x = np.arange(self.n)
y = np.arange(self.min_length, self.max_length + 1)
xx, yy = np.meshgrid(x, y)
individuals = np.transpose(
np.hstack((xx.reshape(-1, 1), yy.reshape(-1, 1))))
# evaluate
self.ret = e(individuals)
# Process / convert results
self.start = individuals[0, :].reshape(-1, 1)
self.length = individuals[1, :].reshape(-1, 1)
self.fitness = self.ret[0, :].reshape(-1, 1)
self.result = pd.DataFrame(
{
'start': self.start.ravel(),
'length': self.length.ravel(),
'fitness': self.fitness.ravel()
},
index=np.arange(len(self.fitness.ravel())))
# Longitudes in input goes from -179.5 to 179.5.
# Here we convert from integer degree (offset) to the proper longitude value.
self.result['start'] -= 179.5
self.save_errors(e.f.n_models)
# aos
obj.save(self)
