def read(cfile): """Read `yaml` formatted config file. Parameters ---------- cfile : str filename of config to read Returns ------- dict Dictionary with config options. """ log = gen.getLogger(__name__) # read config file cfile = pathlib.Path(cfile).resolve() if not cfile.exists(): raise IOError('Config file does not exist: {}'.format(str(cfile))) with open(cfile, 'r') as ymlfile: conf = yaml.safe_load(ymlfile) # check sections # Note that this only really give settings of where to find files. # Thus the key `data_folders` can be substituted with any other system # name, as long as it corresponds to the section in the config file. if 'data_folders' not in conf.keys(): log.critical("'data_folders' not specified in config file '%s'" % cfile) return (conf)
def __init__(self, cfg, time_period=0, models=NAMES_UA): # logging log = gen.getLogger(gen.getClassName(self)) log.info("start creating ua ensemble monthly average") # aos obj = aos.DataObject(('ua_ens_month_avg', time_period, models)) if obj.load(): obj.copy(self) return (None) # init self._time_period = time_period self._models = models self.cfg = dict(cfg) # create list of time_avg_lat_lon 2D-fields ensemble_time_avg_lat_lon = [] if time_period == '1979-2005': directory_prefix = pl.Path( self.cfg['data_folders']['historical']).joinpath('ua/mon') elif time_period == '2070-2099': directory_prefix = pl.Path( self.cfg['data_folders']['rcp85']).joinpath('ua/mon') else: raise Exception("ua_ens_avg: time period '%s' not known" % time_period) print("Load data from time period %s" % time_period) log.info("ua_ens_month_avg: Load data from time period %s" % time_period) for name in self._models: print("Loading %s from file" % name) log.info("Loading %s from file" % name) directory = directory_prefix.joinpath(name).joinpath('r1i1p1') tall = time_avg_month_lat_lon_dir(directory, time_period=time_period) ensemble_time_avg_lat_lon += [tall] # check consistancy of grids in the ensemble of time_avg_lat_lon 2D-fields and store grid tall_0 = ensemble_time_avg_lat_lon[0] for tall in ensemble_time_avg_lat_lon: if not tall_0.grid_equal(tall): raise Exception( "ua_ens_month_avg: error, grids are not consistent") self.points = tall_0.points self.area = tall_0.area # put ensemble into a 3D array with shape (n_points, 24, n_models) tall_0 = ensemble_time_avg_lat_lon[0] n_points = tall_0.data.shape[0] self.n_models = len(ensemble_time_avg_lat_lon) self.data = np.zeros((n_points, 24, self.n_models)) for tall, model in zip(ensemble_time_avg_lat_lon, range(self.n_models)): self.data[:, :, model] = tall.data # aos obj.save(self)
def __init__(self, data): # aos obj = aos.DataObject(('ua', data)) if obj.load(): obj.copy(self) return (None) # logging log = gen.getLogger(gen.getClassName(self)) log.info("creating class") if not data: raise Exception("ua: Initialization error, provide data") if data[0] == 'file': self._init_ua_from_file(data[1]) else: raise Exception("ua: Initialization error, unknow mode") # aos obj.save(self)
def _init_ua_from_file(self, ua_file): # logging log = gen.getLogger(gen.getClassName(self)) log.info("loading data from file '%s'" % ua_file) if not ua_file.exists(): log.critical("Initialization error, file '%s' not found" % ua_file) raise Exception("ua: Initialization error, file '%s' not found" % ua_file) self._ua_file = ua_file g = xr.open_dataset(self._ua_file) self.plev_to_extract = 70000 # store grid self.x = g['lon'].values.reshape(-1) self.y = g['lat'].values.reshape(-1) if 'plev' in g: self.plev = g['plev'].values.reshape(-1) elif 'lev' in g: self.plev = g['lev'].values.reshape(-1) else: log.critical("Initialization error, cannot read 'plev'") raise Exception("ua: Initialization error, cannot read 'plev'") log.info("plev: %s" % str(self.plev)) idc = np.where(self.plev == self.plev_to_extract)[0] if idc.shape[0] != 1: log.critical("ua: Initialization error: cannot find plevel %f" % self.plev_to_extract) raise Exception("ua: Initialization error: cannot find plevel %f" % self.plev_to_extract) self.plev_idx = idc[0] self.t = time_datetime64_to_month(g['time'].values) self.nx = self.x.shape[0] self.ny = self.y.shape[0] self.nt = self.t.shape[0] # store data self.data = g[ 'ua'].values[:, self.plev_idx, :, :] # array for (time, lat, lon)
def __init__(self, x): self.log = gen.getLogger(gen.getClassName(self)) self.X = None self._x_ = x self._hash_ = hsh(x) self._file_ = "%s/data-%s" % (path, self._hash_) self._file_exists_ = os.path.isfile(self._file_) if verbose == 2: print(headline) print("auto=%d" % auto) print("action: init") print(format_list(self._x_)) print("file exists: %d" % self._file_exists_) print("file=%s" % self._file_) print(footline) if 'PYTHONHASHSEED' not in os.environ.keys(): self.log.warning( '`PYTHONHASHSEED` environmental variable not set.' + ' This will prevent `aos.DataObject` from identifying' + ' existing files as each run will have new seeds.')
def __init__(self, directory, time_period=0): # logging log = gen.getLogger(gen.getClassName(self)) log.info("starting") # aos obj = aos.DataObject(('time_avg_lat_lon_dir', directory, time_period)) if obj.load(): obj.copy(self) log.info("loaded") return (None) ua_list = [] for f in directory.iterdir(): if '_reg.nc' in str(f): gen.print_truncate(" Loading file %s" % f) ua_list = ua_list + [ua(['file', directory.joinpath(f)])] super(time_avg_lat_lon_dir, self).__init__(['ua list', ua_list], time_period=time_period) # aos obj.save(self) log.info("created")
def __init__(self, data, time_period=0): # logging log = gen.getLogger(gen.getClassName(self)) log.info("starting") # input if time_period == '1971-2000': self._time_period = '1971-2000' self._n_month = 360 self._month_offset = 0 elif time_period == '2071-2100': self._time_period = '2071-2100' self._n_month = 360 self._month_offset = 100 * 12 elif time_period == '1979-2005': self._time_period = '1979-2005' self._n_month = 27 * 12 self._month_offset = 8 * 12 elif time_period == '2070-2099': self._time_period = '2070-2099' self._n_month = 360 self._month_offset = 99 * 12 else: log.critical("Initialization error, provide time_period") raise Exception( "time_avg_month_lat_lon: Initialization error, provide time_period" ) if not data: log.critical("Initialization error, provide data") raise Exception( "time_avg_month_lat_lon: Initialization error, provide data") # initialize log.info("initializing from %s" % data[0]) values = None if data[0] == 'ua list': # initialize from list for list_item in data[1]: if values is None: values = array_create_unitialized( (self._n_month, list_item.ny, list_item.nx)) self._x = list_item.x self._y = list_item.y if not data[1][0].grid_equal(list_item): log.critical("initialization error, grid mismatch") raise Exception( "time_avg_month_lat_lon: Initialization error, grid mismatch" ) for t_idx in range(list_item.nt): month = list_item.t[t_idx] - self._month_offset if (month < 0) or (month >= self._n_month): continue if not array_is_completely_uninitialized(values[month]): log.error("array not completely uninitialized") print( "WARNING: time_avg_month_lat_lon: array not completely uninitialized" ) values[month] = list_item.data[t_idx] else: log.critical("initialization mode '%s' not recognized" % data[0]) raise Exception( "time_avg_month_lat_lon: initialization mode '%s' not recognized" % data[0]) if not array_is_initialized(values): log.critical( "Initialization error, array not completely initialized from list %s" % str(data[1])) raise Exception( "time_avg_lat_lon: Initialization error, array not completely initialized" ) # create month-averages for 24 month # first 12 month: year 0 ... n-1 # second 12 month: year 1 ... n n_month = values.shape[0] n_years = int(n_month / 12) assert (12 * n_years == n_month) values = values.reshape(n_years, 12, 180, 360) values_1 = np.mean(values[:-1], axis=0) values_2 = np.mean(values[1:], axis=0) values = np.vstack((values_1, values_2)) # output array nx = self._x.shape[0] ny = self._y.shape[0] self.points = np.zeros((nx * ny, 2)) self.data = np.zeros((nx * ny, 24)) self.area = np.zeros((nx * ny, 1)) n = 0 for i in range(nx): for j in range(ny): self.points[n, 0] = self._x[i] self.points[n, 1] = self._y[j] self.data[n, :] = values[:, j, i] self.area[n, 0] = np.cos(np.pi / 180 * self._y[j]) * np.sin( np.pi / 360) / 360 n += 1 log.info("finished loading data from list")
#!/usr/bin/env python # libs # own modules import modules.aos as aos import modules.fmt as fmt import modules.gen as gen import modules.data as data import modules.config as config import modules.parallel as parallel import modules.evaluate as evaluate log = gen.getLogger(__name__) # -------------------------------------------------------------------------------------------------- # init # -------------------------------------------------------------------------------------------------- log.info('Initializing aos.') aos.path = 'objects' aos.verbose = 0 aos.auto = 1 cfg = config.read('conf/2019-07-28-Fumiaki.yml') # -------------------------------------------------------------------------------------------------- # prepare data and model names # -------------------------------------------------------------------------------------------------- # List with names of models to exclude from calculations. Then filter. log.info('Loading data.') excluded_models = []
def __init__(self, e, n=360, max_length=360, min_length=30): """Class initialization Parameters ---------- e : class Evaluation class for a single evaluation to run n : int, optional Range of starting degrees [0, 360]. max_length : int, optional Maximum size of averaging window [1, 360). min_length : int, optional Minumum size of averaging window [1, 359). Returns ------- TYPE Description """ # logging log = gen.getLogger(gen.getClassName(self)) log.info("start evaluating all [JJA]") # Error files self.minima_error_file = 'min_max_error.csv' self.nan_error_file = 'nan_error.csv' # aos obj = aos.DataObject(('EvaluateAll [JJA]', n)) if obj.load(): obj.copy(self) return (None) # prepare evaluate self.n = n self.min_length = min_length self.max_length = max_length x = np.arange(self.n) y = np.arange(self.min_length, self.max_length + 1) xx, yy = np.meshgrid(x, y) individuals = np.transpose( np.hstack((xx.reshape(-1, 1), yy.reshape(-1, 1)))) # evaluate self.ret = e(individuals) # Process / convert results self.start = individuals[0, :].reshape(-1, 1) self.length = individuals[1, :].reshape(-1, 1) self.fitness = self.ret[0, :].reshape(-1, 1) self.result = pd.DataFrame( { 'start': self.start.ravel(), 'length': self.length.ravel(), 'fitness': self.fitness.ravel() }, index=np.arange(len(self.fitness.ravel()))) # Longitudes in input goes from -179.5 to 179.5. # Here we convert from integer degree (offset) to the proper longitude value. self.result['start'] -= 179.5 self.save_errors(e.f.n_models) # aos obj.save(self)