def build_masksample_indices(period, timeframe, mask, min_valid=1): ''' Build resample indices for the supplied timeframe with a validity threshold ''' tc = period_to_tc(period) tf = dt.validate_timeframe(timeframe) new_p = dt.resample_dti(period, tf, as_period=True) _, end_of_p, _ = dt.boundary_funcs(tf) s_mask = mask.copy() sample_indices = [] cur_s = 0 cur_e = 0 for p in new_p: cur_idx = tc.get_index(slice(p.start_time, end_of_p(p.start_time))) v_count = (~mask[cur_idx]).sum() if (v_count >= min_valid): cur_e += v_count sample_indices.append(slice(cur_s, cur_e)) cur_s = cur_e else: s_mask[cur_idx] = True return sample_indices, np.where(s_mask == False)[0]
def stat_percentiles(self, statistic='pearsons_r', freq='m', pctiles=None): ''' Print a summary of percentiles for the specified statistic and timeframe ''' if pctiles is None: pctiles = [0, 5, 25, 50, 75, 95, 100] tf = dt.validate_timeframe(freq).lower() df = pd.DataFrame() for m in self._iter_models(freq): if statistic == "grand_f": m_data = m.stats[tf].loc['fobj', m.stats[tf].columns != 'all'] try: stats = standard_percentiles(m_data) df[m.name] = pd.Series( index=['grand_f'], data=[(stats['25%'] + stats['50%'] + stats['75%'] + stats['100%']) / 4]) except IndexError: logger.warning("no stats for model: %s", m.name) else: m_data = m.stats[tf].loc[statistic, m.stats[tf].columns != 'all'] try: df[m.name] = standard_percentiles(m_data, pctiles) except IndexError: logger.warning("no stats for model: %s", m.name) return df.transpose()
def build_resample_index(period, timeframe, window=None): ''' Return (slice/integer) indices matching the boundaries of a resampled period Optionally supply a window period (ie only produces indices within the window) ''' tc = period_to_tc(period) tf = dt.validate_timeframe(timeframe) if window is None: window = period new_p = dt.resample_dti(window, tf, as_period=True) _, end_of_p, _ = dt.boundary_funcs(tf) if isinstance(period, pd.PeriodIndex): def enforce_freq(ts): return ts.to_period(period.freq) else: def enforce_freq(ts): return ts indices = [] for p in new_p: s = enforce_freq(p.start_time) e = enforce_freq(end_of_p(p.start_time)) indices.append(tc.get_index(slice(s, e))) return indices
def data_percentiles(self, freq='m', pctiles=None): ''' Print a summary of percentiles for the actual data values ''' if pctiles is None: pctiles = [0, 5, 25, 50, 75, 95, 100] tf = dt.validate_timeframe(freq).lower() df = pd.DataFrame() if freq == 'd': # obs won't match model.obs since different obs.valid_idx for each model obs_series = self.obs.data.mean().values.flatten() else: pd_tf = dt.pandas_tf_dict[tf] obs_series = self.obs.data.resample( rule=pd_tf, how=self.aggr_how).mean().values.flatten() df[self.ref_name] = standard_percentiles(obs_series, pctiles) for m in self._iter_models(freq): m_data = pd.DataFrame.from_dict(m.data[tf]).mean().values.flatten() try: df[m.name] = standard_percentiles(m_data, pctiles) except IndexError: logger.warning("no stats for model: %s", m.name) return df.transpose()
def plot_box(self, statistic, freq='m', **kwargs): ''' Show a box-plot for the specified statistic and timeframe ''' tf = dt.validate_timeframe(freq).lower() ax = self._get_ax(kwargs) data = [] colours = [] names = [] for m in self._iter_models(freq): data.append(m.stats[tf].loc[statistic, m.stats[tf].columns != 'all']) colours.append(m.colour) names.append(m.name) box = ax.boxplot(data, patch_artist=True) ax.set_ylabel(statistic) for patch, colour in zip(box['boxes'], colours): patch.set_facecolor(colour) ax.set_xticklabels(names, rotation=90, fontsize=8) for k, v in kwargs.items(): try: ax.set(**{k: v}) except: pass ax.grid() return ax, box
def stat(self, statistic='mean', freq='m'): tf = dt.validate_timeframe(freq).lower() df = pd.DataFrame() for m in self._iter_models(freq): df[m.name] = m.stats[tf].loc[statistic] if statistic == 'mean': df[self.ref_name] = m.stats[tf].loc['obs_mean'] return df
def plot_regression(self, site=None, freq='m', title="", size=20, **kwargs): ''' Plot the model regression(s) for the specified site and frequency ''' if site is None: site = list(self.obs.data.columns) stats_index = 'all' else: stats_index = site site = [site] tf = dt.validate_timeframe(freq).lower() ax = self._get_ax(kwargs) for m in self._iter_models(freq): _site_list = [] for _site in site: if _site in m.data[tf].keys(): _site_list.append(_site) model_data = pd.DataFrame.from_dict(m.data[tf])[_site_list] obs_data = pd.DataFrame.from_dict(m.obs[tf])[_site_list] ax.scatter(obs_data, model_data, color=m.colour, s=size) ax.set_ylabel('model ' + self.var_name) ax.set_xlabel(str(self.ref_name)) if isinstance(site, list): ax.set_title(title) else: ax.set_title(title + " %s" % site) ax.set(**kwargs) ax.grid() # plot regression lines and 1:1 line rl = get_ax_limit(ax) for m in self._iter_models(freq): try: mstats = m.stats[tf][stats_index] except KeyError: continue regress_line = mstats.loc[ 'r_intercept'] + rl * mstats.loc['r_slope'] ax.plot(rl, regress_line, color=m.colour, label=m.name) ax.plot(rl, rl, linestyle='--', color='black', label='1:1') ax.legend(loc='best') return ax
def plot_timeseries(self,site,freq='m',model=None,**kwargs): ''' Plot timeseries of data at the specified site and frequency ''' from functools import partial ax = self._get_ax(kwargs) # def _plot(ax,series,label,colour): def _plot(series,label,colour): #+++ fix for pandas 0.16.1 legend label bug (see https://github.com/pydata/pandas/issues/10119) series.name = label # series.plot(legend=True,axes=ax,color=colour) series.plot(legend=True,color=colour) # plot = partial(_plot,ax=ax) plot = partial(_plot) if freq == 'raw': #self.obs.data[site].plot(legend=True,axes=ax,color='black',label=self.ref_name) plot(series=self.obs.data[site],label=self.ref_name,colour='black') for name in self.selection(): m = self.models[name] #m.data.raw[site].plot(legend=True,axes=ax,color=m.colour,label=m.name) plot(series=m.data.raw[site],label=m.name,colour=m.colour) else: tf = dt.validate_timeframe(freq).lower() _freq = freq == 'y' and 'A' or freq if model is not None: if not model in self.models: logger.critical("%s not found in %s",model,self.models) return None else: plot(series=self.models[model].obs[tf][site].resample(_freq).asfreq(),label=self.ref_name,colour='black') plot(series=self.models[model].data[tf][site].resample(_freq).asfreq(),label=self.models[model].name,colour=self.models[model].colour) else: if freq == 'd': plot(series=self.obs.data[site].resample(_freq).asfreq(),label=self.ref_name,colour='black') elif freq == 'm': plot(series=self.obs.monthly[site].resample(_freq).asfreq(),label=self.ref_name,colour='black') elif freq == 'y': plot(series=self.obs.annual[site].resample(_freq).asfreq(),label=self.ref_name,colour='black') for m in self._iter_models(freq): try: plot(series=m.data[tf][site].resample(_freq).asfreq(),label=m.name,colour=m.colour) except: logger.warning("no data to plot for %s site %s",m.name,site) ax.legend(loc='best') ax.set_title("%s" % site) ax.set_ylabel(self.var_name) ax.set(**kwargs) ax.grid() return ax
def __init__(self, freq, method, coordinates): self.in_coords = coordinates self.freq = dt.validate_timeframe(freq) if method == 'sum': self.method = np.sum elif method == 'mean': self.method = np.mean out_period = dt.resample_dti(self.in_coords.time.index, freq) self.out_coords = CoordinateSet([ TimeCoordinates(awrams_time, out_period), coordinates.latitude, coordinates.longitude ])
def plot_cdf(self,statistic='pearsons_r',freq='m', **kwargs): ''' Plot the empirical CDF for the specified statistic and frequency ''' tf = dt.validate_timeframe(freq).lower() ax = self._get_ax(kwargs) for m in self._iter_models(freq): y = sorted(m.stats[tf].loc[statistic, m.stats[tf].columns != 'all'].dropna()) # temporary fix for broken cdf's ax.plot(np.linspace(0,1.,len(y)),y,color=m.colour,label=m.name) ax.set_xlabel("Catchments below (%)") ax.set_ylabel(statistic) ax.legend(loc='best') ax.set(**kwargs) ax.grid() return ax
def resample_data(in_path, in_pattern, variable, period, out_path, to_freq, method, mode='w', enforce_mask=True, extent=None, use_weights=False): ''' method is 'sum' or 'mean' if no extent is supplied then the full (unmasked) input will be used 'use_weights' should be set for unequally binned conversions (monthly->annual means, for example) ''' from glob import glob import time import numpy as np from awrams.utils.messaging import reader as nr from awrams.utils.messaging import writer as nw from awrams.utils.messaging.brokers import OrderedFanInChunkBroker, FanOutChunkBroker from awrams.utils.messaging.general import message from awrams.utils.messaging.buffers import create_managed_buffers from awrams.utils.processing.chunk_resampler import ChunkedTimeResampler from awrams.utils.catchments import subdivide_extent from awrams.utils import datetools as dt from awrams.utils import mapping_types as mt from awrams.utils.io import data_mapping as dm start = time.time() NWORKERS = 2 read_ahead = 3 writemax = 3 BLOCKSIZE = 128 nbuffers = (NWORKERS * 2) + read_ahead + writemax # Receives all messages from clients ''' Build the 'standard queues' This should be wrapped up somewhere else for various topologies... ''' control_master = mp.Queue() worker_q = mp.Queue() for i in range(NWORKERS): worker_q.put(i) #Reader Queues chunk_out_r = mp.Queue(read_ahead) reader_in = dict(control=mp.Queue()) reader_out = dict(control=control_master, chunks=chunk_out_r) #Writer Queues chunk_in_w = mp.Queue(writemax) writer_in = dict(control=mp.Queue(), chunks=chunk_in_w) writer_out = dict(control=control_master) #FanIn queues fanout_in = dict(control=mp.Queue(), chunks=chunk_out_r, workers=worker_q) fanout_out = dict(control=control_master) fanin_in = dict(control=mp.Queue()) fanin_out = dict(control=control_master, out=chunk_in_w, workers=worker_q) #Worker Queues work_inq = [] work_outq = [] for i in range(NWORKERS): work_inq.append(mp.Queue()) fanout_out[i] = work_inq[-1] work_outq.append(mp.Queue()) fanin_in[i] = work_outq[-1] ''' End standard queues... ''' infiles = glob(in_path + '/' + in_pattern) if len(infiles) > 1: ff = dm.filter_years(period) else: ff = None sfm = dm.SplitFileManager.open_existing(in_path, in_pattern, variable, ff=ff) in_freq = sfm.get_frequency() split_periods = [period] if hasattr(in_freq, 'freqstr'): if in_freq.freqstr == 'D': #Force splitting so that flat files don't end up getting loaded entirely into memory! #Also a bit of a hack to deal with PeriodIndex/DTI issues... split_periods = dt.split_period( dt.resample_dti(period, 'd', as_period=False), 'a') in_periods = [dt.resample_dti(p, in_freq) for p in split_periods] in_pmap = sfm.get_period_map_multi(in_periods) out_periods = [] for p in in_periods: out_periods.append(dt.resample_dti(p, to_freq)) if extent is None: extent = sfm.ref_ds.get_extent(True) if extent.mask.size == 1: extent.mask = (np.ones(extent.shape) * extent.mask).astype(np.bool) sub_extents = subdivide_extent(extent, BLOCKSIZE) chunks = [nr.Chunk(*s.indices()) for s in sub_extents] out_period = dt.resample_dti(period, to_freq) out_cs = mt.gen_coordset(out_period, extent) v = mt.Variable.from_ncvar(sfm.ref_ds.awra_var) in_dtype = sfm.ref_ds.awra_var.dtype sfm.close_all() use_weights = False if method == 'mean': if dt.validate_timeframe(in_freq) == 'MONTHLY': use_weights = True ''' Need a way of formalising multiple buffer pools for different classes of work.. ''' max_inplen = max([len(p) for p in in_periods]) bufshape = (max_inplen, BLOCKSIZE, BLOCKSIZE) shared_buffers = {} shared_buffers['main'] = create_managed_buffers(nbuffers, bufshape, build=False) mvar = mt.MappedVariable(v, out_cs, in_dtype) sfm = dm.FlatFileManager(out_path, mvar) CLOBBER = mode == 'w' sfm.create_files(False, CLOBBER, chunksize=(1, BLOCKSIZE, BLOCKSIZE)) outfile_maps = { v.name: dict(nc_var=v.name, period_map=sfm.get_period_map_multi(out_periods)) } infile_maps = {v.name: dict(nc_var=v.name, period_map=in_pmap)} reader = nr.StreamingReader(reader_in, reader_out, shared_buffers, infile_maps, chunks, in_periods) writer = nw.MultifileChunkWriter(writer_in, writer_out, shared_buffers, outfile_maps, sub_extents, out_periods, enforce_mask=enforce_mask) fanout = FanOutChunkBroker(fanout_in, fanout_out) fanin = OrderedFanInChunkBroker(fanin_in, fanin_out, NWORKERS, len(chunks)) fanout.start() fanin.start() workers = [] w_control = [] for i in range(NWORKERS): w_in = dict(control=mp.Queue(), chunks=work_inq[i]) w_out = dict(control=control_master, chunks=work_outq[i]) w = ChunkedTimeResampler(w_in, w_out, shared_buffers, sub_extents, in_periods, to_freq, method, enforce_mask=enforce_mask, use_weights=use_weights) workers.append(w) w_control.append(w_in['control']) w.start() writer.start() reader.start() writer.join() fanout_in['control'].put(message('terminate')) fanin_in['control'].put(message('terminate')) for i in range(NWORKERS): w_control[i].put(message('terminate')) for x in range(4): control_master.get() for i in range(NWORKERS): workers[i].join() control_master.get() reader.join() fanout.join() fanin.join() end = time.time() logger.info("elapsed time: %ss", end - start)