def load_secoora_ncs(run_name): """ Loads local files using the run_name date. NOTE: Consider moving this inside the notebook. """ fname = '{}-{}.nc'.format OBS_DATA = nc2df(os.path.join(run_name, fname(run_name, 'OBS_DATA'))) SECOORA_OBS_DATA = nc2df(os.path.join(run_name, fname(run_name, 'SECOORA_OBS_DATA'))) ALL_OBS_DATA = concat([OBS_DATA, SECOORA_OBS_DATA], axis=1) index = ALL_OBS_DATA.index dfs = dict(OBS_DATA=ALL_OBS_DATA) for fname in glob(os.path.join(run_name, "*.nc")): if 'OBS_DATA' in fname: continue else: model = fname.split('.')[0].split('-')[-1] df = nc2df(fname) # FIXME: Horrible work around duplicate times. if len(df.index.values) != len(np.unique(df.index.values)): kw = dict(subset='index', keep='last') df = df.reset_index().drop_duplicates(**kw).set_index('index') kw = dict(method='time', limit=30) df = df.reindex(index).interpolate(**kw).ix[index] dfs.update({model: df}) return Panel.fromDict(dfs).swapaxes(0, 2)
def load_secoora_ncs(run_name): """ Loads local files using the run_name date. NOTE: Consider moving this inside the notebook. """ fname = '{}-{}.nc'.format OBS_DATA = nc2df(os.path.join(run_name, fname(run_name, 'OBS_DATA'))) SECOORA_OBS_DATA = nc2df( os.path.join(run_name, fname(run_name, 'SECOORA_OBS_DATA'))) ALL_OBS_DATA = concat([OBS_DATA, SECOORA_OBS_DATA], axis=1) index = ALL_OBS_DATA.index dfs = dict(OBS_DATA=ALL_OBS_DATA) for fname in glob(os.path.join(run_name, "*.nc")): if 'OBS_DATA' in fname: continue else: model = fname.split('.')[0].split('-')[-1] df = nc2df(fname) # FIXME: Horrible work around duplicate times. if len(df.index.values) != len(np.unique(df.index.values)): kw = dict(subset='index', keep='last') df = df.reset_index().drop_duplicates(**kw).set_index('index') kw = dict(method='time', limit=30) df = df.reindex(index).interpolate(**kw).ix[index] dfs.update({model: df}) return Panel.fromDict(dfs).swapaxes(0, 2)
def section(): lon, lat = [], [] fnames = sorted(data_path.glob('CTD/g01mcan*c.cnv.gz')) section = OrderedDict() for fname in fnames: cast = proc_ctd(fname) name = Path(fname).stem section.update({name: cast}) lon.append(cast['longitude'].mean()) lat.append(cast['latitude'].mean()) # Section. section = Panel.fromDict(section) return {'section': section, 'lon': lon, 'lat': lat}
def make_section(data_path=data_path, variable='t090C'): lon, lat = [], [] section = OrderedDict() fnames = sorted(data_path.joinpath('CTD').glob('g01mcan*c.cnv.gz')) for fname in fnames: cast = proc_ctd(fname) name = fname.stem section.update({name: cast}) lon.append(cast['longitude'].mean()) lat.append(cast['latitude'].mean()) section = Panel.fromDict(section) section = section.minor_xs(variable) section.lon, section.lat = lon, lat return section
def setUp(self): lon, lat = [], [] pattern = "./data/CTD/g01mcan*c.cnv.gz" fnames = sorted(glob(pattern), key=alphanum_key) try: section = OrderedDict() except: raise nose.SkipTest for fname in fnames: cast = proc_ctd(fname) name = os.path.basename(fname).split(".")[0] section.update({name: cast}) lon.append(cast.longitude.mean()) lat.append(cast.latitude.mean()) # Section. self.section = Panel.fromDict(section) self.lon, self.lat = lon, lat
def make_section(data_path=data_path, variable="t090C"): lon, lat = [], [] try: section = OrderedDict() except: raise nose.SkipTest pattern = "%s/CTD/g01mcan*c.cnv.gz" % data_path fnames = sorted(glob(pattern), key=alphanum_key) for fname in fnames: cast = proc_ctd(fname) name = os.path.basename(fname).split(".")[0] section.update({name: cast}) lon.append(cast.longitude.mean()) lat.append(cast.latitude.mean()) section = Panel.fromDict(section) section = section.minor_xs(variable) # Section (FIXME: Propagate lon, lat with MetaDataFrame). section.lon, section.lat = lon, lat return section
def load_ncs(run_name): fname = '{}-{}.nc'.format ALL_OBS_DATA = nc2df(os.path.join(run_name, fname(run_name, 'OBS_DATA'))) index = ALL_OBS_DATA.index dfs = dict(OBS_DATA=ALL_OBS_DATA) for fname in glob(os.path.join(run_name, "*.nc")): if 'OBS_DATA' in fname: continue else: model = fname.split('.')[0].split('-')[-1] df = nc2df(fname) # FIXME: Horrible work around duplicate times. if len(df.index.values) != len(np.unique(df.index.values)): kw = dict(subset='index', take_last=True) df = df.reset_index().drop_duplicates(**kw).set_index('index') kw = dict(method='time', limit=30) df = df.reindex(index).interpolate(**kw).ix[index] dfs.update({model: df}) return Panel.fromDict(dfs).swapaxes(0, 2)
fname = '{:%Y-%m-%d}-OBS_DATA.nc'.format(stop) OBS_DATA = nc2df(fname) index = OBS_DATA.index dfs = dict(OBS_DATA=OBS_DATA) for fname in glob("*.nc"): if 'OBS_DATA' in fname: continue else: model = fname.split('.')[0].split('-')[-1] df = nc2df(fname) kw = dict(method='time', limit=30) df = df.reindex(index).interpolate(**kw).ix[index] dfs.update({model: df}) dfs = Panel.fromDict(dfs).swapaxes(0, 2) # Clusters. big_list = [] for fname in glob("*.nc"): if 'OBS_DATA' in fname: continue nc = iris.load_cube(fname) model = fname.split('-')[-1].split('.')[0] lons = nc.coord(axis='X').points lats = nc.coord(axis='Y').points stations = nc.coord('station name').points models = [model]*lons.size lista = zip(models, lons.tolist(), lats.tolist(), stations.tolist()) big_list.extend(lista)
if 'OBS_DATA' in fname: continue else: model = fname.split('.')[0].split('-')[-1] df = nc2df(fname) if len(df.index.values) != len(np.unique(df.index.values)): # FIXME: Horrible work around duplicate times. opts = dict(cols='index', take_last=True) df = df.reset_index().drop_duplicates(**opts).set_index('index') df.index = df.index.tz_localize(start.tzinfo) if False: # if True interpolates to 6 min series. kw = dict(method='time', limit=30) df = df.reindex(index).interpolate(**kw).ix[index] dfs.update({model: df}) dfs = Panel.fromDict(dfs).swapaxes(0, 2) # ### Bias # In[23]: from pandas import DataFrame panel = dfs.copy() means = dict() for station, df in panel.iteritems(): df.dropna(axis=1, how='all', inplace=True) mean = df.mean() df = df - mean + mean['OBS_DATA'] means.update({station: mean.drop('OBS_DATA') - mean['OBS_DATA']})