def load_secoora_ncs(run_name):
"""
Loads local files using the run_name date.
NOTE: Consider moving this inside the notebook.
"""
fname = '{}-{}.nc'.format
OBS_DATA = nc2df(os.path.join(run_name,
fname(run_name, 'OBS_DATA')))
SECOORA_OBS_DATA = nc2df(os.path.join(run_name,
fname(run_name, 'SECOORA_OBS_DATA')))
ALL_OBS_DATA = concat([OBS_DATA, SECOORA_OBS_DATA], axis=1)
index = ALL_OBS_DATA.index
dfs = dict(OBS_DATA=ALL_OBS_DATA)
for fname in glob(os.path.join(run_name, "*.nc")):
if 'OBS_DATA' in fname:
continue
else:
model = fname.split('.')[0].split('-')[-1]
df = nc2df(fname)
# FIXME: Horrible work around duplicate times.
if len(df.index.values) != len(np.unique(df.index.values)):
kw = dict(subset='index', keep='last')
df = df.reset_index().drop_duplicates(**kw).set_index('index')
kw = dict(method='time', limit=30)
df = df.reindex(index).interpolate(**kw).ix[index]
dfs.update({model: df})
return Panel.fromDict(dfs).swapaxes(0, 2)
def load_secoora_ncs(run_name):
"""
Loads local files using the run_name date.
NOTE: Consider moving this inside the notebook.
"""
fname = '{}-{}.nc'.format
OBS_DATA = nc2df(os.path.join(run_name, fname(run_name, 'OBS_DATA')))
SECOORA_OBS_DATA = nc2df(
os.path.join(run_name, fname(run_name, 'SECOORA_OBS_DATA')))
ALL_OBS_DATA = concat([OBS_DATA, SECOORA_OBS_DATA], axis=1)
index = ALL_OBS_DATA.index
dfs = dict(OBS_DATA=ALL_OBS_DATA)
for fname in glob(os.path.join(run_name, "*.nc")):
if 'OBS_DATA' in fname:
continue
else:
model = fname.split('.')[0].split('-')[-1]
df = nc2df(fname)
# FIXME: Horrible work around duplicate times.
if len(df.index.values) != len(np.unique(df.index.values)):
kw = dict(subset='index', keep='last')
df = df.reset_index().drop_duplicates(**kw).set_index('index')
kw = dict(method='time', limit=30)
df = df.reindex(index).interpolate(**kw).ix[index]
dfs.update({model: df})
return Panel.fromDict(dfs).swapaxes(0, 2)
def section():
lon, lat = [], []
fnames = sorted(data_path.glob('CTD/g01mcan*c.cnv.gz'))
section = OrderedDict()
for fname in fnames:
cast = proc_ctd(fname)
name = Path(fname).stem
section.update({name: cast})
lon.append(cast['longitude'].mean())
lat.append(cast['latitude'].mean())
# Section.
section = Panel.fromDict(section)
return {'section': section, 'lon': lon, 'lat': lat}
def make_section(data_path=data_path, variable='t090C'):
lon, lat = [], []
section = OrderedDict()
fnames = sorted(data_path.joinpath('CTD').glob('g01mcan*c.cnv.gz'))
for fname in fnames:
cast = proc_ctd(fname)
name = fname.stem
section.update({name: cast})
lon.append(cast['longitude'].mean())
lat.append(cast['latitude'].mean())
section = Panel.fromDict(section)
section = section.minor_xs(variable)
section.lon, section.lat = lon, lat
return section
def setUp(self):
lon, lat = [], []
pattern = "./data/CTD/g01mcan*c.cnv.gz"
fnames = sorted(glob(pattern), key=alphanum_key)
try:
section = OrderedDict()
except:
raise nose.SkipTest
for fname in fnames:
cast = proc_ctd(fname)
name = os.path.basename(fname).split(".")[0]
section.update({name: cast})
lon.append(cast.longitude.mean())
lat.append(cast.latitude.mean())
# Section.
self.section = Panel.fromDict(section)
self.lon, self.lat = lon, lat
def make_section(data_path=data_path, variable="t090C"):
lon, lat = [], []
try:
section = OrderedDict()
except:
raise nose.SkipTest
pattern = "%s/CTD/g01mcan*c.cnv.gz" % data_path
fnames = sorted(glob(pattern), key=alphanum_key)
for fname in fnames:
cast = proc_ctd(fname)
name = os.path.basename(fname).split(".")[0]
section.update({name: cast})
lon.append(cast.longitude.mean())
lat.append(cast.latitude.mean())
section = Panel.fromDict(section)
section = section.minor_xs(variable)
# Section (FIXME: Propagate lon, lat with MetaDataFrame).
section.lon, section.lat = lon, lat
return section
def load_ncs(run_name):
fname = '{}-{}.nc'.format
ALL_OBS_DATA = nc2df(os.path.join(run_name,
fname(run_name, 'OBS_DATA')))
index = ALL_OBS_DATA.index
dfs = dict(OBS_DATA=ALL_OBS_DATA)
for fname in glob(os.path.join(run_name, "*.nc")):
if 'OBS_DATA' in fname:
continue
else:
model = fname.split('.')[0].split('-')[-1]
df = nc2df(fname)
# FIXME: Horrible work around duplicate times.
if len(df.index.values) != len(np.unique(df.index.values)):
kw = dict(subset='index', take_last=True)
df = df.reset_index().drop_duplicates(**kw).set_index('index')
kw = dict(method='time', limit=30)
df = df.reindex(index).interpolate(**kw).ix[index]
dfs.update({model: df})
return Panel.fromDict(dfs).swapaxes(0, 2)
fname = '{:%Y-%m-%d}-OBS_DATA.nc'.format(stop)
OBS_DATA = nc2df(fname)
index = OBS_DATA.index
dfs = dict(OBS_DATA=OBS_DATA)
for fname in glob("*.nc"):
if 'OBS_DATA' in fname:
continue
else:
model = fname.split('.')[0].split('-')[-1]
df = nc2df(fname)
kw = dict(method='time', limit=30)
df = df.reindex(index).interpolate(**kw).ix[index]
dfs.update({model: df})
dfs = Panel.fromDict(dfs).swapaxes(0, 2)
# Clusters.
big_list = []
for fname in glob("*.nc"):
if 'OBS_DATA' in fname:
continue
nc = iris.load_cube(fname)
model = fname.split('-')[-1].split('.')[0]
lons = nc.coord(axis='X').points
lats = nc.coord(axis='Y').points
stations = nc.coord('station name').points
models = [model]*lons.size
lista = zip(models, lons.tolist(), lats.tolist(), stations.tolist())
big_list.extend(lista)
if 'OBS_DATA' in fname:
continue
else:
model = fname.split('.')[0].split('-')[-1]
df = nc2df(fname)
if len(df.index.values) != len(np.unique(df.index.values)):
# FIXME: Horrible work around duplicate times.
opts = dict(cols='index', take_last=True)
df = df.reset_index().drop_duplicates(**opts).set_index('index')
df.index = df.index.tz_localize(start.tzinfo)
if False: # if True interpolates to 6 min series.
kw = dict(method='time', limit=30)
df = df.reindex(index).interpolate(**kw).ix[index]
dfs.update({model: df})
dfs = Panel.fromDict(dfs).swapaxes(0, 2)
# ### Bias
# In[23]:
from pandas import DataFrame
panel = dfs.copy()
means = dict()
for station, df in panel.iteritems():
df.dropna(axis=1, how='all', inplace=True)
mean = df.mean()
df = df - mean + mean['OBS_DATA']
means.update({station: mean.drop('OBS_DATA') - mean['OBS_DATA']})
