def test_auto_combine(self):
objs = [Dataset({'x': [0]}), Dataset({'x': [1]})]
actual = auto_combine(objs)
expected = Dataset({'x': [0, 1]})
self.assertDatasetIdentical(expected, actual)
actual = auto_combine([actual])
self.assertDatasetIdentical(expected, actual)
objs = [Dataset({'x': [0, 1]}), Dataset({'x': [2]})]
actual = auto_combine(objs)
expected = Dataset({'x': [0, 1, 2]})
self.assertDatasetIdentical(expected, actual)
# ensure auto_combine handles non-sorted dimensions
objs = [Dataset(OrderedDict([('x', ('a', [0])), ('y', ('a', [0]))])),
Dataset(OrderedDict([('y', ('a', [1])), ('x', ('a', [1]))]))]
actual = auto_combine(objs)
expected = Dataset({'x': ('a', [0, 1]), 'y': ('a', [0, 1]), 'a': [0, 0]})
self.assertDatasetIdentical(expected, actual)
objs = [Dataset({'x': [0], 'y': [0]}), Dataset({'y': [1], 'x': [1]})]
with self.assertRaisesRegexp(ValueError, 'too many .* dimensions'):
auto_combine(objs)
objs = [Dataset({'x': 0}), Dataset({'x': 1})]
with self.assertRaisesRegexp(ValueError, 'cannot infer dimension'):
auto_combine(objs)
objs = [Dataset({'x': [0], 'y': [0]}), Dataset({'x': [0]})]
with self.assertRaises(KeyError):
auto_combine(objs)
def test_auto_combine(self):
objs = [Dataset({'x': [0]}), Dataset({'x': [1]})]
actual = auto_combine(objs)
expected = Dataset({'x': [0, 1]})
self.assertDatasetIdentical(expected, actual)
actual = auto_combine([actual])
self.assertDatasetIdentical(expected, actual)
objs = [Dataset({'x': [0, 1]}), Dataset({'x': [2]})]
actual = auto_combine(objs)
expected = Dataset({'x': [0, 1, 2]})
self.assertDatasetIdentical(expected, actual)
# ensure auto_combine handles non-sorted dimensions
objs = [
Dataset(OrderedDict([('x', ('a', [0])), ('y', ('a', [0]))])),
Dataset(OrderedDict([('y', ('a', [1])), ('x', ('a', [1]))]))
]
actual = auto_combine(objs)
expected = Dataset({
'x': ('a', [0, 1]),
'y': ('a', [0, 1]),
'a': [0, 0]
})
self.assertDatasetIdentical(expected, actual)
objs = [Dataset({'x': [0], 'y': [0]}), Dataset({'y': [1], 'x': [1]})]
with self.assertRaisesRegexp(ValueError, 'too many .* dimensions'):
auto_combine(objs)
objs = [Dataset({'x': 0}), Dataset({'x': 1})]
with self.assertRaisesRegexp(ValueError, 'cannot infer dimension'):
auto_combine(objs)
objs = [Dataset({'x': [0], 'y': [0]}), Dataset({'x': [0]})]
with self.assertRaises(KeyError):
auto_combine(objs)
c_DS.append(Get_data(m,sta_id,Vars_ext,Year_rng[-1].strftime('%Y%m%d%H%M'),EndDate.strftime('%Y%m%d%H%M')))
print 'Finished downloading data'
# In[86]:
# check what we got!
print c_DS
# In[87]:
# In case any sites/variables were missing from a given year
# Align each Dataset (fill missing stations/variables with NaN)
#newDS = xray.align(*c_DS, join='outer', copy=True)
newDS = xray.auto_combine(c_DS)
#print newDS
# In[88]:
# In[90]:
# Plot some data (change wind_direction_set_1 to what ever variable you have downloaded)
vartoplot = newDS.wind_direction_set_1
[plt.plot(vartoplot.isel(site=csta)) for csta in range(0,newDS.site.values.size)]
'KN': (('index', 'iter'), numerical_results[:,2,:]), },
coords = {
'iter': range(1, niters+1), 'index': range(nrecs),
},
)
ds.attrs['input_file'] = input_fn
ds.attrs['timestamp'] = timestamp
ds.attrs['ti'] = ti
ds.attrs['n_thresh'] = N_THRESH
ds.attrs['mu_thresh'] = MU_THRESH
# ds.attrs['git_commit'] = get_git_versioning()
ds.attrs['numpy_version'] = np.__version__
ds.attrs['pandas_version'] = pd.version.version
ds.attrs['xray_version'] = xray.version.version
params_ds = xray.Dataset.from_dataframe(df)
combined_ds = xray.auto_combine([ds, params_ds])
# Increment index so order is sequential over all chunks
ds['index'] = ds.index + tot_samples
# Save to disk
combined_ds.to_netcdf(os.path.join(SAVE_DIR, "%s_ti_%04d.nc" % (NAME, ti)))
# Record of how many samples we've processed
tot_samples += len(df.index)
data.close()
