def test_auto_combine(self): objs = [Dataset({'x': [0]}), Dataset({'x': [1]})] actual = auto_combine(objs) expected = Dataset({'x': [0, 1]}) self.assertDatasetIdentical(expected, actual) actual = auto_combine([actual]) self.assertDatasetIdentical(expected, actual) objs = [Dataset({'x': [0, 1]}), Dataset({'x': [2]})] actual = auto_combine(objs) expected = Dataset({'x': [0, 1, 2]}) self.assertDatasetIdentical(expected, actual) # ensure auto_combine handles non-sorted dimensions objs = [Dataset(OrderedDict([('x', ('a', [0])), ('y', ('a', [0]))])), Dataset(OrderedDict([('y', ('a', [1])), ('x', ('a', [1]))]))] actual = auto_combine(objs) expected = Dataset({'x': ('a', [0, 1]), 'y': ('a', [0, 1]), 'a': [0, 0]}) self.assertDatasetIdentical(expected, actual) objs = [Dataset({'x': [0], 'y': [0]}), Dataset({'y': [1], 'x': [1]})] with self.assertRaisesRegexp(ValueError, 'too many .* dimensions'): auto_combine(objs) objs = [Dataset({'x': 0}), Dataset({'x': 1})] with self.assertRaisesRegexp(ValueError, 'cannot infer dimension'): auto_combine(objs) objs = [Dataset({'x': [0], 'y': [0]}), Dataset({'x': [0]})] with self.assertRaises(KeyError): auto_combine(objs)
def test_auto_combine(self): objs = [Dataset({'x': [0]}), Dataset({'x': [1]})] actual = auto_combine(objs) expected = Dataset({'x': [0, 1]}) self.assertDatasetIdentical(expected, actual) actual = auto_combine([actual]) self.assertDatasetIdentical(expected, actual) objs = [Dataset({'x': [0, 1]}), Dataset({'x': [2]})] actual = auto_combine(objs) expected = Dataset({'x': [0, 1, 2]}) self.assertDatasetIdentical(expected, actual) # ensure auto_combine handles non-sorted dimensions objs = [ Dataset(OrderedDict([('x', ('a', [0])), ('y', ('a', [0]))])), Dataset(OrderedDict([('y', ('a', [1])), ('x', ('a', [1]))])) ] actual = auto_combine(objs) expected = Dataset({ 'x': ('a', [0, 1]), 'y': ('a', [0, 1]), 'a': [0, 0] }) self.assertDatasetIdentical(expected, actual) objs = [Dataset({'x': [0], 'y': [0]}), Dataset({'y': [1], 'x': [1]})] with self.assertRaisesRegexp(ValueError, 'too many .* dimensions'): auto_combine(objs) objs = [Dataset({'x': 0}), Dataset({'x': 1})] with self.assertRaisesRegexp(ValueError, 'cannot infer dimension'): auto_combine(objs) objs = [Dataset({'x': [0], 'y': [0]}), Dataset({'x': [0]})] with self.assertRaises(KeyError): auto_combine(objs)
c_DS.append(Get_data(m,sta_id,Vars_ext,Year_rng[-1].strftime('%Y%m%d%H%M'),EndDate.strftime('%Y%m%d%H%M'))) print 'Finished downloading data' # In[86]: # check what we got! print c_DS # In[87]: # In case any sites/variables were missing from a given year # Align each Dataset (fill missing stations/variables with NaN) #newDS = xray.align(*c_DS, join='outer', copy=True) newDS = xray.auto_combine(c_DS) #print newDS # In[88]: # In[90]: # Plot some data (change wind_direction_set_1 to what ever variable you have downloaded) vartoplot = newDS.wind_direction_set_1 [plt.plot(vartoplot.isel(site=csta)) for csta in range(0,newDS.site.values.size)]
'KN': (('index', 'iter'), numerical_results[:,2,:]), }, coords = { 'iter': range(1, niters+1), 'index': range(nrecs), }, ) ds.attrs['input_file'] = input_fn ds.attrs['timestamp'] = timestamp ds.attrs['ti'] = ti ds.attrs['n_thresh'] = N_THRESH ds.attrs['mu_thresh'] = MU_THRESH # ds.attrs['git_commit'] = get_git_versioning() ds.attrs['numpy_version'] = np.__version__ ds.attrs['pandas_version'] = pd.version.version ds.attrs['xray_version'] = xray.version.version params_ds = xray.Dataset.from_dataframe(df) combined_ds = xray.auto_combine([ds, params_ds]) # Increment index so order is sequential over all chunks ds['index'] = ds.index + tot_samples # Save to disk combined_ds.to_netcdf(os.path.join(SAVE_DIR, "%s_ti_%04d.nc" % (NAME, ti))) # Record of how many samples we've processed tot_samples += len(df.index) data.close()