def test_align(self): left = create_test_data() right = left.copy(deep=True) right['dim3'] = ('dim3', list('cdefghijkl')) right['var3'][:-2] = right['var3'][2:] right['var3'][-2:] = np.random.randn(*right['var3'][-2:].shape) intersection = list('cdefghij') union = list('abcdefghijkl') left2, right2 = align(left, right, join='inner') self.assertArrayEqual(left2['dim3'], intersection) self.assertDatasetIdentical(left2, right2) left2, right2 = align(left, right, join='outer') self.assertVariableEqual(left2['dim3'], right2['dim3']) self.assertArrayEqual(left2['dim3'], union) self.assertDatasetIdentical(left2.labeled(dim3=intersection), right2.labeled(dim3=intersection)) self.assertTrue(np.isnan(left2['var3'][-2:]).all()) self.assertTrue(np.isnan(right2['var3'][:2]).all()) left2, right2 = align(left, right, join='left') self.assertVariableEqual(left2['dim3'], right2['dim3']) self.assertVariableEqual(left2['dim3'], left['dim3']) self.assertDatasetIdentical(left2.labeled(dim3=intersection), right2.labeled(dim3=intersection)) self.assertTrue(np.isnan(right2['var3'][:2]).all()) left2, right2 = align(left, right, join='right') self.assertVariableEqual(left2['dim3'], right2['dim3']) self.assertVariableEqual(left2['dim3'], right['dim3']) self.assertDatasetIdentical(left2.labeled(dim3=intersection), right2.labeled(dim3=intersection)) self.assertTrue(np.isnan(left2['var3'][-2:]).all())
def test_align(self): self.ds['x'] = ('x', np.array(list('abcdefghij'))) with self.assertRaises(ValueError): self.dv + self.dv[:5] dv1, dv2 = align(self.dv, self.dv[:5], join='inner') self.assertDataArrayIdentical(dv1, self.dv[:5]) self.assertDataArrayIdentical(dv2, self.dv[:5])
def test_align_dtype(self): # regression test for #264 x1 = np.arange(30) x2 = np.arange(5, 35) a = DataArray(np.random.random((30,)).astype('f32'), {'x': x1}) b = DataArray(np.random.random((30,)).astype('f32'), {'x': x2}) c, d = align(a, b, join='outer') self.assertEqual(c.dtype, np.float32)
def slice_dataset_space(ds_array, ds_to_slice): ''' slices second Dataset to fit first Dataset ''' swe_mask_align, array_align = xray.align(ds_array, ds_to_slice, join='inner', copy=False) return (array_align)
def test_align(self): self.ds['x'] = ('x', np.array(list('abcdefghij'))) dv1, dv2 = align(self.dv, self.dv[:5], join='inner') self.assertDataArrayIdentical(dv1, self.dv[:5]) self.assertDataArrayIdentical(dv2, self.dv[:5])
swe_mask_file = '/raid9/gergel/agg_snowpack/goodleap/SWE/histmeanmask.nc' ## 1s are swe, 0s are no swe swe_mask = xray.open_dataset(swe_mask_file) ## rename dimensions swe_mask.rename({ "Latitude": "lat", "Longitude": "lon", "Time": "time" }, inplace=True) swe_mask = swe_mask.squeeze() ## Dataset join swe_mask_align, pr_full = xray.align(swe_mask, pr, join='inner', copy=False) del pr gc.collect() direc = '/raid/gergel' pdur_file = 'pduration_mod.nc' # this is the one that I regridded to 6 x 6 km using cdo remapcon and /raid9/gergel/agg_snowpack/keepextracopies/grid_info_mine pdur_full = xray.open_dataset(os.path.join( direc, pdur_file)) ## pdur beta parameter for John's transform from Matt Jolly ''' for j in np.arange(len(pr_full.lat)): for k in np.arange(len(pr_full.lon)): if np.isnan(pr_full['precipitation'].values[0,j,k]) == False: lon_ind = np.argmin(np.abs(pdur_full.lon - pr_full.lon[k])) lat_ind = np.argmin(np.abs(pdur_full.lat - pr_full.lat[j]))
print("finished chunking data") ## cut out conus east of 103 for each variable swe_mask_file = '/raid9/gergel/agg_snowpack/goodleap/SWE/histmeanmask.nc' ## 1s are swe, 0s are no swe swe_mask = xray.open_dataset(swe_mask_file) ## rename dimensions swe_mask.rename({ "Latitude": "lat", "Longitude": "lon", "Time": "time" }, inplace=True) swe_mask = swe_mask.squeeze() ## Dataset join swe_mask_align, tmax = xray.align(swe_mask, tmax_f, join='inner', copy=False) swe_mask_align, tmin = xray.align(swe_mask, tmin_f, join='inner', copy=False) swe_mask_align, q = xray.align(swe_mask, q_f, join='inner', copy=False) print("FEEL THE BERN!") ## get julian days julians = pd.DatetimeIndex(np.asarray(tmin.time)).dayofyear ## delete full arrays of each variable for conus del tmax_f, tmin_f, q_f, swe_mask, swe_mask_align gc.collect() # 100-hr and 1000-hr DFM FUNCTION # In[27]:
pr = xray.open_dataset(os.path.join(direc,pr_file)) ## load precip ## adjust lat/lon dimensions since the index names are different lons_new = pr['lon'].values[pr['lon'].values > 180] - 360 pr['lon'] = lons_new swe_mask_file = '/raid9/gergel/agg_snowpack/goodleap/SWE/histmeanmask.nc' ## 1s are swe, 0s are no swe swe_mask = xray.open_dataset(swe_mask_file) ## rename dimensions swe_mask.rename({"Latitude": "lat", "Longitude": "lon", "Time": "time"}, inplace=True) swe_mask = swe_mask.squeeze() ## Dataset join swe_mask_align,pr_full = xray.align(swe_mask,pr,join='inner',copy=False) #swe_mask_align,tmax_align = xray.align(swe_mask,tmax,join='inner',copy=False) del pr gc.collect() direc = '/raid/gergel' pdur_file = 'pduration.nc' pdur_full = xray.open_dataset(os.path.join(direc,pdur_file)) ## pdur beta parameter for John's transform from Matt Jolly # In[13]: # pr_arr = pr_full['precipitation'].values for j in np.arange(len(pr_full.lat)): for k in np.arange(len(pr_full.lon)):
# Check if any data was found, if so add the DS to the list if temp_DS is not None: c_DS.append(temp_DS) # Get last period of data (fence post) print 'Downloading data ' + Year_rng[-1].strftime('%Y%m%d%H%M') + ' through ' + EndDate.strftime('%Y%m%d%H%M') c_DS.append(Get_data(m,sta_id,Vars_ext,Year_rng[-1].strftime('%Y%m%d%H%M'),EndDate.strftime('%Y%m%d%H%M'))) print 'Finished downloading data' # In[8]: # In case any sites/variables were missing from a given year # Align each Dataset (fill missing stations/variables with NaN) newDS = xray.align(*c_DS, join='outer', copy=True) # In[9]: ## Combine Datasets by time ds_ALL = xray.concat(newDS,dim='time') # In[10]: ## Output to netcdf ds_ALL.to_netcdf(ncfilename) # In[ ]:
## adjust lat/lon dimensions since the index names are different tmin_lons_new = tmin_f['lon'].values[tmin_f['lon'].values > 180] - 360 tmin_f['lon'] = tmin_lons_new tmax_f['lon'] = tmin_lons_new q_f['lon'] = tmin_lons_new print("finished chunking data") ## cut out conus east of 103 for each variable swe_mask_file = '/raid9/gergel/agg_snowpack/goodleap/SWE/histmeanmask.nc' ## 1s are swe, 0s are no swe swe_mask = xray.open_dataset(swe_mask_file) ## rename dimensions swe_mask.rename({"Latitude": "lat", "Longitude": "lon", "Time": "time"}, inplace=True) swe_mask = swe_mask.squeeze() ## Dataset join swe_mask_align,tmax = xray.align(swe_mask,tmax_f,join='inner',copy=False) swe_mask_align,tmin = xray.align(swe_mask,tmin_f,join='inner',copy=False) swe_mask_align,q = xray.align(swe_mask,q_f,join='inner',copy=False) print("FEEL THE BERN!") ## get julian days julians = pd.DatetimeIndex(np.asarray(tmin.time)).dayofyear ## delete full arrays of each variable for conus del tmax_f,tmin_f,q_f,swe_mask,swe_mask_align gc.collect() # 100-hr and 1000-hr DFM FUNCTION # In[27]: