def test_dask_array(data_dir):
import dask.array as da
import h5py
from glob import glob
import os
client = Client(n_workers=4, processes=False)
filenames = sorted(
glob(os.path.join(data_dir, 'data', 'weather-big', '*.hdf5')))
dsets = [h5py.File(filename, mode='r')['/t2m'] for filename in filenames]
print(dsets[0])
#fig = plt.figure(figsize=(16, 8))
#plt.imshow(dsets[0][::4, ::4], cmap='RdBu_r')
#plt.show()
arrays = [da.from_array(dset, chunks=(500, 500)) for dset in dsets]
print(arrays)
x = da.stack(arrays, axis=0)
print('X : \n', x)
print('COMPUTE MEAN X')
result = x[0] - x.mean(axis=0)
print('PLOT MEAN X')
fig = plt.figure(figsize=(16, 8))
plt.imshow(result, cmap='RdBu_r')
plt.show()
client.close()
def NEON_create_refl_xarr_from_h5_file(h5file, nid='R10C', nodata=-9999):
# Read H5 file
f = h5.File(h5file, "r")
# spectral
wavelength = f[nid]['Reflectance']['Metadata']['Spectral_Data']['Wavelength'][:]
fwhm = f[nid]['Reflectance']['Metadata']['Spectral_Data']['FWHM'][:]
# CRS
crs_str = f[nid]['Reflectance']['Metadata']['Coordinate_System']['Coordinate_System_String'].value
crs_epsg = f[nid]['Reflectance']['Metadata']['Coordinate_System']['EPSG Code'].value
crs_mapinfo = f[nid]['Reflectance']['Metadata']['Coordinate_System']['Map_Info'].value
crs_proj4 = f[nid]['Reflectance']['Metadata']['Coordinate_System']['Proj4'].value
#arr = f[nid]['Reflectance']['Reflectance_Data'][:]
arr = da.from_array(f[nid]['Reflectance']['Reflectance_Data'], chunks=(256,256,256))
sf = f[nid]['Reflectance']['Reflectance_Data'].attrs['Scale_Factor']
mapinfo_list = [a.strip() for a in str(crs_mapinfo).split(',')]
mapinfo = [float(a) for a in mapinfo_list[1:7]]
mapinfo
pix_size = mapinfo[0]
x = np.arange(mapinfo[2], mapinfo[2] + pix_size*arr.shape[1], pix_size)
y = np.arange(mapinfo[3], mapinfo[3] - pix_size* arr.shape[0], -pix_size)
xr_cube = xr.DataArray(arr, {'y': y, 'x': x, 'bands': wavelength}, dims=['y', 'x', 'bands'])
xr_cube_ma = xr_cube.where(xr_cube != -9999)
return x, y, xr_cube_ma/sf
def dask_out_core_eval(dset):
import dask.array as da
x = da.from_array(dset, chunks=(1000000, ))
result = x.sum()
return result
print('Calcluating total number of CNN parameters.')
n = cnn_model_saver.get_weight_vector_length(cnn_hyperparam_file)
print('\tSuccessfully calcluated total number of CNN parameters.')
p = 50
#print('Creating files to store n-by-p manifold matrix')
#hdf5_file = h5py.File(output_dir + os.sep + 'model_weights' + os.sep + 'tmp_manifold.hdf5', "w")
#print('\tSuccessfully created the file')
print('Found weight vector length: ',n, '\n')
#A_columns = np.random.uniform(-1e8,1e8,size=(p)).astype(np.float32).tolist()
#A = np.stack([np.ones(shape=(n),dtype=np.float32)*col for col in A_columns],axis=1)
A = np.random.uniform(-1.0,1.0,size=(n,p))
assert A.shape==(n,p), 'Shape of A %s'%str(A.shape)
A = da.from_array(A, chunks=(n//100,p))
print(A[:10,:10].compute())
# A_plus = (A'A)−1A' (https://pythonhosted.org/algopy/examples/moore_penrose_pseudoinverse.html)
print('Calculating the psuedo inverse of A')
start_time = time.time()
A_plus = da.from_array(np.linalg.pinv(A),chunks=(p,n//100))
print(A_plus[:10, :10].compute())
start_time = time.time()
print('\tSuccessfully calculated the psuedo inverse of A (%d Secs)\n'%(time.time()-start_time))
#hdf5_A = hdf5_file.create_dataset('A', (n, p), dtype='f')
#hdf5_A_plus = hdf5_file.create_dataset('A_plus', (p, n), dtype='f')
print('Creating a weight vector from weight tensors')
W = session.run(get_weight_vector_with_variables(cnn_ops,n))
W_corrupt_placeholder = tf.placeholder(shape=[n],dtype=tf.float32,name='W_corrupt_ph')
print('\tSample W')