def Smith_net(reuse=tf.AUTO_REUSE): """Fully described network. This method is basically "data as code" Returns: struct with fields: addl_padding: amount of padding needed for an input to model_fn model_fn: function that takes: """ psi = win.fst3d_psi_factory([5, 9, 9]) phi = win.fst3d_phi_window_3D([5, 9, 9]) layer_params = layerO((3, 1, 1), 'valid') def model_fn(x): """ Args: x: tf.placeholder in (height, width, nbands) format, shape must be (25,25,nbands+6) """ return hyper3d_net( x, reuse=reuse, psis=[psi, psi], phi=phi, layer_params=[layer_params, layer_params, layer_params]) return netO(model_fn, (24, 24, 12))
def custom_net(s1=9, s2=9, e1=3, e2=3): psi1 = win.fst3d_psi_factory([e1, s1, s1]) psi2 = win.fst3d_psi_factory([e2, s2, s2]) phi = win.fst3d_phi_window_3D([e2, s2, s2]) lp1 = layerO((min(e1, 5), 1, 1), 'valid') lp2 = layerO((min(e2, 5), 1, 1), 'valid') lp3 = layerO((min(e2, 5), 1, 1), 'valid') def model_fn(x): return hyper3d_net(x, reuse=False, psis=[psi1, psi2], phi=phi, layer_params=[lp1, lp2, lp3]) return netO(model_fn, (s1 + s2 + s2 - 3, s1 + s2 + s2 - 3, e1 + e2 + e2 - 3))
def hyper3d_net(x, reuse=tf.AUTO_REUSE, psis=None, phi=None, layer_params=None): """Computes features for a specific pixel. Args: x: image in (height, width, bands) format psis: array of winO struct, filters are in (bands, height, width) format! phi: winO struct, filters are in (bands, height, width) format! Output: center pixel feature vector """ assert len(layer_params) == 3, 'this network is 2 layers only' assert len(psis) == 2, 'this network is 2 layers only' with tf.variable_scope('Hyper3DNet', reuse=reuse): x = tf.transpose(x, [2, 0, 1]) x = tf.expand_dims(x, 0) x = tf.expand_dims(x, -1) # x is (1, bands, h, w, 1) U1 = scat3d(x, psis[0], layer_params[0]) # U1 is (1, bands, h, w, lambda1) # swap channels with batch U1 = tf.transpose(U1, [4, 1, 2, 3, 0]) # U1 is (lambda1, bands, h, w, 1) U2s = [] # only procede with increasing frequency paths for res_i, used_params in enumerate(psis[0].filter_params): increasing_psi = win.fst3d_psi_factory(psis[1].kernel_size, used_params) if increasing_psi.nfilt > 0: U2s.append(scat3d(U1[res_i:(res_i+1),:,:,:,:], increasing_psi, layer_params[1])) U2 = tf.concat(U2s, 4) # swap channels with batch U2 = tf.transpose(U2, [4, 1, 2, 3, 0]) # convolve with phis S2 = scat3d(U2, phi, layer_params[2]) [p1h, p1w, p1b] = kernel_padding(psis[1].kernel_size) [p2h, p2w, p2b] = kernel_padding(psis[0].kernel_size) p2h += p1h; p2w += p1w; p2b += p1b; S1 = scat3d(U1[:,(p1h):-(p1h), (p1w):-(p1w), (p1b):-(p1b), :], phi, layer_params[2]) S0 = scat3d(x[:,(p2h):-(p2h), (p2w):-(p2w), (p2b):-(p2b), :], phi, layer_params[2]) # flatten everything S2 = tf.reshape(S2, [S2.shape[0] * S2.shape[1]]) # enforces last 3 dimensions being 1 S1 = tf.reshape(S1, [S1.shape[0] * S1.shape[1]]) # enforces last 3 dimensions being 1 S0 = tf.reshape(S0, [S0.shape[1]]) # enforces all but dim1 being 1 return tf.concat([S0,S1,S2], 0)
def KSC_net(reuse=tf.AUTO_REUSE): s = 11 psi1 = win.fst3d_psi_factory([3, s, s]) psi2 = win.fst3d_psi_factory([8, s, s]) phi = win.fst3d_phi_window_3D([8, s, s]) lp1 = layerO((1, 1, 1), 'valid') lp2 = layerO((8, 1, 1), 'valid') lp3 = layerO((8, 1, 1), 'valid') def model_fn(x): """ Args: x: tf.placeholder in (height, width, nbands) format, shape must be (25,25,nbands+6) """ return hyper3d_net(x, reuse=reuse, psis=[psi1, psi2], phi=phi, layer_params=[lp1, lp2, lp3]) return netO(model_fn, ((s - 1) * 3, (s - 1) * 3, 0))
def PaviaR_net(reuse=tf.AUTO_REUSE): psi = win.fst3d_psi_factory([3, 9, 9]) phi = win.fst3d_phi_window_3D([3, 9, 9]) layer_params = layerO((3, 1, 1), 'valid') def model_fn(x): """ Args: x: tf.placeholder in (height, width, nbands) format, shape must be (25,25,nbands+6) """ return hyper3d_net( x, reuse=reuse, psis=[psi, psi], phi=phi, layer_params=[layer_params, layer_params, layer_params]) return netO(model_fn, (24, 24, 6))
def scat2d_to_2d_2layer(x, reuse=tf.AUTO_REUSE, bs=batch_size): """ Args: x: in (batch, h, w, 1) shape Returns (batch, h, w, channels) """ psis = [None, None] layer_params = [None, None, None] with tf.variable_scope('scat2d_to_2d_2layer', reuse=reuse): # TF Estimator input is a dict, in case of multiple inputs psis[0] = win.fst3d_psi_factory([5, 5, 3]) layer_params[0] = layerO((1, 1), 'valid') # 107, 107 U1 = scat2d(x, psis[0], layer_params[0]) psis[1] = win.fst2d_psi_factory([5, 5]) layer_params[1] = layerO((1, 1), 'valid') U2s = [] # only procede with increasing frequency paths for res_i, used_params in enumerate(psis[0].filter_params): increasing_psi = win.fst2d_psi_factory(psis[1].kernel_size, used_params[:2]) if increasing_psi.nfilt > 0: U2s.append( scat2d(U1[:, :, :, res_i:(res_i + 1)], increasing_psi, layer_params[1])) # 101, 101 U2 = tf.concat(U2s, 3) # swap to (batch, chanU2, h, w) U2 = tf.transpose(U2, [0, 3, 1, 2]) # reshape to (batch, h,w, 1) U2os = U2.get_shape() U2 = tf.reshape( U2, (bs * U2.get_shape()[1], U2.get_shape()[2], U2.get_shape()[3], 1)) # swap to (batch, chanU1, h, w) U1 = tf.transpose(U1, [0, 3, 1, 2]) # reshape to (batch, h,w, 1) U1os = U1.get_shape() U1 = tf.reshape( U1, (bs * U1.get_shape()[1], U1.get_shape()[2], U1.get_shape()[3], 1)) # now lo-pass # each layer lo-passed differently so that (h,w) align bc we # want to be able to do 2d convolutions afterwards again layer_params[2] = layerO((1, 1), 'valid') phi = win.fst2d_phi_factory([5, 5]) # filter and separate by original batch via old shape S0 = scat2d(x[:, 4:-4, 4:-4, :], phi, layer_params[2]) S0 = tf.reshape(S0, (bs, 1, S0.get_shape()[1], S0.get_shape()[2])) S1 = scat2d(U1[:, 2:-2, 2:-2, :], phi, layer_params[2]) S1 = tf.reshape(S1, (bs, U1os[1], S1.get_shape()[1], S1.get_shape()[2])) S2 = scat2d(U2, phi, layer_params[2]) S2 = tf.reshape(S2, (bs, U2os[1], S2.get_shape()[1], S2.get_shape()[2])) # (batch, chan, h,w) feat2d = tf.concat([S0, S1, S2], 1) return tf.transpose(feat2d, [0, 2, 3, 1])
def scat3d_to_3d_nxn_2layer(x, reuse=tf.AUTO_REUSE, psis=None, phi=None, layer_params=None, final_size=5, tang_mode=False): """Computes features for a specific pixel. Args: x: image in (height, width, bands) format psis: array of winO struct, filters are in (bands, height, width) format! phi: winO struct, filters are in (bands, height, width) format! final_size: int, the outputs spatial size (will be a spatial square) Output: center pixel feature vector in the s """ assert len(layer_params) == 3, 'this network is 2 layers only' assert len(psis) == 2, 'this network is 2 layers only' with tf.variable_scope('Hyper3DNet', reuse=reuse): x = tf.transpose(x, [2, 0, 1]) x = tf.expand_dims(x, 0) x = tf.expand_dims(x, -1) # x is (1, bands, h, w, 1) U1 = fst.scat3d(x, psis[0], layer_params[0]) # U1 is (1, bands, h, w, lambda1) U1 = tf.transpose(U1, [0, 4, 1, 2, 3]) # U1 is (1, lambda1, bands, h, w) # downsampling amounts ds_amounts = { 9: 7, 7: 5, 5: 3, 3: 1, 1: 1 } def avg_cube_side(kernel_size): # started doing this for cubes with spatial size 1 avg_cube_side_ = int(round(np.prod(kernel_size[:2])**(1/2.0))) if avg_cube_side_ % 2 == 0: avg_cube_side_ += 1 return avg_cube_side_ lambda1_d = ds_amounts[ avg_cube_side(psis[0].kernel_size) ]; lambda2_d = ds_amounts[ avg_cube_side(psis[1].kernel_size) ]; lambdax_d = ds_amounts[ avg_cube_side(phi.kernel_size) ]; band1_d = 3 band2_d = 3 U1 = tf.layers.max_pooling3d(U1, (lambda1_d,band1_d,1), (lambda1_d,band1_d,1), padding='same') U1 = tf.transpose(U1, [1,2,3,4,0]) # U1 is (lambda1, bands, h, w, 1) U2s = [] # only procede with increasing frequency paths if tang_mode: for res_i, used_params in enumerate(psis[0].filter_params[::lambda1_d]): increasing_psi = win.tang_psi_factory(3, 3, psis[1].kernel_size, used_params[0]) if increasing_psi.nfilt > 0: U2s.append(fst.scat3d(U1[res_i:(res_i+1),:,:,:,:], increasing_psi, layer_params[1])) else: for res_i, used_params in enumerate(psis[0].filter_params[::lambda1_d]): increasing_psi = win.fst3d_psi_factory(psis[1].kernel_size, used_params) if increasing_psi.nfilt > 0: U2s.append(fst.scat3d(U1[res_i:(res_i+1),:,:,:,:], increasing_psi, layer_params[1])) U2 = tf.concat(U2s, 4) # U2 is (1,bands,h,w,lambda2) U2 = tf.transpose(U2, [0, 4, 1, 2, 3]) # U2 is (1, lambda2, bands, h, w) U2 = tf.layers.max_pooling3d(U2, (lambda2_d,band2_d,1), (lambda2_d,band2_d,1), padding='same') U2 = tf.transpose(U2, [1, 2, 3, 4, 0]) # U2 is (lambda2, bands, h, w, 1) # convolve with phi S2 = fst.scat3d(U2, phi, layer_params[2]) def slice_idxs(sig_size, kernel_size): """ return slice indexes to slice signal so that after convolving with the kernel it is the desired final size """ def slice_idx(s, k, f): """ s: signal size k: kernel size f: final size """ if k % 2 == 0: raise('not implemented even padding') else: return int((s - k - f)//2) final_size_ = [1,final_size,final_size] return [slice_idx(s,k,f-1) for s,k,f in zip(sig_size, kernel_size,final_size_)] # we will stack S0,S1,S2 so we need to trim them to be the same size # after each of them are convolved with phi [p1b, p1h, p1w] = slice_idxs(U1.shape[1:4], phi.kernel_size) [p2b, p2h, p2w] = slice_idxs(x.shape[1:4], phi.kernel_size) if not (p1h == 0 and p1w == 0): S1 = fst.scat3d(U1[:, :,(p1h):-(p1h), (p1w):-(p1w), :], phi, layer_params[2]) else: # if the size of the spatial kernel is 1 in the spatial dimension we # don't need to do this. S1 = fst.scat3d(U1, phi, layer_params[2]) if not (p2h == 0 and p2w == 0): S0 = fst.scat3d(x[:, :,(p2h):-(p2h), (p2w):-(p2w), :], phi, layer_params[2]) else: S0 = fst.scat3d(x, phi, layer_params[2]) # just to get the size down to 1 (flattening step) S0 = tf.reshape(S0, [-1, final_size, final_size, 1]) S1 = tf.reshape(S1, [-1, final_size, final_size, 1]) S2 = tf.reshape(S2, [-1, final_size, final_size, 1]) SX = tf.concat([S0,S1,S2], 0) # SX is (lambdax, h, w, 1) SX = tf.expand_dims(SX, 0) SX = tf.layers.max_pooling3d(SX, (lambdax_d,1,1), (lambdax_d,1,1), padding='same') SX = tf.squeeze(SX) # SX is (channels, h, w) return tf.transpose(SX, [1, 2, 0])
def preprocess_data(data, st_net_spec, patch_size=51): """ST preprocess the whole data cube. Pad data, then pass in as few subsets of this data. """ s = time.time() reuse = tf.AUTO_REUSE # Network info layer_params = layerO((1,1,1), 'valid') preprocessing_mode = 'ST' if st_net_spec.psi1 and st_net_spec.psi2 and st_net_spec.phi: print('Will perform Scattering...') psi1 = win.fst3d_psi_factory(st_net_spec.psi1) psi2 = win.fst3d_psi_factory(st_net_spec.psi2) psis=[psi1,psi2] phi = win.fst3d_phi_window_3D(st_net_spec.phi) net_addl_padding = net_addl_padding_from_spec(st_net_spec) layer_params=[layer_params, layer_params, layer_params] elif st_net_spec.psi1 and st_net_spec.psi2 is None and st_net_spec.phi is None: # just one layer gabor print('Will perform Gabor filtering...') psis = [win.gabor_psi_factory(st_net_spec.psi1)] b, h, w = list(tupsum(tuple(st_net_spec.psi1), (-1,-1,-1))) net_addl_padding = (h,w,b) layer_params=[layer_params] preprocessing_mode = 'Gabor' elif st_net_spec.psi1 is None and st_net_spec.psi2 is None and st_net_spec.phi is None: # tang WST # OK I am sorry for this temporary kludge, will make a new st_net_spec struct soon print('Will perform Tang-WST...') preprocessing_mode = 'Tang-WST' kernel_size = [7,7,7] max_scale = 3 K = 3 psi = win.tang_psi_factory(max_scale, K, kernel_size) psis=[psi,psi] phi = win.tang_phi_window_3D(max_scale, kernel_size) net_addl_padding = net_addl_padding_from_spec(st_net_spec_struct(kernel_size,kernel_size,kernel_size)) layer_params=[layer_params, layer_params, layer_params] else: raise ValueError('This ST spec is not supported') # END Network info [height, width, nbands] = data.shape hyper_pixel_shape = (1, 1,data.shape[2]) all_pixels = np.array(list(itertools.product(range(width),range(height)))) ap = np.array(net_addl_padding) assert np.all(ap[:2] % 2 == 0), 'Assymetric padding is not supported' padded_data = np.pad(data, ((ap[0]//2,ap[0]//2),(ap[1]//2,ap[1]//2),(ap[2]//2,ap[2]//2)), PAD_TYPE) # cover the data with patches patch_xs = [max(0,width - (x*patch_size)) for x in range(1, width // patch_size + 2)] patch_ys = [max(0,height - (y*patch_size)) for y in range(1, height // patch_size + 2)] patch_ul_corners = itertools.product(patch_xs, patch_ys) # upper left corners addl_spatial_pad = (patch_size-1, patch_size-1, 0) batch_item_shape = tupsum(hyper_pixel_shape, net_addl_padding, addl_spatial_pad) x = tf.placeholder(tf.float32, shape=batch_item_shape) print('Compiling Graph...') compile_start = time.time() if preprocessing_mode == 'ST': feat = scat3d_to_3d_nxn_2layer(x, reuse, psis, phi, layer_params, final_size=patch_size) elif preprocessing_mode == 'Gabor': feat = gabor_mag_filter(x, reuse, psis, layer_params, final_size=patch_size) elif preprocessing_mode == 'Tang-WST': feat = scat3d_to_3d_nxn_2layer(x, reuse, psis, phi, layer_params, final_size=patch_size, tang_mode=True) compile_time = time.time() - compile_start feat_shape = tuple([int(d) for d in feat.shape]) print('Graph Compiled %is. Feature dimension per pixel is now %i.' % (int(compile_time), feat_shape[2])) assert feat_shape[0] == feat_shape[1], 'ST spatial output is not square!' assert feat_shape[0] == patch_size, 'ST spatial output size is %i, expected %i!' % (feat_shape[0], patch_size) new_data = np.zeros((height,width,feat_shape[2])) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for pixel_i, pixel in enumerate(tqdm(patch_ul_corners, desc=('Performing %s: ' % preprocessing_mode), total=len(patch_xs)*len(patch_ys))): [pixel_x, pixel_y] = pixel subimg = padded_data[pixel_y:(patch_size+pixel_y+ap[0]), pixel_x:(patch_size+pixel_x+ap[1]), :] feed_dict = {x: subimg} new_data[pixel_y:(patch_size+pixel_y), pixel_x:(patch_size+pixel_x)] = sess.run(feat, feed_dict) tf.reset_default_graph() print('ST preprocessing finished in %is.' % int(time.time() - s)) return new_data