def Smith_net(reuse=tf.AUTO_REUSE):
"""Fully described network.
This method is basically "data as code"
Returns:
struct with fields:
addl_padding: amount of padding needed for an input to model_fn
model_fn: function that takes:
"""
psi = win.fst3d_psi_factory([5, 9, 9])
phi = win.fst3d_phi_window_3D([5, 9, 9])
layer_params = layerO((3, 1, 1), 'valid')
def model_fn(x):
"""
Args:
x: tf.placeholder in (height, width, nbands) format, shape must be (25,25,nbands+6)
"""
return hyper3d_net(
x,
reuse=reuse,
psis=[psi, psi],
phi=phi,
layer_params=[layer_params, layer_params, layer_params])
return netO(model_fn, (24, 24, 12))
def custom_net(s1=9, s2=9, e1=3, e2=3):
psi1 = win.fst3d_psi_factory([e1, s1, s1])
psi2 = win.fst3d_psi_factory([e2, s2, s2])
phi = win.fst3d_phi_window_3D([e2, s2, s2])
lp1 = layerO((min(e1, 5), 1, 1), 'valid')
lp2 = layerO((min(e2, 5), 1, 1), 'valid')
lp3 = layerO((min(e2, 5), 1, 1), 'valid')
def model_fn(x):
return hyper3d_net(x,
reuse=False,
psis=[psi1, psi2],
phi=phi,
layer_params=[lp1, lp2, lp3])
return netO(model_fn,
(s1 + s2 + s2 - 3, s1 + s2 + s2 - 3, e1 + e2 + e2 - 3))
def hyper3d_net(x, reuse=tf.AUTO_REUSE, psis=None, phi=None, layer_params=None):
"""Computes features for a specific pixel.
Args:
x: image in (height, width, bands) format
psis: array of winO struct, filters are in (bands, height, width) format!
phi: winO struct, filters are in (bands, height, width) format!
Output:
center pixel feature vector
"""
assert len(layer_params) == 3, 'this network is 2 layers only'
assert len(psis) == 2, 'this network is 2 layers only'
with tf.variable_scope('Hyper3DNet', reuse=reuse):
x = tf.transpose(x, [2, 0, 1])
x = tf.expand_dims(x, 0)
x = tf.expand_dims(x, -1)
# x is (1, bands, h, w, 1)
U1 = scat3d(x, psis[0], layer_params[0])
# U1 is (1, bands, h, w, lambda1)
# swap channels with batch
U1 = tf.transpose(U1, [4, 1, 2, 3, 0])
# U1 is (lambda1, bands, h, w, 1)
U2s = []
# only procede with increasing frequency paths
for res_i, used_params in enumerate(psis[0].filter_params):
increasing_psi = win.fst3d_psi_factory(psis[1].kernel_size, used_params)
if increasing_psi.nfilt > 0:
U2s.append(scat3d(U1[res_i:(res_i+1),:,:,:,:], increasing_psi, layer_params[1]))
U2 = tf.concat(U2s, 4)
# swap channels with batch
U2 = tf.transpose(U2, [4, 1, 2, 3, 0])
# convolve with phis
S2 = scat3d(U2, phi, layer_params[2])
[p1h, p1w, p1b] = kernel_padding(psis[1].kernel_size)
[p2h, p2w, p2b] = kernel_padding(psis[0].kernel_size)
p2h += p1h; p2w += p1w; p2b += p1b;
S1 = scat3d(U1[:,(p1h):-(p1h), (p1w):-(p1w), (p1b):-(p1b), :], phi, layer_params[2])
S0 = scat3d(x[:,(p2h):-(p2h), (p2w):-(p2w), (p2b):-(p2b), :], phi, layer_params[2])
# flatten everything
S2 = tf.reshape(S2, [S2.shape[0] * S2.shape[1]]) # enforces last 3 dimensions being 1
S1 = tf.reshape(S1, [S1.shape[0] * S1.shape[1]]) # enforces last 3 dimensions being 1
S0 = tf.reshape(S0, [S0.shape[1]]) # enforces all but dim1 being 1
return tf.concat([S0,S1,S2], 0)
def KSC_net(reuse=tf.AUTO_REUSE):
s = 11
psi1 = win.fst3d_psi_factory([3, s, s])
psi2 = win.fst3d_psi_factory([8, s, s])
phi = win.fst3d_phi_window_3D([8, s, s])
lp1 = layerO((1, 1, 1), 'valid')
lp2 = layerO((8, 1, 1), 'valid')
lp3 = layerO((8, 1, 1), 'valid')
def model_fn(x):
"""
Args:
x: tf.placeholder in (height, width, nbands) format, shape must be (25,25,nbands+6)
"""
return hyper3d_net(x,
reuse=reuse,
psis=[psi1, psi2],
phi=phi,
layer_params=[lp1, lp2, lp3])
return netO(model_fn, ((s - 1) * 3, (s - 1) * 3, 0))
def PaviaR_net(reuse=tf.AUTO_REUSE):
psi = win.fst3d_psi_factory([3, 9, 9])
phi = win.fst3d_phi_window_3D([3, 9, 9])
layer_params = layerO((3, 1, 1), 'valid')
def model_fn(x):
"""
Args:
x: tf.placeholder in (height, width, nbands) format, shape must be (25,25,nbands+6)
"""
return hyper3d_net(
x,
reuse=reuse,
psis=[psi, psi],
phi=phi,
layer_params=[layer_params, layer_params, layer_params])
return netO(model_fn, (24, 24, 6))
def scat2d_to_2d_2layer(x, reuse=tf.AUTO_REUSE, bs=batch_size):
"""
Args:
x: in (batch, h, w, 1) shape
Returns
(batch, h, w, channels)
"""
psis = [None, None]
layer_params = [None, None, None]
with tf.variable_scope('scat2d_to_2d_2layer', reuse=reuse):
# TF Estimator input is a dict, in case of multiple inputs
psis[0] = win.fst3d_psi_factory([5, 5, 3])
layer_params[0] = layerO((1, 1), 'valid')
# 107, 107
U1 = scat2d(x, psis[0], layer_params[0])
psis[1] = win.fst2d_psi_factory([5, 5])
layer_params[1] = layerO((1, 1), 'valid')
U2s = []
# only procede with increasing frequency paths
for res_i, used_params in enumerate(psis[0].filter_params):
increasing_psi = win.fst2d_psi_factory(psis[1].kernel_size,
used_params[:2])
if increasing_psi.nfilt > 0:
U2s.append(
scat2d(U1[:, :, :, res_i:(res_i + 1)], increasing_psi,
layer_params[1]))
# 101, 101
U2 = tf.concat(U2s, 3)
# swap to (batch, chanU2, h, w)
U2 = tf.transpose(U2, [0, 3, 1, 2])
# reshape to (batch, h,w, 1)
U2os = U2.get_shape()
U2 = tf.reshape(
U2,
(bs * U2.get_shape()[1], U2.get_shape()[2], U2.get_shape()[3], 1))
# swap to (batch, chanU1, h, w)
U1 = tf.transpose(U1, [0, 3, 1, 2])
# reshape to (batch, h,w, 1)
U1os = U1.get_shape()
U1 = tf.reshape(
U1,
(bs * U1.get_shape()[1], U1.get_shape()[2], U1.get_shape()[3], 1))
# now lo-pass
# each layer lo-passed differently so that (h,w) align bc we
# want to be able to do 2d convolutions afterwards again
layer_params[2] = layerO((1, 1), 'valid')
phi = win.fst2d_phi_factory([5, 5])
# filter and separate by original batch via old shape
S0 = scat2d(x[:, 4:-4, 4:-4, :], phi, layer_params[2])
S0 = tf.reshape(S0, (bs, 1, S0.get_shape()[1], S0.get_shape()[2]))
S1 = scat2d(U1[:, 2:-2, 2:-2, :], phi, layer_params[2])
S1 = tf.reshape(S1,
(bs, U1os[1], S1.get_shape()[1], S1.get_shape()[2]))
S2 = scat2d(U2, phi, layer_params[2])
S2 = tf.reshape(S2,
(bs, U2os[1], S2.get_shape()[1], S2.get_shape()[2]))
# (batch, chan, h,w)
feat2d = tf.concat([S0, S1, S2], 1)
return tf.transpose(feat2d, [0, 2, 3, 1])
def scat3d_to_3d_nxn_2layer(x, reuse=tf.AUTO_REUSE, psis=None, phi=None, layer_params=None, final_size=5, tang_mode=False):
"""Computes features for a specific pixel.
Args:
x: image in (height, width, bands) format
psis: array of winO struct, filters are in (bands, height, width) format!
phi: winO struct, filters are in (bands, height, width) format!
final_size: int, the outputs spatial size (will be a spatial square)
Output:
center pixel feature vector in the s
"""
assert len(layer_params) == 3, 'this network is 2 layers only'
assert len(psis) == 2, 'this network is 2 layers only'
with tf.variable_scope('Hyper3DNet', reuse=reuse):
x = tf.transpose(x, [2, 0, 1])
x = tf.expand_dims(x, 0)
x = tf.expand_dims(x, -1)
# x is (1, bands, h, w, 1)
U1 = fst.scat3d(x, psis[0], layer_params[0])
# U1 is (1, bands, h, w, lambda1)
U1 = tf.transpose(U1, [0, 4, 1, 2, 3])
# U1 is (1, lambda1, bands, h, w)
# downsampling amounts
ds_amounts = {
9: 7,
7: 5,
5: 3,
3: 1,
1: 1
}
def avg_cube_side(kernel_size):
# started doing this for cubes with spatial size 1
avg_cube_side_ = int(round(np.prod(kernel_size[:2])**(1/2.0)))
if avg_cube_side_ % 2 == 0:
avg_cube_side_ += 1
return avg_cube_side_
lambda1_d = ds_amounts[ avg_cube_side(psis[0].kernel_size) ];
lambda2_d = ds_amounts[ avg_cube_side(psis[1].kernel_size) ];
lambdax_d = ds_amounts[ avg_cube_side(phi.kernel_size) ];
band1_d = 3
band2_d = 3
U1 = tf.layers.max_pooling3d(U1, (lambda1_d,band1_d,1), (lambda1_d,band1_d,1), padding='same')
U1 = tf.transpose(U1, [1,2,3,4,0])
# U1 is (lambda1, bands, h, w, 1)
U2s = []
# only procede with increasing frequency paths
if tang_mode:
for res_i, used_params in enumerate(psis[0].filter_params[::lambda1_d]):
increasing_psi = win.tang_psi_factory(3, 3, psis[1].kernel_size, used_params[0])
if increasing_psi.nfilt > 0:
U2s.append(fst.scat3d(U1[res_i:(res_i+1),:,:,:,:], increasing_psi, layer_params[1]))
else:
for res_i, used_params in enumerate(psis[0].filter_params[::lambda1_d]):
increasing_psi = win.fst3d_psi_factory(psis[1].kernel_size, used_params)
if increasing_psi.nfilt > 0:
U2s.append(fst.scat3d(U1[res_i:(res_i+1),:,:,:,:], increasing_psi, layer_params[1]))
U2 = tf.concat(U2s, 4)
# U2 is (1,bands,h,w,lambda2)
U2 = tf.transpose(U2, [0, 4, 1, 2, 3])
# U2 is (1, lambda2, bands, h, w)
U2 = tf.layers.max_pooling3d(U2, (lambda2_d,band2_d,1), (lambda2_d,band2_d,1), padding='same')
U2 = tf.transpose(U2, [1, 2, 3, 4, 0])
# U2 is (lambda2, bands, h, w, 1)
# convolve with phi
S2 = fst.scat3d(U2, phi, layer_params[2])
def slice_idxs(sig_size, kernel_size):
"""
return slice indexes to slice signal so that after convolving with
the kernel it is the desired final size
"""
def slice_idx(s, k, f):
"""
s: signal size
k: kernel size
f: final size
"""
if k % 2 == 0:
raise('not implemented even padding')
else:
return int((s - k - f)//2)
final_size_ = [1,final_size,final_size]
return [slice_idx(s,k,f-1) for s,k,f in zip(sig_size, kernel_size,final_size_)]
# we will stack S0,S1,S2 so we need to trim them to be the same size
# after each of them are convolved with phi
[p1b, p1h, p1w] = slice_idxs(U1.shape[1:4], phi.kernel_size)
[p2b, p2h, p2w] = slice_idxs(x.shape[1:4], phi.kernel_size)
if not (p1h == 0 and p1w == 0):
S1 = fst.scat3d(U1[:, :,(p1h):-(p1h), (p1w):-(p1w), :], phi, layer_params[2])
else:
# if the size of the spatial kernel is 1 in the spatial dimension we
# don't need to do this.
S1 = fst.scat3d(U1, phi, layer_params[2])
if not (p2h == 0 and p2w == 0):
S0 = fst.scat3d(x[:, :,(p2h):-(p2h), (p2w):-(p2w), :], phi, layer_params[2])
else:
S0 = fst.scat3d(x, phi, layer_params[2])
# just to get the size down to 1 (flattening step)
S0 = tf.reshape(S0, [-1, final_size, final_size, 1])
S1 = tf.reshape(S1, [-1, final_size, final_size, 1])
S2 = tf.reshape(S2, [-1, final_size, final_size, 1])
SX = tf.concat([S0,S1,S2], 0)
# SX is (lambdax, h, w, 1)
SX = tf.expand_dims(SX, 0)
SX = tf.layers.max_pooling3d(SX, (lambdax_d,1,1), (lambdax_d,1,1), padding='same')
SX = tf.squeeze(SX)
# SX is (channels, h, w)
return tf.transpose(SX, [1, 2, 0])
def preprocess_data(data, st_net_spec, patch_size=51):
"""ST preprocess the whole data cube.
Pad data, then pass in as few subsets of this data.
"""
s = time.time()
reuse = tf.AUTO_REUSE
# Network info
layer_params = layerO((1,1,1), 'valid')
preprocessing_mode = 'ST'
if st_net_spec.psi1 and st_net_spec.psi2 and st_net_spec.phi:
print('Will perform Scattering...')
psi1 = win.fst3d_psi_factory(st_net_spec.psi1)
psi2 = win.fst3d_psi_factory(st_net_spec.psi2)
psis=[psi1,psi2]
phi = win.fst3d_phi_window_3D(st_net_spec.phi)
net_addl_padding = net_addl_padding_from_spec(st_net_spec)
layer_params=[layer_params, layer_params, layer_params]
elif st_net_spec.psi1 and st_net_spec.psi2 is None and st_net_spec.phi is None: # just one layer gabor
print('Will perform Gabor filtering...')
psis = [win.gabor_psi_factory(st_net_spec.psi1)]
b, h, w = list(tupsum(tuple(st_net_spec.psi1), (-1,-1,-1)))
net_addl_padding = (h,w,b)
layer_params=[layer_params]
preprocessing_mode = 'Gabor'
elif st_net_spec.psi1 is None and st_net_spec.psi2 is None and st_net_spec.phi is None: # tang WST
# OK I am sorry for this temporary kludge, will make a new st_net_spec struct soon
print('Will perform Tang-WST...')
preprocessing_mode = 'Tang-WST'
kernel_size = [7,7,7]
max_scale = 3
K = 3
psi = win.tang_psi_factory(max_scale, K, kernel_size)
psis=[psi,psi]
phi = win.tang_phi_window_3D(max_scale, kernel_size)
net_addl_padding = net_addl_padding_from_spec(st_net_spec_struct(kernel_size,kernel_size,kernel_size))
layer_params=[layer_params, layer_params, layer_params]
else:
raise ValueError('This ST spec is not supported')
# END Network info
[height, width, nbands] = data.shape
hyper_pixel_shape = (1, 1,data.shape[2])
all_pixels = np.array(list(itertools.product(range(width),range(height))))
ap = np.array(net_addl_padding)
assert np.all(ap[:2] % 2 == 0), 'Assymetric padding is not supported'
padded_data = np.pad(data, ((ap[0]//2,ap[0]//2),(ap[1]//2,ap[1]//2),(ap[2]//2,ap[2]//2)), PAD_TYPE)
# cover the data with patches
patch_xs = [max(0,width - (x*patch_size)) for x in range(1, width // patch_size + 2)]
patch_ys = [max(0,height - (y*patch_size)) for y in range(1, height // patch_size + 2)]
patch_ul_corners = itertools.product(patch_xs, patch_ys) # upper left corners
addl_spatial_pad = (patch_size-1, patch_size-1, 0)
batch_item_shape = tupsum(hyper_pixel_shape, net_addl_padding, addl_spatial_pad)
x = tf.placeholder(tf.float32, shape=batch_item_shape)
print('Compiling Graph...')
compile_start = time.time()
if preprocessing_mode == 'ST':
feat = scat3d_to_3d_nxn_2layer(x, reuse, psis, phi, layer_params, final_size=patch_size)
elif preprocessing_mode == 'Gabor':
feat = gabor_mag_filter(x, reuse, psis, layer_params, final_size=patch_size)
elif preprocessing_mode == 'Tang-WST':
feat = scat3d_to_3d_nxn_2layer(x, reuse, psis, phi, layer_params, final_size=patch_size, tang_mode=True)
compile_time = time.time() - compile_start
feat_shape = tuple([int(d) for d in feat.shape])
print('Graph Compiled %is. Feature dimension per pixel is now %i.' % (int(compile_time), feat_shape[2]))
assert feat_shape[0] == feat_shape[1], 'ST spatial output is not square!'
assert feat_shape[0] == patch_size, 'ST spatial output size is %i, expected %i!' % (feat_shape[0], patch_size)
new_data = np.zeros((height,width,feat_shape[2]))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for pixel_i, pixel in enumerate(tqdm(patch_ul_corners, desc=('Performing %s: ' % preprocessing_mode), total=len(patch_xs)*len(patch_ys))):
[pixel_x, pixel_y] = pixel
subimg = padded_data[pixel_y:(patch_size+pixel_y+ap[0]), pixel_x:(patch_size+pixel_x+ap[1]), :]
feed_dict = {x: subimg}
new_data[pixel_y:(patch_size+pixel_y), pixel_x:(patch_size+pixel_x)] = sess.run(feat, feed_dict)
tf.reset_default_graph()
print('ST preprocessing finished in %is.' % int(time.time() - s))
return new_data