def random_connected_subset(nodes_, sz):
"""Returns a random connected subset of a component.
Because the subset needs to be connected, we grow it outwards from a random
element of the component.
"""
nodes = copy.copy(nodes_)
init_len = len(nodes)
# valid_moves = [(0,1), (-1,0), (0,-1), (1,0)]
assert len(
nodes
) >= sz, 'Cannot find a subset of size %i in a component size %i!' % (
sz, len(nodes))
node_i = random.randint(0, len(nodes) - 1)
queue = [nodes.pop(node_i)]
subset = []
while len(subset) < sz:
node = queue.pop(0)
subset.append(node)
random.shuffle(valid_moves)
moves_made = 0
for m in valid_moves:
adj_n = hsi_data.tupsum(node, m)
if adj_n in nodes and moves_made < 2:
moves_made += 1
i = nodes.index(adj_n)
nodes.pop(i)
queue.append(adj_n)
return subset[:sz]
def net_addl_padding_from_spec(spec):
"""
Here is a potential for huge confusion:
padding is (h,w,b)
BUT
specs are (b,h,w)
"""
b, h, w = list(tupsum(tuple(spec.psi1), tuple(spec.psi2), tuple(spec.phi), (-3,-3,-3)))
return (h,w,b)
def check_no_isolated_nodes(nodes):
for node in nodes:
nlinks = 0
for m in valid_moves:
adj_p = hsi_data.tupsum(m, node)
if adj_p in nodes:
nlinks += 1
if nlinks < 1:
return False
return True
def extract_connected_component(nodes):
"""Takes out one connected component out of nodes.
"""
queue = [nodes.pop(0)]
component = []
while len(queue):
p = queue.pop(0)
component.append(p)
for m in valid_moves:
adj_p = hsi_data.tupsum(m, p)
if adj_p in nodes:
i = nodes.index(adj_p)
queue.append(nodes.pop(i))
return component, nodes
def dlgrf_filter(data, kernel_size=[21,21,21], sigmas=[2,2,2], patch_size=101):
"""
"""
s = time.time()
filter_obj = win.dlrgf_factory(kernel_size, sigmas)
layer_params = layerO((1,1,1), 'valid')
[height, width, nbands] = data.shape
hyper_pixel_shape = (1, 1,data.shape[2])
padding = (kernel_size[0]-1, kernel_size[1]-1, 0)
ap = np.array(padding)
assert np.all(ap[:2] % 2 == 0), 'Assymetric padding is not supported'
padded_data = np.pad(data, ((ap[0]//2,ap[0]//2),(ap[1]//2,ap[1]//2),(ap[2]//2,ap[2]//2)), PAD_TYPE)
# cover the data with patches
patch_xs = [max(0,width - (x*patch_size)) for x in range(1, width // patch_size + 2)]
patch_ys = [max(0,height - (y*patch_size)) for y in range(1, height // patch_size + 2)]
patch_ul_corners = itertools.product(patch_xs, patch_ys) # upper left corners
addl_spatial_pad = (patch_size-1, patch_size-1, 0)
batch_item_shape = tupsum(hyper_pixel_shape, padding, addl_spatial_pad)
x = tf.placeholder(tf.float32, shape=batch_item_shape)
feat = fst.conv3dfeat(x, filter_obj, layer_params, patch_size)
feat_shape = tuple([int(d) for d in feat.shape])
new_data = np.zeros((height,width,feat_shape[2]))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for pixel_i, pixel in enumerate(tqdm(patch_ul_corners, desc='Performing DLGRF: ', total=len(patch_xs)*len(patch_ys))):
[pixel_x, pixel_y] = pixel
subimg = padded_data[pixel_y:(patch_size+pixel_y+ap[0]), pixel_x:(patch_size+pixel_x+ap[1]), :]
feed_dict = {x: subimg}
new_data[pixel_y:(patch_size+pixel_y), pixel_x:(patch_size+pixel_x)] = sess.run(feat, feed_dict)
tf.reset_default_graph()
print('DLGRF preprocessing finished in %is.' % int(time.time() - s))
return new_data
def preprocess_data(data, st_net_spec, patch_size=51):
"""ST preprocess the whole data cube.
Pad data, then pass in as few subsets of this data.
"""
s = time.time()
reuse = tf.AUTO_REUSE
# Network info
layer_params = layerO((1,1,1), 'valid')
preprocessing_mode = 'ST'
if st_net_spec.psi1 and st_net_spec.psi2 and st_net_spec.phi:
print('Will perform Scattering...')
psi1 = win.fst3d_psi_factory(st_net_spec.psi1)
psi2 = win.fst3d_psi_factory(st_net_spec.psi2)
psis=[psi1,psi2]
phi = win.fst3d_phi_window_3D(st_net_spec.phi)
net_addl_padding = net_addl_padding_from_spec(st_net_spec)
layer_params=[layer_params, layer_params, layer_params]
elif st_net_spec.psi1 and st_net_spec.psi2 is None and st_net_spec.phi is None: # just one layer gabor
print('Will perform Gabor filtering...')
psis = [win.gabor_psi_factory(st_net_spec.psi1)]
b, h, w = list(tupsum(tuple(st_net_spec.psi1), (-1,-1,-1)))
net_addl_padding = (h,w,b)
layer_params=[layer_params]
preprocessing_mode = 'Gabor'
elif st_net_spec.psi1 is None and st_net_spec.psi2 is None and st_net_spec.phi is None: # tang WST
# OK I am sorry for this temporary kludge, will make a new st_net_spec struct soon
print('Will perform Tang-WST...')
preprocessing_mode = 'Tang-WST'
kernel_size = [7,7,7]
max_scale = 3
K = 3
psi = win.tang_psi_factory(max_scale, K, kernel_size)
psis=[psi,psi]
phi = win.tang_phi_window_3D(max_scale, kernel_size)
net_addl_padding = net_addl_padding_from_spec(st_net_spec_struct(kernel_size,kernel_size,kernel_size))
layer_params=[layer_params, layer_params, layer_params]
else:
raise ValueError('This ST spec is not supported')
# END Network info
[height, width, nbands] = data.shape
hyper_pixel_shape = (1, 1,data.shape[2])
all_pixels = np.array(list(itertools.product(range(width),range(height))))
ap = np.array(net_addl_padding)
assert np.all(ap[:2] % 2 == 0), 'Assymetric padding is not supported'
padded_data = np.pad(data, ((ap[0]//2,ap[0]//2),(ap[1]//2,ap[1]//2),(ap[2]//2,ap[2]//2)), PAD_TYPE)
# cover the data with patches
patch_xs = [max(0,width - (x*patch_size)) for x in range(1, width // patch_size + 2)]
patch_ys = [max(0,height - (y*patch_size)) for y in range(1, height // patch_size + 2)]
patch_ul_corners = itertools.product(patch_xs, patch_ys) # upper left corners
addl_spatial_pad = (patch_size-1, patch_size-1, 0)
batch_item_shape = tupsum(hyper_pixel_shape, net_addl_padding, addl_spatial_pad)
x = tf.placeholder(tf.float32, shape=batch_item_shape)
print('Compiling Graph...')
compile_start = time.time()
if preprocessing_mode == 'ST':
feat = scat3d_to_3d_nxn_2layer(x, reuse, psis, phi, layer_params, final_size=patch_size)
elif preprocessing_mode == 'Gabor':
feat = gabor_mag_filter(x, reuse, psis, layer_params, final_size=patch_size)
elif preprocessing_mode == 'Tang-WST':
feat = scat3d_to_3d_nxn_2layer(x, reuse, psis, phi, layer_params, final_size=patch_size, tang_mode=True)
compile_time = time.time() - compile_start
feat_shape = tuple([int(d) for d in feat.shape])
print('Graph Compiled %is. Feature dimension per pixel is now %i.' % (int(compile_time), feat_shape[2]))
assert feat_shape[0] == feat_shape[1], 'ST spatial output is not square!'
assert feat_shape[0] == patch_size, 'ST spatial output size is %i, expected %i!' % (feat_shape[0], patch_size)
new_data = np.zeros((height,width,feat_shape[2]))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for pixel_i, pixel in enumerate(tqdm(patch_ul_corners, desc=('Performing %s: ' % preprocessing_mode), total=len(patch_xs)*len(patch_ys))):
[pixel_x, pixel_y] = pixel
subimg = padded_data[pixel_y:(patch_size+pixel_y+ap[0]), pixel_x:(patch_size+pixel_x+ap[1]), :]
feed_dict = {x: subimg}
new_data[pixel_y:(patch_size+pixel_y), pixel_x:(patch_size+pixel_x)] = sess.run(feat, feed_dict)
tf.reset_default_graph()
print('ST preprocessing finished in %is.' % int(time.time() - s))
return new_data