def random_connected_subset(nodes_, sz): """Returns a random connected subset of a component. Because the subset needs to be connected, we grow it outwards from a random element of the component. """ nodes = copy.copy(nodes_) init_len = len(nodes) # valid_moves = [(0,1), (-1,0), (0,-1), (1,0)] assert len( nodes ) >= sz, 'Cannot find a subset of size %i in a component size %i!' % ( sz, len(nodes)) node_i = random.randint(0, len(nodes) - 1) queue = [nodes.pop(node_i)] subset = [] while len(subset) < sz: node = queue.pop(0) subset.append(node) random.shuffle(valid_moves) moves_made = 0 for m in valid_moves: adj_n = hsi_data.tupsum(node, m) if adj_n in nodes and moves_made < 2: moves_made += 1 i = nodes.index(adj_n) nodes.pop(i) queue.append(adj_n) return subset[:sz]
def net_addl_padding_from_spec(spec): """ Here is a potential for huge confusion: padding is (h,w,b) BUT specs are (b,h,w) """ b, h, w = list(tupsum(tuple(spec.psi1), tuple(spec.psi2), tuple(spec.phi), (-3,-3,-3))) return (h,w,b)
def check_no_isolated_nodes(nodes): for node in nodes: nlinks = 0 for m in valid_moves: adj_p = hsi_data.tupsum(m, node) if adj_p in nodes: nlinks += 1 if nlinks < 1: return False return True
def extract_connected_component(nodes): """Takes out one connected component out of nodes. """ queue = [nodes.pop(0)] component = [] while len(queue): p = queue.pop(0) component.append(p) for m in valid_moves: adj_p = hsi_data.tupsum(m, p) if adj_p in nodes: i = nodes.index(adj_p) queue.append(nodes.pop(i)) return component, nodes
def dlgrf_filter(data, kernel_size=[21,21,21], sigmas=[2,2,2], patch_size=101): """ """ s = time.time() filter_obj = win.dlrgf_factory(kernel_size, sigmas) layer_params = layerO((1,1,1), 'valid') [height, width, nbands] = data.shape hyper_pixel_shape = (1, 1,data.shape[2]) padding = (kernel_size[0]-1, kernel_size[1]-1, 0) ap = np.array(padding) assert np.all(ap[:2] % 2 == 0), 'Assymetric padding is not supported' padded_data = np.pad(data, ((ap[0]//2,ap[0]//2),(ap[1]//2,ap[1]//2),(ap[2]//2,ap[2]//2)), PAD_TYPE) # cover the data with patches patch_xs = [max(0,width - (x*patch_size)) for x in range(1, width // patch_size + 2)] patch_ys = [max(0,height - (y*patch_size)) for y in range(1, height // patch_size + 2)] patch_ul_corners = itertools.product(patch_xs, patch_ys) # upper left corners addl_spatial_pad = (patch_size-1, patch_size-1, 0) batch_item_shape = tupsum(hyper_pixel_shape, padding, addl_spatial_pad) x = tf.placeholder(tf.float32, shape=batch_item_shape) feat = fst.conv3dfeat(x, filter_obj, layer_params, patch_size) feat_shape = tuple([int(d) for d in feat.shape]) new_data = np.zeros((height,width,feat_shape[2])) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for pixel_i, pixel in enumerate(tqdm(patch_ul_corners, desc='Performing DLGRF: ', total=len(patch_xs)*len(patch_ys))): [pixel_x, pixel_y] = pixel subimg = padded_data[pixel_y:(patch_size+pixel_y+ap[0]), pixel_x:(patch_size+pixel_x+ap[1]), :] feed_dict = {x: subimg} new_data[pixel_y:(patch_size+pixel_y), pixel_x:(patch_size+pixel_x)] = sess.run(feat, feed_dict) tf.reset_default_graph() print('DLGRF preprocessing finished in %is.' % int(time.time() - s)) return new_data
def preprocess_data(data, st_net_spec, patch_size=51): """ST preprocess the whole data cube. Pad data, then pass in as few subsets of this data. """ s = time.time() reuse = tf.AUTO_REUSE # Network info layer_params = layerO((1,1,1), 'valid') preprocessing_mode = 'ST' if st_net_spec.psi1 and st_net_spec.psi2 and st_net_spec.phi: print('Will perform Scattering...') psi1 = win.fst3d_psi_factory(st_net_spec.psi1) psi2 = win.fst3d_psi_factory(st_net_spec.psi2) psis=[psi1,psi2] phi = win.fst3d_phi_window_3D(st_net_spec.phi) net_addl_padding = net_addl_padding_from_spec(st_net_spec) layer_params=[layer_params, layer_params, layer_params] elif st_net_spec.psi1 and st_net_spec.psi2 is None and st_net_spec.phi is None: # just one layer gabor print('Will perform Gabor filtering...') psis = [win.gabor_psi_factory(st_net_spec.psi1)] b, h, w = list(tupsum(tuple(st_net_spec.psi1), (-1,-1,-1))) net_addl_padding = (h,w,b) layer_params=[layer_params] preprocessing_mode = 'Gabor' elif st_net_spec.psi1 is None and st_net_spec.psi2 is None and st_net_spec.phi is None: # tang WST # OK I am sorry for this temporary kludge, will make a new st_net_spec struct soon print('Will perform Tang-WST...') preprocessing_mode = 'Tang-WST' kernel_size = [7,7,7] max_scale = 3 K = 3 psi = win.tang_psi_factory(max_scale, K, kernel_size) psis=[psi,psi] phi = win.tang_phi_window_3D(max_scale, kernel_size) net_addl_padding = net_addl_padding_from_spec(st_net_spec_struct(kernel_size,kernel_size,kernel_size)) layer_params=[layer_params, layer_params, layer_params] else: raise ValueError('This ST spec is not supported') # END Network info [height, width, nbands] = data.shape hyper_pixel_shape = (1, 1,data.shape[2]) all_pixels = np.array(list(itertools.product(range(width),range(height)))) ap = np.array(net_addl_padding) assert np.all(ap[:2] % 2 == 0), 'Assymetric padding is not supported' padded_data = np.pad(data, ((ap[0]//2,ap[0]//2),(ap[1]//2,ap[1]//2),(ap[2]//2,ap[2]//2)), PAD_TYPE) # cover the data with patches patch_xs = [max(0,width - (x*patch_size)) for x in range(1, width // patch_size + 2)] patch_ys = [max(0,height - (y*patch_size)) for y in range(1, height // patch_size + 2)] patch_ul_corners = itertools.product(patch_xs, patch_ys) # upper left corners addl_spatial_pad = (patch_size-1, patch_size-1, 0) batch_item_shape = tupsum(hyper_pixel_shape, net_addl_padding, addl_spatial_pad) x = tf.placeholder(tf.float32, shape=batch_item_shape) print('Compiling Graph...') compile_start = time.time() if preprocessing_mode == 'ST': feat = scat3d_to_3d_nxn_2layer(x, reuse, psis, phi, layer_params, final_size=patch_size) elif preprocessing_mode == 'Gabor': feat = gabor_mag_filter(x, reuse, psis, layer_params, final_size=patch_size) elif preprocessing_mode == 'Tang-WST': feat = scat3d_to_3d_nxn_2layer(x, reuse, psis, phi, layer_params, final_size=patch_size, tang_mode=True) compile_time = time.time() - compile_start feat_shape = tuple([int(d) for d in feat.shape]) print('Graph Compiled %is. Feature dimension per pixel is now %i.' % (int(compile_time), feat_shape[2])) assert feat_shape[0] == feat_shape[1], 'ST spatial output is not square!' assert feat_shape[0] == patch_size, 'ST spatial output size is %i, expected %i!' % (feat_shape[0], patch_size) new_data = np.zeros((height,width,feat_shape[2])) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for pixel_i, pixel in enumerate(tqdm(patch_ul_corners, desc=('Performing %s: ' % preprocessing_mode), total=len(patch_xs)*len(patch_ys))): [pixel_x, pixel_y] = pixel subimg = padded_data[pixel_y:(patch_size+pixel_y+ap[0]), pixel_x:(patch_size+pixel_x+ap[1]), :] feed_dict = {x: subimg} new_data[pixel_y:(patch_size+pixel_y), pixel_x:(patch_size+pixel_x)] = sess.run(feat, feed_dict) tf.reset_default_graph() print('ST preprocessing finished in %is.' % int(time.time() - s)) return new_data