dr.trainable_variables_nodes = [ v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) ] # y_hat are re-generated after each global updating # gradients of each segments are calculated all according to this y_hat # namely, Wxx, Wxxu, and Wxu are not updated in each segment Wxx = np.identity(3) * (1 - 1.6 * du.get('t_delta')) Wxxu = np.zeros((3, 3)) Wxu = np.array([0., 0., 0.]).reshape(3, 1) # y_regenerated = regenerate_data(du, Wxx, Wxxu, Wxu) data = { 'u': tb.split(du.get('u'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data), 'x_initial': tb.split(du.get('x'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=0), 'h_initial': tb.split(du.get('h'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=1), 'x_whole': tb.split(du.get('x'), n_segment=dr.n_recurrent_step + dr.shift_u_x, n_step=dr.shift_data, shift=0),
# modify spm_data if necessary # iterator = itertools.product(*[package_list, ['H_PARA_INITIAL'], [values]]) # package_list = p.starmap(tm.modify_signel_data_package, iterator) # build graph and training must be done in one function iterator = itertools.product(*[[dr], package_list]) package_list = p.starmap(tm.build_initializer_graph_and_train, iterator) data = package_list[0].data assert 'loss_smooth_normalizer' in data.keys() signal_length = data['x_true_merged'].data.shape[0] check_length = tm.N_SEGMENTS * tm.DATA_SHIFT # test split and merge recovered = tb.merge(tb.split(du.get('x'), 64, 4), 64, 4) original = du.get('x')[:len(recovered)] np.testing.assert_array_almost_equal(recovered, original) # check recorded u and processed u original = du.get('u')[:check_length] recovered = data['u_merged'][:check_length] np.testing.assert_array_almost_equal(recovered, original) plt.plot(original, '-', label="original") plt.plot(recovered, '*', label="processed") plt.show() # check recorded x_true and the original x_true original = du.get('u')[:check_length] recovered = data['u_merged'][:check_length] np.testing.assert_array_almost_equal(recovered, original)
} dr.build_main_graph_parallel(neural_parameter_initial=neural_parameter_initial) # process after building the main graph dr.support_masks = dr.setup_support_mask(mask) dr.x_parameter_nodes = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=dr.variable_scope_name_x_parameter)] dr.trainable_variables_nodes = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] dr.trainable_variables_names = [v.name for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] du_hat.variable_names_in_graph = du_hat.parse_variable_names(dr.trainable_variables_names) # prepare data for training data_hat = { 'x_initial': tb.split(du_hat.get('x'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=0), 'h_initial': tb.split(du_hat.get('h'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=1), } data = { 'u': tb.split(du.get('u'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data), 'y': tb.split(du.get('y'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=dr.shift_u_y)} for k in data_hat.keys(): data[k] = data_hat[k][: N_SEGMENTS] for k in data.keys(): data[k] = data[k][: N_SEGMENTS] batches = tb.make_batches(data['u'], data_hat['x_initial'], data_hat['h_initial'], data['y'], batch_size=dr.batch_size, if_shuffle=True) print('start session') # start session isess = tf.InteractiveSession()
dr.build_main_graph_parallel(neural_parameter_initial=neural_parameter_initial) # process after building the main graph dr.support_masks = dr.setup_support_mask(mask) dr.x_parameter_nodes = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=dr.variable_scope_name_x_parameter)] dr.trainable_variables_nodes = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] dr.trainable_variables_names = [v.name for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] du_hat.variable_names_in_graph = du_hat.parse_variable_names(dr.trainable_variables_names) # prepare data for training # prepare data for training data = { 'u': tb.split(du.get('u'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data), 'x_initial': tb.split(du.get('x'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=0), 'h_initial': tb.split(du.get('h'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=1), 'x_whole': tb.split(du.get('x'), n_segment=dr.n_recurrent_step + dr.shift_u_x, n_step=dr.shift_data, shift=0), 'h_whole': tb.split(du.get('h'), n_segment=dr.n_recurrent_step + dr.shift_x_y, n_step=dr.shift_data, shift=1), 'h_predicted': tb.split(du.get('h'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=dr.shift_u_y), 'y': tb.split(du.get('y'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=dr.shift_u_y)} data_hat = { 'x_initial': tb.split(du_hat.get('x'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=0), 'h_initial': tb.split(du_hat.get('h'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=1), } for k in data_hat.keys(): data[k] = data_hat[k][: N_SEGMENTS] for k in data.keys(): data[k] = data[k][: N_SEGMENTS] batches = tb.make_batches(data['u'], data_hat['x_initial'], data_hat['h_initial'], data['y'],
mask[dr.Wxxu[2].name][0, 1] = 1 mask[dr.Wxu.name][0, 0] = 1 dr.support_masks = dr.setup_support_mask(mask) dr.x_parameter_nodes = [ v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=dr.variable_scope_name_x_parameter) ] dr.trainable_variables_nodes = [ v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) ] # prepare spm_data for training data = { 'u': tb.split(spm_data['u_upsampled'], n_segment=dr.n_recurrent_step, n_step=dr.shift_data), 'y': tb.split(spm_data['y_upsampled'], n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=dr.shift_u_y), } for k in data.keys(): data[k] = data[k][:N_SEGMENTS] N_TEST_SAMPLE = min(N_SEGMENTS, len(data['y'])) def apply_and_check(isess, grads_and_vars, step_size, u, x_connector, h_connector, y_true):
def prepare_data(max_segments=None, node_index=None): global data global loss_x_normalizer global loss_y_normalizer global loss_smooth_normalizer global isess global SEQUENCE_LENGTH global H_STATE_INITIAL global IF_NOISED_Y global SNR global NOISE if IF_NOISED_Y: std = np.std(du.get('y').reshape([-1])) / SNR NOISE = np.random.normal(0, std, du.get('y').shape) else: NOISE = np.zeros(du.get('y').shape) ## saved line. SEQUENCE_LENGTH = dr.n_recurrent_step + (len(spm_data['x_true']) - 1) * dr.shift_data data['y_train'] = tb.split(du.get('y') + NOISE, dr.n_recurrent_step, dr.shift_data, dr.shift_x_y) max_segments_natural = len(data['y_train']) data['y_true'] = tb.split(du.get('y'), dr.n_recurrent_step, dr.shift_data, dr.shift_x_y)[:max_segments_natural] data['h_true_monitor'] = tb.split(du.get('h'), dr.n_recurrent_step, dr.shift_data)[:max_segments_natural] data['x_true'] = tb.split(du.get('x'), dr.n_recurrent_step, dr.shift_data)[:max_segments_natural] data['u'] = tb.split(du.get('u'), dr.n_recurrent_step, dr.shift_data)[:max_segments_natural] if max_segments is not None: if max_segments > max_segments_natural: warnings.warn("max_segments is larger than the length of available spm_data", UserWarning) else: data['u'] = data['u'][:max_segments] data['x_true'] = data['x_true'][:max_segments] data['h_true_monitor'] = data['h_true_monitor'][:max_segments] data['y_true'] = data['y_true'][:max_segments] data['y_train'] = data['y_train'][:max_segments] if node_index is not None: data['x_true'] = [array[:, node_index].reshape(dr.n_recurrent_step, 1) for array in data['x_true']] data['h_true_monitor'] = [np.take(array, node_index, 1) for array in data['h_true_monitor']] data['y_true'] = [array[:, node_index].reshape(dr.n_recurrent_step, 1) for array in data['y_true']] data['y_train'] = [array[:, node_index].reshape(dr.n_recurrent_step, 1) for array in data['y_train']] H_STATE_INITIAL = H_STATE_INITIAL[node_index].reshape(1, 4) # collect merged spm_data (without split and merge, it can be tricky to cut proper part from du) data['u_merged'] = tb.merge(data['u'], dr.n_recurrent_step, dr.shift_data) data['x_true_merged'] = tb.merge(data['x_true'], dr.n_recurrent_step, dr.shift_data) # x_hat is with extra wrapper for easy modification with a single index data['x_hat_merged'] = tb.ArrayWrapper(np.zeros(data['x_true_merged'].shape), dr.n_recurrent_step, dr.shift_data) data['h_true_monitor_merged'] = tb.merge(data['h_true_monitor'], dr.n_recurrent_step, dr.shift_data) data['y_true_merged'] = tb.merge(data['y_true'], dr.n_recurrent_step, dr.shift_data) data['y_train_merged'] = tb.merge(data['y_train'], dr.n_recurrent_step, dr.shift_data) # run forward pass with x_true to show y error caused by error in the network parameters isess.run(tf.global_variables_initializer()) y_hat_x_true_log, h_hat_x_true_monitor_log, h_hat_x_true_connector_log = \ dr.run_initializer_graph(isess, H_STATE_INITIAL, data['x_true']) data['h_hat_x_true_monitor'] = h_hat_x_true_monitor_log data['y_hat_x_true'] = y_hat_x_true_log data['h_hat_x_true_monitor_merged'] = tb.merge(h_hat_x_true_monitor_log, dr.n_recurrent_step, dr.shift_data) data['y_hat_x_true_merged'] = tb.merge(y_hat_x_true_log, dr.n_recurrent_step, dr.shift_data) loss_x_normalizer = np.sum(data['x_true_merged'].flatten() ** 2) loss_y_normalizer = np.sum(data['y_true_merged'].flatten() ** 2) loss_smooth_normalizer = np.std(data['x_true_merged'].flatten()) ** 2 return data
def calculate_log_data(): global isess if 'y_hat_x_true' not in data.keys(): # run forward pass with x_true to show y error caused by error in the network parameters with tf.Session() as sess: sess.run(tf.global_variables_initializer()) y_hat_x_true_log, h_hat_x_true_monitor_log, h_hat_x_true_connector_log = \ dr.run_initializer_graph(sess, H_STATE_INITIAL, data['x_true']) data['h_hat_x_true_monitor'] = h_hat_x_true_monitor_log data['y_hat_x_true'] = y_hat_x_true_log data['h_hat_x_true_monitor_merged'] = tb.merge(h_hat_x_true_monitor_log, dr.n_recurrent_step, dr.shift_data) data['y_hat_x_true_merged'] = tb.merge(y_hat_x_true_log, dr.n_recurrent_step, dr.shift_data) data['loss_x_normalizer'] = np.sum(data['x_true_merged'].flatten() ** 2) data['loss_y_normalizer'] = np.sum(data['y_true_merged'].flatten() ** 2) data['loss_smooth_normalizer'] = np.std(data['x_true_merged'].flatten()) ** 2 data['x_hat'] = tb.split(data['x_hat_merged'].get(), n_segment=dr.n_recurrent_step, n_step=dr.shift_data) if IF_NODE_MODE: data['x_hat'] = [array.reshape(dr.n_recurrent_step, 1) for array in data['x_hat']] isess.run(tf.global_variables_initializer()) y_hat_log, h_hat_monitor_log, h_hat_connector_log = \ dr.run_initializer_graph(isess, H_STATE_INITIAL, data['x_hat']) # collect results # segmented spm_data data['x_hat'] = data['x_hat'] data['x_true'] = data['x_true'] data['h_true_monitor'] = data['h_true_monitor'] data['h_hat_x_true_monitor'] = data['h_hat_x_true_monitor'] data['h_hat_monitor'] = h_hat_monitor_log data['y_train'] = data['y_train'] data['y_true'] = data['y_true'] data['y_hat_x_true'] = data['y_hat_x_true'] data['y_hat'] = y_hat_log # merged spm_data data['x_true_merged'] = data['x_true_merged'] data['x_hat_merged'] = data['x_hat_merged'] data['h_true_monitor_merged'] = data['h_true_monitor_merged'] data['h_hat_x_true_monitor_merged'] = data['h_hat_x_true_monitor_merged'] data['h_hat_monitor_merged'] = tb.merge(h_hat_monitor_log, dr.n_recurrent_step, dr.shift_data) data['y_train_merged'] = data['y_train_merged'] data['y_true_merged'] = data['y_true_merged'] data['y_hat_x_true_merged'] = data['y_hat_x_true_merged'] data['y_hat_merged'] = tb.merge(y_hat_log, dr.n_recurrent_step, dr.shift_data) # calculate loss loss_x = np.sum((data['x_hat_merged'].data.flatten() - data['x_true_merged'].flatten()) ** 2) loss_y = np.sum((data['y_hat_merged'].flatten() - data['y_true_merged'].flatten()) ** 2) loss_smooth = np.sum((data['x_hat_merged'].data[0:-1].flatten() - data['x_hat_merged'].data[1:].flatten()) ** 2) data['loss_x'].append(loss_x / loss_x_normalizer) data['loss_y'].append(loss_y / loss_y_normalizer) data['loss_smooth'].append(loss_smooth / loss_smooth_normalizer) data['loss_total'].append((loss_y + dr.loss_weighting['smooth'] * loss_smooth) / ( loss_y_normalizer + dr.loss_weighting['smooth'] * loss_smooth_normalizer))
dr.trainable_variables_nodes = [ v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) ] dr.trainable_variables_names = [ v.name for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) ] du_hat.variable_names_in_graph = du_hat.parse_variable_names( dr.trainable_variables_names) # prepare data for training y_target = get_target_curve(du_hat.get('y'), confounds, spm_data['y_upsampled']) data = { 'u': tb.split(spm_data['u_upsampled'], n_segment=dr.n_recurrent_step, n_step=dr.shift_data) } data_hat = { 'x_initial': tb.split(du_hat.get('x'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=0), 'h_initial': tb.split(du_hat.get('h'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=1), 'y': tb.split(y_target,
'k': False, 'gamma': False, 'tao': False, 'epsilon': False, 'V0': False, 'TE': False, 'r0': False, 'theta0': False, 'x_h_coupling': False } dr.build_main_graph(neural_parameter_initial=neural_parameter_initial) # prepare spm_data data = { 'u': tb.split(du.get('u'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data), 'y': tb.split(du.get('y'), n_segment=dr.n_recurrent_step, n_step=dr.shift_data, shift=dr.shift_u_y) } n_segment = min([len(data[x]) for x in data.keys()]) for k in data.keys(): data[k] = data[k][:min([n_segment, N_SEGMENTS])] # [val.name for val in tf.global_variables()] # [val.name for val in tf.trainable_variables()] sess = tf.InteractiveSession()
def prepare_data(self, du, dr, data_package): """ Prepare spm_data in du for dr in training. Create a 'spm_data' dictionary :param du: a DataUnit instance, with needed spm_data :param dr: a DcmRnn instance, with needed parameters :param data_package: a dictionary, stores configure and spm_data for a particular experimental case :return: modified """ dp = data_package data = dp.data # create spm_data according to flag if dp.IF_RANDOM_H_PARA: data['H_PARA_INITIAL'] = \ dr.randomly_generate_hemodynamic_parameters(dr.n_region, deviation_constraint=2).astype(np.float32) else: data['H_PARA_INITIAL'] = dr.get_standard_hemodynamic_parameters( n_node=dr.n_region).astype(np.float32) if dp.IF_RANDOM_H_STATE_INIT: data['H_STATE_INITIAL'] = du.get('h')[random.randint( 64, du.get('h').shape[0] - 64)].astype(np.float32) else: data['H_STATE_INITIAL'] = \ dr.set_initial_hemodynamic_state_as_inactivated(n_node=dr.n_region).astype(np.float32) if dp.IF_NOISED_Y: std = np.std(du.get('y').reshape([-1])) / dp.SNR data['NOISE'] = np.random.normal(0, std, du.get('y').shape) else: data['NOISE'] = np.zeros(du.get('y').shape) data['y_train'] = tb.split( du.get('y') + data['NOISE'], dr.n_recurrent_step, dr.shift_data, dr.shift_x_y) max_segments_natural = len(data['y_train']) data['max_segments_natural'] = max_segments_natural data['y_true'] = tb.split(du.get('y'), dr.n_recurrent_step, dr.shift_data, dr.shift_x_y)[:max_segments_natural] data['h_true_monitor'] = tb.split(du.get('h'), dr.n_recurrent_step, dr.shift_data)[:max_segments_natural] data['x_true'] = tb.split(du.get('x'), dr.n_recurrent_step, dr.shift_data)[:max_segments_natural] data['u'] = tb.split(du.get('u'), dr.n_recurrent_step, dr.shift_data)[:max_segments_natural] if dp.N_SEGMENTS is not None: if dp.N_SEGMENTS > max_segments_natural: dp.N_SEGMENTS = max_segments_natural warnings.warn( "dp.n_segments is larger than the length of available spm_data", UserWarning) else: data['u'] = data['u'][:dp.N_SEGMENTS] data['x_true'] = data['x_true'][:dp.N_SEGMENTS] data['h_true_monitor'] = data['h_true_monitor'][:dp.N_SEGMENTS] data['y_true'] = data['y_true'][:dp.N_SEGMENTS] data['y_train'] = data['y_train'][:dp.N_SEGMENTS] if dp.IF_NODE_MODE is True: node_index = dp.NODE_INDEX data['x_true'] = [ array[:, node_index].reshape(dr.n_recurrent_step, 1) for array in data['x_true'] ] data['h_true_monitor'] = [ np.take(array, node_index, 1) for array in data['h_true_monitor'] ] data['y_true'] = [ array[:, node_index].reshape(dr.n_recurrent_step, 1) for array in data['y_true'] ] data['y_train'] = [ array[:, node_index].reshape(dr.n_recurrent_step, 1) for array in data['y_train'] ] data['H_STATE_INITIAL'] = data['H_STATE_INITIAL'][ node_index].reshape(1, 4) # saved dp.SEQUENCE_LENGTH = dr.n_recurrent_step + (len(spm_data['x_true']) - 1) * dr.shift_data # collect merged spm_data (without split and merge, it can be tricky to cut proper part from du) data['u_merged'] = tb.merge(data['u'], dr.n_recurrent_step, dr.shift_data) data['x_true_merged'] = tb.merge(data['x_true'], dr.n_recurrent_step, dr.shift_data) # x_hat is with extra wrapper for easy modification with a single index data['x_hat_merged'] = \ tb.ArrayWrapper(np.zeros(data['x_true_merged'].shape), dr.n_recurrent_step, dr.shift_data) data['h_true_monitor_merged'] = \ tb.merge(data['h_true_monitor'], dr.n_recurrent_step, dr.shift_data) data['y_true_merged'] = tb.merge(data['y_true'], dr.n_recurrent_step, dr.shift_data) data['y_train_merged'] = tb.merge(data['y_train'], dr.n_recurrent_step, dr.shift_data) return data_package