def load_model_gates(model_path, params):
state_dict = torch.load(model_path)
model_params = params['model_params']
fake_init_gates = GateInitializer.get_fake_init_gates(params['gate_init_multi_heuristic_params']['num_gates'])
model = DepthOneModel(fake_init_gates, model_params)
model.load_state_dict(state_dict)
return model.get_gates()
def cross_validate_accuracy_over_saved_results(path_to_results,
stepsize,
n_steps,
nfolds=20,
starting_fold=30):
path_to_params = os.path.join(path_to_results, 'params.yaml')
params = TransformParameterParser(path_to_params).parse_params()
print(params)
cur_params = deepcopy(params)
#evauntually uncomment this leaving asis in order ot keep the same results as before to compare.
set_random_seeds(params)
data_input = DataInput(params['data_params'])
te_accs = []
pushed_gates_per_fold = []
starting_gates_per_fold = []
diffs_per_fold = []
for fold in range(starting_fold):
data_input.split_data()
for fold in range(starting_fold, nfolds + starting_fold):
print('Running fold %d' % fold)
cur_params['save_dir'] = os.path.join(params['save_dir'],
'run%d' % fold)
data_input.split_data()
best_tr_acc, starting_gate, best_gate = push_converged_boundaries_given_data_input_and_params(
cur_params, data_input, stepsize, n_steps, path_to_params)
model = DepthOneModel([[['D1', best_gate[0], best_gate[1]],
['D2', best_gate[2], best_gate[3]]]],
params['model_params'])
fit_classifier_params(
model, data_input,
params['train_params']['learning_rate_classifier'])
te_acc = compute_te_acc(model, data_input)
print('te acc for fold %d is %.3f' % (fold, te_acc))
te_accs.append(te_acc)
pushed_gates_per_fold.append(best_gate)
starting_gates_per_fold.append(starting_gate)
diffs_per_fold.append(get_diff_between_gates(starting_gate, best_gate))
print('Diff: ', get_diff_between_gates(starting_gate, best_gate))
print('Te accs:', te_accs)
print('Diffs per fold:', diffs_per_fold)
with open(
os.path.join(path_to_results,
'expanded_boundaries_te_accs_per_fold.pkl'),
'wb') as f:
pickle.dump(te_accs, f)
with open(
os.path.join(path_to_results,
'expanded_boundaries_diffs_per_fold.pkl'), 'wb') as f:
pickle.dump(diffs_per_fold, f)
with open(
os.path.join(path_to_results,
'expanded_boundaries_best_pushed_gates_per_fold.pkl'),
'wb') as f:
pickle.dump(pushed_gates_per_fold, f)
def get_data_inside_both_visually_correct_and_learned_gate(
path_to_params_learned):
matplotlib.rcParams.update({'font.size': 22})
path_to_params_learned, model_learned, data_input = load_saved_model_and_matching_data_input(
path_to_params_learned)
vis_correct_gate = [[['D1', 0.,
model_learned.get_gate_tree()[0][0][2]],
['D2',
model_learned.get_gate_tree()[0][1][1], .75]]]
vis_correct_model = DepthOneModel(vis_correct_gate,
path_to_params_learned['model_params'])
plotter_model = DataAndGatesPlotterDepthOne(
model_learned, np.concatenate(data_input.x_tr))
plotter_vis_corr = DataAndGatesPlotterDepthOne(
vis_correct_model, np.concatenate(data_input.x_tr))
model_learned_data_inside_gate = get_data_inside_gate(
plotter_model, data_input, model_learned)
vis_corr_data_inside_gate = get_data_inside_gate(plotter_vis_corr,
data_input,
vis_correct_model)
with open('model_feat_diff_data_inside_gate.pkl', 'wb') as f:
pickle.dump(model_learned_data_inside_gate, f)
with open('vis_corr_data_inside_gate.pkl', 'wb') as f:
pickle.dump(vis_corr_data_inside_gate, f)
return model_learned_data_inside_gate, vis_corr_data_inside_gate
def get_model_with_set_position_and_size(self, x_pos, y_pos, size):
model = DepthOneModel(self.get_init_gate(x_pos, y_pos, size),
self.params['model_params'])
train_params = self.params['train_params']
fit_classifier_params(model, self.data_input,\
train_params['learning_rate_classifier'],
l1_reg_strength=train_params['l1_reg_strength'])
return model
def get_next_gate_tree_by_log_loss(unused_gate_trees, data_input, params, model=None):
if model:
losses = []
for gate_tree in unused_gate_trees:
dummy_model_state = deepcopy(model.state_dict())
dummy_model = DepthOneModel(model.get_gate_tree(), params['model_params'])
dummy_model.load_state_dict(dummy_model_state)
dummy_model.add_node(gate_tree)
performance_tracker = run_train_model(dummy_model, params['train_params'], data_input)
losses.append(dummy_model(data_input.x_tr, data_input.y_tr)['log_loss'].cpu().detach().numpy())
losses = np.array(losses)
best_gate_idx = np.argmin(losses[~np.isnan(losses)])
else:
losses = []
for gate_tree in unused_gate_trees:
model = DepthOneModel([gate_tree], params['model_params'])
performance_tracker = run_train_model(model, params['train_params'], data_input)
losses.append(model(data_input.x_tr, data_input.y_tr)['log_loss'].cpu().detach().numpy())
losses = np.array(losses)
best_gate_idx = np.argmin(losses[~np.isnan(losses)])
best_gate = unused_gate_trees[best_gate_idx]
del unused_gate_trees[best_gate_idx]
return best_gate, unused_gate_trees
def load_saved_results(path_to_params, ret_params_too=False):
start_time = time.time()
params = TransformParameterParser(path_to_params).parse_params()
print(params)
torch.manual_seed(params['random_seed'])
np.random.seed(params['random_seed'])
data_input = load_and_prepare_data_input(params)
model = DepthOneModel([[['D1', 0, 0], ['D2', 0, 0]]],
params['model_params'])
model.load_state_dict(
torch.load(os.path.join(params['save_dir'], 'model.pkl')))
if ret_params_too:
return data_input, model, params
return data_input, model
def get_next_best_gate(remaining_gates, data_input, params, model):
losses = []
trackers = []
for gate in remaining_gates:
dummy_model_state = deepcopy(model.state_dict())
init_gates = rectangularize_gates(model)
dummy_model = DepthOneModel(init_gates, params['model_params'])
dummy_model.load_state_dict(dummy_model_state)
dummy_model.add_node(gate)
trackers.append(run_train_model(
dummy_model, params['train_params'], data_input
))
losses.append(
dummy_model(
data_input.x_tr, data_input.y_tr
)['loss'].cpu().detach().numpy()
)
losses = np.array(losses)
best_gate_idx = np.argmin(losses[~np.isnan(losses)])
best_gate = remaining_gates[best_gate_idx]
remaining_gates = [
gate for g, gate in enumerate(remaining_gates)
if not g == best_gate_idx
]
best_tracker = trackers[best_gate_idx]
return best_gate, remaining_gates, best_tracker
def load_saved_model_and_matching_data_input(path_to_params):
def set_random_seeds(params):
torch.manual_seed(params['random_seed'])
np.random.seed(params['random_seed'])
start_time = time.time()
params = TransformParameterParser(path_to_params).parse_params()
print(params)
#evauntually uncomment this leaving asis in order ot keep the same results as before to compare.
set_random_seeds(params)
data_input = DataInput(params['data_params'])
data_input.split_data()
print('%d samples in the training data' % len(data_input.x_tr))
with open(os.path.join(params['save_dir'], 'trackers.pkl'), 'rb') as f:
trackers = pickle.load(f)
with open(os.path.join(params['save_dir'], 'transformer.pkl'), 'rb') as f:
umapper = pickle.load(f)
# FOR DEBUGGING ONLY
#params['transform_params']['cells_to_subsample'] = 10
data_input.embed_data(\
umapper,
cells_to_subsample=params['transform_params']['cells_to_subsample'],
use_labels_to_transform_data=params['transform_params']['use_labels_to_transform_data']
)
data_input.normalize_data()
data_input.convert_all_data_to_tensors()
model = DepthOneModel([[['D1', 0, 0], ['D2', 0, 0]]],
params['model_params'])
model.load_state_dict(
torch.load(os.path.join(params['save_dir'], 'model.pkl')))
return params, model, data_input, umapper
def get_model_with_new_gate(gate_idx, new_gate, params, model_gates, data_input):
gates = model_gates
if type(new_gate[0]) == float:
new_gate = [['D1', new_gate[0], new_gate[1]], ['D2', new_gate[2], new_gate[3]]]
formatted_gates = []
for gate in gates:
formatted_gates.append([
['D1', gate[0], gate[1]],
['D2', gate[2], gate[3]]
])
formatted_gates[gate_idx] = new_gate
train_params = params['train_params']
model = DepthOneModel(formatted_gates, params['model_params'])
fit_classifier_params(model, data_input,\
train_params['learning_rate_classifier'],
l1_reg_strength=train_params['l1_reg_strength'])
return model
def initialize_model_with_best_gate(potential_gates, data_input, params):
losses = []
models = []
for g, gate in enumerate(potential_gates):
model = DepthOneModel([gate], params['model_params'])
tracker = run_train_model(model, params['train_params'], data_input)
losses.append(
model(data_input.x_tr,
data_input.y_tr)['loss'].cpu().detach().numpy())
models.append(model)
best_gate_idx = np.argmin(np.array(losses)[~np.isnan(losses)])
best_model = models[best_gate_idx]
remaining_gates = [
gate for g, gate in enumerate(potential_gates)
if not g == best_gate_idx
]
return best_model, remaining_gates, tracker
def initialize_model(model_params, init_gate_tree):
model = DepthOneModel(init_gate_tree, model_params)
return model
def push_converged_boundaries_given_data_input_and_params(
params, data_input, stepsize, n_steps, path_to_params):
start_time = time.time()
print('%d samples in the training data' % len(data_input.x_tr))
with open(os.path.join(params['save_dir'], 'tracker.pkl'), 'rb') as f:
tracker = pickle.load(f)
with open(os.path.join(params['save_dir'], 'transformer.pkl'), 'rb') as f:
umapper = pickle.load(f)
# FOR DEBUGGING ONLY
#params['transform_params']['cells_to_subsample'] = 10
data_input.embed_data(\
umapper,
cells_to_subsample=params['transform_params']['cells_to_subsample'],
use_labels_to_transform_data=params['transform_params']['use_labels_to_transform_data']
)
data_input.normalize_data()
data_input.convert_all_data_to_tensors()
model = DepthOneModel([[['D1', 0, 0], ['D2', 0, 0]]],
params['model_params'])
model.load_state_dict(
torch.load(os.path.join(params['save_dir'], 'model.pkl')))
init_acc = tracker.metrics['tr_acc'][-1]
cur_best_acc = init_acc
starting_gate = model.get_gates()[0]
cur_gate = copy.deepcopy(starting_gate)
cur_best_gate = copy.deepcopy(cur_gate)
print('Starting gate:', starting_gate)
counter = 0
for left_step in range(n_steps):
cur_gate[0] = starting_gate[0] - left_step * stepsize
for right_step in range(n_steps):
cur_gate[1] = starting_gate[1] + right_step * stepsize
for down_step in range(n_steps):
cur_gate[2] = starting_gate[2] - down_step * stepsize
for up_step in range(n_steps):
cur_gate[3] = starting_gate[3] + up_step * stepsize
model = DepthOneModel([[['D1', cur_gate[0], cur_gate[1]],
['D2', cur_gate[2], cur_gate[3]]]],
params['model_params'])
fit_classifier_params(
model, data_input,
params['train_params']['learning_rate_classifier'])
# model.nodes = None
# model.init_nodes([[['D1', cur_gate[0], cur_gate[1]], ['D2', cur_gate[2], cur_gate[3]]]])
cur_acc = compute_tr_acc(model, data_input)
#cur_acc = performance_tracker.metrics['tr_acc'][-1]
counter += 1
#print(counter)
#print(cur_gate)
#print(cur_acc)
if cur_acc > cur_best_acc:
cur_best_acc = cur_acc
cur_best_gate = copy.deepcopy(cur_gate)
print('Final acc %.3f, Initial acc %.3f' % (cur_best_acc, init_acc))
print('Init/final gates', starting_gate, cur_best_gate)
print('time taken: %d' % (time.time() - start_time))
return cur_best_acc, starting_gate, cur_best_gate
def initialize_model(model_params, init_gate_tree):
if model_params['depth_one_disjunction_of_all_gates']:
model = DisjunctiveDepthOneModel(init_gate_tree, model_params)
else:
model = DepthOneModel(init_gate_tree, model_params)
return model