def get_transformed_data(self, sess, dataset, dataset_type):
dataset.set_current_data_set_type(dataset_type=dataset_type)
leaf_true_labels_dict = {}
leaf_final_features_dict = {}
# network.get_variable_name(name="final_eval_feature", node=node)
while True:
results = self.eval_network(sess=sess, dataset=dataset, use_masking=True)
for node in self.topologicalSortedNodes:
if not node.isLeaf:
continue
final_features = results[self.get_variable_name(name="final_eval_feature", node=node)]
true_labels = results[self.get_variable_name(name="labels", node=node)]
UtilityFuncs.concat_to_np_array_dict(dct=leaf_final_features_dict, key=node.index, array=final_features)
UtilityFuncs.concat_to_np_array_dict(dct=leaf_true_labels_dict, key=node.index, array=true_labels)
if dataset.isNewEpoch:
break
# Concatenate all data
transformed_samples = None
labels = None
for k, v in leaf_final_features_dict.items():
if transformed_samples is None:
transformed_samples = np.array(v)
else:
transformed_samples = np.concatenate((transformed_samples, v))
if labels is None:
labels = np.array(leaf_true_labels_dict[k])
else:
labels = np.concatenate((labels, leaf_true_labels_dict[k]))
return transformed_samples, labels
def create_global_input_drivers(self):
# Branching probability
self.globalInputDrivers[GlobalInputNames.branching_prob_threshold.value] = \
UtilityFuncs.create_parameter_from_train_program(
parameter_name=GlobalInputNames.branching_prob_threshold.value, train_program=self.trainProgram)
# Batch Size
self.globalInputDrivers[GlobalInputNames.batch_size.value] = \
UtilityFuncs.create_parameter_from_train_program(
parameter_name=GlobalInputNames.batch_size.value, train_program=self.trainProgram)
# Evaluation Batch Size
self.globalInputDrivers[GlobalInputNames.evaluation_batch_size.value] = \
UtilityFuncs.create_parameter_from_train_program(
parameter_name=GlobalInputNames.evaluation_batch_size.value, train_program=self.trainProgram)
# Epoch Count
self.globalInputDrivers[GlobalInputNames.epoch_count.value] = \
UtilityFuncs.create_parameter_from_train_program(
parameter_name=GlobalInputNames.epoch_count.value, train_program=self.trainProgram)
# All hyper parameters regarding to parameter training
hyper_parameter_set = {GlobalInputNames.wd.value, GlobalInputNames.lr_initial.value,
GlobalInputNames.momentum.value, GlobalInputNames.weight_update_interval.value,
GlobalInputNames.lr_update_interval.value, GlobalInputNames.lr_decay_ratio.value}
for node in self.nodes.values():
for parameter in node.parametersDict.values():
if parameter.parameterType == parameterTypes.learnable_parameter:
hyper_parameter_dict = {}
for hyper_parameter in hyper_parameter_set:
parameter_hyper_parameter_name = parameter.get_property_name(property_=hyper_parameter)
value_dict = self.trainProgram.load_settings_for_property(property_name=hyper_parameter)
hyper_parameter_value = self.trainProgram.decode_json_element_for_parameter(
parameter_name=parameter_hyper_parameter_name, json_element=value_dict)
hyper_parameter_dict[hyper_parameter] = hyper_parameter_value
# Decaying Parameter for the learning rate
lr_hyper_param_name = parameter.get_property_name(property_=GlobalInputNames.lr.value)
self.globalInputDrivers[lr_hyper_param_name] = \
DecayingParameter(name=lr_hyper_param_name,
value=hyper_parameter_dict[GlobalInputNames.lr_initial.value],
decay=hyper_parameter_dict[GlobalInputNames.lr_decay_ratio.value],
decay_period=hyper_parameter_dict[GlobalInputNames.lr_update_interval.value])
# Fixed Parameter for the wd
wd_hyper_parameter_name = parameter.get_property_name(property_=GlobalInputNames.wd.value)
self.globalInputDrivers[wd_hyper_parameter_name] = \
FixedParameter(name=wd_hyper_parameter_name,
value=hyper_parameter_dict[GlobalInputNames.wd.value])
# Fixed Parameter for the momentum value
momentum_hyper_parameter_name = \
parameter.get_property_name(property_=GlobalInputNames.momentum.value)
self.globalInputDrivers[momentum_hyper_parameter_name] = \
FixedParameter(name=momentum_hyper_parameter_name,
value=hyper_parameter_dict[GlobalInputNames.momentum.value])
# Weight update parameter
weight_update_interval_parameter_name = \
parameter.get_property_name(property_=GlobalInputNames.weight_update_interval.value)
self.globalInputDrivers[weight_update_interval_parameter_name] = \
FixedParameter(name=weight_update_interval_parameter_name,
value=hyper_parameter_dict[GlobalInputNames.weight_update_interval.value])
print("X")
def calculate_accuracy(self, sess, dataset_type):
leaf_posteriors_dict = {}
leaf_true_labels_dict = {}
self.datasets[0].set_current_data_set_type(dataset_type=dataset_type)
while True:
results = self.eval(sess=sess,
dataset=self.datasets[0],
use_masking=True)
batch_sample_count = 0.0
for network, eval_dict in zip(self.networks, results):
for node in network.topologicalSortedNodes:
if node.isLeaf:
posterior_probs = eval_dict[network.get_variable_name(
name="posterior_probs", node=node)]
true_labels = eval_dict["Node{0}_label_tensor".format(
node.index)]
UtilityFuncs.concat_to_np_array_dict(
dct=leaf_posteriors_dict,
key=(network, node.index),
array=posterior_probs)
UtilityFuncs.concat_to_np_array_dict(
dct=leaf_true_labels_dict,
key=(network, node.index),
array=true_labels)
# batch_sample_count += list(leaf_true_labels_dict.values())[0].shape[0]
# if batch_sample_count != GlobalConstants.EVAL_BATCH_SIZE:
# raise Exception("Incorrect batch size:{0}".format(batch_sample_count))
if self.datasets[0].isNewEpoch:
break
ensemble_length = len(leaf_posteriors_dict)
posterior_tensor = np.stack(list(leaf_posteriors_dict.values()),
axis=0)
assert posterior_tensor.shape[0] == len(self.networks)
averaged_posterior = np.mean(posterior_tensor, axis=0)
assert averaged_posterior.shape[0] == self.datasets[
0].get_current_sample_count()
assert averaged_posterior.shape[1] == self.datasets[0].get_label_count(
)
true_labels_list = list(leaf_true_labels_dict.values())
for i in range(len(true_labels_list) - 1):
assert np.array_equal(true_labels_list[i], true_labels_list[i + 1])
true_label_arr = true_labels_list[0]
total_correct = 0.0
total_count = float(true_label_arr.shape[0])
assert averaged_posterior.shape[0] == true_label_arr.shape[0]
for i in range(averaged_posterior.shape[0]):
sample_posterior = averaged_posterior[i, :]
predicted_label = np.argmax(sample_posterior)
true_label = true_label_arr[i]
if predicted_label == true_label:
total_correct += 1.0
print(
"*************Overall {0} samples. Overall Accuracy:{1}*************"
.format(total_count, total_correct / total_count))
return total_correct / total_count
def get_input(input, node, out_filters, first_conv_filter_size):
assert input.get_shape().ndims == 4
name = UtilityFuncs.get_variable_name(name="init_conv", node=node)
input_filters = input.get_shape().as_list()[-1]
x = ResnetGenerator.conv(name, input, first_conv_filter_size,
input_filters, out_filters, ResnetGenerator.stride_arr(1))
return x
def main():
dataset = MnistDataSet(validation_sample_count=10000)
lr_list = [0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.11, 0.12, 0.13, 0.14, 0.15]
lr_periods = [500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500]
threshold_decay_periods = [500, 1000]
threshold_decays = [0.5, 0.75]
list_of_lists = [
lr_list, lr_periods, threshold_decay_periods, threshold_decays
]
for idx in itertools.product(*list_of_lists):
tf.reset_default_graph()
lr = idx[0]
lr_period = idx[1]
threshold_decay_period = idx[2]
threshold_decay = idx[3]
train_program_path = UtilityFuncs.get_absolute_path(
script_file=__file__, relative_path="train_program.json")
train_program = TrainProgram(program_file=train_program_path)
train_program.set_train_program_element(element_name="lr_initial",
keywords=",",
skipwords="",
value=lr)
train_program.set_train_program_element(
element_name="lr_update_interval",
keywords=",",
skipwords="",
value=lr_period)
train_program.set_train_program_element(
element_name="BranchingProbThreshold",
keywords={"decay"},
skipwords={},
value=threshold_decay)
train_program.set_train_program_element(
element_name="BranchingProbThreshold",
keywords={"decayPeriod"},
skipwords={},
value=threshold_decay_period)
cnn_lenet = TreeNetwork(
dataset=dataset,
parameter_file=None,
tree_degree=2,
tree_type=TreeType.hard,
problem_type=ProblemType.classification,
train_program=train_program,
explanation=
"100 Epochs, {0} lr decay period, {1} initial lr params runId17 "
"threshold decay period: {2} threshold decay: {3}".format(
lr_period, lr, threshold_decay_period, threshold_decay),
list_of_node_builder_functions=[root_func, l1_func, leaf_func])
optimizer = SgdOptimizer(network=cnn_lenet,
use_biased_gradient_estimates=True)
cnn_lenet.set_optimizer(optimizer=optimizer)
cnn_lenet.build_network()
cnn_lenet.check_grads()
cnn_lenet.init_session()
cnn_lenet.train()
def load_dataset(self):
self.trainingSamples, self.trainingLabels = self.load(
path_img=self.trainImagesPath, path_lbl=self.trainLabelsPath)
self.testSamples, self.testLabels = self.load(
path_img=self.testImagesPath, path_lbl=self.testLabelsPath)
if self.validationLoadFile is None:
# random_indices = np.random.choice(self.trainingSamples.shape[0], size=self.validationSampleCount, replace=False)
indices = np.arange(0, self.validationSampleCount)
if self.validationSaveFile is not None:
UtilityFuncs.save_npz(file_name=self.validationSaveFile,
arr_dict={"random_indices": indices})
else:
indices = UtilityFuncs.load_npz(
file_name=self.validationLoadFile)["random_indices"]
# print(random_indices[0:5])
self.validationSamples = self.trainingSamples[indices]
self.validationLabels = self.trainingLabels[indices]
self.trainingSamples = np.delete(self.trainingSamples, indices, 0)
self.trainingLabels = np.delete(self.trainingLabels, indices, 0)
def bottleneck_residual(x, node, is_train, in_filter, out_filter, stride, relu_leakiness, activate_before_residual,
bn_momentum):
"""Bottleneck residual unit with 3 sub layers."""
if activate_before_residual:
with tf.variable_scope("Node{0}_common_bn_relu".format(node.index)):
x = ResnetGenerator.batch_norm(UtilityFuncs.get_variable_name(name="init_bn", node=node), x, is_train,
bn_momentum)
x = ResnetGenerator.relu(x, relu_leakiness)
orig_x = x
else:
with tf.variable_scope("Node{0}_residual_bn_relu".format(node.index)):
orig_x = x
x = ResnetGenerator.batch_norm(UtilityFuncs.get_variable_name(name="init_bn", node=node), x, is_train,
bn_momentum)
x = ResnetGenerator.relu(x, relu_leakiness)
with tf.variable_scope("Node{0}_sub1".format(node.index)):
x = ResnetGenerator.conv(UtilityFuncs.get_variable_name(name="conv_1", node=node), x, 1, in_filter,
out_filter / 4, stride)
with tf.variable_scope("Node{0}_sub2".format(node.index)):
x = ResnetGenerator.batch_norm(UtilityFuncs.get_variable_name(name="bn2", node=node), x, is_train,
bn_momentum)
x = ResnetGenerator.relu(x, relu_leakiness)
x = ResnetGenerator.conv(UtilityFuncs.get_variable_name(name="conv2", node=node), x, 3, out_filter / 4,
out_filter / 4, [1, 1, 1, 1])
with tf.variable_scope("Node{0}_sub3".format(node.index)):
x = ResnetGenerator.batch_norm(UtilityFuncs.get_variable_name(name="bn3", node=node), x, is_train,
bn_momentum)
x = ResnetGenerator.relu(x, relu_leakiness)
x = ResnetGenerator.conv(UtilityFuncs.get_variable_name(name="conv3", node=node), x, 1, out_filter / 4,
out_filter, [1, 1, 1, 1])
with tf.variable_scope("Node{0}_sub_add".format(node.index)):
if in_filter != out_filter:
orig_x = ResnetGenerator.conv(UtilityFuncs.get_variable_name(name="project", node=node), orig_x, 1,
in_filter, out_filter, stride)
x += orig_x
return x
def main():
dataset = MnistDataSet(validation_sample_count=5000)
dataset.load_dataset()
train_program_path = UtilityFuncs.get_absolute_path(
script_file=__file__, relative_path="train_program.json")
train_program = TrainProgram(program_file=train_program_path)
cnn_lenet = TreeNetwork(run_id=0,
dataset=dataset,
parameter_file=None,
tree_degree=2,
tree_type=TreeType.hard,
problem_type=ProblemType.classification,
train_program=train_program,
list_of_node_builder_functions=[baseline_network])
optimizer = SgdOptimizer(network=cnn_lenet,
use_biased_gradient_estimates=True)
cnn_lenet.set_optimizer(optimizer=optimizer)
cnn_lenet.build_network()
cnn_lenet.init_session()
cnn_lenet.train()
def build_network(self):
# Create itself
curr_index = 0
is_leaf = 0 == (self.depth - 1)
root_node = Node(index=curr_index, depth=0, is_root=True, is_leaf=is_leaf)
threshold_name = self.get_variable_name(name="threshold", node=root_node)
root_node.probabilityThreshold = tf.placeholder(name=threshold_name, dtype=tf.float32)
softmax_decay_name = self.get_variable_name(name="softmax_decay", node=root_node)
root_node.softmaxDecay = tf.placeholder(name=softmax_decay_name, dtype=tf.float32)
self.dagObject.add_node(node=root_node)
self.nodes[curr_index] = root_node
d = deque()
d.append(root_node)
# Create children if not leaf
while len(d) > 0:
# Dequeue
curr_node = d.popleft()
if not curr_node.isLeaf:
for i in range(self.degreeList[curr_node.depth]):
new_depth = curr_node.depth + 1
is_leaf = new_depth == (self.depth - 1)
curr_index += 1
child_node = Node(index=curr_index, depth=new_depth, is_root=False, is_leaf=is_leaf)
if not child_node.isLeaf:
threshold_name = self.get_variable_name(name="threshold", node=child_node)
child_node.probabilityThreshold = tf.placeholder(name=threshold_name, dtype=tf.float32)
softmax_decay_name = self.get_variable_name(name="softmax_decay", node=child_node)
child_node.softmaxDecay = tf.placeholder(name=softmax_decay_name, dtype=tf.float32)
self.nodes[curr_index] = child_node
self.dagObject.add_edge(parent=curr_node, child=child_node)
d.append(child_node)
# Flags and hyperparameters
self.useThresholding = tf.placeholder(name="threshold_flag", dtype=tf.int64)
self.iterationHolder = tf.placeholder(name="iteration", dtype=tf.int64)
self.isTrain = tf.placeholder(name="is_train_flag", dtype=tf.int64)
self.useMasking = tf.placeholder(name="use_masking_flag", dtype=tf.int64)
self.isDecisionPhase = tf.placeholder(name="is_decision_phase", dtype=tf.int64)
self.decisionDropoutKeepProb = tf.placeholder(name="decision_dropout_keep_prob", dtype=tf.float32)
self.classificationDropoutKeepProb = tf.placeholder(name="classification_dropout_keep_prob", dtype=tf.float32)
self.noiseCoefficient = tf.placeholder(name="noise_coefficient", dtype=tf.float32)
self.informationGainBalancingCoefficient = tf.placeholder(name="info_gain_balance_coefficient",
dtype=tf.float32)
# Build symbolic networks
self.topologicalSortedNodes = self.dagObject.get_topological_sort()
self.isBaseline = len(self.topologicalSortedNodes) == 1
if not GlobalConstants.USE_RANDOM_PARAMETERS:
self.paramsDict = UtilityFuncs.load_npz(file_name="parameters")
# Set up mechanism for probability thresholding
if not self.isBaseline:
self.thresholdFunc(network=self)
# Build all symbolic networks in each node
for node in self.topologicalSortedNodes:
self.nodeBuildFuncs[node.depth](node=node, network=self)
# Disable some properties if we are using a baseline
if self.isBaseline:
GlobalConstants.USE_INFO_GAIN_DECISION = False
GlobalConstants.USE_CONCAT_TRICK = False
GlobalConstants.USE_PROBABILITY_THRESHOLD = False
# Prepare tensors to evaluate
for node in self.topologicalSortedNodes:
# if node.isLeaf:
# continue
# F
f_output = node.fOpsList[-1]
self.evalDict["Node{0}_F".format(node.index)] = f_output
# H
if len(node.hOpsList) > 0:
h_output = node.hOpsList[-1]
self.evalDict["Node{0}_H".format(node.index)] = h_output
# Activations
for k, v in node.activationsDict.items():
self.evalDict["Node{0}_activation_from_{1}".format(node.index, k)] = v
# Decision masks
for k, v in node.maskTensors.items():
self.evalDict["Node{0}_{1}".format(node.index, v.name)] = v
# Evaluation outputs
for k, v in node.evalDict.items():
self.evalDict[k] = v
# Label outputs
if node.labelTensor is not None:
self.evalDict["Node{0}_label_tensor".format(node.index)] = node.labelTensor
# Sample indices
self.evalDict["Node{0}_indices_tensor".format(node.index)] = node.indicesTensor
# One Hot Label outputs
if node.oneHotLabelTensor is not None:
self.evalDict["Node{0}_one_hot_label_tensor".format(node.index)] = node.oneHotLabelTensor
# Get the leaf counts, which are descendants of each node
for node in self.topologicalSortedNodes:
descendants = self.dagObject.descendants(node=node)
descendants.append(node)
for descendant in descendants:
if descendant.isLeaf:
node.leafCountUnderThisNode += 1
# Learning rate, counter
self.globalCounter = tf.Variable(0, dtype=GlobalConstants.DATA_TYPE, trainable=False)
# Prepare the cost function
# ******************** Residue loss ********************
self.build_residue_loss()
# Record all variables into the variable manager (For backwards compatibility)
self.variableManager.get_all_node_variables()
# Unit Test
tf_trainable_vars = set(tf.trainable_variables())
custom_trainable_vars = set(self.variableManager.trainable_variables())
assert tf_trainable_vars == custom_trainable_vars
# Unit Test
# ******************** Residue loss ********************
# ******************** Main losses ********************
self.build_main_loss()
# ******************** Main losses ********************
# ******************** Decision losses ********************
self.build_decision_loss()
# ******************** Decision losses ********************
# ******************** Regularization losses ********************
self.build_regularization_loss()
# ******************** Regularization losses ********************
self.finalLoss = self.mainLoss + self.regularizationLoss + self.decisionLoss
self.evalDict["RegularizerLoss"] = self.regularizationLoss
self.evalDict["PrimaryLoss"] = self.mainLoss
self.evalDict["ResidueLoss"] = self.residueLoss
self.evalDict["DecisionLoss"] = self.decisionLoss
self.evalDict["NetworkLoss"] = self.finalLoss
self.sampleCountTensors = {k: self.evalDict[k] for k in self.evalDict.keys() if "sample_count" in k}
self.isOpenTensors = {k: self.evalDict[k] for k in self.evalDict.keys() if "is_open" in k}
self.gradFunc(network=self)
# training_features = node_5_features_dict["training_features"]
# training_one_hot_labels = node_5_features_dict["training_one_hot_labels"]
# training_compressed_posteriors = node_5_features_dict["training_compressed_posteriors"]
#
# test_features = node_5_features_dict["test_features"]
# test_one_hot_labels = node_5_features_dict["test_one_hot_labels"]
# test_compressed_posteriors = node_5_features_dict["test_compressed_posteriors"]
# class_count = 4
# features_dim = 64
leaves_list = [3, 4, 5, 6]
base_run_id = 300
for node_index in leaves_list:
run_id = base_run_id + node_index
features_dict = UtilityFuncs.load_npz(
file_name="npz_node_{0}_final_features".format(node_index))
training_features = features_dict["training_features"]
training_one_hot_labels = features_dict["training_one_hot_labels"]
training_compressed_posteriors = features_dict[
"training_compressed_posteriors"]
training_logits = features_dict["training_logits"]
training_labels = np.argmax(training_one_hot_labels, axis=1)
test_features = features_dict["test_features"]
test_one_hot_labels = features_dict["test_one_hot_labels"]
test_compressed_posteriors = features_dict["test_compressed_posteriors"]
test_logits = features_dict["test_logits"]
test_labels = np.argmax(test_one_hot_labels, axis=1)
modes = features_dict["leaf_modes"]
print("pop_mean_eval = {0}".format(np.sum(results[pop_mean_eval])))
print("shadow_var = {0}".format(np.sum(shadow_var)))
print("pop_var_eval = {0}".format(np.sum(results[pop_var_eval])))
print("b_mean_eval = {0}".format(np.sum(results[b_mean_eval])))
print("b_var_eval = {0}".format(np.sum(results[b_var_eval])))
print("mu_eval = {0}".format(np.sum(results[mu_eval])))
print("sigma_eval = {0}".format(np.sum(results[sigma_eval])))
print("final_mean_eval = {0}".format(np.sum(results[final_mean_eval])))
print("final_var_eval = {0}".format(np.sum(results[final_var_eval])))
dec_mu = np.sum(results[final_mean_eval])
dec_sigma = np.sum(results[final_var_eval])
grads_0_np = results["grads_0"]
grads_1_np = results["grads_1"]
for g0, g1 in zip(grads_0_np, grads_1_np):
assert (np.allclose(g0, g1))
assert (UtilityFuncs.compare_floats(f1=np.sum(shadow_mean),
f2=np.sum(results[pop_mean_eval])))
assert (UtilityFuncs.compare_floats(f1=np.sum(shadow_var),
f2=np.sum(results[pop_var_eval])))
assert (UtilityFuncs.compare_floats(f1=np.sum(
results[final_mean_eval]),
f2=np.sum(results[mu_eval])))
assert (UtilityFuncs.compare_floats(f1=np.sum(results[final_var_eval]),
f2=np.sum(results[sigma_eval])))
# ************************************ Decision ************************************
# ************************************ Classification ************************************
samples, labels, indices_list, one_hot_labels = dataset.get_next_batch(
batch_size=GlobalConstants.BATCH_SIZE)
samples = np.expand_dims(samples, axis=3)
eval_list = [
final_mean_eval, final_var_eval, b_mean_eval, b_var_eval,
def load_dataset(self):
training_data = UtilityFuncs.unpickle(file=self.trainImagesPath)
test_data = UtilityFuncs.unpickle(file=self.testImagesPath)
self.trainingSamples = training_data[b"data"]
self.testSamples = test_data[b"data"]
self.trainingSamples = self.trainingSamples.reshape((self.trainingSamples.shape[0], 3,
CifarDataSet.CIFAR_SIZE, CifarDataSet.CIFAR_SIZE)) \
.transpose([0, 2, 3, 1])
self.testSamples = self.testSamples.reshape((self.testSamples.shape[0], 3,
CifarDataSet.CIFAR_SIZE, CifarDataSet.CIFAR_SIZE)) \
.transpose([0, 2, 3, 1])
self.trainingIndices = np.arange(self.trainingSamples.shape[0])
self.testIndices = np.arange(self.testSamples.shape[0])
# Unpack fine and coarse labels
training_coarse_labels = np.array(
training_data[b"coarse_labels"]).reshape(
(len(training_data[b"coarse_labels"]), 1))
training_fine_labels = np.array(training_data[b"fine_labels"]).reshape(
(len(training_data[b"fine_labels"]), 1))
test_coarse_labels = np.array(test_data[b"coarse_labels"]).reshape(
(len(test_data[b"coarse_labels"]), 1))
test_fine_labels = np.array(test_data[b"fine_labels"]).reshape(
(len(test_data[b"fine_labels"]), 1))
# Pack
self.trainingLabels = training_fine_labels.reshape(
(training_fine_labels.shape[0], ))
self.trainingCoarseLabels = training_coarse_labels.reshape(
(training_coarse_labels.shape[0], ))
self.testLabels = test_fine_labels.reshape(
(test_fine_labels.shape[0], ))
self.testCoarseLabels = test_coarse_labels.reshape(
(test_coarse_labels.shape[0], ))
# Convert labels to one hot encoding.
self.trainingOneHotLabels = UtilityFuncs.convert_labels_to_one_hot(
labels=self.trainingLabels,
max_label=CifarDataSet.CIFAR100_FINE_LABEL_COUNT)
self.trainingCoarseOneHotLabels = UtilityFuncs.convert_labels_to_one_hot(
labels=self.trainingCoarseLabels,
max_label=CifarDataSet.CIFAR100_COARSE_LABEL_COUNT)
self.testOneHotLabels = UtilityFuncs.convert_labels_to_one_hot(
labels=self.testLabels,
max_label=CifarDataSet.CIFAR100_FINE_LABEL_COUNT)
self.testCoarseOneHotLabels = UtilityFuncs.convert_labels_to_one_hot(
labels=self.testCoarseLabels,
max_label=CifarDataSet.CIFAR100_COARSE_LABEL_COUNT)
# Validation Set
if self.validationLoadFile is None:
indices = np.arange(0, self.validationSampleCount)
if self.validationSaveFile is not None:
UtilityFuncs.save_npz(file_name=self.validationSaveFile,
arr_dict={"random_indices": indices})
else:
indices = UtilityFuncs.load_npz(
file_name=self.validationLoadFile)["random_indices"]
self.validationSamples = self.trainingSamples[indices]
self.validationLabels = self.trainingLabels[indices]
self.validationIndices = self.trainingIndices[indices]
self.validationOneHotLabels = self.trainingOneHotLabels[indices]
self.validationCoarseLabels = self.trainingCoarseLabels[indices]
self.validationCoarseOneHotLabels = self.trainingCoarseOneHotLabels[
indices]
self.trainingSamples = np.delete(self.trainingSamples, indices, 0)
self.trainingLabels = np.delete(self.trainingLabels, indices, 0)
self.trainingIndices = np.delete(self.trainingIndices, indices, 0)
self.trainingOneHotLabels = np.delete(self.trainingOneHotLabels,
indices, 0)
self.trainingCoarseLabels = np.delete(self.trainingCoarseLabels,
indices, 0)
self.trainingCoarseOneHotLabels = np.delete(
self.trainingCoarseOneHotLabels, indices, 0)
# Load into Tensorflow Datasets
# dataset = tf.data.Dataset.from_tensor_slices((self.trainingSamples, self.trainingLabels))
self.trainDataset = tf.data.Dataset.from_tensor_slices(
(self.trainingSamples, self.trainingLabels, self.trainingIndices,
self.trainingOneHotLabels, self.trainingCoarseLabels,
self.trainingCoarseOneHotLabels))
self.validationDataset = tf.data.Dataset.from_tensor_slices(
(self.validationSamples, self.validationLabels,
self.validationIndices, self.validationOneHotLabels,
self.validationCoarseLabels, self.validationCoarseOneHotLabels))
self.testDataset = tf.data.Dataset.from_tensor_slices(
(self.testSamples, self.testLabels, self.testIndices,
self.testOneHotLabels, self.testCoarseLabels,
self.testCoarseOneHotLabels))
# Create augmented training set
self.trainDataset = self.trainDataset.shuffle(
buffer_size=self.trainingSamples.shape[0]
) #self.trainingSamples.shape[0]/10)
self.trainDataset = self.trainDataset.map(
CifarDataSet.augment_training_image_fn)
self.trainDataset = self.trainDataset.repeat(
self.augmentationMultiplier)
self.trainDataset = self.trainDataset.batch(batch_size=self.batchSize)
self.trainDataset = self.trainDataset.prefetch(
buffer_size=self.batchSize)
self.trainIter = tf.data.Iterator.from_structure(
self.trainDataset.output_types, self.trainDataset.output_shapes)
features, labels, indices, one_hot_labels, coarse_labels, coarse_one_hot_labels = self.trainIter.get_next(
)
self.outputsDict[DatasetTypes.training] = [
features, labels, indices, one_hot_labels, coarse_labels,
coarse_one_hot_labels
]
self.trainInitOp = self.trainIter.make_initializer(self.trainDataset)
# Create validation set
self.validationDataset = self.validationDataset.map(
CifarDataSet.augment_test_image_fn)
self.validationDataset = self.validationDataset.batch(
batch_size=self.batchSize)
self.validationDataset = self.validationDataset.prefetch(
buffer_size=self.batchSize)
self.validationIter = tf.data.Iterator.from_structure(
self.validationDataset.output_types,
self.validationDataset.output_shapes)
features, labels, indices, one_hot_labels, coarse_labels, coarse_one_hot_labels = self.validationIter.get_next(
)
self.outputsDict[DatasetTypes.validation] = [
features, labels, indices, one_hot_labels, coarse_labels,
coarse_one_hot_labels
]
self.validationInitOp = self.validationIter.make_initializer(
self.validationDataset)
# Create test set
self.testDataset = self.testDataset.map(
CifarDataSet.augment_test_image_fn)
self.testDataset = self.testDataset.batch(batch_size=self.batchSize)
self.testDataset = self.testDataset.prefetch(
buffer_size=self.batchSize)
self.testIter = tf.data.Iterator.from_structure(
self.testDataset.output_types, self.testDataset.output_shapes)
features, labels, indices, one_hot_labels, coarse_labels, coarse_one_hot_labels = self.testIter.get_next(
)
self.outputsDict[DatasetTypes.test] = [
features, labels, indices, one_hot_labels, coarse_labels,
coarse_one_hot_labels
]
self.testInitOp = self.testIter.make_initializer(self.testDataset)
self.set_current_data_set_type(
dataset_type=DatasetTypes.training,
batch_size=self.batchSizesDict[DatasetTypes.training])
from sklearn.model_selection import cross_val_score
from algorithms.softmax_compresser import SoftmaxCompresser
from auxillary.constants import DatasetTypes
from auxillary.db_logger import DbLogger
from auxillary.general_utility_funcs import UtilityFuncs
from simple_tf.global_params import GlobalConstants, SoftmaxCompressionStrategy, GradientType
from collections import namedtuple
from scipy.stats import expon
import matplotlib.pyplot as plt
from sklearn.svm import LinearSVC
from sklearn.model_selection import GridSearchCV
for leaf_index in {3, 4, 5, 6}:
file_name = "npz_node_{0}_all_data".format(leaf_index)
data = UtilityFuncs.load_npz(file_name=file_name)
training_features = data["training_features"]
training_labels = data["training_labels"]
test_features = data["test_features"]
test_labels = data["test_labels"]
training_posteriors_compressed = data["training_posteriors_compressed"]
training_one_hot_labels_compressed = data[
"training_one_hot_labels_compressed"]
test_posteriors_compressed = data["test_posteriors_compressed"]
test_one_hot_labels_compressed = data["test_one_hot_labels_compressed"]
test_accuracy_full = \
SoftmaxCompresser.calculate_compressed_accuracy(posteriors=test_posteriors_compressed,
one_hot_labels=test_one_hot_labels_compressed)
exponential_distribution = scipy.stats.expon(scale=100)
from algorithms.softmax_compresser import SoftmaxCompresser
from auxillary.general_utility_funcs import UtilityFuncs
class NodeMock:
def __init__(self, index):
self.index = index
# "C:\Users\t67rt\Desktop\phd_work\phd_work\simple_tf\parameters.npz"
node_3_dict = UtilityFuncs.load_npz(
file_name=
"C://Users//t67rt//Desktop//phd_work//phd_work//simple_tf//npz_node_3_distillation"
)
node_4_dict = UtilityFuncs.load_npz(
file_name=
"C://Users//t67rt//Desktop//phd_work//phd_work//simple_tf//npz_node_4_distillation"
)
mock_node = NodeMock(index=4)
SoftmaxCompresser.assert_prob_correctness(
softmax_weights=node_4_dict["softmax_weights"],
softmax_biases=node_4_dict["softmax_biases"],
features=node_4_dict["features"],
logits=node_4_dict["logits"],
probs=node_4_dict["probs"],
leaf_node=mock_node)
cm6 = DbLogger.read_confusion_matrix(run_id=run_id,
dataset=2,
iteration=iteration,
num_of_labels=10,
leaf_id=6)
cm_list = [cm3, cm4, cm5, cm6]
mode_counts = []
for cm in cm_list:
total = np.sum(cm)
label_counts = np.sum(cm, axis=1)
label_distribution = label_counts / total
label_distribution_dict = {}
for i in range(len(label_distribution)):
label_distribution_dict[i] = label_distribution[i]
modes = UtilityFuncs.get_modes_from_distribution(
distribution=label_distribution_dict,
percentile_threshold=GlobalConstants.PERCENTILE_THRESHOLD)
mode_counts.append(len(modes))
test_accuracy = DbLogger.read_test_accuracy(run_id=run_id,
type="\"Regular\"")
mode_counts_sorted = tuple(
sorted(mode_counts, key=lambda cnt: cnt, reverse=True))
total_mode_count = sum(mode_counts_sorted)
if total_mode_count != 10:
print("X")
if mode_counts_sorted not in accuracies_dict:
accuracies_dict[mode_counts_sorted] = []
accuracies_dict[mode_counts_sorted].append(test_accuracy)
mean_accuracies_dict = {}
for k, v in accuracies_dict.items():
mean_accuracies_dict[k] = sum(v) / float(len(v))
def get_output(x, node, is_train, leakiness, bn_momentum):
name = UtilityFuncs.get_variable_name(name="final_bn", node=node)
x = ResnetGenerator.batch_norm(name, x, is_train, bn_momentum)
x = ResnetGenerator.relu(x, leakiness)
x = ResnetGenerator.global_avg_pool(x)