def build_evaluation_network(self):
self.evalOutputs = []
if self.logitTensor is None:
raise Exception("No logit tensor have been found.")
# Calculate the number of correct sample inferences
with NetworkChannel(parent_node=self.parentNode,
parent_node_channel=ChannelTypes.evaluation
) as correct_count_channel:
posterior_probs = correct_count_channel.add_operation(
op=tf.nn.softmax(logits=self.logitTensor))
argmax_label_prediction = correct_count_channel.add_operation(
op=tf.argmax(posterior_probs, 1))
comparison_with_labels = correct_count_channel.add_operation(
op=tf.equal(x=argmax_label_prediction, y=self.labelTensor))
comparison_cast = correct_count_channel.add_operation(
op=tf.cast(comparison_with_labels, tf.float32))
self.evalOutputs.append(
correct_count_channel.add_operation(op=tf.reduce_sum(
input_tensor=comparison_cast)))
# Calculate the total number of samples
with NetworkChannel(parent_node=self.parentNode,
parent_node_channel=ChannelTypes.evaluation
) as total_count_channel:
self.evalOutputs.append(
total_count_channel.add_operation(op=tf.size(
input=comparison_cast)))
def l1_func(network, node):
parent_node = get_parent(network=network, node=node)
conv_input = network.add_nodewise_input(
producer_node=parent_node,
producer_channel=ChannelTypes.f_operator,
producer_channel_index=0,
dest_node=node)
h_input = network.add_nodewise_input(
producer_node=parent_node,
producer_channel=ChannelTypes.h_operator,
producer_channel_index=0,
dest_node=node)
with NetworkChannel(
parent_node=node,
parent_node_channel=ChannelTypes.f_operator) as f_channel:
TfLayerFactory.create_convolutional_layer(
node=node,
channel=f_channel,
input_tensor=conv_input,
conv_filter_shape=[
5, 5, conv_1_feature_map_count, conv_2_feature_map_count
],
conv_stride_shape=[1, 1, 1, 1],
pooling_shape=[1, 2, 2, 1],
conv_padding="SAME",
pooling_stride_shape=[1, 2, 2, 1],
pooling_padding="SAME",
init_type=InitType.custom,
activation_type=ActivationType.relu,
pooling_type=PoolingType.max,
post_fix=ChannelTypes.f_operator.value)
with NetworkChannel(
parent_node=node,
parent_node_channel=ChannelTypes.h_operator) as h_channel:
h_channel.add_operation(op=h_input)
def root_func(network, node):
# Data Input
x = network.add_nodewise_input(producer_channel=ChannelTypes.data_input,
dest_node=node)
# Label Input
y = network.add_nodewise_input(producer_channel=ChannelTypes.label_input,
dest_node=node)
with NetworkChannel(
parent_node=node,
parent_node_channel=ChannelTypes.f_operator) as f_channel:
# Reshape x for convolutions
x_image = tf.reshape(
x, [-1, MnistDataSet.MNIST_SIZE, MnistDataSet.MNIST_SIZE, 1])
# Convolution Filter 1
TfLayerFactory.create_convolutional_layer(
node=node,
channel=f_channel,
input_tensor=x_image,
conv_filter_shape=[5, 5, 1, conv_1_feature_map_count],
conv_stride_shape=[1, 1, 1, 1],
pooling_shape=[1, 2, 2, 1],
conv_padding="SAME",
pooling_stride_shape=[1, 2, 2, 1],
pooling_padding="SAME",
init_type=InitType.custom,
activation_type=ActivationType.relu,
pooling_type=PoolingType.max,
post_fix=ChannelTypes.f_operator.value)
with NetworkChannel(
parent_node=node,
parent_node_channel=ChannelTypes.h_operator) as h_channel:
x_flattened = tf.reshape(
x, [-1, MnistDataSet.MNIST_SIZE * MnistDataSet.MNIST_SIZE])
h_channel.add_operation(op=x_flattened)
def __init__(self, parent_node, feature_list, label_tensor, class_count):
super().__init__(parent_node=parent_node,
loss_type=LossType.objective,
is_differentiable=True)
self.logitTensor = None
self.featureList = feature_list
self.labelTensor = label_tensor
self.classCount = class_count
# Pre-Loss channel
with NetworkChannel(
parent_node=self.parentNode,
parent_node_channel=ChannelTypes.pre_loss) as pre_loss_channel:
if len(self.featureList) > 1:
final_feature = pre_loss_channel.add_operation(
op=tf.concat(values=self.featureList, axis=1))
elif len(self.featureList) == 1:
final_feature = self.featureList[0]
else:
raise Exception(
"No features have been passed to cross entropy objective_loss."
)
final_dimension = final_feature.shape[1].value
self.logitTensor = TfLayerFactory.create_fc_layer(
node=self.parentNode,
channel=pre_loss_channel,
input_tensor=final_feature,
fc_shape=[final_dimension, self.classCount],
init_type=self.parentNode.parentNetwork.lossLayerInit,
activation_type=self.parentNode.parentNetwork.
lossLayerActivation,
post_fix=ChannelTypes.pre_loss.value)
def build_training_network(self):
wd_tensor = self.parentNode.parentNetwork.add_networkwise_input(name=self.get_name(), tensor_type=tf.float32)
with NetworkChannel(parent_node=self.parentNode,
parent_node_channel=ChannelTypes.regularization_loss) as loss_channel:
l2_loss = loss_channel.add_operation(op=tf.nn.l2_loss(self.parameter.tensor))
self.lossOutputs = [wd_tensor * l2_loss]
loss_channel.add_operation(op=(self.lossOutputs[0]))
def attach_decision(self):
# Step 1): Gather all inputs which will enter to decision step.
tensor_list = []
# Get all ancestor activations, as allowed by the related hyperparameter.
tensor_list.extend(self.get_activation_inputs())
# Get all h operators
tensor_list.extend(self.get_outputs_of_given_type(channel_set={ChannelTypes.h_operator}))
# Create activation output
with NetworkChannel(parent_node=self,
parent_node_channel=ChannelTypes.branching_activation) as branching_channel:
# Concatenate all tensors
concatenated_features = branching_channel.add_operation(op=tf.concat(tensor_list, axis=1))
feature_dimension = concatenated_features.shape[1].value
# Apply W'[h_operators,parent_activations] + b, where W' is the transpose of the hyperplanes and
# b are the biases.
activation_tensor = TfLayerFactory.create_fc_layer(node=self, channel=branching_channel,
input_tensor=concatenated_features,
fc_shape=[feature_dimension,
self.parentNetwork.treeDegree],
init_type=self.parentNetwork.activationInit,
activation_type=ActivationType.no_activation,
post_fix=ChannelTypes.branching_activation.value)
# Create branching probabilities
with NetworkChannel(parent_node=self,
parent_node_channel=ChannelTypes.branching_probabilities) as branch_prob_channel:
branch_probabilities_tensor = branch_prob_channel.add_operation(op=tf.nn.softmax(logits=activation_tensor))
# Create the Nxk matrix, where N is the minibatch size and k is the branch count, which contains in its (i,j)
# entry a boolean, indicating whether the sample i will go into the child j of this node.
with NetworkChannel(parent_node=self,
parent_node_channel=ChannelTypes.branching_masks_unified) as branch_masks_unif_channel:
branch_probability_threshold = self.parentNetwork.get_networkwise_input(
name=GlobalInputNames.branching_prob_threshold.value)
unified_branch_mask_tensor = branch_masks_unif_channel.add_operation(
op=tf.greater_equal(x=branch_probabilities_tensor, y=branch_probability_threshold))
# Create binary masks for each branch, which will be boolean vectors of the size (N,). k separate such vectors
# will be generated.
for k in range(self.parentNetwork.treeDegree):
with NetworkChannel(parent_node=self,
parent_node_channel=ChannelTypes.branching_masks_sliced) as branch_masks_sliced_channel:
pre_mask_tensor = branch_masks_sliced_channel.add_operation(
op=tf.slice(unified_branch_mask_tensor, [0, k], [-1, 1]))
branch_masks_sliced_channel.add_operation(op=tf.reshape(pre_mask_tensor, [-1]))
def apply_decision(self, tensor):
parents = self.parentNetwork.dag.parents(node=self)
if len(parents) != 1:
raise Exception("Number of parents is not 1 in tree network.")
parent = parents[0]
child_index_wrt_parent = (self.index - 1) % self.parentNetwork.treeDegree
mask_output = parent.get_output(
producer_triple=(parent, ChannelTypes.branching_masks_sliced, child_index_wrt_parent))
mask_tensor = mask_output.tensor
with NetworkChannel(parent_node=self, parent_node_channel=ChannelTypes.decision) as decision_channel:
branched_tensor = decision_channel.add_operation(op=tf.boolean_mask(tensor=tensor, mask=mask_tensor))
return branched_tensor
def build_evaluation_network(self):
with NetworkChannel(
parent_node=self.parentNode,
parent_node_channel=ChannelTypes.evaluation) as eval_channel:
eval_channel.add_operation(op=self.sampleIndexTensor)
if not self.parentNode.isLeaf:
branch_probs = self.parentNode.get_output(
producer_triple=(self.parentNode,
ChannelTypes.branching_probabilities,
0)).tensor
self.evalOutputs = [self.sampleIndexTensor, branch_probs]
else:
self.evalOutputs = [self.sampleIndexTensor]
def build_training_network(self):
if self.logitTensor is None:
raise Exception("No logit tensor have been found.")
with NetworkChannel(
parent_node=self.parentNode,
parent_node_channel=ChannelTypes.objective_loss,
channel_name=CrossEntropyLoss.Name) as loss_channel:
softmax_cross_entropy = loss_channel.add_operation(
op=tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=self.labelTensor, logits=self.logitTensor))
total_softmax_cross_entropy = loss_channel.add_operation(
op=tf.reduce_sum(input_tensor=softmax_cross_entropy))
batch_size = self.parentNode.parentNetwork.get_networkwise_input(
name=GlobalInputNames.batch_size.value)
avg_softmax_cross_entropy = loss_channel.add_operation(
op=(total_softmax_cross_entropy / batch_size))
self.lossOutputs = [avg_softmax_cross_entropy]
def __init__(self, parent_node):
super().__init__(parent_node=parent_node,
loss_type=LossType.eval_term,
is_differentiable=False)
self.sampleIndexTensor = None
if self.parentNode.isRoot:
# First add as a normal input to the root node, then add into the evaluation channel.
self.sampleIndexTensor = self.parentNode.parentNetwork.add_nodewise_input(
producer_channel=ChannelTypes.indices_input,
dest_node=self.parentNode)
else:
self.sampleIndexTensor = self.parentNode.parentNetwork.add_nodewise_input(
producer_node=self.parentNode.parentNetwork.get_root_node(),
producer_channel=ChannelTypes.indices_input,
dest_node=self.parentNode)
with NetworkChannel(
parent_node=self.parentNode,
parent_node_channel=ChannelTypes.pre_loss) as pre_loss_channel:
pre_loss_channel.add_operation(op=self.sampleIndexTensor)
def __init__(self, name, symbolic_network_object, container_node,
arg_type):
self.name = name
self.inputName = "{0}_{1}".format(
self.name, GlobalInputNames.parameter_update.value)
self.tensor = symbolic_network_object
self.containerNode = container_node
self.valueArray = None
self.gradientArray = None
self.globalParameterIndex = None
self.parameterType = arg_type
self.assignOp = None
# Create update mechanism for the parameter tensor
self.inputTensor = \
self.containerNode.parentNetwork.add_networkwise_input(name=self.inputName, tensor_type=self.tensor.dtype)
with NetworkChannel(parent_node=self.containerNode,
parent_node_channel=ChannelTypes.parameter_update
) as param_update_channel:
self.assignOp = param_update_channel.add_operation(
op=tf.assign(self.tensor, self.inputTensor))
def build_training_network(self):
with NetworkChannel(parent_node=self.parentNode,
parent_node_channel=ChannelTypes.
non_differentiable_loss) as non_dif_loss_channel:
sample_count_tensor = non_dif_loss_channel.add_operation(
op=tf.size(input=self.sampleIndexTensor))
if self.parentNode != self.parentNode.parentNetwork.get_root_node(
):
label_tensor = self.parentNode.parentNetwork.add_nodewise_input(
producer_node=self.parentNode.parentNetwork.get_root_node(
),
producer_channel=ChannelTypes.label_input,
producer_channel_index=0,
dest_node=self.parentNode)
self.lossOutputs = [
sample_count_tensor, self.sampleIndexTensor, label_tensor
]
else:
self.lossOutputs = [
sample_count_tensor, self.sampleIndexTensor
]
def leaf_func(network, node):
parent_node = get_parent(network=network, node=node)
conv_input = network.add_nodewise_input(
producer_node=parent_node,
producer_channel=ChannelTypes.f_operator,
producer_channel_index=0,
dest_node=node)
# F channel
with NetworkChannel(
parent_node=node,
parent_node_channel=ChannelTypes.f_operator) as f_channel:
flattened_conv = f_channel.add_operation(
op=tf.reshape(conv_input, [-1, 7 * 7 * conv_2_feature_map_count]))
TfLayerFactory.create_fc_layer(
node=node,
channel=f_channel,
input_tensor=flattened_conv,
fc_shape=[7 * 7 * conv_2_feature_map_count, fc_unit_count],
init_type=InitType.custom,
activation_type=ActivationType.relu,
post_fix=ChannelTypes.f_operator.value)
def attach_acc_node_loss_eval_channels(self):
# Step 1) Build the final loss layer.
self.parentNetwork.objectiveLossTensors = []
self.parentNetwork.regularizationTensors = []
self.parentNetwork.nonDifferentiableLossTensors = []
self.parentNetwork.allLossTensorsDict = {}
for node in self.parentNetwork.nodes.values():
for loss_object in node.losses.values():
if loss_object.lossOutputs is None:
continue
if loss_object.isDifferentiable:
if loss_object.lossType == LossType.regularization:
self.parentNetwork.regularizationTensors.extend(loss_object.lossOutputs)
elif loss_object.lossType == LossType.objective:
self.parentNetwork.objectiveLossTensors.extend(loss_object.lossOutputs)
else:
self.parentNetwork.nonDifferentiableLossTensors.extend(loss_object.lossOutputs)
self.parentNetwork.trainingTensorsDict[loss_object.get_name()] = loss_object.lossOutputs
# Step 2) Add all objective_loss tensors
if len(self.parentNetwork.objectiveLossTensors) > 1:
with NetworkChannel(parent_node=self,
parent_node_channel=ChannelTypes.total_objective_loss) as total_objective_loss:
self.parentNetwork.totalObjectiveLossTensor = total_objective_loss.add_operation(
op=tf.add_n(self.parentNetwork.objectiveLossTensors))
else:
self.parentNetwork.totalObjectiveLossTensor = self.parentNetwork.objectiveLossTensors[0]
self.parentNetwork.trainingTensorsDict[ChannelTypes.total_objective_loss.value] = \
self.parentNetwork.totalObjectiveLossTensor
# Step 3) Add all regularization_loss tensors
if len(self.parentNetwork.regularizationTensors) > 1:
with NetworkChannel(parent_node=self,
parent_node_channel=ChannelTypes.total_regularization_loss) as total_regularization_loss:
self.parentNetwork.totalRegularizationLossTensor = total_regularization_loss.add_operation(
op=tf.add_n(self.parentNetwork.regularizationTensors))
else:
self.parentNetwork.totalRegularizationLossTensor = self.parentNetwork.regularizationTensors[0]
self.parentNetwork.trainingTensorsDict[ChannelTypes.total_regularization_loss.value] = \
self.parentNetwork.totalRegularizationLossTensor
# Step 4) Gather all learnable parameters
self.parentNetwork.allLearnableParameterTensorsList = []
for node in self.parentNetwork.nodes.values():
if node == self:
continue
for parameter in node.parametersDict.values():
if parameter.parameterType == parameterTypes.learnable_parameter:
# if ChannelTypes.branching_activation.value in parameter.name and \
# self.parentNetwork.ancestorCount == 0:
# continue
parameter.globalParameterIndex = len(self.parentNetwork.allLearnableParameterTensorsList)
self.parentNetwork.allLearnableParameterTensorsList.append(parameter.tensor)
# Step 5) Calculate the objective gradients with respect to parameters
self.parentNetwork.objectiveGradientTensors = tf.gradients(self.parentNetwork.totalObjectiveLossTensor,
self.parentNetwork.allLearnableParameterTensorsList)
self.parentNetwork.trainingTensorsDict[ChannelTypes.objective_gradients.value] = \
self.parentNetwork.objectiveGradientTensors
# Step 6) Calculate the regularization gradients with respect to parameters
self.parentNetwork.regularizationGradientTensors = tf.gradients(
self.parentNetwork.totalRegularizationLossTensor,
self.parentNetwork.allLearnableParameterTensorsList)
self.parentNetwork.trainingTensorsDict[ChannelTypes.regularization_gradients.value] = \
self.parentNetwork.regularizationGradientTensors
# Step 7) Prepare the evaluation tensors
self.parentNetwork.evaluationTensorsDict = {}
for node in self.parentNetwork.nodes.values():
for loss_object in node.losses.values():
if loss_object.evalOutputs is None:
continue
self.parentNetwork.evaluationTensorsDict[loss_object.get_name()] = loss_object.evalOutputs
def baseline_network(network, node):
# Data Input
x = network.add_nodewise_input(producer_channel=ChannelTypes.data_input,
dest_node=node)
# Label Input
y = network.add_nodewise_input(producer_channel=ChannelTypes.label_input,
dest_node=node)
# F channel
with NetworkChannel(
parent_node=node,
parent_node_channel=ChannelTypes.f_operator) as f_channel:
# Reshape x for convolutions
x_image = tf.reshape(
x, [-1, MnistDataSet.MNIST_SIZE, MnistDataSet.MNIST_SIZE, 1])
# Convolution Filter 1
conv_1_feature_map_count = 20
conv_layer_1 = TfLayerFactory.create_convolutional_layer(
node=node,
channel=f_channel,
input_tensor=x_image,
conv_filter_shape=[5, 5, 1, conv_1_feature_map_count],
conv_stride_shape=[1, 1, 1, 1],
pooling_shape=[1, 2, 2, 1],
conv_padding="SAME",
pooling_stride_shape=[1, 2, 2, 1],
pooling_padding="SAME",
init_type=InitType.xavier,
activation_type=ActivationType.tanh,
pooling_type=PoolingType.max,
post_fix="{0}_1".format(ChannelTypes.f_operator.value))
# Convolution Filter 2
conv_2_feature_map_count = 50
conv_layer_2 = TfLayerFactory.create_convolutional_layer(
node=node,
channel=f_channel,
input_tensor=conv_layer_1,
conv_filter_shape=[
5, 5, conv_1_feature_map_count, conv_2_feature_map_count
],
conv_stride_shape=[1, 1, 1, 1],
pooling_shape=[1, 2, 2, 1],
conv_padding="SAME",
pooling_stride_shape=[1, 2, 2, 1],
pooling_padding="SAME",
init_type=InitType.xavier,
activation_type=ActivationType.tanh,
pooling_type=PoolingType.max,
post_fix="{0}_2".format(ChannelTypes.f_operator.value))
# Fully Connected Layer 1
fc_unit_count = 500
flattened_conv = f_channel.add_operation(
op=tf.reshape(conv_layer_2, [-1, 7 * 7 *
conv_2_feature_map_count]))
TfLayerFactory.create_fc_layer(
node=node,
channel=f_channel,
input_tensor=flattened_conv,
fc_shape=[7 * 7 * conv_2_feature_map_count, fc_unit_count],
init_type=InitType.xavier,
activation_type=ActivationType.tanh,
post_fix="{0}_3".format(ChannelTypes.f_operator.value))