def Recurrence(over, go_backwards=False, initial_state=initial_state_default_or_None):
# helper to compute previous value
# can take a single Variable/Function or a tuple
initial_state = initial_state if _is_given(initial_state) else _current_default_options.initial_state
# if initial state is given and a numeric constant, then turn it into a Constant() object
if np.isscalar(initial_state):
initial_state = Constant(initial_state, shape=(1)) # TODO: This should be automatically done inside the API.
def previous_hook(state):
if isinstance (state, tuple): # if multiple then apply to each element
return tuple([previous_hook(s) for s in state])
# not a tuple: must be a 'scalar', i.e. a single element
return past_value (state, initial_state) if not go_backwards else \
future_value(state, initial_state)
x = Placeholder(name='recurrence_arg')
state_forward = over.create_placeholder() # create a placeholder or a tuple of placeholders
prev_state = previous_hook(state_forward) # delay (h, c)
f_x_h_c = over(x, prev_state) # apply the recurrent over
# this returns a Function (x, (h_prev, c_prev)) -> (h, c)
h_c = f_x_h_c.outputs
replacements = { value_forward: value for (value_forward, value) in zip(list(_as_tuple(state_forward)), h_c) }
f_x_h_c.replace_placeholders(replacements) # resolves state_forward := h_c
h = f_x_h_c.outputs[0] # 'h' is a Variable (the output of a Function that computed it)
if _trace_layers:
_log_node(h)
_log_node(combine([h.owner]))
apply_x = combine([h]) # the Function that yielded 'h', so we get to know its inputs
# apply_x is a Function x -> h
return Block(apply_x, 'Recurrence', Record(over=over))
def frcn_predictor(features, rois, n_classes):
# Load the pretrained classification net and find nodes
loaded_model = load_model(model_file)
feature_node = find_by_name(loaded_model, feature_node_name)
conv_node = find_by_name(loaded_model, last_conv_node_name)
pool_node = find_by_name(loaded_model, pool_node_name)
last_node = find_by_name(loaded_model, last_hidden_node_name)
# Clone the conv layers and the fully connected layers of the network
conv_layers = combine([conv_node.owner
]).clone(CloneMethod.freeze,
{feature_node: Placeholder()})
fc_layers = combine([last_node.owner]).clone(CloneMethod.clone,
{pool_node: Placeholder()})
# Create the Fast R-CNN model
feat_norm = features - Constant(114)
conv_out = conv_layers(feat_norm)
roi_out = roipooling(conv_out, rois, (roi_dim, roi_dim))
fc_out = fc_layers(roi_out)
# z = Dense(rois[0], num_classes, map_rank=1)(fc_out) # --> map_rank=1 is not yet supported
W = parameter(shape=(4096, n_classes), init=glorot_uniform())
b = parameter(shape=n_classes, init=0)
z = times(fc_out, W) + b
return z
def LSTM_layer(input,
output_dim,
recurrence_hook_h=past_value,
recurrence_hook_c=past_value):
# we first create placeholders for the hidden state and cell state which we don't have yet
dh = placeholder_variable(shape=(output_dim),
dynamic_axes=input.dynamic_axes)
dc = placeholder_variable(shape=(output_dim),
dynamic_axes=input.dynamic_axes)
# we now create an LSTM_cell function and call it with the input and placeholders
LSTM_cell = LSTM(output_dim)
f_x_h_c = LSTM_cell(input, (dh, dc))
h_c = f_x_h_c.outputs
# we setup the recurrence by specifying the type of recurrence (by default it's `past_value` -- the previous value)
h = recurrence_hook_h(h_c[0])
c = recurrence_hook_c(h_c[1])
replacements = {dh: h.output, dc: c.output}
f_x_h_c.replace_placeholders(replacements)
h = f_x_h_c.outputs[0]
c = f_x_h_c.outputs[1]
# and finally we return the hidden state and cell state as functions (by using `combine`)
return combine([h]), combine([c])
def Recurrence(over, go_backwards=False, initial_state=initial_state_default_or_None):
# helper to compute previous value
# can take a single Variable/Function or a tuple
initial_state = initial_state if _is_given(initial_state) else _current_default_options.initial_state
# if initial state is given and a numeric constant, then turn it into a Constant() object
if np.isscalar(initial_state):
initial_state = Constant(initial_state, shape=(1)) # TODO: This should be automatically done inside the API.
def previous_hook(state):
if isinstance (state, tuple): # if multiple then apply to each element
return tuple([previous_hook(s) for s in state])
# not a tuple: must be a 'scalar', i.e. a single element
return past_value (state, initial_state) if not go_backwards else \
future_value(state, initial_state)
x = Placeholder(name='recurrence_arg')
state_forward = over.create_placeholder() # create a placeholder or a tuple of placeholders
prev_state = previous_hook(state_forward) # delay (h, c)
f_x_h_c = over(x, prev_state) # apply the recurrent over
# this returns a Function (x, (h_prev, c_prev)) -> (h, c)
h_c = f_x_h_c.outputs
replacements = { value_forward: value for (value_forward, value) in zip(list(_as_tuple(state_forward)), h_c) }
f_x_h_c.replace_placeholders(replacements) # resolves state_forward := h_c
h = f_x_h_c.outputs[0] # 'h' is a Variable (the output of a Function that computed it)
if _trace_layers:
_log_node(h)
_log_node(combine([h.owner]))
apply_x = combine([h]) # the Function that yielded 'h', so we get to know its inputs
# apply_x is a Function x -> h
return Block(apply_x, 'Recurrence', Record(over=over))
def Recurrence(over, _inf=None, go_backwards=False, initial_state=None):
# helper to compute previous value
# can take a single Variable/Function or a tuple
if go_backwards:
UntestedBranchError("Recurrence, go_backwards option")
def previous_hook(state):
if hasattr(state, 'outputs'):
outputs = state.outputs
if len(outputs) > 1: # if multiple then apply to each element
return tuple([previous_hook(s) for s in outputs])
# not a tuple: must be a 'scalar', i.e. a single element
return past_value (state, initial_state) if not go_backwards else \
future_value(state, initial_state)
x = Placeholder(_inf=_inf, name='recurrence_arg')
prev_state_forward = over.create_placeholder() # create a placeholder or a tuple of placeholders
f_x_h_c = over(x, prev_state_forward) # apply the recurrent over
# this returns a Function (x, (h_prev, c_prev)) -> (h, c)
h = f_x_h_c.outputs[0] # 'h' is a Variable (the output of a Function that computed it)
if _trace_layers:
_log_node(h)
_log_node(combine([h.owner]))
prev_state = previous_hook(f_x_h_c) # delay (h, c)
replacements = { value_forward: value.output for (value_forward, value) in zip(list(prev_state_forward), list(prev_state)) }
f_x_h_c.replace_placeholders(replacements) # binds _h_c := prev_state
apply_x = combine([h.owner]) # the Function that yielded 'h', so we get to know its inputs
# apply_x is a Function x -> h
_name_and_extend_Function(apply_x, 'Recurrence')
if _trace_layers:
_log_node(apply_x)
return apply_x
def create_model(base_model_file,
input_features,
num_classes,
dropout_rate=0.5,
freeze_weights=False):
# Load the pretrained classification net and find nodes
base_model = load_model(base_model_file)
feature_node = find_by_name(base_model, 'features')
beforePooling_node = find_by_name(base_model, "z.x.x.r")
#graph.plot(base_model, filename="base_model.pdf") # Write graph visualization
# Clone model until right before the pooling layer, ie. until including z.x.x.r
modelCloned = combine([beforePooling_node.owner]).clone(
CloneMethod.freeze if freeze_weights else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Center the input around zero and set model input.
# Do this early, to avoid CNTK bug with wrongly estimated layer shapes
feat_norm = input_features - constant(114)
model = modelCloned(feat_norm)
# Pool over all spatial dimensions and add dropout layer
avgPool = GlobalAveragePooling(name="poolingLayer")(model)
if dropout_rate > 0:
avgPoolDrop = Dropout(dropout_rate)(avgPool)
else:
avgPoolDrop = avgPool
# Add new dense layer for class prediction
finalModel = Dense(num_classes, activation=None,
name="prediction")(avgPoolDrop)
return finalModel
def runCntkModel(model, map_file, node_name=[], mb_size=1):
# Get minibatch source
num_classes = model.shape[0]
(image_width, image_height) = find_by_name(model, "input").shape[1:]
minibatch_source = create_mb_source(map_file, image_width, image_height, 3,
num_classes, False)
features_si = minibatch_source['features']
# Set output node
if node_name == []:
output_node = model
else:
node_in_graph = model.find_by_name(node_name)
output_node = combine([node_in_graph.owner])
# Evaluate DNN for all images
feats = []
sample_counts = 0
imgPaths = getColumn(readTable(map_file), 0)
while sample_counts < len(imgPaths):
sample_count = min(mb_size, len(imgPaths) - sample_counts)
mb = minibatch_source.next_minibatch(sample_count)
output = output_node.eval(mb[features_si])
feats += [o.flatten() for o in output]
sample_counts += sample_count
if sample_counts % 100 < mb_size:
print(
"Evaluating DNN (output dimension = {}) for image {} of {}: {}"
.format(len(feats[-1]), sample_counts, len(imgPaths),
imgPaths[sample_counts - 1]))
data = [[imgPath, feat] for imgPath, feat in zip(imgPaths, feats)]
return data
def create_model(base_model_file, feature_node_name, last_hidden_node_name, num_classes, input_features, freeze=False):
# Load the pretrained classification net and find nodes
base_model = load_model(base_model_file)
feature_node = None
last_node = None
# feature node
for n in cntk.logging.graph.depth_first_search(base_model, (lambda x: True)):
if (n.name.strip() == feature_node_name) or (n.uid.strip() == feature_node_name):
feature_node = n
print("feature node:", feature_node)
if feature_node is None:
raise Exception("Failed to locate feature node: " + feature_node_name)
# last hidden node
for n in cntk.logging.get_node_outputs(base_model):
if (n.name.strip() == last_hidden_node_name) or (n.uid.strip() == last_hidden_node_name):
last_node = n
print("last hidden node:", last_node)
if last_node is None:
raise Exception("Failed to locate last hidden node: " + last_hidden_node_name)
# Clone the desired layers with fixed weights
cloned_layers = combine([last_node.owner]).clone(
CloneMethod.freeze if freeze else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Add new dense layer for class prediction
feat_norm = input_features - Constant(114)
cloned_out = cloned_layers(feat_norm)
z = Dense(num_classes, activation=None, name=new_output_node_name) (cloned_out)
return z
def get_extractor():
"""Load the CNN."""
node_name = "z.x"
loaded_model = load_model(MODEL_PATH)
node_in_graph = loaded_model.find_by_name(node_name)
output_nodes = combine([node_in_graph.owner])
return output_nodes
def print_all_node_names(model_file, is_BrainScript=True):
loaded_model = load_model(model_file)
if is_BrainScript:
loaded_model = combine([loaded_model.outputs[0]])
node_list = graph.depth_first_search(loaded_model, lambda x: x.is_output)
print("printing node information in the format")
print("node name (tensor shape)")
for node in node_list:
print(node.name, node.shape)
def cifar_resnet():
dev = 0
cntk_dev = cntk_device(dev)
epoch_size = sys.maxsize
mbs = create_mb_source(epoch_size)
stream_infos = mbs.stream_infos()
for si in stream_infos:
if si.m_name == 'features':
features_si = si
elif si.m_name == 'labels':
labels_si = si
image_shape = features_si.m_sample_layout.dimensions()
image_shape = (image_shape[2], image_shape[0], image_shape[1])
num_classes = labels_si.m_sample_layout.dimensions()[0]
image_input = variable(image_shape,
features_si.m_element_type,
needs_gradient=False,
name="Images")
classifier_output = resnet_classifer(image_input, num_classes, dev,
"classifierOutput")
label_var = variable((num_classes),
features_si.m_element_type,
needs_gradient=False,
name="Labels")
ce = cross_entropy_with_softmax(classifier_output, label_var)
pe = classification_error(classifier_output, label_var)
#TODO: add save and load module code
image_classifier = combine([ce, pe, classifier_output], "ImageClassifier")
lr = learning_rates_per_sample(0.0078125)
mb_size = 32
num_mbs = 1000
trainer = Trainer(classifier_output, ce, pe,
[sgdlearner(classifier_output.owner.parameters(), lr)])
for i in range(0, num_mbs):
mb = mbs.get_next_minibatch(mb_size, cntk_dev)
arguments = dict()
arguments[image_input] = mb[features_si].m_data
arguments[label_var] = mb[labels_si].m_data
trainer.train_minibatch(arguments, cntk_dev)
freq = 20
if i % freq == 0:
training_loss = get_train_loss(trainer)
eval_crit = get_train_eval_criterion(trainer)
print(
"Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}"
.format(i, training_loss, eval_crit))
def print_all_node_names(model_file, is_BrainScript=True):
loaded_model = load_model(model_file)
if is_BrainScript:
loaded_model = combine([loaded_model.outputs[0]])
node_list = graph.depth_first_search(loaded_model, lambda x: x.is_output)
print("printing node information in the format")
print("node name (tensor shape)")
for node in node_list:
print(node.name, node.shape)
def _test_cifar_resnet():
dev = 0
cntk_dev = cntk_device(dev)
epoch_size = sys.maxsize
mbs = create_mb_source(epoch_size)
stream_infos = mbs.stream_infos()
for si in stream_infos:
if si.m_name == 'features':
features_si = si
elif si.m_name == 'labels':
labels_si = si
image_shape = features_si.m_sample_layout.dimensions()
image_shape = (image_shape[2], image_shape[0], image_shape[1])
num_classes = labels_si.m_sample_layout.dimensions()[0]
image_input = variable(image_shape,
features_si.m_element_type,
needs_gradient=False,
name="Images")
classifer_output = resnet_classifer(image_input, num_classes, dev,
"classifierOutput")
label_var = variable((num_classes, ),
features_si.m_element_type,
needs_gradient=False,
name="Labels")
ce = cross_entropy_with_softmax(classifer_output.output(), label_var)
pe = classification_error(classifer_output.output(), label_var)
image_classifier = combine([ce, pe, classifer_output], "ImageClassifier")
learning_rate_per_sample = cntk_py.learning_rates_per_sample(0.0078125)
trainer = cntk_py.Trainer(image_classifier, ce.output(), [
cntk_py.sgdlearner(image_classifier.parameters(),
learning_rate_per_sample)
])
mb_size = 32
num_mbs = 100
minibatch_size_limits = dict()
minibatch_size_limits[features_si] = (0, mb_size)
minibatch_size_limits[labels_si] = (0, mb_size)
for i in range(0, num_mbs):
mb = mbs.get_next_minibatch(minibatch_size_limits, cntk_dev)
arguments = dict()
arguments[image_input] = mb[features_si].m_data
arguments[label_var] = mb[labels_si].m_data
trainer.train_minibatch(arguments, cntk_dev)
freq = 20
if i % freq == 0:
print(str(i + freq) + ": " + str(get_train_loss(trainer)))
def print_all_node_names(model_file, is_BrainScript=True):
loaded_model = load_model(model_file)
if is_BrainScript:
loaded_model = combine([loaded_model.outputs[0]])
node_list = graph.depth_first_search(loaded_model,
lambda x: isinstance(x, Function))
print("printing node information in the format")
for node in node_list:
print("Node name:", node.name)
for out in node.outputs:
print("Output name and shape:", out.name, out.shape)
def translate(tokens,
model_decoding,
vocab,
i2w,
show_attention=False,
max_label_length=20):
vdict = {v: i for i, v in enumerate(vocab)}
try:
w = [vdict["<s>"]] + [vdict[c] for c in tokens] + [vdict["</s>"]]
except:
print('Input contains an unexpected token.')
return []
# convert to one_hot
query = one_hot([w], len(vdict))
pred = model_decoding(query)
pred = pred[0] # first sequence (we only have one) -> [len, vocab size]
if use_attention:
pred = pred[:, 0, 0, :] # attention has extra dimensions
# print out translation and stop at the sequence-end tag
prediction = np.argmax(pred, axis=-1)
translation = [i2w[i] for i in prediction]
# show attention window (requires matplotlib, seaborn, and pandas)
if use_attention and show_attention:
#att_value = model_decoding.attention_model.attention_weights(query)
# BUGBUG: fails with "Forward: Feature Not Implemented"
q = combine([model_decoding.attention_model.attention_weights])
att_value = q(query)
# get the attention data up to the length of the output (subset of the full window)
att_value = att_value[0, 0:len(prediction), 0:len(w), 0,
0] # -> (len, span)
# set up the actual words/letters for the heatmap axis labels
columns = [i2w[ww] for ww in prediction]
index = [i2w[ww] for ww in w]
# show the attention weight heatmap
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
dframe = pd.DataFrame(data=np.fliplr(att_value.T),
columns=columns,
index=index)
sns.heatmap(dframe)
print('close the heatmap window to continue')
plt.show()
return translation
def test_simple_mnist():
input_dim = 784
num_output_classes = 10
num_hidden_layers = 1
hidden_layers_dim = 200
epoch_size = sys.maxsize
minibatch_size = 32
num_samples_per_sweep = 60000
num_sweeps_to_train_with = 3
num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size
lr = cntk_py.learning_rates_per_sample(0.003125)
input = variable((input_dim,), np.float32, needs_gradient=False, name="features")
scaled_input = element_times(constant((), 0.00390625), input)
label = variable((num_output_classes,), np.float32, needs_gradient=False, name="labels")
dev = cntk_py.DeviceDescriptor.cpudevice()
netout = fully_connected_classifier_net(scaled_input.output(), num_output_classes, hidden_layers_dim, num_hidden_layers, dev, sigmoid)
ce = cross_entropy_with_softmax(netout.output(), label)
pe = classification_error(netout.output(), label)
ffnet = combine([ce, pe, netout], "classifier_model")
cm = create_mb_source(input_dim, num_output_classes, epoch_size)
stream_infos = cm.stream_infos()
for si in stream_infos:
if si.m_name == 'features':
features_si = si
elif si.m_name == 'labels':
labels_si = si
minibatch_size_limits = dict()
minibatch_size_limits[features_si] = (0,minibatch_size)
minibatch_size_limits[labels_si] = (0,minibatch_size)
trainer = cntk_py.Trainer(ffnet, ce.output(), [cntk_py.sgdlearner(ffnet.parameters(), lr)])
for i in range(0,int(num_minibatches_to_train)):
mb=cm.get_next_minibatch(minibatch_size_limits, dev)
arguments = dict()
arguments[input] = mb[features_si].m_data
arguments[label] = mb[labels_si].m_data
trainer.train_minibatch(arguments, dev)
freq = 20
if i % freq == 0:
training_loss = get_train_loss(trainer)
print(str(i+freq) + ": " + str(training_loss))
#TODO: move the testing code into a separate test module ?
assert np.allclose(training_loss, 0.6142425537109375, atol=TOLERANCE_ABSOLUTE)
def modify_model(features, n_classes):
loaded_model = load_model(model_file)
feature_node = find_by_name(loaded_model, 'features')
last_node = find_by_name(loaded_model, 'h2_d')
all_layers = combine([last_node.owner
]).clone(CloneMethod.freeze,
{feature_node: placeholder()})
feat_norm = features - Constant(114)
fc_out = all_layers(feat_norm)
z = Dense(num_classes)(fc_out)
return (z)
def debug_attention(model, input):
q = combine([model, model.attention_model.attention_weights])
#words, p = q(input) # Python 3
words_p = q(input)
words = words_p[0]
p = words_p[1]
len = words.shape[attention_axis-1]
span = 7 #attention_span #7 # test sentence is 7 tokens long
p_sq = np.squeeze(p[0,:len,:span,0,:]) # (batch, len, attention_span, 1, vector_dim)
opts = np.get_printoptions()
np.set_printoptions(precision=5)
print(p_sq)
np.set_printoptions(**opts)
def load_model(self):
if self.__model:
raise Exception("Model already loaded")
trained_frcnn_model = load_model(self.__model_path)
# cache indices of the model arguments
args_indices = {}
for i, arg in enumerate(trained_frcnn_model.arguments):
args_indices[arg.name] = i
self.__nr_rois = trained_frcnn_model.arguments[
args_indices["rois"]].shape[0]
self.__resize_width = trained_frcnn_model.arguments[
args_indices["features"]].shape[1]
self.__resize_height = trained_frcnn_model.arguments[
args_indices["features"]].shape[2]
self.labels_count = trained_frcnn_model.arguments[
args_indices["roiLabels"]].shape[1]
# next, we adjust the clone the model and create input nodes just for the features (image) and ROIs
# This will make sure that only the calculations that are needed for evaluating images are performed
# during test time
#
# find the original features and rois input nodes
features_node = find_by_name(trained_frcnn_model, "features")
rois_node = find_by_name(trained_frcnn_model, "rois")
# find the output "z" node
z_node = find_by_name(trained_frcnn_model, 'z')
# define new input nodes for the features (image) and rois
image_input = input_variable(features_node.shape, name='features')
roi_input = input_variable(rois_node.shape, name='rois')
# Clone the desired layers with fixed weights and place holder for the new input nodes
cloned_nodes = combine([z_node.owner]).clone(
CloneMethod.freeze, {
features_node: placeholder(name='features'),
rois_node: placeholder(name='rois')
})
# apply the cloned nodes to the input nodes to obtain the model for evaluation
self.__model = cloned_nodes(image_input, roi_input)
# cache the indices of the input nodes
self.__args_indices = {}
for i, arg in enumerate(self.__model.arguments):
self.__args_indices[arg.name] = i
def debug_attention(model, input):
q = combine([model, model.attention_model.attention_weights])
#words, p = q(input) # Python 3
words_p = q(input)
words = words_p[0]
p = words_p[1]
len = words.shape[attention_axis - 1]
span = 7 #attention_span #7 # test sentence is 7 tokens long
p_sq = np.squeeze(p[0, :len, :span,
0, :]) # (batch, len, attention_span, 1, vector_dim)
opts = np.get_printoptions()
np.set_printoptions(precision=5)
print(p_sq)
np.set_printoptions(**opts)
def frcn_predictor(features, rois, n_classes, base_path):
# model specific variables for AlexNet
model_file = base_path + "/../../../resources/cntk/AlexNet.model"
roi_dim = 6
feature_node_name = "features"
last_conv_node_name = "conv5.y"
pool_node_name = "pool3"
last_hidden_node_name = "h2_d"
# Load the pretrained classification net and find nodes
print("Loading pre-trained model...")
loaded_model = load_model(model_file)
print("Loading pre-trained model... DONE.")
feature_node = find_by_name(loaded_model, feature_node_name)
conv_node = find_by_name(loaded_model, last_conv_node_name)
pool_node = find_by_name(loaded_model, pool_node_name)
last_node = find_by_name(loaded_model, last_hidden_node_name)
# Clone the conv layers and the fully connected layers of the network
conv_layers = combine([conv_node.owner
]).clone(CloneMethod.freeze,
{feature_node: placeholder()})
fc_layers = combine([last_node.owner]).clone(CloneMethod.clone,
{pool_node: placeholder()})
# Create the Fast R-CNN model
feat_norm = features - constant(114)
conv_out = conv_layers(feat_norm)
roi_out = roipooling(conv_out, rois, (roi_dim, roi_dim))
fc_out = fc_layers(roi_out)
#fc_out.set_name("fc_out")
# z = Dense(rois[0], num_classes, map_rank=1)(fc_out) # --> map_rank=1 is not yet supported
W = parameter(shape=(4096, n_classes), init=glorot_uniform())
b = parameter(shape=n_classes, init=0)
z = times(fc_out, W) + b
return z, fc_out
def ffnet():
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
epoch_size = sys.maxsize
minibatch_size = 25
num_samples_per_sweep = 10000
num_sweeps_to_train_with = 2
num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size
lr = learning_rates_per_sample(0.02)
input = variable((input_dim,), np.float32, needs_gradient=False, name="features")
label = variable((num_output_classes,), np.float32, needs_gradient=False, name="labels")
dev = -1
cntk_dev = cntk_device(dev)
netout = fully_connected_classifier_net(input, num_output_classes, hidden_layers_dim, num_hidden_layers, dev, sigmoid)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
#TODO: add save and load module code
ffnet = combine([ce, pe, netout], "classifier_model")
rel_path = r"../../../../Examples/Other/Simple2d/Data/SimpleDataTrain_cntk_text.txt"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
cm = create_text_mb_source(path, input_dim, num_output_classes, epoch_size)
stream_infos = cm.stream_infos()
for si in stream_infos:
if si.m_name == 'features':
features_si = si
elif si.m_name == 'labels':
labels_si = si
trainer = Trainer(netout, ce, pe, [sgdlearner(netout.owner.parameters(), lr)])
for i in range(0,int(num_minibatches_to_train)):
mb=cm.get_next_minibatch(minibatch_size, cntk_dev)
arguments = dict()
arguments[input] = mb[features_si].m_data
arguments[label] = mb[labels_si].m_data
trainer.train_minibatch(arguments, cntk_dev)
freq = 20
if i % freq == 0:
training_loss = get_train_loss(trainer)
eval_crit = get_train_eval_criterion(trainer)
print ("Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}".format(i, training_loss, eval_crit))
def frcn_predictor(features, rois, n_classes):
# Load the pretrained classification net and find nodes
loaded_model = load_model(model_file)
feature_node = find_by_name(loaded_model, feature_node_name)
conv_node = find_by_name(loaded_model, last_conv_node_name)
pool_node = find_by_name(loaded_model, pool_node_name)
last_node = find_by_name(loaded_model, last_hidden_node_name)
# Clone the conv layers and the fully connected layers of the network
conv_layers = combine([conv_node.owner]).clone(CloneMethod.freeze, {feature_node: Placeholder()})
fc_layers = combine([last_node.owner]).clone(CloneMethod.clone, {pool_node: Placeholder()})
# Create the Fast R-CNN model
feat_norm = features - Constant(114)
conv_out = conv_layers(feat_norm)
roi_out = roipooling(conv_out, rois, (roi_dim, roi_dim))
fc_out = fc_layers(roi_out)
# z = Dense(rois[0], num_classes, map_rank=1)(fc_out) # --> map_rank=1 is not yet supported
W = parameter(shape=(4096, n_classes), init=glorot_uniform())
b = parameter(shape=n_classes, init=0)
z = times(fc_out, W) + b
return z
def sanitize_function(arg):
'''
Tries to retrieve a Function from the argument or throws an exception if
that's not possible.
'''
from cntk.ops import combine
if isinstance(arg, cntk_py.Variable):
arg = combine([arg])
if not isinstance(arg, cntk_py.Function):
raise TypeError("Object of type %s cannot be cast to Variable" %
str(type(arg)))
return arg
def sanitize_function(arg):
'''
Tries to retrieve a Function from the argument or throws an exception if
that's not possible.
'''
from cntk.ops import combine
if isinstance(arg, cntk_py.Variable):
arg = combine([arg])
if not isinstance(arg, cntk_py.Function):
raise TypeError("Object of type %s cannot be cast to Variable" %
str(type(arg)))
return arg
def translate(tokens, model_decoding, vocab, i2w, show_attention=False, max_label_length=20):
vdict = {v:i for i,v in enumerate(vocab)}
try:
w = [vdict["<s>"]] + [vdict[c] for c in tokens] + [vdict["</s>"]]
except:
print('Input contains an unexpected token.')
return []
# convert to one_hot
query = one_hot([w], len(vdict))
pred = model_decoding(query)
pred = pred[0] # first sequence (we only have one) -> [len, vocab size]
if use_attention:
pred = pred[:,0,0,:] # attention has extra dimensions
# print out translation and stop at the sequence-end tag
prediction = np.argmax(pred, axis=-1)
translation = [i2w[i] for i in prediction]
# show attention window (requires matplotlib, seaborn, and pandas)
if use_attention and show_attention:
#att_value = model_decoding.attention_model.attention_weights(query)
# BUGBUG: fails with "Forward: Feature Not Implemented"
q = combine([model_decoding.attention_model.attention_weights])
att_value = q(query)
# get the attention data up to the length of the output (subset of the full window)
att_value = att_value[0,0:len(prediction),0:len(w),0,0] # -> (len, span)
# set up the actual words/letters for the heatmap axis labels
columns = [i2w[ww] for ww in prediction]
index = [i2w[ww] for ww in w]
# show the attention weight heatmap
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
dframe = pd.DataFrame(data=np.fliplr(att_value.T), columns=columns, index=index)
sns.heatmap(dframe)
print('close the heatmap window to continue')
plt.show()
return translation
def runCntkModelAllImages(model, classes, imgDir, mbs, node_name, mb_size=1):
log = logging.getLogger("neuralnets.utils.runCntkModelAllImages")
# Create empty dnn output dictionary
#dnnOutput = dict()
#for label in classes:
# dnnOutput[label] = dict()
imgPaths = [line.strip('\n')
for line in open(imgDir)] # Prepare cntk input
# Run CNTK model for each image
num_classes = model.shape[0]
image_dimensions = find_by_name(model, "input").shape[::-1]
# Set output node
if node_name == []:
output_node = model # use final pred layer
else:
node_in_graph = model.find_by_name(
node_name) # gives poolingLayer output 512*1*1
output_node = combine([node_in_graph.owner]) # Set output node
# Evaluate DNN for all images
feats = []
labels = []
sample_counts = 0
while sample_counts < len(imgPaths):
sample_count = min(mb_size, len(imgPaths) - sample_counts)
mb = mbs.next_minibatch(sample_count)
output = output_node.eval(mb[mbs['features']])
prev_count = sample_counts
sample_counts += sample_count
for path, o in zip(imgPaths[prev_count:sample_counts], output):
feat = o.flatten()
feat /= np.linalg.norm(feat,
2) #normalizing the features for this image
(imgFilename, imgSubdir) = os.path.basename(path).split()
#dnnOutput[int(imgSubdir)][imgFilename] = feat
feats.append(np.float32(feat))
labels.append(int(imgSubdir))
if sample_counts % 100 < mb_size:
log.info(
"Evaluating DNN (output dimension = %s) for image %s of %s: %s"
% (len(feats[-1]), sample_counts, len(imgPaths), imgFilename))
#repeat for train and test
return feats, labels
def sanitize_function(arg):
'''
Tries to retrieve a Function from the argument or throws an exception if
that's not possible.
'''
from cntk.ops import combine
if isinstance(arg, cntk_py.Variable):
arg = combine([arg])
if len(arg.outputs) != 1: # BUGBUG: This seems to happen with BlockFunctions?
raise TypeError("casting Variable to Function unexpectedly returned a tuple")
if not isinstance(arg, cntk_py.Function):
raise TypeError("Object of type %s cannot be cast to Variable" %
str(type(arg)))
return arg
def eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects):
# load model and pick desired node as output
loaded_model = load_model(model_file)
node_in_graph = loaded_model.find_by_name(node_name)
output_nodes = combine([node_in_graph.owner])
# evaluate model and get desired node output
print("Evaluating model for output node %s" % node_name)
features_si = minibatch_source['features']
with open(output_file, 'wb') as results_file:
for i in range(0, num_objects):
mb = minibatch_source.next_minibatch(1)
output = output_nodes.eval(mb[features_si])
# write results to file
out_values = output[0,0].flatten()
np.savetxt(results_file, out_values[np.newaxis], fmt="%.6f")
def eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects):
# load model and pick desired node as output
loaded_model = load_model(model_file)
node_in_graph = loaded_model.find_by_name(node_name)
output_nodes = combine([node_in_graph.owner])
# evaluate model and get desired node output
print("Evaluating model for output node %s" % node_name)
features_si = minibatch_source['features']
with open(output_file, 'wb') as results_file:
for i in range(0, num_objects):
mb = minibatch_source.next_minibatch(1)
output = output_nodes.eval(mb[features_si])
# write results to file
out_values = output[0,0].flatten()
np.savetxt(results_file, out_values[np.newaxis], fmt="%.6f")
def sanitize_function(arg):
'''
Tries to retrieve a Function from the argument or raises a TypeError if
that's not possible.
'''
from cntk.ops import combine
if isinstance(arg, cntk_py.Variable):
arg = combine([arg])
if len(arg.outputs) != 1: # BUGBUG: This seems to happen with BlockFunctions?
raise TypeError("casting Variable to Function unexpectedly returned a tuple")
if not isinstance(arg, cntk_py.Function):
raise TypeError("Object of type %s cannot be cast to Variable" %
str(type(arg)))
return arg
def create_model(base_model_file, feature_node_name, last_hidden_node_name, num_classes, input_features, freeze=False):
# Load the pretrained classification net and find nodes
base_model = load_model(base_model_file)
feature_node = find_by_name(base_model, feature_node_name)
last_node = find_by_name(base_model, last_hidden_node_name)
# Clone the desired layers with fixed weights
cloned_layers = combine([last_node.owner]).clone(
CloneMethod.freeze if freeze else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Add new dense layer for class prediction
feat_norm = input_features - Constant(114)
cloned_out = cloned_layers(feat_norm)
z = Dense(num_classes, activation=None, name=new_output_node_name) (cloned_out)
return z
def create_model(base_model_file, input_features, params):
num_classes = params['num_classes']
dropout_rate = params['dropout_rate']
freeze_weights = params['freeze_weights']
# Load the pretrained classification net and find nodes
base_model = load_model(base_model_file)
log = logging.getLogger("neuralnets1.utils.create_model")
log.info('Loaded base model - %s with layers:' % base_model_file)
node_outputs = get_node_outputs(base_model)
[log.info('%s , %s' % (layer.name, layer.shape)) for layer in node_outputs]
graph.plot(base_model,
filename="base_model.pdf") # Write graph visualization
feature_node = find_by_name(base_model, 'features')
beforePooling_node = find_by_name(base_model, "z.x.x.r")
# Clone model until right before the pooling layer, ie. until including z.x.x.r
modelCloned = combine([beforePooling_node.owner]).clone(
CloneMethod.freeze if freeze_weights else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Center the input around zero and set model input.
# Do this early, to avoid CNTK bug with wrongly estimated layer shapes
feat_norm = input_features - constant(114)
model = modelCloned(feat_norm)
# Add pool layer
avgPool = GlobalAveragePooling(name="poolingLayer")(
model) # assign name to the layer and add to the model
# Add drop out layer
if dropout_rate > 0:
avgPoolDrop = Dropout(dropout_rate)(
avgPool
) # add drop out layer with specified drop out rate and add it to the model
else:
avgPoolDrop = avgPool
# Add new dense layer for class prediction
finalModel = Dense(num_classes, activation=None, name="Dense")(avgPoolDrop)
return finalModel
def instance_functions(self, cntk_sorted_layers, cntk_layers):
'''
Instace all nodes into CNTK ops
Args:
cntk_sorted_layers (list): the list contains the name of instaced layers with
traversal order
cntk_layers (dict): the dict contains all layers definition
Return:
None
'''
unused_func = set()
for cntk_sorted_layer in cntk_sorted_layers:
cntk_layer = cntk_layers[cntk_sorted_layer]
local_inputs = []
for local_input in cntk_layer.inputs:
local_inputs.append(self._functions[local_input])
if self._functions[local_input] in unused_func:
unused_func.remove(self._functions[local_input])
self._functions[cntk_layer.op_name] = getattr(ApiSetup, cntk_layer.op_type.name\
)(cntk_layer, local_inputs)
unused_func.add(self._functions[cntk_layer.op_name])
self._output = ops.combine(list(unused_func), name='outputs')
def instance_functions(self, cntk_sorted_layers, cntk_layers):
'''
Instace all nodes into CNTK ops
Args:
cntk_sorted_layers (list): the list contains the name of instaced layers with
traversal order
cntk_layers (dict): the dict contains all layers definition
Return:
None
'''
unused_func = set()
for cntk_sorted_layer in cntk_sorted_layers:
cntk_layer = cntk_layers[cntk_sorted_layer]
local_inputs = []
for local_input in cntk_layer.inputs:
local_inputs.append(self._functions[local_input])
if self._functions[local_input] in unused_func:
unused_func.remove(self._functions[local_input])
self._functions[cntk_layer.op_name] = getattr(ApiSetup, cntk_layer.op_type.name\
)(cntk_layer, local_inputs)
unused_func.add(self._functions[cntk_layer.op_name])
self._output = ops.combine(list(unused_func), name='outputs')
def generate_visualization(use_brain_script_model, testing=False):
num_objects_to_eval = 5
if (use_brain_script_model):
model_file_name = "07_Deconvolution_BS.model"
encoder_output_file_name = "encoder_output_BS.txt"
decoder_output_file_name = "decoder_output_BS.txt"
enc_node_name = "z.pool1"
input_node_name = "f2"
output_node_name = "z"
else:
model_file_name = "07_Deconvolution_PY.model"
encoder_output_file_name = "encoder_output_PY.txt"
decoder_output_file_name = "decoder_output_PY.txt"
enc_node_name = "pooling_node"
input_node_name = "input_node"
output_node_name = "output_node"
# define location of output, model and data and check existence
output_path = os.path.join(abs_path, "Output")
model_file = os.path.join(model_path, model_file_name)
data_file = os.path.join(data_path, "Test-28x28_cntk_text.txt")
if not (os.path.exists(model_file) and os.path.exists(data_file)):
print(
"Cannot find required data or model. "
"Please get the MNIST data set and run 'cntk configFile=07_Deconvolution_BS.cntk' or 'python 07_Deconvolution_PY.py' to create the model."
)
exit(0)
# create minibatch source
minibatch_source = MinibatchSource(CTFDeserializer(
data_file,
StreamDefs(features=StreamDef(field='features', shape=(28 * 28)),
labels=StreamDef(field='labels', shape=10))),
randomize=False,
max_sweeps=1)
# use this to print all node names in the model
# print_all_node_names(model_file, use_brain_script_model)
# load model and pick desired nodes as output
loaded_model = load_model(model_file)
output_nodes = combine([
loaded_model.find_by_name(input_node_name).owner,
loaded_model.find_by_name(enc_node_name).owner,
loaded_model.find_by_name(output_node_name).owner
])
# evaluate model save output
features_si = minibatch_source['features']
with open(os.path.join(output_path, decoder_output_file_name),
'wb') as decoder_text_file:
with open(os.path.join(output_path, encoder_output_file_name),
'wb') as encoder_text_file:
for i in range(0, num_objects_to_eval):
mb = minibatch_source.next_minibatch(1)
raw_dict = output_nodes.eval(mb[features_si])
output_dict = {}
for key in raw_dict.keys():
output_dict[key.name] = raw_dict[key]
encoder_input = output_dict[input_node_name]
encoder_output = output_dict[enc_node_name]
decoder_output = output_dict[output_node_name]
in_values = (encoder_input[0, 0].flatten())[np.newaxis]
enc_values = (encoder_output[0, 0].flatten())[np.newaxis]
out_values = (decoder_output[0, 0].flatten())[np.newaxis]
if not testing:
# write results as text and png
np.savetxt(decoder_text_file, out_values, fmt="%.6f")
np.savetxt(encoder_text_file, enc_values, fmt="%.6f")
save_as_png(
in_values,
os.path.join(output_path,
"imageAutoEncoder_%s__input.png" % i))
save_as_png(
out_values,
os.path.join(output_path,
"imageAutoEncoder_%s_output.png" % i))
# visualizing the encoding is only possible and meaningful with a single conv filter
enc_dim = 7
if (enc_values.size == enc_dim * enc_dim):
save_as_png(
enc_values,
os.path.join(
output_path,
"imageAutoEncoder_%s_encoding.png" % i),
dim=enc_dim)
print("Done. Wrote output to %s" % output_path)
def train_sequence_classifier(device):
input_dim = 2000
cell_dim = 25
hidden_dim = 25
embedding_dim = 50
num_output_classes = 5
features = variable(shape=input_dim, is_sparse=True, name="features")
classifier_output = LSTM_sequence_classifer_net(features,
num_output_classes,
embedding_dim, hidden_dim,
cell_dim, device)
label = variable(num_output_classes,
dynamic_axes=[Axis.default_batch_axis()],
name="labels")
ce = cross_entropy_with_softmax(classifier_output, label)
pe = classification_error(classifier_output, label)
#TODO: add save and load module code
lstm_net = combine([ce, pe, classifier_output], "classifier_model")
rel_path = r"../../../../Tests/EndToEndTests/Text/SequenceClassification/Data/Train.ctf"
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_path)
cm = create_text_mb_source(path, input_dim, num_output_classes, 0, True,
False, "x", "y")
stream_infos = cm.stream_infos()
for si in stream_infos:
if si.m_name == 'features':
features_si = si
elif si.m_name == 'labels':
labels_si = si
minibatch_size = 200
lr = lr = learning_rates_per_sample(0.0005)
trainer = Trainer(classifier_output, ce, pe,
[sgdlearner(classifier_output.owner.parameters(), lr)])
freq = 1
i = 0
cntk_dev = cntk_device(device)
while True:
mb = cm.get_next_minibatch(minibatch_size, cntk_dev)
if len(mb) == 0:
break
arguments = dict()
arguments[features] = mb[features_si].m_data
arguments[label] = mb[labels_si].m_data
trainer.train_minibatch(arguments, cntk_dev)
if i % freq == 0:
training_loss = get_train_loss(trainer)
eval_crit = get_train_eval_criterion(trainer)
print(
"Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}"
.format(i, training_loss, eval_crit))
i += 1
def generate_visualization(use_brain_script_model, testing=False):
num_objects_to_eval = 5
if (use_brain_script_model):
model_file_name = "07_Deconvolution_BS.model"
encoder_output_file_name = "encoder_output_BS.txt"
decoder_output_file_name = "decoder_output_BS.txt"
enc_node_name = "z.pool1"
input_node_name = "f2"
output_node_name = "z"
else:
model_file_name = "07_Deconvolution_PY.model"
encoder_output_file_name = "encoder_output_PY.txt"
decoder_output_file_name = "decoder_output_PY.txt"
enc_node_name = "pooling_node"
input_node_name = "input_node"
output_node_name = "output_node"
# define location of output, model and data and check existence
output_path = os.path.join(abs_path, "Output")
model_file = os.path.join(model_path, model_file_name)
data_file = os.path.join(data_path, "Test-28x28_cntk_text.txt")
if not (os.path.exists(model_file) and os.path.exists(data_file)):
print("Cannot find required data or model. "
"Please get the MNIST data set and run 'cntk configFile=07_Deconvolution_BS.cntk' or 'python 07_Deconvolution_PY.py' to create the model.")
exit(0)
# create minibatch source
minibatch_source = MinibatchSource(CTFDeserializer(data_file, StreamDefs(
features = StreamDef(field='features', shape=(28*28)),
labels = StreamDef(field='labels', shape=10)
)), randomize=False, max_sweeps = 1)
# use this to print all node names in the model
# print_all_node_names(model_file, use_brain_script_model)
# load model and pick desired nodes as output
loaded_model = load_model(model_file)
output_nodes = combine(
[loaded_model.find_by_name(input_node_name).owner,
loaded_model.find_by_name(enc_node_name).owner,
loaded_model.find_by_name(output_node_name).owner])
# evaluate model save output
features_si = minibatch_source['features']
with open(os.path.join(output_path, decoder_output_file_name), 'wb') as decoder_text_file:
with open(os.path.join(output_path, encoder_output_file_name), 'wb') as encoder_text_file:
for i in range(0, num_objects_to_eval):
mb = minibatch_source.next_minibatch(1)
raw_dict = output_nodes.eval(mb[features_si])
output_dict = {}
for key in raw_dict.keys(): output_dict[key.name] = raw_dict[key]
encoder_input = output_dict[input_node_name]
encoder_output = output_dict[enc_node_name]
decoder_output = output_dict[output_node_name]
in_values = (encoder_input[0,0].flatten())[np.newaxis]
enc_values = (encoder_output[0,0].flatten())[np.newaxis]
out_values = (decoder_output[0,0].flatten())[np.newaxis]
if not testing:
# write results as text and png
np.savetxt(decoder_text_file, out_values, fmt="%.6f")
np.savetxt(encoder_text_file, enc_values, fmt="%.6f")
save_as_png(in_values, os.path.join(output_path, "imageAutoEncoder_%s__input.png" % i))
save_as_png(out_values, os.path.join(output_path, "imageAutoEncoder_%s_output.png" % i))
# visualizing the encoding is only possible and meaningful with a single conv filter
enc_dim = 7
if(enc_values.size == enc_dim*enc_dim):
save_as_png(enc_values, os.path.join(output_path, "imageAutoEncoder_%s_encoding.png" % i), dim=enc_dim)
print("Done. Wrote output to %s" % output_path)
num_hidden_layers = 2
hidden_layers_dim = 50
minibatch_size = 25
num_samples_per_sweep = 10000
num_sweeps_to_train_with = 2
num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size
lr = 0.02
input = variable((input_dim,), np.float32, needs_gradient=False, name="features")
label = variable((num_output_classes,), np.float32, needs_gradient=False, name="labels")
dev = cntk_py.DeviceDescriptor.cpudevice()
netout = fully_connected_classifier_net(input, num_output_classes, hidden_layers_dim, num_hidden_layers, dev, sigmoid)
ce = cross_entropy_with_softmax(netout.output(), label)
pe = classification_error(netout.output(), label)
ffnet = combine([ce, pe, netout], "classifier_model")
cm = create_mb_source()
streamInfos = cm.stream_infos();
minibatchSizeLimits = dict()
minibatchSizeLimits[streamInfos[0]] = (0,minibatch_size)
minibatchSizeLimits[streamInfos[1]] = (0,minibatch_size)
trainer = cntk_py.Trainer(ffnet, ce.output(), [cntk_py.sgdlearner(ffnet.parameters(), lr)])
for i in range(0,int(num_minibatches_to_train)):
mb=cm.get_next_minibatch(minibatchSizeLimits, dev)
"Please get the MNIST data set and run 'cntk configFile=07_Deconvolution_BS.cntk' or 'python 07_Deconvolution_PY.py' to create the model.")
exit(0)
# create minibatch source
minibatch_source = MinibatchSource(CTFDeserializer(data_file, StreamDefs(
features = StreamDef(field='features', shape=(28*28)),
labels = StreamDef(field='labels', shape=10)
)), randomize=False, max_sweeps = 1)
# use this to print all node names in the model
# print_all_node_names(model_file, use_brain_script_model)
# load model and pick desired nodes as output
loaded_model = load_model(model_file)
output_nodes = combine(
[loaded_model.find_by_name(input_node_name).owner,
loaded_model.find_by_name(enc_node_name).owner,
loaded_model.find_by_name(output_node_name).owner])
# evaluate model save output
features_si = minibatch_source['features']
with open(os.path.join(output_path, decoder_output_file_name), 'wb') as decoder_text_file:
with open(os.path.join(output_path, encoder_output_file_name), 'wb') as encoder_text_file:
for i in range(0, num_objects_to_eval):
mb = minibatch_source.next_minibatch(1)
raw_dict = output_nodes.eval(mb[features_si])
output_dict = {}
for key in raw_dict.keys(): output_dict[key.name] = raw_dict[key]
encoder_input = output_dict[input_node_name]
encoder_output = output_dict[enc_node_name]
decoder_output = output_dict[output_node_name]
# create minibatch source
minibatch_source = MinibatchSource(CTFDeserializer(
data_file,
StreamDefs(features=StreamDef(field='features', shape=(28 * 28)),
labels=StreamDef(field='labels', shape=10))),
randomize=False,
epoch_size=FULL_DATA_SWEEP)
# use this to print all node names in the model
# print_all_node_names(model_file)
# load model and pick desired nodes as output
loaded_model = load_model(model_file)
output_nodes = combine([
loaded_model.find_by_name('f1').owner,
loaded_model.find_by_name('z.p1').owner,
loaded_model.find_by_name('z').owner
])
# evaluate model save output
features_si = minibatch_source['features']
with open(os.path.join(output_path, "decoder_output_py.txt"),
'wb') as decoder_text_file:
with open(os.path.join(output_path, "encoder_output_py.txt"),
'wb') as encoder_text_file:
for i in range(0, num_objects_to_eval):
mb = minibatch_source.next_minibatch(1)
raw_dict = output_nodes.eval(mb[features_si])
output_dict = {}
for key in raw_dict.keys():
output_dict[key.name] = raw_dict[key]