def load_models(self):
feature_model = C.load_model('dnns/feature_predicter_GRP.dnn')(self._input, self._target)
feature_model = feature_model.clone(C.CloneMethod.freeze)
print(feature_model)
inputs = C.ops.splice(self._input,feature_model,axis=0)
print(inputs)
action_model = C.load_model('dnns/GRP_f.dnn')(inputs,self._target)
action_model = action_model.clone(C.CloneMethod.freeze)
print(action_model)
node_outputs = C.logging.get_node_outputs(action_model)
for out in node_outputs: print("{0} {1}".format(out.name, out.shape))
print('model arguments',action_model.arguments)
return action_model
def main():
# generate data from physical model
T = 16 # time interval
dt = 0.1 # integration step
# train data (oscillation with initial angle = pi/6):
t, train = phys.simulate_pendulum(0, np.array([np.pi / 6, 0]), T, dt=dt)
# test data (oscillation with initial angle = pi/4):
_, test1 = phys.simulate_pendulum(0, np.array([np.pi / 4, 0]), T, dt=dt)
# test data (fixed point with angle = 0):
_, test2 = phys.simulate_pendulum(0, np.array([0.0, 0.0]), T, dt=dt)
N = 5 # size of the history window (for LSTM)
# train lstm model
# model_lstm = models.train_lstm(train, N)
# model_lstm.save('bin_models/pendulum_lstm')
# or load pre-built one
model_lstm = load_model('bin_models/pendulum_lstm')
# train linear model
# model_linear = models.train_linear(train, N)
# model_linear.save('bin_models/pendulum_linear')
# or load pre-built one
model_linear = load_model('bin_models/pendulum_linear')
for model, n, title in zip([model_lstm, model_linear], [N, 1], [
"LSTM: non-physical predictions, training data are memorized",
"Linear map: physical behaviour generalization"
]):
# use model for prediction with training initial angle = pi/6
pred = models.predict(model,
train[:n],
step_count=int(T / dt - n),
N=n)
# use model for prediction with initial angle = pi/4
pred1 = models.predict(model,
test1[:n],
step_count=int(T / dt - n),
N=n)
# use lstm for prediction with initial angle = 0
pred2 = models.predict(model,
test2[:n],
step_count=int(T / dt - n),
N=n)
plot(title, train, test1, test2, pred, pred1, pred2)
plt.show()
return 0
def create_model(base_model_file,
input_features,
num_classes,
dropout_rate=0.5,
freeze_weights=False):
# Load the pretrained classification net and find nodes
base_model = load_model(base_model_file)
feature_node = find_by_name(base_model, 'features')
beforePooling_node = find_by_name(base_model, "z.x.x.r")
#graph.plot(base_model, filename="base_model.pdf") # Write graph visualization
# Clone model until right before the pooling layer, ie. until including z.x.x.r
modelCloned = combine([beforePooling_node.owner]).clone(
CloneMethod.freeze if freeze_weights else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Center the input around zero and set model input.
# Do this early, to avoid CNTK bug with wrongly estimated layer shapes
feat_norm = input_features - constant(114)
model = modelCloned(feat_norm)
# Pool over all spatial dimensions and add dropout layer
avgPool = GlobalAveragePooling(name="poolingLayer")(model)
if dropout_rate > 0:
avgPoolDrop = Dropout(dropout_rate)(avgPool)
else:
avgPoolDrop = avgPool
# Add new dense layer for class prediction
finalModel = Dense(num_classes, activation=None,
name="prediction")(avgPoolDrop)
return finalModel
def init():
# Get the path to the model asset
# local_path = get_local_path('mymodel.model.link')
# Load model using appropriate library and function
# global model
# model = model_load_function(local_path)
# model = 42
try:
print("Executing init() method...")
print("Python version: " + str(sys.version) + ", CNTK version: " +
C.__version__)
print("Start downloading model...")
model_path = download_model()
base_model = load_model(model_path)
data = C.input_variable(shape=(3, C.FreeDimension, C.FreeDimension),
name="data")
predictor = clone_model(base_model, ['data'],
["Mconv7_stage6_L1", "Mconv7_stage6_L2"],
CloneMethod.freeze)
global pred_net
pred_net = predictor(data)
except Exception as e:
print("Exception in init:")
print(str(e))
def predictor_from_cntk_model(modelFile):
"""Loads a CNTK model and returns an ELL.NeuralNetworkPredictor"""
print("Loading...")
z = load_model(modelFile)
print("\nFinished loading.")
print("Pre-processing...")
modelLayers = get_model_layers(z)
# Get the relevant CNTK layers that we will convert to ELL
layersToConvert = get_filtered_layers_list(modelLayers)
print("\nFinished pre-processing.")
predictor = None
try:
# Create a list of ELL layers from the CNTK layers
ellLayers = convert_cntk_layers_to_ell_layers(layersToConvert)
# Create an ELL neural network predictor from the layers
predictor = ELL.FloatNeuralNetworkPredictor(ellLayers)
except:
print("Error occurrred attempting to convert cntk layers to ELL layers")
traceback.print_exc()
return predictor
def create_model(self, input_dim, output_dim, hidden_dim, feature_input):
"""
Create a model with the layers library.
"""
model_file = "nh_autoencoder.model"
if os.path.exists(model_file):
print("Resuming training")
netout = C.load_model(model_file)
return netout(feature_input)
netout = C.layers.Sequential([
C.layers.Embedding(300, name="Embedding"),
C.layers.Dense(300, activation=C.ops.sigmoid, name="Hidden"),
C.layers.Dense(output_dim, activation=C.ops.sigmoid, name="Out")
])(feature_input)
# c1 = C.layers.Convolution2D((3,3), cmap, strides=2, reduction_rank=0)(feature_input)
# p1 = C.layers.MaxPooling( (3,3), (2,2))(c1)
# d1 = C.layers.Dense((20,5,19))(p1)
# u1 = C.layers.MaxUnpooling((3,3), (2,2))(p1, c1)
# #C.layers.Dense(output_dim)
# d1 = C.layers.ConvolutionTranspose2D((3,3), num_channels, bias=False, init=C.glorot_uniform(0.001))(u1)
# netout = C.layers.Dense(output_dim)(d1)
return (netout)
def test(test_data, model_path, model_file, config_file):
polymath = PolyMath(config_file)
model = C.load_model(os.path.join(model_path, model_file if model_file else model_name))
begin_logits = model.outputs[0]
end_logits = model.outputs[1]
loss = C.as_composite(model.outputs[2].owner)
begin_prediction = C.sequence.input_variable(1, sequence_axis=begin_logits.dynamic_axes[1], needs_gradient=True)
end_prediction = C.sequence.input_variable(1, sequence_axis=end_logits.dynamic_axes[1], needs_gradient=True)
best_span_score = symbolic_best_span(begin_prediction, end_prediction)
predicted_span = C.layers.Recurrence(C.plus)(begin_prediction - C.sequence.past_value(end_prediction))
batch_size = 32 # in sequences
misc = {'rawctx':[], 'ctoken':[], 'answer':[], 'uid':[]}
tsv_reader = create_tsv_reader(loss, test_data, polymath, batch_size, 1, is_test=True, misc=misc)
results = {}
with open('{}_out.json'.format(model_file), 'w', encoding='utf-8') as json_output:
for data in tsv_reader:
out = model.eval(data, outputs=[begin_logits,end_logits,loss], as_numpy=False)
g = best_span_score.grad({begin_prediction:out[begin_logits], end_prediction:out[end_logits]}, wrt=[begin_prediction,end_prediction], as_numpy=False)
other_input_map = {begin_prediction: g[begin_prediction], end_prediction: g[end_prediction]}
span = predicted_span.eval((other_input_map))
for seq, (raw_text, ctokens, answer, uid) in enumerate(zip(misc['rawctx'], misc['ctoken'], misc['answer'], misc['uid'])):
seq_where = np.argwhere(span[seq])[:,0]
span_begin = np.min(seq_where)
span_end = np.max(seq_where)
predict_answer = get_answer(raw_text, ctokens, span_begin, span_end)
results['query_id'] = int(uid)
results['answers'] = [predict_answer]
json.dump(results, json_output)
json_output.write("\n")
misc['rawctx'] = []
misc['ctoken'] = []
misc['answer'] = []
misc['uid'] = []
def run_cntk():
text, chars, char_indices, x_train, y_train = get_data(one_hot_encode_features=False)
alphabet_size = len(chars)
print('alphabet_size=', alphabet_size)
model = build_model_cntk(alphabet_size=alphabet_size)
model_filename = 'ch8-1_cntk.model'
model.save(model_filename)
model = None
model = cntk.load_model(model_filename)
x = cntk.sequence.input_variable(shape=(), dtype=np.float32)
y = cntk.input_variable(shape=(), dtype=np.float32)
model.replace_placeholders({model.placeholders[0]: x})
y_oneHot = cntk.one_hot(y, num_classes=alphabet_size)
loss_function = cntk.cross_entropy_with_softmax(model.output, y_oneHot)
learner = cntk.adam(model.parameters, cntk.learning_parameter_schedule_per_sample(0.001), cntk.learning_parameter_schedule_per_sample(0.9))
trainer = cntk.Trainer(model, (loss_function, loss_function), [learner],)
for epoch in range(1, 60):
print('epoch', epoch)
cntk_train(x, y, x_train, y_train, max_epochs=32, batch_size=128, trainer=trainer)
model_filename = 'final_ch8-1_cntk.model'
model.save(model_filename)
generate_text_cntk(char_indices, chars, model, text)
def benchmark_cntk(data_path, model_path):
import cntk as C
model = C.load_model(model_path)
vocab_file_path = os.path.join(data_path, "vocabulary.txt")
label_file_path = os.path.join(data_path, "labels.txt")
dev_file_path = os.path.join(data_path, "dev.ctf")
minibatch_size = 1024
x_dim = get_size(vocab_file_path)
y_dim = get_size(label_file_path)
x = C.sequence.input_variable(x_dim, is_sparse=True)
model = model(x)
y = C.input_variable(y_dim, is_sparse=True)
streamDefs = C.io.StreamDefs(
sentence=C.io.StreamDef(
field='S0', shape=x_dim, is_sparse=True),
label=C.io.StreamDef(
field='S1', shape=y_dim, is_sparse=True)
)
dev_reader = C.io.MinibatchSource(
C.io.CTFDeserializer(dev_file_path,
streamDefs),
randomize=True, max_sweeps=1)
error = C.classification_error(model, y)
progress_printer = C.logging.ProgressPrinter(
freq=100, tag='evaluate', test_freq=1000
)
evaluator = C.eval.Evaluator(error, [progress_printer])
input_map = {x: dev_reader.streams.sentence, y: dev_reader.streams.label}
data = dev_reader.next_minibatch(minibatch_size, input_map=input_map)
while data:
evaluator.test_minibatch(data)
data = dev_reader.next_minibatch(minibatch_size, input_map=input_map)
evaluator.summarize_test_progress()
def main():
with open('cartpole.json', encoding='utf-8') as config_file:
config = json.load(config_file)
if not os.path.exists('cartpole.cdqn'):
print(
'Run `python -m experiments.train_cartpole` to train and save a model for CartPole-v0'
)
return
env = gym.make('CartPole-v0')
model = C.load_model('cartpole.cdqn')
z = np.linspace(config['vmin'],
config['vmax'],
config['n'],
dtype=np.float32)
rewards = 0.0
s = env.reset()
done = False
while not done:
env.render()
a = get_action(model, s, z)
s, r, done, info = env.step(a)
rewards += r
print('Total reward: {}'.format(rewards))
def create_model(model_details,
num_classes,
input_features,
new_prediction_node_name='prediction',
freeze=False):
# Load the pretrained classification net and find nodes
base_model = C.load_model(model_details['model_file'])
feature_node = C.logging.find_by_name(base_model,
model_details['feature_node_name'])
last_node = C.logging.find_by_name(base_model,
model_details['last_hidden_node_name'])
# Clone the desired layers with fixed weights
cloned_layers = C.combine([last_node.owner]).clone(
C.CloneMethod.freeze if freeze else C.CloneMethod.clone,
{feature_node: C.placeholder(name='features')})
# Add new dense layer for class prediction
feat_norm = input_features - C.Constant(114)
cloned_out = cloned_layers(feat_norm)
z = C.layers.Dense(num_classes,
activation=None,
name=new_prediction_node_name)(cloned_out)
return z
def test(test_data, model_path, model_file, config_file):
polymath = PolyMath(config_file)
model = C.load_model(os.path.join(model_path, model_file if model_file else model_name))
begin_logits = model.outputs[0]
end_logits = model.outputs[1]
loss = C.as_composite(model.outputs[2].owner)
begin_prediction = C.sequence.input_variable(1, sequence_axis=begin_logits.dynamic_axes[1], needs_gradient=True)
end_prediction = C.sequence.input_variable(1, sequence_axis=end_logits.dynamic_axes[1], needs_gradient=True)
best_span_score = symbolic_best_span(begin_prediction, end_prediction)
predicted_span = C.layers.Recurrence(C.plus)(begin_prediction - C.sequence.past_value(end_prediction))
batch_size = 32 # in sequences
misc = {'rawctx':[], 'ctoken':[], 'answer':[], 'uid':[]}
tsv_reader = create_tsv_reader(loss, test_data, polymath, batch_size, 1, is_test=True, misc=misc)
results = {}
with open('{}_out.json'.format(model_file), 'w', encoding='utf-8') as json_output:
for data in tsv_reader:
out = model.eval(data, outputs=[begin_logits,end_logits,loss], as_numpy=False)
g = best_span_score.grad({begin_prediction:out[begin_logits], end_prediction:out[end_logits]}, wrt=[begin_prediction,end_prediction], as_numpy=False)
other_input_map = {begin_prediction: g[begin_prediction], end_prediction: g[end_prediction]}
span = predicted_span.eval((other_input_map))
for seq, (raw_text, ctokens, answer, uid) in enumerate(zip(misc['rawctx'], misc['ctoken'], misc['answer'], misc['uid'])):
seq_where = np.argwhere(span[seq])[:,0]
span_begin = np.min(seq_where)
span_end = np.max(seq_where)
predict_answer = get_answer(raw_text, ctokens, span_begin, span_end)
results['query_id'] = int(uid)
results['answers'] = [predict_answer]
json.dump(results, json_output)
json_output.write("\n")
misc['rawctx'] = []
misc['ctoken'] = []
misc['answer'] = []
misc['uid'] = []
def main(input_map_file, label_file, output_file, trained_model_file):
model = cntk.load_model(trained_model_file)
# Get the class names
idx_to_label_dict = {}
with open(label_file, 'r') as f:
for line in f:
label, idx = line.strip().split('\t')
idx_to_label_dict[idx] = label
n_classes = len(list(idx_to_label_dict.keys()))
# Process each image and store true vs. predicted labels
df = []
with open(input_map_file, 'r') as f:
for line in f:
filename, label_idx = line.strip().split('\t')
image_data = load_image(filename)
raw_pred_result = model.eval({model.arguments[0]: image_data})
pred_label_idx = np.argmax(np.squeeze(raw_pred_result))
df.append([
filename, idx_to_label_dict[str(label_idx)],
idx_to_label_dict[str(pred_label_idx)]
])
df = pd.DataFrame(df, columns=['filename', 'true_label', 'pred_label'])
accuracy = len(df.loc[df['true_label'] == df['pred_label']].index) / \
len(df.index)
print('Overall accuracy on test set: {:0.3f}'.format(accuracy))
for label in idx_to_label_dict.values():
print_class_specific_statistics(df, label)
return
def frcn_predictor(features, rois, n_classes):
# Load the pretrained classification net and find nodes
loaded_model = load_model(model_file)
feature_node = find_by_name(loaded_model, feature_node_name)
conv_node = find_by_name(loaded_model, last_conv_node_name)
pool_node = find_by_name(loaded_model, pool_node_name)
last_node = find_by_name(loaded_model, last_hidden_node_name)
# Clone the conv layers and the fully connected layers of the network
conv_layers = combine([conv_node.owner
]).clone(CloneMethod.freeze,
{feature_node: Placeholder()})
fc_layers = combine([last_node.owner]).clone(CloneMethod.clone,
{pool_node: Placeholder()})
# Create the Fast R-CNN model
feat_norm = features - Constant(114)
conv_out = conv_layers(feat_norm)
roi_out = roipooling(conv_out, rois, (roi_dim, roi_dim))
fc_out = fc_layers(roi_out)
# z = Dense(rois[0], num_classes, map_rank=1)(fc_out) # --> map_rank=1 is not yet supported
W = parameter(shape=(4096, n_classes), init=glorot_uniform())
b = parameter(shape=n_classes, init=0)
z = times(fc_out, W) + b
return z
def stacking_recurrent_layers_cntk(float_data, epochs=20):
train_gen, val_gen, test_gen, val_steps, test_steps, lookback, step = create_generators(
float_data)
input_shape = (float_data.shape[-1], )
model = create_model_cntk(model_type=2,
input_shape=input_shape,
dropout=0.2,
recurrent_dropout=0.5)
model_filename = 'ch6-3_model_type_2.model'
model.save(model_filename)
print('Saved ', model_filename)
model = None
model = cntk.load_model(model_filename)
print('Loaded ', model_filename)
x = cntk.input_variable(shape=input_shape,
dtype=np.float32,
dynamic_axes=[
cntk.Axis.default_batch_axis(),
cntk.Axis.default_dynamic_axis()
])
model.replace_placeholders({model.placeholders[0]: x})
y = cntk.input_variable(shape=(), dtype=np.float32)
train_mse_cntk(x, y, model, train_gen, val_gen, epochs, val_steps)
def evaluate_model(map_filename, output_dir, num_classes):
''' Evaluate the model on the test set, storing predictions to a file '''
inds_to_labels = {}
with open(os.path.join(output_dir, 'labels_to_inds.tsv'), 'r') as f:
for line in f:
label, ind = line.strip().split('\t')
inds_to_labels[int(ind)] = label
loaded_model = cntk.load_model(os.path.join(output_dir, 'retrained.model'))
with open(map_filename, 'r') as f:
with open(os.path.join(output_dir, 'predictions.csv'), 'w') as g:
g.write('filename,label,pred_label\n')
for line in f:
filename, true_ind = line.strip().split('\t')
image_data = np.array(Image.open(filename), dtype=np.float32)
image_data = np.ascontiguousarray(
np.transpose(image_data[:, :, ::-1], (2, 0, 1)))
dnn_output = loaded_model.eval(
{loaded_model.arguments[0]: [image_data]})
true_label = inds_to_labels[int(true_ind)]
pred_label = inds_to_labels[np.argmax(np.squeeze(dnn_output))]
g.write('{},{},{}\n'.format(filename, true_label, pred_label))
df = pd.read_csv(os.path.join(output_dir, 'predictions.csv'))
num_correct = len(df.loc[df['true_label'] == df['pred_label']].index)
print('Overall accuracy on test set: {:0.3f}'.format(
len(df.loc[df['true_label'] == df['pred_label']].index) /
len(df.index)))
return
def load_alexnet_model(image_input, num_classes, model_filename,
retraining_type):
''' Load pretrained AlexNet for desired level of retraining '''
loaded_model = cntk.load_model(model_filename)
# Load the convolutional layers, freezing if desired
feature_node = cntk.logging.graph.find_by_name(loaded_model, 'features')
last_conv_node = cntk.logging.graph.find_by_name(loaded_model, 'conv5.y')
conv_layers = cntk.ops.combine([last_conv_node.owner]).clone(
cntk.ops.functions.CloneMethod.clone if retraining_type == 'all' \
else cntk.ops.functions.CloneMethod.freeze,
{feature_node: cntk.ops.placeholder()})
# Load the fully connected layers, freezing if desired
last_node = cntk.logging.graph.find_by_name(loaded_model, 'h2_d')
fully_connected_layers = cntk.ops.combine([last_node.owner]).clone(
cntk.ops.functions.CloneMethod.freeze if retraining_type == \
'last_only' else cntk.ops.functions.CloneMethod.clone,
{last_conv_node: cntk.ops.placeholder()})
# Define the network using the loaded layers
feat_norm = image_input - cntk.layers.Constant(114)
conv_out = conv_layers(feat_norm)
fc_out = fully_connected_layers(conv_out)
new_model = cntk.layers.Dense(shape=num_classes, name='lastlayer')(fc_out)
return (new_model)
def predictor_from_cntk_model(modelFile, plotModel=False):
"""Loads a CNTK model and returns an ELL.NeuralNetworkPredictor"""
print("Loading...")
z = load_model(modelFile)
print("\nFinished loading.")
if plotModel:
filename = os.path.join(os.path.dirname(modelFile),
os.path.basename(modelFile) + ".png")
cntk_utilities.plot_model(z, filename)
print("Pre-processing...")
modelLayers = cntk_utilities.get_model_layers(z)
# Get the relevant CNTK layers that we will convert to ELL
layersToConvert = cntk_layers.get_filtered_layers_list(modelLayers)
print("\nFinished pre-processing.")
predictor = None
try:
# Create a list of ELL layers from the CNTK layers
ellLayers = cntk_layers.convert_cntk_layers_to_ell_layers(
layersToConvert)
# Create an ELL neural network predictor from the layers
predictor = ELL.FloatNeuralNetworkPredictor(ellLayers)
except BaseException as exception:
print("Error occurred attempting to convert cntk layers to ELL layers")
raise exception
return predictor
def predictor_from_cntk_model_using_new_engine(modelFile, plotModel=True):
"""
Loads a CNTK model and returns an ell.neural.NeuralNetworkPredictor
"""
_logger = logger.get()
_logger.info("Loading...")
z = load_model(modelFile)
_logger.info("\nFinished loading.")
if plotModel:
filename = os.path.join(os.path.dirname(modelFile), os.path.basename(modelFile) + ".svg")
cntk_utilities.plot_model(z, filename)
try:
_logger.info("Pre-processing...")
# Get the relevant nodes from CNTK that make up the model
importer_nodes = cntk_utilities.Utilities.get_model_nodes(z)
_logger.info("\nFinished pre-processing.")
# Create an ImporterModel from the CNTK nodes
importer_model = import_nodes(importer_nodes)
# Use the common importer engine to drive conversion of the
# ImporterModel to ELL layers
importer_engine = common.importer.ImporterEngine()
ell_layers = importer_engine.convert(importer_model)
# Create an ELL neural network predictor from the layers
predictor = ell.neural.NeuralNetworkPredictor(ell_layers)
except BaseException as exception:
_logger.error("Error occurred attempting to convert cntk layers to ELL layers: " + str(exception))
raise exception
return predictor
def load_model(model_filename: str):
"""A helper function to load the acoustic model from disc.
Args:
model_filename (str): The file path to the acoustic model.
"""
cntk_model = cntk.load_model(model_filename)
print(cntk_model)
# First try and find output by name
model_output = cntk_model.find_by_name('ScaledLogLikelihood')
print(model_output)
# Fall back to first defined output
if model_output is None:
model_output = cntk_model.outputs[0]
print(model_output)
# Create an object restricted to the desired output.
cntk_model = cntk.combine(model_output)
print(cntk_model)
# Optimized RNN models won't run on CPU without conversion.
if 0 == cntk.use_default_device().type():
cntk_model = cntk.misc.convert_optimized_rnnstack(cntk_model)
print("cntk model loaded")
print(cntk_model)
return cntk_model
def resnet_model(name, scaled_input):
print('Loading Resnet model from {}.'.format(name))
base_model = C.load_model(os.path.join(model_path, name))
feature_node = C.logging.find_by_name(base_model, 'features')
last_node = C.logging.find_by_name(base_model, 'z.x')
# Clone the desired layers with fixed weights
cloned_layers = C.combine([last_node.owner]).clone(
C.CloneMethod.freeze, {feature_node: C.placeholder(name='features')})
cloned_out = cloned_layers(scaled_input)
z = C.layers.GlobalAveragePooling()(cloned_out)
z = C.layers.Dropout(dropout_rate=0.3, name='d1')(z)
z = C.layers.Dense(128, activation=C.ops.relu, name='fc1')(cloned_out)
z = C.layers.BatchNormalization(map_rank=1)(z)
z = C.layers.Dropout(dropout_rate=0.6, name='d2')(z)
z = C.layers.Dense(128, activation=C.ops.relu, name='fc2')(cloned_out)
z = C.layers.BatchNormalization(map_rank=1)(z)
z = C.layers.Dropout(dropout_rate=0.3, name='d2')(z)
z = C.layers.Dense(num_classes, activation=None, name='prediction')(z)
return z
def inference(model, test):
p = Model()
model = C.load_model(model)
cos = model.outputs[0]
loss = C.as_composite(model.outputs[1].owner)
mb_test, map_test = deserialize(loss,
test,
p,
randomize=False,
repeat=False,
is_test=True)
c1 = argument_by_name(loss, 'c1')
c2 = argument_by_name(loss, 'c2')
results = []
if 'test' in test:
total_samples = 3000
else:
total_samples = num_validation
with tqdm(total=total_samples) as progress_bar:
while True:
data = mb_test.next_minibatch(minibatch_size, input_map=map_test)
progress_bar.update(len(data))
if not data:
break
out = model.eval(data, outputs=[cos])
results.extend(out)
assert (len(results) == total_samples)
return results
def create_transfer_learning_model(input, num_classes, model_file, freeze=False):
base_model = load_model(model_file)
base_model = C.as_composite(base_model[3].owner)
# Load the pretrained classification net and find nodes
feature_node = C.logging.find_by_name(base_model, feature_node_name)
last_node = C.logging.find_by_name(base_model, last_hidden_node_name)
base_model = C.combine([last_node.owner]).clone(C.CloneMethod.freeze if freeze else C.CloneMethod.clone, {feature_node: C.placeholder(name='features')})
base_model = base_model(C.input_variable((num_channels, image_height, image_width)))
r1 = C.logging.find_by_name(base_model, "z.x.x.r")
r2_2 = C.logging.find_by_name(base_model, "z.x.x.x.x.r")
r3_2 = C.logging.find_by_name(base_model, "z.x.x.x.x.x.x.r")
r4_2 = C.logging.find_by_name(base_model, "z.x.x.x.x.x.x.x.x.r")
up_r1 = OneByOneConvAndUpSample(r1, 3, num_classes)
up_r2_2 = OneByOneConvAndUpSample(r2_2, 2, num_classes)
up_r3_2 = OneByOneConvAndUpSample(r3_2, 1, num_classes)
up_r4_2 = OneByOneConvAndUpSample(r4_2, 0, num_classes)
merged = C.splice(up_r1, up_r3_2, up_r2_2, axis=0)
resnet_fcn_out = Convolution((1, 1), num_classes, init=he_normal(), activation=sigmoid, pad=True)(merged)
z = UpSampling2DPower(resnet_fcn_out,2)
return z
def create_model(base_model_file, feature_node_name, last_hidden_node_name, num_classes, input_features, freeze=False):
# Load the pretrained classification net and find nodes
base_model = load_model(base_model_file)
feature_node = None
last_node = None
# feature node
for n in cntk.logging.graph.depth_first_search(base_model, (lambda x: True)):
if (n.name.strip() == feature_node_name) or (n.uid.strip() == feature_node_name):
feature_node = n
print("feature node:", feature_node)
if feature_node is None:
raise Exception("Failed to locate feature node: " + feature_node_name)
# last hidden node
for n in cntk.logging.get_node_outputs(base_model):
if (n.name.strip() == last_hidden_node_name) or (n.uid.strip() == last_hidden_node_name):
last_node = n
print("last hidden node:", last_node)
if last_node is None:
raise Exception("Failed to locate last hidden node: " + last_hidden_node_name)
# Clone the desired layers with fixed weights
cloned_layers = combine([last_node.owner]).clone(
CloneMethod.freeze if freeze else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Add new dense layer for class prediction
feat_norm = input_features - Constant(114)
cloned_out = cloned_layers(feat_norm)
z = Dense(num_classes, activation=None, name=new_output_node_name) (cloned_out)
return z
def create_faster_rcnn_model(features, scaled_gt_boxes, dims_input, cfg):
# Load the pre-trained classification net and clone layers
base_model = load_model(cfg['BASE_MODEL_PATH'])
conv_layers = clone_conv_layers(base_model, cfg)
fc_layers = clone_model(base_model, [cfg["MODEL"].POOL_NODE_NAME],
[cfg["MODEL"].LAST_HIDDEN_NODE_NAME],
clone_method=CloneMethod.clone)
# Normalization and conv layers
feat_norm = features - Constant([[[v]]
for v in cfg["MODEL"].IMG_PAD_COLOR])
conv_out = conv_layers(feat_norm)
# RPN and prediction targets
rpn_rois, rpn_losses = create_rpn(conv_out, scaled_gt_boxes, dims_input,
cfg)
rois, label_targets, bbox_targets, bbox_inside_weights = \
create_proposal_target_layer(rpn_rois, scaled_gt_boxes, cfg)
# Fast RCNN and losses
cls_score, bbox_pred = create_fast_rcnn_predictor(conv_out, rois,
fc_layers, cfg)
detection_losses = create_detection_losses(cls_score, label_targets,
bbox_pred, rois, bbox_targets,
bbox_inside_weights, cfg)
loss = rpn_losses + detection_losses
pred_error = classification_error(cls_score, label_targets, axis=1)
return loss, pred_error
def train_faster_rcnn(cfg):
# Train only if no model exists yet
model_path = cfg['MODEL_PATH']
if os.path.exists(model_path) and cfg["CNTK"].MAKE_MODE:
print("Loading existing model from %s" % model_path)
eval_model = load_model(model_path)
else:
if cfg["CNTK"].TRAIN_E2E:
eval_model = train_faster_rcnn_e2e(cfg)
else:
eval_model = train_faster_rcnn_alternating(cfg)
eval_model.save(model_path)
if cfg["CNTK"].DEBUG_OUTPUT:
plot(
eval_model,
os.path.join(
cfg.OUTPUT_PATH, "graph_frcn_eval_{}_{}.{}".format(
cfg["MODEL"].BASE_MODEL,
"e2e" if cfg["CNTK"].TRAIN_E2E else "4stage",
cfg["CNTK"].GRAPH_TYPE)))
print("Stored eval model at %s" % model_path)
if cfg.DISTRIBUTED_FLG:
distributed.Communicator.finalize()
return eval_model
def train_faster_rcnn(cfg):
# Train only if no model exists yet
model_path = cfg['MODEL_PATH']
if os.path.exists(model_path) and cfg["CNTK"].MAKE_MODE:
print("Loading existing model from %s" % model_path)
eval_model = load_model(model_path)
else:
if cfg["CNTK"].TRAIN_E2E:
eval_model = train_faster_rcnn_e2e(cfg)
else:
eval_model = train_faster_rcnn_alternating(cfg)
eval_model.save(model_path)
print("Stored eval model at %s" % model_path)
if cfg["CNTK"].DEBUG_OUTPUT:
plot(
eval_model,
os.path.join(
cfg.OUTPUT_PATH, "graph_frcn_eval_{}_{}.{}".format(
cfg["MODEL"].BASE_MODEL,
"e2e" if cfg["CNTK"].TRAIN_E2E else "4stage",
cfg["CNTK"].GRAPH_TYPE)))
upload_checkpoint_file(os.path.join(
os.environ['AZUREML_NATIVE_SHARE_DIRECTORY'], 'output',
cfg['OUTPUT_MODEL_NAME']),
cfg['OUTPUT_MODEL_NAME'],
add_timestamp=True)
print("Model saved to Azure blob")
return eval_model
def create_faster_rcnn_predictor(base_model_file_name, features,
scaled_gt_boxes, dims_input):
# Load the pre-trained classification net and clone layers
base_model = load_model(base_model_file_name)
conv_layers = clone_conv_layers(base_model)
fc_layers = clone_model(base_model, [pool_node_name],
[last_hidden_node_name],
clone_method=CloneMethod.clone)
# Normalization and conv layers
feat_norm = features - normalization_const
conv_out = conv_layers(feat_norm)
# RPN and prediction targets
rpn_rois, rpn_losses = \
create_rpn(conv_out, scaled_gt_boxes, dims_input, proposal_layer_param_string=cfg["CNTK"].PROPOSAL_LAYER_PARAMS)
rois, label_targets, bbox_targets, bbox_inside_weights = \
create_proposal_target_layer(rpn_rois, scaled_gt_boxes, num_classes=globalvars['num_classes'])
# Fast RCNN and losses
cls_score, bbox_pred = create_fast_rcnn_predictor(conv_out, rois,
fc_layers)
detection_losses = create_detection_losses(cls_score, label_targets, rois,
bbox_pred, bbox_targets,
bbox_inside_weights)
loss = rpn_losses + detection_losses
pred_error = classification_error(cls_score, label_targets, axis=1)
return loss, pred_error
def predictor_from_cntk_model(modelFile):
"""Loads a CNTK model and returns an ELL.NeuralNetworkPredictor"""
print("Loading...")
z = load_model(modelFile)
print("\nFinished loading.")
print("Pre-processing...")
modelLayers = get_model_layers(z)
# Get the relevant CNTK layers that we will convert to ELL
layersToConvert = get_filtered_layers_list(modelLayers)
print("\nFinished pre-processing.")
predictor = None
try:
# Create a list of ELL layers from the CNTK layers
ellLayers = convert_cntk_layers_to_ell_layers(layersToConvert)
# Create an ELL neural network predictor from the layers
predictor = ELL.FloatNeuralNetworkPredictor(ellLayers)
except:
print(
"Error occurrred attempting to convert cntk layers to ELL layers")
traceback.print_exc()
return predictor
def __init__(self, para, creator, valid, mapping, valid_batch, valid_iter,
input_key):
self.para = para
network = creator(para)
temp_err = 0
for i in range(valid_iter):
data = valid.next_minibatch(valid_batch, input_map=mapping(valid))
temp_err += network.test_minibatch(data)
self.accuracy = 1 - temp_err / valid_iter
model_name = os.path.join('module', '_'.join(map(str, para)))
network.model.save(model_name)
cpu_timer = cntk.load_model(model_name, device=cntk.cpu())
time_cost = []
for i in range(valid_iter):
data = valid.next_minibatch(valid_batch, input_map=mapping(valid))
arr = numpy.array(data[input_key].as_sequences())
arr = numpy.reshape(arr, (-1, ) + input_key.shape)
current_time = time.clock()
cpu_timer.eval(arr, device=cntk.cpu())
current_time = time.clock() - current_time
time_cost.append(current_time)
self.time = numpy.mean(time_cost)
def test_sequence_to_sequence(device_id):
# import code after setting the device, otherwise some part of the code picks up "default device"
# which causes an inconsistency if there is already another job using GPU #0
from Sequence2Sequence import create_reader, DATA_DIR, MODEL_DIR, TRAINING_DATA, VALIDATION_DATA, TESTING_DATA, \
VOCAB_FILE, get_vocab, create_model, model_path_stem, train, evaluate_metric
from cntk.ops.tests.ops_test_utils import cntk_device
set_default_device(cntk_device(device_id))
# hook up data (train_reader gets False randomization to get consistent error)
train_reader = create_reader(os.path.join(DATA_DIR, TRAINING_DATA), False)
valid_reader = create_reader(os.path.join(DATA_DIR, VALIDATION_DATA), True)
test_reader = create_reader(os.path.join(DATA_DIR, TESTING_DATA), False)
vocab, i2w, _ = get_vocab(os.path.join(DATA_DIR, VOCAB_FILE))
# create model
model = create_model()
# train (with small numbers to finish within a reasonable amount of time)
train(train_reader, valid_reader, vocab, i2w, model, max_epochs=1, epoch_size=5000)
# now test the model and print out test error (for automated test)
model_filename = os.path.join(MODEL_DIR, model_path_stem + ".cmf.0")
model = load_model(model_filename)
error = evaluate_metric(test_reader, model, 10)
print(error)
#expected_error = 0.9943119920022192 # when run separately
expected_error = 0.9912881900980582 # when run inside the harness--random-initialization?
assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def create_model(model_details, num_classes, input_features, new_prediction_node_name="prediction", freeze=False):
# Load the pre-trained classification net and find nodes
base_model = cntk.load_model(model_details["model_file"])
feature_node = cntk.logging.find_by_name(base_model, model_details["feature_node_name"])
last_node = cntk.logging.find_by_name(base_model, model_details["last_hidden_node_name"])
if model_details["inception"]:
node_outputs = cntk.logging.get_node_outputs(base_model)
last_node = node_outputs[5]
feature_node = cntk.logging.find_all_with_name(base_model, "")[-5]
if model_details["vgg"]:
last_node = cntk.logging.find_by_name(base_model, "prob")
feature_node = cntk.logging.find_by_name(base_model, "data")
# Clone the desired layers with fixed weights
cloned_layers = cntk.combine([last_node.owner]).clone(
cntk.CloneMethod.freeze if freeze else cntk.CloneMethod.clone,
{feature_node: cntk.placeholder(name="features")},
)
# Add new dense layer for class prediction
feat_norm = input_features - cntk.Constant(114)
cloned_out = cloned_layers(feat_norm)
z = cntk.layers.Dense(num_classes, activation=None, name=new_prediction_node_name)(cloned_out)
return z
def print_all_node_names(model_file, is_BrainScript=True):
loaded_model = load_model(model_file)
if is_BrainScript:
loaded_model = combine([loaded_model.outputs[0]])
node_list = graph.depth_first_search(loaded_model, lambda x: x.is_output)
print("printing node information in the format")
print("node name (tensor shape)")
for node in node_list:
print(node.name, node.shape)
def test_custom_attributes(tmpdir):
root = 0 + C.input_variable(())
assert not root.custom_attributes.keys()
root.custom_attributes['cleared'] = 'none'
assert 'none' == root.custom_attributes['cleared']
# replace the custom attributes entirely, so 'cleared' is dropped
root.custom_attributes = {'test':'abc', 'dict':{'a':1, 'b':2}, 'list':[1,2,3]}
root.custom_attributes['test2'] = 'def'
model_file = os.path.join(str(tmpdir), 'custom_attr.dnn')
root.save(model_file)
root2 = C.load_model(model_file)
assert 'abc' == root2.custom_attributes['test']
assert {'a':1, 'b':2} == root2.custom_attributes['dict']
assert [1,2,3]==root2.custom_attributes['list']
assert 'def' == root2.custom_attributes['test2']
def test_transfer_learning(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # due to batch normalization in ResNet_18
try_set_default_device(cntk_device(device_id))
base_path = os.path.dirname(os.path.abspath(__file__))
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
print("Reading data and model from %s" % extPath)
model_file = os.path.join(extPath, *"PreTrainedModels/ResNet/v1/ResNet_18.model".split("/"))
map_file = os.path.join(extPath, *"Image/CIFAR/v0/cifar-10-batches-py/test_map.txt".split("/"))
else:
model_file = os.path.join(base_path, *"../../../../Examples/Image/PretrainedModels/ResNet_18.model".split("/"))
map_file = os.path.join(base_path, *"../../../../Examples/Image/DataSets/CIFAR-10/test_map.txt".split("/"))
os.chdir(os.path.join(os.path.dirname(map_file), '..'))
feature_node_name = "features"
last_hidden_node_name = "z.x"
image_width = 224
image_height = 224
num_channels = 3
num_classes = 10
num_epochs = 10
num_train_images = 10
num_test_images = 2
node_outputs = get_node_outputs(load_model(model_file))
assert len(node_outputs) == 83
output_file = os.path.join(base_path, "tl_output.txt")
trained_model = train_model(model_file, feature_node_name, last_hidden_node_name,
image_width, image_height, num_channels, num_classes, map_file,
num_epochs=num_epochs, max_images=num_train_images, freeze=True)
# since we do not use a reader for evaluation we need unzipped data
grocery_path = prepare_Grocery_data()
eval_map_file = os.path.join(grocery_path, "test.txt")
os.chdir(grocery_path)
eval_test_images(trained_model, output_file, eval_map_file, image_width, image_height,
max_images=num_test_images, column_offset=1)
expected_output_file = os.path.join(base_path, "tl_expected_output.txt")
output = np.fromfile(output_file)
expected_output = np.fromfile(expected_output_file)
assert np.allclose(output, expected_output, atol=TOLERANCE_ABSOLUTE)
def create_fast_rcnn_model(features, roi_proposals, label_targets, bbox_targets, bbox_inside_weights, cfg):
# Load the pre-trained classification net and clone layers
base_model = load_model(cfg['BASE_MODEL_PATH'])
conv_layers = clone_conv_layers(base_model, cfg)
fc_layers = clone_model(base_model, [cfg["MODEL"].POOL_NODE_NAME], [cfg["MODEL"].LAST_HIDDEN_NODE_NAME], clone_method=CloneMethod.clone)
# Normalization and conv layers
feat_norm = features - Constant([[[v]] for v in cfg["MODEL"].IMG_PAD_COLOR])
conv_out = conv_layers(feat_norm)
# Fast RCNN and losses
cls_score, bbox_pred = create_fast_rcnn_predictor(conv_out, roi_proposals, fc_layers, cfg)
detection_losses = create_detection_losses(cls_score, label_targets, bbox_pred, roi_proposals, bbox_targets, bbox_inside_weights, cfg)
pred_error = classification_error(cls_score, label_targets, axis=1)
return detection_losses, pred_error
def evaluateimage(file_path, mode, eval_model=None):
#from plot_helpers import eval_and_plot_faster_rcnn
if eval_model==None:
print("Loading existing model from %s" % model_path)
eval_model = load_model(model_path)
img_shape = (num_channels, image_height, image_width)
results_folder = globalvars['temppath']
results=eval_faster_rcnn(eval_model, file_path, img_shape,
results_folder, feature_node_name, globalvars['classes'], mode,
drawUnregressedRois=cfg["CNTK"].DRAW_UNREGRESSED_ROIS,
drawNegativeRois=cfg["CNTK"].DRAW_NEGATIVE_ROIS,
nmsThreshold=cfg["CNTK"].RESULTS_NMS_THRESHOLD,
nmsConfThreshold=cfg["CNTK"].RESULTS_NMS_CONF_THRESHOLD,
bgrPlotThreshold=cfg["CNTK"].RESULTS_BGR_PLOT_THRESHOLD)
return results
def eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects):
# load model and pick desired node as output
loaded_model = load_model(model_file)
node_in_graph = loaded_model.find_by_name(node_name)
output_nodes = combine([node_in_graph.owner])
# evaluate model and get desired node output
print("Evaluating model for output node %s" % node_name)
features_si = minibatch_source['features']
with open(output_file, 'wb') as results_file:
for i in range(0, num_objects):
mb = minibatch_source.next_minibatch(1)
output = output_nodes.eval(mb[features_si])
# write results to file
out_values = output[0].flatten()
np.savetxt(results_file, out_values[np.newaxis], fmt="%.6f")
def create_model(base_model_file, feature_node_name, last_hidden_node_name, num_classes, input_features, freeze=False):
# Load the pretrained classification net and find nodes
base_model = load_model(base_model_file)
feature_node = find_by_name(base_model, feature_node_name)
last_node = find_by_name(base_model, last_hidden_node_name)
# Clone the desired layers with fixed weights
cloned_layers = combine([last_node.owner]).clone(
CloneMethod.freeze if freeze else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Add new dense layer for class prediction
feat_norm = input_features - Constant(114)
cloned_out = cloned_layers(feat_norm)
z = Dense(num_classes, activation=None, name=new_output_node_name) (cloned_out)
return z
def test_custom_op_with_int8_params(tmpdir):
model_file = str(tmpdir/'test_model_params.bin')
delete_if_file_exists(model_file)
W1 = C.Parameter((1, 42), dtype=np.int8)
W1.value = np.arange(42).reshape(1, 42)
W2 = C.Parameter((1, 42), dtype=np.int8)
W3 = C.Parameter((1, 42), dtype=np.float)
X = C.input_variable((1, 42), dtype=np.float)
# custom_op, output_shape, output_data_type, *operands, **kw_name
z = C.custom_proxy_op("times", (21, 2), np.int8, X, W1, W2, W3, name ="custom_proxy")
z.save(model_file)
newz = C.load_model(model_file)
assert(newz.parameters[0].shape == (1, 42))
assert(newz.output.shape == (21, 2))
assert (np.array_equal(W1.value, newz.parameters[0].value))
def train_faster_rcnn(cfg):
# Train only if no model exists yet
model_path = cfg['MODEL_PATH']
if os.path.exists(model_path) and cfg["CNTK"].MAKE_MODE:
print("Loading existing model from %s" % model_path)
eval_model = load_model(model_path)
else:
if cfg["CNTK"].TRAIN_E2E:
eval_model = train_faster_rcnn_e2e(cfg)
else:
eval_model = train_faster_rcnn_alternating(cfg)
eval_model.save(model_path)
if cfg["CNTK"].DEBUG_OUTPUT:
plot(eval_model, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_eval_{}_{}.{}"
.format(cfg["MODEL"].BASE_MODEL, "e2e" if cfg["CNTK"].TRAIN_E2E else "4stage", cfg["CNTK"].GRAPH_TYPE)))
print("Stored eval model at %s" % model_path)
return eval_model
def test_saving_and_loading_int16_ndarray_as_attribute(tmpdir):
model_file = str(tmpdir/'test_model_int16.bin')
delete_if_file_exists(model_file)
data = np.arange(0,64, dtype=np.int16).reshape(16,4)
dict_val = C._to_cntk_dict_value(data)
W = C.Parameter((C.InferredDimension, 42), init=C.glorot_uniform(), dtype=np.float)
x = C.input_variable(12, dtype=np.float)
y = C.times(x, W)
y.custom_attributes = {'int16_nd':dict_val}
y.save(model_file)
assert(os.path.isfile(model_file))
z = C.load_model(model_file)
int16_data = z.custom_attributes['int16_nd']
assert(int16_data.shape == (16,4))
assert (np.array_equal(int16_data, data))
def create_faster_rcnn_predictor(base_model_file_name, features, scaled_gt_boxes, dims_input):
# Load the pre-trained classification net and clone layers
base_model = load_model(base_model_file_name)
conv_layers = clone_conv_layers(base_model)
fc_layers = clone_model(base_model, [pool_node_name], [last_hidden_node_name], clone_method=CloneMethod.clone)
# Normalization and conv layers
feat_norm = features - normalization_const
conv_out = conv_layers(feat_norm)
# RPN and prediction targets
rpn_rois, rpn_losses = \
create_rpn(conv_out, scaled_gt_boxes, dims_input, proposal_layer_param_string=cfg["CNTK"].PROPOSAL_LAYER_PARAMS)
rois, label_targets, bbox_targets, bbox_inside_weights = \
create_proposal_target_layer(rpn_rois, scaled_gt_boxes, num_classes=globalvars['num_classes'])
# Fast RCNN and losses
cls_score, bbox_pred = create_fast_rcnn_predictor(conv_out, rois, fc_layers)
detection_losses = create_detection_losses(cls_score, label_targets, rois, bbox_pred, bbox_targets, bbox_inside_weights)
loss = rpn_losses + detection_losses
pred_error = classification_error(cls_score, label_targets, axis=1)
return loss, pred_error
def frcn_predictor(features, rois, n_classes):
# Load the pretrained classification net and find nodes
loaded_model = load_model(model_file)
feature_node = find_by_name(loaded_model, feature_node_name)
conv_node = find_by_name(loaded_model, last_conv_node_name)
pool_node = find_by_name(loaded_model, pool_node_name)
last_node = find_by_name(loaded_model, last_hidden_node_name)
# Clone the conv layers and the fully connected layers of the network
conv_layers = combine([conv_node.owner]).clone(CloneMethod.freeze, {feature_node: Placeholder()})
fc_layers = combine([last_node.owner]).clone(CloneMethod.clone, {pool_node: Placeholder()})
# Create the Fast R-CNN model
feat_norm = features - Constant(114)
conv_out = conv_layers(feat_norm)
roi_out = roipooling(conv_out, rois, (roi_dim, roi_dim))
fc_out = fc_layers(roi_out)
# z = Dense(rois[0], num_classes, map_rank=1)(fc_out) # --> map_rank=1 is not yet supported
W = parameter(shape=(4096, n_classes), init=glorot_uniform())
b = parameter(shape=n_classes, init=0)
z = times(fc_out, W) + b
return z
#!flask/bin/python
from flask import Flask, jsonify, request, make_response, send_file
import os
os.environ['PATH'] = r'D:\home\python354x64;' + os.environ['PATH']
import uuid
from config import cfg
from cntk import load_model
app = Flask(__name__)
model_path = os.path.join(cfg["CNTK"].MODEL_DIRECTORY, cfg["CNTK"].MODEL_NAME)
print("Loading existing model from %s" % model_path)
loadedModel = load_model(model_path)
@app.errorhandler(404)
def not_found(error):
return make_response(jsonify({'error': 'Not found'}), 404)
@app.route('/')
def index():
return "<html>" \
"<body>" \
"Hello, World!<br>" \
"This is a sample web service written in Python using <a href=""http://flask.pocoo.org/"">Flask</a> module.<br>" \
"Use one of the following urls to evaluate images:<br>" \
"<a href=""/hotelidentifier/api/v1.0/evaluate/returntags"">/hotelidentifier/api/v1.0/evaluate/returntags</a> - takes image as parameter and returns cloud of tags<br>" \
"<a href=""/hotelidentifier/api/v1.0/evaluate/returntags"">/hotelidentifier/api/v1.0/evaluate/returnimage</a> - takes image as parameter and returns tagged image<br>" \
"</body>" \
"</html>"
def test_eye_like(operand, sparse_output, device_id, precision):
np_eye_like = lambda matrix: np.eye(matrix.shape[0], matrix.shape[1], dtype=np.float32)
operand = AA(operand).astype(np.float32)
expected = np_eye_like(operand)
expected_grad = np.zeros_like(operand).reshape(expected.shape)
my_eval = (lambda f, arg: f.eval(arg).todense()) if sparse_output else (lambda f, arg: f.eval(arg))
from .. import eye_like
import cntk as C
#testing with direct numpy input
y = C.eye_like(operand, sparse_output=sparse_output)
actual = y.eval().todense() if sparse_output else y.eval()
np.testing.assert_almost_equal(actual, expected)
#testing through input_variable
#test load and save:
import tempfile
import os
x = C.input_variable(operand.shape[1:], dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected)
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected)
os.remove(tempdir)
cntk_eye_like = C.eye_like(C.unpack_batch(x), sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected)
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test2')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected)
os.remove(tempdir)
cntk_eye_like = C.eye_like(C.transpose(C.unpack_batch(x), (1,0)), sparse_output=sparse_output)
actual = my_eval(cntk_eye_like, {x: operand})
grad = cntk_eye_like.grad({x: operand})
np.testing.assert_almost_equal(actual, expected.transpose())
np.testing.assert_almost_equal(grad, expected_grad)
tempdir = os.path.join(tempfile.gettempdir(), 'eye_like_test3')
cntk_eye_like.save(tempdir)
cntk_eye_like2 = C.load_model(tempdir)
np.testing.assert_almost_equal(my_eval(cntk_eye_like2, {cntk_eye_like2.arguments[0]: operand}), expected.transpose())
os.remove(tempdir)
#test expecting exception with sequence axis
with pytest.raises(Exception) as info:
#no sequence axis is allowed
x = C.sequence.input_variable(operand.shape[1:], dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
with pytest.raises(Exception) as info:
#no more than 2 axes is allowed (including any dynamic axes)
x = C.input_variable((3, 3), dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
with pytest.raises(Exception) as info:
#no less than 2 axes is allowed (including any dynamic axes)
x = C.input_variable((), dtype=np.float32, needs_gradient=True)
cntk_eye_like = C.eye_like(x, sparse_output=sparse_output)
os.makedirs(os.path.join(abs_path, "Output"))
if not os.path.exists(os.path.join(abs_path, "Output", cfg["CNTK"].DATASET)):
os.makedirs(os.path.join(abs_path, "Output", cfg["CNTK"].DATASET))
else:
# disable debug and plot outputs when running on GPU cluster
cfg["CNTK"].DEBUG_OUTPUT = False
cfg["CNTK"].VISUALIZE_RESULTS = False
set_global_vars()
model_path = os.path.join(globalvars['output_path'], "faster_rcnn_eval_{}_{}.model"
.format(cfg["CNTK"].BASE_MODEL, "e2e" if globalvars['train_e2e'] else "4stage"))
# Train only if no model exists yet
if os.path.exists(model_path) and cfg["CNTK"].MAKE_MODE:
print("Loading existing model from %s" % model_path)
eval_model = load_model(model_path)
else:
if globalvars['train_e2e']:
eval_model = train_faster_rcnn_e2e(base_model_file, debug_output=cfg["CNTK"].DEBUG_OUTPUT)
else:
eval_model = train_faster_rcnn_alternating(base_model_file, debug_output=cfg["CNTK"].DEBUG_OUTPUT)
eval_model.save(model_path)
if cfg["CNTK"].DEBUG_OUTPUT:
plot(eval_model, os.path.join(globalvars['output_path'], "graph_frcn_eval_{}_{}.{}"
.format(cfg["CNTK"].BASE_MODEL, "e2e" if globalvars['train_e2e'] else "4stage", cfg["CNTK"].GRAPH_TYPE)))
print("Stored eval model at %s" % model_path)
# Compute mean average precision on test set
eval_faster_rcnn_mAP(eval_model)
def generate_visualization(use_brain_script_model, testing=False):
num_objects_to_eval = 5
if (use_brain_script_model):
model_file_name = "07_Deconvolution_BS.model"
encoder_output_file_name = "encoder_output_BS.txt"
decoder_output_file_name = "decoder_output_BS.txt"
enc_node_name = "z.pool1"
input_node_name = "f2"
output_node_name = "z"
else:
model_file_name = "07_Deconvolution_PY.model"
encoder_output_file_name = "encoder_output_PY.txt"
decoder_output_file_name = "decoder_output_PY.txt"
enc_node_name = "pooling_node"
input_node_name = "input_node"
output_node_name = "output_node"
# define location of output, model and data and check existence
output_path = os.path.join(abs_path, "Output")
model_file = os.path.join(model_path, model_file_name)
data_file = os.path.join(data_path, "Test-28x28_cntk_text.txt")
if not (os.path.exists(model_file) and os.path.exists(data_file)):
print("Cannot find required data or model. "
"Please get the MNIST data set and run 'cntk configFile=07_Deconvolution_BS.cntk' or 'python 07_Deconvolution_PY.py' to create the model.")
exit(0)
# create minibatch source
minibatch_source = MinibatchSource(CTFDeserializer(data_file, StreamDefs(
features = StreamDef(field='features', shape=(28*28)),
labels = StreamDef(field='labels', shape=10)
)), randomize=False, max_sweeps = 1)
# use this to print all node names in the model
# print_all_node_names(model_file, use_brain_script_model)
# load model and pick desired nodes as output
loaded_model = load_model(model_file)
output_nodes = combine(
[loaded_model.find_by_name(input_node_name).owner,
loaded_model.find_by_name(enc_node_name).owner,
loaded_model.find_by_name(output_node_name).owner])
# evaluate model save output
features_si = minibatch_source['features']
with open(os.path.join(output_path, decoder_output_file_name), 'wb') as decoder_text_file:
with open(os.path.join(output_path, encoder_output_file_name), 'wb') as encoder_text_file:
for i in range(0, num_objects_to_eval):
mb = minibatch_source.next_minibatch(1)
raw_dict = output_nodes.eval(mb[features_si])
output_dict = {}
for key in raw_dict.keys(): output_dict[key.name] = raw_dict[key]
encoder_input = output_dict[input_node_name]
encoder_output = output_dict[enc_node_name]
decoder_output = output_dict[output_node_name]
in_values = (encoder_input[0,0].flatten())[np.newaxis]
enc_values = (encoder_output[0,0].flatten())[np.newaxis]
out_values = (decoder_output[0,0].flatten())[np.newaxis]
if not testing:
# write results as text and png
np.savetxt(decoder_text_file, out_values, fmt="%.6f")
np.savetxt(encoder_text_file, enc_values, fmt="%.6f")
save_as_png(in_values, os.path.join(output_path, "imageAutoEncoder_%s__input.png" % i))
save_as_png(out_values, os.path.join(output_path, "imageAutoEncoder_%s_output.png" % i))
# visualizing the encoding is only possible and meaningful with a single conv filter
enc_dim = 7
if(enc_values.size == enc_dim*enc_dim):
save_as_png(enc_values, os.path.join(output_path, "imageAutoEncoder_%s_encoding.png" % i), dim=enc_dim)
print("Done. Wrote output to %s" % output_path)
print ("{0} out of {1} predictions were correct {2}.".format(correct_count, pred_count, (float(correct_count) / pred_count)))
if __name__ == '__main__':
try_set_default_device(gpu(0))
# check for model and data existence
if not (os.path.exists(_base_model_file) and os.path.exists(_train_map_file) and os.path.exists(_test_map_file)):
print("Please run 'python install_data_and_model.py' first to get the required data and model.")
exit(0)
# You can use the following to inspect the base model and determine the desired node names
# node_outputs = get_node_outputs(load_model(_base_model_file))
# for out in node_outputs: print("{0} {1}".format(out.name, out.shape))
# Train only if no model exists yet or if make_mode is set to False
if os.path.exists(tl_model_file) and make_mode:
print("Loading existing model from %s" % tl_model_file)
trained_model = load_model(tl_model_file)
else:
trained_model = train_model(_base_model_file, _feature_node_name, _last_hidden_node_name,
_image_width, _image_height, _num_channels, _num_classes, _train_map_file,
max_epochs, freeze=freeze_weights)
trained_model.save(tl_model_file)
print("Stored trained model at %s" % tl_model_file)
# Evaluate the test set
eval_test_images(trained_model, output_file, _test_map_file, _image_width, _image_height)
print("Done. Wrote output to %s" % output_file)
def train_faster_rcnn_alternating(base_model_file_name, debug_output=False):
'''
4-Step Alternating Training scheme from the Faster R-CNN paper:
# Create initial network, only rpn, without detection network
# --> train only the rpn (and conv3_1 and up for VGG16)
# buffer region proposals from rpn
# Create full network, initialize conv layers with imagenet, use buffered proposals
# --> train only detection network (and conv3_1 and up for VGG16)
# Keep conv weights from detection network and fix them
# --> train only rpn
# buffer region proposals from rpn
# Keep conv and rpn weights from step 3 and fix them
# --> train only detection network
'''
# Learning parameters
rpn_lr_factor = globalvars['rpn_lr_factor']
rpn_lr_per_sample_scaled = [x * rpn_lr_factor for x in cfg["CNTK"].RPN_LR_PER_SAMPLE]
frcn_lr_factor = globalvars['frcn_lr_factor']
frcn_lr_per_sample_scaled = [x * frcn_lr_factor for x in cfg["CNTK"].FRCN_LR_PER_SAMPLE]
l2_reg_weight = cfg["CNTK"].L2_REG_WEIGHT
mm_schedule = momentum_schedule(globalvars['momentum_per_mb'])
rpn_epochs = globalvars['rpn_epochs']
frcn_epochs = globalvars['frcn_epochs']
print("Using base model: {}".format(cfg["CNTK"].BASE_MODEL))
print("rpn_lr_per_sample: {}".format(rpn_lr_per_sample_scaled))
print("frcn_lr_per_sample: {}".format(frcn_lr_per_sample_scaled))
if debug_output:
print("Storing graphs and models to %s." % globalvars['output_path'])
# Input variables denoting features, labeled ground truth rois (as 5-tuples per roi) and image dimensions
image_input = input_variable((num_channels, image_height, image_width), dynamic_axes=[Axis.default_batch_axis()],
name=feature_node_name)
feat_norm = image_input - normalization_const
roi_input = input_variable((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()])
scaled_gt_boxes = alias(roi_input, name='roi_input')
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
dims_node = alias(dims_input, name='dims_input')
rpn_rois_input = input_variable((cfg["TRAIN"].RPN_POST_NMS_TOP_N, 4), dynamic_axes=[Axis.default_batch_axis()])
rpn_rois_buf = alias(rpn_rois_input, name='rpn_rois')
# base image classification model (e.g. VGG16 or AlexNet)
base_model = load_model(base_model_file_name)
print("stage 1a - rpn")
if True:
# Create initial network, only rpn, without detection network
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: init new yes
# frcn: - -
# conv layers
conv_layers = clone_conv_layers(base_model)
conv_out = conv_layers(feat_norm)
# RPN and losses
rpn_rois, rpn_losses = create_rpn(conv_out, scaled_gt_boxes, dims_node, proposal_layer_param_string=cfg["CNTK"].PROPOSAL_LAYER_PARAMS)
stage1_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output: plot(stage1_rpn_network, os.path.join(globalvars['output_path'], "graph_frcn_train_stage1a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, rpn_losses, rpn_losses,
rpn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, epochs_to_train=rpn_epochs)
print("stage 1a - buffering rpn proposals")
buffered_proposals_s1 = compute_rpn_proposals(stage1_rpn_network, image_input, roi_input, dims_input)
print("stage 1b - frcn")
if True:
# Create full network, initialize conv layers with imagenet, fix rpn weights
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: stage1a rpn model no --> use buffered proposals
# frcn: base_model + new yes
# conv_layers
conv_layers = clone_conv_layers(base_model)
conv_out = conv_layers(feat_norm)
# use buffered proposals in target layer
rois, label_targets, bbox_targets, bbox_inside_weights = \
create_proposal_target_layer(rpn_rois_buf, scaled_gt_boxes, num_classes=globalvars['num_classes'])
# Fast RCNN and losses
fc_layers = clone_model(base_model, [pool_node_name], [last_hidden_node_name], CloneMethod.clone)
cls_score, bbox_pred = create_fast_rcnn_predictor(conv_out, rois, fc_layers)
detection_losses = create_detection_losses(cls_score, label_targets, rois, bbox_pred, bbox_targets, bbox_inside_weights)
pred_error = classification_error(cls_score, label_targets, axis=1, name="pred_error")
stage1_frcn_network = combine([rois, cls_score, bbox_pred, detection_losses, pred_error])
# train
if debug_output: plot(stage1_frcn_network, os.path.join(globalvars['output_path'], "graph_frcn_train_stage1b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, detection_losses, pred_error,
frcn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, epochs_to_train=frcn_epochs,
rpn_rois_input=rpn_rois_input, buffered_rpn_proposals=buffered_proposals_s1)
buffered_proposals_s1 = None
print("stage 2a - rpn")
if True:
# Keep conv weights from detection network and fix them
# initial weights train?
# conv: stage1b frcn model no
# rpn: stage1a rpn model yes
# frcn: - -
# conv_layers
conv_layers = clone_model(stage1_frcn_network, [feature_node_name], [last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# RPN and losses
rpn = clone_model(stage1_rpn_network, [last_conv_node_name, "roi_input", "dims_input"], ["rpn_rois", "rpn_losses"], CloneMethod.clone)
rpn_net = rpn(conv_out, dims_node, scaled_gt_boxes)
rpn_rois = rpn_net.outputs[0]
rpn_losses = rpn_net.outputs[1]
stage2_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output: plot(stage2_rpn_network, os.path.join(globalvars['output_path'], "graph_frcn_train_stage2a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, rpn_losses, rpn_losses,
rpn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, epochs_to_train=rpn_epochs)
print("stage 2a - buffering rpn proposals")
buffered_proposals_s2 = compute_rpn_proposals(stage2_rpn_network, image_input, roi_input, dims_input)
print("stage 2b - frcn")
if True:
# Keep conv and rpn weights from step 3 and fix them
# initial weights train?
# conv: stage2a rpn model no
# rpn: stage2a rpn model no --> use buffered proposals
# frcn: stage1b frcn model yes -
# conv_layers
conv_layers = clone_model(stage2_rpn_network, [feature_node_name], [last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# Fast RCNN and losses
frcn = clone_model(stage1_frcn_network, [last_conv_node_name, "rpn_rois", "roi_input"],
["cls_score", "bbox_regr", "rpn_target_rois", "detection_losses", "pred_error"], CloneMethod.clone)
stage2_frcn_network = frcn(conv_out, rpn_rois_buf, scaled_gt_boxes)
detection_losses = stage2_frcn_network.outputs[3]
pred_error = stage2_frcn_network.outputs[4]
# train
if debug_output: plot(stage2_frcn_network, os.path.join(globalvars['output_path'], "graph_frcn_train_stage2b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, detection_losses, pred_error,
frcn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, epochs_to_train=frcn_epochs,
rpn_rois_input=rpn_rois_input, buffered_rpn_proposals=buffered_proposals_s2)
buffered_proposals_s2 = None
return create_eval_model(stage2_frcn_network, image_input, dims_input, rpn_model=stage2_rpn_network)
results_file_path = base_path + "test.z"
with open(results_file_path, 'wb') as results_file:
for i in range(0, num_test_images):
data = test_minibatch_source.next_minibatch(1, input_map=input_map)
output = model.eval(data)
out_values = output[0, 0].flatten()
np.savetxt(results_file, out_values[np.newaxis], fmt="%.6f")
if (i+1) % 100 == 0:
print("Evaluated %s images.." % (i+1))
return
# The main method trains and evaluates a Fast R-CNN model.
# If a trained model is already available it is loaded an no training will be performed.
if __name__ == '__main__':
os.chdir(base_path)
model_path = os.path.join(abs_path, "Output", "frcn_py.model")
# Train only is no model exists yet
if os.path.exists(model_path):
print("Loading existing model from %s" % model_path)
trained_model = load_model(model_path)
else:
trained_model = train_fast_rcnn()
trained_model.save(model_path)
print("Stored trained model at %s" % model_path)
# Evaluate the test set
test_fast_rcnn(trained_model)
def train_fast_rcnn(cfg):
# Train only if no model exists yet
model_path = cfg['MODEL_PATH']
if os.path.exists(model_path) and cfg["CNTK"].MAKE_MODE:
print("Loading existing model from %s" % model_path)
return load_model(model_path)
else:
# Input variables denoting features and labeled ground truth rois (as 5-tuples per roi)
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=cfg["MODEL"].FEATURE_NODE_NAME)
roi_proposals = input_variable((cfg.NUM_ROI_PROPOSALS, 4), dynamic_axes=[Axis.default_batch_axis()], name = "roi_proposals")
label_targets = input_variable((cfg.NUM_ROI_PROPOSALS, cfg["DATA"].NUM_CLASSES), dynamic_axes=[Axis.default_batch_axis()])
bbox_targets = input_variable((cfg.NUM_ROI_PROPOSALS, 4*cfg["DATA"].NUM_CLASSES), dynamic_axes=[Axis.default_batch_axis()])
bbox_inside_weights = input_variable((cfg.NUM_ROI_PROPOSALS, 4*cfg["DATA"].NUM_CLASSES), dynamic_axes=[Axis.default_batch_axis()])
# Instantiate the Fast R-CNN prediction model and loss function
loss, pred_error = create_fast_rcnn_model(image_input, roi_proposals, label_targets, bbox_targets, bbox_inside_weights, cfg)
if isinstance(loss, cntk.Variable):
loss = combine([loss])
if cfg["CNTK"].DEBUG_OUTPUT:
print("Storing graphs and models to %s." % cfg.OUTPUT_PATH)
plot(loss, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train." + cfg["CNTK"].GRAPH_TYPE))
# Set learning parameters
lr_factor = cfg["CNTK"].LR_FACTOR
lr_per_sample_scaled = [x * lr_factor for x in cfg["CNTK"].LR_PER_SAMPLE]
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
l2_reg_weight = cfg["CNTK"].L2_REG_WEIGHT
epochs_to_train = cfg["CNTK"].MAX_EPOCHS
print("Using base model: {}".format(cfg["MODEL"].BASE_MODEL))
print("lr_per_sample: {}".format(lr_per_sample_scaled))
# --- train ---
# Instantiate the learners and the trainer object
params = loss.parameters
biases = [p for p in params if '.b' in p.name or 'b' == p.name]
others = [p for p in params if not p in biases]
bias_lr_mult = cfg["CNTK"].BIAS_LR_MULT
lr_schedule = learning_rate_schedule(lr_per_sample_scaled, unit=UnitType.sample)
learner = momentum_sgd(others, lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight, unit_gain=False, use_mean_gradient=True)
bias_lr_per_sample = [v * bias_lr_mult for v in cfg["CNTK"].LR_PER_SAMPLE]
bias_lr_schedule = learning_rate_schedule(bias_lr_per_sample, unit=UnitType.sample)
bias_learner = momentum_sgd(biases, bias_lr_schedule, mm_schedule, l2_regularization_weight=l2_reg_weight, unit_gain=False, use_mean_gradient=True)
trainer = Trainer(None, (loss, pred_error), [learner, bias_learner])
# Get minibatches of images and perform model training
print("Training model for %s epochs." % epochs_to_train)
log_number_of_parameters(loss)
# Create the minibatch source
if cfg.USE_PRECOMPUTED_PROPOSALS:
proposal_provider = ProposalProvider.fromfile(cfg["DATA"].TRAIN_PRECOMPUTED_PROPOSALS_FILE, cfg.NUM_ROI_PROPOSALS)
else:
proposal_provider = ProposalProvider.fromconfig(cfg)
od_minibatch_source = ObjectDetectionMinibatchSource(
cfg["DATA"].TRAIN_MAP_FILE, cfg["DATA"].TRAIN_ROI_FILE,
max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE,
pad_width=cfg.IMAGE_WIDTH,
pad_height=cfg.IMAGE_HEIGHT,
pad_value=cfg["MODEL"].IMG_PAD_COLOR,
randomize=True,
use_flipping=cfg["TRAIN"].USE_FLIPPED,
max_images=cfg["DATA"].NUM_TRAIN_IMAGES,
num_classes=cfg["DATA"].NUM_CLASSES,
proposal_provider=proposal_provider,
provide_targets=True,
proposal_iou_threshold = cfg.BBOX_THRESH,
normalize_means = None if not cfg.BBOX_NORMALIZE_TARGETS else cfg.BBOX_NORMALIZE_MEANS,
normalize_stds = None if not cfg.BBOX_NORMALIZE_TARGETS else cfg.BBOX_NORMALIZE_STDS)
# define mapping from reader streams to network inputs
input_map = {
od_minibatch_source.image_si: image_input,
od_minibatch_source.proposals_si: roi_proposals,
od_minibatch_source.label_targets_si: label_targets,
od_minibatch_source.bbox_targets_si: bbox_targets,
od_minibatch_source.bbiw_si: bbox_inside_weights
}
progress_printer = ProgressPrinter(tag='Training', num_epochs=epochs_to_train, gen_heartbeat=True)
for epoch in range(epochs_to_train): # loop over epochs
sample_count = 0
while sample_count < cfg["DATA"].NUM_TRAIN_IMAGES: # loop over minibatches in the epoch
data = od_minibatch_source.next_minibatch(min(cfg.MB_SIZE, cfg["DATA"].NUM_TRAIN_IMAGES - sample_count), input_map=input_map)
trainer.train_minibatch(data) # update model with it
sample_count += trainer.previous_minibatch_sample_count # count samples processed so far
progress_printer.update_with_trainer(trainer, with_metric=True) # log progress
if sample_count % 100 == 0:
print("Processed {} samples".format(sample_count))
progress_printer.epoch_summary(with_metric=True)
eval_model = create_fast_rcnn_eval_model(loss, image_input, roi_proposals, cfg)
eval_model.save(cfg['MODEL_PATH'])
return eval_model
def train_faster_rcnn_alternating(cfg):
'''
4-Step Alternating Training scheme from the Faster R-CNN paper:
# Create initial network, only rpn, without detection network
# --> train only the rpn (and conv3_1 and up for VGG16)
# buffer region proposals from rpn
# Create full network, initialize conv layers with imagenet, use buffered proposals
# --> train only detection network (and conv3_1 and up for VGG16)
# Keep conv weights from detection network and fix them
# --> train only rpn
# buffer region proposals from rpn
# Keep conv and rpn weights from step 3 and fix them
# --> train only detection network
'''
# setting pre- and post-nms top N to training values since buffered proposals are used for further training
test_pre = cfg["TEST"].RPN_PRE_NMS_TOP_N
test_post = cfg["TEST"].RPN_POST_NMS_TOP_N
cfg["TEST"].RPN_PRE_NMS_TOP_N = cfg["TRAIN"].RPN_PRE_NMS_TOP_N
cfg["TEST"].RPN_POST_NMS_TOP_N = cfg["TRAIN"].RPN_POST_NMS_TOP_N
# Learning parameters
rpn_lr_factor = cfg["MODEL"].RPN_LR_FACTOR
rpn_lr_per_sample_scaled = [x * rpn_lr_factor for x in cfg["CNTK"].RPN_LR_PER_SAMPLE]
frcn_lr_factor = cfg["MODEL"].FRCN_LR_FACTOR
frcn_lr_per_sample_scaled = [x * frcn_lr_factor for x in cfg["CNTK"].FRCN_LR_PER_SAMPLE]
l2_reg_weight = cfg["CNTK"].L2_REG_WEIGHT
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
rpn_epochs = cfg["CNTK"].RPN_EPOCHS
frcn_epochs = cfg["CNTK"].FRCN_EPOCHS
feature_node_name = cfg["MODEL"].FEATURE_NODE_NAME
last_conv_node_name = cfg["MODEL"].LAST_CONV_NODE_NAME
print("Using base model: {}".format(cfg["MODEL"].BASE_MODEL))
print("rpn_lr_per_sample: {}".format(rpn_lr_per_sample_scaled))
print("frcn_lr_per_sample: {}".format(frcn_lr_per_sample_scaled))
debug_output=cfg["CNTK"].DEBUG_OUTPUT
if debug_output:
print("Storing graphs and models to %s." % cfg.OUTPUT_PATH)
# Input variables denoting features, labeled ground truth rois (as 5-tuples per roi) and image dimensions
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=feature_node_name)
feat_norm = image_input - Constant([[[v]] for v in cfg["MODEL"].IMG_PAD_COLOR])
roi_input = input_variable((cfg.INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()])
scaled_gt_boxes = alias(roi_input, name='roi_input')
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
dims_node = alias(dims_input, name='dims_input')
rpn_rois_input = input_variable((cfg["TRAIN"].RPN_POST_NMS_TOP_N, 4), dynamic_axes=[Axis.default_batch_axis()])
rpn_rois_buf = alias(rpn_rois_input, name='rpn_rois')
# base image classification model (e.g. VGG16 or AlexNet)
base_model = load_model(cfg['BASE_MODEL_PATH'])
print("stage 1a - rpn")
if True:
# Create initial network, only rpn, without detection network
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: init new yes
# frcn: - -
# conv layers
conv_layers = clone_conv_layers(base_model, cfg)
conv_out = conv_layers(feat_norm)
# RPN and losses
rpn_rois, rpn_losses = create_rpn(conv_out, scaled_gt_boxes, dims_node, cfg)
stage1_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output: plot(stage1_rpn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage1a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, rpn_losses, rpn_losses,
rpn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, rpn_epochs, cfg)
print("stage 1a - buffering rpn proposals")
buffered_proposals_s1 = compute_rpn_proposals(stage1_rpn_network, image_input, roi_input, dims_input, cfg)
print("stage 1b - frcn")
if True:
# Create full network, initialize conv layers with imagenet, fix rpn weights
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: stage1a rpn model no --> use buffered proposals
# frcn: base_model + new yes
# conv_layers
conv_layers = clone_conv_layers(base_model, cfg)
conv_out = conv_layers(feat_norm)
# use buffered proposals in target layer
rois, label_targets, bbox_targets, bbox_inside_weights = \
create_proposal_target_layer(rpn_rois_buf, scaled_gt_boxes, cfg)
# Fast RCNN and losses
fc_layers = clone_model(base_model, [cfg["MODEL"].POOL_NODE_NAME], [cfg["MODEL"].LAST_HIDDEN_NODE_NAME], CloneMethod.clone)
cls_score, bbox_pred = create_fast_rcnn_predictor(conv_out, rois, fc_layers, cfg)
detection_losses = create_detection_losses(cls_score, label_targets, bbox_pred, rois, bbox_targets, bbox_inside_weights, cfg)
pred_error = classification_error(cls_score, label_targets, axis=1, name="pred_error")
stage1_frcn_network = combine([rois, cls_score, bbox_pred, detection_losses, pred_error])
# train
if debug_output: plot(stage1_frcn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage1b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, detection_losses, pred_error,
frcn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, frcn_epochs, cfg,
rpn_rois_input=rpn_rois_input, buffered_rpn_proposals=buffered_proposals_s1)
buffered_proposals_s1 = None
print("stage 2a - rpn")
if True:
# Keep conv weights from detection network and fix them
# initial weights train?
# conv: stage1b frcn model no
# rpn: stage1a rpn model yes
# frcn: - -
# conv_layers
conv_layers = clone_model(stage1_frcn_network, [feature_node_name], [last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# RPN and losses
rpn = clone_model(stage1_rpn_network, [last_conv_node_name, "roi_input", "dims_input"], ["rpn_rois", "rpn_losses"], CloneMethod.clone)
rpn_net = rpn(conv_out, dims_node, scaled_gt_boxes)
rpn_rois = rpn_net.outputs[0]
rpn_losses = rpn_net.outputs[1]
stage2_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output: plot(stage2_rpn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage2a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, rpn_losses, rpn_losses,
rpn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, rpn_epochs, cfg)
print("stage 2a - buffering rpn proposals")
buffered_proposals_s2 = compute_rpn_proposals(stage2_rpn_network, image_input, roi_input, dims_input, cfg)
print("stage 2b - frcn")
if True:
# Keep conv and rpn weights from step 3 and fix them
# initial weights train?
# conv: stage2a rpn model no
# rpn: stage2a rpn model no --> use buffered proposals
# frcn: stage1b frcn model yes -
# conv_layers
conv_layers = clone_model(stage2_rpn_network, [feature_node_name], [last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# Fast RCNN and losses
frcn = clone_model(stage1_frcn_network, [last_conv_node_name, "rpn_rois", "roi_input"],
["cls_score", "bbox_regr", "rpn_target_rois", "detection_losses", "pred_error"], CloneMethod.clone)
stage2_frcn_network = frcn(conv_out, rpn_rois_buf, scaled_gt_boxes)
detection_losses = stage2_frcn_network.outputs[3]
pred_error = stage2_frcn_network.outputs[4]
# train
if debug_output: plot(stage2_frcn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage2b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, detection_losses, pred_error,
frcn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, frcn_epochs, cfg,
rpn_rois_input=rpn_rois_input, buffered_rpn_proposals=buffered_proposals_s2)
buffered_proposals_s2 = None
# resetting config values to original test values
cfg["TEST"].RPN_PRE_NMS_TOP_N = test_pre
cfg["TEST"].RPN_POST_NMS_TOP_N = test_post
return create_faster_rcnn_eval_model(stage2_frcn_network, image_input, dims_input, cfg, rpn_model=stage2_rpn_network)
def sequence_to_sequence_translator(debug_output=False, run_test=False):
input_vocab_dim = 69
label_vocab_dim = 69
# network complexity; initially low for faster testing
hidden_dim = 256
num_layers = 1
# Source and target inputs to the model
batch_axis = Axis.default_batch_axis()
input_seq_axis = Axis('inputAxis')
label_seq_axis = Axis('labelAxis')
input_dynamic_axes = [batch_axis, input_seq_axis]
raw_input = input_variable(
shape=(input_vocab_dim), dynamic_axes=input_dynamic_axes, name='raw_input')
label_dynamic_axes = [batch_axis, label_seq_axis]
raw_labels = input_variable(
shape=(label_vocab_dim), dynamic_axes=label_dynamic_axes, name='raw_labels')
# Instantiate the sequence to sequence translation model
input_sequence = raw_input
# Drop the sentence start token from the label, for decoder training
label_sequence = sequence.slice(raw_labels, 1, 0) # <s> A B C </s> --> A B C </s>
label_sentence_start = sequence.first(raw_labels) # <s>
is_first_label = sequence.is_first(label_sequence) # <s> 0 0 0 ...
label_sentence_start_scattered = sequence.scatter(
label_sentence_start, is_first_label)
# Encoder
encoder_outputH = stabilize(input_sequence)
for i in range(0, num_layers):
(encoder_outputH, encoder_outputC) = LSTMP_component_with_self_stabilization(
encoder_outputH.output, hidden_dim, hidden_dim, future_value, future_value)
thought_vectorH = sequence.first(encoder_outputH)
thought_vectorC = sequence.first(encoder_outputC)
thought_vector_broadcastH = sequence.broadcast_as(
thought_vectorH, label_sequence)
thought_vector_broadcastC = sequence.broadcast_as(
thought_vectorC, label_sequence)
# Decoder
decoder_history_hook = alias(label_sequence, name='decoder_history_hook') # copy label_sequence
decoder_input = element_select(is_first_label, label_sentence_start_scattered, past_value(
decoder_history_hook))
decoder_outputH = stabilize(decoder_input)
for i in range(0, num_layers):
if (i > 0):
recurrence_hookH = past_value
recurrence_hookC = past_value
else:
isFirst = sequence.is_first(label_sequence)
recurrence_hookH = lambda operand: element_select(
isFirst, thought_vector_broadcastH, past_value(operand))
recurrence_hookC = lambda operand: element_select(
isFirst, thought_vector_broadcastC, past_value(operand))
(decoder_outputH, encoder_outputC) = LSTMP_component_with_self_stabilization(
decoder_outputH.output, hidden_dim, hidden_dim, recurrence_hookH, recurrence_hookC)
decoder_output = decoder_outputH
# Softmax output layer
z = linear_layer(stabilize(decoder_output), label_vocab_dim)
# Criterion nodes
ce = cross_entropy_with_softmax(z, label_sequence)
errs = classification_error(z, label_sequence)
# network output for decoder history
net_output = hardmax(z)
# make a clone of the graph where the ground truth is replaced by the network output
ng = z.clone(CloneMethod.share, {decoder_history_hook.output : net_output.output})
# Instantiate the trainer object to drive the model training
lr_per_minibatch = learning_rate_schedule(0.5, UnitType.minibatch)
momentum_time_constant = momentum_as_time_constant_schedule(1100)
clipping_threshold_per_sample = 2.3
gradient_clipping_with_truncation = True
learner = momentum_sgd(z.parameters,
lr_per_minibatch, momentum_time_constant,
gradient_clipping_threshold_per_sample=clipping_threshold_per_sample,
gradient_clipping_with_truncation=gradient_clipping_with_truncation)
trainer = Trainer(z, ce, errs, learner)
# setup data
train_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "Data", "cmudict-0.7b.train-dev-20-21.ctf")
valid_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "Data", "tiny.ctf")
# readers
randomize_data = True
if run_test:
randomize_data = False # because we want to get an exact error
train_reader = create_reader(train_path, randomize_data, input_vocab_dim, label_vocab_dim)
train_bind = {
raw_input : train_reader.streams.features,
raw_labels : train_reader.streams.labels
}
# get the vocab for printing output sequences in plaintext
vocab_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "Data", "cmudict-0.7b.mapping")
vocab = [w.strip() for w in open(vocab_path).readlines()]
i2w = { i:ch for i,ch in enumerate(vocab) }
# Get minibatches of sequences to train with and perform model training
i = 0
mbs = 0
minibatch_size = 72
epoch_size = 908241
max_epochs = 10
training_progress_output_freq = 500
# make things more basic for running a quicker test
if run_test:
epoch_size = 5000
max_epochs = 1
training_progress_output_freq = 30
valid_reader = create_reader(valid_path, False, input_vocab_dim, label_vocab_dim)
valid_bind = {
find_arg_by_name('raw_input',ng) : valid_reader.streams.features,
find_arg_by_name('raw_labels',ng) : valid_reader.streams.labels
}
for epoch in range(max_epochs):
loss_numer = 0
metric_numer = 0
denom = 0
while i < (epoch+1) * epoch_size:
# get next minibatch of training data
mb_train = train_reader.next_minibatch(minibatch_size, input_map=train_bind)
trainer.train_minibatch(mb_train)
# collect epoch-wide stats
samples = trainer.previous_minibatch_sample_count
loss_numer += trainer.previous_minibatch_loss_average * samples
metric_numer += trainer.previous_minibatch_evaluation_average * samples
denom += samples
# every N MBs evaluate on a test sequence to visually show how we're doing
if mbs % training_progress_output_freq == 0:
mb_valid = valid_reader.next_minibatch(minibatch_size, input_map=valid_bind)
e = ng.eval(mb_valid)
print_sequences(e, i2w)
print_training_progress(trainer, mbs, training_progress_output_freq)
i += mb_train[raw_labels].num_samples
mbs += 1
print("--- EPOCH %d DONE: loss = %f, errs = %f ---" % (epoch, loss_numer/denom, 100.0*(metric_numer/denom)))
error1 = translator_test_error(z, trainer, input_vocab_dim, label_vocab_dim)
save_model(z, "seq2seq.dnn")
z = load_model("seq2seq.dnn")
label_seq_axis = Axis('labelAxis')
label_sequence = sequence.slice(find_arg_by_name('raw_labels',z), 1, 0)
ce = cross_entropy_with_softmax(z, label_sequence)
errs = classification_error(z, label_sequence)
trainer = Trainer(z, ce, errs, [momentum_sgd(
z.parameters, lr_per_minibatch, momentum_time_constant, clipping_threshold_per_sample, gradient_clipping_with_truncation)])
error2 = translator_test_error(z, trainer, input_vocab_dim, label_vocab_dim)
assert error1 == error2
return error1
