def testSelectRecordByContext(self):
float_features = self.model.input_feature_schema.float_features
float_array = np.array([1.0, 2.0], dtype=np.float32)
schema.FeedRecord(float_features, [float_array])
with Tags(Tags.EXCLUDE_FROM_PREDICTION):
log_float_features = self.model.Log(float_features, 1)
joined = self.model.SelectRecordByContext(
schema.Struct(
(InstantiationContext.PREDICTION, float_features),
(InstantiationContext.TRAINING, log_float_features),
# TODO: TRAIN_ONLY layers are also generated in eval
(InstantiationContext.EVAL, log_float_features),
))
# model.output_schema has to a struct
self.model.output_schema = schema.Struct(('joined', joined))
predict_net = layer_model_instantiator.generate_predict_net(self.model)
workspace.RunNetOnce(predict_net)
predict_output = schema.FetchRecord(predict_net.output_record())
npt.assert_array_equal(float_array, predict_output['joined']())
eval_net = layer_model_instantiator.generate_eval_net(self.model)
workspace.RunNetOnce(eval_net)
eval_output = schema.FetchRecord(eval_net.output_record())
npt.assert_array_equal(np.log(float_array), eval_output['joined']())
_, train_net = (
layer_model_instantiator.generate_training_nets_forward_only(
self.model))
workspace.RunNetOnce(train_net)
train_output = schema.FetchRecord(train_net.output_record())
npt.assert_array_equal(np.log(float_array), train_output['joined']())
def testLastNWindowCollector(self, X, num_to_collect):
input_record = self.new_record(schema.Scalar(np.float32))
schema.FeedRecord(input_record, [X])
last_n = self.model.LastNWindowCollector(input_record, num_to_collect)
self.run_train_net_forward_only()
output_record = schema.FetchRecord(last_n.last_n)
start = max(0, 5 - num_to_collect)
npt.assert_array_equal(X[start:], output_record())
num_visited = schema.FetchRecord(last_n.num_visited)
npt.assert_array_equal([5], num_visited())
def train_with_eval(
self,
num_epoch=1,
report_interval=0,
eval_during_training=False,
):
''' Fastest mode: report_interval = 0
Medium mode: report_interval > 0, eval_during_training=False
Slowest mode: report_interval > 0, eval_during_training=True
'''
num_batch_per_epoch = int(self.input_data_store['train'][1] /
self.batch_size)
if not self.input_data_store['train'][1] % self.batch_size == 0:
num_batch_per_epoch += 1
print('[Warning]: batch_size cannot be divided. ' +
'Run on {} example instead of {}'.format(
num_batch_per_epoch *
self.batch_size, self.input_data_store['train'][1]))
print('<<< Run {} iteration'.format(num_epoch * num_batch_per_epoch))
train_net = self.net_store['train_net']
if report_interval > 0:
print('>>> Training with Reports')
num_eval = int(num_epoch / report_interval)
num_unit_iter = int((num_batch_per_epoch * num_epoch) / num_eval)
if eval_during_training and 'eval_net' in self.net_store:
print('>>> Training with Eval Reports (Slowest mode)')
eval_net = self.net_store['eval_net']
for i in range(num_eval):
workspace.RunNet(train_net.Proto().name,
num_iter=num_unit_iter)
self.reports['epoch'].append((i + 1) * report_interval)
train_loss = np.asscalar(schema.FetchRecord(self.loss).get())
self.reports['train_loss'].append(train_loss)
if eval_during_training and 'eval_net' in self.net_store:
workspace.RunNet(eval_net.Proto().name,
num_iter=num_unit_iter)
eval_loss = np.asscalar(
schema.FetchRecord(self.loss).get())
self.reports['eval_loss'].append(eval_loss)
else:
print('>>> Training without Reports (Fastest mode)')
num_iter = num_epoch * num_batch_per_epoch
workspace.RunNet(train_net, num_iter=num_iter)
print('>>> Saving test model')
exporter.save_net(self.net_store['pred_net'], self.model,
self.model_name + '_init',
self.model_name + '_predict')
def testConv(self):
batch_size = 50
H = 1
W = 10
C = 50
output_dims = 32
kernel_h = 1
kernel_w = 3
stride_h = 1
stride_w = 1
pad_t = 0
pad_b = 0
pad_r = None
pad_l = None
input_record = self.new_record(schema.Scalar((np.float32, (H, W, C))))
X = np.random.random((batch_size, H, W, C)).astype(np.float32)
schema.FeedRecord(input_record, [X])
conv = self.model.Conv(input_record,
output_dims,
kernel_h=kernel_h,
kernel_w=kernel_w,
stride_h=stride_h,
stride_w=stride_w,
pad_t=pad_t,
pad_b=pad_b,
pad_r=pad_r,
pad_l=pad_l,
order='NHWC')
self.assertEqual(schema.Scalar((np.float32, (output_dims, ))), conv)
self.run_train_net_forward_only()
output_record = schema.FetchRecord(conv)
# check the number of output channels is the same as input in this example
assert output_record.field_types()[0].shape == (H, W, output_dims)
assert output_record().shape == (batch_size, H, W, output_dims)
train_init_net, train_net = self.get_training_nets()
# Init net assertions
init_ops = self.assertNetContainOps(train_init_net, [
OpSpec("XavierFill", None, None),
OpSpec("ConstantFill", None, None),
])
conv_spec = OpSpec("Conv", [
input_record.field_blobs()[0],
init_ops[0].output[0],
init_ops[1].output[0],
], conv.field_blobs())
# Train net assertions
self.assertNetContainOps(train_net, [conv_spec])
# Predict net assertions
predict_net = self.get_predict_net()
self.assertNetContainOps(predict_net, [conv_spec])
# Eval net assertions
eval_net = self.get_eval_net()
self.assertNetContainOps(eval_net, [conv_spec])
def predict_ids(self, vg, vd):
# preproc the input
vg = vg.astype(np.float32)
vd = vd.astype(np.float32)
if len(self.preproc_param) == 0:
self.preproc_param = pickle.load(open(self.pickle_file_name, "rb"))
dummy_ids = np.zeros(len(vg))
preproc_data_arrays = preproc.dc_iv_preproc(
vg,
vd,
dummy_ids,
self.preproc_param['scale'],
self.preproc_param['vg_shift'],
)
_preproc_data_arrays = [
np.expand_dims(x, axis=1) for x in preproc_data_arrays
]
workspace.FeedBlob('DBInput_train/sig_input', _preproc_data_arrays[0])
workspace.FeedBlob('DBInput_train/tanh_input', _preproc_data_arrays[1])
pred_net = self.net_store['pred_net']
workspace.RunNet(pred_net)
_ids = np.squeeze(schema.FetchRecord(self.pred).get())
restore_id_func, _ = preproc.get_restore_id_func(
self.preproc_param['scale'],
self.preproc_param['vg_shift'],
)
ids = restore_id_func(_ids)
return _ids, ids
def avg_loss_full_epoch(self, net_name):
num_batch_per_epoch = int(self.input_data_store['train'][1] /
self.batch_size)
if not self.input_data_store['train'][1] % self.batch_size == 0:
num_batch_per_epoch += 1
print('[Warning]: batch_size cannot be divided. ' +
'Run on {} example instead of {}'.format(
num_batch_per_epoch *
self.batch_size, self.input_data_store['train'][1]))
# Get the average loss of all data
loss = 0.
for j in range(num_batch_per_epoch):
workspace.RunNet(self.net_store[net_name])
loss += np.asscalar(schema.FetchRecord(self.loss).get())
loss /= num_batch_per_epoch
return loss
def predict_ids(self, vg, vd):
# preproc the input
vg = vg.astype(np.float32)
vd = vd.astype(np.float32)
if len(self.preproc_param) == 0:
self.preproc_param = pickle.load(open(self.pickle_file_name, "rb"))
dummy_ids = np.zeros(len(vg))
preproc_data_arrays = [vg, vd, dummy_ids]
if self.train_target == TrainTarget.ORIGIN:
preproc_data_arrays = preproc.dc_iv_preproc(
vg,
vd,
dummy_ids,
self.preproc_param['scale'],
self.preproc_param['vg_shift'],
)
_preproc_data_arrays = [
np.expand_dims(x, axis=1) if x.ndim == 1 else x
for x in preproc_data_arrays
]
workspace.FeedBlob('DBInput_train/sig_input', _preproc_data_arrays[0])
workspace.FeedBlob('DBInput_train/tanh_input', _preproc_data_arrays[1])
if self.train_target == TrainTarget.ADJOINT:
adjoint_input = np.ones((vg.shape[0], 1))
workspace.FeedBlob('DBInput_train/adjoint_input', adjoint_input)
pred_net = self.net_store['pred_net']
workspace.RunNet(pred_net)
_ids = np.squeeze(schema.FetchRecord(self.pred).get())
ids = _ids
if self.train_target == TrainTarget.ORIGIN:
restore_id_func, _ = preproc.get_restore_id_func(
self.preproc_param['scale'],
self.preproc_param['vg_shift'],
)
ids = restore_id_func(_ids)
return _ids, ids
),
bias_optim=optimizer.AdagradOptimizer(
alpha=0.01,
epsilon=1e-4,
))
# Train the model
train_init_net, train_net = instantiator.generate_training_nets(model)
workspace.RunNetOnce(train_init_net)
workspace.CreateNet(train_net)
num_iter = 1000
eval_num_iter = 100
# loss_lst = []
for i in range(eval_num_iter):
workspace.RunNet(train_net.Proto().name, num_iter=num_iter)
print(schema.FetchRecord(loss).get())
X_pred = np.squeeze(
schema.FetchRecord(model.input_feature_schema.sig_input).get()).reshape(
X.shape)
Y_pred = np.squeeze(
schema.FetchRecord(model.input_feature_schema.tanh_input).get()).reshape(
X.shape)
Z_pred = np.squeeze(schema.FetchRecord(origin_pred).get()).reshape(X.shape)
gy_restore, gx_restore = np.gradient(Z_pred, 0.2, 0.2)
gx_pred = schema.FetchRecord(sig_adjoint_pred).get().reshape(X.shape)
gy_pred = schema.FetchRecord(tanh_adjoint_pred).get().reshape(X.shape)
gx_label = schema.FetchRecord(
model.trainer_extra_schema.sig_loss_record.label).get().reshape(X.shape)
gy_label = schema.FetchRecord(
model.trainer_extra_schema.tanh_loss_record.label).get().reshape(X.shape)
def train_with_eval(
self,
num_epoch=1,
report_interval=0,
eval_during_training=False,
):
''' Fastest mode: report_interval = 0
Medium mode: report_interval > 0, eval_during_training=False
Slowest mode: report_interval > 0, eval_during_training=True
Debug mode: DEBUG flag set to true
'''
if DEBUG:
train_net = self.net_store['train_net']
workspace.RunNet(train_net, num_iter=5000)
if self.net_builder == TrainTarget.ORIGIN:
print(workspace.FetchBlob('DBInput_train/tanh_input'))
print(workspace.FetchBlob('DBInput_train/sig_input'))
print(workspace.FetchBlob('DBInput_train/label'))
print(workspace.FetchBlob('prediction'))
# print(workspace.FetchBlob('Sigmoid_auto_1/sig_tranfer_layer_2'))
# print(workspace.FetchBlob('Tanh_auto_1/tanh_tranfer_layer_2'))
# print(workspace.FetchBlob('Sigmoid/sig_tranfer_layer_0'))
# print(workspace.FetchBlob('sig_fc_layer_0/w'))
# print(workspace.FetchBlob('sig_fc_layer_0/b'))
if self.net_builder == TrainTarget.ADJOINT:
print(workspace.FetchBlob('DBInput_train/tanh_input'))
print(workspace.FetchBlob('DBInput_train/sig_input'))
print(workspace.FetchBlob('DBInput_train/label'))
print(workspace.FetchBlob('origin/' + 'Mul/origin_pred'))
print(
workspace.FetchBlob('adjoint/' +
'Sigmoid_auto_1/sig_tranfer_layer_2'))
print(
workspace.FetchBlob('origin/' +
'Tanh_auto_1/tanh_tranfer_layer_2'))
print(
workspace.FetchBlob('origin/' +
'Sigmoid/sig_tranfer_layer_0'))
print(workspace.FetchBlob('adjoint/' + 'sig_fc_layer_0/w'))
print(workspace.FetchBlob('adjoint/' + 'sig_fc_layer_0/b'))
print('-' * 50)
eval_net = self.net_store['eval_net']
workspace.RunNet(eval_net.Proto().name)
if self.net_builder == TrainTarget.ORIGIN:
print(workspace.FetchBlob('DBInput_eval/eval_sig_input'))
print(workspace.FetchBlob('DBInput_eval/eval_sig_input'))
eval_label_debug = workspace.FetchBlob(
'DBInput_eval/eval_label')
eval_pred_debug = workspace.FetchBlob('prediction')
print(
workspace.FetchBlob(
'batch_direct_weighted_l1_loss/l1_metric'))
print(np.average(np.abs(eval_label_debug - eval_pred_debug)))
print("-" * 10 + "Debug BatchDirectWeightedL1Loss" + "-" * 10)
print(eval_pred_debug)
print(
workspace.FetchBlob(
'batch_direct_weighted_l1_loss/scaler'))
print(
workspace.FetchBlob(
'batch_direct_weighted_l1_loss/scaler_no_clip'))
print(
workspace.FetchBlob(
'batch_direct_weighted_l1_loss/scaled_loss'))
print(
workspace.FetchBlob(
'batch_direct_weighted_l1_loss/scaled_loss_no_clip'))
print(
workspace.FetchBlob('batch_direct_weighted_l1_loss/loss'))
quit()
num_batch_per_epoch = int(self.input_data_store['train'][1] /
self.batch_size)
if not self.input_data_store['train'][1] % self.batch_size == 0:
num_batch_per_epoch += 1
print('[Warning]: batch_size cannot be divided. ' +
'Run on {} example instead of {}'.format(
num_batch_per_epoch *
self.batch_size, self.input_data_store['train'][1]))
print('<<< Run {} iteration'.format(num_epoch * num_batch_per_epoch))
train_net = self.net_store['train_net']
if report_interval > 0:
print('>>> Training with Reports')
num_eval = int(num_epoch / report_interval)
num_unit_iter = int((num_batch_per_epoch * num_epoch) / num_eval)
if eval_during_training and 'eval_net' in self.net_store:
print('>>> Training with Eval Reports (Slowest mode)')
eval_net = self.net_store['eval_net']
for i in range(num_eval):
print('>>> Done with Iter: ' + str(i * num_unit_iter))
workspace.RunNet(train_net.Proto().name,
num_iter=num_unit_iter)
self.reports['epoch'].append((i + 1) * report_interval)
train_loss = np.asscalar(schema.FetchRecord(self.loss).get())
self.reports['train_loss'].append(train_loss)
print(' train_loss = ' + str(train_loss))
# Add metrics
train_l1_metric = np.asscalar(
schema.FetchRecord(
self.model.metrics_schema.l1_metric).get())
self.reports['train_l1_metric'].append(train_l1_metric)
# print(' train_l1_metric = ' + str(train_l1_metric))
train_scaled_l1_metric = np.asscalar(
schema.FetchRecord(
self.model.metrics_schema.scaled_l1_metric).get())
self.reports['train_scaled_l1_metric'].append(
train_scaled_l1_metric)
if eval_during_training and 'eval_net' in self.net_store:
workspace.RunNet(eval_net.Proto().name)
eval_loss = np.asscalar(
schema.FetchRecord(self.loss).get())
# Add metrics
self.reports['eval_loss'].append(eval_loss)
print(' eval_loss = ' + str(eval_loss))
eval_l1_metric = np.asscalar(
schema.FetchRecord(
self.model.metrics_schema.l1_metric).get())
self.reports['eval_l1_metric'].append(eval_l1_metric)
# print(' eval_l1_metric = ' + str(eval_l1_metric))
eval_scaled_l1_metric = np.asscalar(
schema.FetchRecord(
self.model.metrics_schema.scaled_l1_metric).get())
self.reports['eval_scaled_l1_metric'].append(
eval_scaled_l1_metric)
# Save Net
exporter.save_net(self.net_store['pred_net'], self.model,
self.model_name + '_init',
self.model_name + '_predict')
else:
print('>>> Training without Reports (Fastest mode)')
workspace.RunNet(train_net,
num_iter=num_epoch * num_batch_per_epoch)
print('>>> Saving test model')
# Save Net
exporter.save_net(self.net_store['pred_net'], self.model,
self.model_name + '_init',
self.model_name + '_predict')
# Save Loss Trend
if report_interval > 0:
self.save_loss_trend(self.model_name)
origin_pred, adjoint_pred, loss = build_adjoint_mlp( model, input_dim=input_dim, hidden_dims=hidden_dims, output_dim=output_dim, optim=optimizer.AdagradOptimizer(alpha=0.01, epsilon=1e-4,)) # Train the model train_init_net, train_net = instantiator.generate_training_nets(model) workspace.RunNetOnce(train_init_net) workspace.CreateNet(train_net) num_iter = 10000 eval_num_iter = 1 for i in range(eval_num_iter): workspace.RunNet(train_net.Proto().name, num_iter=num_iter) print(schema.FetchRecord(loss).get()) import matplotlib.pyplot as plt origin_pred_array = np.squeeze(schema.FetchRecord(origin_pred).get()) plt.plot(x_array, np.gradient(origin_pred_array, np.squeeze(x_array)), 'r') plt.plot(x_array, origin_pred_array, 'r') plt.plot(x_array, schema.FetchRecord(adjoint_pred).get(), 'b') plt.plot(x_array, adjoint_label, 'b--') plt.show() # Eval eval_net = instantiator.generate_eval_net(model) graph = net_drawer.GetPydotGraph(eval_net.Proto().op, rankdir='TB') with open(eval_net.Name() + ".png",'wb') as f: f.write(graph.create_png()) # Predict1 # pred_net = instantiator.generate_predict_net(model) # graph = net_drawer.GetPydotGraph(pred_net.Proto().op, rankdir='TB')
