def testSelectRecordByContext(self): float_features = self.model.input_feature_schema.float_features float_array = np.array([1.0, 2.0], dtype=np.float32) schema.FeedRecord(float_features, [float_array]) with Tags(Tags.EXCLUDE_FROM_PREDICTION): log_float_features = self.model.Log(float_features, 1) joined = self.model.SelectRecordByContext( schema.Struct( (InstantiationContext.PREDICTION, float_features), (InstantiationContext.TRAINING, log_float_features), # TODO: TRAIN_ONLY layers are also generated in eval (InstantiationContext.EVAL, log_float_features), )) # model.output_schema has to a struct self.model.output_schema = schema.Struct(('joined', joined)) predict_net = layer_model_instantiator.generate_predict_net(self.model) workspace.RunNetOnce(predict_net) predict_output = schema.FetchRecord(predict_net.output_record()) npt.assert_array_equal(float_array, predict_output['joined']()) eval_net = layer_model_instantiator.generate_eval_net(self.model) workspace.RunNetOnce(eval_net) eval_output = schema.FetchRecord(eval_net.output_record()) npt.assert_array_equal(np.log(float_array), eval_output['joined']()) _, train_net = ( layer_model_instantiator.generate_training_nets_forward_only( self.model)) workspace.RunNetOnce(train_net) train_output = schema.FetchRecord(train_net.output_record()) npt.assert_array_equal(np.log(float_array), train_output['joined']())
def testLastNWindowCollector(self, X, num_to_collect): input_record = self.new_record(schema.Scalar(np.float32)) schema.FeedRecord(input_record, [X]) last_n = self.model.LastNWindowCollector(input_record, num_to_collect) self.run_train_net_forward_only() output_record = schema.FetchRecord(last_n.last_n) start = max(0, 5 - num_to_collect) npt.assert_array_equal(X[start:], output_record()) num_visited = schema.FetchRecord(last_n.num_visited) npt.assert_array_equal([5], num_visited())
def train_with_eval( self, num_epoch=1, report_interval=0, eval_during_training=False, ): ''' Fastest mode: report_interval = 0 Medium mode: report_interval > 0, eval_during_training=False Slowest mode: report_interval > 0, eval_during_training=True ''' num_batch_per_epoch = int(self.input_data_store['train'][1] / self.batch_size) if not self.input_data_store['train'][1] % self.batch_size == 0: num_batch_per_epoch += 1 print('[Warning]: batch_size cannot be divided. ' + 'Run on {} example instead of {}'.format( num_batch_per_epoch * self.batch_size, self.input_data_store['train'][1])) print('<<< Run {} iteration'.format(num_epoch * num_batch_per_epoch)) train_net = self.net_store['train_net'] if report_interval > 0: print('>>> Training with Reports') num_eval = int(num_epoch / report_interval) num_unit_iter = int((num_batch_per_epoch * num_epoch) / num_eval) if eval_during_training and 'eval_net' in self.net_store: print('>>> Training with Eval Reports (Slowest mode)') eval_net = self.net_store['eval_net'] for i in range(num_eval): workspace.RunNet(train_net.Proto().name, num_iter=num_unit_iter) self.reports['epoch'].append((i + 1) * report_interval) train_loss = np.asscalar(schema.FetchRecord(self.loss).get()) self.reports['train_loss'].append(train_loss) if eval_during_training and 'eval_net' in self.net_store: workspace.RunNet(eval_net.Proto().name, num_iter=num_unit_iter) eval_loss = np.asscalar( schema.FetchRecord(self.loss).get()) self.reports['eval_loss'].append(eval_loss) else: print('>>> Training without Reports (Fastest mode)') num_iter = num_epoch * num_batch_per_epoch workspace.RunNet(train_net, num_iter=num_iter) print('>>> Saving test model') exporter.save_net(self.net_store['pred_net'], self.model, self.model_name + '_init', self.model_name + '_predict')
def testConv(self): batch_size = 50 H = 1 W = 10 C = 50 output_dims = 32 kernel_h = 1 kernel_w = 3 stride_h = 1 stride_w = 1 pad_t = 0 pad_b = 0 pad_r = None pad_l = None input_record = self.new_record(schema.Scalar((np.float32, (H, W, C)))) X = np.random.random((batch_size, H, W, C)).astype(np.float32) schema.FeedRecord(input_record, [X]) conv = self.model.Conv(input_record, output_dims, kernel_h=kernel_h, kernel_w=kernel_w, stride_h=stride_h, stride_w=stride_w, pad_t=pad_t, pad_b=pad_b, pad_r=pad_r, pad_l=pad_l, order='NHWC') self.assertEqual(schema.Scalar((np.float32, (output_dims, ))), conv) self.run_train_net_forward_only() output_record = schema.FetchRecord(conv) # check the number of output channels is the same as input in this example assert output_record.field_types()[0].shape == (H, W, output_dims) assert output_record().shape == (batch_size, H, W, output_dims) train_init_net, train_net = self.get_training_nets() # Init net assertions init_ops = self.assertNetContainOps(train_init_net, [ OpSpec("XavierFill", None, None), OpSpec("ConstantFill", None, None), ]) conv_spec = OpSpec("Conv", [ input_record.field_blobs()[0], init_ops[0].output[0], init_ops[1].output[0], ], conv.field_blobs()) # Train net assertions self.assertNetContainOps(train_net, [conv_spec]) # Predict net assertions predict_net = self.get_predict_net() self.assertNetContainOps(predict_net, [conv_spec]) # Eval net assertions eval_net = self.get_eval_net() self.assertNetContainOps(eval_net, [conv_spec])
def predict_ids(self, vg, vd): # preproc the input vg = vg.astype(np.float32) vd = vd.astype(np.float32) if len(self.preproc_param) == 0: self.preproc_param = pickle.load(open(self.pickle_file_name, "rb")) dummy_ids = np.zeros(len(vg)) preproc_data_arrays = preproc.dc_iv_preproc( vg, vd, dummy_ids, self.preproc_param['scale'], self.preproc_param['vg_shift'], ) _preproc_data_arrays = [ np.expand_dims(x, axis=1) for x in preproc_data_arrays ] workspace.FeedBlob('DBInput_train/sig_input', _preproc_data_arrays[0]) workspace.FeedBlob('DBInput_train/tanh_input', _preproc_data_arrays[1]) pred_net = self.net_store['pred_net'] workspace.RunNet(pred_net) _ids = np.squeeze(schema.FetchRecord(self.pred).get()) restore_id_func, _ = preproc.get_restore_id_func( self.preproc_param['scale'], self.preproc_param['vg_shift'], ) ids = restore_id_func(_ids) return _ids, ids
def avg_loss_full_epoch(self, net_name): num_batch_per_epoch = int(self.input_data_store['train'][1] / self.batch_size) if not self.input_data_store['train'][1] % self.batch_size == 0: num_batch_per_epoch += 1 print('[Warning]: batch_size cannot be divided. ' + 'Run on {} example instead of {}'.format( num_batch_per_epoch * self.batch_size, self.input_data_store['train'][1])) # Get the average loss of all data loss = 0. for j in range(num_batch_per_epoch): workspace.RunNet(self.net_store[net_name]) loss += np.asscalar(schema.FetchRecord(self.loss).get()) loss /= num_batch_per_epoch return loss
def predict_ids(self, vg, vd): # preproc the input vg = vg.astype(np.float32) vd = vd.astype(np.float32) if len(self.preproc_param) == 0: self.preproc_param = pickle.load(open(self.pickle_file_name, "rb")) dummy_ids = np.zeros(len(vg)) preproc_data_arrays = [vg, vd, dummy_ids] if self.train_target == TrainTarget.ORIGIN: preproc_data_arrays = preproc.dc_iv_preproc( vg, vd, dummy_ids, self.preproc_param['scale'], self.preproc_param['vg_shift'], ) _preproc_data_arrays = [ np.expand_dims(x, axis=1) if x.ndim == 1 else x for x in preproc_data_arrays ] workspace.FeedBlob('DBInput_train/sig_input', _preproc_data_arrays[0]) workspace.FeedBlob('DBInput_train/tanh_input', _preproc_data_arrays[1]) if self.train_target == TrainTarget.ADJOINT: adjoint_input = np.ones((vg.shape[0], 1)) workspace.FeedBlob('DBInput_train/adjoint_input', adjoint_input) pred_net = self.net_store['pred_net'] workspace.RunNet(pred_net) _ids = np.squeeze(schema.FetchRecord(self.pred).get()) ids = _ids if self.train_target == TrainTarget.ORIGIN: restore_id_func, _ = preproc.get_restore_id_func( self.preproc_param['scale'], self.preproc_param['vg_shift'], ) ids = restore_id_func(_ids) return _ids, ids
), bias_optim=optimizer.AdagradOptimizer( alpha=0.01, epsilon=1e-4, )) # Train the model train_init_net, train_net = instantiator.generate_training_nets(model) workspace.RunNetOnce(train_init_net) workspace.CreateNet(train_net) num_iter = 1000 eval_num_iter = 100 # loss_lst = [] for i in range(eval_num_iter): workspace.RunNet(train_net.Proto().name, num_iter=num_iter) print(schema.FetchRecord(loss).get()) X_pred = np.squeeze( schema.FetchRecord(model.input_feature_schema.sig_input).get()).reshape( X.shape) Y_pred = np.squeeze( schema.FetchRecord(model.input_feature_schema.tanh_input).get()).reshape( X.shape) Z_pred = np.squeeze(schema.FetchRecord(origin_pred).get()).reshape(X.shape) gy_restore, gx_restore = np.gradient(Z_pred, 0.2, 0.2) gx_pred = schema.FetchRecord(sig_adjoint_pred).get().reshape(X.shape) gy_pred = schema.FetchRecord(tanh_adjoint_pred).get().reshape(X.shape) gx_label = schema.FetchRecord( model.trainer_extra_schema.sig_loss_record.label).get().reshape(X.shape) gy_label = schema.FetchRecord( model.trainer_extra_schema.tanh_loss_record.label).get().reshape(X.shape)
def train_with_eval( self, num_epoch=1, report_interval=0, eval_during_training=False, ): ''' Fastest mode: report_interval = 0 Medium mode: report_interval > 0, eval_during_training=False Slowest mode: report_interval > 0, eval_during_training=True Debug mode: DEBUG flag set to true ''' if DEBUG: train_net = self.net_store['train_net'] workspace.RunNet(train_net, num_iter=5000) if self.net_builder == TrainTarget.ORIGIN: print(workspace.FetchBlob('DBInput_train/tanh_input')) print(workspace.FetchBlob('DBInput_train/sig_input')) print(workspace.FetchBlob('DBInput_train/label')) print(workspace.FetchBlob('prediction')) # print(workspace.FetchBlob('Sigmoid_auto_1/sig_tranfer_layer_2')) # print(workspace.FetchBlob('Tanh_auto_1/tanh_tranfer_layer_2')) # print(workspace.FetchBlob('Sigmoid/sig_tranfer_layer_0')) # print(workspace.FetchBlob('sig_fc_layer_0/w')) # print(workspace.FetchBlob('sig_fc_layer_0/b')) if self.net_builder == TrainTarget.ADJOINT: print(workspace.FetchBlob('DBInput_train/tanh_input')) print(workspace.FetchBlob('DBInput_train/sig_input')) print(workspace.FetchBlob('DBInput_train/label')) print(workspace.FetchBlob('origin/' + 'Mul/origin_pred')) print( workspace.FetchBlob('adjoint/' + 'Sigmoid_auto_1/sig_tranfer_layer_2')) print( workspace.FetchBlob('origin/' + 'Tanh_auto_1/tanh_tranfer_layer_2')) print( workspace.FetchBlob('origin/' + 'Sigmoid/sig_tranfer_layer_0')) print(workspace.FetchBlob('adjoint/' + 'sig_fc_layer_0/w')) print(workspace.FetchBlob('adjoint/' + 'sig_fc_layer_0/b')) print('-' * 50) eval_net = self.net_store['eval_net'] workspace.RunNet(eval_net.Proto().name) if self.net_builder == TrainTarget.ORIGIN: print(workspace.FetchBlob('DBInput_eval/eval_sig_input')) print(workspace.FetchBlob('DBInput_eval/eval_sig_input')) eval_label_debug = workspace.FetchBlob( 'DBInput_eval/eval_label') eval_pred_debug = workspace.FetchBlob('prediction') print( workspace.FetchBlob( 'batch_direct_weighted_l1_loss/l1_metric')) print(np.average(np.abs(eval_label_debug - eval_pred_debug))) print("-" * 10 + "Debug BatchDirectWeightedL1Loss" + "-" * 10) print(eval_pred_debug) print( workspace.FetchBlob( 'batch_direct_weighted_l1_loss/scaler')) print( workspace.FetchBlob( 'batch_direct_weighted_l1_loss/scaler_no_clip')) print( workspace.FetchBlob( 'batch_direct_weighted_l1_loss/scaled_loss')) print( workspace.FetchBlob( 'batch_direct_weighted_l1_loss/scaled_loss_no_clip')) print( workspace.FetchBlob('batch_direct_weighted_l1_loss/loss')) quit() num_batch_per_epoch = int(self.input_data_store['train'][1] / self.batch_size) if not self.input_data_store['train'][1] % self.batch_size == 0: num_batch_per_epoch += 1 print('[Warning]: batch_size cannot be divided. ' + 'Run on {} example instead of {}'.format( num_batch_per_epoch * self.batch_size, self.input_data_store['train'][1])) print('<<< Run {} iteration'.format(num_epoch * num_batch_per_epoch)) train_net = self.net_store['train_net'] if report_interval > 0: print('>>> Training with Reports') num_eval = int(num_epoch / report_interval) num_unit_iter = int((num_batch_per_epoch * num_epoch) / num_eval) if eval_during_training and 'eval_net' in self.net_store: print('>>> Training with Eval Reports (Slowest mode)') eval_net = self.net_store['eval_net'] for i in range(num_eval): print('>>> Done with Iter: ' + str(i * num_unit_iter)) workspace.RunNet(train_net.Proto().name, num_iter=num_unit_iter) self.reports['epoch'].append((i + 1) * report_interval) train_loss = np.asscalar(schema.FetchRecord(self.loss).get()) self.reports['train_loss'].append(train_loss) print(' train_loss = ' + str(train_loss)) # Add metrics train_l1_metric = np.asscalar( schema.FetchRecord( self.model.metrics_schema.l1_metric).get()) self.reports['train_l1_metric'].append(train_l1_metric) # print(' train_l1_metric = ' + str(train_l1_metric)) train_scaled_l1_metric = np.asscalar( schema.FetchRecord( self.model.metrics_schema.scaled_l1_metric).get()) self.reports['train_scaled_l1_metric'].append( train_scaled_l1_metric) if eval_during_training and 'eval_net' in self.net_store: workspace.RunNet(eval_net.Proto().name) eval_loss = np.asscalar( schema.FetchRecord(self.loss).get()) # Add metrics self.reports['eval_loss'].append(eval_loss) print(' eval_loss = ' + str(eval_loss)) eval_l1_metric = np.asscalar( schema.FetchRecord( self.model.metrics_schema.l1_metric).get()) self.reports['eval_l1_metric'].append(eval_l1_metric) # print(' eval_l1_metric = ' + str(eval_l1_metric)) eval_scaled_l1_metric = np.asscalar( schema.FetchRecord( self.model.metrics_schema.scaled_l1_metric).get()) self.reports['eval_scaled_l1_metric'].append( eval_scaled_l1_metric) # Save Net exporter.save_net(self.net_store['pred_net'], self.model, self.model_name + '_init', self.model_name + '_predict') else: print('>>> Training without Reports (Fastest mode)') workspace.RunNet(train_net, num_iter=num_epoch * num_batch_per_epoch) print('>>> Saving test model') # Save Net exporter.save_net(self.net_store['pred_net'], self.model, self.model_name + '_init', self.model_name + '_predict') # Save Loss Trend if report_interval > 0: self.save_loss_trend(self.model_name)
origin_pred, adjoint_pred, loss = build_adjoint_mlp( model, input_dim=input_dim, hidden_dims=hidden_dims, output_dim=output_dim, optim=optimizer.AdagradOptimizer(alpha=0.01, epsilon=1e-4,)) # Train the model train_init_net, train_net = instantiator.generate_training_nets(model) workspace.RunNetOnce(train_init_net) workspace.CreateNet(train_net) num_iter = 10000 eval_num_iter = 1 for i in range(eval_num_iter): workspace.RunNet(train_net.Proto().name, num_iter=num_iter) print(schema.FetchRecord(loss).get()) import matplotlib.pyplot as plt origin_pred_array = np.squeeze(schema.FetchRecord(origin_pred).get()) plt.plot(x_array, np.gradient(origin_pred_array, np.squeeze(x_array)), 'r') plt.plot(x_array, origin_pred_array, 'r') plt.plot(x_array, schema.FetchRecord(adjoint_pred).get(), 'b') plt.plot(x_array, adjoint_label, 'b--') plt.show() # Eval eval_net = instantiator.generate_eval_net(model) graph = net_drawer.GetPydotGraph(eval_net.Proto().op, rankdir='TB') with open(eval_net.Name() + ".png",'wb') as f: f.write(graph.create_png()) # Predict1 # pred_net = instantiator.generate_predict_net(model) # graph = net_drawer.GetPydotGraph(pred_net.Proto().op, rankdir='TB')