def testFunctionalLayer(self): def normalize(net, in_record, out_record): mean = net.ReduceFrontMean(in_record(), 1) net.Sub( [in_record(), mean], out_record(), broadcast=1) normalized = self.model.Functional( self.model.input_feature_schema.float_features, 1, normalize, name="normalizer") # Attach metadata to one of the outputs and use it in FC normalized.set_type((np.float32, 32)) self.model.output_schema = self.model.FC(normalized, 2) predict_net = layer_model_instantiator.generate_predict_net( self.model) ops = predict_net.Proto().op assert len(ops) == 3 assert ops[0].type == "ReduceFrontMean" assert ops[1].type == "Sub" assert ops[2].type == "FC" assert len(ops[0].input) == 1 assert ops[0].input[0] ==\ self.model.input_feature_schema.float_features() assert len(ops[1].output) == 1 assert ops[1].output[0] in ops[2].input
def testFunctionalLayer(self): def normalize(net, in_record, out_record): mean = net.ReduceFrontMean(in_record(), 1) net.Sub([in_record(), mean], out_record[0](), broadcast=1) normalized = self.model.Functional( self.model.input_feature_schema.float_features, 1, normalize, name="normalizer") # Attach metadata to one of the outputs and use it in FC normalized[0].set_type((np.float32, 32)) self.model.FC(normalized[0], 2) predict_net = layer_model_instantiator.generate_predict_net(self.model) ops = predict_net.Proto().op assert len(ops) == 3 assert ops[0].type == "ReduceFrontMean" assert ops[1].type == "Sub" assert ops[2].type == "FC" assert len(ops[0].input) == 1 assert ops[0].input[0] ==\ self.model.input_feature_schema.float_features() assert len(ops[1].output) == 1 assert ops[1].output[0] in ops[2].input
def testSelectRecordByContext(self): float_features = self.model.input_feature_schema.float_features float_array = np.array([1.0, 2.0], dtype=np.float32) schema.FeedRecord(float_features, [float_array]) with Tags(Tags.EXCLUDE_FROM_PREDICTION): log_float_features = self.model.Log(float_features, 1) joined = self.model.SelectRecordByContext( schema.Struct( (InstantiationContext.PREDICTION, float_features), (InstantiationContext.TRAINING, log_float_features), # TODO: TRAIN_ONLY layers are also generated in eval (InstantiationContext.EVAL, log_float_features), )) # model.output_schema has to a struct self.model.output_schema = schema.Struct(('joined', joined)) predict_net = layer_model_instantiator.generate_predict_net(self.model) workspace.RunNetOnce(predict_net) predict_output = schema.FetchRecord(predict_net.output_record()) npt.assert_array_equal(float_array, predict_output['joined']()) eval_net = layer_model_instantiator.generate_eval_net(self.model) workspace.RunNetOnce(eval_net) eval_output = schema.FetchRecord(eval_net.output_record()) npt.assert_array_equal(np.log(float_array), eval_output['joined']()) _, train_net = ( layer_model_instantiator.generate_training_nets_forward_only( self.model)) workspace.RunNetOnce(train_net) train_output = schema.FetchRecord(train_net.output_record()) npt.assert_array_equal(np.log(float_array), train_output['joined']())
def testFunctionalLayerHelperAutoInference(self): softsign = self.model.Softsign( schema.Tuple(self.model.input_feature_schema.float_features), 1) assert softsign.field_type().base == np.float32 assert softsign.field_type().shape == (32, ) self.model.output_schema = self.model.FC(softsign, 2) predict_net = layer_model_instantiator.generate_predict_net(self.model) ops = predict_net.Proto().op assert len(ops) == 2 assert ops[0].type == "Softsign" assert ops[1].type == "FC" assert len(ops[0].input) == 1 assert ops[0].input[0] ==\ self.model.input_feature_schema.float_features() assert len(ops[0].output) == 1 assert ops[0].output[0] in ops[1].input
def testFunctionalLayerHelperAutoInference(self): softsign = self.model.Softsign( schema.Tuple(self.model.input_feature_schema.float_features), 1) assert softsign.field_type().base == np.float32 assert softsign.field_type().shape == (32,) self.model.output_schema = self.model.FC(softsign, 2) predict_net = layer_model_instantiator.generate_predict_net( self.model) ops = predict_net.Proto().op assert len(ops) == 2 assert ops[0].type == "Softsign" assert ops[1].type == "FC" assert len(ops[0].input) == 1 assert ops[0].input[0] ==\ self.model.input_feature_schema.float_features() assert len(ops[0].output) == 1 assert ops[0].output[0] in ops[1].input
def testSelectRecordByContext(self): float_features = self.model.input_feature_schema.float_features float_array = np.array([1.0, 2.0], dtype=np.float32) schema.FeedRecord(float_features, [float_array]) with Tags(Tags.EXCLUDE_FROM_PREDICTION): log_float_features = self.model.Log(float_features, 1) joined = self.model.SelectRecordByContext( schema.Struct( (InstantiationContext.PREDICTION, float_features), (InstantiationContext.TRAINING, log_float_features), # TODO: TRAIN_ONLY layers are also generated in eval (InstantiationContext.EVAL, log_float_features), ) ) # model.output_schema has to a struct self.model.output_schema = schema.Struct(( 'joined', joined )) predict_net = layer_model_instantiator.generate_predict_net(self.model) workspace.RunNetOnce(predict_net) predict_output = schema.FetchRecord(predict_net.output_record()) npt.assert_array_equal(float_array, predict_output['joined']()) eval_net = layer_model_instantiator.generate_eval_net(self.model) workspace.RunNetOnce(eval_net) eval_output = schema.FetchRecord(eval_net.output_record()) npt.assert_array_equal(np.log(float_array), eval_output['joined']()) _, train_net = ( layer_model_instantiator.generate_training_nets_forward_only( self.model ) ) workspace.RunNetOnce(train_net) train_output = schema.FetchRecord(train_net.output_record()) npt.assert_array_equal(np.log(float_array), train_output['joined']())
def testFunctionalLayerHelper(self): mean = self.model.ReduceFrontMean( self.model.input_feature_schema.float_features, 1) normalized = self.model.Sub(schema.Tuple( self.model.input_feature_schema.float_features, mean[0]), 1, broadcast=1) # Attach metadata to one of the outputs and use it in FC normalized[0].set_type((np.float32, (32, ))) self.model.FC(normalized[0], 2) predict_net = layer_model_instantiator.generate_predict_net(self.model) ops = predict_net.Proto().op assert len(ops) == 3 assert ops[0].type == "ReduceFrontMean" assert ops[1].type == "Sub" assert ops[2].type == "FC" assert len(ops[0].input) == 1 assert ops[0].input[0] ==\ self.model.input_feature_schema.float_features() assert len(ops[1].output) == 1 assert ops[1].output[0] in ops[2].input
def testFunctionalLayerHelper(self): mean = self.model.ReduceFrontMean( self.model.input_feature_schema.float_features, 1) normalized = self.model.Sub( schema.Tuple( self.model.input_feature_schema.float_features, mean), 1, broadcast=1) # Attach metadata to one of the outputs and use it in FC normalized.set_type((np.float32, (32,))) self.model.output_schema = self.model.FC(normalized, 2) predict_net = layer_model_instantiator.generate_predict_net( self.model) ops = predict_net.Proto().op assert len(ops) == 3 assert ops[0].type == "ReduceFrontMean" assert ops[1].type == "Sub" assert ops[2].type == "FC" assert len(ops[0].input) == 1 assert ops[0].input[0] ==\ self.model.input_feature_schema.float_features() assert len(ops[1].output) == 1 assert ops[1].output[0] in ops[2].input
def build_nets( self, hidden_sig_dims, hidden_tanh_dims, train_batch_size=1, eval_batch_size=1, weight_optim_method='AdaGrad', weight_optim_param={ 'alpha': 0.01, 'epsilon': 1e-4 }, bias_optim_method='AdaGrad', bias_optim_param={ 'alpha': 0.01, 'epsilon': 1e-4 }, loss_function='scaled_l1', max_loss_scale=1e6, ): assert len(self.input_data_store) > 0, 'Input data store is empty.' assert 'train' in self.input_data_store, 'Missing training data.' self.batch_size = train_batch_size # Build the date reader net for train net input_data_train = data_reader.build_input_reader( self.model, self.input_data_store['train'][0], 'minidb', ['sig_input', 'tanh_input', 'label'], batch_size=train_batch_size, data_type='train', ) if 'eval' in self.input_data_store: # Build the data reader net for eval net input_data_eval = data_reader.build_input_reader( self.model, self.input_data_store['eval'][0], 'minidb', ['eval_sig_input', 'eval_tanh_input', 'eval_label'], batch_size=eval_batch_size, data_type='eval', ) # Build the computational nets # Create train net self.model.input_feature_schema.sig_input.set_value( input_data_train[0].get(), unsafe=True) self.model.input_feature_schema.tanh_input.set_value( input_data_train[1].get(), unsafe=True) self.model.trainer_extra_schema.label.set_value( input_data_train[2].get(), unsafe=True) self.pred, self.loss = build_pinn( self.model, sig_net_dim=hidden_sig_dims, tanh_net_dim=hidden_tanh_dims, weight_optim=_build_optimizer(weight_optim_method, weight_optim_param), bias_optim=_build_optimizer(bias_optim_method, bias_optim_param), loss_function=loss_function, max_loss_scale=max_loss_scale) train_init_net, train_net = instantiator.generate_training_nets( self.model) workspace.RunNetOnce(train_init_net) workspace.CreateNet(train_net) self.net_store['train_net'] = train_net pred_net = instantiator.generate_predict_net(self.model) workspace.CreateNet(pred_net) self.net_store['pred_net'] = pred_net if 'eval' in self.input_data_store: # Create eval net self.model.input_feature_schema.sig_input.set_value( input_data_eval[0].get(), unsafe=True) self.model.input_feature_schema.tanh_input.set_value( input_data_eval[1].get(), unsafe=True) self.model.trainer_extra_schema.label.set_value( input_data_eval[2].get(), unsafe=True) eval_net = instantiator.generate_eval_net(self.model) workspace.CreateNet(eval_net) self.net_store['eval_net'] = eval_net
def build_nets( self, hidden_sig_dims, hidden_tanh_dims, train_batch_size=1, eval_batch_size=1, weight_optim_method='AdaGrad', weight_optim_param={ 'alpha': 0.01, 'epsilon': 1e-4 }, bias_optim_method='AdaGrad', bias_optim_param={ 'alpha': 0.01, 'epsilon': 1e-4 }, loss_function='scaled_l1', max_loss_scale=1., ): assert len(self.input_data_store) > 0, 'Input data store is empty.' assert 'train' in self.input_data_store, 'Missing training data.' self.batch_size = train_batch_size # Build the date reader net for train net if self.train_target == TrainTarget.ORIGIN: input_data_train = data_reader.build_input_reader( self.model, self.input_data_store['train'][0], 'minidb', ['sig_input', 'tanh_input', 'label'], batch_size=train_batch_size, data_type='train', ) if 'eval' in self.input_data_store: # Build the data reader net for eval net input_data_eval = data_reader.build_input_reader( self.model, self.input_data_store['eval'][0], 'minidb', ['eval_sig_input', 'eval_tanh_input', 'eval_label'], batch_size=eval_batch_size, data_type='eval', ) if self.net_builder == TrainTarget.ADJOINT: # Use Adjoint net so output adjoint net # for training origin, use origin_loss_record self.model.trainer_extra_schema.origin_loss_record.label.set_value( input_data_train[2].get(), unsafe=True) elif self.net_builder == TrainTarget.ORIGIN: self.model.trainer_extra_schema.label.set_value( input_data_train[2].get(), unsafe=True) if self.train_target == TrainTarget.ADJOINT: raise Exception('Not Implemented') # Build the computational nets # Create train net self.model.input_feature_schema.sig_input.set_value( input_data_train[0].get(), unsafe=True) self.model.input_feature_schema.tanh_input.set_value( input_data_train[1].get(), unsafe=True) if self.net_builder == TrainTarget.ADJOINT: (self.pred, self.sig_adjoint_pred, self.tanh_adjoint_pred, self.loss) = build_adjoint_pinn( self.model, sig_input_dim=self.sig_input_dim, tanh_input_dim=self.tanh_input_dim, sig_net_dim=hidden_sig_dims, tanh_net_dim=hidden_tanh_dims, weight_optim=_build_optimizer(weight_optim_method, weight_optim_param), bias_optim=_build_optimizer(bias_optim_method, bias_optim_param), adjoint_tag=self.adjoint_tag, train_target=self.train_target, loss_function=loss_function, max_loss_scale=max_loss_scale, ) elif self.net_builder == TrainTarget.ORIGIN: self.pred, self.loss = build_pinn( self.model, sig_net_dim=hidden_sig_dims, tanh_net_dim=hidden_tanh_dims, weight_optim=_build_optimizer(weight_optim_method, weight_optim_param), bias_optim=_build_optimizer(bias_optim_method, bias_optim_param), loss_function=loss_function, max_loss_scale=max_loss_scale, ) train_init_net, train_net = instantiator.generate_training_nets( self.model) workspace.RunNetOnce(train_init_net) workspace.CreateNet(train_net) self.net_store['train_net'] = train_net pred_net = instantiator.generate_predict_net(self.model) workspace.CreateNet(pred_net) self.net_store['pred_net'] = pred_net if 'eval' in self.input_data_store: # Create eval net self.model.input_feature_schema.sig_input.set_value( input_data_eval[0].get(), unsafe=True) self.model.input_feature_schema.tanh_input.set_value( input_data_eval[1].get(), unsafe=True) if self.train_target == TrainTarget.ORIGIN: if self.net_builder == TrainTarget.ADJOINT: self.model.trainer_extra_schema.origin_loss_record.label.set_value( input_data_eval[2].get(), unsafe=True) elif self.net_builder == TrainTarget.ORIGIN: self.model.trainer_extra_schema.label.set_value( input_data_eval[2].get(), unsafe=True) if self.train_target == TrainTarget.ADJOINT: raise Exception('Not Implemented') eval_net = instantiator.generate_eval_net(self.model) workspace.CreateNet(eval_net) self.net_store['eval_net'] = eval_net
def get_predict_net(self): return layer_model_instantiator.generate_predict_net(self.model)
def build_nets( self, hidden_sig_dims, hidden_tanh_dims, train_batch_size=1, eval_batch_size=1, weight_optim_method='AdaGrad', weight_optim_param={ 'alpha': 0.01, 'epsilon': 1e-4 }, bias_optim_method='AdaGrad', bias_optim_param={ 'alpha': 0.01, 'epsilon': 1e-4 }, loss_function='scaled_l1', max_loss_scale=1., # used to scale up the loss signal for small input neg_grad_penalty=None, # whether and how to apply neg_grad_penalty init_model=None, # do postfix matching i.e. adjoint/<blob_nanme> == <blob_nanme> ): assert len(self.input_data_store) > 0, 'Input data store is empty.' assert 'train' in self.input_data_store, 'Missing training data.' assert (neg_grad_penalty is None or (neg_grad_penalty and self.train_target == TrainTarget.ORIGIN and self.net_builder == TrainTarget.ADJOINT) ), '''When set neg_grad_penalty, train target should be ORIGIN, but net builder should be ADJOINT''' self.has_neg_grad_penalty = True if neg_grad_penalty else False self.batch_size = train_batch_size # Build the date reader net for train net if self.train_target == TrainTarget.ORIGIN: input_data_train = data_reader.build_input_reader( self.model, self.input_data_store['train'][0], 'minidb', ['sig_input', 'tanh_input', 'label'], batch_size=train_batch_size, data_type='train', ) if 'eval' in self.input_data_store: # Build the data reader net for eval net input_data_eval = data_reader.build_input_reader( self.model, self.input_data_store['eval'][0], 'minidb', ['eval_sig_input', 'eval_tanh_input', 'eval_label'], batch_size=eval_batch_size, data_type='eval', ) if self.net_builder == TrainTarget.ADJOINT: # Use Adjoint net so output adjoint net # for training origin, use origin_loss_record self.model.trainer_extra_schema.origin_loss_record.label.set_value( input_data_train[2].get(), unsafe=True) elif self.net_builder == TrainTarget.ORIGIN: self.model.trainer_extra_schema.label.set_value( input_data_train[2].get(), unsafe=True) if self.train_target == TrainTarget.ADJOINT: raise Exception('Not Implemented') # Build the computational nets # Create train net self.model.input_feature_schema.sig_input.set_value( input_data_train[0].get(), unsafe=True) self.model.input_feature_schema.tanh_input.set_value( input_data_train[1].get(), unsafe=True) if self.net_builder == TrainTarget.ADJOINT: # decide adjoint tag adjoint_tag = 'no_tag' if self.train_target == TrainTarget.ORIGIN and neg_grad_penalty is None: adjoint_tag = Tags.PREDICTION_ONLY (self.pred, self.sig_adjoint_pred, self.tanh_adjoint_pred, self.loss) = build_adjoint_pinn( self.model, sig_input_dim=self.sig_input_dim, tanh_input_dim=self.tanh_input_dim, sig_net_dim=hidden_sig_dims, tanh_net_dim=hidden_tanh_dims, weight_optim=_build_optimizer(weight_optim_method, weight_optim_param), bias_optim=_build_optimizer(bias_optim_method, bias_optim_param), adjoint_tag=adjoint_tag, train_target=self.train_target, loss_function=loss_function, max_loss_scale=max_loss_scale, neg_grad_penalty=neg_grad_penalty, ) elif self.net_builder == TrainTarget.ORIGIN: self.pred, self.loss = build_pinn( self.model, sig_net_dim=hidden_sig_dims, tanh_net_dim=hidden_tanh_dims, weight_optim=_build_optimizer(weight_optim_method, weight_optim_param), bias_optim=_build_optimizer(bias_optim_method, bias_optim_param), loss_function=loss_function, max_loss_scale=max_loss_scale, ) train_init_net, train_net = instantiator.generate_training_nets( self.model) workspace.RunNetOnce(train_init_net) if init_model: model_name = init_model['name'] print('[INFO] Init params from ' + model_name) given_init_net = exporter.load_init_net(model_name) if 'prefix' in init_model.keys(): print('[INFO] Append ' + init_model['prefix'] + ' to all blob names.') for op in given_init_net.op: op.output[0] = init_model['prefix'] + op.output[0] workspace.RunNetOnce(given_init_net) workspace.CreateNet(train_net) self.net_store['train_net'] = train_net pred_net = instantiator.generate_predict_net(self.model) workspace.CreateNet(pred_net) self.net_store['pred_net'] = pred_net if 'eval' in self.input_data_store: # Create eval net self.model.input_feature_schema.sig_input.set_value( input_data_eval[0].get(), unsafe=True) self.model.input_feature_schema.tanh_input.set_value( input_data_eval[1].get(), unsafe=True) if self.train_target == TrainTarget.ORIGIN: if self.net_builder == TrainTarget.ADJOINT: self.model.trainer_extra_schema.origin_loss_record.label.set_value( input_data_eval[2].get(), unsafe=True) elif self.net_builder == TrainTarget.ORIGIN: self.model.trainer_extra_schema.label.set_value( input_data_eval[2].get(), unsafe=True) if self.train_target == TrainTarget.ADJOINT: raise Exception('Not Implemented') eval_net = instantiator.generate_eval_net(self.model) workspace.CreateNet(eval_net) self.net_store['eval_net'] = eval_net
def get_predict_net(self): return layer_model_instantiator.generate_predict_net(self.model)
def build_nets( self, hidden_dims, batch_size=1, optim_method='AdaGrad', optim_param={ 'alpha': 0.01, 'epsilon': 1e-4 }, ): assert len(self.input_data_store) > 0, 'Input data store is empty.' assert 'train' in self.input_data_store, 'Missing training data.' self.batch_size = batch_size # Build the date reader net for train net input_data_train = data_reader.build_input_reader( self.model, self.input_data_store['train'][0], 'minidb', ['origin_input', 'adjoint_input', 'label'], batch_size=batch_size, data_type='train', ) if 'eval' in self.input_data_store: # Build the data reader net for eval net input_data_eval = data_reader.build_input_reader( self.model, self.input_data_store['eval'][0], 'minidb', ['origin_input', 'adjoint_input', 'label'], batch_size=batch_size, data_type='eval', ) # Build the computational nets # Create train net self.model.input_feature_schema.origin_input.set_value( input_data_train[0].get(), unsafe=True) self.model.input_feature_schema.adjoint_input.set_value( input_data_train[1].get(), unsafe=True) self.model.trainer_extra_schema.label.set_value( input_data_train[2].get(), unsafe=True) self.origin_pred, self.adjoint_pred, self.loss = build_adjoint_mlp( self.model, input_dim=self.input_dim, hidden_dims=hidden_dims, output_dim=self.output_dim, optim=_build_optimizer(optim_method, optim_param), ) train_init_net, train_net = instantiator.generate_training_nets( self.model) workspace.RunNetOnce(train_init_net) workspace.CreateNet(train_net) self.net_store['train_net'] = train_net pred_net = instantiator.generate_predict_net(self.model) workspace.CreateNet(pred_net) self.net_store['pred_net'] = pred_net if 'eval' in self.input_data_store: # Create eval net self.model.input_feature_schema.origin_input.set_value( input_data_eval[0].get(), unsafe=True) self.model.input_feature_schema.adjoint_input.set_value( input_data_eval[1].get(), unsafe=True) self.model.trainer_extra_schema.label.set_value( input_data_eval[2].get(), unsafe=True) eval_net = instantiator.generate_eval_net(self.model) workspace.CreateNet(eval_net) self.net_store['eval_net'] = eval_net