def test_regularizer_context(self, X):
weight_reg_out = L1Norm(0.2)
bias_reg_out = L1Norm(0)
regularizers = {
'WEIGHT': weight_reg_out,
'BIAS': bias_reg_out
}
output_dims = 2
input_record = self.new_record(schema.Scalar((np.float32, (5,))))
schema.FeedRecord(input_record, [X])
with UseRegularizer(regularizers):
weight_reg = RegularizerContext.current().get_regularizer('WEIGHT')
bias_reg = RegularizerContext.current().get_regularizer('BIAS')
optim = SgdOptimizer(0.15)
assert weight_reg == weight_reg_out, \
'fail to get correct weight reg from context'
assert bias_reg == bias_reg_out, \
'fail to get correct bias reg from context'
fc_output = self.model.FC(
input_record,
output_dims,
weight_optim=optim,
bias_optim=optim,
weight_reg=weight_reg,
bias_reg=bias_reg
)
# model.output_schema has to a struct
self.model.output_schema = schema.Struct((
'fc_output', fc_output
))
self.assertEqual(
schema.Scalar((np.float32, (output_dims, ))),
fc_output
)
_, train_net = layer_model_instantiator.generate_training_nets(self.model)
ops = train_net.Proto().op
ops_type_list = [ops[i].type for i in range(len(ops))]
assert ops_type_list.count('LpNorm') == 2
assert ops_type_list.count('Scale') == 4
assert ops_type_list.count('LpNormGradient') == 2
def test_regularizer_context(self, X):
weight_reg_out = L1Norm(0.2)
bias_reg_out = L1Norm(0)
regularizers = {"WEIGHT": weight_reg_out, "BIAS": bias_reg_out}
output_dims = 2
input_record = self.new_record(schema.Scalar((np.float32, (5, ))))
schema.FeedRecord(input_record, [X])
with UseRegularizer(regularizers):
weight_reg = RegularizerContext.current().get_regularizer("WEIGHT")
bias_reg = RegularizerContext.current().get_regularizer("BIAS")
optim = SgdOptimizer(0.15)
assert (weight_reg == weight_reg_out
), "fail to get correct weight reg from context"
assert bias_reg == bias_reg_out, "fail to get correct bias reg from context"
fc_output = self.model.FC(
input_record,
output_dims,
weight_optim=optim,
bias_optim=optim,
weight_reg=weight_reg,
bias_reg=bias_reg,
)
# model.output_schema has to a struct
self.model.output_schema = schema.Struct(("fc_output", fc_output))
self.assertEqual(schema.Scalar((np.float32, (output_dims, ))),
fc_output)
_, train_net = layer_model_instantiator.generate_training_nets(
self.model)
ops = train_net.Proto().op
ops_type_list = [ops[i].type for i in range(len(ops))]
assert ops_type_list.count("LpNorm") == 2
assert ops_type_list.count("Scale") == 4
assert ops_type_list.count("LpNormGradient") == 2
# Build model
(origin_pred, sig_adjoint_pred, tanh_adjoint_pred,
loss) = build_adjoint_pinn(model,
sig_net_dim=[10, 1],
tanh_net_dim=[10, 1],
weight_optim=optimizer.AdagradOptimizer(
alpha=0.01,
epsilon=1e-4,
),
bias_optim=optimizer.AdagradOptimizer(
alpha=0.01,
epsilon=1e-4,
))
# Train the model
train_init_net, train_net = instantiator.generate_training_nets(model)
workspace.RunNetOnce(train_init_net)
workspace.CreateNet(train_net)
num_iter = 1000
eval_num_iter = 100
# loss_lst = []
for i in range(eval_num_iter):
workspace.RunNet(train_net.Proto().name, num_iter=num_iter)
print(schema.FetchRecord(loss).get())
X_pred = np.squeeze(
schema.FetchRecord(model.input_feature_schema.sig_input).get()).reshape(
X.shape)
Y_pred = np.squeeze(
schema.FetchRecord(model.input_feature_schema.tanh_input).get()).reshape(
X.shape)
def build_nets(
self,
hidden_sig_dims,
hidden_tanh_dims,
train_batch_size=1,
eval_batch_size=1,
weight_optim_method='AdaGrad',
weight_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
bias_optim_method='AdaGrad',
bias_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
loss_function='scaled_l1',
max_loss_scale=1e6,
):
assert len(self.input_data_store) > 0, 'Input data store is empty.'
assert 'train' in self.input_data_store, 'Missing training data.'
self.batch_size = train_batch_size
# Build the date reader net for train net
input_data_train = data_reader.build_input_reader(
self.model,
self.input_data_store['train'][0],
'minidb',
['sig_input', 'tanh_input', 'label'],
batch_size=train_batch_size,
data_type='train',
)
if 'eval' in self.input_data_store:
# Build the data reader net for eval net
input_data_eval = data_reader.build_input_reader(
self.model,
self.input_data_store['eval'][0],
'minidb',
['eval_sig_input', 'eval_tanh_input', 'eval_label'],
batch_size=eval_batch_size,
data_type='eval',
)
# Build the computational nets
# Create train net
self.model.input_feature_schema.sig_input.set_value(
input_data_train[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_train[1].get(), unsafe=True)
self.model.trainer_extra_schema.label.set_value(
input_data_train[2].get(), unsafe=True)
self.pred, self.loss = build_pinn(
self.model,
sig_net_dim=hidden_sig_dims,
tanh_net_dim=hidden_tanh_dims,
weight_optim=_build_optimizer(weight_optim_method,
weight_optim_param),
bias_optim=_build_optimizer(bias_optim_method, bias_optim_param),
loss_function=loss_function,
max_loss_scale=max_loss_scale)
train_init_net, train_net = instantiator.generate_training_nets(
self.model)
workspace.RunNetOnce(train_init_net)
workspace.CreateNet(train_net)
self.net_store['train_net'] = train_net
pred_net = instantiator.generate_predict_net(self.model)
workspace.CreateNet(pred_net)
self.net_store['pred_net'] = pred_net
if 'eval' in self.input_data_store:
# Create eval net
self.model.input_feature_schema.sig_input.set_value(
input_data_eval[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_eval[1].get(), unsafe=True)
self.model.trainer_extra_schema.label.set_value(
input_data_eval[2].get(), unsafe=True)
eval_net = instantiator.generate_eval_net(self.model)
workspace.CreateNet(eval_net)
self.net_store['eval_net'] = eval_net
def build_nets(
self,
hidden_sig_dims,
hidden_tanh_dims,
train_batch_size=1,
eval_batch_size=1,
weight_optim_method='AdaGrad',
weight_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
bias_optim_method='AdaGrad',
bias_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
loss_function='scaled_l1',
max_loss_scale=1.,
):
assert len(self.input_data_store) > 0, 'Input data store is empty.'
assert 'train' in self.input_data_store, 'Missing training data.'
self.batch_size = train_batch_size
# Build the date reader net for train net
if self.train_target == TrainTarget.ORIGIN:
input_data_train = data_reader.build_input_reader(
self.model,
self.input_data_store['train'][0],
'minidb',
['sig_input', 'tanh_input', 'label'],
batch_size=train_batch_size,
data_type='train',
)
if 'eval' in self.input_data_store:
# Build the data reader net for eval net
input_data_eval = data_reader.build_input_reader(
self.model,
self.input_data_store['eval'][0],
'minidb',
['eval_sig_input', 'eval_tanh_input', 'eval_label'],
batch_size=eval_batch_size,
data_type='eval',
)
if self.net_builder == TrainTarget.ADJOINT: # Use Adjoint net so output adjoint net
# for training origin, use origin_loss_record
self.model.trainer_extra_schema.origin_loss_record.label.set_value(
input_data_train[2].get(), unsafe=True)
elif self.net_builder == TrainTarget.ORIGIN:
self.model.trainer_extra_schema.label.set_value(
input_data_train[2].get(), unsafe=True)
if self.train_target == TrainTarget.ADJOINT:
raise Exception('Not Implemented')
# Build the computational nets
# Create train net
self.model.input_feature_schema.sig_input.set_value(
input_data_train[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_train[1].get(), unsafe=True)
if self.net_builder == TrainTarget.ADJOINT:
(self.pred, self.sig_adjoint_pred, self.tanh_adjoint_pred,
self.loss) = build_adjoint_pinn(
self.model,
sig_input_dim=self.sig_input_dim,
tanh_input_dim=self.tanh_input_dim,
sig_net_dim=hidden_sig_dims,
tanh_net_dim=hidden_tanh_dims,
weight_optim=_build_optimizer(weight_optim_method,
weight_optim_param),
bias_optim=_build_optimizer(bias_optim_method,
bias_optim_param),
adjoint_tag=self.adjoint_tag,
train_target=self.train_target,
loss_function=loss_function,
max_loss_scale=max_loss_scale,
)
elif self.net_builder == TrainTarget.ORIGIN:
self.pred, self.loss = build_pinn(
self.model,
sig_net_dim=hidden_sig_dims,
tanh_net_dim=hidden_tanh_dims,
weight_optim=_build_optimizer(weight_optim_method,
weight_optim_param),
bias_optim=_build_optimizer(bias_optim_method,
bias_optim_param),
loss_function=loss_function,
max_loss_scale=max_loss_scale,
)
train_init_net, train_net = instantiator.generate_training_nets(
self.model)
workspace.RunNetOnce(train_init_net)
workspace.CreateNet(train_net)
self.net_store['train_net'] = train_net
pred_net = instantiator.generate_predict_net(self.model)
workspace.CreateNet(pred_net)
self.net_store['pred_net'] = pred_net
if 'eval' in self.input_data_store:
# Create eval net
self.model.input_feature_schema.sig_input.set_value(
input_data_eval[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_eval[1].get(), unsafe=True)
if self.train_target == TrainTarget.ORIGIN:
if self.net_builder == TrainTarget.ADJOINT:
self.model.trainer_extra_schema.origin_loss_record.label.set_value(
input_data_eval[2].get(), unsafe=True)
elif self.net_builder == TrainTarget.ORIGIN:
self.model.trainer_extra_schema.label.set_value(
input_data_eval[2].get(), unsafe=True)
if self.train_target == TrainTarget.ADJOINT:
raise Exception('Not Implemented')
eval_net = instantiator.generate_eval_net(self.model)
workspace.CreateNet(eval_net)
self.net_store['eval_net'] = eval_net
def run_train_net(self):
self.model.output_schema = schema.Struct()
train_init_net, train_net = \
layer_model_instantiator.generate_training_nets(self.model)
workspace.RunNetOnce(train_init_net)
workspace.RunNetOnce(train_net)
def build_nets(
self,
hidden_sig_dims,
hidden_tanh_dims,
train_batch_size=1,
eval_batch_size=1,
weight_optim_method='AdaGrad',
weight_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
bias_optim_method='AdaGrad',
bias_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
loss_function='scaled_l1',
max_loss_scale=1., # used to scale up the loss signal for small input
neg_grad_penalty=None, # whether and how to apply neg_grad_penalty
init_model=None, # do postfix matching i.e. adjoint/<blob_nanme> == <blob_nanme>
):
assert len(self.input_data_store) > 0, 'Input data store is empty.'
assert 'train' in self.input_data_store, 'Missing training data.'
assert (neg_grad_penalty is None or
(neg_grad_penalty and self.train_target == TrainTarget.ORIGIN
and self.net_builder == TrainTarget.ADJOINT)
), '''When set neg_grad_penalty, train target should be ORIGIN,
but net builder should be ADJOINT'''
self.has_neg_grad_penalty = True if neg_grad_penalty else False
self.batch_size = train_batch_size
# Build the date reader net for train net
if self.train_target == TrainTarget.ORIGIN:
input_data_train = data_reader.build_input_reader(
self.model,
self.input_data_store['train'][0],
'minidb',
['sig_input', 'tanh_input', 'label'],
batch_size=train_batch_size,
data_type='train',
)
if 'eval' in self.input_data_store:
# Build the data reader net for eval net
input_data_eval = data_reader.build_input_reader(
self.model,
self.input_data_store['eval'][0],
'minidb',
['eval_sig_input', 'eval_tanh_input', 'eval_label'],
batch_size=eval_batch_size,
data_type='eval',
)
if self.net_builder == TrainTarget.ADJOINT: # Use Adjoint net so output adjoint net
# for training origin, use origin_loss_record
self.model.trainer_extra_schema.origin_loss_record.label.set_value(
input_data_train[2].get(), unsafe=True)
elif self.net_builder == TrainTarget.ORIGIN:
self.model.trainer_extra_schema.label.set_value(
input_data_train[2].get(), unsafe=True)
if self.train_target == TrainTarget.ADJOINT:
raise Exception('Not Implemented')
# Build the computational nets
# Create train net
self.model.input_feature_schema.sig_input.set_value(
input_data_train[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_train[1].get(), unsafe=True)
if self.net_builder == TrainTarget.ADJOINT:
# decide adjoint tag
adjoint_tag = 'no_tag'
if self.train_target == TrainTarget.ORIGIN and neg_grad_penalty is None:
adjoint_tag = Tags.PREDICTION_ONLY
(self.pred, self.sig_adjoint_pred, self.tanh_adjoint_pred,
self.loss) = build_adjoint_pinn(
self.model,
sig_input_dim=self.sig_input_dim,
tanh_input_dim=self.tanh_input_dim,
sig_net_dim=hidden_sig_dims,
tanh_net_dim=hidden_tanh_dims,
weight_optim=_build_optimizer(weight_optim_method,
weight_optim_param),
bias_optim=_build_optimizer(bias_optim_method,
bias_optim_param),
adjoint_tag=adjoint_tag,
train_target=self.train_target,
loss_function=loss_function,
max_loss_scale=max_loss_scale,
neg_grad_penalty=neg_grad_penalty,
)
elif self.net_builder == TrainTarget.ORIGIN:
self.pred, self.loss = build_pinn(
self.model,
sig_net_dim=hidden_sig_dims,
tanh_net_dim=hidden_tanh_dims,
weight_optim=_build_optimizer(weight_optim_method,
weight_optim_param),
bias_optim=_build_optimizer(bias_optim_method,
bias_optim_param),
loss_function=loss_function,
max_loss_scale=max_loss_scale,
)
train_init_net, train_net = instantiator.generate_training_nets(
self.model)
workspace.RunNetOnce(train_init_net)
if init_model:
model_name = init_model['name']
print('[INFO] Init params from ' + model_name)
given_init_net = exporter.load_init_net(model_name)
if 'prefix' in init_model.keys():
print('[INFO] Append ' + init_model['prefix'] +
' to all blob names.')
for op in given_init_net.op:
op.output[0] = init_model['prefix'] + op.output[0]
workspace.RunNetOnce(given_init_net)
workspace.CreateNet(train_net)
self.net_store['train_net'] = train_net
pred_net = instantiator.generate_predict_net(self.model)
workspace.CreateNet(pred_net)
self.net_store['pred_net'] = pred_net
if 'eval' in self.input_data_store:
# Create eval net
self.model.input_feature_schema.sig_input.set_value(
input_data_eval[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_eval[1].get(), unsafe=True)
if self.train_target == TrainTarget.ORIGIN:
if self.net_builder == TrainTarget.ADJOINT:
self.model.trainer_extra_schema.origin_loss_record.label.set_value(
input_data_eval[2].get(), unsafe=True)
elif self.net_builder == TrainTarget.ORIGIN:
self.model.trainer_extra_schema.label.set_value(
input_data_eval[2].get(), unsafe=True)
if self.train_target == TrainTarget.ADJOINT:
raise Exception('Not Implemented')
eval_net = instantiator.generate_eval_net(self.model)
workspace.CreateNet(eval_net)
self.net_store['eval_net'] = eval_net
def run_train_net(self):
self.model.output_schema = schema.Struct()
train_init_net, train_net = \
layer_model_instantiator.generate_training_nets(self.model)
workspace.RunNetOnce(train_init_net)
workspace.RunNetOnce(train_net)
def build_nets(
self,
hidden_dims,
batch_size=1,
optim_method='AdaGrad',
optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
):
assert len(self.input_data_store) > 0, 'Input data store is empty.'
assert 'train' in self.input_data_store, 'Missing training data.'
self.batch_size = batch_size
# Build the date reader net for train net
input_data_train = data_reader.build_input_reader(
self.model,
self.input_data_store['train'][0],
'minidb',
['origin_input', 'adjoint_input', 'label'],
batch_size=batch_size,
data_type='train',
)
if 'eval' in self.input_data_store:
# Build the data reader net for eval net
input_data_eval = data_reader.build_input_reader(
self.model,
self.input_data_store['eval'][0],
'minidb',
['origin_input', 'adjoint_input', 'label'],
batch_size=batch_size,
data_type='eval',
)
# Build the computational nets
# Create train net
self.model.input_feature_schema.origin_input.set_value(
input_data_train[0].get(), unsafe=True)
self.model.input_feature_schema.adjoint_input.set_value(
input_data_train[1].get(), unsafe=True)
self.model.trainer_extra_schema.label.set_value(
input_data_train[2].get(), unsafe=True)
self.origin_pred, self.adjoint_pred, self.loss = build_adjoint_mlp(
self.model,
input_dim=self.input_dim,
hidden_dims=hidden_dims,
output_dim=self.output_dim,
optim=_build_optimizer(optim_method, optim_param),
)
train_init_net, train_net = instantiator.generate_training_nets(
self.model)
workspace.RunNetOnce(train_init_net)
workspace.CreateNet(train_net)
self.net_store['train_net'] = train_net
pred_net = instantiator.generate_predict_net(self.model)
workspace.CreateNet(pred_net)
self.net_store['pred_net'] = pred_net
if 'eval' in self.input_data_store:
# Create eval net
self.model.input_feature_schema.origin_input.set_value(
input_data_eval[0].get(), unsafe=True)
self.model.input_feature_schema.adjoint_input.set_value(
input_data_eval[1].get(), unsafe=True)
self.model.trainer_extra_schema.label.set_value(
input_data_eval[2].get(), unsafe=True)
eval_net = instantiator.generate_eval_net(self.model)
workspace.CreateNet(eval_net)
self.net_store['eval_net'] = eval_net
