def testFunctionalLayer(self):
def normalize(net, in_record, out_record):
mean = net.ReduceFrontMean(in_record(), 1)
net.Sub(
[in_record(), mean],
out_record(),
broadcast=1)
normalized = self.model.Functional(
self.model.input_feature_schema.float_features, 1,
normalize, name="normalizer")
# Attach metadata to one of the outputs and use it in FC
normalized.set_type((np.float32, 32))
self.model.output_schema = self.model.FC(normalized, 2)
predict_net = layer_model_instantiator.generate_predict_net(
self.model)
ops = predict_net.Proto().op
assert len(ops) == 3
assert ops[0].type == "ReduceFrontMean"
assert ops[1].type == "Sub"
assert ops[2].type == "FC"
assert len(ops[0].input) == 1
assert ops[0].input[0] ==\
self.model.input_feature_schema.float_features()
assert len(ops[1].output) == 1
assert ops[1].output[0] in ops[2].input
def testFunctionalLayer(self):
def normalize(net, in_record, out_record):
mean = net.ReduceFrontMean(in_record(), 1)
net.Sub([in_record(), mean], out_record[0](), broadcast=1)
normalized = self.model.Functional(
self.model.input_feature_schema.float_features,
1,
normalize,
name="normalizer")
# Attach metadata to one of the outputs and use it in FC
normalized[0].set_type((np.float32, 32))
self.model.FC(normalized[0], 2)
predict_net = layer_model_instantiator.generate_predict_net(self.model)
ops = predict_net.Proto().op
assert len(ops) == 3
assert ops[0].type == "ReduceFrontMean"
assert ops[1].type == "Sub"
assert ops[2].type == "FC"
assert len(ops[0].input) == 1
assert ops[0].input[0] ==\
self.model.input_feature_schema.float_features()
assert len(ops[1].output) == 1
assert ops[1].output[0] in ops[2].input
def testSelectRecordByContext(self):
float_features = self.model.input_feature_schema.float_features
float_array = np.array([1.0, 2.0], dtype=np.float32)
schema.FeedRecord(float_features, [float_array])
with Tags(Tags.EXCLUDE_FROM_PREDICTION):
log_float_features = self.model.Log(float_features, 1)
joined = self.model.SelectRecordByContext(
schema.Struct(
(InstantiationContext.PREDICTION, float_features),
(InstantiationContext.TRAINING, log_float_features),
# TODO: TRAIN_ONLY layers are also generated in eval
(InstantiationContext.EVAL, log_float_features),
))
# model.output_schema has to a struct
self.model.output_schema = schema.Struct(('joined', joined))
predict_net = layer_model_instantiator.generate_predict_net(self.model)
workspace.RunNetOnce(predict_net)
predict_output = schema.FetchRecord(predict_net.output_record())
npt.assert_array_equal(float_array, predict_output['joined']())
eval_net = layer_model_instantiator.generate_eval_net(self.model)
workspace.RunNetOnce(eval_net)
eval_output = schema.FetchRecord(eval_net.output_record())
npt.assert_array_equal(np.log(float_array), eval_output['joined']())
_, train_net = (
layer_model_instantiator.generate_training_nets_forward_only(
self.model))
workspace.RunNetOnce(train_net)
train_output = schema.FetchRecord(train_net.output_record())
npt.assert_array_equal(np.log(float_array), train_output['joined']())
def testFunctionalLayerHelperAutoInference(self):
softsign = self.model.Softsign(
schema.Tuple(self.model.input_feature_schema.float_features), 1)
assert softsign.field_type().base == np.float32
assert softsign.field_type().shape == (32, )
self.model.output_schema = self.model.FC(softsign, 2)
predict_net = layer_model_instantiator.generate_predict_net(self.model)
ops = predict_net.Proto().op
assert len(ops) == 2
assert ops[0].type == "Softsign"
assert ops[1].type == "FC"
assert len(ops[0].input) == 1
assert ops[0].input[0] ==\
self.model.input_feature_schema.float_features()
assert len(ops[0].output) == 1
assert ops[0].output[0] in ops[1].input
def testFunctionalLayerHelperAutoInference(self):
softsign = self.model.Softsign(
schema.Tuple(self.model.input_feature_schema.float_features),
1)
assert softsign.field_type().base == np.float32
assert softsign.field_type().shape == (32,)
self.model.output_schema = self.model.FC(softsign, 2)
predict_net = layer_model_instantiator.generate_predict_net(
self.model)
ops = predict_net.Proto().op
assert len(ops) == 2
assert ops[0].type == "Softsign"
assert ops[1].type == "FC"
assert len(ops[0].input) == 1
assert ops[0].input[0] ==\
self.model.input_feature_schema.float_features()
assert len(ops[0].output) == 1
assert ops[0].output[0] in ops[1].input
def testSelectRecordByContext(self):
float_features = self.model.input_feature_schema.float_features
float_array = np.array([1.0, 2.0], dtype=np.float32)
schema.FeedRecord(float_features, [float_array])
with Tags(Tags.EXCLUDE_FROM_PREDICTION):
log_float_features = self.model.Log(float_features, 1)
joined = self.model.SelectRecordByContext(
schema.Struct(
(InstantiationContext.PREDICTION, float_features),
(InstantiationContext.TRAINING, log_float_features),
# TODO: TRAIN_ONLY layers are also generated in eval
(InstantiationContext.EVAL, log_float_features),
)
)
# model.output_schema has to a struct
self.model.output_schema = schema.Struct((
'joined', joined
))
predict_net = layer_model_instantiator.generate_predict_net(self.model)
workspace.RunNetOnce(predict_net)
predict_output = schema.FetchRecord(predict_net.output_record())
npt.assert_array_equal(float_array,
predict_output['joined']())
eval_net = layer_model_instantiator.generate_eval_net(self.model)
workspace.RunNetOnce(eval_net)
eval_output = schema.FetchRecord(eval_net.output_record())
npt.assert_array_equal(np.log(float_array),
eval_output['joined']())
_, train_net = (
layer_model_instantiator.generate_training_nets_forward_only(
self.model
)
)
workspace.RunNetOnce(train_net)
train_output = schema.FetchRecord(train_net.output_record())
npt.assert_array_equal(np.log(float_array),
train_output['joined']())
def testFunctionalLayerHelper(self):
mean = self.model.ReduceFrontMean(
self.model.input_feature_schema.float_features, 1)
normalized = self.model.Sub(schema.Tuple(
self.model.input_feature_schema.float_features, mean[0]),
1,
broadcast=1)
# Attach metadata to one of the outputs and use it in FC
normalized[0].set_type((np.float32, (32, )))
self.model.FC(normalized[0], 2)
predict_net = layer_model_instantiator.generate_predict_net(self.model)
ops = predict_net.Proto().op
assert len(ops) == 3
assert ops[0].type == "ReduceFrontMean"
assert ops[1].type == "Sub"
assert ops[2].type == "FC"
assert len(ops[0].input) == 1
assert ops[0].input[0] ==\
self.model.input_feature_schema.float_features()
assert len(ops[1].output) == 1
assert ops[1].output[0] in ops[2].input
def testFunctionalLayerHelper(self):
mean = self.model.ReduceFrontMean(
self.model.input_feature_schema.float_features, 1)
normalized = self.model.Sub(
schema.Tuple(
self.model.input_feature_schema.float_features, mean),
1, broadcast=1)
# Attach metadata to one of the outputs and use it in FC
normalized.set_type((np.float32, (32,)))
self.model.output_schema = self.model.FC(normalized, 2)
predict_net = layer_model_instantiator.generate_predict_net(
self.model)
ops = predict_net.Proto().op
assert len(ops) == 3
assert ops[0].type == "ReduceFrontMean"
assert ops[1].type == "Sub"
assert ops[2].type == "FC"
assert len(ops[0].input) == 1
assert ops[0].input[0] ==\
self.model.input_feature_schema.float_features()
assert len(ops[1].output) == 1
assert ops[1].output[0] in ops[2].input
def build_nets(
self,
hidden_sig_dims,
hidden_tanh_dims,
train_batch_size=1,
eval_batch_size=1,
weight_optim_method='AdaGrad',
weight_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
bias_optim_method='AdaGrad',
bias_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
loss_function='scaled_l1',
max_loss_scale=1e6,
):
assert len(self.input_data_store) > 0, 'Input data store is empty.'
assert 'train' in self.input_data_store, 'Missing training data.'
self.batch_size = train_batch_size
# Build the date reader net for train net
input_data_train = data_reader.build_input_reader(
self.model,
self.input_data_store['train'][0],
'minidb',
['sig_input', 'tanh_input', 'label'],
batch_size=train_batch_size,
data_type='train',
)
if 'eval' in self.input_data_store:
# Build the data reader net for eval net
input_data_eval = data_reader.build_input_reader(
self.model,
self.input_data_store['eval'][0],
'minidb',
['eval_sig_input', 'eval_tanh_input', 'eval_label'],
batch_size=eval_batch_size,
data_type='eval',
)
# Build the computational nets
# Create train net
self.model.input_feature_schema.sig_input.set_value(
input_data_train[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_train[1].get(), unsafe=True)
self.model.trainer_extra_schema.label.set_value(
input_data_train[2].get(), unsafe=True)
self.pred, self.loss = build_pinn(
self.model,
sig_net_dim=hidden_sig_dims,
tanh_net_dim=hidden_tanh_dims,
weight_optim=_build_optimizer(weight_optim_method,
weight_optim_param),
bias_optim=_build_optimizer(bias_optim_method, bias_optim_param),
loss_function=loss_function,
max_loss_scale=max_loss_scale)
train_init_net, train_net = instantiator.generate_training_nets(
self.model)
workspace.RunNetOnce(train_init_net)
workspace.CreateNet(train_net)
self.net_store['train_net'] = train_net
pred_net = instantiator.generate_predict_net(self.model)
workspace.CreateNet(pred_net)
self.net_store['pred_net'] = pred_net
if 'eval' in self.input_data_store:
# Create eval net
self.model.input_feature_schema.sig_input.set_value(
input_data_eval[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_eval[1].get(), unsafe=True)
self.model.trainer_extra_schema.label.set_value(
input_data_eval[2].get(), unsafe=True)
eval_net = instantiator.generate_eval_net(self.model)
workspace.CreateNet(eval_net)
self.net_store['eval_net'] = eval_net
def build_nets(
self,
hidden_sig_dims,
hidden_tanh_dims,
train_batch_size=1,
eval_batch_size=1,
weight_optim_method='AdaGrad',
weight_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
bias_optim_method='AdaGrad',
bias_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
loss_function='scaled_l1',
max_loss_scale=1.,
):
assert len(self.input_data_store) > 0, 'Input data store is empty.'
assert 'train' in self.input_data_store, 'Missing training data.'
self.batch_size = train_batch_size
# Build the date reader net for train net
if self.train_target == TrainTarget.ORIGIN:
input_data_train = data_reader.build_input_reader(
self.model,
self.input_data_store['train'][0],
'minidb',
['sig_input', 'tanh_input', 'label'],
batch_size=train_batch_size,
data_type='train',
)
if 'eval' in self.input_data_store:
# Build the data reader net for eval net
input_data_eval = data_reader.build_input_reader(
self.model,
self.input_data_store['eval'][0],
'minidb',
['eval_sig_input', 'eval_tanh_input', 'eval_label'],
batch_size=eval_batch_size,
data_type='eval',
)
if self.net_builder == TrainTarget.ADJOINT: # Use Adjoint net so output adjoint net
# for training origin, use origin_loss_record
self.model.trainer_extra_schema.origin_loss_record.label.set_value(
input_data_train[2].get(), unsafe=True)
elif self.net_builder == TrainTarget.ORIGIN:
self.model.trainer_extra_schema.label.set_value(
input_data_train[2].get(), unsafe=True)
if self.train_target == TrainTarget.ADJOINT:
raise Exception('Not Implemented')
# Build the computational nets
# Create train net
self.model.input_feature_schema.sig_input.set_value(
input_data_train[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_train[1].get(), unsafe=True)
if self.net_builder == TrainTarget.ADJOINT:
(self.pred, self.sig_adjoint_pred, self.tanh_adjoint_pred,
self.loss) = build_adjoint_pinn(
self.model,
sig_input_dim=self.sig_input_dim,
tanh_input_dim=self.tanh_input_dim,
sig_net_dim=hidden_sig_dims,
tanh_net_dim=hidden_tanh_dims,
weight_optim=_build_optimizer(weight_optim_method,
weight_optim_param),
bias_optim=_build_optimizer(bias_optim_method,
bias_optim_param),
adjoint_tag=self.adjoint_tag,
train_target=self.train_target,
loss_function=loss_function,
max_loss_scale=max_loss_scale,
)
elif self.net_builder == TrainTarget.ORIGIN:
self.pred, self.loss = build_pinn(
self.model,
sig_net_dim=hidden_sig_dims,
tanh_net_dim=hidden_tanh_dims,
weight_optim=_build_optimizer(weight_optim_method,
weight_optim_param),
bias_optim=_build_optimizer(bias_optim_method,
bias_optim_param),
loss_function=loss_function,
max_loss_scale=max_loss_scale,
)
train_init_net, train_net = instantiator.generate_training_nets(
self.model)
workspace.RunNetOnce(train_init_net)
workspace.CreateNet(train_net)
self.net_store['train_net'] = train_net
pred_net = instantiator.generate_predict_net(self.model)
workspace.CreateNet(pred_net)
self.net_store['pred_net'] = pred_net
if 'eval' in self.input_data_store:
# Create eval net
self.model.input_feature_schema.sig_input.set_value(
input_data_eval[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_eval[1].get(), unsafe=True)
if self.train_target == TrainTarget.ORIGIN:
if self.net_builder == TrainTarget.ADJOINT:
self.model.trainer_extra_schema.origin_loss_record.label.set_value(
input_data_eval[2].get(), unsafe=True)
elif self.net_builder == TrainTarget.ORIGIN:
self.model.trainer_extra_schema.label.set_value(
input_data_eval[2].get(), unsafe=True)
if self.train_target == TrainTarget.ADJOINT:
raise Exception('Not Implemented')
eval_net = instantiator.generate_eval_net(self.model)
workspace.CreateNet(eval_net)
self.net_store['eval_net'] = eval_net
def get_predict_net(self):
return layer_model_instantiator.generate_predict_net(self.model)
def build_nets(
self,
hidden_sig_dims,
hidden_tanh_dims,
train_batch_size=1,
eval_batch_size=1,
weight_optim_method='AdaGrad',
weight_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
bias_optim_method='AdaGrad',
bias_optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
loss_function='scaled_l1',
max_loss_scale=1., # used to scale up the loss signal for small input
neg_grad_penalty=None, # whether and how to apply neg_grad_penalty
init_model=None, # do postfix matching i.e. adjoint/<blob_nanme> == <blob_nanme>
):
assert len(self.input_data_store) > 0, 'Input data store is empty.'
assert 'train' in self.input_data_store, 'Missing training data.'
assert (neg_grad_penalty is None or
(neg_grad_penalty and self.train_target == TrainTarget.ORIGIN
and self.net_builder == TrainTarget.ADJOINT)
), '''When set neg_grad_penalty, train target should be ORIGIN,
but net builder should be ADJOINT'''
self.has_neg_grad_penalty = True if neg_grad_penalty else False
self.batch_size = train_batch_size
# Build the date reader net for train net
if self.train_target == TrainTarget.ORIGIN:
input_data_train = data_reader.build_input_reader(
self.model,
self.input_data_store['train'][0],
'minidb',
['sig_input', 'tanh_input', 'label'],
batch_size=train_batch_size,
data_type='train',
)
if 'eval' in self.input_data_store:
# Build the data reader net for eval net
input_data_eval = data_reader.build_input_reader(
self.model,
self.input_data_store['eval'][0],
'minidb',
['eval_sig_input', 'eval_tanh_input', 'eval_label'],
batch_size=eval_batch_size,
data_type='eval',
)
if self.net_builder == TrainTarget.ADJOINT: # Use Adjoint net so output adjoint net
# for training origin, use origin_loss_record
self.model.trainer_extra_schema.origin_loss_record.label.set_value(
input_data_train[2].get(), unsafe=True)
elif self.net_builder == TrainTarget.ORIGIN:
self.model.trainer_extra_schema.label.set_value(
input_data_train[2].get(), unsafe=True)
if self.train_target == TrainTarget.ADJOINT:
raise Exception('Not Implemented')
# Build the computational nets
# Create train net
self.model.input_feature_schema.sig_input.set_value(
input_data_train[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_train[1].get(), unsafe=True)
if self.net_builder == TrainTarget.ADJOINT:
# decide adjoint tag
adjoint_tag = 'no_tag'
if self.train_target == TrainTarget.ORIGIN and neg_grad_penalty is None:
adjoint_tag = Tags.PREDICTION_ONLY
(self.pred, self.sig_adjoint_pred, self.tanh_adjoint_pred,
self.loss) = build_adjoint_pinn(
self.model,
sig_input_dim=self.sig_input_dim,
tanh_input_dim=self.tanh_input_dim,
sig_net_dim=hidden_sig_dims,
tanh_net_dim=hidden_tanh_dims,
weight_optim=_build_optimizer(weight_optim_method,
weight_optim_param),
bias_optim=_build_optimizer(bias_optim_method,
bias_optim_param),
adjoint_tag=adjoint_tag,
train_target=self.train_target,
loss_function=loss_function,
max_loss_scale=max_loss_scale,
neg_grad_penalty=neg_grad_penalty,
)
elif self.net_builder == TrainTarget.ORIGIN:
self.pred, self.loss = build_pinn(
self.model,
sig_net_dim=hidden_sig_dims,
tanh_net_dim=hidden_tanh_dims,
weight_optim=_build_optimizer(weight_optim_method,
weight_optim_param),
bias_optim=_build_optimizer(bias_optim_method,
bias_optim_param),
loss_function=loss_function,
max_loss_scale=max_loss_scale,
)
train_init_net, train_net = instantiator.generate_training_nets(
self.model)
workspace.RunNetOnce(train_init_net)
if init_model:
model_name = init_model['name']
print('[INFO] Init params from ' + model_name)
given_init_net = exporter.load_init_net(model_name)
if 'prefix' in init_model.keys():
print('[INFO] Append ' + init_model['prefix'] +
' to all blob names.')
for op in given_init_net.op:
op.output[0] = init_model['prefix'] + op.output[0]
workspace.RunNetOnce(given_init_net)
workspace.CreateNet(train_net)
self.net_store['train_net'] = train_net
pred_net = instantiator.generate_predict_net(self.model)
workspace.CreateNet(pred_net)
self.net_store['pred_net'] = pred_net
if 'eval' in self.input_data_store:
# Create eval net
self.model.input_feature_schema.sig_input.set_value(
input_data_eval[0].get(), unsafe=True)
self.model.input_feature_schema.tanh_input.set_value(
input_data_eval[1].get(), unsafe=True)
if self.train_target == TrainTarget.ORIGIN:
if self.net_builder == TrainTarget.ADJOINT:
self.model.trainer_extra_schema.origin_loss_record.label.set_value(
input_data_eval[2].get(), unsafe=True)
elif self.net_builder == TrainTarget.ORIGIN:
self.model.trainer_extra_schema.label.set_value(
input_data_eval[2].get(), unsafe=True)
if self.train_target == TrainTarget.ADJOINT:
raise Exception('Not Implemented')
eval_net = instantiator.generate_eval_net(self.model)
workspace.CreateNet(eval_net)
self.net_store['eval_net'] = eval_net
def get_predict_net(self):
return layer_model_instantiator.generate_predict_net(self.model)
def build_nets(
self,
hidden_dims,
batch_size=1,
optim_method='AdaGrad',
optim_param={
'alpha': 0.01,
'epsilon': 1e-4
},
):
assert len(self.input_data_store) > 0, 'Input data store is empty.'
assert 'train' in self.input_data_store, 'Missing training data.'
self.batch_size = batch_size
# Build the date reader net for train net
input_data_train = data_reader.build_input_reader(
self.model,
self.input_data_store['train'][0],
'minidb',
['origin_input', 'adjoint_input', 'label'],
batch_size=batch_size,
data_type='train',
)
if 'eval' in self.input_data_store:
# Build the data reader net for eval net
input_data_eval = data_reader.build_input_reader(
self.model,
self.input_data_store['eval'][0],
'minidb',
['origin_input', 'adjoint_input', 'label'],
batch_size=batch_size,
data_type='eval',
)
# Build the computational nets
# Create train net
self.model.input_feature_schema.origin_input.set_value(
input_data_train[0].get(), unsafe=True)
self.model.input_feature_schema.adjoint_input.set_value(
input_data_train[1].get(), unsafe=True)
self.model.trainer_extra_schema.label.set_value(
input_data_train[2].get(), unsafe=True)
self.origin_pred, self.adjoint_pred, self.loss = build_adjoint_mlp(
self.model,
input_dim=self.input_dim,
hidden_dims=hidden_dims,
output_dim=self.output_dim,
optim=_build_optimizer(optim_method, optim_param),
)
train_init_net, train_net = instantiator.generate_training_nets(
self.model)
workspace.RunNetOnce(train_init_net)
workspace.CreateNet(train_net)
self.net_store['train_net'] = train_net
pred_net = instantiator.generate_predict_net(self.model)
workspace.CreateNet(pred_net)
self.net_store['pred_net'] = pred_net
if 'eval' in self.input_data_store:
# Create eval net
self.model.input_feature_schema.origin_input.set_value(
input_data_eval[0].get(), unsafe=True)
self.model.input_feature_schema.adjoint_input.set_value(
input_data_eval[1].get(), unsafe=True)
self.model.trainer_extra_schema.label.set_value(
input_data_eval[2].get(), unsafe=True)
eval_net = instantiator.generate_eval_net(self.model)
workspace.CreateNet(eval_net)
self.net_store['eval_net'] = eval_net
