def __eval(self):
# import
from mindspore import Model, load_param_into_net, load_checkpoint
from mindspore.nn.metrics import Accuracy
# load params
if self.__ckpt_path:
load_param_into_net(self.__network,
load_checkpoint(self.__ckpt_path))
else:
print(
"Warning: `ckpt_path` is None, Please call func: `set_ckpt_path($ckpt_path)`."
)
return
# loss_fn & optimizer & metrics
model = Model(self.__network,
loss_fn=self.__loss_fn,
optimizer=self.__optimizer,
metrics={"Accuracy": Accuracy()}
if self.__metrics is None else self.__metrics)
# eval
print(">>>>>>>>>>>>>>>>>>>>> eval start ... <<<<<<<<<<<<<<<<<<<<<<")
result = model.eval(self.__dataset)
print(
">>>>>>>>>>>>>>>>>>>>> eval success ~ <<<<<<<<<<<<<<<<<<<<<<: result=",
result)
def __init__(self, config, network):
super(OneHopBert, self).__init__(auto_prefix=False)
self.network = network
self.mlp = Model()
param_dict = load_checkpoint(config.onehop_mlp_path)
load_param_into_net(self.mlp, param_dict)
self.cast = P.Cast()
def epoch_end(self, run_context):
if self.step_eval:
return None
flag = ''
cb_params = run_context.original_args()
cur_epoch = cb_params.cur_epoch_num
if cur_epoch > 0 and cur_epoch % self.eval_epoch == 0:
acc = self.eval()
if acc > self.best_acc:
flag = '↑'
self.best_acc = acc
save_checkpoint(self.model._network, self.best_ckpt)
self.logger.update_acc_ckpt(acc, self.best_ckpt)
else:
if acc > self.threshold:
self.patience_count += 1
if self.patience_count > self.patience:
param_dict = load_checkpoint(
ckpt_file_name=self.best_ckpt,
net=self.model._network)
load_param_into_net(net=self.model._network,
parameter_dict=param_dict)
self.patience_count = 0
print(
f'* acc for epoch: {cur_epoch} is {acc * 100}%{flag}, best acc is {self.best_acc * 100}%'
)
def __init__(self, requires_grad=False):
super(Vgg19, self).__init__()
##load vgg16
vgg = Vgg(cfg['19'], phase="test", args=imagenet_cfg)
# model = os.path.join(opt.data_url, 'vgg19_ImageNet.ckpt')
model = os.path.join('./', 'vgg19_ImageNet.ckpt')
print(model)
param_dict = load_checkpoint(model)
# print(param_dict)
load_param_into_net(vgg, param_dict)
vgg.set_train(False)
vgg_pretrained_features = vgg.layers
self.slice1 = nn.SequentialCell()
self.slice2 = nn.SequentialCell()
self.slice3 = nn.SequentialCell()
self.slice4 = nn.SequentialCell()
self.slice5 = nn.SequentialCell()
for x in range(2):
self.slice1.append(vgg_pretrained_features[x])
for x in range(2, 7):
self.slice2.append(vgg_pretrained_features[x])
for x in range(7, 12):
self.slice3.append(vgg_pretrained_features[x])
for x in range(12, 21):
self.slice4.append(vgg_pretrained_features[x])
for x in range(21, 30):
self.slice5.append(vgg_pretrained_features[x])
if not requires_grad:
for param in self.get_parameters():
param.requires_grad = False
def progress(self):
self.current_scale += 1
# Lower scale discriminators are not used in later ... replace append to assign?
if self.current_scale % 4 == 0:
self.nf *= 2
tmp_discriminator = nn.CellList()
tmp_discriminator.append(
nn.SequentialCell(nn.Conv2d(3, self.nf, 3, 1, 1),
nn.LeakyReLU(2e-1)))
for _ in range(3):
tmp_discriminator.append(
nn.SequentialCell(nn.Conv2d(self.nf, self.nf, 3, 1, 1),
nn.BatchNorm2d(self.nf), nn.LeakyReLU(2e-1)))
tmp_discriminator.append(
nn.SequentialCell(nn.Conv2d(self.nf, 1, 3, 1, 1)))
tmp_discriminator = nn.SequentialCell(*tmp_discriminator)
if self.current_scale % 4 != 0:
prev_discriminator = self.sub_discriminators[-1]
# Initialize layers via copy
if self.current_scale >= 1:
mindspore.load_param_into_net(
tmp_discriminator, prev_discriminator.parameters_dict())
self.sub_discriminators.append(tmp_discriminator)
print("DISCRIMINATOR PROGRESSION DONE")
def __train(self):
# import
from mindspore import Model, load_param_into_net, load_checkpoint
# load params
if self.__ckpt_path and os.path.isfile(
self.__ckpt_path) and os.path.exists(self.__ckpt_path):
load_param_into_net(self.__network,
load_checkpoint(self.__ckpt_path))
# loss_fn & optimizer & metrics
model = Model(self.__network,
loss_fn=self.__loss_fn,
optimizer=self.__optimizer,
metrics=self.__metrics)
# train
print(">>>>>>>>>>>>>>>>>>>>> train start ... <<<<<<<<<<<<<<<<<<<<<<")
self.__update_cbs()
model.train(self.__epoch_size,
self.__dataset,
callbacks=self.__callbacks,
dataset_sink_mode=self.__do_sink,
sink_size=self.__sink_size)
print(">>>>>>>>>>>>>>>>>>>>> train success ~ <<<<<<<<<<<<<<<<<<<<<<")
def load_weights(self, weights):
"""Load the model weights passed in as a parameter."""
for name, weight in weights.items():
weights[name] = mindspore.Parameter(weight, name=name)
# One can also use `self.model.load_parameter_slice(weights)', which
# seems to be equivalent to mindspore.load_param_into_net() in its effects
mindspore.load_param_into_net(self.model, weights, strict_load=True)
def test_net(network, model, mnist_path):
"""Define the evaluation method"""
print("==================== Starting Testing ===============")
param_dict = load_checkpoint(
"./model/ckpt/mindspore_quick_start/checkpoint_lenet-1_1874.ckpt")
load_param_into_net(network, param_dict)
ds_eval = create_dataset(os.path.join(mnist_path, "test"))
acc = model.eval(ds_eval, dataset_sink_mode=False)
print("==================== Accuracy:{} ===============".format(acc))
def test_net(network, network_model, data_path):
"""Define the evaluation method."""
print("============== Starting Testing ==============")
# load the saved model for evaluation
param_dict = load_checkpoint("checkpoint_lenet-1_1875.ckpt")
# load parameter to the network
load_param_into_net(network, param_dict)
# load testing dataset
ds_eval = create_dataset(os.path.join(data_path, "test"))
acc = network_model.eval(ds_eval, dataset_sink_mode=False)
print("============== Accuracy:{} ==============".format(acc))
def test_eval(model_ckpt_path):
'''eval model'''
target = args_opt.device_target
if target == "Ascend":
devid = int(os.getenv('DEVICE_ID'))
context.set_context(mode=context.GRAPH_MODE,
device_target="Ascend",
device_id=devid)
bert_net_cfg.batch_size = 1
poetrymodel = BertPoetryModel(bert_net_cfg, False, 3191, dropout_prob=0.0)
poetrymodel.set_train(False)
param_dict = load_checkpoint(model_ckpt_path)
load_param_into_net(poetrymodel, param_dict)
# random generation/continue
start_time = time.time()
output = generate_random_poetry(poetrymodel, s='')
end_to_end_delay = (time.time() - start_time) * 1000
a = re.findall(r'[\u4e00-\u9fa5]*[\uff0c\u3002]', output)
print("\n**********************************")
print("随机生成: \n")
for poem in a:
print(poem)
print("\ncost time: {:.1f} ms".format(end_to_end_delay))
print("\n")
start = "天下为公"
start_time = time.time()
output = generate_random_poetry(poetrymodel, s=start)
end_to_end_delay = (time.time() - start_time) * 1000
a = re.findall(r'[\u4e00-\u9fa5]*[\uff0c\u3002]', output)
print("\n**********************************")
print("续写 【{}】: \n".format(start))
for poem in a:
print(poem)
print("\ncost time: {:.1f} ms".format(end_to_end_delay))
print("\n")
# hidden poetry
s = "人工智能"
start_time = time.time()
output = generate_hidden(poetrymodel, head=s)
end_to_end_delay = (time.time() - start_time) * 1000
a = re.findall(r'[\u4e00-\u9fa5]*[\uff0c\u3002]', output)
print("\n**********************************")
print("藏头诗 【{}】: \n".format(s))
for poem in a:
print(poem)
print("\ncost time: {:.1f} ms".format(end_to_end_delay))
print("\n")
def export_net(model_ckpt_path):
bert_net_cfg.batch_size = 1
poetrymodel = BertPoetryModel(bert_net_cfg, False, 3191, dropout_prob=0.0)
poetrymodel.set_train(False)
param_dict = load_checkpoint(model_ckpt_path)
load_param_into_net(poetrymodel, param_dict)
input_id = np.ones(shape=(1, 128))
token_type_id = np.ones(shape=(1, 128))
pad_mask = np.ones(shape=(1, 128))
export(poetrymodel, Tensor(input_id, mstype.int32),\
Tensor(token_type_id, mstype.int32),\
Tensor(pad_mask, mstype.float32),\
file_name='poetry.pb', file_format='MINDIR')
def __init__(self, batch_size, encoder_ck_file, downstream_ck_file):
"""init function"""
super(Reranker, self).__init__(auto_prefix=False)
self.encoder = Albert(batch_size)
param_dict = load_checkpoint(encoder_ck_file)
not_load_params_1 = load_param_into_net(self.encoder, param_dict)
print(f"re-ranker albert not loaded params: {not_load_params_1}")
self.no_answer_mlp = Rerank_Downstream()
param_dict = load_checkpoint(downstream_ck_file)
not_load_params_2 = load_param_into_net(self.no_answer_mlp, param_dict)
print(f"re-ranker downstream not loaded params: {not_load_params_2}")
def run_export(args):
""" export """
device_id = int(os.getenv('DEVICE_ID', '0'))
context.set_context(mode=context.GRAPH_MODE,
device_target="Ascend",
device_id=device_id)
net = RCAN(arg)
param_dict = load_checkpoint(args.ckpt_path)
load_param_into_net(net, param_dict)
net.set_train(False)
print('load mindspore net and checkpoint successfully.')
inputs = Tensor(np.zeros([args.batch_size, 3, 678, 1020], np.float32))
export(net, inputs, file_name=args.file_name, file_format=args.file_format)
print('export successfully!')
def __init__(self, batch_size, encoder_ck_file, downstream_ck_file):
"""init function"""
super(Reader, self).__init__(auto_prefix=False)
self.encoder = Reader_Albert(batch_size)
param_dict = load_checkpoint(encoder_ck_file)
not_load_params = load_param_into_net(self.encoder, param_dict)
print(f"not loaded: {not_load_params}")
self.downstream = Reader_Downstream()
param_dict = load_checkpoint(downstream_ck_file)
not_load_params = load_param_into_net(self.downstream, param_dict)
print(f"not loaded: {not_load_params}")
self.bmm = BatchMatMul()
def test_net(network, network_model, data_path):
"""Define the evaluation method."""
print("============== Starting Testing ==============")
# load the saved model for evaluation
latest_file = newest(data_path)
print(latest_file)
param_dict = load_checkpoint(latest_file)
# load parameter to the network
load_param_into_net(network, param_dict)
# load testing dataset
ds_eval = dm.create_dataset(os.path.join(
data_path, "./MindSpore_train_images_dataset/test"),
do_train=False)
acc = network_model.eval(ds_eval, dataset_sink_mode=False)
print("============== Accuracy:{} ==============".format(acc))
def progress(self):
self.current_scale += 1
if self.current_scale % 4 == 0:
self.nf *= 2
tmp_generator = nn.CellList()
tmp_generator.append(
nn.SequentialCell(nn.Conv2d(3, self.nf, 3, 1),
nn.BatchNorm2d(self.nf), nn.LeakyReLU(2e-1)))
for _ in range(3):
tmp_generator.append(
nn.SequentialCell(nn.Conv2d(self.nf, self.nf, 3, 1),
nn.BatchNorm2d(self.nf), nn.LeakyReLU(2e-1)))
tmp_generator.append(
nn.SequentialCell(nn.Conv2d(self.nf, 3, 3, 1), nn.Tanh()))
tmp_generator = nn.SequentialCell(*tmp_generator)
if self.current_scale % 4 != 0:
prev_generator = self.sub_generators[-1]
# Initialize layers via copy
if self.current_scale >= 1:
tmp_generator = mindspore.load_param_into_net(
tmp_generator,
prev_generator.parameters_dict) # 以python的字典格式加载存储
self.sub_generators.append(tmp_generator)
print("GENERATOR PROGRESSION DONE")
def eval_quant():
context.set_context(mode=context.GRAPH_MODE, device_target=device_target)
cfg = quant_cfg
ds_eval = create_dataset(os.path.join(data_path, "test"), cfg.batch_size,
1)
ckpt_path = './ckpt_lenet_quant-10_937.ckpt'
# define fusion network
network = LeNet5Fusion(cfg.num_classes)
# convert fusion network to quantization aware network
quantizer = QuantizationAwareTraining(quant_delay=0,
bn_fold=False,
freeze_bn=10000,
per_channel=[True, False],
symmetric=[True, False])
network = quantizer.quantize(network)
# define loss
net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
# define network optimization
net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum)
# call back and monitor
model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()})
# load quantization aware network checkpoint
param_dict = load_checkpoint(ckpt_path)
not_load_param = load_param_into_net(network, param_dict)
if not_load_param:
raise ValueError("Load param into net fail!")
print("============== Starting Testing ==============")
acc = model.eval(ds_eval, dataset_sink_mode=True)
print("============== {} ==============".format(acc))
assert acc['Accuracy'] > 0.98
def load_model(self, filename=None):
"""Loading pre-trained model weights from a file."""
model_name = Config().trainer.model_name
model_dir = Config().params['pretrained_model_dir']
if filename is not None:
model_path = f'{model_dir}{filename}'
else:
model_path = f'{model_dir}{model_name}.ckpt'
if self.client_id == 0:
logging.info("[Server #%d] Loading a model from %s.", os.getpid(),
model_path)
else:
logging.info("[Client #%d] Loading a model from %s.",
self.client_id, model_path)
param_dict = mindspore.load_checkpoint(model_path)
mindspore.load_param_into_net(self.model, param_dict)
def source_checker(py_path, ckpt_path):
"""Check source model script and source checkpoint file."""
sys.path.append(os.path.dirname(py_path))
model = getattr(
import_module(os.path.basename(py_path).replace('.py', '')), 'Model')()
try:
param_dict = load_checkpoint(ckpt_path)
not_load_name = load_param_into_net(model, param_dict)
return not not_load_name
except (ValueError, TypeError, RuntimeError):
return False
def test_net(network, model, data_path):
"""Define the evaluation method."""
print("============== Starting Testing ==============")
# load the saved model for evaluation
checkpoint = "./101.ckpt"
_t = time.time()
param_dict = load_checkpoint(checkpoint)
# load parameter to the network
load_param_into_net(network, param_dict)
print(f"Load params to model cost: {time.time() - _t} s")
# inference
print("============== Infer Testing ==============")
input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]),
mindspore.float32)
infer_time = 0
while infer_time < 21:
_t = time.time()
model.predict(input_data)
print(f"Model infer cost: {time.time() - _t} s")
infer_time += 1
def eval_lenet():
context.set_context(mode=context.GRAPH_MODE, device_target=device_target)
cfg = nonquant_cfg
ds_eval = create_dataset(os.path.join(data_path, "test"), cfg.batch_size,
1)
ckpt_path = './ckpt_lenet_noquant-10_1875.ckpt'
# define fusion network
network = LeNet5(cfg.num_classes)
net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum)
# call back and monitor
model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()})
# load quantization aware network checkpoint
param_dict = load_checkpoint(ckpt_path)
not_load_param = load_param_into_net(network, param_dict)
if not_load_param:
raise ValueError("Load param into net fail!")
print("============== Starting Testing ==============")
acc = model.eval(ds_eval, dataset_sink_mode=True)
print("============== {} ==============".format(acc))
assert acc['Accuracy'] > 0.98
required=True,
help='inceptionv3 ckpt file.')
parser.add_argument('--output_file',
type=str,
default='inceptionv3.air',
help='inceptionv3 output air name.')
parser.add_argument('--file_format',
type=str,
choices=["AIR", "ONNX", "MINDIR"],
default='AIR',
help='file format')
parser.add_argument('--width', type=int, default=299, help='input width')
parser.add_argument('--height', type=int, default=299, help='input height')
args = parser.parse_args()
if __name__ == '__main__':
net = InceptionV3(num_classes=cfg.num_classes, is_training=False)
param_dict = load_checkpoint(args.ckpt_file)
load_param_into_net(net, param_dict)
input_arr = Tensor(
np.random.uniform(0.0,
1.0,
size=[cfg.batch_size, 3, args.width, args.height]),
ms.float32)
export(net,
input_arr,
file_name=args.output_file,
file_format=args.file_format)
from src.lenet import LeNet5
parser = argparse.ArgumentParser(description='MindSpore MNIST Example')
parser.add_argument(
'--device_target',
type=str,
default="Ascend",
choices=['Ascend', 'GPU'],
help='device where the code will be implemented (default: Ascend)')
parser.add_argument(
'--ckpt_path',
type=str,
default="",
help='if mode is test, must provide path where the trained ckpt file')
args = parser.parse_args()
if __name__ == "__main__":
context.set_context(mode=context.GRAPH_MODE,
device_target=args.device_target)
# define fusion network
network = LeNet5(cfg.num_classes)
# load network checkpoint
param_dict = load_checkpoint(args.ckpt_path)
load_param_into_net(network, param_dict)
# export network
inputs = Tensor(np.ones([1, 1, cfg.image_height, cfg.image_width]),
mindspore.float32)
export(network, inputs, file_name=cfg.air_name, file_format='AIR')
def evaluation(d_id):
"""evaluation"""
context.set_context(mode=context.GRAPH_MODE,
device_target='Ascend',
device_id=d_id,
save_graphs=False)
print('********************** loading corpus ********************** ')
s_lc = time.time()
data_generator = DataGen(config)
queries = read_query(config, d_id)
print("loading corpus time (h):", (time.time() - s_lc) / 3600)
print('********************** loading model ********************** ')
s_lm = time.time()
model_onehop_bert = ModelOneHop(256)
param_dict = load_checkpoint(config.onehop_bert_path)
load_param_into_net(model_onehop_bert, param_dict)
model_twohop_bert = ModelOneHop(448)
param_dict2 = load_checkpoint(config.twohop_bert_path)
load_param_into_net(model_twohop_bert, param_dict2)
onehop = OneHopBert(config, model_onehop_bert)
twohop = TwoHopBert(config, model_twohop_bert)
print("loading model time (h):", (time.time() - s_lm) / 3600)
print('********************** evaluation ********************** ')
f_dev = open(config.dev_path, 'rb')
dev_data = json.load(f_dev)
f_dev.close()
q_gold = {}
q_2id = {}
for onedata in dev_data:
if onedata["question"] not in q_gold:
q_gold[onedata["question"]] = [
get_new_title(get_raw_title(item)) for item in onedata['path']
]
q_2id[onedata["question"]] = onedata['_id']
val, true_count, count, step = 0, 0, 0, 0
batch_queries = split_queries(config, queries)
output_path = []
for _, batch in enumerate(batch_queries):
print("###step###: ", str(step) + "_" + str(d_id))
query = batch[0]
temp_dict = {}
temp_dict['q_id'] = q_2id[query]
temp_dict['question'] = query
gold_path = q_gold[query]
input_ids_1, token_type_ids_1, input_mask_1 = data_generator.convert_onehop_to_features(
batch)
start = 0
TOTAL = len(input_ids_1)
split_chunk = 8
while start < TOTAL:
end = min(start + split_chunk - 1, TOTAL - 1)
chunk_len = end - start + 1
input_ids_1_ = input_ids_1[start:start + chunk_len]
input_ids_1_ = Tensor(input_ids_1_, mstype.int32)
token_type_ids_1_ = token_type_ids_1[start:start + chunk_len]
token_type_ids_1_ = Tensor(token_type_ids_1_, mstype.int32)
input_mask_1_ = input_mask_1[start:start + chunk_len]
input_mask_1_ = Tensor(input_mask_1_, mstype.int32)
cls_out = onehop(input_ids_1_, token_type_ids_1_, input_mask_1_)
if start == 0:
out = cls_out
else:
out = P.Concat(0)((out, cls_out))
start = end + 1
out = P.Squeeze(1)(out)
onehop_prob, onehop_index = P.TopK(sorted=True)(out, config.topk)
onehop_prob = P.Softmax()(onehop_prob)
sample, path_raw, last_out = data_generator.get_samples(
query, onehop_index, onehop_prob)
input_ids_2, token_type_ids_2, input_mask_2 = data_generator.convert_twohop_to_features(
sample)
start_2 = 0
TOTAL_2 = len(input_ids_2)
split_chunk = 8
while start_2 < TOTAL_2:
end_2 = min(start_2 + split_chunk - 1, TOTAL_2 - 1)
chunk_len = end_2 - start_2 + 1
input_ids_2_ = input_ids_2[start_2:start_2 + chunk_len]
input_ids_2_ = Tensor(input_ids_2_, mstype.int32)
token_type_ids_2_ = token_type_ids_2[start_2:start_2 + chunk_len]
token_type_ids_2_ = Tensor(token_type_ids_2_, mstype.int32)
input_mask_2_ = input_mask_2[start_2:start_2 + chunk_len]
input_mask_2_ = Tensor(input_mask_2_, mstype.int32)
cls_out = twohop(input_ids_2_, token_type_ids_2_, input_mask_2_)
if start_2 == 0:
out_2 = cls_out
else:
out_2 = P.Concat(0)((out_2, cls_out))
start_2 = end_2 + 1
out_2 = P.Softmax()(out_2)
last_out = Tensor(last_out, mstype.float32)
out_2 = P.Mul()(out_2, last_out)
val, true_count, topk_titles = eval_output(out_2, last_out, path_raw,
gold_path, val, true_count)
temp_dict['topk_titles'] = topk_titles
output_path.append(temp_dict)
step += 1
count += 1
return {
'val': val,
'count': count,
'true_count': true_count,
'path': output_path
}
def _load_weights(self, model_path: str = None) -> None:
param_dict = load_checkpoint(model_path)
load_param_into_net(self, param_dict)
help="device target (default: Ascend)")
args = parser.parse_args()
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target)
if args.device_target == "Ascend":
context.set_context(device_id=args.device_id)
if __name__ == '__main__':
widedeep_config = WideDeepConfig()
widedeep_config.argparse_init()
net_builder = ModelBuilder()
_, eval_net = net_builder.get_net(widedeep_config)
param_dict = load_checkpoint(args.ckpt_file)
load_param_into_net(eval_net, param_dict)
eval_net.set_train(False)
ids = Tensor(
np.ones([widedeep_config.eval_batch_size,
widedeep_config.field_size]).astype(np.int32))
wts = Tensor(
np.ones([widedeep_config.eval_batch_size,
widedeep_config.field_size]).astype(np.float32))
label = Tensor(
np.ones([widedeep_config.eval_batch_size, 1]).astype(np.float32))
input_tensor_list = [ids, wts, label]
export(eval_net,
*input_tensor_list,
file_name=args.file_name,
file_format=args.file_format)
def main():
set_seed(1)
date = time.strftime("%Y%m%d%H%M%S", time.localtime())
print(f'* Preparing to train model {date}')
# ************** configuration ****************
# - training setting
resume = config['resume']
if config['mode'] == 'PYNATIVE':
mode = context.PYNATIVE_MODE
else:
mode = context.GRAPH_MODE
device = config['device']
device_id = config['device_id']
dataset_sink_mode = config['dataset_sink_mode']
# use in dataset
div = 8
# setting bias and padding
if resume:
print('* Resuming model...')
resume_config_log = config['resume_config_log']
resume_config = get_eval_config(resume_config_log)
if 'best_ckpt' in resume_config.keys():
resume_model_path = resume_config['best_ckpt']
else:
resume_model_path = resume_config['latest_model']
print('* [WARNING] Not using the best model, but latest saved model instead.')
has_bias = resume_config['has_bias']
use_dropout = resume_config['use_dropout']
pad_mode = resume_config['pad_mode']
if pad_mode == 'pad':
padding = resume_config['padding']
elif pad_mode == 'same':
padding = 0
else:
raise ValueError(f"invalid pad mode: {pad_mode}!")
best_acc = resume_config['best_acc']
best_ckpt = resume_config['best_ckpt']
print('* The best accuracy in dev dataset for the current resumed model is {:.2f}%'.format(best_acc * 100))
else:
has_bias = config['has_bias']
use_dropout = config['use_dropout']
pad_mode = config['pad_mode']
if pad_mode == 'pad':
padding = config['padding']
elif pad_mode == 'same':
padding = 0
else:
raise ValueError(f"invalid pad mode: {pad_mode}!")
# hyper-parameters
if resume:
batch_size = resume_config['batch_size']
opt_type = resume_config['opt']
use_dynamic_lr = resume_config['use_dynamic_lr']
warmup_step = resume_config['warmup_step']
warmup_ratio = resume_config['warmup_ratio']
else:
batch_size = config['batch_size']
opt_type = config['opt']
use_dynamic_lr = config['use_dynamic_lr']
warmup_step = config['warmup_step']
warmup_ratio = config['warmup_ratio']
test_dev_batch_size = config['test_dev_batch_size']
learning_rate = float(config['learning_rate'])
epochs = config['epochs']
loss_scale = config['loss_scale']
# configuration of saving model checkpoint
save_checkpoint_steps = config['save_checkpoint_steps']
keep_checkpoint_max = config['keep_checkpoint_max']
prefix = config['prefix'] + '_' + date
model_dir = config['model_dir']
# loss monitor
loss_monitor_step = config['loss_monitor_step']
# whether to use mindInsight summary
use_summary = config['use_summary']
# step_eval
use_step_eval = config['use_step_eval']
eval_step = config['eval_step']
eval_epoch = config['eval_epoch']
patience = config['patience']
# eval in steps or epochs
step_eval = True
if eval_step == -1:
step_eval = False
# ************** end of configuration **************
if device == 'GPU':
context.set_context(mode=mode, device_target=device, device_id=device_id)
elif device == 'Ascend':
import moxing as mox
from utils.const import DATA_PATH, MODEL_PATH, BEST_MODEL_PATH, LOG_PATH
obs_datapath = config['obs_datapath']
obs_saved_model = config['obs_saved_model']
obs_best_model = config['obs_best_model']
obs_log = config['obs_log']
mox.file.copy_parallel(obs_datapath, DATA_PATH)
mox.file.copy_parallel(MODEL_PATH, obs_saved_model)
mox.file.copy_parallel(BEST_MODEL_PATH, obs_best_model)
mox.file.copy_parallel(LOG_PATH, obs_log)
context.set_context(mode=mode, device_target=device)
use_summary = False
# callbacks function
callbacks = []
# data
train_loader, idx2label, label2idx = get_dataset(batch_size=batch_size, phase='train',
test_dev_batch_size=test_dev_batch_size, div=div,
num_parallel_workers=4)
if eval_step == 0:
eval_step = train_loader.get_dataset_size()
# network
net = DFCNN(num_classes=len(label2idx), padding=padding, pad_mode=pad_mode,
has_bias=has_bias, use_dropout=use_dropout)
# Criterion
criterion = CTCLoss()
# resume
if resume:
print("* Loading parameters...")
param_dict = load_checkpoint(resume_model_path)
# load the parameter into net
load_param_into_net(net, param_dict)
print(f'* Parameters loading from {resume_model_path} succeeded!')
net.set_train(True)
net.set_grad(True)
# lr schedule
if use_dynamic_lr:
dataset_size = train_loader.get_dataset_size()
learning_rate = Tensor(dynamic_lr(base_lr=learning_rate, warmup_step=warmup_step,
warmup_ratio=warmup_ratio, epochs=epochs,
steps_per_epoch=dataset_size), mstype.float32)
print('* Using dynamic learning rate, which will be set up as :', learning_rate.asnumpy())
# optim
if opt_type == 'adam':
opt = nn.Adam(net.trainable_params(), learning_rate=learning_rate, beta1=0.9, beta2=0.999, weight_decay=0.0,
eps=10e-8)
elif opt_type == 'rms':
opt = nn.RMSProp(params=net.trainable_params(),
centered=True,
learning_rate=learning_rate,
momentum=0.9,
loss_scale=loss_scale)
elif opt_type == 'sgd':
opt = nn.SGD(params=net.trainable_params(), learning_rate=learning_rate)
else:
raise ValueError(f"optimizer: {opt_type} is not supported for now!")
if resume:
# load the parameter into optimizer
load_param_into_net(opt, param_dict)
# save_model
config_ck = CheckpointConfig(save_checkpoint_steps=save_checkpoint_steps, keep_checkpoint_max=keep_checkpoint_max)
ckpt_cb = ModelCheckpoint(prefix=prefix, directory=model_dir, config=config_ck)
# logger
the_logger = logger(config, date)
log = Logging(logger=the_logger, model_ckpt=ckpt_cb)
callbacks.append(ckpt_cb)
callbacks.append(log)
net = WithLossCell(net, criterion)
scaling_sens = Tensor(np.full((1), loss_scale), dtype=mstype.float32)
net = DFCNNCTCTrainOneStepWithLossScaleCell(net, opt, scaling_sens)
net.set_train(True)
model = Model(net)
if use_step_eval:
# step evaluation
step_eval = StepAccInfo(model=model, name=prefix, div=div, test_dev_batch_size=test_dev_batch_size,
step_eval=step_eval, eval_step=eval_step, eval_epoch=eval_epoch,
logger=the_logger, patience=patience, dataset_size=train_loader.get_dataset_size())
callbacks.append(step_eval)
# loss monitor
loss_monitor = LossMonitor(loss_monitor_step)
callbacks.append(loss_monitor)
if use_summary:
summary_dir = os.path.join(SUMMARY_DIR, date)
if not os.path.exists(summary_dir):
os.mkdir(summary_dir)
# mindInsight
summary_collector = SummaryCollector(summary_dir=summary_dir, collect_freq=1, max_file_size=4 * 1024 ** 3)
callbacks.append(summary_collector)
if resume:
the_logger.update_acc_ckpt(best_acc, best_ckpt)
print(f'* Start training...')
model.train(epochs,
train_loader,
callbacks=callbacks,
dataset_sink_mode=dataset_sink_mode)
def test_train():
'''
finetune function
'''
target = args_opt.device_target
if target == "Ascend":
devid = int(os.getenv('DEVICE_ID'))
context.set_context(mode=context.GRAPH_MODE,
device_target="Ascend",
device_id=devid)
poetry, tokenizer, keep_words = create_tokenizer()
print(len(keep_words))
dataset = create_poetry_dataset(bert_net_cfg.batch_size, poetry, tokenizer)
num_tokens = 3191
poetrymodel = BertPoetryModel(bert_net_cfg,
True,
num_tokens,
dropout_prob=0.1)
netwithloss = BertPoetry(poetrymodel, bert_net_cfg, True, dropout_prob=0.1)
callback = LossCallBack(poetrymodel)
# optimizer
steps_per_epoch = dataset.get_dataset_size()
print("============ steps_per_epoch is {}".format(steps_per_epoch))
lr_schedule = BertLearningRate(
learning_rate=cfg.AdamWeightDecay.learning_rate,
end_learning_rate=cfg.AdamWeightDecay.end_learning_rate,
warmup_steps=1000,
decay_steps=cfg.epoch_num * steps_per_epoch,
power=cfg.AdamWeightDecay.power)
optimizer = AdamWeightDecay(netwithloss.trainable_params(), lr_schedule)
# load checkpoint into network
ckpt_config = CheckpointConfig(save_checkpoint_steps=steps_per_epoch,
keep_checkpoint_max=1)
ckpoint_cb = ModelCheckpoint(prefix=cfg.ckpt_prefix,
directory=cfg.ckpt_dir,
config=ckpt_config)
param_dict = load_checkpoint(cfg.pre_training_ckpt)
new_dict = {}
# load corresponding rows of embedding_lookup
for key in param_dict:
if "bert_embedding_lookup" not in key:
new_dict[key] = param_dict[key]
else:
value = param_dict[key]
np_value = value.data.asnumpy()
np_value = np_value[keep_words]
tensor_value = Tensor(np_value, mstype.float32)
parameter_value = Parameter(tensor_value, name=key)
new_dict[key] = parameter_value
load_param_into_net(netwithloss, new_dict)
update_cell = DynamicLossScaleUpdateCell(loss_scale_value=2**32,
scale_factor=2,
scale_window=1000)
netwithgrads = BertPoetryCell(netwithloss,
optimizer=optimizer,
scale_update_cell=update_cell)
model = Model(netwithgrads)
model.train(cfg.epoch_num,
dataset,
callbacks=[callback, ckpoint_cb],
dataset_sink_mode=True)
parser.add_argument('--device_target',
type=str,
default=None,
help='Run device target')
args_opt = parser.parse_args()
if __name__ == '__main__':
context.set_context(mode=context.GRAPH_MODE,
device_target=args_opt.device_target,
save_graphs=False)
# define manual quantization network
network = resnet50_quant(class_num=config_quant.class_num)
# load checkpoint
if args_opt.checkpoint_path:
param_dict = load_checkpoint(args_opt.checkpoint_path)
not_load_param = load_param_into_net(network, param_dict)
if not_load_param:
raise ValueError("Load param into network fail!")
# export network
print("============== Starting export ==============")
inputs = Tensor(np.ones([1, 3, 224, 224]), mindspore.float32)
export(network,
inputs,
file_name="resnet50_quant",
file_format=args_opt.file_format,
quant_mode='QUANT',
mean=0.,
std_dev=48.106)
print("============== End export ==============")
parser.add_argument("--device_target",
type=str,
choices=["Ascend", "GPU", "CPU"],
default="GPU",
help="device target")
args = parser.parse_args()
context.set_context(mode=context.GRAPH_MODE,
device_target=args.device_target,
device_id=args.device_id)
if __name__ == "__main__":
if args.device_target != "GPU":
raise ValueError("Only supported GPU now.")
net = efficientnet_b0(
num_classes=cfg.num_classes,
drop_rate=cfg.drop,
drop_connect_rate=cfg.drop_connect,
global_pool=cfg.gp,
bn_tf=cfg.bn_tf,
)
ckpt = load_checkpoint(args.ckpt_file)
load_param_into_net(net, ckpt)
net.set_train(False)
image = Tensor(
np.ones([cfg.batch_size, 3, args.height, args.width], np.float32))
export(net, image, file_name=args.file_name, file_format=args.file_format)
