def __eval(self): # import from mindspore import Model, load_param_into_net, load_checkpoint from mindspore.nn.metrics import Accuracy # load params if self.__ckpt_path: load_param_into_net(self.__network, load_checkpoint(self.__ckpt_path)) else: print( "Warning: `ckpt_path` is None, Please call func: `set_ckpt_path($ckpt_path)`." ) return # loss_fn & optimizer & metrics model = Model(self.__network, loss_fn=self.__loss_fn, optimizer=self.__optimizer, metrics={"Accuracy": Accuracy()} if self.__metrics is None else self.__metrics) # eval print(">>>>>>>>>>>>>>>>>>>>> eval start ... <<<<<<<<<<<<<<<<<<<<<<") result = model.eval(self.__dataset) print( ">>>>>>>>>>>>>>>>>>>>> eval success ~ <<<<<<<<<<<<<<<<<<<<<<: result=", result)
def __init__(self, config, network): super(OneHopBert, self).__init__(auto_prefix=False) self.network = network self.mlp = Model() param_dict = load_checkpoint(config.onehop_mlp_path) load_param_into_net(self.mlp, param_dict) self.cast = P.Cast()
def epoch_end(self, run_context): if self.step_eval: return None flag = '' cb_params = run_context.original_args() cur_epoch = cb_params.cur_epoch_num if cur_epoch > 0 and cur_epoch % self.eval_epoch == 0: acc = self.eval() if acc > self.best_acc: flag = '↑' self.best_acc = acc save_checkpoint(self.model._network, self.best_ckpt) self.logger.update_acc_ckpt(acc, self.best_ckpt) else: if acc > self.threshold: self.patience_count += 1 if self.patience_count > self.patience: param_dict = load_checkpoint( ckpt_file_name=self.best_ckpt, net=self.model._network) load_param_into_net(net=self.model._network, parameter_dict=param_dict) self.patience_count = 0 print( f'* acc for epoch: {cur_epoch} is {acc * 100}%{flag}, best acc is {self.best_acc * 100}%' )
def __init__(self, requires_grad=False): super(Vgg19, self).__init__() ##load vgg16 vgg = Vgg(cfg['19'], phase="test", args=imagenet_cfg) # model = os.path.join(opt.data_url, 'vgg19_ImageNet.ckpt') model = os.path.join('./', 'vgg19_ImageNet.ckpt') print(model) param_dict = load_checkpoint(model) # print(param_dict) load_param_into_net(vgg, param_dict) vgg.set_train(False) vgg_pretrained_features = vgg.layers self.slice1 = nn.SequentialCell() self.slice2 = nn.SequentialCell() self.slice3 = nn.SequentialCell() self.slice4 = nn.SequentialCell() self.slice5 = nn.SequentialCell() for x in range(2): self.slice1.append(vgg_pretrained_features[x]) for x in range(2, 7): self.slice2.append(vgg_pretrained_features[x]) for x in range(7, 12): self.slice3.append(vgg_pretrained_features[x]) for x in range(12, 21): self.slice4.append(vgg_pretrained_features[x]) for x in range(21, 30): self.slice5.append(vgg_pretrained_features[x]) if not requires_grad: for param in self.get_parameters(): param.requires_grad = False
def progress(self): self.current_scale += 1 # Lower scale discriminators are not used in later ... replace append to assign? if self.current_scale % 4 == 0: self.nf *= 2 tmp_discriminator = nn.CellList() tmp_discriminator.append( nn.SequentialCell(nn.Conv2d(3, self.nf, 3, 1, 1), nn.LeakyReLU(2e-1))) for _ in range(3): tmp_discriminator.append( nn.SequentialCell(nn.Conv2d(self.nf, self.nf, 3, 1, 1), nn.BatchNorm2d(self.nf), nn.LeakyReLU(2e-1))) tmp_discriminator.append( nn.SequentialCell(nn.Conv2d(self.nf, 1, 3, 1, 1))) tmp_discriminator = nn.SequentialCell(*tmp_discriminator) if self.current_scale % 4 != 0: prev_discriminator = self.sub_discriminators[-1] # Initialize layers via copy if self.current_scale >= 1: mindspore.load_param_into_net( tmp_discriminator, prev_discriminator.parameters_dict()) self.sub_discriminators.append(tmp_discriminator) print("DISCRIMINATOR PROGRESSION DONE")
def __train(self): # import from mindspore import Model, load_param_into_net, load_checkpoint # load params if self.__ckpt_path and os.path.isfile( self.__ckpt_path) and os.path.exists(self.__ckpt_path): load_param_into_net(self.__network, load_checkpoint(self.__ckpt_path)) # loss_fn & optimizer & metrics model = Model(self.__network, loss_fn=self.__loss_fn, optimizer=self.__optimizer, metrics=self.__metrics) # train print(">>>>>>>>>>>>>>>>>>>>> train start ... <<<<<<<<<<<<<<<<<<<<<<") self.__update_cbs() model.train(self.__epoch_size, self.__dataset, callbacks=self.__callbacks, dataset_sink_mode=self.__do_sink, sink_size=self.__sink_size) print(">>>>>>>>>>>>>>>>>>>>> train success ~ <<<<<<<<<<<<<<<<<<<<<<")
def load_weights(self, weights): """Load the model weights passed in as a parameter.""" for name, weight in weights.items(): weights[name] = mindspore.Parameter(weight, name=name) # One can also use `self.model.load_parameter_slice(weights)', which # seems to be equivalent to mindspore.load_param_into_net() in its effects mindspore.load_param_into_net(self.model, weights, strict_load=True)
def test_net(network, model, mnist_path): """Define the evaluation method""" print("==================== Starting Testing ===============") param_dict = load_checkpoint( "./model/ckpt/mindspore_quick_start/checkpoint_lenet-1_1874.ckpt") load_param_into_net(network, param_dict) ds_eval = create_dataset(os.path.join(mnist_path, "test")) acc = model.eval(ds_eval, dataset_sink_mode=False) print("==================== Accuracy:{} ===============".format(acc))
def test_net(network, network_model, data_path): """Define the evaluation method.""" print("============== Starting Testing ==============") # load the saved model for evaluation param_dict = load_checkpoint("checkpoint_lenet-1_1875.ckpt") # load parameter to the network load_param_into_net(network, param_dict) # load testing dataset ds_eval = create_dataset(os.path.join(data_path, "test")) acc = network_model.eval(ds_eval, dataset_sink_mode=False) print("============== Accuracy:{} ==============".format(acc))
def test_eval(model_ckpt_path): '''eval model''' target = args_opt.device_target if target == "Ascend": devid = int(os.getenv('DEVICE_ID')) context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) bert_net_cfg.batch_size = 1 poetrymodel = BertPoetryModel(bert_net_cfg, False, 3191, dropout_prob=0.0) poetrymodel.set_train(False) param_dict = load_checkpoint(model_ckpt_path) load_param_into_net(poetrymodel, param_dict) # random generation/continue start_time = time.time() output = generate_random_poetry(poetrymodel, s='') end_to_end_delay = (time.time() - start_time) * 1000 a = re.findall(r'[\u4e00-\u9fa5]*[\uff0c\u3002]', output) print("\n**********************************") print("随机生成: \n") for poem in a: print(poem) print("\ncost time: {:.1f} ms".format(end_to_end_delay)) print("\n") start = "天下为公" start_time = time.time() output = generate_random_poetry(poetrymodel, s=start) end_to_end_delay = (time.time() - start_time) * 1000 a = re.findall(r'[\u4e00-\u9fa5]*[\uff0c\u3002]', output) print("\n**********************************") print("续写 【{}】: \n".format(start)) for poem in a: print(poem) print("\ncost time: {:.1f} ms".format(end_to_end_delay)) print("\n") # hidden poetry s = "人工智能" start_time = time.time() output = generate_hidden(poetrymodel, head=s) end_to_end_delay = (time.time() - start_time) * 1000 a = re.findall(r'[\u4e00-\u9fa5]*[\uff0c\u3002]', output) print("\n**********************************") print("藏头诗 【{}】: \n".format(s)) for poem in a: print(poem) print("\ncost time: {:.1f} ms".format(end_to_end_delay)) print("\n")
def export_net(model_ckpt_path): bert_net_cfg.batch_size = 1 poetrymodel = BertPoetryModel(bert_net_cfg, False, 3191, dropout_prob=0.0) poetrymodel.set_train(False) param_dict = load_checkpoint(model_ckpt_path) load_param_into_net(poetrymodel, param_dict) input_id = np.ones(shape=(1, 128)) token_type_id = np.ones(shape=(1, 128)) pad_mask = np.ones(shape=(1, 128)) export(poetrymodel, Tensor(input_id, mstype.int32),\ Tensor(token_type_id, mstype.int32),\ Tensor(pad_mask, mstype.float32),\ file_name='poetry.pb', file_format='MINDIR')
def __init__(self, batch_size, encoder_ck_file, downstream_ck_file): """init function""" super(Reranker, self).__init__(auto_prefix=False) self.encoder = Albert(batch_size) param_dict = load_checkpoint(encoder_ck_file) not_load_params_1 = load_param_into_net(self.encoder, param_dict) print(f"re-ranker albert not loaded params: {not_load_params_1}") self.no_answer_mlp = Rerank_Downstream() param_dict = load_checkpoint(downstream_ck_file) not_load_params_2 = load_param_into_net(self.no_answer_mlp, param_dict) print(f"re-ranker downstream not loaded params: {not_load_params_2}")
def run_export(args): """ export """ device_id = int(os.getenv('DEVICE_ID', '0')) context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=device_id) net = RCAN(arg) param_dict = load_checkpoint(args.ckpt_path) load_param_into_net(net, param_dict) net.set_train(False) print('load mindspore net and checkpoint successfully.') inputs = Tensor(np.zeros([args.batch_size, 3, 678, 1020], np.float32)) export(net, inputs, file_name=args.file_name, file_format=args.file_format) print('export successfully!')
def __init__(self, batch_size, encoder_ck_file, downstream_ck_file): """init function""" super(Reader, self).__init__(auto_prefix=False) self.encoder = Reader_Albert(batch_size) param_dict = load_checkpoint(encoder_ck_file) not_load_params = load_param_into_net(self.encoder, param_dict) print(f"not loaded: {not_load_params}") self.downstream = Reader_Downstream() param_dict = load_checkpoint(downstream_ck_file) not_load_params = load_param_into_net(self.downstream, param_dict) print(f"not loaded: {not_load_params}") self.bmm = BatchMatMul()
def test_net(network, network_model, data_path): """Define the evaluation method.""" print("============== Starting Testing ==============") # load the saved model for evaluation latest_file = newest(data_path) print(latest_file) param_dict = load_checkpoint(latest_file) # load parameter to the network load_param_into_net(network, param_dict) # load testing dataset ds_eval = dm.create_dataset(os.path.join( data_path, "./MindSpore_train_images_dataset/test"), do_train=False) acc = network_model.eval(ds_eval, dataset_sink_mode=False) print("============== Accuracy:{} ==============".format(acc))
def progress(self): self.current_scale += 1 if self.current_scale % 4 == 0: self.nf *= 2 tmp_generator = nn.CellList() tmp_generator.append( nn.SequentialCell(nn.Conv2d(3, self.nf, 3, 1), nn.BatchNorm2d(self.nf), nn.LeakyReLU(2e-1))) for _ in range(3): tmp_generator.append( nn.SequentialCell(nn.Conv2d(self.nf, self.nf, 3, 1), nn.BatchNorm2d(self.nf), nn.LeakyReLU(2e-1))) tmp_generator.append( nn.SequentialCell(nn.Conv2d(self.nf, 3, 3, 1), nn.Tanh())) tmp_generator = nn.SequentialCell(*tmp_generator) if self.current_scale % 4 != 0: prev_generator = self.sub_generators[-1] # Initialize layers via copy if self.current_scale >= 1: tmp_generator = mindspore.load_param_into_net( tmp_generator, prev_generator.parameters_dict) # 以python的字典格式加载存储 self.sub_generators.append(tmp_generator) print("GENERATOR PROGRESSION DONE")
def eval_quant(): context.set_context(mode=context.GRAPH_MODE, device_target=device_target) cfg = quant_cfg ds_eval = create_dataset(os.path.join(data_path, "test"), cfg.batch_size, 1) ckpt_path = './ckpt_lenet_quant-10_937.ckpt' # define fusion network network = LeNet5Fusion(cfg.num_classes) # convert fusion network to quantization aware network quantizer = QuantizationAwareTraining(quant_delay=0, bn_fold=False, freeze_bn=10000, per_channel=[True, False], symmetric=[True, False]) network = quantizer.quantize(network) # define loss net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") # define network optimization net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) # call back and monitor model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) # load quantization aware network checkpoint param_dict = load_checkpoint(ckpt_path) not_load_param = load_param_into_net(network, param_dict) if not_load_param: raise ValueError("Load param into net fail!") print("============== Starting Testing ==============") acc = model.eval(ds_eval, dataset_sink_mode=True) print("============== {} ==============".format(acc)) assert acc['Accuracy'] > 0.98
def load_model(self, filename=None): """Loading pre-trained model weights from a file.""" model_name = Config().trainer.model_name model_dir = Config().params['pretrained_model_dir'] if filename is not None: model_path = f'{model_dir}{filename}' else: model_path = f'{model_dir}{model_name}.ckpt' if self.client_id == 0: logging.info("[Server #%d] Loading a model from %s.", os.getpid(), model_path) else: logging.info("[Client #%d] Loading a model from %s.", self.client_id, model_path) param_dict = mindspore.load_checkpoint(model_path) mindspore.load_param_into_net(self.model, param_dict)
def source_checker(py_path, ckpt_path): """Check source model script and source checkpoint file.""" sys.path.append(os.path.dirname(py_path)) model = getattr( import_module(os.path.basename(py_path).replace('.py', '')), 'Model')() try: param_dict = load_checkpoint(ckpt_path) not_load_name = load_param_into_net(model, param_dict) return not not_load_name except (ValueError, TypeError, RuntimeError): return False
def test_net(network, model, data_path): """Define the evaluation method.""" print("============== Starting Testing ==============") # load the saved model for evaluation checkpoint = "./101.ckpt" _t = time.time() param_dict = load_checkpoint(checkpoint) # load parameter to the network load_param_into_net(network, param_dict) print(f"Load params to model cost: {time.time() - _t} s") # inference print("============== Infer Testing ==============") input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]), mindspore.float32) infer_time = 0 while infer_time < 21: _t = time.time() model.predict(input_data) print(f"Model infer cost: {time.time() - _t} s") infer_time += 1
def eval_lenet(): context.set_context(mode=context.GRAPH_MODE, device_target=device_target) cfg = nonquant_cfg ds_eval = create_dataset(os.path.join(data_path, "test"), cfg.batch_size, 1) ckpt_path = './ckpt_lenet_noquant-10_1875.ckpt' # define fusion network network = LeNet5(cfg.num_classes) net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) # call back and monitor model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) # load quantization aware network checkpoint param_dict = load_checkpoint(ckpt_path) not_load_param = load_param_into_net(network, param_dict) if not_load_param: raise ValueError("Load param into net fail!") print("============== Starting Testing ==============") acc = model.eval(ds_eval, dataset_sink_mode=True) print("============== {} ==============".format(acc)) assert acc['Accuracy'] > 0.98
required=True, help='inceptionv3 ckpt file.') parser.add_argument('--output_file', type=str, default='inceptionv3.air', help='inceptionv3 output air name.') parser.add_argument('--file_format', type=str, choices=["AIR", "ONNX", "MINDIR"], default='AIR', help='file format') parser.add_argument('--width', type=int, default=299, help='input width') parser.add_argument('--height', type=int, default=299, help='input height') args = parser.parse_args() if __name__ == '__main__': net = InceptionV3(num_classes=cfg.num_classes, is_training=False) param_dict = load_checkpoint(args.ckpt_file) load_param_into_net(net, param_dict) input_arr = Tensor( np.random.uniform(0.0, 1.0, size=[cfg.batch_size, 3, args.width, args.height]), ms.float32) export(net, input_arr, file_name=args.output_file, file_format=args.file_format)
from src.lenet import LeNet5 parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument( '--device_target', type=str, default="Ascend", choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument( '--ckpt_path', type=str, default="", help='if mode is test, must provide path where the trained ckpt file') args = parser.parse_args() if __name__ == "__main__": context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target) # define fusion network network = LeNet5(cfg.num_classes) # load network checkpoint param_dict = load_checkpoint(args.ckpt_path) load_param_into_net(network, param_dict) # export network inputs = Tensor(np.ones([1, 1, cfg.image_height, cfg.image_width]), mindspore.float32) export(network, inputs, file_name=cfg.air_name, file_format='AIR')
def evaluation(d_id): """evaluation""" context.set_context(mode=context.GRAPH_MODE, device_target='Ascend', device_id=d_id, save_graphs=False) print('********************** loading corpus ********************** ') s_lc = time.time() data_generator = DataGen(config) queries = read_query(config, d_id) print("loading corpus time (h):", (time.time() - s_lc) / 3600) print('********************** loading model ********************** ') s_lm = time.time() model_onehop_bert = ModelOneHop(256) param_dict = load_checkpoint(config.onehop_bert_path) load_param_into_net(model_onehop_bert, param_dict) model_twohop_bert = ModelOneHop(448) param_dict2 = load_checkpoint(config.twohop_bert_path) load_param_into_net(model_twohop_bert, param_dict2) onehop = OneHopBert(config, model_onehop_bert) twohop = TwoHopBert(config, model_twohop_bert) print("loading model time (h):", (time.time() - s_lm) / 3600) print('********************** evaluation ********************** ') f_dev = open(config.dev_path, 'rb') dev_data = json.load(f_dev) f_dev.close() q_gold = {} q_2id = {} for onedata in dev_data: if onedata["question"] not in q_gold: q_gold[onedata["question"]] = [ get_new_title(get_raw_title(item)) for item in onedata['path'] ] q_2id[onedata["question"]] = onedata['_id'] val, true_count, count, step = 0, 0, 0, 0 batch_queries = split_queries(config, queries) output_path = [] for _, batch in enumerate(batch_queries): print("###step###: ", str(step) + "_" + str(d_id)) query = batch[0] temp_dict = {} temp_dict['q_id'] = q_2id[query] temp_dict['question'] = query gold_path = q_gold[query] input_ids_1, token_type_ids_1, input_mask_1 = data_generator.convert_onehop_to_features( batch) start = 0 TOTAL = len(input_ids_1) split_chunk = 8 while start < TOTAL: end = min(start + split_chunk - 1, TOTAL - 1) chunk_len = end - start + 1 input_ids_1_ = input_ids_1[start:start + chunk_len] input_ids_1_ = Tensor(input_ids_1_, mstype.int32) token_type_ids_1_ = token_type_ids_1[start:start + chunk_len] token_type_ids_1_ = Tensor(token_type_ids_1_, mstype.int32) input_mask_1_ = input_mask_1[start:start + chunk_len] input_mask_1_ = Tensor(input_mask_1_, mstype.int32) cls_out = onehop(input_ids_1_, token_type_ids_1_, input_mask_1_) if start == 0: out = cls_out else: out = P.Concat(0)((out, cls_out)) start = end + 1 out = P.Squeeze(1)(out) onehop_prob, onehop_index = P.TopK(sorted=True)(out, config.topk) onehop_prob = P.Softmax()(onehop_prob) sample, path_raw, last_out = data_generator.get_samples( query, onehop_index, onehop_prob) input_ids_2, token_type_ids_2, input_mask_2 = data_generator.convert_twohop_to_features( sample) start_2 = 0 TOTAL_2 = len(input_ids_2) split_chunk = 8 while start_2 < TOTAL_2: end_2 = min(start_2 + split_chunk - 1, TOTAL_2 - 1) chunk_len = end_2 - start_2 + 1 input_ids_2_ = input_ids_2[start_2:start_2 + chunk_len] input_ids_2_ = Tensor(input_ids_2_, mstype.int32) token_type_ids_2_ = token_type_ids_2[start_2:start_2 + chunk_len] token_type_ids_2_ = Tensor(token_type_ids_2_, mstype.int32) input_mask_2_ = input_mask_2[start_2:start_2 + chunk_len] input_mask_2_ = Tensor(input_mask_2_, mstype.int32) cls_out = twohop(input_ids_2_, token_type_ids_2_, input_mask_2_) if start_2 == 0: out_2 = cls_out else: out_2 = P.Concat(0)((out_2, cls_out)) start_2 = end_2 + 1 out_2 = P.Softmax()(out_2) last_out = Tensor(last_out, mstype.float32) out_2 = P.Mul()(out_2, last_out) val, true_count, topk_titles = eval_output(out_2, last_out, path_raw, gold_path, val, true_count) temp_dict['topk_titles'] = topk_titles output_path.append(temp_dict) step += 1 count += 1 return { 'val': val, 'count': count, 'true_count': true_count, 'path': output_path }
def _load_weights(self, model_path: str = None) -> None: param_dict = load_checkpoint(model_path) load_param_into_net(self, param_dict)
help="device target (default: Ascend)") args = parser.parse_args() context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target) if args.device_target == "Ascend": context.set_context(device_id=args.device_id) if __name__ == '__main__': widedeep_config = WideDeepConfig() widedeep_config.argparse_init() net_builder = ModelBuilder() _, eval_net = net_builder.get_net(widedeep_config) param_dict = load_checkpoint(args.ckpt_file) load_param_into_net(eval_net, param_dict) eval_net.set_train(False) ids = Tensor( np.ones([widedeep_config.eval_batch_size, widedeep_config.field_size]).astype(np.int32)) wts = Tensor( np.ones([widedeep_config.eval_batch_size, widedeep_config.field_size]).astype(np.float32)) label = Tensor( np.ones([widedeep_config.eval_batch_size, 1]).astype(np.float32)) input_tensor_list = [ids, wts, label] export(eval_net, *input_tensor_list, file_name=args.file_name, file_format=args.file_format)
def main(): set_seed(1) date = time.strftime("%Y%m%d%H%M%S", time.localtime()) print(f'* Preparing to train model {date}') # ************** configuration **************** # - training setting resume = config['resume'] if config['mode'] == 'PYNATIVE': mode = context.PYNATIVE_MODE else: mode = context.GRAPH_MODE device = config['device'] device_id = config['device_id'] dataset_sink_mode = config['dataset_sink_mode'] # use in dataset div = 8 # setting bias and padding if resume: print('* Resuming model...') resume_config_log = config['resume_config_log'] resume_config = get_eval_config(resume_config_log) if 'best_ckpt' in resume_config.keys(): resume_model_path = resume_config['best_ckpt'] else: resume_model_path = resume_config['latest_model'] print('* [WARNING] Not using the best model, but latest saved model instead.') has_bias = resume_config['has_bias'] use_dropout = resume_config['use_dropout'] pad_mode = resume_config['pad_mode'] if pad_mode == 'pad': padding = resume_config['padding'] elif pad_mode == 'same': padding = 0 else: raise ValueError(f"invalid pad mode: {pad_mode}!") best_acc = resume_config['best_acc'] best_ckpt = resume_config['best_ckpt'] print('* The best accuracy in dev dataset for the current resumed model is {:.2f}%'.format(best_acc * 100)) else: has_bias = config['has_bias'] use_dropout = config['use_dropout'] pad_mode = config['pad_mode'] if pad_mode == 'pad': padding = config['padding'] elif pad_mode == 'same': padding = 0 else: raise ValueError(f"invalid pad mode: {pad_mode}!") # hyper-parameters if resume: batch_size = resume_config['batch_size'] opt_type = resume_config['opt'] use_dynamic_lr = resume_config['use_dynamic_lr'] warmup_step = resume_config['warmup_step'] warmup_ratio = resume_config['warmup_ratio'] else: batch_size = config['batch_size'] opt_type = config['opt'] use_dynamic_lr = config['use_dynamic_lr'] warmup_step = config['warmup_step'] warmup_ratio = config['warmup_ratio'] test_dev_batch_size = config['test_dev_batch_size'] learning_rate = float(config['learning_rate']) epochs = config['epochs'] loss_scale = config['loss_scale'] # configuration of saving model checkpoint save_checkpoint_steps = config['save_checkpoint_steps'] keep_checkpoint_max = config['keep_checkpoint_max'] prefix = config['prefix'] + '_' + date model_dir = config['model_dir'] # loss monitor loss_monitor_step = config['loss_monitor_step'] # whether to use mindInsight summary use_summary = config['use_summary'] # step_eval use_step_eval = config['use_step_eval'] eval_step = config['eval_step'] eval_epoch = config['eval_epoch'] patience = config['patience'] # eval in steps or epochs step_eval = True if eval_step == -1: step_eval = False # ************** end of configuration ************** if device == 'GPU': context.set_context(mode=mode, device_target=device, device_id=device_id) elif device == 'Ascend': import moxing as mox from utils.const import DATA_PATH, MODEL_PATH, BEST_MODEL_PATH, LOG_PATH obs_datapath = config['obs_datapath'] obs_saved_model = config['obs_saved_model'] obs_best_model = config['obs_best_model'] obs_log = config['obs_log'] mox.file.copy_parallel(obs_datapath, DATA_PATH) mox.file.copy_parallel(MODEL_PATH, obs_saved_model) mox.file.copy_parallel(BEST_MODEL_PATH, obs_best_model) mox.file.copy_parallel(LOG_PATH, obs_log) context.set_context(mode=mode, device_target=device) use_summary = False # callbacks function callbacks = [] # data train_loader, idx2label, label2idx = get_dataset(batch_size=batch_size, phase='train', test_dev_batch_size=test_dev_batch_size, div=div, num_parallel_workers=4) if eval_step == 0: eval_step = train_loader.get_dataset_size() # network net = DFCNN(num_classes=len(label2idx), padding=padding, pad_mode=pad_mode, has_bias=has_bias, use_dropout=use_dropout) # Criterion criterion = CTCLoss() # resume if resume: print("* Loading parameters...") param_dict = load_checkpoint(resume_model_path) # load the parameter into net load_param_into_net(net, param_dict) print(f'* Parameters loading from {resume_model_path} succeeded!') net.set_train(True) net.set_grad(True) # lr schedule if use_dynamic_lr: dataset_size = train_loader.get_dataset_size() learning_rate = Tensor(dynamic_lr(base_lr=learning_rate, warmup_step=warmup_step, warmup_ratio=warmup_ratio, epochs=epochs, steps_per_epoch=dataset_size), mstype.float32) print('* Using dynamic learning rate, which will be set up as :', learning_rate.asnumpy()) # optim if opt_type == 'adam': opt = nn.Adam(net.trainable_params(), learning_rate=learning_rate, beta1=0.9, beta2=0.999, weight_decay=0.0, eps=10e-8) elif opt_type == 'rms': opt = nn.RMSProp(params=net.trainable_params(), centered=True, learning_rate=learning_rate, momentum=0.9, loss_scale=loss_scale) elif opt_type == 'sgd': opt = nn.SGD(params=net.trainable_params(), learning_rate=learning_rate) else: raise ValueError(f"optimizer: {opt_type} is not supported for now!") if resume: # load the parameter into optimizer load_param_into_net(opt, param_dict) # save_model config_ck = CheckpointConfig(save_checkpoint_steps=save_checkpoint_steps, keep_checkpoint_max=keep_checkpoint_max) ckpt_cb = ModelCheckpoint(prefix=prefix, directory=model_dir, config=config_ck) # logger the_logger = logger(config, date) log = Logging(logger=the_logger, model_ckpt=ckpt_cb) callbacks.append(ckpt_cb) callbacks.append(log) net = WithLossCell(net, criterion) scaling_sens = Tensor(np.full((1), loss_scale), dtype=mstype.float32) net = DFCNNCTCTrainOneStepWithLossScaleCell(net, opt, scaling_sens) net.set_train(True) model = Model(net) if use_step_eval: # step evaluation step_eval = StepAccInfo(model=model, name=prefix, div=div, test_dev_batch_size=test_dev_batch_size, step_eval=step_eval, eval_step=eval_step, eval_epoch=eval_epoch, logger=the_logger, patience=patience, dataset_size=train_loader.get_dataset_size()) callbacks.append(step_eval) # loss monitor loss_monitor = LossMonitor(loss_monitor_step) callbacks.append(loss_monitor) if use_summary: summary_dir = os.path.join(SUMMARY_DIR, date) if not os.path.exists(summary_dir): os.mkdir(summary_dir) # mindInsight summary_collector = SummaryCollector(summary_dir=summary_dir, collect_freq=1, max_file_size=4 * 1024 ** 3) callbacks.append(summary_collector) if resume: the_logger.update_acc_ckpt(best_acc, best_ckpt) print(f'* Start training...') model.train(epochs, train_loader, callbacks=callbacks, dataset_sink_mode=dataset_sink_mode)
def test_train(): ''' finetune function ''' target = args_opt.device_target if target == "Ascend": devid = int(os.getenv('DEVICE_ID')) context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) poetry, tokenizer, keep_words = create_tokenizer() print(len(keep_words)) dataset = create_poetry_dataset(bert_net_cfg.batch_size, poetry, tokenizer) num_tokens = 3191 poetrymodel = BertPoetryModel(bert_net_cfg, True, num_tokens, dropout_prob=0.1) netwithloss = BertPoetry(poetrymodel, bert_net_cfg, True, dropout_prob=0.1) callback = LossCallBack(poetrymodel) # optimizer steps_per_epoch = dataset.get_dataset_size() print("============ steps_per_epoch is {}".format(steps_per_epoch)) lr_schedule = BertLearningRate( learning_rate=cfg.AdamWeightDecay.learning_rate, end_learning_rate=cfg.AdamWeightDecay.end_learning_rate, warmup_steps=1000, decay_steps=cfg.epoch_num * steps_per_epoch, power=cfg.AdamWeightDecay.power) optimizer = AdamWeightDecay(netwithloss.trainable_params(), lr_schedule) # load checkpoint into network ckpt_config = CheckpointConfig(save_checkpoint_steps=steps_per_epoch, keep_checkpoint_max=1) ckpoint_cb = ModelCheckpoint(prefix=cfg.ckpt_prefix, directory=cfg.ckpt_dir, config=ckpt_config) param_dict = load_checkpoint(cfg.pre_training_ckpt) new_dict = {} # load corresponding rows of embedding_lookup for key in param_dict: if "bert_embedding_lookup" not in key: new_dict[key] = param_dict[key] else: value = param_dict[key] np_value = value.data.asnumpy() np_value = np_value[keep_words] tensor_value = Tensor(np_value, mstype.float32) parameter_value = Parameter(tensor_value, name=key) new_dict[key] = parameter_value load_param_into_net(netwithloss, new_dict) update_cell = DynamicLossScaleUpdateCell(loss_scale_value=2**32, scale_factor=2, scale_window=1000) netwithgrads = BertPoetryCell(netwithloss, optimizer=optimizer, scale_update_cell=update_cell) model = Model(netwithgrads) model.train(cfg.epoch_num, dataset, callbacks=[callback, ckpoint_cb], dataset_sink_mode=True)
parser.add_argument('--device_target', type=str, default=None, help='Run device target') args_opt = parser.parse_args() if __name__ == '__main__': context.set_context(mode=context.GRAPH_MODE, device_target=args_opt.device_target, save_graphs=False) # define manual quantization network network = resnet50_quant(class_num=config_quant.class_num) # load checkpoint if args_opt.checkpoint_path: param_dict = load_checkpoint(args_opt.checkpoint_path) not_load_param = load_param_into_net(network, param_dict) if not_load_param: raise ValueError("Load param into network fail!") # export network print("============== Starting export ==============") inputs = Tensor(np.ones([1, 3, 224, 224]), mindspore.float32) export(network, inputs, file_name="resnet50_quant", file_format=args_opt.file_format, quant_mode='QUANT', mean=0., std_dev=48.106) print("============== End export ==============")
parser.add_argument("--device_target", type=str, choices=["Ascend", "GPU", "CPU"], default="GPU", help="device target") args = parser.parse_args() context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target, device_id=args.device_id) if __name__ == "__main__": if args.device_target != "GPU": raise ValueError("Only supported GPU now.") net = efficientnet_b0( num_classes=cfg.num_classes, drop_rate=cfg.drop, drop_connect_rate=cfg.drop_connect, global_pool=cfg.gp, bn_tf=cfg.bn_tf, ) ckpt = load_checkpoint(args.ckpt_file) load_param_into_net(net, ckpt) net.set_train(False) image = Tensor( np.ones([cfg.batch_size, 3, args.height, args.width], np.float32)) export(net, image, file_name=args.file_name, file_format=args.file_format)