def _fit_model(self, train_reader, valid_reader): best_score = 1e5 tqdm_trange = trange(0, self.n_epoch) for epoch in tqdm_trange: self._train_model(train_reader) self._valid_model(valid_reader) train_loss = self.acc['train'][-1] valid_loss = self.acc['valid'][-1] train_loss_str = '{:.3f}'.format(self.acc['train'][-1]) valid_loss_str = '{:.3f}'.format(self.acc['valid'][-1]) tqdm_trange.set_description( f'tr=>epoch={epoch} Valid Loss: {valid_loss_str}, Train Loss: {train_loss_str}' ) unit = { 'epoch': epoch, 'state_dict': self.predictor.state_dict(), 'score': valid_loss, 'best_score': best_score, 'optimizer': self.optimizer.state_dict() } if valid_loss < best_score: best_score = valid_loss unit['best_score'] = valid_loss self._save_checkpoint(unit, epoch, is_best=True) if epoch % self.n_epoch_saved == 0: self._save_checkpoint(unit, epoch, is_best=False) self._save_checkpoint(unit, -1, is_best=False)
def predict(self): self.input_array /= 255 for i in range(3): self.input_array[:,:,:,i] -= self.mean[i] self.input_array[:,:,:,i] /= self.std[i] if self.input_array.shape[0] <= self.batch_size : image_tensor = Variable(torch.from_numpy(self.input_array.copy().transpose((0,3, 1, 2))).float().cuda()) output = self.model(image_tensor) output_predict = F.softmax(output) output_predict = output_predict.cuda().data.cpu().numpy() else: batch_count = self.input_array.shape[0]// self.batch_size # with tqdm(dataloaders[phase], desc = phase, file=sys.stdout, disable=not (self.verbose)) as iterator: for i in trange(batch_count + 1,disable=not (self.verbose)): top = i * self.batch_size bottom = min(self.input_array.shape[0], (i+1) * self.batch_size) if top < self.input_array.shape[0]: image_tensor = Variable(torch.from_numpy(self.input_array[top:bottom,:,:,:].copy().transpose((0,3, 1, 2))).float().cuda()) output = self.model(image_tensor) output_predict_batch = F.softmax(output) output_predict_batch = output_predict_batch.cuda().data.cpu().numpy() if i ==0 : output_predict = output_predict_batch.copy() else: output_predict = np.row_stack((output_predict,output_predict_batch.copy())) del image_tensor,output_predict_batch,output gc.collect() return output_predict
def predict(self): self.input_array /= 255 for i in range(3): self.input_array[:, :, :, i] -= self.mean[i] self.input_array[:, :, :, i] /= self.std[i] if self.input_array.shape[0] <= self.batch_size: output_predict = self.model.predict(self.input_array, batch_size=self.batch_size) else: batch_count = self.input_array.shape[0] // self.batch_size # with tqdm(dataloaders[phase], desc = phase, file=sys.stdout, disable=not (self.verbose)) as iterator: for i in trange(batch_count + 1, disable=not (self.verbose)): top = i * self.batch_size bottom = min(self.input_array.shape[0], (i + 1) * self.batch_size) if top < self.input_array.shape[0]: output_predict_batch = self.model.predict( self.input_array[top:bottom, :, :, :].copy(), batch_size=self.batch_size) if i == 0: output_predict = output_predict_batch.copy() else: output_predict = np.row_stack( (output_predict, output_predict_batch.copy())) del image_tensor, output_predict_batch, output gc.collect() return output_predict
def evaluate_player(old_player, new_player, game_size=11, each_times=40, thresh_num=0.55, competitive=False): newPlayerWin = 0 #let new_player win in competitive if (competitive == True): game = HexGame(game_size) _, winner = comparePlay(new_player, old_player, game, competitive=True) if (winner == chess.blue): print('|competitive loss') return old_player, False game = HexGame(game_size) _, winner = comparePlay(old_player, new_player, game, competitive=True) if (winner == chess.red): print('|competitive loss') return old_player, False print('red evaluate: ') for i in trange(int(each_times)): game = HexGame(game_size) _, winner = comparePlay(new_player, old_player, game) if (winner == chess.red): newPlayerWin += 1 print('blue evaluate: ') for i in trange(int(each_times)): game = HexGame(game_size) _, winner = comparePlay(old_player, new_player, game) if (winner == chess.blue): newPlayerWin += 1 print('|victory_pros: ' + str(newPlayerWin / (int(each_times) * 2) * 100) + '%') if (newPlayerWin / (int(each_times) * 2) >= thresh_num): winPlayer = new_player isBeat = True else: winPlayer = old_player isBeat = False return winPlayer, isBeat
def genGraph(self): idx = 0 for x in trange(self.h): for y in range(self.w): for (i, j) in [(x, y + 1), (x + 1, y - 1), (x + 1, y), (x + 1, y + 1)]: if (0 <= i < self.h) and (0 <= j < self.w): self.addEdge(idx, (x, y), (i, j)) idx += 1
def Run(self): """execute function""" random.seed() self.Initialize(self.max_gen, self.env) for i in trange(self.max_gen): time.sleep(0.01) # logging.info("\n------This is the %d generation------" % i) random.shuffle(self.rat_list) self.Crossover_and_Mutation() # self.Select() self.Elite_select() self.rat_best.BestRatDisplay_log()
def segementRegion(self): """ separating the region :return: """ self.sortEdges() print("union regions...") for i in trange(self.G.edges_num): edge = self.G.edges[i, :] r_1 = self.regions.findRoot(edge[0]) r_2 = self.regions.findRoot(edge[1]) if r_1 != r_2: if edge[2] <= min(self.int_dif[r_1], self.int_dif[r_2]): self.regions.union(r_1, r_2) root_idx = self.regions.findRoot(r_1) self.int_dif[root_idx] = edge[2] + self.tau( self.regions.getSetSize(root_idx))
def data_get(): n_class = 10 data_name_list = os.listdir('/home/flr/Desktop/Dadou/') data_x = [] y_list = [] index = np.arange(7218) np.random.shuffle(index) for i in trange(len(data_name_list)): picture_data = cv2.imread( '/home/flr/Desktop/Dadou/' + data_name_list[i], 1) data_x.append(picture_data) y_list.append(data_name_list[i][0].split('-')[0]) data_y = np_utils.to_categorical(y_list, 17) index_train = index[0:6000] index_test = index[6000:7218] data_x_train = np.array(data_x)[index_train, :, :, :] data_y_train = np.array(data_y)[index_train, :] data_x_test = np.array(data_x)[index_test, :, :, :] data_y_test = np.array(data_y)[index_test, :] return data_x_train, data_y_train, data_x_test, data_y_test, n_class
def create_player(king_player, base_player=None, game_size=11, total_times=1000, reTrain=False, train_only_win=False, plot=False, debug=False, train_scale=9): train_data = [] train_winner = [] print("SelfPlay Progress:") for i in trange(total_times): #tqdm.set_description(f"SelfPlay Progress:") #print(i) if (debug == True): debug_str = 'self play progress : ' + str(i + 1) + '/' + str( total_times) print(debug_str) game = HexGame(game_size) new_data, new_winner = selfPlay(king_player, game) train_data += new_data train_winner.append(new_winner) if (train_only_win == True): train_data = __pickUp_data(train_data, new_winner) if (reTrain == True): new_model = __reTrain_model(train_data, game_size, plot=plot, train_scale=train_scale) else: new_model = __train_model(base_player.model, train_data, game_size, plot=plot, train_scale=train_scale) new_player = DeepLearningPlayer(new_model) return new_player
def get_dissim_distribution(str_A, str_B): i = 0 dissim_distribution = np.zeros((50, )) for i in trange(len(str_A)): for j in range(0, len(str_B)): result = Levenshtein.editops( str_A[i], str_B[j]) # catch the editting steps of two sequences record = [] # Get the different region for item in result: if item[0] == 'delete' or item[0] == 'replace' or ( item[0] == 'insert' and item[2] == 50): record.append(item[1]) record = list(set(record)) #remove repeating location tmp_dissim_region = [] # Get the dissimilar region in the str_A for k in range(0, 50): if k in record: tmp_dissim_region.append(k) for item in tmp_dissim_region: dissim_distribution[item] = dissim_distribution[item] + 1 print(i) return dissim_distribution
def get_exp_data(self, sel_task='diagnose', shuffle=True, split_ratio=[0.64, 0.16, 0.2], data_root='', n_limit=-1): """ Parameters ---------- task : str, optional (default='phenotyping') name of current healthcare task shuffle : bool, optional (default=True) determine whether shuffle data or not split_ratio : list, optional (default=[0.64,0.16,0.2]) used for split whole data into train/valid/test data_root : str, (default='') use data in data_root n_limit : int, optional (default = -1) used for sample N-data not for all data, if n_limit==-1, use all data """ self.sel_task = sel_task if data_root == '': raise Exception('fill in correct data_root') all_list = [] l_list = [] episode_dir = os.path.join(data_root, 'x_data') feat_n, label_n = 0, 0 label_seq = pd.read_csv( os.path.join(data_root, 'y_data', self.sel_task + '.csv')).values for row_id in trange(len(label_seq)): if n_limit > 0 and row_id > n_limit: break time.sleep(0.01) row = label_seq[row_id, :] concrete_path = os.path.join(episode_dir, row[0]) if os.path.exists(concrete_path) is False: continue all_list.append([concrete_path] + row[1:].astype(float).tolist()) label_n = len(row[1:]) # shuffle the list if shuffle: random.shuffle(all_list) N = len(all_list) x_list = [] y_list = [] for item in all_list: x_list.append(item[0]) y_list.append(np.array(item[1:]).astype(float)) train_ratio = split_ratio[0] valid_ratio = split_ratio[1] training_x = x_list[:int(train_ratio * N)] validing_x = x_list[int(train_ratio * N):int((train_ratio + valid_ratio) * N)] testing_x = x_list[int((train_ratio + valid_ratio) * N):] training_y = y_list[:int(train_ratio * N)] validing_y = y_list[int(train_ratio * N):int((train_ratio + valid_ratio) * N)] testing_y = y_list[int((train_ratio + valid_ratio) * N):] if os.path.exists(self.expdata_dir) is False: os.makedirs(self.expdata_dir) pickle.dump(training_x, open(os.path.join(self.expdata_dir, 'train_x.pkl'), 'wb')) pickle.dump(validing_x, open(os.path.join(self.expdata_dir, 'valid_x.pkl'), 'wb')) pickle.dump(testing_x, open(os.path.join(self.expdata_dir, 'test_x.pkl'), 'wb')) print('finished X generate') pickle.dump(training_y, open(os.path.join(self.expdata_dir, 'train_y.pkl'), 'wb')) pickle.dump(validing_y, open(os.path.join(self.expdata_dir, 'valid_y.pkl'), 'wb')) pickle.dump(testing_y, open(os.path.join(self.expdata_dir, 'test_y.pkl'), 'wb')) print('finished Y generate') expdata_statistic = { 'task': self.sel_task, 'raio': split_ratio, 'label_n': label_n, 'len_train': len(training_x), 'len_valid': len(validing_x), 'len_test': len(testing_x) } pickle.dump( expdata_statistic, open(os.path.join(self.expdata_dir, 'expdata_statistic.pkl'), 'wb')) self.train = {'x': training_x, 'y': training_y, 'label_n': label_n} self.valid = {'x': validing_x, 'y': validing_y, 'label_n': label_n} self.test = {'x': testing_x, 'y': testing_y, 'label_n': label_n} print('generate finished') print('target Task:', expdata_statistic['task']) print('N of labels:', expdata_statistic['label_n']) print('N of TrainData:', expdata_statistic['len_train']) print('N of ValidData:', expdata_statistic['len_valid']) print('N of TestData:', expdata_statistic['len_test'])
from tqdm import tqdm import time from tqdm._tqdm import trange for i in tqdm(range(50)): time.sleep(0.1) pass for j in trange(50): time.sleep(0.05)
rel2idx = {relation:i for i,relation in enumerate(relations)} with open(ROOT_PATH+"/data/ent2idx.csv","w+") as f: [print(str(k)+","+str(v),file=f) for k,v in ent2idx.items()] with open(ROOT_PATH+"/data/rel2idx.csv","w+") as f: [print(str(k)+","+str(v),file=f) for k,v in rel2idx.items()] # df = df.head(10) heads = df["head"] tails = df["tail"] relations = df["relation"] idx_df = pd.DataFrame(columns=["head","tail","relation"]) for i in trange(len(df)): # print(heads[i]) heads[i] = ent2idx[heads[i]] # print(heads[i]) tails[i] = ent2idx[tails[i]] relations[i] = rel2idx[relations[i]] idx_df["head"] = heads idx_df["tail"] = tails idx_df["relation"] = relations # idx_df = idx_df.join(heads) # idx_df = idx_df.join(tails) # idx_df = idx_df.join(relations) idx_df.to_csv(ROOT_PATH+"/data/graph.csv",index=False)
def test(tracker, cfg): def transform(data): dets = mot_reader.detection['bboxes'][data['index']] image = data['raw'] if dets.size == 0: return image, dets, None, None if len(dets) > cfg.datasets.max_object: dets = dets[:cfg.datasets.max_object, :] input_dets = dets.copy() input_dets[:, 2:] += input_dets[:, :2] input_image = image.copy() input_image, _, input_dets, _, _ = augmentation(img_pre=input_image, boxes_pre=input_dets) input_image = input_image.unsqueeze(0).to(cfg.solver.device[0]) input_dets = input_dets.unsqueeze(0).to(cfg.solver.device[0]) return image, dets, input_image, input_dets mot_reader = MOTReader(args.sequence_folder, vis_thresh=cfg.datasets.min_visibility, detection_thresh=cfg.datasets.detection_thresh) sequence_name = mot_reader.video_info['name'] w, h = mot_reader.video_info['shape'] print('=> Preparing DataLoader for {}...'.format(sequence_name)) tracker.reset() augmentation = DANAugmentation(cfg, type='test') method = {'transform': transform} loader = DataLoader(os.path.join(args.sequence_folder, 'img1'), max_size=20, **method) loader.start() time.sleep(2) result_file = os.path.join(args.result_folder, sequence_name, '.txt') result_video = os.path.join(args.result_folder, sequence_name + '.avi') if args.save_video: video_writer = cv2.VideoWriter( result_video, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 10, (w, h)) print('=> Begin to Track {}...'.format(sequence_name)) result = [] for i in trange(len(mot_reader)): image, dets, input_image, input_dets = loader.getData()['output'] if dets.size == 0: continue draw = tracker.update(image, dets, input_image, input_dets, i, args.show_image) if args.show_image and not image_org is None: cv2.imshow(sequence_name, draw) cv2.waitKey(1) if args.save_video: video_writer.write(draw) for t in tracker.tracks: n = t.nodes[-1] if t.age == 1: b = n.get_box(tracker.frame_index - 1, tracker.recorder) result.append([i] + [t.id] + [b[0], b[1], b[2], b[3]] + [-1, -1, -1, -1]) np.savetxt(result_file, np.array(result).astype(int), fmt='%i') if args.evaluate and mot_reader.ground_truth is not None: mota, _, idf1, idsw = evaluation_mot(mot_reader.ground_truth, result) print('MOTA={:.4f}, IDF1={:.4f}, ID Sw.={}'.format(mota, idf1, idsw)) loader.stop()
# # # ############################################################################ # # return { # 'src': np.asarray( src ), # 'dst': np.asarray( dst ), # 'linear': np.array( < double [ :3, :2 ] > linear ), # 'weights': np.array( < double [ :n, :2 ] > W ), # 'be': be[0], # } ################################################################################ cylst = [] cy2lst = [] for n in trange(5, 100, 1, leave=True, nested=False): np.random.seed(1337) src = np.random.rand(n, 2) * 10 dst = src + np.random.rand(n, 2) * 0.1 tmp1 = [] tmp2 = [] for r in trange(0, 10, 1, nested=True): with Capturing(): with Timer("Cy") as t1: gCy(src, dst) tmp1.append(t1.interval)
def train(self, load_model=False, model_path=None): if load_model: if model_path is not None: self.load_weights(model_path) ## Training utterances all_input_ids, all_input_len, all_label_ids = convert_examples_to_features( self.train_examples, self.label_list, args.max_seq_length, self.tokenizer, args.max_turn_length) print('all input ids size: ', all_input_ids.size()) num_train_batches = all_input_ids.size(0) num_train_steps = int( num_train_batches / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs) logger.info("***** training *****") logger.info(" Num examples = %d", len(self.train_examples)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_steps) all_input_ids, all_input_len, all_label_ids = all_input_ids.to(DEVICE), all_input_len.to( DEVICE), all_label_ids.to(DEVICE) train_data = TensorDataset(all_input_ids, all_input_len, all_label_ids) train_sampler = RandomSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) all_input_ids_dev, all_input_len_dev, all_label_ids_dev = convert_examples_to_features( self.dev_examples, self.label_list, args.max_seq_length, self.tokenizer, args.max_turn_length) logger.info("***** validation *****") logger.info(" Num examples = %d", len(self.dev_examples)) logger.info(" Batch size = %d", args.dev_batch_size) all_input_ids_dev, all_input_len_dev, all_label_ids_dev = \ all_input_ids_dev.to(DEVICE), all_input_len_dev.to(DEVICE), all_label_ids_dev.to(DEVICE) dev_data = TensorDataset(all_input_ids_dev, all_input_len_dev, all_label_ids_dev) dev_sampler = SequentialSampler(dev_data) dev_dataloader = DataLoader(dev_data, sampler=dev_sampler, batch_size=args.dev_batch_size) logger.info("Loaded data!") if args.fp16: self.sumbt_model.half() self.sumbt_model.to(DEVICE) # ## Get domain-slot-type embeddings # slot_token_ids, slot_len = \ # get_label_embedding(self.processor.target_slot, args.max_label_length, self.tokenizer, DEVICE) # # for slot_idx, slot_str in zip(slot_token_ids, self.processor.target_slot): # # self.idx2slot[slot_idx] = slot_str # ## Get slot-value embeddings # label_token_ids, label_len = [], [] # for slot_idx, labels in zip(slot_token_ids, self.label_list): # # self.idx2value[slot_idx] = {} # token_ids, lens = get_label_embedding(labels, args.max_label_length, self.tokenizer, DEVICE) # label_token_ids.append(token_ids) # label_len.append(lens) # # for label, token_id in zip(labels, token_ids): # # self.idx2value[slot_idx][token_id] = label # logger.info('embeddings prepared') # if USE_CUDA and N_GPU > 1: # self.sumbt_model.module.initialize_slot_value_lookup(label_token_ids, slot_token_ids) # else: # self.sumbt_model.initialize_slot_value_lookup(label_token_ids, slot_token_ids) def get_optimizer_grouped_parameters(model): param_optimizer = [(n, p) for n, p in model.named_parameters() if p.requires_grad] no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [ {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01, 'lr': args.learning_rate}, {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0, 'lr': args.learning_rate}, ] return optimizer_grouped_parameters if not USE_CUDA or N_GPU == 1: optimizer_grouped_parameters = get_optimizer_grouped_parameters(self.sumbt_model) else: optimizer_grouped_parameters = get_optimizer_grouped_parameters(self.sumbt_model.module) t_total = num_train_steps if args.fp16: try: from apex.optimizers import FP16_Optimizer from apex.optimizers import FusedAdam except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") optimizer = FusedAdam(optimizer_grouped_parameters, lr=args.learning_rate, bias_correction=False, max_grad_norm=1.0) if args.fp16_loss_scale == 0: optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) else: optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.fp16_loss_scale) else: optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, correct_bias=False) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_proportion*t_total, num_training_steps=t_total) logger.info(optimizer) # Training code ############################################################################### print(torch.cuda.memory_allocated()) logger.info("Training...") global_step = 0 last_update = None best_loss = None model = self.sumbt_model if not args.do_not_use_tensorboard: summary_writer = None else: summary_writer = SummaryWriter("./tensorboard_summary/logs_1214/") for epoch in trange(int(args.num_train_epochs), desc="Epoch"): # Train model.train() tr_loss = 0 nb_tr_examples = 0 nb_tr_steps = 0 for step, batch in enumerate(tqdm(train_dataloader)): batch = tuple(t.to(DEVICE) for t in batch) input_ids, input_len, label_ids = batch # print(input_ids.size()) # Forward if N_GPU == 1: loss, loss_slot, acc, acc_slot, _ = model(input_ids, input_len, label_ids, N_GPU) else: loss, _, acc, acc_slot, _ = model(input_ids, input_len, label_ids, N_GPU) # average to multi-gpus loss = loss.mean() acc = acc.mean() acc_slot = acc_slot.mean(0) if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps # Backward if args.fp16: optimizer.backward(loss) else: loss.backward() # tensrboard logging if summary_writer is not None: summary_writer.add_scalar("Epoch", epoch, global_step) summary_writer.add_scalar("Train/Loss", loss, global_step) summary_writer.add_scalar("Train/JointAcc", acc, global_step) if N_GPU == 1: for i, slot in enumerate(self.processor.target_slot): summary_writer.add_scalar("Train/Loss_%s" % slot.replace(' ', '_'), loss_slot[i], global_step) summary_writer.add_scalar("Train/Acc_%s" % slot.replace(' ', '_'), acc_slot[i], global_step) tr_loss += loss.item() nb_tr_examples += input_ids.size(0) nb_tr_steps += 1 if (step + 1) % args.gradient_accumulation_steps == 0: # modify lealrning rate with special warm up BERT uses lr_this_step = args.learning_rate * warmup_linear(global_step / t_total, args.warmup_proportion) if summary_writer is not None: summary_writer.add_scalar("Train/LearningRate", lr_this_step, global_step) for param_group in optimizer.param_groups: param_group['lr'] = lr_this_step if scheduler is not None: torch.nn.utils.clip_grad_norm_(optimizer_grouped_parameters, 1.0) optimizer.step() if scheduler is not None: scheduler.step() optimizer.zero_grad() global_step += 1 # Perform evaluation on validation dataset model.eval() dev_loss = 0 dev_acc = 0 dev_loss_slot, dev_acc_slot = None, None nb_dev_examples, nb_dev_steps = 0, 0 for step, batch in enumerate(tqdm(dev_dataloader, desc="Validation")): batch = tuple(t.to(DEVICE) for t in batch) input_ids, input_len, label_ids = batch if input_ids.dim() == 2: input_ids = input_ids.unsqueeze(0) input_len = input_len.unsqueeze(0) label_ids = label_ids.unsuqeeze(0) with torch.no_grad(): if N_GPU == 1: loss, loss_slot, acc, acc_slot, _ = model(input_ids, input_len, label_ids, N_GPU) else: loss, _, acc, acc_slot, _ = model(input_ids, input_len, label_ids, N_GPU) # average to multi-gpus loss = loss.mean() acc = acc.mean() acc_slot = acc_slot.mean(0) num_valid_turn = torch.sum(label_ids[:, :, 0].view(-1) > -1, 0).item() dev_loss += loss.item() * num_valid_turn dev_acc += acc.item() * num_valid_turn if N_GPU == 1: if dev_loss_slot is None: dev_loss_slot = [l * num_valid_turn for l in loss_slot] dev_acc_slot = acc_slot * num_valid_turn else: for i, l in enumerate(loss_slot): dev_loss_slot[i] = dev_loss_slot[i] + l * num_valid_turn dev_acc_slot += acc_slot * num_valid_turn nb_dev_examples += num_valid_turn dev_loss = dev_loss / nb_dev_examples dev_acc = dev_acc / nb_dev_examples if N_GPU == 1: dev_acc_slot = dev_acc_slot / nb_dev_examples # tensorboard logging if summary_writer is not None: summary_writer.add_scalar("Validate/Loss", dev_loss, global_step) summary_writer.add_scalar("Validate/Acc", dev_acc, global_step) if N_GPU == 1: for i, slot in enumerate(self.processor.target_slot): summary_writer.add_scalar("Validate/Loss_%s" % slot.replace(' ', '_'), dev_loss_slot[i] / nb_dev_examples, global_step) summary_writer.add_scalar("Validate/Acc_%s" % slot.replace(' ', '_'), dev_acc_slot[i], global_step) dev_loss = round(dev_loss, 6) output_model_file = os.path.join(os.path.join(SUMBT_PATH, args.output_dir), "pytorch_model.bin") if last_update is None or dev_loss < best_loss: last_update = epoch best_loss = dev_loss best_acc = dev_acc if not USE_CUDA or N_GPU == 1: torch.save(model.state_dict(), output_model_file) else: torch.save(model.module.state_dict(), output_model_file) logger.info( "*** Model Updated: Epoch=%d, Validation Loss=%.6f, Validation Acc=%.6f, global_step=%d ***" % ( last_update, best_loss, best_acc, global_step)) else: logger.info( "*** Model NOT Updated: Epoch=%d, Validation Loss=%.6f, Validation Acc=%.6f, global_step=%d ***" % ( epoch, dev_loss, dev_acc, global_step)) if last_update + args.patience <= epoch: break
def stat(self): # bar = tqdm(self.urls) for _ in trange(len(self.urls) // 20): time.sleep(0.01)
def modelToDB(self): # 读取文件 if 'table' in self.flag or 'constraint' in self.flag or 'index' in self.flag: result = self.flag.split(',') for object in result: # 表对象 if object == 'table': with open(self.filedir_tab + os.sep + '%s2%s_model.sql' % (self.dbtype, self.dbtype_tag), 'r', encoding='utf-8') as f: lines = f.readlines() array_tab = ''.join(lines).split(';') # 剔除空元素 while '' in array_tab: array_tab.remove('') if len(array_tab) > 0: mdtool.log.info("表对象创建进度:") for elem in trange(len(array_tab)): elem_sql = array_tab[elem] try: self.dbtag_executor.sql_execute(elem_sql) self.dbtag_executor.dbclose() except Exception as err: mdtool.log.error("失败的建表语句:%s;" % elem_sql) mdtool.log.error("表对象创建失败:" + str(err)) f.close() else: mdtool.log.warning("模型生成器未生成表对象, 请检查是否正确") mdtool.log.info("表对象创建完成") # 索引对象 # 添加唯一性索引后可以再添加主键,故先执行索引对象 elif object == 'index': with open( self.filedir_idx + os.sep + '%s2%s_index_model.sql' % (self.dbtype, self.dbtype_tag), 'r') as f: lines = f.readlines() array_idx = ''.join(lines).split(';') while '' in array_idx: array_idx.remove('') if len(array_idx) > 0: mdtool.log.info("索引对象创建进度:") for elem in trange(len(array_idx)): elem_sql = array_idx[elem] try: self.dbtag_executor.sql_execute(elem_sql) self.dbtag_executor.dbclose() except Exception as err: mdtool.log.error("失败的索引语句:%s;" % elem_sql) mdtool.log.error("索引对象创建失败:" + str(err)) f.close() else: mdtool.log.warning("模型生成器未生成索引对象, 请检查是否正确") mdtool.log.info("索引对象创建完成") # 约束对象 elif object == 'constraint': # 主键 、 检查 、 唯一性 with open( self.filedir_cst + os.sep + '%s2%s_constraint_model.sql' % (self.dbtype, self.dbtype_tag), 'r') as f: lines = f.readlines() array_cst = ''.join(lines).split(';') while '' in array_cst: array_cst.remove('') if len(array_cst) > 0: mdtool.log.info("约束对象创建进度:") for elem in trange(len(array_cst)): elem_sql = array_cst[elem] try: self.dbtag_executor.sql_execute(elem_sql) self.dbtag_executor.dbclose() except Exception as err: mdtool.log.error("失败的约束语句:%s;" % elem_sql) mdtool.log.error("约束对象创建失败:" + str(err)) f.close() else: mdtool.log.warning("模型生成器未生成约束对象, 请检查是否正确") # 外键最后处理 with open( self.filedir_cst + os.sep + '%s2%s_constraint_fk_model.sql' % (self.dbtype, self.dbtype_tag), 'r') as f: lines = f.readlines() array_fk = ''.join(lines).split(';') while '' in array_fk: array_fk.remove('') if len(array_fk) > 0: mdtool.log.info("约束-外键对象创建进度:") for elem in trange(len(array_fk)): elem_sql = array_fk[elem] try: self.dbtag_executor.sql_execute(elem_sql) self.dbtag_executor.dbclose() except Exception as err: mdtool.log.error("失败的约束-外键语句:%s;" % elem_sql) mdtool.log.error("约束-外键对象创建失败:" + str(err)) f.close() else: mdtool.log.warning("模型生成器未生成约束-外键对象, 请检查是否正确") mdtool.log.info("约束(&外键)对象创建完成")