def __init__(self, train=True, transform=None, target_transform=None, dataset='cifar10', noise_type='symmetric', noise_rate=0.5, split_per=0.9, random_seed=1, num_class=10): self.transform = transform self.target_transform = target_transform self.train = train original_images = np.load('data/cifar10/train_images.npy') original_labels = np.load('data/cifar10/train_labels.npy') # clean images and noisy labels (training and validation) self.train_data, self.val_data, self.train_labels, self.val_labels = tools.dataset_split( original_images, original_labels, dataset, noise_type, noise_rate, split_per, random_seed, num_class) if self.train: self.train_data = self.train_data.reshape((-1, 3, 32, 32)) self.train_data = self.train_data.transpose((0, 2, 3, 1)) else: self.val_data = self.val_data.reshape((-1, 3, 32, 32)) self.val_data = self.val_data.transpose((0, 2, 3, 1))
def __init__(self, train=True, transform=None, target_transform=None, noise_rate=0.5, split_per=0.9, random_seed=1, num_class=10): self.transform = transform self.target_transform = target_transform self.train = train original_images = np.load('data/cifar10/train_images.npy') original_labels = np.load('data/cifar10/train_labels.npy') self.train_data, self.val_data, self.train_labels, self.val_labels = tools.dataset_split( original_images, original_labels, noise_rate, split_per, random_seed, num_class) if self.train: self.train_data = self.train_data.reshape((45000, 3, 32, 32)) self.train_data = self.train_data.transpose((0, 2, 3, 1)) else: self.val_data = self.val_data.reshape((5000, 3, 32, 32)) self.val_data = self.val_data.transpose((0, 2, 3, 1))
def dev_loss(args): ''' 调参优化目标函数,使用了hold out验证 ''' tune_args = Parameters(**args) if tune_args.get('batch_size'): tune_args.batch_size = int(tune_args.batch_size) net_params.update(tune_args) train_params.update(tune_args) logger.info(f'\n\n \t <<<<<< ** parameters: {tune_args.to_str()} >>>>>>\n') HOLD_OUT = 5 # hold out 次数 min_dev_loss = [] max_dev_acc = [] epochs = [] for _ in range(HOLD_OUT): time_str = str(time.time()).replace('.', '')[:11] train_params.set('model_name', f'{time_str}__{tune_args.to_str()}') # 模型保存位置 # 数据集划分 train_x, train_y, val_x, val_y, _, _ = tools.dataset_split( texts, labels, train_percent=train_percent) train_params.extend({ 'train_x': train_x, # 训练数据 'train_y': train_y, # 训练数据标记 'dev_x': val_x, # 验证数据 'dev_y': val_y, # 验证数据标记 }) model = dm.DNNModel(net_params) model.build() min_dev_loss_, max_dev_acc_, epoch = dm.train(model, train_params) min_dev_loss.append(min_dev_loss_) max_dev_acc.append(max_dev_acc_) epochs.append(epoch) global lowerst_dev_loss if lowerst_dev_loss > np.average(min_dev_loss): # 仅保留最好的参数对应的模型,其余删除 lowerst_dev_loss = np.average(min_dev_loss) tools.rm_dirs('./checkpoints/best', tune_args.to_str(), False) else: tools.rm_dirs('./checkpoints/best', tune_args.to_str(), True) logger.info( f'\n\n\t >>>>>> Acc {str(max_dev_acc)}\t{np.average(max_dev_acc)}\ \n\t >>>>>> Loss {str(min_dev_loss)}\t{np.average(min_dev_loss)}\n\t >>>>>> Epoch {str(epochs)}\n' ) return { 'status': STATUS_OK, 'loss': np.average(min_dev_loss), 'epochs': epochs, 'metrics': { 'accuracys': max_dev_acc, 'accuracy': np.average(max_dev_acc), 'losses': min_dev_loss } }
def __init__(self, train=True, transform=None, target_transform=None, noise_rate=0.5, split_per=0.9, random_seed=1, num_class=10): self.transform = transform self.target_transform = target_transform self.train = train original_images = np.load('data/mnist/train_images.npy') original_labels = np.load('data/mnist/train_labels.npy') self.train_data, self.val_data, self.train_labels, self.val_labels = tools.dataset_split( original_images, original_labels, noise_rate, split_per, random_seed, num_class) pass
def __init__(self, train=True, transform=None, target_transform=None, dataset='fmnist', noise_type='symmetric', noise_rate=0.5, split_per=0.9, random_seed=1, num_class=10): self.transform = transform self.target_transform = target_transform self.train = train original_images = np.load( 'data/fashionmnist/train_images.npy').reshape((-1, 1, 28, 28)) original_labels = np.load('data/fashionmnist/train_labels.npy') self.train_data, self.val_data, self.train_labels, self.val_labels = tools.dataset_split( original_images, original_labels, dataset, noise_type, noise_rate, split_per, random_seed, num_class)
def __init__(self, train=True, transform=None, target_transform=None, dataset='mnist', noise_type='symmetric', noise_rate=0.5, split_per=0.9, random_seed=1, num_class=10): self.transform = transform self.target_transform = target_transform self.train = train original_images = np.load('data/mnist/train_images.npy') original_labels = np.load('data/mnist/train_labels.npy') # clean images and noisy labels (training and validation) self.train_data, self.val_data, self.train_labels, self.val_labels = tools.dataset_split( original_images, original_labels, dataset, noise_type, noise_rate, split_per, random_seed, num_class)
# vocab_to_int, embedding_matrix = tools.load_embedding(data_path + "word_embedding_300_new.txt") # 英文词向量 # vocab_to_int, embedding_matrix = tools.load_embedding(data_path + "glove.6B.200d.txt") # 英文词向量 # vocab_to_int, embedding_matrix = tools.load_embedding(data_path + "sgns.weibo.word.txt") # 中文词向量 net_params.set('embedding_matrix', embedding_matrix) # 添加词向量矩阵参数 net_params.set('vocab_size', len(vocab_to_int)) # 添加词典大小参数 logger.info(f"dictionary length: {len(vocab_to_int)}") texts = tools.wordlists2idlists(texts, vocab_to_int) # 将句子转成词典id列表 texts, labels = tools.drop_empty_texts(texts, labels) # 清除预处理后文本为空的数据 labels = tools.labels2onehot(labels, net_params.class_num) # 将类别标记转为one-hot形式 texts = tools.dataset_padding(texts, sent_len=net_params.max_sent_len) # 左侧补0 # 数据集划分 train_x, train_y, val_x, val_y, test_x, test_y = tools.dataset_split( texts, labels, train_percent=train_percent) train_params.extend({ 'train_x': train_x, # 训练数据 'train_y': train_y, # 训练数据标记 'dev_x': val_x, # 验证数据 'dev_y': val_y, # 验证数据标记 }) # ================== step2: 构建模型 ================= model = dm.DNNModel(net_params) model.build() # ================== step3: 训练 ================= min_dev_loss = dm.train(model, train_params) logger.info(f' ** The minimum dev_loss is {min_dev_loss}') # ================== step4: 测试 =================