def DogDataLoader(opt): transform_train = T.Compose([ T.Resize(256), T.CenterCrop(224), T.RandomFlipLeftRight(), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) transform_valid = T.Compose([ T.Resize(224), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) train_set = vision.ImageFolderDataset(opt.train_dir, flag=1) valid_set = vision.ImageFolderDataset(opt.valid_dir, flag=1) loader = gluon.data.DataLoader train_loader = loader(train_set.transform_first(transform_train), batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers, last_batch='rollover') valid_loader = loader(valid_set.transform_first(transform_valid), batch_size=opt.batch_size, shuffle=False, num_workers=opt.num_workers, last_batch='keep') return train_loader, valid_loader
def load_data(self): def transform_train(data, label): im = data.astype("float32") / 255 auglist = image.CreateAugmenter(data_shape=(3, 32, 32), resize=0, rand_crop=True, rand_resize=True, rand_gray=0.5, rand_mirror=True, mean=np.array([0., 0., 0.]), std=np.array([1, 1, 1]), brightness=0.5, contrast=0.5, saturation=0.5, hue=0.5, pca_noise=0, inter_method=2) for aug in auglist: im = aug(im) # 将数据格式从"高*宽*通道"改为"通道*高*宽"。 im = nd.transpose(im, (2, 0, 1)) return im, nd.array([label]).asscalar().astype("float32") # 测试时,无需对图像做标准化以外的增强数据处理。 def transform_test(data, label): im = data.astype("float32") / 255 auglist = image.CreateAugmenter(data_shape=(3, 32, 32), mean=np.array([0., 0., 0.]), std=np.array([1, 1, 1])) for aug in auglist: im = aug(im) im = nd.transpose(im, (2, 0, 1)) return im, nd.array([label]).asscalar().astype("float32") # 读取原始图像文件。flag=1 说明输入图像有三个通道(彩色)。 self.__train_ds = vision.ImageFolderDataset( os.path.join(self.__input_path, self.__folder_list[0]), flag=1, transform=transform_train) self.__valid_ds = vision.ImageFolderDataset( os.path.join(self.__input_path, self.__folder_list[1]), flag=1, transform=transform_test) self.__train_valid_ds = vision.ImageFolderDataset( os.path.join(self.__input_path, self.__folder_list[2]), flag=1, transform=transform_train) self.__test_ds = vision.ImageFolderDataset( os.path.join(self.__input_path, self.__folder_list[3]), flag=1, transform=transform_test) # print(len(self.__test_ds)) # 样本个数 self.__train_data = gluon.data.DataLoader( self.__train_ds, self.__batch_size, shuffle=True, last_batch="keep") self.__valid_data = gluon.data.DataLoader( self.__valid_ds, self.__batch_size, shuffle=True, last_batch="keep") self.__train_valid_data = gluon.data.DataLoader( self.__train_valid_ds, self.__batch_size, shuffle=True, last_batch="keep") self.__test_data = gluon.data.DataLoader( self.__test_ds, self.__batch_size, shuffle=False, last_batch="keep") return ( self.__train_ds, self.__valid_ds, self.__train_valid_ds, self.__test_ds, self.__train_data, self.__valid_data, self.__train_valid_data, self.__test_data )
def ReadData(batchSize): transform_train = transforms.Compose([ # transforms.CenterCrop(32) # transforms.RandomFlipTopBottom(), # transforms.RandomColorJitter(brightness=0.0, contrast=0.0,saturation=0.0, hue=0.0), # transforms.RandomLighting(0.0), # transforms.Cast('float32'), # transforms.Resize(32), # 随机按照 scale 和 ratio 裁剪,并放缩为 32x32 的正⽅形 transforms.RandomResizedCrop(32, scale=(0.08, 1.0), ratio=(3.0 / 4.0, 4.0 / 3.0)), # 随机左右翻转图⽚ transforms.RandomFlipLeftRight(), # 将图⽚像素值缩⼩到 (0,1) 内,并将数据格式从" ⾼ * 宽 * 通道" 改为" 通道 * ⾼ * 宽" transforms.ToTensor(), # 对图⽚的每个通道做标准化 transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]) transform_test = transforms.Compose([ # transforms.RandomResizedCrop(32, scale=(0.08, 1.0), ratio=(3.0 / 4.0, 4.0 / 3.0)), transforms.ToTensor(), transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]) data_dir = "D:\program\python\lijing\TrainUI" input_dir = "data" input_str = data_dir + '/' + input_dir + '/' train_ds = vision.ImageFolderDataset(input_str + 'train', flag=1) valid_ds = vision.ImageFolderDataset(input_str + 'valid', flag=1) # test_ds = vision.ImageFolderDataset(input_str + 'valid', flag=1) test_ds = vision.ImageFolderDataset(input_str + 'test', flag=1) loader = gluon.data.DataLoader trainData = loader(train_ds.transform_first(transform_train), batchSize, shuffle=True, last_batch='keep') validData = loader(valid_ds.transform_first(transform_test), batchSize, shuffle=True, last_batch='keep') testData = loader(test_ds.transform_first(transform_test), batchSize, shuffle=False, last_batch='keep') return trainData, validData, testData
def classification(ctx, num_outputs, filename, test_data): net = get_net(ctx, num_outputs) net.load_params(filename, ctx) net.hybridize() preds = [] for data, _ in test_data: output = net(data.as_in_context(ctx)) preds.extend(output.argmax(axis=1).astype(int).asnumpy()) test_ds = vision.ImageFolderDataset(os.path.join(input_str, 'test'), flag=1) sorted_ids = list(range(1, len(test_ds) + 1)) sorted_ids.sort(key=lambda x: str(x)) train_valid_ds = vision.ImageFolderDataset(os.path.join( input_str, 'train_valid'), flag=1) df = pd.DataFrame({'id': sorted_ids, 'label': preds}) df['label'] = df['label'].apply(lambda x: train_valid_ds.synsets[x]) df.to_csv('data/submission.csv', index=False)
def build_features_with_net_input_shape(scope="train", batch_size=32): imgs = vision.ImageFolderDataset('input/%s' % scope, flag=1, transform=transform_callback) data = gluon.data.DataLoader(imgs, batch_size) for net_shape in net_list_dict: augs = get_image_augs(net_shape, scope) net_list = net_list_dict[net_shape] batch_num = 0 for X, y in tqdm(data): features_extract(net_list, net_shape, X, y, batch_num, scope) batch_num += 1
def classify(fname): train_ds = vision.ImageFolderDataset('train', flag=1, transform=transform_train) with open(fname, 'rb') as f: img = image.imdecode(f.read()) data = image.imresize(img.astype('float32') / 255, 32, 32) data = nd.transpose(data, (2, 0, 1)) data = data.expand_dims(axis=0) net = get_net(mx.cpu(0)) net.load_params('model.params', mx.cpu(0)) out = net(data.as_in_context(mx.cpu(0))) out = nd.SoftmaxActivation(out) pred = int(nd.argmax(out, axis=1).asscalar()) label = train_ds.synsets return label[pred]
def data_process(subset_root, batch_size = 10, shuffle = True): transform_train = transforms.Compose([ transforms.Resize(224), # transforms.RandomResizedCrop(224) transforms.ToTensor() ]) transform_test = transforms.Compose([ transforms.Resize(224), transforms.ToTensor() ]) loader = gluon.data.DataLoader subset_ds = vision.ImageFolderDataset(subset_root, flag = 1) # test_ds = SimpleDataset(load_data(test_root, test_index)) subset_data = loader(subset_ds.transform_first(transform_train), batch_size, shuffle = shuffle, last_batch = "keep") # print("train data load done") # test_data = loader(test_ds.transform_first(transform_test), batch_size, shuffle = False, last_batch = "keep") # return train_data, test_data return subset_data
print("please input params file") exit print(param_file) data_dir = '/gpu_data/datasets/cifar-10' label_file = 'trainLabels.csv' train_dir = 'train' test_dir = 'test' input_dir = 'train_valid_test' batch_size = 128 input_str = data_dir + '/' + input_dir + '/' # 读取原始图像文件。flag=1说明输入图像有三个通道(彩色)。 train_ds = vision.ImageFolderDataset(input_str + 'train', flag=1, transform=transform_train) test_ds = vision.ImageFolderDataset(input_str + 'test', flag=1, transform=transform_test) loader = gluon.data.DataLoader test_data = loader(test_ds, batch_size, shuffle=False, last_batch='keep') # 交叉熵损失函数。 softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss() def get_net(ctx): num_outputs = 10 #net = gluon.model_zoo.vision.densenet121(classes=num_outputs,ctx=ctx) net = ResNet(num_outputs) net.initialize(ctx=ctx, init=init.Xavier()) return net
data_dir = 'E:\\WorkSpace\\gluon-tutorials-zh_aa12356jm\\data\\kaggle_dog' train_dir = 'train' test_dir = 'test' valid_dir = 'valid' input_dir = 'train_valid_test' train_valid_dir = 'train_valid' input_str = data_dir + '/' + input_dir + '/' batch_size = 128 #训练数据,从train_valid_test/train文件夹中加载,使用函数transform_train进行数据转换和增强 # flag为0,则表示加载为灰度图,为1则为三通道彩色图 train_ds = vision.ImageFolderDataset(input_str + train_dir, flag=1, transform=transform_train) #验证数据,从train_valid_test/valid文件夹中加载,使用函数transform_test进行数据转换和增强 valid_ds = vision.ImageFolderDataset(input_str + valid_dir, flag=1, transform=transform_test) #训练验证数据,从train_valid_test/train_valid文件夹中加载,使用函数transform_train进行数据转换和增强 train_valid_ds = vision.ImageFolderDataset(input_str + train_valid_dir, flag=1, transform=transform_train) loader = gluon.data.DataLoader #gluon提供的数据加载函数,Loads data from a dataset and returns mini-batches of data #从数据集中每次加载指定batchsize大小的数据 train_data = loader(train_ds, batch_size, shuffle=True, last_batch='keep') valid_data = loader(valid_ds, batch_size, shuffle=True, last_batch='keep') train_valid_data = loader(train_valid_ds,
def transform_test(data, label): im = data.astype('float32') / 255 - 0.5 # auglist = image.CreateAugmenter(data_shape=(3, 112, 112), resize=0, # rand_crop=False, rand_resize=False, rand_mirror=True, # brightness=0, contrast=0, # saturation=0, hue=0, # pca_noise=0, rand_gray=0, inter_method=2) # for aug in auglist: # im = aug(im) im = nd.transpose(im, (2, 0, 1)) return im, nd.array([label]).asscalar().astype('float32') # 读取原始图像文件。flag=1说明输入图像有三个通道(彩色)。 train_ds = vision.ImageFolderDataset(data_dir + train_dir, flag=1, transform=transform_train) valid_ds = vision.ImageFolderDataset(data_dir + valid_dir, flag=1, transform=transform_test) loader = gluon.data.DataLoader train_data = loader(train_ds, batch_size, shuffle=True, last_batch='keep', num_workers=6) valid_data = loader(valid_ds, batch_size, shuffle=True,
input_dir = 'D:/dataset/gluon/train_valid_test' valid_ratio = 0.1 pre_deal_flag = True vis = Visualizer(env='CIFAR10') # sorting the dataset and transform if not pre_deal_flag: pre_deal.reorg_cifar10_data(data_dir, label_file, train_dir, test_dir, input_dir, valid_ratio) input_str = input_dir + '/' # 读取原始图像文件。flag=1说明输入图像有三个通道(彩色)。 train_ds = vision.ImageFolderDataset(input_str + 'train', flag=1, transform=pre_deal.transform_train) valid_ds = vision.ImageFolderDataset(input_str + 'valid', flag=1, transform=pre_deal.transform_test) train_valid_ds = vision.ImageFolderDataset(input_str + 'train_valid', flag=1, transform=pre_deal.transform_train) test_ds = vision.ImageFolderDataset(input_str + 'test', flag=1, transform=pre_deal.transform_test) loader = gluon.data.DataLoader train_data = loader(train_ds, batch_size, shuffle=True, last_batch='keep') valid_data = loader(valid_ds, batch_size, shuffle=True, last_batch='keep') train_valid_data = loader(train_valid_ds,
output = model(img) ids = output.argmax(axis=1).astype('int').asnumpy()[0] cls_id.append(ids) c_scores.append(mx.nd.softmax(output, axis=1)[0][ids].asnumpy()[0]) plot_bbox(orig_image, c_bboxes, c_scores, cls_id, cls_name, thresh=0.1) # model.hybridize() im_name = 'saluki3.jpg' x, orig_img = gluoncv.data.transforms.presets.rcnn.load_test(im_name) model = models.vgg16_faster_rcnn() model.load_parameters('/home/qinliang/Desktop/dog_detection_gluoncv/checkpoints/epoch9_map_0.8939.params') box_ids, scores, bboxes = model(x) del model sets = vision.ImageFolderDataset('/home/qinliang/dataset/stanford_dog_dataset/cut_images_train', flag=1) cls_name = sets.synsets # model = models.VGG16(num_classes=len(cls_name)) model = models.Resnet50_v2(num_classes=len(cls_name)) model.load_parameters('/home/qinliang/Desktop/github/dog_recognition_gluoncv/checkpoints/epoch8_acc_86.87.params') plot(orig_img, model, box_ids, scores, bboxes, cls_name, thresh=0.5) plt.show()
# 将图片像素值缩小到(0,1)内,并将数据格式从"高*宽*通道"改为"通道*高*宽" transforms.ToTensor(), # 对图片的每个通道做标准化 transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]) # 测试时,无需对图像做标准化以外的增强数据处理。 transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]) input_str = data_dir + '/' + input_dir + '/' # 读取原始图像文件。flag=1说明输入图像有三个通道(彩色)。 train_ds = vision.ImageFolderDataset(input_str + 'train', flag=1) valid_ds = vision.ImageFolderDataset(input_str + 'valid', flag=1) train_valid_ds = vision.ImageFolderDataset(input_str + 'train_valid', flag=1) test_ds = vision.ImageFolderDataset(input_str + 'test', flag=1) loader = gluon.data.DataLoader train_data = loader(train_ds.transform_first(transform_train), batch_size, shuffle=True, last_batch='keep') valid_data = loader(valid_ds.transform_first(transform_test), batch_size, shuffle=True, last_batch='keep') train_valid_data = loader(train_valid_ds.transform_first(transform_train), batch_size,
def get_features(net, data): features = [] labels = [] #print('In get_features') for X, y in tqdm(data): feature = net.features(X.as_in_context(ctx)) features.append(feature.asnumpy()) labels.append(y.asnumpy()) features = np.concatenate(features, axis=0) labels = np.concatenate(labels, axis=0) return features, labels preprocessing[0] = image.ForceResizeAug((224, 224)) imgs = vision.ImageFolderDataset('for_train_stanford', transform=transform) data = gluon.data.DataLoader(imgs, 64) features_vgg, labels = get_features(models.vgg16_bn(pretrained=True, ctx=ctx), data) features_resnet, _ = get_features( models.resnet152_v1(pretrained=True, ctx=ctx), data) features_densenet, _ = get_features( models.densenet161(pretrained=True, ctx=ctx), data) preprocessing[0] = image.ForceResizeAug((299, 299)) imgs_299 = vision.ImageFolderDataset('for_train_stanford', transform=transform) data_299 = gluon.data.DataLoader(imgs_299, 64) features_inception, _ = get_features( models.inception_v3(pretrained=True, ctx=ctx), data) with h5py.File('features_train_stanford.h5', 'w') as f:
# -*- coding: utf-8 -*- # @Time : 18-1-30 下午1:25 # @Author : lyyang from mxnet.gluon.data import vision from mxnet import image from mxnet import nd train_test_augs = [ image.RandomCropAug((image_size,image_size)) ] def transform(data, label, augs=None): data = data.astype('float32') if augs: for aug in augs: data = aug(data) data = nd.transpose(data, (2,0,1)) return data, nd.array([label]).asscalar().astype('float32') # 读取原始图像文件。flag=1说明输入图像有三个通道(彩色)。 train_ds = vision.ImageFolderDataset(input_str + 'train', flag=1,transform=lambda X, y: transform(X, y, train_test_augs)) test_ds = vision.ImageFolderDataset(input_str + 'test', flag=1,transform=lambda X, y: transform(X, y, train_test_augs)) loader = gluon.data.DataLoader train_data = loader(train_ds, batch_size, shuffle=True, last_batch='keep') test_data = loader(train_ds, batch_size, shuffle=True, last_batch='keep')
def get_features(net, data): features = [] labels = [] for X, y in tqdm(data): feature = net.features(X.as_in_context(ctx)) features.append(feature.asnumpy()) labels.append(y.asnumpy()) features = np.concatenate(features, axis=0) labels = np.concatenate(labels, axis=0) return features, labels preprocessing[0] = image.ForceResizeAug((224, 224)) imgs = vision.ImageFolderDataset('for_train', transform=transform) data = gluon.data.DataLoader(imgs, 64) features_vgg, labels = get_features(models.vgg16_bn(pretrained=True, ctx=ctx), data) features_resnet, _ = get_features( models.resnet152_v1(pretrained=True, ctx=ctx), data) features_densenet, _ = get_features( models.densenet161(pretrained=True, ctx=ctx), data) # # preprocessing[0] = image.ForceResizeAug((299,299)) # imgs_299 = vision.ImageFolderDataset('for_train', transform=transform) # data_299 = gluon.data.DataLoader(imgs_299, 64) # # features_inception, _ = get_features(models.inception_v3(pretrained=True, ctx=ctx), data_299)
net = get_net(mx.cpu(0)) net.load_params('model.params', mx.cpu(0)) out = net(data.as_in_context(mx.cpu(0))) out = nd.SoftmaxActivation(out) out = out.asnumpy()[0] label = train_ds.synsets return label, out #主函数 if __name__ == '__main__': batch_size = 32 #读取训练集和验证集 train_ds = vision.ImageFolderDataset('train', flag=1, transform=transform_train) valid_ds = vision.ImageFolderDataset('train', flag=1, transform=transform_test) loader = gluon.data.DataLoader train_data = loader(train_ds, batch_size, shuffle=True, last_batch='keep') valid_data = loader(valid_ds, batch_size, shuffle=True, last_batch='keep') #设置参数 ctx = mx.gpu(0) epochs = 49 learning_rate = 0.01 wd = 0.0001 net = get_net(ctx) net.hybridize() loss = gluon.loss.SoftmaxCrossEntropyLoss()
[0.2023, 0.1994, 0.2010]) ]) # 测试时,无需对图像做标准化以外的增强数据处理 transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]) input_str = os.path.join(data_dir, input_dir) batch_size = 128 # 读取原始图像文件,flag=1说明输入图像有三个通道(彩色) train_valid_ds = vision.ImageFolderDataset(os.path.join( input_str, 'train_valid'), flag=1) train_ds = vision.ImageFolderDataset(os.path.join(input_str, 'train'), flag=1) valid_ds = vision.ImageFolderDataset(os.path.join(input_str, 'valid'), flag=1) test_ds = vision.ImageFolderDataset(os.path.join(input_str, 'test'), flag=1) loader = gluon.data.DataLoader train_valid_data = loader(train_valid_ds.transform_first(transform_train), batch_size, shuffle=True, last_batch='keep') train_data = loader(train_ds.transform_first(transform_train), batch_size,