def data_preprare(self, test=False): if len(os.path.basename(self.data_dir).split('.')) == 2: with open(self.data_dir, 'rb') as fo: list_data = pickle.load(fo, encoding='bytes') else: list_data = get_files(self.data_dir, test) with open(os.path.join(self.data_dir, "MFPTCWT.pkl"), 'wb') as fo: pickle.dump(list_data, fo) if test: test_dataset = dataset(list_data=list_data, test=True, transform=data_transforms( 'val', self.normalise_type)) return test_dataset else: print(len(list_data[0])) print(len(list_data[1])) data_pd = pd.DataFrame({ "data": list_data[0], "label": list_data[1] }) train_pd, val_pd = train_test_split(data_pd, test_size=0.2, random_state=40, stratify=data_pd["label"]) train_dataset = dataset(list_data=train_pd, transform=data_transforms( 'train', self.normalise_type)) val_dataset = dataset(list_data=val_pd, transform=data_transforms( 'val', self.normalise_type)) return train_dataset, val_dataset
def data_preprare(self, test=False): if len(os.path.basename(self.data_dir).split('.')) == 2: with open(self.data_dir, 'rb') as fo: list_data = pickle.load(fo, encoding='bytes') else: list_data = get_files(self.data_dir, test) with open(os.path.join(self.data_dir, "CWRUCWT.pkl"), 'wb') as fo: pickle.dump(list_data, fo) if test: test_dataset = dataset(list_data=list_data, test=True, transform=None) return test_dataset else: data_pd = pd.DataFrame({ "data": list_data[0], "label": list_data[1] }) train_pd, val_pd = train_test_split_order(data_pd, test_size=0.2, num_classes=10) train_dataset = dataset(list_data=train_pd, transform=data_transforms( 'train', self.normlizetype)) val_dataset = dataset(list_data=val_pd, transform=data_transforms( 'val', self.normlizetype)) return train_dataset, val_dataset
def data_preprare(self, test=False): if len(os.path.basename(self.data_dir).split('.')) == 2: with open(self.data_dir, 'rb') as fo: list_data = pickle.load(fo, encoding='bytes') else: list_data = get_files(self.data_dir, test) with open(os.path.join(self.data_dir, "CWRUSlice.pkl"), 'wb') as fo: pickle.dump(list_data, fo) if test: test_dataset = dataset(list_data=list_data, test=True, transform=None) return test_dataset else: data_pd = pd.DataFrame({ "data": list_data[0], "label": list_data[1] }) train_pd, val_pd = train_test_split_order(data_pd, test_size=0.6, num_classes=10) train_dataset = dataset(list_data=train_pd, transform=data_transforms( 'train', self.normlizetype)) val_dataset = dataset(list_data=val_pd, transform=data_transforms( 'val', self.normlizetype)) return train_dataset, val_dataset # path = r'/media/ubuntu/Data/hcy/dl_based_bearing_diagnosis/tmp' # data = CWRUSlice(path, 'mean-std') # datasets = {} # datasets['train'], datasets['val'] = data.data_preprare() # dataloaders = {x: torch.utils.data.DataLoader(datasets[x], batch_size=64, # shuffle=(True if x == 'train' else False), # num_workers=0) # for x in ['train', 'val']} # a = next(iter(dataloaders['train'])) # # print(a[0].shape) # path = r'E:\learning\现代信号处理\dl_based_bearing_diagnosis\tmp' # data = CWRUSlice(path, 'mean-std') # # # print(max(train_dataset[600][0][0]), len(val_dataset)) # # plt.plot(train_dataset[600][0][0]) # # plt.show() # # datasets = {} # datasets['train'], datasets['val'] = data.data_preprare() # dataloaders = {x: torch.utils.data.DataLoader(datasets[x], batch_size=64, # shuffle=(True if x == 'train' else False), # num_workers=0) # for x in ['train', 'val']} # a = next(iter(dataloaders['train'])) # # print(a[0][5][0],a[0].shape) # print(len(datasets['train']), len(datasets['val'])) # plt.imshow(a[0][8][0]) # plt.show()