def prepare_data(csv_file, image_folder, batch_size=10): trainformations = transforms.Compose( [Rescale(250), RandomCrop(224), Normalize(), ToTensor()]) dataset = FacialKeyPointDataProccessing(csv_file, image_folder, transformation=trainformations) validation_split = .1 shuffle_dataset = True random_seed = 42 # Creating data indices for training and validation splits: dataset_size = len(dataset) indices = list(range(dataset_size)) split = int(np.floor(validation_split * dataset_size)) if shuffle_dataset: np.random.seed(random_seed) np.random.shuffle(indices) train_indices, val_indices = indices[split:], indices[:split] # Creating PT data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler) validation_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler) return train_loader, validation_loader
def main(): with open("config.json", "r") as f: config = json.load(f) ## Prepare data data = CifarDataset(config) all_transforms = transforms.Compose( [ToGrayscale(), Normalize(), ToTensor()]) train_data_transformed = CifarDataLoader(config, data.X_train, data.y_train, transform=all_transforms) train_loader = DataLoader(train_data_transformed, batch_size=config["data_loader"]["batch_size"], shuffle=False, num_workers=4) if config["validation"]["split"]: valid_data_transformed = CifarDataLoader(config, data.X_valid, data.y_valid, transform=all_transforms) valid_loader = DataLoader( valid_data_transformed, batch_size=config["data_loader"]["batch_size"], shuffle=False, num_workers=4) test_data_transformed = CifarDataLoader(config, data.X_test, data.y_test, transform=all_transforms) test_loader = DataLoader(test_data_transformed, batch_size=config["data_loader"]["batch_size"], shuffle=False, num_workers=4) ## Create neural net net = LeNet() ## Training trainer = Trainer(model=net, config=config, train_data_loader=train_loader, valid_data_loader=valid_loader, test_data_loader=test_loader) trainer.train() ## Saving model parameters trainer.save_model_params() ## Evaluate test data trainer.evaluate()
crop = Crop([args.crop, args.crop]) boundary = None lung_values = None if args.boundary_transform: lung_values = [ config.mask_values["right_lung"], config.mask_values["left_lung"] ] boundary = TransformMaskToBoundary( value_to_erode=config.mask_values["covid_tissue"], iterations=parameters["iterations"], fill_with_value=config.mask_values["right_lung"], lung_values=lung_values) normalize = Normalize(config.cgan_parameters["min"], config.cgan_parameters["max"]) mask_lungs = ApplyMask(config.mask_values["non_lung_tissue"]) mask_covid = ApplyMask(config.mask_values["covid_tissue"], args.mask_covid) dataset = CoivdLungMaskLungDataset(images=config.cgan_data_train, mask_covid=mask_covid, mask_lungs=mask_lungs, max_rotation=max_rotation, rotation=rotation, crop=crop, normalize=normalize, boundary=boundary, lung_values=lung_values) valid_dataset = CoivdLungMaskLungDataset(images=config.cgan_data_test, mask_covid=mask_covid,
testset_df = pd.read_csv( '../data/CLEAN_REFIT/CLEAN_' + test[test_index] + '.csv', skiprows=range(1, round(0.8 * house_lengths[test[test_index]]))) agg, iam = np.copy(np.array(testset_df['Aggregate'])), np.copy( np.array(testset_df[params.fridge_channels[test[test_index]]])) del testset_df agg, iam = data.create_windows(agg, iam, params.MICROWAVE_WINDOW_SIZE) testset = [agg, iam] testset = datasets.RefitDataset(testset) test_index += 1 # for i in range(10): # agg, iam = testset[i] # show_example(agg, iam, window_size=params.MICROWAVE_WINDOW_SIZE) #Setting up Normalization transformation of each item returned by the testset object testset.init_transformation( torchvision.transforms.Compose([Normalize(mean=mean, sd=std)])) #Evaluation of best model - before start of training best_model.eval() best_scores = scores.get_scores(best_model, testset, 1, params.MICROWAVE_WINDOW_SIZE, std, mean) #Mean Squared error is chosen as a loss function criterion = nn.MSELoss() #Stohastic Gradient Descent with learning rate 0.001 and momentum 0.9 optimimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) for epoch in range(1): net.train() #House indicators indicate current positions within .csv files of eache house included in training set house_indicators = {house: 1 for house in training_houses.keys()}
def k_fold(): images, masks = load_train_data(TRAIN_IMAGES_PATH, TRAIN_MASKS_PATH) test_file_paths, test_images = load_test_data(TEST_IMAGES_PATH, load_images=True, to256=False) train_transformer = transforms.Compose([ CropAugmenter(), AffineAugmenter(), MasksAdder(), ToTensor(), Normalize(), ClassAdder() ]) eval_transformer = transforms.Compose( [MasksAdder(), ToTensor(), Normalize(), ClassAdder()]) predict_transformer = transforms.Compose( [ToTensor(predict=True), Normalize(predict=True)]) test_images_loader = build_data_loader(test_images, None, predict_transformer, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, predict=True) k_fold = KFold(n_splits=FOLDS, random_state=RANDOM_SEED, shuffle=True) test_masks_folds = [] config = AttrDict({ 'cuda_index': CUDA_ID, 'momentum': MOMENTUM, 'lr': LR, 'tune_lr': TUNE_LR, 'min_lr': MIN_LR, 'bce_epochs': BCE_EPOCHS, 'intermediate_epochs': INTERMEDIATE_EPOCHS, 'cycle_length': CYCLE_LENGTH, 'logs_dir': LOGS_DIR, 'masks_weight': MASKS_WEIGHT, 'class_weight': CLASS_WEIGHT, 'val_metric_criterion': 'comp_metric' }) for index, (train_index, valid_index) in list(enumerate(k_fold.split(images))): print('fold_{}\n'.format(index)) x_train_fold, x_valid = images[train_index], images[valid_index] y_train_fold, y_valid = masks[train_index], masks[valid_index] train_data_loader = build_data_loader(x_train_fold, y_train_fold, train_transformer, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, predict=False) val_data_loader = build_data_loader(x_valid, y_valid, eval_transformer, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, predict=False) test_data_loader = build_data_loader(x_valid, y_valid, eval_transformer, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, predict=False) data_loaders = AttrDict({ 'train': train_data_loader, 'val': val_data_loader, 'test': test_data_loader }) zers = np.zeros(BCE_EPOCHS) zers += 0.1 lovasz_ratios = np.linspace(0.1, 0.9, INTERMEDIATE_EPOCHS) lovasz_ratios = np.hstack((zers, lovasz_ratios)) bce_ratios = 1.0 - lovasz_ratios loss_weights = [ (bce_ratio, lovasz_ratio) for bce_ratio, lovasz_ratio in zip(bce_ratios, lovasz_ratios) ] loss = LossAggregator((nn.BCEWithLogitsLoss(), LovaszLoss()), weights=[0.9, 0.1]) metrics = { 'binary_accuracy': BinaryAccuracy, 'dice_coefficient': DiceCoefficient, 'comp_metric': CompMetric } segmentor = SawSeenNet(base_channels=64, pretrained=True, frozen=False).cuda(config.cuda_index) trainer = Trainer(config=config, model=segmentor, loss=loss, loss_weights=loss_weights, metrics=metrics, data_loaders=data_loaders) segmentor = trainer.train(num_epochs=NUM_EPOCHS, model_pattern=MODEL_FILE_PATH + '_{}_fold.pth'.format(index)) test_masks = predict(config, segmentor, test_images_loader, thresholding=False) test_masks = trim_masks(test_masks, height=IMG_SIZE_ORIGIN, width=IMG_SIZE_ORIGIN) test_masks_folds.append(test_masks) np.save(FOLDS_FILE_PATH.format(index), test_masks) result_masks = np.zeros_like(test_masks_folds[0]) for test_masks in test_masks_folds: result_masks += test_masks result_masks = result_masks.astype(dtype=np.float32) result_masks /= FOLDS result_masks = result_masks > THRESHOLD return test_file_paths, result_masks
from models import * from torch.autograd import Variable import torchvision from scores import * test_individual_networks = {"House16":"Appliance2"} #Reading mean and standard deviation features of all houses's aggregate consumption with open('fridge_data_features.pickle', 'rb') as f: features = pickle.load(f) mean, std = features['mean'], features['std'] #Reading data lengths of each house written inside of pickle binary file with open('fridge_house_lengths.pickle','rb') as f: house_lengths = pickle.load(f) for test, channel in test_individual_networks.items(): print('Testing network for: ', test) testset_df = pd.read_csv('../data/CLEAN_REFIT/CLEAN_' + test + '.csv', skiprows=range(1,round(0.8 * house_lengths[test]))) agg, iam = np.copy(np.array(testset_df['Aggregate'])), np.copy(np.array(testset_df[channel])) del testset_df agg, iam = data.create_windows(agg, iam, params.REFRIGERATOR_WINDOW_SIZE) testset = [agg, iam] testset = RefitDataset(testset) testset.init_transformation(torchvision.transforms.Compose([Normalize(mean=mean, sd=std)])) best_model = torch.load('models/refit_refrigerator_trained_model'+test+'2.pt') if torch.cuda.is_available(): best_model = best_model.cuda() best_model.eval() best_scores= get_scores(best_model, testset, 1, params.REFRIGERATOR_WINDOW_SIZE, std, mean)
inputs_train = get_filenames_of_path(root, 'train/TP/5') targets_train = [ pathlib.Path( str(inp).replace('crops_fixed_scale_uint8', 'crops_lbl_fixed_rev')) for inp in inputs_train ] inputs_valid = get_filenames_of_path(root, 'valid/TP/5') targets_valid = [ pathlib.Path( str(inp).replace('crops_fixed_scale_uint8', 'crops_lbl_fixed_rev')) for inp in inputs_valid ] transforms_training = Compose([Normalize(), Unsqueeze()]) transforms_validation = Compose([Normalize(), Unsqueeze()]) dataset_train = SegmentationDataset(inputs=inputs_train, targets=targets_train, transform=transforms_training) dataset_valid = SegmentationDataset(inputs=inputs_valid, targets=targets_valid, transform=transforms_validation) dataloder_training = DataLoader(dataset=dataset_train, batch_size=2, shuffle=True) dataloader_validation = DataLoader(dataset=dataset_valid, batch_size=2, shuffle=True)
""" Implements loaders for training and testing data. """ from torch.utils.data import DataLoader from torchvision.transforms import Compose from dataset import FacialLandmarksDataset from transformations import (Rescale, RandomCrop, Normalize, ToTensor) __author__ = "Victor mawusi Ayi <*****@*****.**>" data_transform = Compose( [Rescale(250), RandomCrop(224), Normalize(), ToTensor()]) def train_dataset(transforms_pipe=data_transform): return FacialLandmarksDataset( keypoints_file='/data/training_frames_keypoints.csv', images_dir='/data/training/', transforms=transforms_pipe) def test_dataset(transforms_pipe=data_transform): return FacialLandmarksDataset( keypoints_file='/data/test_frames_keypoints.csv', images_dir='/data/test/', transforms=transforms_pipe)
def get_filenames_of_path(path: pathlib.Path, ext: str = '*'): """Returns a list of files in a directory/path. Uses pathlib.""" filenames = [ file for file in path.glob(ext + '/**/*.npy') if file.is_file() ] return filenames root = pathlib.Path('/data/dk/datasets_CROPS/crops_fixed_scale_uint8') inputs = get_filenames_of_path(root, 'valid/TP') targets = [ pathlib.Path( str(inp).replace('crops_fixed_scale_uint8', 'crops_lbl_fixed_rev')) for inp in inputs ] transforms = Compose([Normalize()]) dataset_valid = SegmentationDataset(inputs=inputs, targets=targets, transform=transforms) dataloader_valid = DataLoader(dataset=dataset_valid, batch_size=4, shuffle=True) x, y = next(iter(dataloader_valid)) print(f'x = shape: {x.shape}; type: {x.dtype}') print(f'y = shape: {y.shape}; type: {y.dtype}') y_r = np.reshape(y, (4, -1)) print(y_r.shape) print(y_r.sum(axis=1))