def prepare_data(csv_file, image_folder, batch_size=10):
trainformations = transforms.Compose(
[Rescale(250), RandomCrop(224),
Normalize(), ToTensor()])
dataset = FacialKeyPointDataProccessing(csv_file,
image_folder,
transformation=trainformations)
validation_split = .1
shuffle_dataset = True
random_seed = 42
# Creating data indices for training and validation splits:
dataset_size = len(dataset)
indices = list(range(dataset_size))
split = int(np.floor(validation_split * dataset_size))
if shuffle_dataset:
np.random.seed(random_seed)
np.random.shuffle(indices)
train_indices, val_indices = indices[split:], indices[:split]
# Creating PT data samplers and loaders:
train_sampler = SubsetRandomSampler(train_indices)
valid_sampler = SubsetRandomSampler(val_indices)
train_loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
sampler=train_sampler)
validation_loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
sampler=valid_sampler)
return train_loader, validation_loader
def main():
with open("config.json", "r") as f:
config = json.load(f)
## Prepare data
data = CifarDataset(config)
all_transforms = transforms.Compose(
[ToGrayscale(), Normalize(), ToTensor()])
train_data_transformed = CifarDataLoader(config,
data.X_train,
data.y_train,
transform=all_transforms)
train_loader = DataLoader(train_data_transformed,
batch_size=config["data_loader"]["batch_size"],
shuffle=False,
num_workers=4)
if config["validation"]["split"]:
valid_data_transformed = CifarDataLoader(config,
data.X_valid,
data.y_valid,
transform=all_transforms)
valid_loader = DataLoader(
valid_data_transformed,
batch_size=config["data_loader"]["batch_size"],
shuffle=False,
num_workers=4)
test_data_transformed = CifarDataLoader(config,
data.X_test,
data.y_test,
transform=all_transforms)
test_loader = DataLoader(test_data_transformed,
batch_size=config["data_loader"]["batch_size"],
shuffle=False,
num_workers=4)
## Create neural net
net = LeNet()
## Training
trainer = Trainer(model=net,
config=config,
train_data_loader=train_loader,
valid_data_loader=valid_loader,
test_data_loader=test_loader)
trainer.train()
## Saving model parameters
trainer.save_model_params()
## Evaluate test data
trainer.evaluate()
crop = Crop([args.crop, args.crop])
boundary = None
lung_values = None
if args.boundary_transform:
lung_values = [
config.mask_values["right_lung"], config.mask_values["left_lung"]
]
boundary = TransformMaskToBoundary(
value_to_erode=config.mask_values["covid_tissue"],
iterations=parameters["iterations"],
fill_with_value=config.mask_values["right_lung"],
lung_values=lung_values)
normalize = Normalize(config.cgan_parameters["min"],
config.cgan_parameters["max"])
mask_lungs = ApplyMask(config.mask_values["non_lung_tissue"])
mask_covid = ApplyMask(config.mask_values["covid_tissue"], args.mask_covid)
dataset = CoivdLungMaskLungDataset(images=config.cgan_data_train,
mask_covid=mask_covid,
mask_lungs=mask_lungs,
max_rotation=max_rotation,
rotation=rotation,
crop=crop,
normalize=normalize,
boundary=boundary,
lung_values=lung_values)
valid_dataset = CoivdLungMaskLungDataset(images=config.cgan_data_test,
mask_covid=mask_covid,
testset_df = pd.read_csv(
'../data/CLEAN_REFIT/CLEAN_' + test[test_index] + '.csv',
skiprows=range(1, round(0.8 * house_lengths[test[test_index]])))
agg, iam = np.copy(np.array(testset_df['Aggregate'])), np.copy(
np.array(testset_df[params.fridge_channels[test[test_index]]]))
del testset_df
agg, iam = data.create_windows(agg, iam, params.MICROWAVE_WINDOW_SIZE)
testset = [agg, iam]
testset = datasets.RefitDataset(testset)
test_index += 1
# for i in range(10):
# agg, iam = testset[i]
# show_example(agg, iam, window_size=params.MICROWAVE_WINDOW_SIZE)
#Setting up Normalization transformation of each item returned by the testset object
testset.init_transformation(
torchvision.transforms.Compose([Normalize(mean=mean, sd=std)]))
#Evaluation of best model - before start of training
best_model.eval()
best_scores = scores.get_scores(best_model, testset, 1,
params.MICROWAVE_WINDOW_SIZE, std, mean)
#Mean Squared error is chosen as a loss function
criterion = nn.MSELoss()
#Stohastic Gradient Descent with learning rate 0.001 and momentum 0.9
optimimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
for epoch in range(1):
net.train()
#House indicators indicate current positions within .csv files of eache house included in training set
house_indicators = {house: 1 for house in training_houses.keys()}
def k_fold():
images, masks = load_train_data(TRAIN_IMAGES_PATH, TRAIN_MASKS_PATH)
test_file_paths, test_images = load_test_data(TEST_IMAGES_PATH,
load_images=True,
to256=False)
train_transformer = transforms.Compose([
CropAugmenter(),
AffineAugmenter(),
MasksAdder(),
ToTensor(),
Normalize(),
ClassAdder()
])
eval_transformer = transforms.Compose(
[MasksAdder(), ToTensor(),
Normalize(), ClassAdder()])
predict_transformer = transforms.Compose(
[ToTensor(predict=True),
Normalize(predict=True)])
test_images_loader = build_data_loader(test_images,
None,
predict_transformer,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=4,
predict=True)
k_fold = KFold(n_splits=FOLDS, random_state=RANDOM_SEED, shuffle=True)
test_masks_folds = []
config = AttrDict({
'cuda_index': CUDA_ID,
'momentum': MOMENTUM,
'lr': LR,
'tune_lr': TUNE_LR,
'min_lr': MIN_LR,
'bce_epochs': BCE_EPOCHS,
'intermediate_epochs': INTERMEDIATE_EPOCHS,
'cycle_length': CYCLE_LENGTH,
'logs_dir': LOGS_DIR,
'masks_weight': MASKS_WEIGHT,
'class_weight': CLASS_WEIGHT,
'val_metric_criterion': 'comp_metric'
})
for index, (train_index,
valid_index) in list(enumerate(k_fold.split(images))):
print('fold_{}\n'.format(index))
x_train_fold, x_valid = images[train_index], images[valid_index]
y_train_fold, y_valid = masks[train_index], masks[valid_index]
train_data_loader = build_data_loader(x_train_fold,
y_train_fold,
train_transformer,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=4,
predict=False)
val_data_loader = build_data_loader(x_valid,
y_valid,
eval_transformer,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=4,
predict=False)
test_data_loader = build_data_loader(x_valid,
y_valid,
eval_transformer,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=4,
predict=False)
data_loaders = AttrDict({
'train': train_data_loader,
'val': val_data_loader,
'test': test_data_loader
})
zers = np.zeros(BCE_EPOCHS)
zers += 0.1
lovasz_ratios = np.linspace(0.1, 0.9, INTERMEDIATE_EPOCHS)
lovasz_ratios = np.hstack((zers, lovasz_ratios))
bce_ratios = 1.0 - lovasz_ratios
loss_weights = [
(bce_ratio, lovasz_ratio)
for bce_ratio, lovasz_ratio in zip(bce_ratios, lovasz_ratios)
]
loss = LossAggregator((nn.BCEWithLogitsLoss(), LovaszLoss()),
weights=[0.9, 0.1])
metrics = {
'binary_accuracy': BinaryAccuracy,
'dice_coefficient': DiceCoefficient,
'comp_metric': CompMetric
}
segmentor = SawSeenNet(base_channels=64, pretrained=True,
frozen=False).cuda(config.cuda_index)
trainer = Trainer(config=config,
model=segmentor,
loss=loss,
loss_weights=loss_weights,
metrics=metrics,
data_loaders=data_loaders)
segmentor = trainer.train(num_epochs=NUM_EPOCHS,
model_pattern=MODEL_FILE_PATH +
'_{}_fold.pth'.format(index))
test_masks = predict(config,
segmentor,
test_images_loader,
thresholding=False)
test_masks = trim_masks(test_masks,
height=IMG_SIZE_ORIGIN,
width=IMG_SIZE_ORIGIN)
test_masks_folds.append(test_masks)
np.save(FOLDS_FILE_PATH.format(index), test_masks)
result_masks = np.zeros_like(test_masks_folds[0])
for test_masks in test_masks_folds:
result_masks += test_masks
result_masks = result_masks.astype(dtype=np.float32)
result_masks /= FOLDS
result_masks = result_masks > THRESHOLD
return test_file_paths, result_masks
from models import *
from torch.autograd import Variable
import torchvision
from scores import *
test_individual_networks = {"House16":"Appliance2"}
#Reading mean and standard deviation features of all houses's aggregate consumption
with open('fridge_data_features.pickle', 'rb') as f:
features = pickle.load(f)
mean, std = features['mean'], features['std']
#Reading data lengths of each house written inside of pickle binary file
with open('fridge_house_lengths.pickle','rb') as f:
house_lengths = pickle.load(f)
for test, channel in test_individual_networks.items():
print('Testing network for: ', test)
testset_df = pd.read_csv('../data/CLEAN_REFIT/CLEAN_' + test + '.csv', skiprows=range(1,round(0.8 * house_lengths[test])))
agg, iam = np.copy(np.array(testset_df['Aggregate'])), np.copy(np.array(testset_df[channel]))
del testset_df
agg, iam = data.create_windows(agg, iam, params.REFRIGERATOR_WINDOW_SIZE)
testset = [agg, iam]
testset = RefitDataset(testset)
testset.init_transformation(torchvision.transforms.Compose([Normalize(mean=mean, sd=std)]))
best_model = torch.load('models/refit_refrigerator_trained_model'+test+'2.pt')
if torch.cuda.is_available():
best_model = best_model.cuda()
best_model.eval()
best_scores= get_scores(best_model, testset, 1, params.REFRIGERATOR_WINDOW_SIZE, std, mean)
inputs_train = get_filenames_of_path(root, 'train/TP/5')
targets_train = [
pathlib.Path(
str(inp).replace('crops_fixed_scale_uint8', 'crops_lbl_fixed_rev'))
for inp in inputs_train
]
inputs_valid = get_filenames_of_path(root, 'valid/TP/5')
targets_valid = [
pathlib.Path(
str(inp).replace('crops_fixed_scale_uint8', 'crops_lbl_fixed_rev'))
for inp in inputs_valid
]
transforms_training = Compose([Normalize(), Unsqueeze()])
transforms_validation = Compose([Normalize(), Unsqueeze()])
dataset_train = SegmentationDataset(inputs=inputs_train,
targets=targets_train,
transform=transforms_training)
dataset_valid = SegmentationDataset(inputs=inputs_valid,
targets=targets_valid,
transform=transforms_validation)
dataloder_training = DataLoader(dataset=dataset_train,
batch_size=2,
shuffle=True)
dataloader_validation = DataLoader(dataset=dataset_valid,
batch_size=2,
shuffle=True)
"""
Implements loaders for training and testing data.
"""
from torch.utils.data import DataLoader
from torchvision.transforms import Compose
from dataset import FacialLandmarksDataset
from transformations import (Rescale, RandomCrop, Normalize, ToTensor)
__author__ = "Victor mawusi Ayi <*****@*****.**>"
data_transform = Compose(
[Rescale(250), RandomCrop(224),
Normalize(), ToTensor()])
def train_dataset(transforms_pipe=data_transform):
return FacialLandmarksDataset(
keypoints_file='/data/training_frames_keypoints.csv',
images_dir='/data/training/',
transforms=transforms_pipe)
def test_dataset(transforms_pipe=data_transform):
return FacialLandmarksDataset(
keypoints_file='/data/test_frames_keypoints.csv',
images_dir='/data/test/',
transforms=transforms_pipe)
def get_filenames_of_path(path: pathlib.Path, ext: str = '*'):
"""Returns a list of files in a directory/path. Uses pathlib."""
filenames = [
file for file in path.glob(ext + '/**/*.npy') if file.is_file()
]
return filenames
root = pathlib.Path('/data/dk/datasets_CROPS/crops_fixed_scale_uint8')
inputs = get_filenames_of_path(root, 'valid/TP')
targets = [
pathlib.Path(
str(inp).replace('crops_fixed_scale_uint8', 'crops_lbl_fixed_rev'))
for inp in inputs
]
transforms = Compose([Normalize()])
dataset_valid = SegmentationDataset(inputs=inputs,
targets=targets,
transform=transforms)
dataloader_valid = DataLoader(dataset=dataset_valid,
batch_size=4,
shuffle=True)
x, y = next(iter(dataloader_valid))
print(f'x = shape: {x.shape}; type: {x.dtype}')
print(f'y = shape: {y.shape}; type: {y.dtype}')
y_r = np.reshape(y, (4, -1))
print(y_r.shape)
print(y_r.sum(axis=1))
