def __init__(self,
directory,
transforms=None,
target_transforms=None,
test=False,
has_mask=True):
"""CBDataset: Cranberry Dataset.
The sample images of this dataset must be all inside one directory.
Inside the same directory, there must be one CSV file.
This file must contain one row per image.
It can contain as many columns as wanted, i.e, filename, count...
:param directory: Directory with all the images and the CSV file.
:param transform: Transform to be applied to each image.
:param test: if this dataset is to be tested
:param has_mask: if the data has masks
"""
self.test = test
self.has_mask = has_mask
self.root_dir = directory
self.transforms = transforms
self.target_transforms = target_transforms
self.image_path = self.root_dir + "images/"
self.mask_path = self.root_dir + "masks/"
self.image_paths = sorted(
utils.dictionary_contents(self.image_path, types=IMG_EXTENSIONS))
self.mask_paths = sorted(
utils.dictionary_contents(self.mask_path, types=IMG_EXTENSIONS))
if len(self.image_paths) == 0:
raise ValueError("There are no images in {}".format(directory))
def __init__(self, directory, transforms=None, readsave=False):
self.transforms = transforms
self.root_dir = directory
self.images_path = self.root_dir + "images/"
self.masks_path = self.root_dir + "masks/"
self.image_paths = sorted(
utils.dictionary_contents(self.images_path, types=IMG_EXTENSIONS))
self.mask_paths = sorted(
utils.dictionary_contents(self.masks_path, types=IMG_EXTENSIONS))
def __init__(self,
directory,
transforms=None,
target_transforms=None,
test=False,
has_mask=True):
"""
CBDataset: Cranberry Dataset.
The sample images of this dataset must be all inside one directory.
:param directory: Directory with all the images and the CSV file.
:param transform: Transform to be applied to each image.
:param test: if this dataset is to be tested
:param has_mask: if the data has masks
"""
self.test = test
self.has_mask = has_mask
self.root_dir = directory
self.transforms = transforms
self.target_transforms = target_transforms
self.image_path = self.root_dir + "images/"
self.mask_path = self.root_dir + "masks/"
self.image_paths = sorted(
utils.dictionary_contents(self.image_path, types=IMG_EXTENSIONS))
self.mask_paths = sorted(
utils.dictionary_contents(self.mask_path, types=IMG_EXTENSIONS))
self.csv_path = None
self.csv_path = utils.dictionary_contents(self.root_dir,
types=["*.csv"])[0]
if len(self.image_paths) == 0:
raise ValueError("There are no images in {}".format(directory))
elif self.csv_path == None:
raise ValueError("There is no groundtruth in {}".format(directory))
self.csv_df = pd.read_csv(self.csv_path)
self.csv_df = self.csv_df.sample(frac=1).reset_index(drop=True)
def __init__(self,
directory,
transforms=None,
target_transforms=None,
test=False,
has_mask=True):
"""
CBDataset: Cranberry Dataset.
The sample images of this dataset must be all inside one directory.
:param directory: Directory with all the images and the CSV file.
:param transform: Transform to be applied to each image.
:param test: if this dataset is to be tested
:param has_mask: if the data has masks
"""
self.root_dir = directory
self.has_mask = has_mask
self.multi_masks_path = self.root_dir + "combined_multi_mask/"
self.single_masks_path = self.root_dir + "combined_single_mask/"
self.images_path = self.root_dir + "images/"
self.masks_path = self.root_dir + "masks/"
self.transforms = transforms
self.target_transforms = target_transforms
self.image_paths = sorted(
utils.dictionary_contents(self.images_path, types=IMG_EXTENSIONS))
self.mask_paths = sorted(
utils.dictionary_contents(self.masks_path, types=IMG_EXTENSIONS))
self.multi_masks_paths = sorted(
utils.dictionary_contents(self.multi_masks_path,
types=IMG_EXTENSIONS))
self.single_masks_paths = sorted(
utils.dictionary_contents(self.single_masks_path,
types=IMG_EXTENSIONS))
if len(self.image_paths) == 0:
raise ValueError("There are no images in {}".format(directory))
def __init__(self,
directory,
transforms=None,
target_transforms=None,
test=False,
has_mask=True):
"""CBDataset: Cranberry Dataset.
The sample images of this dataset must be all inside one directory.
:param directory: Directory with all the images and the CSV file.
:param transform: Transform to be applied to each image.
:param test: if this dataset is to be tested
:param has_mask: if the data has masks
"""
self.transforms = transforms
self.target_transforms = target_transforms
self.root_dir = directory
self.full_supervision = True
self.images_path = self.root_dir + "images/"
self.masks_path = self.root_dir + "masks/"
self.image_paths = sorted(
utils.dictionary_contents(self.images_path, types=IMG_EXTENSIONS))
self.mask_paths = sorted(
utils.dictionary_contents(self.masks_path, types=IMG_EXTENSIONS))
self.test = test
if not self.test and not self.full_supervision:
self.back_masks_path = self.root_dir + "back_masks/"
self.back_mask_paths = sorted(
utils.dictionary_contents(self.back_masks_path,
types=IMG_EXTENSIONS))
self.has_mask = has_mask
if len(self.image_paths) == 0:
raise ValueError(f"There are no images in {self.images_path}")
def build_train_validation_loaders(config):
transformations = {
'img':transforms.Compose([
transforms.ToPILImage(),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor()
# transforms.Normalize([0.485,0.485,0.406], [0.229, 0.224, 0.225])
]),
'mask':transforms.Compose([
transforms.ToPILImage(),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor()
])
}
location = config['location']
type = config[location]['dataset']['type']
data_dir = config[location]['dataset']['data_dir']
year = config[location]['dataset']['year']
type = config[location]['dataset']['type']
batch_size = config['data_loaders']['batch_size']
num_workers = config['data_loaders']['num_workers']
annotation_dir = data_dir + "/"+ str(year)+"/annotations/"
annotaiton_files = utils.dictionary_contents(annotation_dir,types=['*.json'])
datasets_types = ['train','val','test']
datasets = {}
for annotation_file in annotaiton_files:
for datasets_type in datasets_types:
if (datasets_type+str(year) in annotation_file) and (type in annotation_file):
root_dir = data_dir +str(year)+'/' + datasets_type+str(year)+"/"
# print(root_dir)
datasets[datasets_type] = CocoDetection(root=root_dir,annFile=annotation_file,transform=transformations['img'],target_transform=transformations['mask'])
# print(datasets)
dataloaders = {}
for datasets_type in datasets.keys():
dataloaders[datasets_type] = torch.utils.data.DataLoader(datasets[datasets_type],batch_size, shuffle=True,num_workers=num_workers)
if 'test' in datasets.keys():
return dataloaders['train'],dataloaders['val'],dataloaders['test']
else:
return dataloaders['train'],dataloaders['val']
return total_loss / self.val_loader.__len__(), mean_iou, count_metrics
def forward(self, cometml_experiment):
with cometml_experiment.validate():
self.evaluate(cometml_experiment)
return
if __name__ == "__main__":
project_name = f"{current_path[-3]}_{current_path[-1]}" #_{datetime.datetime.today().strftime('%Y-%m-%d-%H:%M')}"
experiment = comet_ml.Experiment(api_key="9GTK1r9PK4NzMAoLsnC6XxI7p",
project_name=project_name,
workspace="periakiva")
config_path = utils.dictionary_contents(os.getcwd() + "/",
types=["*.yaml"])[0]
config = utils.config_parser(config_path, experiment_type="training")
torch.set_default_dtype(torch.float32)
device_cpu = torch.device('cpu')
device = torch.device('cuda:0') if config['use_cuda'] else device_cpu
# data_dictionary,batch_size,num_workers,instance_seg = False):
test_loader = cranberry_dataset.build_single_loader(
data_dictionary=config['data']['eval_dir'],
batch_size=config['testing']['batch_size'],
num_workers=config['testing']['num_workers'],
instance_seg=config['data']['instance_seg'],
test=config['testing']['img_only'],
has_mask=config['data']['has_mask'])
def build_train_validation_loaders(data_dictionary,
batch_size,
num_workers,
type='',
train_val_test_split=[0.8, 0.1, 0.1]):
if type == "points_expand":
transformer = utils.ComposeJoint(
# [utils.RandomHorizontalFlipJoint(),
[[transforms.ToTensor(), None],
[transforms.Normalize(*mean_std), None],
[utils.ToFloat(), utils.ToLong()]])
else:
transformer = utils.ComposeJoint(
# [utils.RandomHorizontalFlipJoint(),
[
[transforms.ToTensor(), None],
# [transforms.Normalize(*mean_std), None],
[utils.ToFloat(), utils.ToLong()]
])
if type.lower() == 'instance_seg':
dataset = CBDatasetInstanceSeg(directory=data_dictionary,
transforms=transformer,
target_transforms=transformer)
elif type.lower() == 'points':
dataset = CBDatasetPoints(directory=data_dictionary,
transforms=transformer,
target_transforms=transformer)
elif type.lower() == 'points_expand':
dataset = CBDatasetPointsPixelExpansion(directory=data_dictionary,
transforms=transformer,
target_transforms=transformer)
elif type.lower() == 'semantic_seg':
dataset = CBDatasetSemanticSeg(directory=data_dictionary,
transforms=transformer,
target_transforms=transformer)
data_dictionary = data_dictionary + "/images/"
train_size = math.ceil(
int(
len(
utils.dictionary_contents(data_dictionary,
types=IMG_EXTENSIONS))) *
train_val_test_split[0])
val_size = math.floor(
int(
len(
utils.dictionary_contents(data_dictionary,
types=IMG_EXTENSIONS))) *
train_val_test_split[1])
test_size = int(
len(utils.dictionary_contents(
data_dictionary, types=IMG_EXTENSIONS))) - train_size - val_size
train_val_split_lengths = [train_size, val_size, test_size]
print(train_val_split_lengths)
print(
"Number of images for training: {}\nnumber of images for validation:{}\nnumber of images for testing: {}\n"
.format(train_val_split_lengths[0], train_val_split_lengths[1],
train_val_split_lengths[2]))
train_dataset, val_dataset, test_dataset = random_split(
dataset, train_val_split_lengths)
train_dataloader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
test_dataloader = torch.utils.data.DataLoader(test_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
return train_dataloader, val_dataloader, test_dataloader
yaml.dump(config, file)
# torch.save(self.model,model_save_dir+model_save_name)
torch.save(
{
'epoch': epoch,
'model': self.model.state_dict(),
'optimizer': self.optimizer.state_dict(),
'loss': train_loss
}, model_save_dir + model_save_name)
return train_losses, val_losses, mean_ious_val
if __name__ == "__main__":
config_path = utils.dictionary_contents(os.getcwd() + "/",
types=["*.yaml"])[0]
config = utils.config_parser(config_path, experiment_type="training")
config['start_time'] = datetime.datetime.today().strftime(
'%Y-%m-%d-%H:%M:%S')
project_name = f"{current_path[-3]}_{current_path[-1]}" #_{datetime.datetime.today().strftime('%Y-%m-%d-%H:%M')}"
experiment = comet_ml.Experiment(api_key=config['cometml_api_key'],
project_name=project_name,
workspace="periakiva")
# location = config['location']
torch.set_default_dtype(torch.float32)
device_cpu = torch.device('cpu')
device = torch.device('cuda:0') if config['use_cuda'] else device_cpu
train_dataloader, validation_dataloader, test_dataloader = cranberry_dataset.build_train_validation_loaders(
