def get_cat_dogs_dataset(
dirs: str = "/app/data/data_cat_dogs/*",
extension: str = "*.jpg",
test_size: float = 0.2,
random_state: int = 42,
tag_file_path: tp.Optional[str] = None,
) -> tp.Tuple[tp.Dict[str, tp.Any], tp.Dict[str, tp.Any], int]:
dataset = utils.create_dataset(dirs=dirs, extension=extension)
df = utils.create_dataframe(dataset, columns=["class", "filepath"])
tag_to_label = utils.get_dataset_labeling(df, "class")
if tag_file_path is not None:
with open(tag_file_path, "w") as file:
json.dump(tag_to_label, file)
df_with_labels = utils.map_dataframe(
df,
tag_column="class",
class_column="label",
tag2class=tag_to_label,
verbose=False,
)
train_data, valid_data = utils.split_dataframe_train_test(
df_with_labels, test_size=test_size, random_state=random_state)
return (
train_data.to_dict("records"),
valid_data.to_dict("records"),
len(tag_to_label),
)
def build(self):
SEED = 42
data = pd.read_csv(self.data)
ab = data.label
############################################
transforms = [
RescaleIntensity((0, 1)),
RandomAffine(),
transformss.ToTensor(),
]
transform = Compose(transforms)
#############################################
dataset_dir = self.dataset_dir
dataset_dir = Path(dataset_dir)
images_dir = dataset_dir
labels_dir = dataset_dir
image_paths = sorted(images_dir.glob('**/*.nii'))
label_paths = sorted(labels_dir.glob('**/*.nii'))
assert len(image_paths) == len(label_paths)
# These two names are arbitrary
MRI = 'features'
BRAIN = 'targets'
#split dataset into training and validation
from catalyst.utils import split_dataframe_train_test
train_image_paths, valid_image_paths = split_dataframe_train_test(
image_paths, test_size=0.2, random_state=SEED)
#training data
subjects = []
i = 0
for (image_path, label_path) in zip(train_image_paths, label_paths):
subject_dict = {
MRI: torchio.Image(image_path, torchio.INTENSITY),
BRAIN: ab[i],
}
i = i + 1
subject = torchio.Subject(subject_dict)
subjects.append(subject)
train_data = torchio.ImagesDataset(subjects)
#validation data
subjects = []
for (image_path, label_path) in zip(valid_image_paths, label_paths):
subject_dict = {
MRI: torchio.Image(image_path, torchio.INTENSITY),
BRAIN: ab[i],
}
i = i + 1
subject = torchio.Subject(subject_dict)
subjects.append(subject)
test_data = torchio.ImagesDataset(subjects)
return train_data, test_data
def balance_data(csv_path: str,
test_size: float = 0.2,
random_state: int = 123):
df = pd.read_csv(csv_path)
# first class has large number of samples as compares to others
# one way to balance is by sampling smaller amount of data
class_0 = df[df['diagnosis'] == 0]
class_0 = class_0.sample(400)
class_0_train, class_0_test = split_dataframe_train_test(
class_0, test_size=test_size, random_state=random_state)
df_train = class_0_train
df_test = class_0_test
class_1 = df[df['diagnosis'] == 1]
class_1_train, class_1_test = split_dataframe_train_test(
class_1, test_size=test_size, random_state=random_state)
df_train = df_train.append(class_1_train)
df_test = df_test.append(class_1_test)
# sub sampling data for Moderate category
class_2 = df[df['diagnosis'] == 2]
class_2 = class_2.sample(400)
class_2_train, class_2_test = split_dataframe_train_test(
class_2, test_size=test_size, random_state=random_state)
df_train = df_train.append(class_2_train)
df_test = df_test.append(class_2_test)
class_3 = df[df['diagnosis'] == 3]
class_3_train, class_3_test = split_dataframe_train_test(
class_3, test_size=test_size, random_state=random_state)
df_train = df_train.append(class_3_train)
df_test = df_test.append(class_3_test)
class_4 = df[df['diagnosis'] == 4]
class_4_train, class_4_test = split_dataframe_train_test(
class_4, test_size=test_size, random_state=random_state)
df_train = df_train.append(class_4_train)
df_test = df_test.append(class_4_test)
return df_train, df_test
def get_datasets(config):
train_df = pd.read_csv(config.root + 'train.csv')
test_df = pd.read_csv(config.root + 'test.csv')
dataset = create_dataset(root_dir=config.root_images,
mask="Train*.jpg",
config=config)
df_path = create_dataframe(dataset, columns=["image_id", "filepath"])
df_with_labels = pd.merge(df_path,
train_df,
left_on='image_id',
right_on='image_id')
df_with_labels["disease_type"] = df_with_labels["healthy"] * 0 + df_with_labels["multiple_diseases"] * 1 + \
df_with_labels["rust"] * 2 + df_with_labels["scab"] * 3
df_with_labels.head(10)
train_data, valid_data = split_dataframe_train_test(
df_with_labels, test_size=0.3, random_state=config.seed)
train_data, valid_data = train_data.to_dict('records'), valid_data.to_dict(
'records')
return train_data, valid_data
utils.prepare_cudnn(deterministic=True)
train_df = pd.read_csv(config.root + 'train.csv')
test_df = pd.read_csv(config.root + 'test.csv')
dataset = create_dataset(root_dir=config.root_images, mask="Train*.jpg")
df_path = create_dataframe(dataset, columns=["image_id", "filepath"])
df_with_labels = pd.merge(df_path,
train_df,
left_on='image_id',
right_on='image_id')
df_with_labels["disease_type"] = df_with_labels["healthy"] * 0 + df_with_labels["multiple_diseases"] * 1 + \
df_with_labels["rust"] * 2 + df_with_labels["scab"] * 3
df_with_labels.head(10)
train_data, valid_data = split_dataframe_train_test(df_with_labels,
test_size=0.3,
random_state=config.seed)
train_data, valid_data = train_data.to_dict('records'), valid_data.to_dict(
'records')
open_fn = ReaderCompose([
ImageReader(input_key="filepath",
output_key="features",
rootpath=config.root_images),
ScalarReader(input_key="disease_type",
output_key="targets",
default_value=-1,
dtype=np.int64),
ScalarReader(input_key="disease_type",
output_key="targets_one_hot",
default_value=-1,