def __init__(self, hparams):
super(Unet3D, self).__init__()
self.hparams = hparams
self.learning_rate = hparams.learning_rate
self.data_set_dir = hparams.data_set_dir
self.loader_kwargs = {
'batch_size': hparams.batch_size,
'num_workers': hparams.num_workers,
'pin_memory': True
}
self.valid_split = hparams.valid_split
num_pool = hparams.num_pool
num_features = hparams.num_features
patch_size = (hparams.patch_x, hparams.patch_y, hparams.patch_z)
def encode_kwargs_fn(level):
num_stacks = max(level, 1)
return {'num_stacks': num_stacks}
paired_features = generate_paired_features(num_pool, num_features)
self.net = Unet(in_channels=1,
out_channels=1,
paired_features=paired_features,
pool_block=ResBlock,
pool_kwargs={'stride': 2},
up_kwargs={'attention': True},
encode_block=ResBlockStack,
encode_kwargs_fn=encode_kwargs_fn,
decode_block=ResBlock)
self.loss_fn = FocalDiceCoefLoss()
self.metrics = {'kd_dsc': DiceCoef()}
self.tr_transform = Compose([
RandomRescaleCrop(0.1,
patch_size,
crop_mode='random',
enforce_label_indices=[1]),
RandomMirror((0.2, 0, 0)),
RandomContrast(0.1),
RandomBrightness(0.1),
RandomGamma(0.1),
CombineLabels([1, 2], 3),
ToTensor()
])
self.vd_transform = Compose(
[RandomCrop(patch_size),
CombineLabels([1, 2], 3),
ToTensor()])
def __init__(self):
self.trans = v_transforms.Compose([
Normalize(bound=[-1300., 500.], cover=[0., 1.]),
CenterCrop([48, 96, 96]),
ToTensor(),
Resize([48, 96, 96]),
])
def get_datasets(label_columns=['study'],
transform=None,
balance_train=True,
ratio=0.1):
cancer_data_dir = '/home/noskill/projects/cancer.old/data'
dataset_dict = load_merged_dataset(cancer_data_dir)
merged = dataset_dict['merged']
bmc = dataset_dict['bmc']
genes_features = dataset_dict['genes_features']
genes_columns = genes_features.columns.to_list()[1:]
feature_columns = genes_columns
if transform is None:
to_tensor = ToTensor()
transform = DataLabelCompose(to_tensor, to_tensor)
train_data, train_labels, val_data, val_labels = random_split(
merged,
feature_columns,
label_columns,
balance_train=balance_train,
balance_by_study=False,
ratio=ratio)
# assert val_labels.mean() == 0.5
train_set = GeneDataset(train_data, train_labels, transform)
test_set = GeneDataset(val_data, val_labels, transform)
return train_set, test_set
def get_data_loader(cfg, data_dir, batch_size=None):
batch_size = cfg["batch_size"] if batch_size is None else batch_size
transform = transforms.Compose([
RandomCrop(cfg["hr_crop_size"], cfg["scale"]),
ToTensor()])
dataset = DIV2K(data_dir, transform=transform)
return DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=8)
def __init__(self, num=1):
from fem import noise
from fem.training import make_noisy_transformers
self.noisy = make_noisy_transformers()
self.imgcrop = noise.RandomCropTransform(size=256, beta=0)
self.resize = noise.Resize((256, 256))
self.to_tensor = ToTensor()
self.homography = HomographySamplerTransformer(
num=1,
beta=14,
theta=0.08,
random_scale_range=(0.8, 1.3),
perspective=85)
self.num = num
def make_noisy_transformers():
from torchvision.transforms import Compose
from transform import RandomTransformer, ToTensor
from noise import AdditiveGaussian, RandomBrightness, AdditiveShade, MotionBlur, SaltPepper, RandomContrast
totensor = ToTensor()
# ColorInversion doesn't seem to be usefull on most datasets
transformer = [
AdditiveGaussian(var=30),
RandomBrightness(range=(-50, 50)),
AdditiveShade(kernel_size_range=[45, 85],
transparency_range=(-0.25, .45)),
SaltPepper(),
MotionBlur(max_kernel_size=5),
RandomContrast([0.6, 1.05])
]
return Compose([RandomTransformer(transformer), totensor])
def __init__(self, cfg):
super(Data, self).__init__()
self.cfg = cfg
# 下面是数据增强等
self.randombrig = RandomBrightness()
self.normalize = Normalize(mean=cfg.mean, std=cfg.std)
self.randomcrop = RandomCrop()
self.blur = RandomBlur()
self.randomvflip = RandomVorizontalFlip()
self.randomhflip = RandomHorizontalFlip()
self.resize = Resize(384, 384)
self.totensor = ToTensor()
# 读数据
with open(cfg.datapath + '/' + cfg.mode + '.txt', 'r') as lines:
self.samples = []
for line in lines:
self.samples.append(line.strip())
def __init__(self, root):
self.size = (192, 192)
self.root = root
if not os.path.exists(self.root):
raise Exception("[!] {} not exists.".format(root))
self.img_transform = Compose([
ToTensor(),
CenterCrop(self.size),
# RangeNormalize(min_val=-1,max_val=1),
# RandomFlip(),
])
#sort file names
self.input_paths = sorted(
glob(os.path.join(self.root, '{}/*.npy'.format("val"))))
self.name = os.path.basename(root)
if len(self.input_paths) == 0:
raise Exception("No validations are found in {}".format(self.root))
def get_metagx_dataset(ratio=0.1):
data = load_metagx_dataset('/home/noskill/projects/cancer/data/metaGxBreast/', min_genes=5000)
merged = data['merged']
genes_list = data['genes_features']
train_data, train_labels, val_data, val_labels = util.random_split(merged,
genes_list,
['study'],
balance_validation=False,
balance_by_study=False,
ratio=ratio)
to_tensor = ToTensor()
transform = DataLabelCompose(to_tensor, to_tensor)
# assert val_labels.mean() == 0.5
train_set = GeneDataset(train_data, train_labels, transform)
test_set = GeneDataset(val_data, val_labels, transform)
return train_set, test_set
def read_image(dir):
transform = transforms.Compose(
[Rescale(224), RandomCrop(223),
Normalize(), ToTensor()])
image = cv2.imread(dir)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
sample = {'image': image, 'keypoints': None}
transformed_image = transform(sample)
images = transformed_image['image']
print(images.shape)
images = images.type(torch.FloatTensor)
images = images.unsqueeze(0)
out = model(images)
out = out.view(-1, 2)
out = out.data * 50.0 + 100
images = np.transpose(images, (0, 2, 3, 1)).numpy().squeeze(3)
display(images.squeeze(0), out)
import cv2
import torch
from torch.utils.data import DataLoader
from dataset import FightDataset
from transform import Compose, ToTensor, Resize
from model import MyNet
torch.cuda.set_device(0)
transform_ = Compose([Resize((112, 112)), ToTensor()])
xx = FightDataset("./fight_classify", tranform=transform_)
dataloader = DataLoader(xx, batch_size=1, shuffle=True)
# for i_batch, sample_batched in enumerate(dataloader):
# print(i_batch)
# print(sample_batched["image"].size())
dev = torch.device("cuda:0")
model = MyNet().to(dev)
criterion = torch.nn.MSELoss(reduction='sum')
optimizer = torch.optim.SGD(model.parameters(), lr=1e-8, momentum=0.9)
for t in range(20):
# Forward pass: Compute predicted y by passing x to the model
for i_batch, sample_batched in enumerate(dataloader):
image = sample_batched["image"]
label = sample_batched["label"]
# label = torch.transpose(label, 0,1)
y_pred = model(image)
# print(y_pred)
# print(label)
def main():
args = parser.parse_args()
save_path = 'Trainid_' + args.id
writer = SummaryWriter(log_dir='runs/' + args.tag + str(time.time()))
if not os.path.isdir(save_path):
os.mkdir(save_path)
os.mkdir(save_path + '/Checkpoint')
train_dataset_path = 'data/train'
val_dataset_path = 'data/valid'
train_transform = transforms.Compose([ToTensor()])
val_transform = transforms.Compose([ToTensor()])
train_dataset = TrainDataset(path=train_dataset_path,
transform=train_transform)
val_dataset = TrainDataset(path=val_dataset_path, transform=val_transform)
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=4)
val_dataloader = DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=4)
size_train = len(train_dataloader)
size_val = len(val_dataloader)
print('Number of Training Images: {}'.format(size_train))
print('Number of Validation Images: {}'.format(size_val))
start_epoch = 0
model = Res(n_ch=4, n_classes=9)
class_weights = torch.Tensor([1, 1, 1, 1, 1, 1, 1, 1, 0]).cuda()
criterion = DiceLoss()
criterion1 = torch.nn.CrossEntropyLoss(weight=class_weights)
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
if args.gpu:
model = model.cuda()
criterion = criterion.cuda()
criterion1 = criterion1.cuda()
if args.resume is not None:
weight_path = sorted(os.listdir(save_path + '/Checkpoint/'),
key=lambda x: float(x[:-8]))[0]
checkpoint = torch.load(save_path + '/Checkpoint/' + weight_path)
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print('Loaded Checkpoint of Epoch: {}'.format(args.resume))
for epoch in range(start_epoch, int(args.epoch) + start_epoch):
adjust_learning_rate(optimizer, epoch)
train(model, train_dataloader, criterion, criterion1, optimizer, epoch,
writer, size_train)
print('')
val_loss = val(model, val_dataloader, criterion, criterion1, epoch,
writer, size_val)
print('')
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
},
filename=save_path + '/Checkpoint/' + str(val_loss) + '.pth.tar')
writer.export_scalars_to_json(save_path + '/log.json')
import numpy as np
import matplotlib.pyplot as plt
from transform import Normalize, RandomScaleCrop, RandomGaussianBlur, RandomHorizontalFlip, ToTensor
from torchvision import transforms
from torch.utils.data import DataLoader
trainset = JinNanDataset(
images_dir="../data/jinnan/restricted",
maskes_dir="../data/jinnan/mask",
images_list="../data/jinnan/cross_validation/train_1.txt",
transform=transforms.Compose([
# RandomGaussianBlur(),
RandomScaleCrop(550, 512),
RandomHorizontalFlip(),
Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
ToTensor()
]))
print(len(trainset))
trainloader = DataLoader(trainset,
batch_size=2,
shuffle=True,
num_workers=0)
for i, sample in enumerate(trainloader):
for j in range(sample["image"].size()[0]):
image = sample["image"][j].numpy()
mask = sample["mask"][j].numpy()
image = image.transpose([1, 2, 0])
image *= (0.229, 0.224, 0.225)
image += (0.485, 0.456, 0.406)
image = image * 255
def get_merged_common_dataset(opt, skip_study=None, dataset_dict_cache=[], data_cache=[]):
cancer_data_dir = opt.curated_breast_data_dir
if dataset_dict_cache:
dataset_dict = dataset_dict_cache[0]
else:
dataset_dict = util.load_curated(cancer_data_dir)
dataset_dict_cache.append(dataset_dict)
mergedCurated = dataset_dict['merged'].copy()
if data_cache:
data = data_cache[0]
else:
data = metagx_util.load_metagx_dataset(opt.metagx_data_dir, min_genes=opt.min_genes)
data_cache.append(data)
merged = data['merged'].copy()
genes_list = data['genes_features'].copy()
metagx_pos_outcome = merged[merged.posOutcome.isin([-1, 1])]
print('num pos outcome studies {0}'.format(len(metagx_pos_outcome.study.unique())))
if skip_study is not None:
study_to_skip = metagx_pos_outcome.study.unique()[skip_study]
else:
study_to_skip = None
merged_common = util.merge_metagx_curated(merged, mergedCurated)
merged_treatments = list(metagx_util.treatment_columns_metagx) + util.treatment_columns_bmc
merged_treatments = [x for x in merged_treatments if x in merged_common]
merged_treatments = list(set(merged_treatments))
# add continious covariates to genes
cont_columns = [x for x in merged_treatments if len(merged_common[x].unique()) > 20]
merged_treatments = [x for x in merged_treatments if x not in cont_columns]
common_genes_list = [x for x in genes_list if x in merged_common]
if opt.use_covars:
non_genes = cont_columns + merged_treatments + ['posOutcome']
else:
non_genes = []
if study_to_skip is None:
train_data, train_labels, val_data, val_labels = util.random_split(merged_common,
common_genes_list + non_genes,
['study', 'posOutcome'],
balance_validation=False,
balance_by_study=False,
ratio=opt.test_ratio,
to_numpy=False)
else:
train_data, train_labels, val_data, val_labels = next(util.split_by_study(merged_common,
common_genes_list + non_genes,
['study', 'posOutcome'],
study=study_to_skip,
to_numpy=False))
# it's ok to use gene expression in unsupervised model
copy = val_data.copy()
copy.loc[:, non_genes] = 0
val_copy = val_labels.copy()
val_copy.loc[:, 'posOutcome'] = 0
train_data = pandas.concat([train_data, copy], ignore_index=True)
train_labels = pandas.concat([train_labels, val_copy], ignore_index=True)
print('validation study {0}'.format(study_to_skip))
print(val_data.shape)
train_data.fillna(0, inplace=True)
val_data.fillna(0, inplace=True)
to_tensor = ToTensor()
to_float = ToType('float')
add_age = AdditiveUniform(-0.5, 0.5, 'age')
add_tumor_size = AdditiveUniform(-0.5, 0.5, 'tumor_size')
add_posOutcome = AdditiveUniformTriary(0.0, 0.05, 'posOutcome')
add_treat = Compose([AdditiveUniformTriary(0.0, 0.05, x) for x in merged_treatments])
lst = []
if 'posOutcome' in train_data.columns:
lst = [add_age, add_tumor_size, add_posOutcome, add_treat]
compose = Compose(lst + [to_tensor, to_float])
compose_label = Compose([add_posOutcome, to_tensor, to_float])
num_binary = len(merged_treatments + ['posOutcome'])
num_binary = 0
transform = DataLabelCompose(compose, compose_label)
train_set = GeneDataset(train_data, train_labels, transform, binary=num_binary)
test_set = GeneDataset(val_data, val_labels, transform, binary=num_binary)
return train_set, test_set
'kd_dsc': DiceCoef(weight=[0, 1, 0]),
'ca_dsc': DiceCoef(weight=[0, 0, 1])
}
scheduler = ReduceLROnPlateau(optimizer, factor=0.2, patience=30)
dataset = CaseDataset('data/Task00_Kidney/region_norm')
patch_size = (128, 128, 128)
train_transform = Compose([
RandomRescaleCrop(0.1,
patch_size,
crop_mode='random',
enforce_label_indices=[1]),
RandomMirror((0.5, 0.5, 0.5)),
RandomContrast(0.1),
RandomBrightness(0.1),
RandomGamma(0.1),
ToTensor()
])
valid_transform = Compose([RandomCrop(patch_size), ToTensor()])
trainer = Trainer(model=model,
optimizer=optimizer,
loss=loss,
metrics=metrics,
dataset=dataset,
scheduler=scheduler,
train_transform=train_transform,
valid_transform=valid_transform,
batch_size=2,
valid_split=0)
ims = ims[:, :32]
return ims
def get_multiple(self, ):
pass
if __name__ == '__main__':
from transform import (Compose, Normalize, Scale, CenterCrop, CornerCrop,
MultiScaleCornerCrop, MultiScaleRandomCrop,
RandomHorizontalFlip, ToTensor)
D = EPIC_KITCHENS(
'/mnt/nisho_data2/hyf/EPIC-annotations/EPIC_train_action_labels.csv',
transform=Compose([
Scale([224, 224]),
ToTensor(255),
Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]))
loader = DataLoader(
dataset=D,
batch_size=2,
shuffle=False,
num_workers=8,
pin_memory=True,
)
print(len(loader))
from tqdm import tqdm
for i, sample in tqdm(enumerate(loader)):
pass
# print(sample['ims'].size()) #(b, 3, cliplen, 224, 224)
# print(sample['vid']) #['P01_01', 'P01_01']
print(m)
def save_models(models, path):
for i, model in enumerate(models):
target_dir = f'{path}/model_{i}.pt'
torch.save(model.state_dict(), target_dir)
WORKING_DIRECTORY = 'Documents/studia/mgr/master-thesis'
if WORKING_DIRECTORY not in os.getcwd():
os.chdir(WORKING_DIRECTORY)
dataset = FMASpectrogramsDataset(csv_file='track_mapping.txt', lookup_table='genres.txt', root_dir='spectrograms',
transform=transforms.Compose([ToRGB(), Rescale((128, 128)), MelspectrogramNormalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)), ToTensor()]))
torch.manual_seed(100)
train_size = int(0.8 * len(dataset))
validation_size = int(0.1 * len(dataset))
test_size = len(dataset) - validation_size - train_size
train_dataset, test_dataset, validation_dataset = random_split(
dataset, [train_size, validation_size, test_size])
torch.cuda.empty_cache()
# hyper
num_epochs = 30
num_classes = 8
batch_size = 16
learning_rate = 0.001
return data
def show_data(n_images, nrow=5):
dataloader = DataLoader(dataset,
batch_size=n_images,
shuffle=True,
num_workers=4)
plt.figure()
batch = next(iter(dataloader))
images = batch['image']
for idx, image in enumerate(images):
image = image.permute(1, 2, 0)
plt.subplot(n_images // nrow, nrow, idx + 1)
plt.imshow(image)
plt.axis('off')
plt.show()
if __name__ == "__main__":
dataset = BrushStrokeDataset('calligraphy_100k/labels.csv',
'calligraphy_100k/images/',
transform=transforms.Compose([ToTensor()]))
show_data(25)
loss = FocalDiceCoefLoss()
metrics = {'kd_dsc': Dice()}
scheduler = ReduceLROnPlateau(optimizer, factor=0.2, patience=60)
dataset = CaseDataset('data/Task00_Kidney/norm')
patch_size = (144, 144, 96)
train_transform = Compose([
RandomRescaleCrop(0.1,
patch_size,
crop_mode='random',
enforce_label_indices=[1]),
RandomMirror((0.2, 0, 0)),
RandomContrast(0.1),
RandomBrightness(0.1),
RandomGamma(0.1),
CombineLabels([1, 2], 3),
ToTensor()
])
valid_transform = Compose(
[RandomCrop(patch_size),
CombineLabels([1, 2], 3),
ToTensor()])
ckpt = torch.load('logs/Task00_Kidney/kd-2004052152-epoch=314.pt')
model.load_state_dict(ckpt['model_state_dict'])
optimizer.load_state_dict(ckpt['optimizer_state_dict'])
trainer = Trainer(model=model,
optimizer=optimizer,
loss=loss,
metrics=metrics,