def load_model(args):
trainer = Trainer()
trainer.load(from_directory=args.load_directory, best=False)
# trainer.load(from_directory=args.load_directory, best=False)
trainer.set_max_num_epochs(args.epochs + trainer.epoch_count)
model = trainer.model
trainer.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
trainer.save_to_directory(args.save_directory)
return (model, trainer)
def my_train(load, folder):
click.echo('starting training')
os.makedirs(folder, exist_ok=True)
# joint_transform = Compose(
# RandomRotate(),
# RandomTranspose(),
# RandomFlip()
# )
# setup logger
os.makedirs('derived_data/log', exist_ok=True)
Logger.instance().setup('derived_data/log')
vae = Vae()
ds = HemoDataset(root_folder=root_folder, image_transform=None, training=True)
train_loader = torch.utils.data.DataLoader(ds, batch_size=1536, num_workers=8)
# Build trainer
trainer = Trainer(vae)
trainer.save_to_directory(folder)
if load:
trainer.load()
# trainer.cuda(devices=[0, 1])
trainer.cuda()
trainer.build_criterion(vae.loss_function())
trainer.build_optimizer('Adam', lr=0.001)
# trainer.validate_every((2, 'epochs'))
trainer.save_every((1, 'epochs'))
trainer.set_max_num_epochs(100)
trainer.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never',
# log_images_every=(1, 'iteration'),
log_directory=folder))
# Bind loaders
trainer.bind_loader('train', train_loader, num_inputs=1, num_targets=1)
# bind callbacks
trainer.register_callback(GarbageCollection(), trigger='end_of_training_iteration')
trainer.register_callback(ShowMinimalConsoleInfo(), trigger='end_of_training_iteration')
# trainer.bind_loader('train', train_loader, num_inputs=3, num_targets=1)
trainer.fit()
pushover_notification.send('embeddings generated')
def run(args):
model = SpeechModel(args)
trainer = Trainer()
if os.path.exists(
os.path.join(args.save_directory, trainer._checkpoint_filename)):
trainer.load(from_directory=args.save_directory)
model.load_state_dict(trainer.model.state_dict())
if args.cuda:
model = model.cuda()
else:
train_loader = make_loader('train', args, batch_size=args.batch_size)
dev_loader = make_loader('dev', args, batch_size=args.batch_size)
# Build trainer
trainer = Trainer(model) \
.build_criterion(CTCCriterion, size_average=True) \
.build_optimizer('Adam', weight_decay=1e-7, lr=5e-5) \
.validate_every((1, 'epochs')) \
.save_every((1, 'epochs')) \
.save_to_directory(args.save_directory) \
.set_max_num_epochs(args.epochs) \
.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'),
log_images_every='never'),
log_directory=args.save_directory)
# Bind loaders
trainer.bind_loader('train', train_loader, num_inputs=3, num_targets=1)
trainer.bind_loader('validate',
dev_loader,
num_inputs=3,
num_targets=1)
if args.cuda:
trainer.cuda()
# Go!
trainer.fit()
trainer.save()
test_loader = make_loader('test',
args=args,
shuffle=False,
batch_size=1,
test=True)
run_logits(loader=test_loader, model=model, args=args)
def __init__(self, features: bool=False, out_size: int=28):
'''
:param features:
If True -> returns map of features for each window
If False -> returns map of responces for each window
:param out_size:
Size of output image. Influence on stride.
'''
trainer = Trainer(ICL_DenseNet_3fc)
if torch.cuda.is_available():
trainer = trainer.load(from_directory='../centrioles/models/ICL_DenseNet_3fc/true_save/weights/',
best=True)
else:
trainer = trainer.load(from_directory='../centrioles/models/ICL_DenseNet_3fc/true_save/weights/',
best=True, map_location='cpu')
self.model = trainer.model
self.model.features_needed = True
self.features = features
self.out_size = out_size
self.to_torch = inftransforms.generic.AsTorchBatch(dimensionality=2)
import numpy as np
import src.implemented_models as impl_models
import torch.nn.functional as F
import torch
from src.datasets import CentriollesDatasetPatients
from src.utils import log_info, get_basic_transforms
from src.utils import init_weights
import src.implemented_models as impl_models
from inferno.trainers.basic import Trainer
from tqdm import tqdm
model = impl_models.MIL_DenseNet_3fc_bn
path_to_model_weights = '../centrioles/models/MIL_DenseNet_3fc_bn/3fc_bn_rerun/weights'
trainer = Trainer(model)
if torch.cuda.is_available():
trainer = trainer.load(from_directory=path_to_model_weights,
best=True)
else:
trainer = trainer.load(from_directory=path_to_model_weights,
filename='best_checkpoint.pytorch',
best=True, map_location='cpu')
model = trainer.model
train_tr, test_tr = get_basic_transforms()
dataset = CentriollesDatasetPatients(nums=[396, 397, 401, 409, 40311, 40318, 40918, 406180, 406183],
main_dir='../centrioles/dataset/new_edition/new_data_png',
all_data=True, transform=test_tr, inp_size=512)
false_positives = []
false_negatives = []
true_resp = 0
wsize=(args.wsize, args.wsize),
crop=True,
stride=args.stride,
train=False)
log_info('Bags dataset is used')
print('Test output: ', train_ds[0][0].shape, train_ds[0][1])
train_dl = DataLoader(train_ds, batch_size=1, shuffle=True, num_workers=0)
test_dl = DataLoader(test_ds, batch_size=1, shuffle=True, num_workers=0)
log_info('Datasets are initialized!')
ref_trainer = Trainer(ICL_DenseNet_3fc)
if torch.cuda.is_available():
ref_trainer = ref_trainer.load(
from_directory=
'../centrioles/models/ICL_DenseNet_3fc/true_save/weights/',
best=True)
else:
ref_trainer = ref_trainer.load(
from_directory=
'../centrioles/models/ICL_DenseNet_3fc/true_save/weights/',
best=True,
map_location='cpu')
ref_model = ref_trainer.model
model = ICL_MIL_DS3fc
init_weights(model, ref_model)
model.freeze_weights()
# DIRS AND MODEL
model_dir = os.path.join('models', args.model_name)
def predict(folder, n, show_umap, out_dir):
if not os.path.exists(out_dir):
os.makedirs(out_dir)
LOG_DIRECTORY = folder
SAVE_DIRECTORY = folder
DATASET_DIRECTORY = folder
click.echo('Predict')
model = Model(in_channels=38, k=3)
ds_train = SpatialTranscriptomicsDs(root_folder=root_folder, k=model.k, training=False,
divideable_by=model.divideable_by)
train_loader = data.DataLoader(ds_train, batch_size=1,
num_workers=0)
if n == 0:
n = len(ds_train)
print('N', n)
trainer = Trainer(model)
trainer.save_to_directory(SAVE_DIRECTORY)
trainer.load()
trainer.eval_mode()
with torch.no_grad():
all_res = []
all_muh = []
all_targets = []
all_ids = []
num_cells = []
for i in range(n):
torch.cuda.empty_cache()
gc.collect()
print(i)
img, mask, neighbours, targets = ds_train[i]
n_cells = neighbours.shape[0]
img = trainer.to_device(torch.from_numpy(img))
mask = trainer.to_device(torch.from_numpy(mask))
neighbours = trainer.to_device(torch.from_numpy(neighbours))
unetres, rec, muh, logvar, nn = trainer.apply_model(img[None, ...], mask[None, ...], neighbours[None, ...])
# img.detach()
mask = mask.cpu().numpy()
nn.detach()
logvar.detach()
rec = rec.detach().cpu().numpy()
image = img.detach().cpu().numpy() # [0,:,:].swapaxes(0,1)
unetres = unetres.detach().cpu().numpy() # [0,:,:].swapaxes(0,1)
if show_umap:
app = pg.mkQApp()
viewer = LayerViewerWidget()
viewer.setWindowTitle('LayerViewer')
viewer.show()
image = numpy.moveaxis(image, 0, 2)
image = numpy.swapaxes(image, 0, 1)
unetres = numpy.moveaxis(unetres[0, ...], 0, 2)
unetres = numpy.swapaxes(unetres, 0, 1)
mask = numpy.swapaxes(mask[0, ...], 0, 1)
layer = MultiChannelImageLayer(name='img', data=image)
viewer.addLayer(layer=layer)
layer = MultiChannelImageLayer(name='umap', data=unetres)
viewer.addLayer(layer=layer)
# labels = numpy.zeros(image.shape[0:2], dtype='uint8')
label_layer = ObjectLayer(name='labels', data=mask)
viewer.addLayer(layer=label_layer)
QtGui.QApplication.instance().exec_()
muh = muh.detach().cpu().numpy() # [0,:,:].swapaxes(0,1)
res = muh
all_res.append(res)
all_ids.append(numpy.ones(n_cells) * i)
# num_cells.append(n_cells)
img_name = ds_train.image_filenames[i]
img_name = os.path.basename(img_name)
print(i, img_name)
fname = os.path.join(out_dir, f'{img_name}.h5')
print(fname)
with h5py.File(fname, 'w') as f5:
try:
del f5['labels']
except:
pass
f5['labels'] = numpy.swapaxes(mask[0, ...], 0, 1)
try:
del f5['vae_embedding']
except:
pass
f5['vae_embedding'] = res
try:
del f5['targets']
except:
pass
f5['targets'] = targets
try:
del f5['masks']
except:
pass
f5['masks'] = mask
try:
del f5['rec']
except:
pass
f5['rec'] = rec
# res = numpy.concatenate(all_res, axis=0)
all_ids = numpy.concatenate(all_ids, axis=0)
if False:
# import seaborn
# import pandas as pd
# seaborn.pairplot(pd.DataFrame(res[:,0:5]))
# pylab.show()
embedding, embedding_list = list_umap(x_list=all_res, n_neighbors=30, min_dist=0.0)
for i in range(n):
img_name = ds_train.image_filenames[i]
img_name = os.path.basename(img_name)
print(i, img_name)
fname = os.path.join(out_dir, f'{img_name}.h5')
f5file = h5py.File(fname, 'r+')
# f5file['labels'] = numpy.swapaxes(mask[0,...],0,1)
f5file['vae_embedding'] = all_res[i]
f5file['umap_embedding'] = embedding_list[i]
f5file.close()
# A random colormap for matplotlib
cmap = matplotlib.colors.ListedColormap(numpy.random.rand(1000000, 3))
n = len(all_ids)
perm = numpy.random.permutation(numpy.arange(n))
pylab.scatter(x=embedding[perm, 0], y=embedding[perm, 1], c=all_ids[perm], s=12, edgecolors='face',
cmap=cmap)
pylab.show()
import pushover_notification
pushover_notification.send('predictions generated')
def my_embed(folder, n):
click.echo('computing embeddings')
vae = Vae()
ds_train = HemoDataset(root_folder=root_folder, training=True)
ds_test = HemoDataset(root_folder=root_folder, training=False)
batch_size = 512
train_loader = data.DataLoader(ds_train, batch_size=batch_size, num_workers=8)
test_loader = data.DataLoader(ds_test, batch_size=batch_size, num_workers=8)
trainer = Trainer(vae)
trainer.save_to_directory(folder)
trainer.load()
trainer.cuda()
trainer.eval_mode()
i = 0
def compute_embedding_for_loader(loader, training_set: bool):
with torch.no_grad():
all_mu = []
all_filenames = []
with tqdm(total=len(loader)) as bar:
for k, x in enumerate(loader):
nonlocal i
i += 1
if n != 0 and i > n:
break
if training_set:
image, targets = x
else:
image = x
image = trainer.to_device(image)
_, x_reconstructed_batch, mu_batch, _ = trainer.apply_model(image)
image = image.detach().cpu().numpy()
x_reconstructed_batch = x_reconstructed_batch.detach().cpu().numpy()
mu_batch = mu_batch.detach().cpu().numpy()
all_mu.append(mu_batch)
start = k * loader.batch_size
end = start + loader.batch_size
if training_set:
paths = loader.dataset.training_file_paths
else:
paths = loader.dataset.testing_file_paths
filenames = [os.path.basename(paths[j]) for j in range(start, end)]
all_filenames.append(filenames)
bar.update(1)
all_mu = np.concatenate(all_mu, axis=0)
all_filenames = list(itertools.chain.from_iterable(all_filenames))
torch.cuda.empty_cache()
gc.collect()
return all_mu, all_filenames
h5_path = os.path.join(folder, 'embeddings.h5')
with h5py.File(h5_path, 'w') as f5:
embeddings, filenames = compute_embedding_for_loader(train_loader, training_set=True)
# breakpoint()
f5['training_data/embeddings'] = embeddings
pickle_path = os.path.join(folder, 'embeddings_training_data_row_names.pickle')
pickle.dump(filenames, open(pickle_path, 'wb'))
pushover_notification.send('training embeddings generated')
embeddings, filenames = compute_embedding_for_loader(test_loader, training_set=False)
f5['testing_data/embeddings'] = embeddings
pickle_path = os.path.join(folder, 'embeddings_testing_data_row_names.pickle')
pickle.dump(filenames, open(pickle_path, 'wb'))
pushover_notification.send('testing embeddings generated')
def main():
laptop = True
class LossWrapper(nn.Module):
def __init__(self):
super(LossWrapper, self).__init__()
#self.loss = torch.nn.CrossEntropyLoss(weight=torch.tensor([0.5, 2.0]), reduce=False)
self.loss = torch.nn.BCELoss(reduce=False)
def forward(self, input, target):
gt, sizes = target
input = torch.squeeze(input)
gt = torch.squeeze(gt)
w = (gt + 1)**2.75
sizes = torch.squeeze(sizes.float())
#print('it', input.size(), target.size())
l = self.loss(input, gt)
l = l * w
l = torch.sum(l * sizes) / torch.sum(sizes)
print("l", l.item())
return l
# Fill these in:
if laptop:
LOG_DIRECTORY = '/home/tbeier/src/extension-cpp/log/'
SAVE_DIRECTORY = '/home/tbeier/src/extension-cpp/savedir_new/'
RES_DIRECTORY = '/home/tbeier/src/extension-cpp/danu/'
pmap_root = "/home/tbeier/src/holy-edge/hed-data/out/"
bsd_root = "/home/tbeier/datasets/BSR/BSDS500/"
else:
LOG_DIRECTORY = '/export/home/tbeier/src/extension-cpp/log/'
SAVE_DIRECTORY = '/export/home/tbeier/src/extension-cpp/savedir_new/'
RES_DIRECTORY = '/export/home/tbeier/src/extension-cpp/danu/'
pmap_root = "/export/home/tbeier/bsd500_HED/"
bsd_root = "/home/tbeier/dataset/BSR/BSDS500/"
USE_CUDA = bool(1)
# Device configuration
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print(device)
# Define transforms (1)
#transforms = transforms.Compose([transforms.CenterCrop(100), transforms.ToTensor()])
#split = "train"
bsd_test = Bsd500Sp(bsd_root=bsd_root, pmap_root=pmap_root, split='test')
model = ConvNet() #.double()
#model.double()
# Build trainer
trainer = Trainer(model) \
.build_criterion(LossWrapper) \
.build_criterion(LossWrapper) \
.build_optimizer('Adam') \
.validate_every((4, 'epochs')) \
.save_every((4, 'epochs')) \
.save_to_directory(SAVE_DIRECTORY) \
.set_max_num_epochs(100) \
# .build_logger(TensorboardLogger(log_scalars_every='never',
# log_images_every='never'),
# log_directory=LOG_DIRECTORY)
# Bind loaders
#train_loader = torch.utils.data.DataLoader(dataset=bsd_train, num_workers=8)
#val_loader = torch.utils.data.DataLoader(dataset=bsd_val, num_workers=8)
#trainer \
# .bind_loader('train', train_loader, num_inputs=7, num_targets=2) \
# .bind_loader('validate', val_loader, num_inputs=7, num_targets=2)
#checkpoint = torch.load('savedir/checkpoint.pytorch')
# args.start_epoch = checkpoint['epoch']
# best_prec1 = checkpoint['best_prec1']
#model.load_state_dict(checkpoint['state_dict'])
trainer.cuda()
trainer.load(best=True,
from_directory=SAVE_DIRECTORY,
filename='best_checkpoint.pytorch')
meval = trainer.model.eval() #.cpu()
#model.load_state_dict(torch.load('savedir/checkpoint.pytorch'))
acc_vi_ds = 0.0
acc_ri_ds = 0.0
count = 0
for i in range(0, 200):
predictor = Predictor(model=meval,
ds=bsd_test,
output_folder=RES_DIRECTORY)
vi_img, ri_img = predictor.predict_mc(i)
acc_vi_ds += vi_img
acc_ri_ds += ri_img
count += 1
print("\n", i)
print("IMG ", vi_img, ri_img)
print("DS ", acc_vi_ds / count, acc_ri_ds / count)
transform=test_tr,
inp_size=512,
train=False,
check=args.check)
train_dl = DataLoader(train_ds, batch_size=4, shuffle=True, num_workers=0)
test_dl = DataLoader(test_ds, batch_size=4, shuffle=True, num_workers=0)
log_info('Datasets are initialized!')
# MODEL MIL_DenseNet_3fc
exec("model = impl_models.%s" % (args.model_name))
if args.continue_training:
trainer = Trainer(model)
if torch.cuda.is_available():
trainer = trainer.load(from_directory=args.init_model_path,
best=True)
else:
trainer = trainer.load(from_directory=args.init_model_path,
best=True,
map_location='cpu')
init_model = trainer.model
init_model.features_needed = True
init_weights(model, init_model, freeze_gradients=args.freeze)
else:
icl_model = impl_models.ICL_DenseNet_3fc
# path_to_model_weights = '../centrioles/icl_weights/ICL_DenseNet_3fc/fold_{}/weights/'.format(args.fold)
path_to_model_weights = '../centrioles/run_history/ICL_DenseNet_3fc/true_save/weights/'
trainer = Trainer(icl_model)
if torch.cuda.is_available():
trainer.set_max_num_epochs(200)
trainer.register_callback(
SaveAtBestValidationScore(smoothness=smoothness, verbose=True))
trainer.register_callback(
AutoLR(factor=0.5,
patience='1 epochs',
monitor_while='validating',
monitor='validation_loss',
monitor_momentum=smoothness,
consider_improvement_with_respect_to='previous',
verbose=True))
trainer.register_callback(TQDMProgressBar())
# Bind loaders
train_loader = torch.utils.data.DataLoader(dataset=bsd_train,
num_workers=8)
val_loader = torch.utils.data.DataLoader(dataset=bsd_val, num_workers=8)
num_inputs = bsd_train.num_inputs()
num_targets = bsd_train.num_targets()
trainer.load()
trainer\
.bind_loader('train', train_loader, num_inputs=num_inputs, num_targets=num_targets) \
.bind_loader('validate', val_loader, num_inputs=num_inputs, num_targets=num_targets) \
trainer.cuda()
#E
trainer.fit()
if args.flr:
trainer = trainer.register_callback(
AutoLR(args.decey, (1, 'epochs'),
monitor_momentum=0.9,
monitor_while='validating',
consider_improvement_with_respect_to='best'))
else:
trainer = trainer.register_callback(
AutoLR(0.9, (1, 'epochs'),
consider_improvement_with_respect_to='previous'))
if args.init_model_path != '':
init_trainer = Trainer(model)
if torch.cuda.is_available():
init_trainer = init_trainer.load(
from_directory=args.init_model_path, best=True)
else:
init_trainer = init_trainer.load(
from_directory=args.init_model_path,
best=True,
map_location='cpu')
init_model = init_trainer.model
init_weights(model, init_model, freeze_gradients=False)
# Bind loaders
trainer \
.bind_loader('train', train_dl) \
.bind_loader('validate', test_dl)
if torch.cuda.is_available():
trainer.cuda()
if '--inherit' in sys.argv:
i = sys.argv.index('--inherit') + 1
if sys.argv[i].endswith(('.yml', '.yaml')):
sys.argv[i] = os.path.join(config_path, sys.argv[i])
else:
sys.argv[i] = os.path.join(experiments_path, sys.argv[i])
if '--update' in sys.argv:
i = sys.argv.index('--update') + 1
sys.argv[i] = os.path.join(config_path, sys.argv[i])
i = 0
while True:
if f'--update{i}' in sys.argv:
ind = sys.argv.index(f'--update{i}') + 1
sys.argv[ind] = os.path.join(config_path, sys.argv[ind])
i += 1
else:
break
cls = BaseCremiExperiment
trainer = Trainer()
trainer.load(from_directory='../../runs/cremi/speedrun/run_0', best=False)
loader = BaseCremiExperiment().build_train_loader()
trainer.cuda()
target, loss = trainer.apply_model_and_loss(
loader.dataset[0][0].unsqueeze(0).unsqueeze(0),
loader.dataset[0][1].unsqueeze(0).unsqueeze(0))
print('hi')
def __init__(
self,
model,
path_to_model_weights='../centrioles/models/ICL_DenseNet_3fc/true_save/weights/',
nums=[397, 402, 403, 406, 396, 3971, 4021],
check=False,
main_dir='../centrioles/dataset/new_edition/filtered',
train=True,
all_data=False,
inp_size=512,
repeat_rate=10):
self.samples = []
self.patient = []
self.name = []
self.inp_size = inp_size
if train:
self.transform, _ = get_resps_transforms()
else:
_, self.transform = get_resps_transforms()
self.classes = []
self.images = []
trainer = Trainer(model)
if torch.cuda.is_available():
trainer = trainer.load(from_directory=path_to_model_weights,
best=True)
else:
trainer = trainer.load(from_directory=path_to_model_weights,
best=True,
map_location='cpu')
self.model = trainer.model
def get_img(img_name):
im = Image.open(img_name).convert('L')
im.load()
im.thumbnail((inp_size, inp_size), Image.ANTIALIAS)
cp_img = im.copy()
im.close()
mask = Image.fromarray(get_the_central_cell_mask(cp_img, wsize=0))
rot_mask = Image.new('L', (inp_size, inp_size), (1))
return Image.merge("RGB", [cp_img, mask, rot_mask])
def get_img_names(dir_name):
img_names = [f for f in os.listdir(dir_name) if f.endswith('.png')]
if all_data:
return img_names
delimetr = int(0.9 * len(img_names))
if train:
img_names = img_names[:delimetr]
else:
img_names = img_names[delimetr:]
return img_names
for num in nums:
pos_dir = os.path.join(main_dir, str(num) + '_centrioles')
neg_dir = os.path.join(main_dir, str(num) + '_nocentrioles')
# Positive samples
for img_name in get_img_names(pos_dir):
for i in range(repeat_rate):
self.name.append(os.path.join(pos_dir, img_name))
print(i, self.name[-1])
img = get_img(self.name[-1])
img = self.transform(img)
self.samples.append(img)
self.classes.append(1)
self.patient.append(num)
# Negative sampless
for img_name in get_img_names(neg_dir):
for i in range(repeat_rate):
self.name.append(os.path.join(neg_dir, img_name))
img = get_img(self.name[-1])
img = self.transform(img)
self.samples.append(img)
self.classes.append(0)
self.patient.append(num)
trans2 = transforms.Compose([fromnumpy, squeeze])
imageset_val = HDF5VolumeLoader(
path='./val-volume.h5',
path_in_h5_dataset='data',
transforms=trans,
**yaml2dict('config_val.yml')['slicing_config'])
labelset_val = HDF5VolumeLoader(
path='./stardistance_val.h5',
path_in_h5_dataset='data',
transforms=trans2,
**yaml2dict('config_val.yml')['slicing_config_truth'])
trainer = Trainer()
trainer.load(from_directory='checkpoints', map_location='cpu', best=False)
result, loss = trainer.apply_model_and_loss(
imageset_val[5].unsqueeze(0).unsqueeze(0), #.to('cuda'),
labelset_val[5].unsqueeze(0).unsqueeze(0)) #.to('cuda'))
print(loss)
print(result)
plt.figure()
plt.imshow(result.squeeze().detach().numpy()[0])
#plt.imshow(result[0].detach().squeeze().cpu().numpy())
plt.title('result')
plt.figure()
plt.imshow(imageset_val[5].detach().squeeze().cpu().numpy())
