def load_model(args): trainer = Trainer() trainer.load(from_directory=args.load_directory, best=False) # trainer.load(from_directory=args.load_directory, best=False) trainer.set_max_num_epochs(args.epochs + trainer.epoch_count) model = trainer.model trainer.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'), log_directory=args.save_directory) trainer.save_to_directory(args.save_directory) return (model, trainer)
def my_train(load, folder): click.echo('starting training') os.makedirs(folder, exist_ok=True) # joint_transform = Compose( # RandomRotate(), # RandomTranspose(), # RandomFlip() # ) # setup logger os.makedirs('derived_data/log', exist_ok=True) Logger.instance().setup('derived_data/log') vae = Vae() ds = HemoDataset(root_folder=root_folder, image_transform=None, training=True) train_loader = torch.utils.data.DataLoader(ds, batch_size=1536, num_workers=8) # Build trainer trainer = Trainer(vae) trainer.save_to_directory(folder) if load: trainer.load() # trainer.cuda(devices=[0, 1]) trainer.cuda() trainer.build_criterion(vae.loss_function()) trainer.build_optimizer('Adam', lr=0.001) # trainer.validate_every((2, 'epochs')) trainer.save_every((1, 'epochs')) trainer.set_max_num_epochs(100) trainer.build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never', # log_images_every=(1, 'iteration'), log_directory=folder)) # Bind loaders trainer.bind_loader('train', train_loader, num_inputs=1, num_targets=1) # bind callbacks trainer.register_callback(GarbageCollection(), trigger='end_of_training_iteration') trainer.register_callback(ShowMinimalConsoleInfo(), trigger='end_of_training_iteration') # trainer.bind_loader('train', train_loader, num_inputs=3, num_targets=1) trainer.fit() pushover_notification.send('embeddings generated')
def run(args): model = SpeechModel(args) trainer = Trainer() if os.path.exists( os.path.join(args.save_directory, trainer._checkpoint_filename)): trainer.load(from_directory=args.save_directory) model.load_state_dict(trainer.model.state_dict()) if args.cuda: model = model.cuda() else: train_loader = make_loader('train', args, batch_size=args.batch_size) dev_loader = make_loader('dev', args, batch_size=args.batch_size) # Build trainer trainer = Trainer(model) \ .build_criterion(CTCCriterion, size_average=True) \ .build_optimizer('Adam', weight_decay=1e-7, lr=5e-5) \ .validate_every((1, 'epochs')) \ .save_every((1, 'epochs')) \ .save_to_directory(args.save_directory) \ .set_max_num_epochs(args.epochs) \ .build_logger(TensorboardLogger(log_scalars_every=(1, 'iteration'), log_images_every='never'), log_directory=args.save_directory) # Bind loaders trainer.bind_loader('train', train_loader, num_inputs=3, num_targets=1) trainer.bind_loader('validate', dev_loader, num_inputs=3, num_targets=1) if args.cuda: trainer.cuda() # Go! trainer.fit() trainer.save() test_loader = make_loader('test', args=args, shuffle=False, batch_size=1, test=True) run_logits(loader=test_loader, model=model, args=args)
def __init__(self, features: bool=False, out_size: int=28): ''' :param features: If True -> returns map of features for each window If False -> returns map of responces for each window :param out_size: Size of output image. Influence on stride. ''' trainer = Trainer(ICL_DenseNet_3fc) if torch.cuda.is_available(): trainer = trainer.load(from_directory='../centrioles/models/ICL_DenseNet_3fc/true_save/weights/', best=True) else: trainer = trainer.load(from_directory='../centrioles/models/ICL_DenseNet_3fc/true_save/weights/', best=True, map_location='cpu') self.model = trainer.model self.model.features_needed = True self.features = features self.out_size = out_size self.to_torch = inftransforms.generic.AsTorchBatch(dimensionality=2)
import numpy as np import src.implemented_models as impl_models import torch.nn.functional as F import torch from src.datasets import CentriollesDatasetPatients from src.utils import log_info, get_basic_transforms from src.utils import init_weights import src.implemented_models as impl_models from inferno.trainers.basic import Trainer from tqdm import tqdm model = impl_models.MIL_DenseNet_3fc_bn path_to_model_weights = '../centrioles/models/MIL_DenseNet_3fc_bn/3fc_bn_rerun/weights' trainer = Trainer(model) if torch.cuda.is_available(): trainer = trainer.load(from_directory=path_to_model_weights, best=True) else: trainer = trainer.load(from_directory=path_to_model_weights, filename='best_checkpoint.pytorch', best=True, map_location='cpu') model = trainer.model train_tr, test_tr = get_basic_transforms() dataset = CentriollesDatasetPatients(nums=[396, 397, 401, 409, 40311, 40318, 40918, 406180, 406183], main_dir='../centrioles/dataset/new_edition/new_data_png', all_data=True, transform=test_tr, inp_size=512) false_positives = [] false_negatives = [] true_resp = 0
wsize=(args.wsize, args.wsize), crop=True, stride=args.stride, train=False) log_info('Bags dataset is used') print('Test output: ', train_ds[0][0].shape, train_ds[0][1]) train_dl = DataLoader(train_ds, batch_size=1, shuffle=True, num_workers=0) test_dl = DataLoader(test_ds, batch_size=1, shuffle=True, num_workers=0) log_info('Datasets are initialized!') ref_trainer = Trainer(ICL_DenseNet_3fc) if torch.cuda.is_available(): ref_trainer = ref_trainer.load( from_directory= '../centrioles/models/ICL_DenseNet_3fc/true_save/weights/', best=True) else: ref_trainer = ref_trainer.load( from_directory= '../centrioles/models/ICL_DenseNet_3fc/true_save/weights/', best=True, map_location='cpu') ref_model = ref_trainer.model model = ICL_MIL_DS3fc init_weights(model, ref_model) model.freeze_weights() # DIRS AND MODEL model_dir = os.path.join('models', args.model_name)
def predict(folder, n, show_umap, out_dir): if not os.path.exists(out_dir): os.makedirs(out_dir) LOG_DIRECTORY = folder SAVE_DIRECTORY = folder DATASET_DIRECTORY = folder click.echo('Predict') model = Model(in_channels=38, k=3) ds_train = SpatialTranscriptomicsDs(root_folder=root_folder, k=model.k, training=False, divideable_by=model.divideable_by) train_loader = data.DataLoader(ds_train, batch_size=1, num_workers=0) if n == 0: n = len(ds_train) print('N', n) trainer = Trainer(model) trainer.save_to_directory(SAVE_DIRECTORY) trainer.load() trainer.eval_mode() with torch.no_grad(): all_res = [] all_muh = [] all_targets = [] all_ids = [] num_cells = [] for i in range(n): torch.cuda.empty_cache() gc.collect() print(i) img, mask, neighbours, targets = ds_train[i] n_cells = neighbours.shape[0] img = trainer.to_device(torch.from_numpy(img)) mask = trainer.to_device(torch.from_numpy(mask)) neighbours = trainer.to_device(torch.from_numpy(neighbours)) unetres, rec, muh, logvar, nn = trainer.apply_model(img[None, ...], mask[None, ...], neighbours[None, ...]) # img.detach() mask = mask.cpu().numpy() nn.detach() logvar.detach() rec = rec.detach().cpu().numpy() image = img.detach().cpu().numpy() # [0,:,:].swapaxes(0,1) unetres = unetres.detach().cpu().numpy() # [0,:,:].swapaxes(0,1) if show_umap: app = pg.mkQApp() viewer = LayerViewerWidget() viewer.setWindowTitle('LayerViewer') viewer.show() image = numpy.moveaxis(image, 0, 2) image = numpy.swapaxes(image, 0, 1) unetres = numpy.moveaxis(unetres[0, ...], 0, 2) unetres = numpy.swapaxes(unetres, 0, 1) mask = numpy.swapaxes(mask[0, ...], 0, 1) layer = MultiChannelImageLayer(name='img', data=image) viewer.addLayer(layer=layer) layer = MultiChannelImageLayer(name='umap', data=unetres) viewer.addLayer(layer=layer) # labels = numpy.zeros(image.shape[0:2], dtype='uint8') label_layer = ObjectLayer(name='labels', data=mask) viewer.addLayer(layer=label_layer) QtGui.QApplication.instance().exec_() muh = muh.detach().cpu().numpy() # [0,:,:].swapaxes(0,1) res = muh all_res.append(res) all_ids.append(numpy.ones(n_cells) * i) # num_cells.append(n_cells) img_name = ds_train.image_filenames[i] img_name = os.path.basename(img_name) print(i, img_name) fname = os.path.join(out_dir, f'{img_name}.h5') print(fname) with h5py.File(fname, 'w') as f5: try: del f5['labels'] except: pass f5['labels'] = numpy.swapaxes(mask[0, ...], 0, 1) try: del f5['vae_embedding'] except: pass f5['vae_embedding'] = res try: del f5['targets'] except: pass f5['targets'] = targets try: del f5['masks'] except: pass f5['masks'] = mask try: del f5['rec'] except: pass f5['rec'] = rec # res = numpy.concatenate(all_res, axis=0) all_ids = numpy.concatenate(all_ids, axis=0) if False: # import seaborn # import pandas as pd # seaborn.pairplot(pd.DataFrame(res[:,0:5])) # pylab.show() embedding, embedding_list = list_umap(x_list=all_res, n_neighbors=30, min_dist=0.0) for i in range(n): img_name = ds_train.image_filenames[i] img_name = os.path.basename(img_name) print(i, img_name) fname = os.path.join(out_dir, f'{img_name}.h5') f5file = h5py.File(fname, 'r+') # f5file['labels'] = numpy.swapaxes(mask[0,...],0,1) f5file['vae_embedding'] = all_res[i] f5file['umap_embedding'] = embedding_list[i] f5file.close() # A random colormap for matplotlib cmap = matplotlib.colors.ListedColormap(numpy.random.rand(1000000, 3)) n = len(all_ids) perm = numpy.random.permutation(numpy.arange(n)) pylab.scatter(x=embedding[perm, 0], y=embedding[perm, 1], c=all_ids[perm], s=12, edgecolors='face', cmap=cmap) pylab.show() import pushover_notification pushover_notification.send('predictions generated')
def my_embed(folder, n): click.echo('computing embeddings') vae = Vae() ds_train = HemoDataset(root_folder=root_folder, training=True) ds_test = HemoDataset(root_folder=root_folder, training=False) batch_size = 512 train_loader = data.DataLoader(ds_train, batch_size=batch_size, num_workers=8) test_loader = data.DataLoader(ds_test, batch_size=batch_size, num_workers=8) trainer = Trainer(vae) trainer.save_to_directory(folder) trainer.load() trainer.cuda() trainer.eval_mode() i = 0 def compute_embedding_for_loader(loader, training_set: bool): with torch.no_grad(): all_mu = [] all_filenames = [] with tqdm(total=len(loader)) as bar: for k, x in enumerate(loader): nonlocal i i += 1 if n != 0 and i > n: break if training_set: image, targets = x else: image = x image = trainer.to_device(image) _, x_reconstructed_batch, mu_batch, _ = trainer.apply_model(image) image = image.detach().cpu().numpy() x_reconstructed_batch = x_reconstructed_batch.detach().cpu().numpy() mu_batch = mu_batch.detach().cpu().numpy() all_mu.append(mu_batch) start = k * loader.batch_size end = start + loader.batch_size if training_set: paths = loader.dataset.training_file_paths else: paths = loader.dataset.testing_file_paths filenames = [os.path.basename(paths[j]) for j in range(start, end)] all_filenames.append(filenames) bar.update(1) all_mu = np.concatenate(all_mu, axis=0) all_filenames = list(itertools.chain.from_iterable(all_filenames)) torch.cuda.empty_cache() gc.collect() return all_mu, all_filenames h5_path = os.path.join(folder, 'embeddings.h5') with h5py.File(h5_path, 'w') as f5: embeddings, filenames = compute_embedding_for_loader(train_loader, training_set=True) # breakpoint() f5['training_data/embeddings'] = embeddings pickle_path = os.path.join(folder, 'embeddings_training_data_row_names.pickle') pickle.dump(filenames, open(pickle_path, 'wb')) pushover_notification.send('training embeddings generated') embeddings, filenames = compute_embedding_for_loader(test_loader, training_set=False) f5['testing_data/embeddings'] = embeddings pickle_path = os.path.join(folder, 'embeddings_testing_data_row_names.pickle') pickle.dump(filenames, open(pickle_path, 'wb')) pushover_notification.send('testing embeddings generated')
def main(): laptop = True class LossWrapper(nn.Module): def __init__(self): super(LossWrapper, self).__init__() #self.loss = torch.nn.CrossEntropyLoss(weight=torch.tensor([0.5, 2.0]), reduce=False) self.loss = torch.nn.BCELoss(reduce=False) def forward(self, input, target): gt, sizes = target input = torch.squeeze(input) gt = torch.squeeze(gt) w = (gt + 1)**2.75 sizes = torch.squeeze(sizes.float()) #print('it', input.size(), target.size()) l = self.loss(input, gt) l = l * w l = torch.sum(l * sizes) / torch.sum(sizes) print("l", l.item()) return l # Fill these in: if laptop: LOG_DIRECTORY = '/home/tbeier/src/extension-cpp/log/' SAVE_DIRECTORY = '/home/tbeier/src/extension-cpp/savedir_new/' RES_DIRECTORY = '/home/tbeier/src/extension-cpp/danu/' pmap_root = "/home/tbeier/src/holy-edge/hed-data/out/" bsd_root = "/home/tbeier/datasets/BSR/BSDS500/" else: LOG_DIRECTORY = '/export/home/tbeier/src/extension-cpp/log/' SAVE_DIRECTORY = '/export/home/tbeier/src/extension-cpp/savedir_new/' RES_DIRECTORY = '/export/home/tbeier/src/extension-cpp/danu/' pmap_root = "/export/home/tbeier/bsd500_HED/" bsd_root = "/home/tbeier/dataset/BSR/BSDS500/" USE_CUDA = bool(1) # Device configuration device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') print(device) # Define transforms (1) #transforms = transforms.Compose([transforms.CenterCrop(100), transforms.ToTensor()]) #split = "train" bsd_test = Bsd500Sp(bsd_root=bsd_root, pmap_root=pmap_root, split='test') model = ConvNet() #.double() #model.double() # Build trainer trainer = Trainer(model) \ .build_criterion(LossWrapper) \ .build_criterion(LossWrapper) \ .build_optimizer('Adam') \ .validate_every((4, 'epochs')) \ .save_every((4, 'epochs')) \ .save_to_directory(SAVE_DIRECTORY) \ .set_max_num_epochs(100) \ # .build_logger(TensorboardLogger(log_scalars_every='never', # log_images_every='never'), # log_directory=LOG_DIRECTORY) # Bind loaders #train_loader = torch.utils.data.DataLoader(dataset=bsd_train, num_workers=8) #val_loader = torch.utils.data.DataLoader(dataset=bsd_val, num_workers=8) #trainer \ # .bind_loader('train', train_loader, num_inputs=7, num_targets=2) \ # .bind_loader('validate', val_loader, num_inputs=7, num_targets=2) #checkpoint = torch.load('savedir/checkpoint.pytorch') # args.start_epoch = checkpoint['epoch'] # best_prec1 = checkpoint['best_prec1'] #model.load_state_dict(checkpoint['state_dict']) trainer.cuda() trainer.load(best=True, from_directory=SAVE_DIRECTORY, filename='best_checkpoint.pytorch') meval = trainer.model.eval() #.cpu() #model.load_state_dict(torch.load('savedir/checkpoint.pytorch')) acc_vi_ds = 0.0 acc_ri_ds = 0.0 count = 0 for i in range(0, 200): predictor = Predictor(model=meval, ds=bsd_test, output_folder=RES_DIRECTORY) vi_img, ri_img = predictor.predict_mc(i) acc_vi_ds += vi_img acc_ri_ds += ri_img count += 1 print("\n", i) print("IMG ", vi_img, ri_img) print("DS ", acc_vi_ds / count, acc_ri_ds / count)
transform=test_tr, inp_size=512, train=False, check=args.check) train_dl = DataLoader(train_ds, batch_size=4, shuffle=True, num_workers=0) test_dl = DataLoader(test_ds, batch_size=4, shuffle=True, num_workers=0) log_info('Datasets are initialized!') # MODEL MIL_DenseNet_3fc exec("model = impl_models.%s" % (args.model_name)) if args.continue_training: trainer = Trainer(model) if torch.cuda.is_available(): trainer = trainer.load(from_directory=args.init_model_path, best=True) else: trainer = trainer.load(from_directory=args.init_model_path, best=True, map_location='cpu') init_model = trainer.model init_model.features_needed = True init_weights(model, init_model, freeze_gradients=args.freeze) else: icl_model = impl_models.ICL_DenseNet_3fc # path_to_model_weights = '../centrioles/icl_weights/ICL_DenseNet_3fc/fold_{}/weights/'.format(args.fold) path_to_model_weights = '../centrioles/run_history/ICL_DenseNet_3fc/true_save/weights/' trainer = Trainer(icl_model) if torch.cuda.is_available():
trainer.set_max_num_epochs(200) trainer.register_callback( SaveAtBestValidationScore(smoothness=smoothness, verbose=True)) trainer.register_callback( AutoLR(factor=0.5, patience='1 epochs', monitor_while='validating', monitor='validation_loss', monitor_momentum=smoothness, consider_improvement_with_respect_to='previous', verbose=True)) trainer.register_callback(TQDMProgressBar()) # Bind loaders train_loader = torch.utils.data.DataLoader(dataset=bsd_train, num_workers=8) val_loader = torch.utils.data.DataLoader(dataset=bsd_val, num_workers=8) num_inputs = bsd_train.num_inputs() num_targets = bsd_train.num_targets() trainer.load() trainer\ .bind_loader('train', train_loader, num_inputs=num_inputs, num_targets=num_targets) \ .bind_loader('validate', val_loader, num_inputs=num_inputs, num_targets=num_targets) \ trainer.cuda() #E trainer.fit()
if args.flr: trainer = trainer.register_callback( AutoLR(args.decey, (1, 'epochs'), monitor_momentum=0.9, monitor_while='validating', consider_improvement_with_respect_to='best')) else: trainer = trainer.register_callback( AutoLR(0.9, (1, 'epochs'), consider_improvement_with_respect_to='previous')) if args.init_model_path != '': init_trainer = Trainer(model) if torch.cuda.is_available(): init_trainer = init_trainer.load( from_directory=args.init_model_path, best=True) else: init_trainer = init_trainer.load( from_directory=args.init_model_path, best=True, map_location='cpu') init_model = init_trainer.model init_weights(model, init_model, freeze_gradients=False) # Bind loaders trainer \ .bind_loader('train', train_dl) \ .bind_loader('validate', test_dl) if torch.cuda.is_available(): trainer.cuda()
if '--inherit' in sys.argv: i = sys.argv.index('--inherit') + 1 if sys.argv[i].endswith(('.yml', '.yaml')): sys.argv[i] = os.path.join(config_path, sys.argv[i]) else: sys.argv[i] = os.path.join(experiments_path, sys.argv[i]) if '--update' in sys.argv: i = sys.argv.index('--update') + 1 sys.argv[i] = os.path.join(config_path, sys.argv[i]) i = 0 while True: if f'--update{i}' in sys.argv: ind = sys.argv.index(f'--update{i}') + 1 sys.argv[ind] = os.path.join(config_path, sys.argv[ind]) i += 1 else: break cls = BaseCremiExperiment trainer = Trainer() trainer.load(from_directory='../../runs/cremi/speedrun/run_0', best=False) loader = BaseCremiExperiment().build_train_loader() trainer.cuda() target, loss = trainer.apply_model_and_loss( loader.dataset[0][0].unsqueeze(0).unsqueeze(0), loader.dataset[0][1].unsqueeze(0).unsqueeze(0)) print('hi')
def __init__( self, model, path_to_model_weights='../centrioles/models/ICL_DenseNet_3fc/true_save/weights/', nums=[397, 402, 403, 406, 396, 3971, 4021], check=False, main_dir='../centrioles/dataset/new_edition/filtered', train=True, all_data=False, inp_size=512, repeat_rate=10): self.samples = [] self.patient = [] self.name = [] self.inp_size = inp_size if train: self.transform, _ = get_resps_transforms() else: _, self.transform = get_resps_transforms() self.classes = [] self.images = [] trainer = Trainer(model) if torch.cuda.is_available(): trainer = trainer.load(from_directory=path_to_model_weights, best=True) else: trainer = trainer.load(from_directory=path_to_model_weights, best=True, map_location='cpu') self.model = trainer.model def get_img(img_name): im = Image.open(img_name).convert('L') im.load() im.thumbnail((inp_size, inp_size), Image.ANTIALIAS) cp_img = im.copy() im.close() mask = Image.fromarray(get_the_central_cell_mask(cp_img, wsize=0)) rot_mask = Image.new('L', (inp_size, inp_size), (1)) return Image.merge("RGB", [cp_img, mask, rot_mask]) def get_img_names(dir_name): img_names = [f for f in os.listdir(dir_name) if f.endswith('.png')] if all_data: return img_names delimetr = int(0.9 * len(img_names)) if train: img_names = img_names[:delimetr] else: img_names = img_names[delimetr:] return img_names for num in nums: pos_dir = os.path.join(main_dir, str(num) + '_centrioles') neg_dir = os.path.join(main_dir, str(num) + '_nocentrioles') # Positive samples for img_name in get_img_names(pos_dir): for i in range(repeat_rate): self.name.append(os.path.join(pos_dir, img_name)) print(i, self.name[-1]) img = get_img(self.name[-1]) img = self.transform(img) self.samples.append(img) self.classes.append(1) self.patient.append(num) # Negative sampless for img_name in get_img_names(neg_dir): for i in range(repeat_rate): self.name.append(os.path.join(neg_dir, img_name)) img = get_img(self.name[-1]) img = self.transform(img) self.samples.append(img) self.classes.append(0) self.patient.append(num)
trans2 = transforms.Compose([fromnumpy, squeeze]) imageset_val = HDF5VolumeLoader( path='./val-volume.h5', path_in_h5_dataset='data', transforms=trans, **yaml2dict('config_val.yml')['slicing_config']) labelset_val = HDF5VolumeLoader( path='./stardistance_val.h5', path_in_h5_dataset='data', transforms=trans2, **yaml2dict('config_val.yml')['slicing_config_truth']) trainer = Trainer() trainer.load(from_directory='checkpoints', map_location='cpu', best=False) result, loss = trainer.apply_model_and_loss( imageset_val[5].unsqueeze(0).unsqueeze(0), #.to('cuda'), labelset_val[5].unsqueeze(0).unsqueeze(0)) #.to('cuda')) print(loss) print(result) plt.figure() plt.imshow(result.squeeze().detach().numpy()[0]) #plt.imshow(result[0].detach().squeeze().cpu().numpy()) plt.title('result') plt.figure() plt.imshow(imageset_val[5].detach().squeeze().cpu().numpy())