def get_dataloader(image_resize, mean, std, fast_train, batch_size, img_dir=None): val_transform = transforms.Compose([ transforms.Resize((image_resize, image_resize)), transforms.ToTensor(), transforms.Normalize(mean, std) ]) # noinspection PyTypeChecker fill_color = tuple(np.array(np.random.rand(3) * 255, dtype='int').tolist()) train_transform = transforms.Compose([ transforms.Resize((image_resize, image_resize)), transforms.RandomHorizontalFlip(), transforms.ColorJitter(), transforms.RandomAffine(degrees=20, translate=(0.2, 0.2), scale=(0.5, 1.5), shear=None, resample=False, fillcolor=fill_color), transforms.ToTensor(), transforms.Normalize(mean, std) ]) def dset(creator, ts): return creator(transforms=ts, cached=True, img_dir=img_dir, return_ids=True) if fast_train: train_dataset = test_label_dataset else: train_dataset = train_label_dataset train_dataset = dset(train_dataset, ts=train_transform) return { PHASE_TRAIN: dataloader(train_dataset, batch_size), PHASE_VAL: dataloader(dset(validation_label_dataset, ts=val_transform), batch_size) }
def get_dataloader(self): loader = dataloader(data_file=__TEST_DATA__, val_mode=True, data_source=self.source, data_val_origin=self.origin, vocab_file=self.model_path, num_labels=self.num_labels, train_percent=100, batch_size=1, drop_last=False, num_workers=2 if torch.cuda.is_available() else 0) return loader
def explain(*wandb_groups, origin='unseen_answers'): if not os.path.isdir(__EXPLANATIONS_DIR__): os.mkdir(__EXPLANATIONS_DIR__) for run in get_runs(*wandb_groups): attr_directory = os.path.join(__EXPLANATIONS_DIR__, run.config['group']) if not os.path.isdir(attr_directory): os.mkdir(attr_directory) path_to_attribution_file = os.path.join(attr_directory, run.id + '.pkl') if os.path.isfile( os.path.join(__EXPLANATIONS_DIR__, run.config['group'], run.id + '.pkl')): print('Run already explained:', run.id) continue print('Explaining run:', run.id) try: model, cfg = load_model_from_run(run) except: continue loader = dataloader(data_file=__TEST_DATA__, val_mode=True, data_source=run.config['data_source'], data_val_origin=origin, vocab_file=run.config['model_name'], num_labels=run.config['num_labels'], train_percent=100, batch_size=1, drop_last=False, num_workers=__num_workers__ if __CUDA__ else 0) if not 'token_types' in run.config: run.config.update({'token_types': False}) attr_configs = load_configs_from_file( os.path.join('configs', 'explain.yml'))['EXPLAIN'] if 'large' in run.config['model_name']: for attribution_method in attr_configs.keys(): if attr_configs[ attribution_method] and 'internal_batch_size' in attr_configs[ attribution_method]: attr_configs[attribution_method][ 'internal_batch_size'] = attr_configs[ attribution_method]['internal_batch_size'] // 2 df = explain_model(loader, model, run.config, attr_configs, origin, __CUDA__) df['run_id'] = run.id df['model'] = run.config['name'] df['model_path'] = run.config['model_name'] df['source'] = run.config['data_source'] df['origin'] = origin df['num_labels'] = run.config['num_labels'] df['group'] = run.config['group'] df['token_types'] = run.config['token_types'] df.to_pickle(path_to_attribution_file)
def __init__(self): self.device = torch.device(cfg.device) self.max_epoch = cfg.max_epoch self.train_dataloader = dataloader() self.len_train_dataset = self.train_dataloader.num_annotations self.model = yolov3().to(self.device) self.optimizer = torch.optim.SGD(self.model.parameters(), lr=cfg.lr_start, momentum=cfg.momentum, weight_decay=cfg.weight_decay) self.scheduler = adjust_lr(self.optimizer, self.max_epoch * self.len_train_dataset, cfg.lr_start, cfg.lr_end, cfg.warmup) self.writer = SummaryWriter(cfg.tensorboard_path) self.iter = 0
def main(mode, epoches, learning_rate, train_path, val_path, batch_size, model_path): device = "cuda:0" if torch.cuda.is_available() else "cpu" model = Network(7).to(device) if os.path.isfile(model_path): print("loading model") model.load_state_dict(torch.load(model_path)) if mode == "train": train_loader = dataloader(train_path, batch_size) val_loader = dataloader(val_path, batch_size) optimizer = optim.Adam(model.parameters(), lr=learning_rate) loss_F = nn.NLLLoss() train(epoches, model, train_loader, val_loader, optimizer, loss_F, device) torch.save(model.state_dict(), model_path) else: val_loader = dataloader(val_path, batch_size) acc = validation(model, val_loader, device) print("\taccuracy: %.2f%%" % (acc))
def explain(source, name, token_types = False, origin = 'unseen_answers'): num_labels = 2 group = source + ('-token_types' if token_types else '') attr_directory = os.path.join(__EXPLANATIONS_DIR__, 'random_'+ group) os.makedirs(attr_directory, exist_ok=True) path_to_attribution_file = os.path.join(attr_directory, name + '.pkl') model_path = load_configs_from_file(os.path.join('configs', 'main.yml'))[name]['model_name'] model = transformers.AutoModelForSequenceClassification.from_pretrained(model_path, num_labels = num_labels) config = { 'token_types' : token_types, 'num_labels' : num_labels } model.init_weights() loader = dataloader( data_file = __TEST_DATA__, val_mode = True, data_source = source, data_val_origin = origin, vocab_file = model_path, num_labels = num_labels, train_percent = 100, batch_size = 1, drop_last = False, num_workers = __num_workers__ if __CUDA__ else 0) attr_configs = load_configs_from_file(os.path.join('configs','explain.yml'))['EXPLAIN'] if 'large' in name: for attribution_method in attr_configs.keys(): if attr_configs[attribution_method] and 'internal_batch_size' in attr_configs[attribution_method]: attr_configs[attribution_method]['internal_batch_size'] = attr_configs[attribution_method]['internal_batch_size']//2 df = explain_model(loader, model, config, attr_configs, origin, __CUDA__) df['run_id'] = 'random' df['model'] = name df['model_path'] = model_path df['source'] = source df['origin'] = origin df['num_labels'] = num_labels df['group'] = group df['token_types'] = token_types df.to_pickle(path_to_attribution_file)
from balance_dataloader import BalancedBatchSampler from densenet import densenet121, densenet201, densenet161 from preprocess import preproc from ArcMarginModel import ArcMarginModel_AutoMargin from FocalLoss import FocalLoss import timm os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' training_csv = './main/train.csv' validate_csv = './main/val.csv' data = './data/ISIC2018_Task3_Training_Input' labels_names = ['MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC'] training = dataloader(training_csv, data, preproc(), 'training') validation = dataloader(validate_csv, data, preproc(), 'validate') device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") dataloaders = {'train': training, 'val': validation} def visualizing(phase, epoch, step, epoch_loss, epoch_acc): # training visualizing if epoch == 0 and step == 0: writer.add_scalar(f'{phase} loss', epoch_loss, 0) writer.add_scalar(f'{phase} accuracy', 0, 0) ###################### else: writer.add_scalar(f'{phase} loss',
warnings.filterwarnings("ignore", category=SourceChangeWarning) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # data and model paths # validate_csv = '../../data/ISIC2018_Task3_Training_GroundTruth/ISIC2018_Task3_Training_GroundTruth.csv' validate_csv = './main/val.csv' data = '../../data/ISIC2018_Task3_Training_Input' models = os.listdir('./weights_test') scores = models labels_names = ['MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC'] # dataloader validation = dataloader(validate_csv, data, lowcost_center_preproc(), 'validate') device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") def _confusion_matrix(model_name, score, y_true, y_pred): cfs = confusion_matrix(y_true, y_pred) cfs.dtype = np.float for i in range(len(cfs)): cfs[i] = cfs[i]/sum(cfs[i]) df_cm = pd.DataFrame(cfs, index = ['MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC'], columns = ['pMEL', 'pNV', 'pBCC', 'pAKIEC', 'pBKL', 'pDF', 'pVASC']) plt.figure(figsize = (10,7)) sns_plot = sn.heatmap(df_cm, annot=True, square=True, cmap="YlGnBu").set_title(f'Score: {score}') fig = sns_plot.get_figure() # fig.savefig(f"./confusion_matrix/full_data/{model_name}.png", dpi=400) fig.savefig(f"./confusion_matrix/{model_name}_val.png", dpi=400)
from preprocess import preproc, lowcost_center_preproc import warnings from torch.serialization import SourceChangeWarning warnings.filterwarnings("ignore", category=SourceChangeWarning) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # data and model paths test_data = '../../data/ISIC2018_Task3_Test_Input' model_path = './weights/efficientnet_b2_ArcMargin_2022-02-20_15-1_epoch63.tar' labels_names = ['MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC'] # dataloader test_loader = dataloader(None, test_data, lowcost_center_preproc(), 'test') device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") def summision_generate(model, batch_size, arccos): result = {} since = time.time() print('-' * 10) model.eval() # Set model to evaluate mode batch_iterator = iter( DataLoader(test_loader, batch_size, shuffle=False, num_workers=4))
from dataset import dataloader from preprocess import preproc import pickle warnings.filterwarnings("ignore", category=SourceChangeWarning) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # data and model paths test_data = '../../data/ISIC2018_Task3_Training_Input' model_path = './weights/densenet121_ArcMargin_2021-01-22_1-12_epoch99.tar' labels_names = ['MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC'] # dataloader test_loader = dataloader(None, test_data, preproc(), 'test') device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") def full_crop(image): img_t = [] for r in range(0, 4): for c in range(0, 5): img_t.append(image[:, 50*r:50*r+300, 50*c:50*c+400]) return img_t def summision_generate(model, batch_size, arccos=None, voting=True): result = {}
def main(): if not os.path.exists(config.out_image): os.mkdir(config.out_image) if not os.path.exists(config.save_model): os.mkdir(config.save_model) writer = SummaryWriter() train_lists, val_lists = dataset.train_val_split(config.image_path, config.val, config.seed) #print(train_lists) train_transfoms = Compose([ Resize((config.resize, config.resize)), RandomHorizontalFlip(), ToTensor(), Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ]) val_transfoms = Compose([ Resize((config.resize, config.resize)), ToTensor(), Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ]) train_datasets = dataset.dataloader(train_lists, train_transfoms) val_datasets = dataset.dataloader(val_lists, val_transfoms) train_dataloader = DataLoader(train_datasets, batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers, pin_memory=True, drop_last=True) val_dataloader = DataLoader(val_datasets, batch_size=4, shuffle=False, num_workers=config.num_workers, pin_memory=True) cuda = torch.cuda.is_available() genrate_model = model.genrate_model() D_model = model.style_model() if config.load_checkpoint: genrate_model.load_state_dict(torch.load(config.checkpoint)) if cuda: genrate_model.cuda() D_model.cuda() #optim = torch.optim.sgd.SGD(genrate_model.parameters(),config.lr,) optim = torch.optim.Adam(genrate_model.parameters(), config.lr) style_criterion = torch.nn.MSELoss() constant_criterion = torch.nn.MSELoss() if cuda: style_criterion.cuda() constant_criterion.cuda() ########################style image########################### style_image = Image.open(config.style_image) style_image = Resize((config.resize, config.resize))(style_image) style_image = ToTensor()(style_image) style_image = Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))(style_image) style_image = style_image.unsqueeze(0) if cuda: style_image = style_image.cuda() style_ori_image = [ style_ori_image_.detach() for style_ori_image_ in D_model(style_image, 'style') ] Gram_style = [Gram()(style_ori_image[a]) for a in range(4)] best_loss = float('inf') ########################################train########################################## for epoch in range(config.epochs): genrate_model.train() for i, image in enumerate(train_dataloader): optim.zero_grad() if cuda: image = image.cuda() genrate_image = genrate_model(image) constant_ori_image = D_model(image, 'constant').detach() constant_genrate_image = D_model(genrate_image, 'constant') constant_loss = constant_criterion(constant_ori_image, constant_genrate_image) #style_ori_image = [style_ori_image_.detach() for style_ori_image_ in D_model(style_image,'style')] style_genrate_image = [ style_genrate_image_ for style_genrate_image_ in D_model(genrate_image, 'style') ] Gram_genrate = [Gram()(style_genrate_image[a]) for a in range(4)] style_losses = [ config.style_weight * style_criterion( Gram_genrate[a], Gram_style[a].expand_as(Gram_genrate[a])) for a in range(4) ] style_loss = sum(style_losses) loss = constant_loss + style_loss loss.backward() optim.step() print('epoch:{} iter:{} loss:{}'.format(epoch, i, loss)) writer.add_scalar('scalar/loss', float(loss), epoch * len(train_dataloader) + i) if float(loss) < best_loss: best_loss = float(loss) torch.save(genrate_model.state_dict(), config.save_model + '/best.pth') if (epoch + 1) % config.val_epoch == 0: genrate_model.eval() with torch.no_grad(): for i, image in enumerate(val_dataloader): if cuda: image = image.cuda() genrate_image = genrate_model(image) torchvision.utils.save_image( torch.cat([ image * 0.5 + 0.5, genrate_image * 0.5 + 0.5 ]), config.out_image + '/{}.jpg'.format(i)) print('val_epoch:{} iter:{}'.format(epoch, i))
# TODO: training and test datasets print('==> Preparing data..') dataset = args.dataset dataset_path = os.path.join('outputs', dataset) time = datetime.now().strftime('%Y-%m-%d-%H:%M') save_path = os.path.join(dataset_path, time) if not os.path.isdir(dataset_path): os.mkdir(dataset_path) os.mkdir(save_path) if not os.path.isdir(save_path): os.mkdir(save_path) print('Dataset:', dataset) print('Checkpoint save dir:', save_path) # TODO: datasets loader train_loader, test_loader = dataloader(dataset, args.batch_size, args.test_batch_size) # TODO: optimizer / scheduler / loss function optimizer = optim.Adadelta(model.parameters()) # optimizer = optim.SGD(model.parameters(), lr=args.base_lr, momentum=0.9, weight_decay=5e-4) # loss_func = nn.CrossEntropyLoss() # scheduler = MultiStepLR(optimizer, milestones=[180, 240], gamma=0.1) # TODO: train and test best_test_acc = 0.85 if dataset == 'MNIST-rot+': epochs = int(np.ceil(args.epochs / 8)) elif dataset == 'MNIST' or dataset == 'MNIST-rot': epochs = args.epochs
from balance_dataloader import BalancedBatchSampler from preprocess import preproc, lowcost_center_preproc from ArcMarginModel import ArcMarginModel from FocalLoss import FocalLoss import timm os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' training_csv = './main/train.csv' validate_csv = './main/val.csv' data = './ISIC2018_Task3_Training_Input' labels_names = ['MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC'] training = dataloader(training_csv, data, lowcost_center_preproc(), 'training') validation = dataloader(validate_csv, data, lowcost_center_preproc(), 'validate') device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") dataloaders = {'train': training, 'val': validation} def visualizing(phase, epoch, step, epoch_loss, epoch_acc): # training visualizing if epoch == 0 and step == 0: writer.add_scalar(f'{phase} loss', epoch_loss, 0) writer.add_scalar(f'{phase} accuracy', 0, 0) ###################### else: writer.add_scalar(f'{phase} loss',
def run(*configs, group=None): config = configuration.load(*configs) if config.group: config.group = config.data_source + '-' + config.group else: config.group = config.data_source if group: config.group = config.group + "-" + str(group) if config.from_scratch: config.group = 'scratch-' + config.group config.name = 'scratch-' + config.name if config.log: wandb.init(project='explainable-asag', group=config.group, name=config.name, config=config) config = wandb.config model = transformers.AutoModelForSequenceClassification.from_pretrained( config.model_name, num_labels=config.num_labels) if config.token_types: embedding_size = model.config.__dict__.get('embedding_size', model.config.hidden_size) update_token_type_embeddings(model, embedding_size, model.config.initializer_range) if config.from_scratch: model.init_weights() cuda = torch.cuda.is_available() if cuda: model.cuda() train_dataloader = dataset.dataloader(val_mode=False, data_file=config.train_data, data_source=config.data_source, vocab_file=config.model_name, num_labels=config.num_labels, train_percent=config.train_percent, batch_size=config.batch_size, drop_last=config.drop_last, num_workers=config.num_workers) val_dataloader = dataset.dataloader(val_mode=True, data_file=config.val_data, data_source=config.data_source, vocab_file=config.model_name, num_labels=config.num_labels, train_percent=config.val_percent, batch_size=config.batch_size, drop_last=config.drop_last, num_workers=config.num_workers) optimizer = torch.optim.__dict__[config.optimizer]( model.parameters(), lr=config.learn_rate, **config.optimizer_kwargs) # Hack to get any scheduler we want. transformers.get_scheduler does not implement e.g. linear_with_warmup. get_scheduler = { 'linear_with_warmup': transformers.get_linear_schedule_with_warmup, 'cosine_with_warmup': transformers.get_cosine_schedule_with_warmup, 'constant_with_warmup': transformers.get_constant_schedule_with_warmup, 'cosine_with_hard_restarts_with_warmup': transformers.get_cosine_with_hard_restarts_schedule_with_warmup } lr_scheduler = get_scheduler[config.scheduler](optimizer, *config.scheduler_args, **config.scheduler_kwargs) best_f1 = 0.0 patience = 0 epoch = 0 log_line = '' try: #while lr_scheduler.last_epoch <= total_steps: while epoch < config.max_epochs: epoch += 1 av_epoch_loss = training.train_epoch( train_dataloader, model, optimizer, lr_scheduler, config.num_labels, cuda, log=config.log, token_types=config.token_types) #tidy stuff up every epoch gc.collect() torch.cuda.empty_cache() metrics_weighted, metrics_macro = training.val_loop( model, val_dataloader, cuda, token_types=config.token_types) p, r, f1, val_acc = metrics_weighted p_m, r_m, f1_m, val_acc_m = metrics_macro log_line = f'model: {config.model_name} | epoch: {epoch} | av_epoch_loss {av_epoch_loss:.5f} | f1: {f1:.5f} | accuracy: {val_acc:.5f} \n' print(log_line[:-1]) if config.log: wandb.log({ 'precision': p, 'recall': r, 'f1': f1, 'accuracy': val_acc, 'av_epoch_loss': av_epoch_loss }) wandb.log({ 'precision-macro': p_m, 'recall-macro': r_m, 'f1-macro': f1_m, 'accuracy-macro': val_acc_m }) if f1 > best_f1: if config.log: this_model = os.path.join(wandb.run.dir, 'best_f1.pt') print("saving to: ", this_model) torch.save([model.state_dict(), config.__dict__], this_model) wandb.save('*.pt') best_f1 = f1 patience = 0 #max((0, patience-1)) elif config.max_patience: patience += 1 if patience >= config.max_patience: break # Move stuff off the gpu model.cpu() #This is for sure a kinda dumb way of doing it, but the least mentally taxing right now optimizer = torch.optim.__dict__[config.optimizer]( model.parameters(), lr=config.learn_rate) gc.collect() torch.cuda.empty_cache() #return model #Gives Error except KeyboardInterrupt: if config.log: wandb.save('*.pt') #Move stuff off the gpu model.cpu() optimizer = torch.optim.__dict__[config.optimizer]( model.parameters(), lr=config.learn_rate) gc.collect() torch.cuda.empty_cache()
def __init__(self, ): args = parser.parse_args() self.device = torch.device(args.device) ######################## LOAD DATA ############################# self.trainval_set = dataloader(root=args.dataset_path, split='trainval', device=self.device) #train_set = dataloader(root=args.dataset_path,split='train', device=self.device) self.test_set = dataloader(root=args.dataset_path, split='test_unseen', device=self.device) self.test_set_seen = dataloader(root=args.dataset_path, split='test_seen', device=self.device) #val_set = dataloader(root=args.dataset_path,split='val', device=self.device) self.trainloader = data.DataLoader(self.trainval_set, batch_size=args.batch_size, shuffle=True) #self.testloader = data.DataLoader(self.test_set, batch_size=args.batch_size, shuffle=False) self.input_dim = self.trainval_set.__getlen__() self.atts_dim = self.test_set.__get_attlen__() self.num_classes = self.trainval_set.__totalClasses__() print(20 * ('-')) print("Input_dimension=%d" % self.input_dim) print("Attribute_dimension=%d" % self.atts_dim) print("z=%d" % args.latent_size) print("num_classes=%d" % self.num_classes) print(20 * ('-')) ####################### INITIALIZE THE MODEL AND OPTIMIZER ##################### self.model_encoder = encoder_cada(input_dim=self.input_dim, atts_dim=self.atts_dim, z=args.latent_size).to(self.device) self.model_decoder = decoder_cada(input_dim=self.input_dim, atts_dim=self.atts_dim, z=args.latent_size).to(self.device) learnable_params = list(self.model_encoder.parameters()) + list( self.model_decoder.parameters()) self.optimizer = optim.Adam(learnable_params, lr=0.00015, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=True) self.classifier = Classifier(input_dim=args.latent_size, num_class=self.num_classes) self.cls_optimizer = optim.Adam(self.classifier.parameters(), lr=0.001, betas=(0.5, 0.999)) print(self.model_encoder) print(self.model_decoder) print(self.classifier) ########## LOSS ############ self.l1_loss = nn.L1Loss(reduction='sum') self.lossfunction_classifier = nn.NLLLoss() self.gamma = torch.zeros(1, device=self.device).float() self.beta = torch.zeros(1, device=self.device).float() self.delta = torch.zeros(1, device=self.device).float()
import os import time import torch as t import pdb import torch.nn as nn import torch.optim as optim from model import CNN_BiLSTM from dataset import dataloader from tensorboardX import SummaryWriter sheet_name = 213 train_loader, test_loader, torque = dataloader(sheet_name) device = t.device('cuda' if t.cuda.is_available() else 'cpu') def train(model: CNN_BiLSTM, optimizer: optim.Adam, criterion: nn.modules.loss, model_path, epochs: int = 100): """ model: CNN_BiLSTM optimizer: usualy adam criterion: loss function epochs: training epochs model_path: path of model storage """ best_test_loss = float('inf') for epoch in range(1, epochs + 1): start_time = time.time()