def run(): from config import get_config config = get_config() print('%s/train_embeddings.csv' % config.result_dir) result_dir = config.result_dir #.replace("results", "best_results") print('%s/train_embeddings.csv' % result_dir) if os.path.exists( '%s/train_embeddings.csv' % result_dir) and os.path.exists( '%s/test_embeddings.csv' % result_dir): return True if not os.path.exists(result_dir): os.makedirs(result_dir) print("Saved Module Trainer Not Return") import losses import models from utils.make_dirs import create_dirs from datasets import loaders from torchsample.modules import ModuleTrainer create_dirs() cuda_device = -1 model = getattr(models, config.network).get_network()( channel=config.network_channel, embedding_size=config.embedding) check_point = os.path.join(config.result_dir, "ckpt.pth.tar") if os.path.isfile(check_point): print("=> loading checkpoint '{}'".format(check_point)) checkpoint = torch.load(check_point) model.load_state_dict(checkpoint['state_dict']) print("=> loaded checkpoint '{}' (epoch {})".format( check_point, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(check_point)) #criterion = getattr(losses, config.loss)() criterion = CrossEntropyLoss() if config.cuda: model.cuda() criterion.cuda() trainer = ModuleTrainer(model) trainer.compile(loss=criterion, optimizer='adam') if config.cuda: cuda_device = 0 tr_data_loader, val_data_loader, te_data_loader = getattr( loaders, config.loader_name)(train=False, val=True) tr_y_pred = trainer.predict_loader(tr_data_loader, cuda_device=cuda_device) save_embeddings(tr_y_pred, '%s/train_embeddings.csv' % result_dir) save_labels(tr_data_loader, '%s/train_labels.csv' % result_dir) val_y_pred = trainer.predict_loader(val_data_loader, cuda_device=cuda_device) save_embeddings(val_y_pred, '%s/val_embeddings.csv' % result_dir) save_labels(val_data_loader, '%s/val_labels.csv' % result_dir) te_y_pred = trainer.predict_loader(te_data_loader, cuda_device=cuda_device) save_embeddings(te_y_pred, '%s/test_embeddings.csv' % result_dir) save_labels(te_data_loader, '%s/test_labels.csv' % result_dir)
def run(): from config import get_config config = get_config() print('%s/ckpt.pth.tar' % config.result_dir) if os.path.exists('%s/ckpt.pth.tar' % config.result_dir): return True print("Contrastive Trainer Not Return") import models from utils.make_dirs import create_dirs from datasets import loaders from torchsample.callbacks import EarlyStopping, ModelCheckpoint, CSVLogger from torchsample.metrics import CategoricalAccuracy from torchsample.modules import ModuleTrainer create_dirs() cuda_device = -1 tr_data_loader, val_data_loader, te_data_loader = loaders.online_pair_loaders() model = getattr(models, config.network).get_network()(channel=config.network_channel, embedding_size=config.embedding) from losses.online_cosine import OnlineCosineLoss from datasets.data_utils import AllPositivePairSelector, HardNegativePairSelector margin = 0.5 if args.selector == 'AllPositivePairSelector': criterion = OnlineCosineLoss(margin, AllPositivePairSelector()) elif args.selector == 'HardNegativePairSelector': criterion = OnlineCosineLoss(margin, HardNegativePairSelector()) if config.cuda: model.cuda() criterion.cuda() trainer = ModuleTrainer(model) epochs = config.epochs callbacks = [EarlyStopping(monitor='val_loss', patience=50), ModelCheckpoint(config.result_dir, save_best_only=True, verbose=1), CSVLogger("%s/logger.csv" % config.result_dir)] metrics = [] if config.loader_name == 'data_loaders' and 'Angle' not in config.loss: metrics.append(CategoricalAccuracy(top_k=1)) trainer.compile(loss=criterion, optimizer='adam', metrics=metrics) trainer.set_callbacks(callbacks) if config.cuda: cuda_device = 0 start_time = time.time() trainer.fit_loader(tr_data_loader, val_loader=val_data_loader, num_epoch=epochs, verbose=2, cuda_device=cuda_device) end_time = time.time() with open("%s/app.log" % config.result_dir, mode="a") as f: f.write("%s\n" % str(model)) f.write("%s %s\n" % (config.loss, str(end_time - start_time))) tr_loss = trainer.evaluate_loader(tr_data_loader, cuda_device=cuda_device) print(tr_loss) val_loss = trainer.evaluate_loader(val_data_loader, cuda_device=cuda_device) te_loss = trainer.evaluate_loader(te_data_loader, cuda_device=cuda_device) print(te_loss) with open(config.log_path, "a") as f: f.write('Train %s\nVal:%s\nTest:%s\n' % (str(tr_loss), str(val_loss), te_loss)) tr_data_loader, val_data_loader, te_data_loader = loaders.data_loaders(train=False, val=True) tr_y_pred = trainer.predict_loader(tr_data_loader, cuda_device=cuda_device) save_embeddings(tr_y_pred, '%s/train_embeddings.csv' % config.result_dir) save_labels(tr_data_loader, '%s/train_labels.csv' % config.result_dir) val_y_pred = trainer.predict_loader(val_data_loader, cuda_device=cuda_device) save_embeddings(val_y_pred, '%s/val_embeddings.csv' % config.result_dir) save_labels(val_data_loader, '%s/val_labels.csv' % config.result_dir) te_y_pred = trainer.predict_loader(te_data_loader, cuda_device=cuda_device) save_embeddings(te_y_pred, '%s/test_embeddings.csv' % config.result_dir) save_labels(te_data_loader, '%s/test_labels.csv' % config.result_dir)
transforms.ToTensor(), transforms.Normalize(mean=[0.55803171, 0.54565148, 0.57604222], std=[0.30346842, 0.3020077, 0.30457914]), ])) test_loader = DataLoader( test_dataset, #sampler = SubsetSampler(100), batch_size=64, num_workers=1, pin_memory=True if th.cuda.is_available() else False) test_trainer = ModuleTrainer(model) predictions = test_trainer.predict_loader( test_loader, verbose=1, cuda_device=0 if th.cuda.is_available() else -1) species = predictions[0].data[:, :8].topk(8) species_softmax = predictions[0].data[:, :8].exp() ll = len(predictions[0]) print(ll) video_filename__base_frame = test_dataset.video_index_frame.ix[range(ll), [ test_dataset.video_index_frame.columns.get_loc('video_id'), test_dataset.video_index_frame.columns.get_loc('base_frame') ]].values length = predictions[0].data[:, 8].cpu().numpy() species_fourspot__species_no_fish = species_softmax.cpu().numpy()
def run(): from config import get_config config = get_config() print('%s/train_embeddings.csv' % config.result_dir) result_dir = config.result_dir print('%s/train_embeddings.csv' % result_dir) if os.path.exists( '%s/train_embeddings.csv' % result_dir) and os.path.exists( '%s/test_embeddings.csv' % result_dir): return True if not os.path.exists(result_dir): os.makedirs(result_dir) print("Saved Module Trainer Not Return") import losses import models from utils.make_dirs import create_dirs from datasets import loaders from torchsample.modules import ModuleTrainer create_dirs() cuda_device = -1 model = getattr(models, config.network).get_network()( channel=config.network_channel, embedding_size=config.embedding) check_point = os.path.join(config.result_dir, "ckpt.pth.tar") if os.path.isfile(check_point): print("=> loading checkpoint '{}'".format(check_point)) checkpoint = torch.load(check_point) model.load_state_dict(checkpoint['state_dict']) print("=> loaded checkpoint '{}' (epoch {})".format( check_point, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(check_point)) criterion = getattr(losses, config.loss)() if config.cuda: model.cuda() criterion.cuda() trainer = ModuleTrainer(model) metrics = [] if config.loader_name == 'data_loaders': metrics.append(CategoricalAccuracy(top_k=1)) trainer.compile(loss=criterion, optimizer='adam', metrics=metrics) if config.cuda: cuda_device = 0 tr_data_loader, val_data_loader, te_data_loader = getattr( loaders, config.loader_name)(train=False, val=True) tr_loss = trainer.evaluate_loader(tr_data_loader, cuda_device=cuda_device) val_loss = trainer.evaluate_loader(val_data_loader, cuda_device=cuda_device) te_loss = trainer.evaluate_loader(te_data_loader, cuda_device=cuda_device) tr_y_pred = trainer.predict_loader(tr_data_loader, cuda_device=cuda_device) save_embeddings(tr_y_pred, '%s/train_embeddings.csv' % result_dir) save_labels(tr_data_loader, '%s/train_labels.csv' % result_dir) val_y_pred = trainer.predict_loader(val_data_loader, cuda_device=cuda_device) save_embeddings(val_y_pred, '%s/val_embeddings.csv' % result_dir) save_labels(val_data_loader, '%s/val_labels.csv' % result_dir) te_y_pred = trainer.predict_loader(te_data_loader, cuda_device=cuda_device) save_embeddings(te_y_pred, '%s/test_embeddings.csv' % result_dir) save_labels(te_data_loader, '%s/test_labels.csv' % result_dir) with open(config.log_path.replace("results", "best_results"), "a") as f: f.write('Best Train %s\nBest Val:%s\nBest Test:%s\n' % (str(tr_loss), str(val_loss), te_loss))
def run(): config = BaseConfig() logging.info('%s/train_embeddings.csv' % config.result_dir) result_dir = config.result_dir logging.info('%s/train_embeddings.csv' % result_dir) if os.path.exists( '%s/train_embeddings.csv' % result_dir) and os.path.exists( '%s/test_embeddings.csv' % result_dir): return True if not os.path.exists(result_dir): os.makedirs(result_dir) logging.info("Saved Module Trainer Not Return") create_dirs() device = 0 if torch.cuda.is_available() else -1 tr_data_loader, val_data_loader, te_data_loader = loaders.data_loaders() model = getattr( models, config.network).get_network()(embedding_size=config.embedding) check_point = os.path.join(config.result_dir, "ckpt.pth.tar") if os.path.isfile(check_point): logging.info("=> loading checkpoint '{}'".format(check_point)) checkpoint = torch.load(check_point) model.load_state_dict(checkpoint['state_dict']) logging.info("=> loaded checkpoint '{}' (epoch {})".format( check_point, checkpoint['epoch'])) else: logging.info("=> no checkpoint found at '{}'".format(check_point)) return margin = 1. criterion = OnlineTripletLoss(margin, SemihardNegativeTripletSelector(margin)) if device == 0: model.cuda() criterion.cuda() trainer = ModuleTrainer(model) trainer.compile(loss=criterion, optimizer='adam') logging.info('Train Prediction') tr_y_pred = trainer.predict_loader(tr_data_loader, cuda_device=device) logging.info('Train Save Embeddings') save_embeddings(tr_y_pred, '%s/train_embeddings.csv' % config.result_dir) logging.info('Train Save Labels') save_labels(tr_data_loader, '%s/train_labels.csv' % config.result_dir) tr_data_loader.dataset.id_list.to_csv('%s/train_ids.csv' % config.result_dir, header=None, index=None) logging.info('Validation Prediction') val_y_pred = trainer.predict_loader(val_data_loader, cuda_device=device) logging.info('Validation Save Embeddings') save_embeddings(val_y_pred, '%s/val_embeddings.csv' % config.result_dir) logging.info('Validation Save Labels') save_labels(val_data_loader, '%s/val_labels.csv' % config.result_dir) val_data_loader.dataset.id_list.to_csv('%s/val_ids.csv' % config.result_dir, header=None, index=None) logging.info('Test Prediction') te_y_pred = trainer.predict_loader(te_data_loader, cuda_device=device) logging.info('Test Save Embeddings') save_embeddings(te_y_pred, '%s/test_embeddings.csv' % config.result_dir) logging.info('Test Save Labels') save_labels(te_data_loader, '%s/test_labels.csv' % config.result_dir) te_data_loader.dataset.id_list.to_csv('%s/test_ids.csv' % config.result_dir, header=None, index=None)