def run():
from config import get_config
config = get_config()
print('%s/train_embeddings.csv' % config.result_dir)
result_dir = config.result_dir #.replace("results", "best_results")
print('%s/train_embeddings.csv' % result_dir)
if os.path.exists(
'%s/train_embeddings.csv' % result_dir) and os.path.exists(
'%s/test_embeddings.csv' % result_dir):
return True
if not os.path.exists(result_dir):
os.makedirs(result_dir)
print("Saved Module Trainer Not Return")
import losses
import models
from utils.make_dirs import create_dirs
from datasets import loaders
from torchsample.modules import ModuleTrainer
create_dirs()
cuda_device = -1
model = getattr(models, config.network).get_network()(
channel=config.network_channel, embedding_size=config.embedding)
check_point = os.path.join(config.result_dir, "ckpt.pth.tar")
if os.path.isfile(check_point):
print("=> loading checkpoint '{}'".format(check_point))
checkpoint = torch.load(check_point)
model.load_state_dict(checkpoint['state_dict'])
print("=> loaded checkpoint '{}' (epoch {})".format(
check_point, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(check_point))
#criterion = getattr(losses, config.loss)()
criterion = CrossEntropyLoss()
if config.cuda:
model.cuda()
criterion.cuda()
trainer = ModuleTrainer(model)
trainer.compile(loss=criterion, optimizer='adam')
if config.cuda:
cuda_device = 0
tr_data_loader, val_data_loader, te_data_loader = getattr(
loaders, config.loader_name)(train=False, val=True)
tr_y_pred = trainer.predict_loader(tr_data_loader, cuda_device=cuda_device)
save_embeddings(tr_y_pred, '%s/train_embeddings.csv' % result_dir)
save_labels(tr_data_loader, '%s/train_labels.csv' % result_dir)
val_y_pred = trainer.predict_loader(val_data_loader,
cuda_device=cuda_device)
save_embeddings(val_y_pred, '%s/val_embeddings.csv' % result_dir)
save_labels(val_data_loader, '%s/val_labels.csv' % result_dir)
te_y_pred = trainer.predict_loader(te_data_loader, cuda_device=cuda_device)
save_embeddings(te_y_pred, '%s/test_embeddings.csv' % result_dir)
save_labels(te_data_loader, '%s/test_labels.csv' % result_dir)
def run():
from config import get_config
config = get_config()
print('%s/ckpt.pth.tar' % config.result_dir)
if os.path.exists('%s/ckpt.pth.tar' % config.result_dir):
return True
print("Contrastive Trainer Not Return")
import models
from utils.make_dirs import create_dirs
from datasets import loaders
from torchsample.callbacks import EarlyStopping, ModelCheckpoint, CSVLogger
from torchsample.metrics import CategoricalAccuracy
from torchsample.modules import ModuleTrainer
create_dirs()
cuda_device = -1
tr_data_loader, val_data_loader, te_data_loader = loaders.online_pair_loaders()
model = getattr(models, config.network).get_network()(channel=config.network_channel,
embedding_size=config.embedding)
from losses.online_cosine import OnlineCosineLoss
from datasets.data_utils import AllPositivePairSelector, HardNegativePairSelector
margin = 0.5
if args.selector == 'AllPositivePairSelector':
criterion = OnlineCosineLoss(margin, AllPositivePairSelector())
elif args.selector == 'HardNegativePairSelector':
criterion = OnlineCosineLoss(margin, HardNegativePairSelector())
if config.cuda:
model.cuda()
criterion.cuda()
trainer = ModuleTrainer(model)
epochs = config.epochs
callbacks = [EarlyStopping(monitor='val_loss', patience=50),
ModelCheckpoint(config.result_dir, save_best_only=True, verbose=1),
CSVLogger("%s/logger.csv" % config.result_dir)]
metrics = []
if config.loader_name == 'data_loaders' and 'Angle' not in config.loss:
metrics.append(CategoricalAccuracy(top_k=1))
trainer.compile(loss=criterion, optimizer='adam', metrics=metrics)
trainer.set_callbacks(callbacks)
if config.cuda:
cuda_device = 0
start_time = time.time()
trainer.fit_loader(tr_data_loader, val_loader=val_data_loader, num_epoch=epochs, verbose=2,
cuda_device=cuda_device)
end_time = time.time()
with open("%s/app.log" % config.result_dir, mode="a") as f:
f.write("%s\n" % str(model))
f.write("%s %s\n" % (config.loss, str(end_time - start_time)))
tr_loss = trainer.evaluate_loader(tr_data_loader, cuda_device=cuda_device)
print(tr_loss)
val_loss = trainer.evaluate_loader(val_data_loader, cuda_device=cuda_device)
te_loss = trainer.evaluate_loader(te_data_loader, cuda_device=cuda_device)
print(te_loss)
with open(config.log_path, "a") as f:
f.write('Train %s\nVal:%s\nTest:%s\n' % (str(tr_loss), str(val_loss), te_loss))
tr_data_loader, val_data_loader, te_data_loader = loaders.data_loaders(train=False, val=True)
tr_y_pred = trainer.predict_loader(tr_data_loader, cuda_device=cuda_device)
save_embeddings(tr_y_pred, '%s/train_embeddings.csv' % config.result_dir)
save_labels(tr_data_loader, '%s/train_labels.csv' % config.result_dir)
val_y_pred = trainer.predict_loader(val_data_loader, cuda_device=cuda_device)
save_embeddings(val_y_pred, '%s/val_embeddings.csv' % config.result_dir)
save_labels(val_data_loader, '%s/val_labels.csv' % config.result_dir)
te_y_pred = trainer.predict_loader(te_data_loader, cuda_device=cuda_device)
save_embeddings(te_y_pred, '%s/test_embeddings.csv' % config.result_dir)
save_labels(te_data_loader, '%s/test_labels.csv' % config.result_dir)
transforms.ToTensor(),
transforms.Normalize(mean=[0.55803171, 0.54565148, 0.57604222],
std=[0.30346842, 0.3020077, 0.30457914]),
]))
test_loader = DataLoader(
test_dataset,
#sampler = SubsetSampler(100),
batch_size=64,
num_workers=1,
pin_memory=True if th.cuda.is_available() else False)
test_trainer = ModuleTrainer(model)
predictions = test_trainer.predict_loader(
test_loader,
verbose=1,
cuda_device=0 if th.cuda.is_available() else -1)
species = predictions[0].data[:, :8].topk(8)
species_softmax = predictions[0].data[:, :8].exp()
ll = len(predictions[0])
print(ll)
video_filename__base_frame = test_dataset.video_index_frame.ix[range(ll), [
test_dataset.video_index_frame.columns.get_loc('video_id'),
test_dataset.video_index_frame.columns.get_loc('base_frame')
]].values
length = predictions[0].data[:, 8].cpu().numpy()
species_fourspot__species_no_fish = species_softmax.cpu().numpy()
def run():
from config import get_config
config = get_config()
print('%s/train_embeddings.csv' % config.result_dir)
result_dir = config.result_dir
print('%s/train_embeddings.csv' % result_dir)
if os.path.exists(
'%s/train_embeddings.csv' % result_dir) and os.path.exists(
'%s/test_embeddings.csv' % result_dir):
return True
if not os.path.exists(result_dir):
os.makedirs(result_dir)
print("Saved Module Trainer Not Return")
import losses
import models
from utils.make_dirs import create_dirs
from datasets import loaders
from torchsample.modules import ModuleTrainer
create_dirs()
cuda_device = -1
model = getattr(models, config.network).get_network()(
channel=config.network_channel, embedding_size=config.embedding)
check_point = os.path.join(config.result_dir, "ckpt.pth.tar")
if os.path.isfile(check_point):
print("=> loading checkpoint '{}'".format(check_point))
checkpoint = torch.load(check_point)
model.load_state_dict(checkpoint['state_dict'])
print("=> loaded checkpoint '{}' (epoch {})".format(
check_point, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(check_point))
criterion = getattr(losses, config.loss)()
if config.cuda:
model.cuda()
criterion.cuda()
trainer = ModuleTrainer(model)
metrics = []
if config.loader_name == 'data_loaders':
metrics.append(CategoricalAccuracy(top_k=1))
trainer.compile(loss=criterion, optimizer='adam', metrics=metrics)
if config.cuda:
cuda_device = 0
tr_data_loader, val_data_loader, te_data_loader = getattr(
loaders, config.loader_name)(train=False, val=True)
tr_loss = trainer.evaluate_loader(tr_data_loader, cuda_device=cuda_device)
val_loss = trainer.evaluate_loader(val_data_loader,
cuda_device=cuda_device)
te_loss = trainer.evaluate_loader(te_data_loader, cuda_device=cuda_device)
tr_y_pred = trainer.predict_loader(tr_data_loader, cuda_device=cuda_device)
save_embeddings(tr_y_pred, '%s/train_embeddings.csv' % result_dir)
save_labels(tr_data_loader, '%s/train_labels.csv' % result_dir)
val_y_pred = trainer.predict_loader(val_data_loader,
cuda_device=cuda_device)
save_embeddings(val_y_pred, '%s/val_embeddings.csv' % result_dir)
save_labels(val_data_loader, '%s/val_labels.csv' % result_dir)
te_y_pred = trainer.predict_loader(te_data_loader, cuda_device=cuda_device)
save_embeddings(te_y_pred, '%s/test_embeddings.csv' % result_dir)
save_labels(te_data_loader, '%s/test_labels.csv' % result_dir)
with open(config.log_path.replace("results", "best_results"), "a") as f:
f.write('Best Train %s\nBest Val:%s\nBest Test:%s\n' %
(str(tr_loss), str(val_loss), te_loss))
def run():
config = BaseConfig()
logging.info('%s/train_embeddings.csv' % config.result_dir)
result_dir = config.result_dir
logging.info('%s/train_embeddings.csv' % result_dir)
if os.path.exists(
'%s/train_embeddings.csv' % result_dir) and os.path.exists(
'%s/test_embeddings.csv' % result_dir):
return True
if not os.path.exists(result_dir):
os.makedirs(result_dir)
logging.info("Saved Module Trainer Not Return")
create_dirs()
device = 0 if torch.cuda.is_available() else -1
tr_data_loader, val_data_loader, te_data_loader = loaders.data_loaders()
model = getattr(
models, config.network).get_network()(embedding_size=config.embedding)
check_point = os.path.join(config.result_dir, "ckpt.pth.tar")
if os.path.isfile(check_point):
logging.info("=> loading checkpoint '{}'".format(check_point))
checkpoint = torch.load(check_point)
model.load_state_dict(checkpoint['state_dict'])
logging.info("=> loaded checkpoint '{}' (epoch {})".format(
check_point, checkpoint['epoch']))
else:
logging.info("=> no checkpoint found at '{}'".format(check_point))
return
margin = 1.
criterion = OnlineTripletLoss(margin,
SemihardNegativeTripletSelector(margin))
if device == 0:
model.cuda()
criterion.cuda()
trainer = ModuleTrainer(model)
trainer.compile(loss=criterion, optimizer='adam')
logging.info('Train Prediction')
tr_y_pred = trainer.predict_loader(tr_data_loader, cuda_device=device)
logging.info('Train Save Embeddings')
save_embeddings(tr_y_pred, '%s/train_embeddings.csv' % config.result_dir)
logging.info('Train Save Labels')
save_labels(tr_data_loader, '%s/train_labels.csv' % config.result_dir)
tr_data_loader.dataset.id_list.to_csv('%s/train_ids.csv' %
config.result_dir,
header=None,
index=None)
logging.info('Validation Prediction')
val_y_pred = trainer.predict_loader(val_data_loader, cuda_device=device)
logging.info('Validation Save Embeddings')
save_embeddings(val_y_pred, '%s/val_embeddings.csv' % config.result_dir)
logging.info('Validation Save Labels')
save_labels(val_data_loader, '%s/val_labels.csv' % config.result_dir)
val_data_loader.dataset.id_list.to_csv('%s/val_ids.csv' %
config.result_dir,
header=None,
index=None)
logging.info('Test Prediction')
te_y_pred = trainer.predict_loader(te_data_loader, cuda_device=device)
logging.info('Test Save Embeddings')
save_embeddings(te_y_pred, '%s/test_embeddings.csv' % config.result_dir)
logging.info('Test Save Labels')
save_labels(te_data_loader, '%s/test_labels.csv' % config.result_dir)
te_data_loader.dataset.id_list.to_csv('%s/test_ids.csv' %
config.result_dir,
header=None,
index=None)
