def __init__(self, model,use_cuda_if_available=True):
self.model = model
self.cuda = False
if use_cuda_if_available and cuda.is_available():
self.cuda = True
self.loss_history = []
self.__input_hooks = []
def __init__(self, gen_model,disc_model,use_cuda_if_available=True):
self.model_dir = os.getcwd()
self.gen_model = gen_model
self.disc_model = disc_model
self.cuda = False
if use_cuda_if_available and cuda.is_available():
self.cuda = True
self.gen_loss_history = []
self.disc_loss_history = []
self.__input_hooks = []
def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, bias=False, momentum=0.01,
dilation=1, groups=1, device = 'cuda:0' if cuda.is_available() else 'cpu',norm_layer=nn.BatchNorm2d, **kwargs):
super(ConvBNMish, self).__init__()
self.cb = ConvBN(in_channels, out_channels, kernel_size, stride, padding, bias, momentum, dilation, groups, device, norm_layer)
self.act = Mish()
top = (height - new_height) / 2
right = (width + new_width) / 2
bottom = (height + new_height) / 2
img = img.crop((left, top, right, bottom))
img = np.array(img).transpose((2, 0, 1)) / 256
means = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1))
stds = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1))
img = img - means
img = img / stds
img_tensor = torch.Tensor(img)
return img_tensor
checkpoint_path = 'vgg16-transfer-4.pth'
train_on_gpu = cuda.is_available()
if train_on_gpu:
gpu_count = cuda.device_count()
if gpu_count > 1:
multi_gpu = True
else:
multi_gpu = False
loaded_model, optimizer = load_checkpoint(path=checkpoint_path)
print(predict("pictures/test.jpg", loaded_model))
model_name = 'resnet50V4'
data_root = '/Users/yefeichen/Database/furniture/collect_from_matterport_all0823/'
data_root_ext = '.png'
test_root = '/Users/yefeichen/Database/furniture/chair_from_digital/'
test_root_ext = '.jpg'
# N_Cls = 109
N_Cls = 253
model_root = Path(model_root)
model = getattr(models, model_name)(num_classes=N_Cls, pretrained='imagenet')
# model = torch.nn.DataParallel(model)
use_cuda = cuda.is_available()
if use_cuda:
model = model.cuda()
model_path = os.path.join(model_root, ckpt)
load_par_gpu_model_cpu(model, model_path)
print('weights loaded!')
fcnt = 0
ecnt = 0
#loading npy
featList = []
nameList = []
preprocess.process_fact(fact)
print("Vocab size = {}".format(preprocess.vocab_size))
# Define dimensions
INPUT_DIM = preprocess.vocab_size
OUTPUT_DIM = preprocess.vocab_size # This is the output vocabulary size
EMBEDDING_DIM = 100
HIDDEN_DIM = 100
MAX_CONTEXT_LEN = 60
MAX_FACT_LEN = 60
MAX_DESC_LEN = 60
BATCH_SIZE = 100
USE_CUDA = False
if cuda.is_available():
USE_CUDA = True
#****************************** INPUT MODULE ********************************************
#****************************************************************************************
class PositionalFactEncoder(nn.Module):
def __init__(self):
super(PositionalFactEncoder, self).__init__()
def forward(self, embedded_sentence, fact_lengths):
_, slen, elen = embedded_sentence.size()
l = [[(1 - s/(slen-1)) - (e/(elen-1)) * (1 - 2*s/(slen-1)) for e in range(elen)] for s in range(slen)]
l = torch.FloatTensor(l)
l = l.unsqueeze(0)
from utils.datasets import UsageBasedDataset, RecognitionDataset
train_root = args.t
eval_root = args.e
eval_indices = args.ei
eval_files = args.ef
logfile = args.l
save = args.w
best = args.b
force_cuda = args.nc
# Construct model, optimizer, and criterion
darknet19: RPYOLOv2 = RPYOLOv2()
darknet19.recognizing()
if force_cuda and cuda.is_available():
darknet19 = darknet19.cuda()
adam = Adam(darknet19.parameters(), lr=.0001, weight_decay=.0005)
crit = CrossEntropyLoss()
# Construct dataset variables
image_resolution = (224, 224)
# Construct training dataset and loader
train_transform = Compose([
Lambda(maybe_blur),
Lambda(maybe_darken_a_lot),
Lambda(maybe_rotate),
Lambda(maybe_random_perspective),
Lambda(maybe_random_crop),
Lambda(maybe_random_erase),
def validate_model(Net,
seed,
mini_batch_size=100,
optimizer=optim.Adam,
criterion=nn.CrossEntropyLoss(),
n_epochs=40,
eta=1e-3,
lambda_l2=0,
alpha=0.5,
beta=0.5,
plot=True,
rotate=False,
translate=False,
swap_channel=False,
GPU=False):
"""
General :
- Train a network model which weights has been initialized with a specific seed over n_epochs
- Data is created with the same seed : train,validation and test calling the prologue
- Record the train and validation accuracy and loss and can display they evolution curve
Input :
- Net : A network dictionnary from the <Nets> class
- seed : seed for pseudo random number generator used in weight initialization and data loading
-> mini_batch_size,optimizer, criterion, n_epochs, eta, lambda_2, alpha, beta see training.py
- plot : if true plot the learning curve evolution over the epochs -> default true
-> rotate,translate and swap_channels -> data augmentation see loader.py
Output : printed loss and accuracy of the network after training on the test set and learning curve if plot true
"""
# set the pytorch seed
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
# set the seed for random spliting of the dataset in training and validation
random.seed(0)
# create the dataset
data = PairSetMNIST()
train_data = Training_set(data)
test_data = Test_set(data)
train_data_split = Training_set_split(train_data, rotate, translate,
swap_channel)
validation_data = Validation_set(train_data)
# construct the net type with default parameter
if (Net['net_type'] == 'Net2c'):
model = Net['net'](nb_hidden=Net['hidden_layers'],
dropout_prob=Net['drop_prob'])
if (Net['net_type'] == 'LeNet_sharing'):
model = Net['net'](nb_hidden=Net['hidden_layers'],
dropout_ws=Net['drop_prob_ws'],
dropout_comp=Net['drop_prob_comp'])
if (Net['net_type'] == 'LeNet_sharing_aux'):
# check if any data augmentation has been called
# if none construct with tuned parameters without data augmentation
# if yes construct with tuned parameters with data augmentation
if (rotate == False and translate == False and swap_channel == False):
model = Net['net'](nbhidden_aux=Net['hidden_layers_aux'],
nbhidden_comp=Net['hidden_layers_comp'],
drop_prob_aux=Net['drop_prob_aux'],
drop_prob_comp=Net['drop_prob_comp'])
else:
Net['learning rate'] = Net['learning rate augm']
model = Net['net'](nbhidden_aux=Net['hidden_layers_aux'],
nbhidden_comp=Net['hidden_layers_comp'],
drop_prob_aux=Net['drop_prob_aux_augm'],
drop_prob_comp=Net['drop_prob_comp_augm'])
if (Net['net_type'] == 'Google_Net'):
model = Net['net'](channels_1x1=Net['channels_1x1'],
channels_3x3=Net['channels_3x3'],
channels_5x5=Net['channels_5x5'],
pool_channels=Net['pool_channels'],
nhidden=Net['hidden_layers'],
drop_prob_comp=Net['drop_prob_comp'],
drop_prob_aux=Net['drop_prob_aux'])
if GPU and cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
model = model.to(device)
# train the model on the train set and validate at each epoch
train_losses, train_acc, valid_losses, valid_acc = train_model(
model, train_data_split, validation_data, device, mini_batch_size,
optimizer, criterion, n_epochs, Net['learning rate'], lambda_l2, alpha,
beta)
if plot:
learning_curve(train_losses, train_acc, valid_losses, valid_acc)
# loss and accuracy of the network on the test
test_loss, test_accuracy = compute_metrics(model, test_data, device)
print('\nTest Set | Loss: {:.4f} | Accuracy: {:.2f}%\n'.format(
test_loss, test_accuracy))
import torch
import dataset
import model
import cv2
import os
import torch.nn as nn
import numpy as np
import time
import torch.cuda as cuda
import glob
if __name__ == '__main__':
gpu_avail = cuda.is_available()
os.environ["CUDA_VISIBLE_DEVICES"] = "3"
scale = 0.8
img_dir = '/home/zhangtk/data/test/0/'
# img_dir = r'E:\data\rain_full\rain_imgs\test\2'
# img_list = os.listdir(img_dir)
img_list_ = glob.glob(os.path.join(img_dir, '*.jpg'))
img_list = [os.path.basename(i) for i in img_list_]
# save_dir = f'/home/zhangtk/data/test/0_test_ztkvgg1_{scale}/'
save_dir = os.path.join(img_dir, 'result_mobilenet_conv1x1_new_930')
if not os.path.exists(save_dir):
os.makedirs(save_dir)
start = time.time()
def main():
#cmd and arg parser
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('--mode', choices=['input_grad'], default='input_grad')
arg('--run_root', default='result/location_recommend_model_v6_5city_191113')
arg('--fold', type=int, default=0)
arg('--model', default='location_recommend_model_v6')
arg('--ckpt', type=str, default='model_loss_best.pt')
arg('--pretrained', type=str, default='imagenet')#resnet 1, resnext imagenet
arg('--batch-size', type=int, default=1)
arg('--step', type=str, default=8)#update the gradients every 8 batch(sample num = step*batch-size*inner_size)
arg('--workers', type=int, default=16)
arg('--lr', type=float, default=3e-4)
arg('--patience', type=int, default=4)
arg('--clean', action='store_true')
arg('--n-epochs', type=int, default=1)
arg('--epoch-size', type=int)
arg('--tta', type=int, default=1)
arg('--use-sample', action='store_true', help='use a sample of the dataset')
arg('--debug', action='store_true')
arg('--imgsize',type=int, default = 256)
arg('--finetuning',action='store_true')
arg('--cos_sim_loss',action='store_true')
arg('--ensemble', action='store_true')
arg('--sample_rate',type=float,default=1.0)#sample part of testing data for evaluating during training
arg('--testStep',type=int,default=500000)
arg('--query_location',action='store_true',help='use location as query')
arg('--apps',type=str,default='_191113.csv')
arg('--pre_name', type=str, default='sampled_ww_')
#cuda version T/F
use_cuda = cuda.is_available()
args = parser.parse_args()
#run_root: model/weights root
run_root = Path(args.run_root)
global model_name
model_name = args.model
df_comp_feat = pd.read_csv(pjoin(TR_DATA_ROOT,'company_feat'+args.apps),index_col=0)
df_loc_feat = pd.read_csv(pjoin(TR_DATA_ROOT,'location_feat'+args.apps),index_col=0)
feat_name = list(df_comp_feat.columns)[1:]+list(df_loc_feat.columns)[1:]
print(len(feat_name))
clfile = ['PA', 'SF', 'SJ', 'LA', 'NY']
cfile = ['dnb_pa.csv', 'dnb_sf.csv', 'dnb_sj.csv', 'dnb_Los_Angeles.csv', 'dnb_New_York.csv']
lfile = 'location_scorecard_191113.csv'
clfile = [c + args.apps for c in clfile]
pre_name = args.pre_name
pred_save_name = [ pre_name + c.replace(args.apps,'') + '_similarity'+args.apps for c in clfile ]
#Dont use ensemble score
df_ensemble = pd.DataFrame(columns=['Blank'])
loc_name_dict = translocname2dict(df_loc_feat)
print('Location Embedding Number: %d'%len(loc_name_dict))
##::DataLoader
def make_loader(df_comp_feat: pd.DataFrame, df_loc_feat: pd.DataFrame, df_pair: pd.DataFrame, emb_dict:dict,df_ensemble,
name='train',flag_ensemble=args.ensemble,testStep=args.testStep,shuffle=True) -> DataLoader:
return DataLoader(
TrainDatasetLocationRS(df_comp_feat=df_comp_feat, df_loc_feat=df_loc_feat, df_pair=df_pair,df_ensemble_score=df_ensemble,
emb_dict=emb_dict, name=name,flag_ensemble=flag_ensemble,
negN=nNegTr, posN=nPosTr, testStep=testStep),
shuffle=shuffle,
batch_size=args.batch_size,
num_workers=args.workers,
collate_fn=collate_TrainDatasetLocationRS
)
#Not used in this version
criterion = softmax_loss
lossType = 'softmax'
# se- ception dpn can only use finetuned model from imagenet
# model = getattr(models, args.model)(feat_comp_dim=102, feat_loc_dim=23) #location_recommend_model_v1
model = getattr(rsmodels, args.model)(feat_comp_dim=102,feat_loc_dim=23,embedding_num=len(loc_name_dict)) #location_recommend_model_v3
md_path = Path(str(run_root) + '/' + args.ckpt)
if md_path.exists():
print('load weights from md_path')
load_model(model, md_path)
model.freeze()
all_params = filter(lambda p: p.requires_grad, model.parameters())
#gpu first
if use_cuda:
model = model.cuda()
#print(model)
if args.mode == 'input_grad':
if run_root.exists() and args.clean:
shutil.rmtree(run_root)
run_root.mkdir(exist_ok=True, parents=True)
Path(str(run_root) + '/params.json').write_text(
json.dumps(vars(args), indent=4, sort_keys=True))
for ind_city in range(1):
print('Operating %s...'%pred_save_name[ind_city])
testing_pair = pd.read_csv(pjoin(TR_DATA_ROOT, pred_save_name[ind_city]))[['atlas_location_uuid', 'duns_number']]
testing_pair['label'] = 0
testing_pair = testing_pair[['duns_number', 'atlas_location_uuid','label']]
predict_loader = make_loader(df_comp_feat=df_comp_feat, df_loc_feat=df_loc_feat, df_pair=testing_pair,
emb_dict=loc_name_dict,df_ensemble=df_ensemble, name='valid',shuffle=False)
predict_with_reason(model=model,criterion=criterion,predict_loader=predict_loader,use_cuda=use_cuda,test_pair=testing_pair,
feat_name=feat_name,save_name=pred_save_name[ind_city],pre_name='dlsub2_')
def get_model_ddp(
cls,
rank,
local_leader_rank,
log,
net_version,
pretrained,
freeze,
): # @DontTrace
'''
Determine whether this process is the
master node. If so, obtain the pretrained
resnet18 model. Then distributed the model
to the other nodes.
:param rank: this process' rank
in the distributed data processing sense
:type rank: int
:param local_leader_rank: the lowest rank on this machine
:type local_leader_rank: int
:param log: logging service to log to
:type log: LoggingService
:param net_version: which resnet version to obtain
:type net_version: int
:param pretrained: if true, the pre-trained version
is obtained. Else initial weights are undefined
:type pretrained: bool
:param freeze: how many layers to
freeze, protecting them from training.
:type freeze: int
'''
if net_version not in (18, 50):
raise ValueError("Resnet version must be 18 or 50")
hostname = socket.gethostname()
# Let the local leader download
# the model from the Internet,
# in case it is not already cached
# locally:
# Case 1: not on a GPU machine:
device = device('cuda' if cuda.is_available() else 'cpu')
if device == device('cpu'):
model = hub.load('pytorch/vision:v0.6.0',
'resnet18' if net_version == 18 else 'resnet50',
pretrained=pretrained)
# Case2a: GPU machine, and this is this machine's
# leader process. So it is reponsible for
# downloading the model if it is not cached:
elif rank == local_leader_rank:
log.info(f"Procss with rank {rank} on {hostname} loading model")
model = hub.load('pytorch/vision:v0.6.0',
'resnet18' if net_version == 18 else 'resnet50',
pretrained=pretrained)
# Allow the others on this machine
# to load the model (guaranteed to
# be locally cached now):
log.info(
f"Procss with rank {rank} on {hostname} waiting for others to laod model"
)
dist.barrier()
# Case 2b: GPU machine, but not the local leader. Just
# wait for the local leader to be done downloading:
else:
# Wait for leader to download the
# model for everyone on this machine:
log.info(
f"Process with rank {rank} on {hostname} waiting for leader to laod model"
)
dist.barrier()
# Get the cached version:
log.info(f"Procss with rank {rank} on {hostname} laoding model")
model = hub.load('pytorch/vision:v0.6.0',
'resnet18' if net_version == 18 else 'resnet50',
pretrained=pretrained)
model = cls.freeze_model_layers(model, freeze)
return model
def test():
voc_size = 233
timesteps = 13
batch = 311
hidden_size = 141
num_layers = 2
epsilon = Epsilon(0, 0)
storage = Storage(0, multistep=True, tdlambda=0.9)
step = Step(1)
sReward = 0
on = "cuda" if cuda.is_available() else "cpu"
print("on: {}".format(on))
lr = 1e-3
SOS = 0
PAD = 1
dis = D(voc_size, hidden_size, on=on, num_layers=1).to(on)
gen = G(
voc_size,
hidden_size,
timesteps,
on=on,
sos_pad=(SOS, PAD),
num_layers=num_layers,
).to(on)
rec = R(voc_size,
hidden_size,
timesteps,
on=on,
sos=SOS,
num_layers=num_layers).to(on)
scr = S(voc_size, hidden_size, on=on, num_layers=1).to(on)
do = SGD(dis.parameters(), lr=lr)
go = SGD(gen.parameters(), lr=lr)
ro = SGD(rec.parameters(), lr=lr)
so = SGD(scr.parameters(), lr=lr)
print("S")
summarize_input(
gen,
wi_iw=[
{
"0": 0,
"1": 1,
"2": 2,
"__UNK__": "u",
"__PAD__": 99
},
[str(i) for i in range(voc_size)],
],
)
print("F")
F_train_one_batch(
(
torch.randint(0, voc_size, (batch, timesteps)).to(on),
torch.randint(0, voc_size, (batch, timesteps)).to(on),
torch.randint(0, 5, (batch, )).to(on),
),
dis,
gen,
gen,
rec,
scr,
do,
go,
ro,
so,
F.binary_cross_entropy,
F.mse_loss,
F.cross_entropy,
F.cross_entropy,
epsilon,
step,
storage,
sReward,
"weight",
)
print("N")
class TestDataset(Dataset):
def __init__(self, low, high, shape):
self.data = torch.randint(low, high, shape)
def __len__(self):
return len(self.data)
def __getitem__(self, index):
return self.data[index]
def to(self, device):
self.data = self.data.to(device)
return self
dataset = TestDataset(0, voc_size, (timesteps, batch)).to(on)
print(dataset[:].shape)
train_one_batch(
dataset[:],
dis,
gen,
gen,
rec,
do,
go,
ro,
F.binary_cross_entropy,
F.mse_loss,
F.cross_entropy,
epsilon,
step,
storage,
sReward,
"weight",
)
def __init__(self, in_channels, out_channels, kernel_size=1, stride=1, padding=0, bias=False, momentum=0.01, dilation=1, groups=1,
device = 'cuda:0' if cuda.is_available() else 'cpu', norm_layer=nn.BatchNorm2d, **kwargs):
super(ConvBN, self).__init__(
nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, groups=groups, bias=False),
nn.BatchNorm2d(out_channels, momentum=momentum),
)
def train_model(model: Module, optimizer: optim.Optimizer, epochs: int):
"""Train vanilla cnn pytorch model.
Args:
model: Module, pytorch model
epochs: int, training epochs
Returns:
model: Module, trained model
history: dict, history
"""
data_transforms = {
'train':
transforms.Compose([
transforms.Resize((128, 128)),
transforms.RandomHorizontalFlip(),
transforms.RandomCrop(120),
transforms.Resize((128, 128)),
transforms.RandomRotation(40),
transforms.ToTensor(),
]),
'val':
transforms.Compose([
transforms.Resize((128, 128)),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(10),
transforms.ToTensor(),
])
}
data_dir = '../../data/AsianSampleCategory/'
image_datasets = {
x: ImageFolder(os.path.join(data_dir, x), data_transforms[x])
for x in ['train', 'val']
}
dataloaders = {
x: DataLoader(image_datasets[x], batch_size=32, shuffle=True)
for x in ['train', 'val']
}
data_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}
use_gpu = cuda.is_available()
criterion = CrossEntropyLoss()
# Observe that all parameters are being optimized
# optimizer = optim.Adam(model.parameters(), lr=1e-4)
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0
history = {
'loss': {
'train': [],
'val': []
},
'acc': {
'train': [],
'val': []
}
}
for epoch in range(epochs):
print('Epoch {}/{}:\t'.format(epoch, epochs - 1), end='')
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
model.train(True) # Set model to training mode
else:
model.train(False) # Set model to evaluate mode
running_loss = 0.0
running_corrects = 0
# Iterate over data.
for data in dataloaders[phase]:
# get the inputs
inputs, labels = data
if use_gpu:
inputs = Variable(inputs.cuda())
labels = Variable(labels.cuda())
model = model.cuda()
else:
inputs, labels = Variable(inputs), Variable(labels)
# zero the parameter gradients
optimizer.zero_grad()
# forward
outputs = model(inputs)
_, preds = torch.max(outputs.data, 1)
loss = criterion(outputs, labels)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.data[0] * inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / data_sizes[phase]
epoch_acc = running_corrects / data_sizes[phase]
history['loss'][phase].append(epoch_loss)
history['acc'][phase].append(epoch_acc)
print('{} Loss: {:.4f} Acc: {:.4f}\t'.format(
phase, epoch_loss, epoch_acc),
end='')
# deep copy the model
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
print()
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
# load best model weights
model.load_state_dict(best_model_wts)
return model, history
def linux_with_gpu():
"""Returns if machine is running an Linux OS and has a GPU"""
has_gpu = is_available()
return is_linux() and has_gpu
def train(
self,
base_path: Union[Path, str],
sequence_length: int,
learning_rate: float = 20,
mini_batch_size: int = 100,
anneal_factor: float = 0.25,
patience: int = 10,
clip=0.25,
max_epochs: int = 1000,
checkpoint: bool = False,
grow_to_sequence_length: int = 0,
num_workers: int = 2,
use_amp: bool = False,
amp_opt_level: str = "O1",
**kwargs,
):
if use_amp:
if sys.version_info < (3, 0):
raise RuntimeError(
"Apex currently only supports Python 3. Aborting.")
if amp is None:
raise RuntimeError(
"Failed to import apex. Please install apex from https://www.github.com/nvidia/apex "
"to enable mixed-precision training.")
# cast string to Path
if type(base_path) is str:
base_path = Path(base_path)
add_file_handler(log, base_path / "training.log")
number_of_splits: int = len(self.corpus.train)
val_data = self._batchify(self.corpus.valid, mini_batch_size)
# error message if the validation dataset is too small
if val_data.size(0) == 1:
raise RuntimeError(
f"ERROR: Your validation dataset is too small. For your mini_batch_size, the data needs to "
f"consist of at least {mini_batch_size * 2} characters!")
base_path.mkdir(parents=True, exist_ok=True)
loss_txt = base_path / "loss.txt"
savefile = base_path / "best-lm.pt"
try:
best_val_loss = self.loss
optimizer = self.optimizer(self.model.parameters(),
lr=learning_rate,
**kwargs)
if self.optimizer_state is not None:
optimizer.load_state_dict(self.optimizer_state)
if isinstance(optimizer, (AdamW, SGDW)):
scheduler: ReduceLRWDOnPlateau = ReduceLRWDOnPlateau(
optimizer,
verbose=True,
factor=anneal_factor,
patience=patience)
else:
scheduler: ReduceLROnPlateau = ReduceLROnPlateau(
optimizer,
verbose=True,
factor=anneal_factor,
patience=patience)
if use_amp:
self.model, optimizer = amp.initialize(self.model,
optimizer,
opt_level=amp_opt_level)
training_generator = DataLoader(self.corpus.train,
shuffle=False,
num_workers=num_workers)
for epoch in range(self.epoch, max_epochs):
epoch_start_time = time.time()
# Shuffle training files randomly after serially iterating through corpus one
if epoch > 0:
training_generator = DataLoader(self.corpus.train,
shuffle=True,
num_workers=num_workers)
self.model.save_checkpoint(
base_path / f"epoch_{epoch}.pt",
optimizer,
epoch,
0,
best_val_loss,
)
# iterate through training data, starting at self.split (for checkpointing)
for curr_split, train_slice in enumerate(
training_generator, self.split):
if sequence_length < grow_to_sequence_length:
sequence_length += 1
log.info(f"Sequence length is {sequence_length}")
split_start_time = time.time()
# off by one for printing
curr_split += 1
train_data = self._batchify(train_slice.flatten(),
mini_batch_size)
log.info(
"Split %d" % curr_split +
"\t - ({:%H:%M:%S})".format(datetime.datetime.now()))
for group in optimizer.param_groups:
learning_rate = group["lr"]
# go into train mode
self.model.train()
# reset variables
hidden = self.model.init_hidden(mini_batch_size)
# not really sure what this does
ntokens = len(self.corpus.dictionary)
total_loss = 0
start_time = time.time()
for batch, i in enumerate(
range(0,
train_data.size(0) - 1, sequence_length)):
data, targets = self._get_batch(
train_data, i, sequence_length)
if not data.is_cuda and cuda.is_available():
log.info(
"Batch %d is not on CUDA, training will be very slow"
% (batch))
raise Exception("data isnt on cuda")
self.model.zero_grad()
optimizer.zero_grad()
# do the forward pass in the model
output, rnn_output, hidden = self.model.forward(
data, hidden)
# try to predict the targets
loss = self.loss_function(output.view(-1, ntokens),
targets)
# Backward
if use_amp:
with amp.scale_loss(loss,
optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm_(self.model.parameters(),
clip)
optimizer.step()
total_loss += loss.data
# We detach the hidden state from how it was previously produced.
# If we didn't, the model would try backpropagating all the way to start of the dataset.
hidden = self._repackage_hidden(hidden)
# explicitly remove loss to clear up memory
del loss, output, rnn_output
if batch % self.log_interval == 0 and batch > 0:
cur_loss = total_loss.item() / self.log_interval
elapsed = time.time() - start_time
log.info(
"| split {:3d} /{:3d} | {:5d}/{:5d} batches | ms/batch {:5.2f} | "
"loss {:5.2f} | ppl {:8.2f}".format(
curr_split,
number_of_splits,
batch,
len(train_data) // sequence_length,
elapsed * 1000 / self.log_interval,
cur_loss,
math.exp(cur_loss),
))
total_loss = 0
start_time = time.time()
log.info("%d seconds for train split %d" %
(time.time() - split_start_time, curr_split))
###############################################################################
self.model.eval()
val_loss = self.evaluate(val_data, mini_batch_size,
sequence_length)
scheduler.step(val_loss)
log.info("best loss so far {:5.2f}".format(best_val_loss))
log.info(self.model.generate_text())
if checkpoint:
self.model.save_checkpoint(
base_path / "checkpoint.pt",
optimizer,
epoch,
curr_split,
best_val_loss,
)
# Save the model if the validation loss is the best we've seen so far.
if val_loss < best_val_loss:
self.model.best_score = best_val_loss
self.model.save(savefile)
best_val_loss = val_loss
###############################################################################
# print info
###############################################################################
log.info("-" * 89)
summary = (
"| end of split {:3d} /{:3d} | epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | "
"valid ppl {:8.2f} | learning rate {:3.4f}".format(
curr_split,
number_of_splits,
epoch + 1,
(time.time() - split_start_time),
val_loss,
math.exp(val_loss),
learning_rate,
))
with open(loss_txt, "a") as myfile:
myfile.write("%s\n" % summary)
log.info(summary)
log.info("-" * 89)
log.info("Epoch time: %.2f" % (time.time() - epoch_start_time))
except KeyboardInterrupt:
log.info("-" * 89)
log.info("Exiting from training early")
###############################################################################
# final testing
###############################################################################
test_data = self._batchify(self.corpus.test, mini_batch_size)
test_loss = self.evaluate(test_data, mini_batch_size, sequence_length)
summary = "TEST: valid loss {:5.2f} | valid ppl {:8.2f}".format(
test_loss, math.exp(test_loss))
with open(loss_txt, "a") as myfile:
myfile.write("%s\n" % summary)
log.info(summary)
log.info("-" * 89)
def evaluate(
self,
num_samples: int = 10,
report_every: int = 100,
train_or_test: str = "test",
) -> None:
""" eval code for validation/test set and saves the evaluation results in self.results.
Args:
num_samples: number of samples (clips) of the validation set to test
report_every: print line of results every n times
train_or_test: use train or test set
"""
# asset train or test valid
assert train_or_test in ["train", "test"]
# set device and num_gpus
num_gpus = num_devices()
device = torch_device()
torch.backends.cudnn.benchmark = True if cuda.is_available() else False
# init model with gpu (or not)
self.model.to(device)
if num_gpus > 1:
self.model = nn.DataParallel(self.model)
self.model.eval()
# set train or test
ds = (
self.dataset.test_ds
if train_or_test == "test"
else self.dataset.train_ds
)
# set num_samples
ds.dataset.num_samples = num_samples
print(
f"{len(self.dataset.test_ds)} samples of {self.dataset.test_ds[0][0][0].shape}"
)
# Loop over all examples in the test set and compute accuracies
ret = dict(
infer_times=[],
video_preds=[],
video_trues=[],
clip_preds=[],
clip_trues=[],
)
report_every = 100
# inference
with torch.no_grad():
for i in range(
1, len(ds)
): # [::10]: # Skip some examples to speed up accuracy computation
if i % report_every == 0:
print(
f"Processsing {i} of {len(self.dataset.test_ds)} samples.."
)
# Get model inputs
inputs, label = ds[i]
inputs = inputs.to(device, non_blocking=True)
# Run inference
start_time = time()
outputs = self.model(inputs)
outputs = outputs.cpu().numpy()
infer_time = time() - start_time
ret["infer_times"].append(infer_time)
# Store results
ret["video_preds"].append(outputs.sum(axis=0).argmax())
ret["video_trues"].append(label)
ret["clip_preds"].extend(outputs.argmax(axis=1))
ret["clip_trues"].extend([label] * num_samples)
print(
f"Avg. inference time per video ({len(ds)} clips) =",
round(np.array(ret["infer_times"]).mean() * 1000, 2),
"ms",
)
print(
"Video prediction accuracy =",
round(accuracy_score(ret["video_trues"], ret["video_preds"]), 2),
)
print(
"Clip prediction accuracy =",
round(accuracy_score(ret["clip_trues"], ret["clip_preds"]), 2),
)
return ret
#Transformations and data augmentation
transformations = transforms.Compose([
transforms.Resize(28),
transforms.ToTensor(),
transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))
])
#Load the training and test sets
train_set = MNIST(root="./data",transform=transformations,download=True)
test_set = MNIST(root="./data",train=False,transform=transformations,download=True)
train_loader = DataLoader(train_set,shuffle=True,batch_size=batch_size,num_workers=4)
test_loader = DataLoader(test_set,shuffle=False,batch_size=batch_size,num_workers=4)
#Move to GPU if available
if cuda.is_available():
net.cuda()
#Setup the optimize and a loss function
optimizer = Adam(net.parameters(),lr=0.001)
loss_fn = nn.CrossEntropyLoss()
#Top 1 Train accuracy
train_metrics = tf.Accuracy(topK=1)
#Top 1 and Top 2 train accuracy
test_metrics_top1 = tf.Accuracy(name="Top 1 Acc ",topK=1)
test_metrics_top2 = tf.Accuracy(name="Top 2 Acc ",topK=2)
#Create an instance of the StandardModel
model = tf.StandardModel(net)
def fit(self, train_cfgs):
train_cfgs = Config(train_cfgs)
model_dir = train_cfgs.get('model_dir', "checkpoints")
os.makedirs(model_dir, exist_ok=True)
if cuda.is_available():
device = torch.device("cuda")
num_devices = cuda.device_count()
# Look for the optimal set of algorithms to use in cudnn. Use this only with fixed-size inputs.
torch.backends.cudnn.benchmark = True
else:
device = torch.device("cpu")
num_devices = 1
data_loaders = {}
if self.train_ds is not None:
data_loaders['train'] = DataLoader(
self.train_ds,
batch_size=train_cfgs.get('batch_size', 8) * num_devices,
shuffle=True,
num_workers=
0, # Torch 1.2 has a bug when num-workers > 0 (0 means run a main-processor worker)
pin_memory=True,
)
if self.valid_ds is not None:
data_loaders['valid'] = DataLoader(
self.valid_ds,
batch_size=train_cfgs.get('batch_size', 8) * num_devices,
shuffle=False,
num_workers=0,
pin_memory=True,
)
# Move model to gpu before constructing optimizers and amp.initialize
self.model.to(device)
named_params_to_update = {}
total_params = 0
for name, param in self.model.named_parameters():
total_params += 1
if param.requires_grad:
named_params_to_update[name] = param
print("Params to learn:")
if len(named_params_to_update) == total_params:
print("\tfull network")
else:
for name in named_params_to_update:
print("\t{}".format(name))
optimizer = optim.SGD(
list(named_params_to_update.values()),
lr=train_cfgs.lr,
momentum=train_cfgs.momentum,
weight_decay=train_cfgs.weight_decay,
)
# Use mixed-precision if available
# Currently, only O1 works with DataParallel: See issues https://github.com/NVIDIA/apex/issues/227
if train_cfgs.get('mixed_prec', False) and AMP_AVAILABLE:
# 'O0': Full FP32, 'O1': Conservative, 'O2': Standard, 'O3': Full FP16
self.model, optimizer = amp.initialize(
self.model,
optimizer,
opt_level="O1",
loss_scale="dynamic",
# keep_batchnorm_fp32=True doesn't work on 'O1'
)
# Learning rate scheduler
scheduler = None
warmup_pct = train_cfgs.get('warmup_pct', None)
lr_decay_steps = train_cfgs.get('lr_decay_steps', None)
if warmup_pct is not None:
# Use warmup with the one-cycle policy
lr_decay_total_steps = train_cfgs.epochs if lr_decay_steps is None else lr_decay_steps
scheduler = torch.optim.lr_scheduler.OneCycleLR(
optimizer,
max_lr=train_cfgs.lr,
total_steps=lr_decay_total_steps,
pct_start=train_cfgs.get('warmup_pct', 0.3),
base_momentum=0.9 * train_cfgs.momentum,
max_momentum=train_cfgs.momentum,
final_div_factor=1 / train_cfgs.get('lr_decay_factor', 0.0001),
)
elif lr_decay_steps is not None:
lr_decay_total_steps = train_cfgs.epochs
# Simple step-decay
scheduler = torch.optim.lr_scheduler.StepLR(
optimizer,
step_size=lr_decay_steps,
gamma=train_cfgs.get('lr_decay_factor', 0.1),
)
# DataParallel after amp.initialize
if num_devices > 1:
model = nn.DataParallel(self.model)
else:
model = self.model
criterion = nn.CrossEntropyLoss().to(device)
for e in range(1, train_cfgs.epochs + 1):
print("Epoch {} ==========".format(e))
if scheduler is not None:
print("lr={}".format(scheduler.get_lr()))
self.train_an_epoch(
model,
data_loaders,
device,
criterion,
optimizer,
grad_steps=train_cfgs.grad_steps,
mixed_prec=train_cfgs.mixed_prec,
)
if scheduler is not None and e < lr_decay_total_steps:
scheduler.step()
self.save(
os.path.join(
model_dir,
"{model_name}_{epoch}.pt".format(model_name=train_cfgs.get(
'model_name', self.model_name),
epoch=str(e).zfill(3))))
def __init__(self, out_planes, number, kernel_size=1, bias=True, device = 'cuda:0' if cuda.is_available() else 'cpu', norm_layer = nn.BatchNorm2d):
super(SEModule, self).__init__()
self.bn = norm_layer
self.avg_pool = nn.AdaptiveAvgPool2d(1)
self.fc = nn.Sequential(
nn.Conv2d(out_planes, number, kernel_size=1, bias=bias),
nn.Conv2d(number, out_planes, kernel_size=1, bias=bias),
Sigmoid()
)
def evaluate_model(Net,
seeds,
mini_batch_size=100,
optimizer=optim.Adam,
criterion=nn.CrossEntropyLoss(),
n_epochs=40,
eta=1e-3,
lambda_l2=0,
alpha=0.5,
beta=0.5,
plot=True,
statistics=True,
rotate=False,
translate=False,
swap_channel=False,
GPU=False):
"""
General : 10 rounds of network training / validation with statistics
- Repeat the training/validation procedure 10 times for ten different seeds in seeds
1) At every seed -> reinitializes a network and a dataset : train,validation and test
2) Weights initialization and data loading are using the seed
3) Record the train and validation accuracy and loss and can display their evolution curve
4) Compute the statistics at the end of each training for performance evaluation
i) Mean training accuracy for each seed -> value at the end of the last epoch
ii) Mean validation accuracy for each seed -> value at the end of the last epoch
iii) Mean test accuracy for each seed -> compute the accuracy on the test after each training
-> display a boxplot of the statistics if statistics is true and print the mean and standard deviation
Input :
- Net : A network dictionnary from the <Nets> class
- seeds : a list of seed to iterate over for pseudo random number generator used in weight initialization and data loading
-> mini_batch_size,optimizer, criterion, n_epochs, eta, lambda_2, alpha, beta see training.py
- plot : if true plot the learning curve evolution over the epochs -> default true
- statistics : if true display the boxplot of the train accuracies, validations and test and print the mean and standard deviation
statistics
-> rotate,translate and swap_channels -> data augmentation see loader.py
Output :
- train_result : A (10x4xn_epochs) tensor
10 -> seed
4 -> train loss ,train accuracy, validation loss, validation accuracy
n_epochs -> evolution during training
- test_losses : A tensor of shape (10,) containing the test loss at each seed
- test_accuracies : A tensor of shape (10,) containing the test loss at each seed
"""
# tensor initialization to store the metrics
train_results = torch.empty(len(seeds), 4, n_epochs)
test_losses = []
test_accuracies = []
for n, seed in enumerate(seeds):
# set the pytorch seed
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
# set the seed for random spliting of the dataset in training and validation
random.seed(0)
# load the dataset train,validation and test
data = PairSetMNIST()
train_data = Training_set(data)
test_data = Test_set(data)
train_data_split = Training_set_split(train_data, rotate, translate,
swap_channel)
validation_data = Validation_set(train_data)
# construct the net type with default parameter
if (Net['net_type'] == 'Net2c'):
model = Net['net'](nb_hidden=Net['hidden_layers'],
dropout_prob=Net['drop_prob'])
if (Net['net_type'] == 'LeNet_sharing'):
model = Net['net'](nb_hidden=Net['hidden_layers'],
dropout_ws=Net['drop_prob_ws'],
dropout_comp=Net['drop_prob_comp'])
if (Net['net_type'] == 'LeNet_sharing_aux'):
# check if any data augmentation has been called
# if none construct with tuned parameters without data augmentation
# if yes construct with tuned parameters with data augmentation
if (rotate == False and translate == False
and swap_channel == False):
model = Net['net'](nbhidden_aux=Net['hidden_layers_aux'],
nbhidden_comp=Net['hidden_layers_comp'],
drop_prob_aux=Net['drop_prob_aux'],
drop_prob_comp=Net['drop_prob_comp'])
else:
Net['learning rate'] = Net['learning rate augm']
model = Net['net'](nbhidden_aux=Net['hidden_layers_aux'],
nbhidden_comp=Net['hidden_layers_comp'],
drop_prob_aux=Net['drop_prob_aux_augm'],
drop_prob_comp=Net['drop_prob_comp_augm'])
if (Net['net_type'] == 'Google_Net'):
model = Net['net'](channels_1x1=Net['channels_1x1'],
channels_3x3=Net['channels_3x3'],
channels_5x5=Net['channels_5x5'],
pool_channels=Net['pool_channels'],
nhidden=Net['hidden_layers'],
drop_prob_comp=Net['drop_prob_comp'],
drop_prob_aux=Net['drop_prob_aux'])
if GPU and cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
model = model.to(device)
# train the model on the train set and validate at each epoch
train_losses, train_acc, valid_losses, valid_acc = train_model(
model, train_data_split, validation_data, device, mini_batch_size,
optimizer, criterion, n_epochs, Net['learning rate'], lambda_l2,
alpha, beta)
# store the training and validation accuracies and losses during the training
train_results[n, ] = torch.tensor(
[train_losses, train_acc, valid_losses, valid_acc])
# compute the loss and accuracy of the model on the test set
test_loss, test_acc = compute_metrics(model, test_data, device)
# store the test metrics in the list
test_losses.append(test_loss)
test_accuracies.append(test_acc)
# learning curve
if plot:
learning_curve(train_losses, train_acc, valid_losses, valid_acc)
print(
'Seed {:d} | Test Loss: {:.4f} | Test Accuracy: {:.2f}%\n'.format(
n, test_loss, test_acc))
# store the train, validation and test accuracies in a tensor for the boxplot
data = torch.stack([
train_results[:, 1, (n_epochs - 1)], train_results[:, 3,
(n_epochs - 1)],
torch.tensor(test_accuracies)
])
data = data.view(1, 3, 10)
# boxplot
if statistics:
Title = " Models accuracies"
models = [Net['net_type']]
boxplot(data, Title, models, True)
return train_results, torch.tensor(test_losses), torch.tensor(
test_accuracies)
def main(train_args):
if cuda.is_available():
net = fcn8s.FCN8s(num_classes=voc.num_classes, pretrained=False).cuda()
#net = MBO.MBO().cuda()
#net = deeplab_resnet.Res_Deeplab().cuda()
else:
print('cuda is not available')
net = fcn8s.FCN8s(num_classes=voc.num_classes, pretrained=True)
net.train()
mean_std = ([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
input_transform = standard_transforms.Compose([
standard_transforms.ToTensor(),
standard_transforms.Normalize(*mean_std)
])
target_transform = extended_transforms.MaskToTensor()
restore_transform = standard_transforms.Compose([
extended_transforms.DeNormalize(*mean_std),
standard_transforms.ToPILImage(),
])
visualize = standard_transforms.Compose([
standard_transforms.Scale(400),
standard_transforms.CenterCrop(400),
standard_transforms.ToTensor()
])
train_set = voc.VOC('train',
set='benchmark',
transform=input_transform,
target_transform=target_transform)
train_loader = DataLoader(train_set,
batch_size=bsz,
num_workers=8,
shuffle=True)
val_set = voc.VOC('val',
set='voc',
transform=input_transform,
target_transform=target_transform)
val_loader = DataLoader(val_set,
batch_size=1,
num_workers=4,
shuffle=False)
criterion = CrossEntropyLoss2d(size_average=False,
ignore_index=voc.ignore_label).cuda()
optimizer = optim.Adam([{
'params': [
param
for name, param in net.named_parameters() if name[-4:] == 'bias'
],
'lr':
train_args['lr']
}, {
'params': [
param
for name, param in net.named_parameters() if name[-4:] != 'bias'
],
'lr':
train_args['lr']
}],
betas=(train_args['momentum'], 0.999))
scheduler = ReduceLROnPlateau(optimizer,
'min',
patience=2,
min_lr=1e-10,
verbose=True)
lr0 = 1e-7
max_epoch = 50
max_iter = max_epoch * len(train_loader)
#optimizer = optim.SGD(net.parameters(),lr = lr0, momentum = 0.9, weight_decay = 0.0005)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.5)
log_dir = os.path.join(root, 'logs', 'voc-fcn')
time = datetime.datetime.now().strftime('%d-%m-%H-%M')
train_file = 'train_log' + time + '.txt'
val_file = 'val_log' + time + '.txt'
#os.makedirs(log_dir,exist_ok=True)
training_log = open(os.path.join(log_dir, train_file), 'w')
val_log = open(os.path.join(log_dir, val_file), 'w')
curr_epoch = 1
for epoch in range(curr_epoch, train_args['epoch_num'] + 1):
train(train_loader, net, criterion, optimizer, epoch, train_args,
training_log, max_iter, lr0)
val_loss = validate(val_loader, net, criterion, optimizer, epoch,
train_args, restore_transform, visualize, val_log)
scheduler.step(val_loss)
lr_tmp = 0.0
k = 0
for param_group in optimizer.param_groups:
lr_tmp += param_group['lr']
k += 1
val_log.write('learning rate = {}'.format(str(lr_tmp / k)) + '\n')
from torch import cuda, device
DEVICE = device('cuda:0' if cuda.is_available() else 'cpu')
DATASET_PATH = '/run/timeshift/backup/thesis/EnhanceIt/src/Datasets/DIV2K/Train'
TEST_DATAPATH = '../src/Datasets/DIV2K/Validation'
ARCNN_DATASET = '../src/Datasets/BSDS500'
BATCH_SIZE = 4
UPSCALE_FACTOR = 4
EPOCHS = 500
ARCNN_MODEL = '../src/models/ARCNN.pth'
SIZE = 128
MODEL_SAVE_PATH = '/run/timeshift/backup/thesis/EnhanceIt/src/experiment_models/SRGAN_with_blur/models/bestSRGAN.pth'
DIS_PATH = '/run/timeshift/backup/thesis/EnhanceIt/src/experiment_models/disTest.pth'
LEARNING_RATE = 10e-5
CHECKPOINT_DIR = '../src/models/testSrgan/cp/'
MODEL_X2_DIR = '../src/model_x2.pth'
MODEL_X4_DIR = '../src/model_x4.pth'
CP = '../src/models/checkpoints/testcp'
EXTRACTED_FRAMES_DIR = '../src/extracted_frames/'
ENHANCED_FRAMES_DIR = '../src/enhanced_frames/'
ENHANCED_IMG_DIR = '../src/Single_Image_Results/'
ARCNN_FRAMES_DIR = '../src/arcnn_frames/'
BICUBIC_FRAMES_DIR = '../src/bicubic_resampling/'
arg('--num_classes', type=int, default=4,
help='num tags to predict') # Fixed
arg('--model', type=str, default='Resnet50')
arg('--input_size', type=int, default=296)
arg('--test_augments', default='resize, horizontal_flip', type=str)
arg('--augment_ratio',
default=0.5,
type=float,
help='probability of implementing transforms')
arg('--device', type=int, default=0)
arg('--hidden_size', type=int, default=128)
args = parser.parse_args()
device = args.device
use_gpu = cuda.is_available()
SEED = 2019
seed_everything(SEED)
global model
model = models.densenet201(pretrained=False)
model.classifier = nn.Linear(1920, args.num_classes)
bind_model(model)
if args.mode == 'train':
nsml.save('last')
if use_gpu:
model = model.to(device)
if args.pause:
def main():
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('mode', choices=['train', 'validate', 'predict_valid', 'predict_test'])
arg('run_root')
arg('--model', default='se_resnet50')
arg('--pretrained', type=int, default=1)
arg('--batch-size', type=int, default=64)
arg('--step', type=int, default=1)
arg('--workers', type=int, default=2 if ON_KAGGLE else 4)
arg('--lr', type=float, default=1e-4)
arg('--patience', type=int, default=4)
arg('--clean', action='store_true')
arg('--n-epochs', type=int, default=100)
arg('--epoch-size', type=int)
arg('--tta', type=int, default=4)
arg('--use-sample', action='store_true', help='use a sample of the dataset')
arg('--debug', action='store_true')
arg('--limit', type=int)
arg('--fold', type=int, default=0)
arg('--multi', type=int, default=0)
arg('--half', type=int, default=0)
args = parser.parse_args()
run_root = Path(args.run_root)
folds = pd.read_csv('../folds.csv')
train_root = DATA_ROOT / ('train_sample' if args.use_sample else 'train')
if args.use_sample:
folds = folds[folds['Id'].isin(set(get_ids(train_root)))]
train_fold = folds[folds['fold'] != args.fold]
valid_fold = folds[folds['fold'] == args.fold]
if args.limit:
train_fold = train_fold[:args.limit]
valid_fold = valid_fold[:args.limit]
def make_loader(df: pd.DataFrame, image_transform) -> DataLoader:
return DataLoader(
TrainDataset(train_root, df, image_transform, debug=args.debug),
shuffle=True,
batch_size=args.batch_size,
num_workers=args.workers,
)
criterion = nn.BCEWithLogitsLoss(reduction='none')
# criterion = FocalLoss()
model = getattr(models, args.model)()
feature_dim = model.last_linear.in_features
class AvgPool(nn.Module):
def forward(self, x):
# print (x.size())
return F.avg_pool2d(x, x.shape[2:])
model.avg_pool = AvgPool()
model.avgpool = AvgPool()
model.last_linear = nn.Linear(feature_dim, N_CLASSES)
use_cuda = cuda.is_available()
fresh_params = list(model.last_linear.parameters())
all_params = list(model.parameters())
if use_cuda:
if args.multi:
model = torch.nn.DataParallel(model).cuda()
else:
model = model.cuda()
if args.mode == 'train':
if run_root.exists() and args.clean:
shutil.rmtree(run_root)
run_root.mkdir(exist_ok=True, parents=True)
(run_root / 'params.json').write_text(
json.dumps(vars(args), indent=4, sort_keys=True))
writer = SummaryWriter(str(run_root/'tbx'))
train_loader = make_loader(train_fold, train_transform)
valid_loader = make_loader(valid_fold, test_transform)
print(f'{len(train_loader.dataset):,} items in train, '
f'{len(valid_loader.dataset):,} in valid')
train_kwargs = dict(
args=args,
model=model,
criterion=criterion,
train_loader=train_loader,
valid_loader=valid_loader,
patience=args.patience,
init_optimizer=lambda params, lr: Adam(params, lr),
use_cuda=use_cuda,
)
if args.pretrained:
if train(params=fresh_params, writer=writer, n_epochs=1, **train_kwargs):
train(params=all_params, writer=writer, **train_kwargs)
else:
train(params=all_params, writer=writer, **train_kwargs)
writer.export_scalars_to_json(str(run_root/'tbx/all_scalars.json'))
writer.close()
elif args.mode == 'validate':
valid_loader = make_loader(valid_fold, test_transform)
load_model(model, run_root / 'model.pt')
validation(model, criterion, tqdm.tqdm(valid_loader, desc='Validation'),
use_cuda=use_cuda)
elif args.mode.startswith('predict'):
load_model(model, run_root / 'best-model.pt')
predict_kwargs = dict(
batch_size=args.batch_size,
tta=args.tta,
use_cuda=use_cuda,
workers=args.workers,
)
if args.mode == 'predict_valid':
predict(model, df=valid_fold, root=train_root,
out_path=run_root / 'val.h5',
**predict_kwargs)
elif args.mode == 'predict_test':
test_root = DATA_ROOT / (
'test_sample' if args.use_sample else 'test')
ss = pd.read_csv(DATA_ROOT / 'sample_submission.csv')
if args.use_sample:
ss = ss[ss['id'].isin(set(get_ids(test_root)))]
if args.limit:
ss = ss[:args.limit]
predict(model, df=ss, root=test_root,
out_path=run_root / 'test.h5',
**predict_kwargs)
sys.path.append(os.getcwd())
import torch as tc
from torch import cuda
import wargs
from tools.inputs_handler import *
from tools.inputs import Input
from tools.optimizer import Optim
from models.losser import Classifier
from models.embedding import WordEmbedding
from models.model_builder import build_NMT
from tools.utils import init_dir, wlog
# Check if CUDA is available
if cuda.is_available():
wlog(
'CUDA is available, specify device by gpu_id argument (i.e. gpu_id=[0, 1, 2])'
)
else:
wlog('Warning: CUDA is not available, train on CPU')
if wargs.gpu_id is not None:
#cuda.set_device(wargs.gpu_id[0])
device = tc.device(
'cuda:{}'.format(wargs.gpu_id[0]) if cuda.is_available() else 'cpu')
wlog('Set device {}, will use {} GPUs {}'.format(wargs.gpu_id[0],
len(wargs.gpu_id),
wargs.gpu_id))
from trainer import *
DATA = "/data/horse2zebra/"
trainA = "trainA/"
trainB = "trainB/"
forge = "/data/forge/"
HOME = "/home/ubuntu/"
CLIP_LIMIT = 1e-3
BS = 4
# In[3]:
from models import discriminative
from models import generative_chimney as generative
from torch import cuda
CUDA = cuda.is_available()
# In[4]:
from torch.optim import Adam
class cycle(nn.Module):
def __init__(self,
g_fn=[64, 64, 64, 64, 128, 128, 128, 128, 64],
d_fn=[64, 64, 64, 128, 128, 128]):
"""
g_fn: filter numbers for genrative model, a list of int
d_fn: filter numbers for discriminative model, a list of int, downsampling count is len(self.d_fn)-1
"""
super(cycle, self).__init__()
# confirm PyTorch sees the GPU from torch import cuda assert cuda.is_available() assert cuda.device_count() > 0 print(cuda.get_device_name(cuda.current_device()))
def train(model, optimizer, criterion, epoch, train_data, val_data, save_file,
best_file):
global force_cuda
if os.path.exists(best_file):
out('Loading best model')
loader = torch.load(best_file)
model.load_state_dict(loader['state_dict'])
best_val_acc = loader['best_val']
else:
best_val_acc = None
model.train()
total_loss = 0.0
total_acc = 0.0
total_img = 0
iteration_losses = []
for i, data in enumerate(train_data):
inputs, labels = data
if force_cuda and cuda.is_available():
inputs, labels = inputs.cuda(), labels.cuda()
optimizer.zero_grad()
outputs = model(inputs)
_, predictions = torch.max(outputs.data, 1)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
iteration_losses.append(loss.data.item())
total_loss += loss.data.item()
total_acc += torch.sum(predictions == labels.data).item()
total_img += labels.size(0)
del inputs, labels, outputs, predictions
cuda.empty_cache()
out(f'Training #{epoch}: {total_acc / total_img} accuracy and {total_loss / total_img} loss'
)
val_loss, val_acc = eval(model, criterion, val_data)
out(f'Validation #{epoch}: {val_acc} accuracy and {val_loss} loss')
if best_val_acc is None or best_val_acc < val_acc:
best_val_acc = val_acc
torch.save({
'state_dict': model.state_dict(),
'best_val': best_val_acc
}, best_file)
# Save per epoch
saved_iteration_losses = iteration_losses
if os.path.exists(save_file):
saved_iteration_losses = torch.load(save_file)['iteration_losses']
saved_iteration_losses.extend(iteration_losses)
torch.save(
{
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
'criterion': criterion.state_dict(),
'iteration_losses': saved_iteration_losses,
'last_epoch': epoch
}, save_file)
return model, optimizer, criterion
def setup():
parser = argparse.ArgumentParser()
parser.add_argument('--actions',
default='../actions_bb.txt',
help='define actions txt')
parser.add_argument('--strategy',
default='../strategy_bb.txt',
help='define strategy txt')
parser.add_argument('--dataset',
default='nt4096_ls128_nzf8_nzd32.pth',
help='folder | synth | pth | stead | ann2bb | deepbns')
parser.add_argument(
'--dataroot', default='../database/stead', help='Path to dataset'
) # '/home/filippo/Data/Filippo/aeolus/ann2bb_as4_') # '/home/filippo/Data/Filippo/aeolus/STEAD/waveforms_11_13_19.hdf5',help='path to dataset')
parser.add_argument('--inventory',
default='RM07.xml,LXRA.xml,SRN.xml',
help='inventories')
parser.add_argument('--workers',
type=int,
help='number of data loading workers',
default=2)
parser.add_argument('--batchSize',
type=int,
default=5,
help='input batch size')
parser.add_argument('--batchPercent',
type=int,
nargs='+',
default=[0.8, 0.1, 0.1],
help='train/test/validation %')
parser.add_argument('--niter',
type=int,
default=2,
help='number of epochs to train for')
parser.add_argument(
'--imageSize',
type=int,
default=4096,
help='the height / width of the input image to network')
parser.add_argument(
'--latentSize',
type=int,
default=128,
help='the height / width of the input image to network')
parser.add_argument('--cutoff',
type=float,
default=1.,
help='cutoff frequency')
parser.add_argument('--nzd',
type=int,
default=32,
help='size of the latent space')
parser.add_argument('--nzf',
type=int,
default=8,
help='size of the latent space')
parser.add_argument('--ngf',
type=int,
default=32,
help='size of G input layer')
parser.add_argument('--ndf',
type=int,
default=32,
help='size of D input layer')
parser.add_argument('--glr',
type=float,
default=0.0001,
help='AE learning rate, default=0.0001')
parser.add_argument('--rlr',
type=float,
default=0.0001,
help='GAN learning rate, default=0.00005')
parser.add_argument('--b1',
type=float,
default=0.5,
help='beta1 for Adam. default=0.5')
parser.add_argument('--cuda', action='store_true', help='enables cuda')
parser.add_argument('--ngpu',
type=int,
default=2,
help='number of GPUs to use')
parser.add_argument('--plot',
action='store_true',
help="flag for plotting")
parser.add_argument('--outf',
default='./imgs',
help='folder to output images and model checkpoints')
parser.add_argument('--manualSeed', type=int, help='manual seed')
parser.add_argument('--mw', type=float, default=4.5, help='magnitude [Mw]')
parser.add_argument('--dtm',
type=float,
default=0.01,
help='time-step [s]')
parser.add_argument('--dep',
type=float,
default=50.,
help='epicentral distance [km]')
parser.add_argument('--scc', type=int, default=0, help='site-class')
parser.add_argument('--sst', type=int, default=1, help='site')
parser.add_argument('--scl', type=int, default=1, help='scale [1]')
parser.add_argument('--nsy',
type=int,
default=83,
help='number of synthetics [1]')
parser.add_argument('--save_checkpoint',
type=int,
default=1,
help='Number of epochs for each checkpoint')
parser.set_defaults(stack=False, ftune=False, feat=False, plot=True)
opt = parser.parse_args()
u'''Set-up GPU and CUDA'''
opt.cuda = True if (tcuda.is_available() and opt.cuda) else False
device = tdev("cuda:0" if opt.cuda else "cpu")
FloatTensor = tcuda.FloatTensor if opt.cuda else tFT
LongTensor = tcuda.LongTensor if opt.cuda else tLT
ngpu = int(opt.ngpu)
try:
os.makedirs(opt.outf)
except OSError:
pass
if opt.manualSeed is None:
opt.manualSeed = random.randint(1, 10000)
print("Random Seed: ", opt.manualSeed)
random.seed(opt.manualSeed)
mseed(opt.manualSeed)
cudnn.benchmark = True
if tcuda.is_available() and not opt.cuda:
print(
"WARNING: You have a CUDA device, so you should probably run with --cuda"
)
if opt.dataset == 'synth':
sd_mw = np.log(1. + (2. / opt.mw)**2)
mu_mw = np.log(opt.mw) - 0.5 * sd_mw
l=get_truncated_normal(mean=mu_mw,\
sd=sd_mw,\
low=np.log(3.5),\
upp=np.log(8))
x = l.ppf(np.random.rand(opt.nsy))
opt.mw = np.round(np.exp(x), 1)
opt.dep = get_truncated_normal(mean=opt.dep,\
sd=50.,low=10.,upp=200.).ppf(np.random.rand(opt.nsy))
opt.dep = opt.dep.round(0)
md = {'mw':opt.mw,'dep':opt.dep,'scc':opt.scc,\
'sst':opt.sst,'dtm':opt.dtm,'scl':opt.scl,'cutoff':opt.cutoff}
md['vTn'] = np.arange(0.0, 3.05, 0.05, dtype=np.float64)
md['nTn'] = md['vTn'].size
ths_trn,ths_tst,ths_vld,\
vtm,fsc = synth_dataset(opt.batchPercent,opt.imageSize,opt.latentSize,\
opt.nzd,opt.nzf,md=md,nsy=opt.nsy,device=device)
md['fsc'] = fsc
opt.ncls = md['fsc']['ncat']
# Create natural period vector
opt.vTn = np.arange(0.0, 3.05, 0.05, dtype=np.float64)
opt.nTn = md['vTn'].size
tsave(ths_trn, './ths_trn.pth')
tsave(ths_tst, './ths_tst.pth')
tsave(ths_vld, './ths_vld.pth')
tsave(vtm, './vtm.pth')
with open('md.p', 'wb') as handle:
pickle.dump(md, handle)
handle.close()
with open('opt.p', 'wb') as handle:
pickle.dump(opt, handle)
handle.close()
elif opt.dataset == 'folder':
src = opt.dataroot.split(',')
for n in range(len(src)):
src[n] = osj(src[n], '*.*.*.mseed')
print('dataroots:')
print(src)
inv = opt.inventory.split(',')
for n in range(len(inv)):
inv[n] = osj(opt.dataroot, 'sxml', inv[n])
print('inventories:')
print(inv)
ths_trn,ths_tst,ths_vld,thf_trn,thf_tst,thf_vld,\
vtm,fsc = load_dataset(opt.batchPercent,\
source=src,inventory=inv)
md = {'mw':[],'dep':[],'scc':[],\
'sst':[],'dtm':vtm[1]-vtm[0],\
'ntm':vtm.size,'scl':[]}
opt.ncls = 1
tsave(ths_trn, './ths_trn.pth')
tsave(ths_tst, './ths_tst.pth')
tsave(ths_vld, './ths_vld.pth')
tsave(thf_trn, './thf_trn.pth')
tsave(thf_tst, './thf_tst.pth')
tsave(thf_vld, './thf_vld.pth')
tsave(vtm, './vtm.pth')
elif '.pth' in opt.dataset:
ths_trn = tload(osj(opt.dataroot, 'ths_trn_' + opt.dataset))
ths_tst = tload(osj(opt.dataroot, 'ths_tst_' + opt.dataset))
ths_vld = tload(osj(opt.dataroot, 'ths_vld_' + opt.dataset))
vtm = tload(osj(opt.dataroot, 'vtm.pth'))
pickle_off = open(osj(opt.dataroot, "md.p"), "rb")
md = pickle.load(pickle_off)
pickle_off.close()
pickle_off = open(osj(opt.dataroot, "opt.p"), "rb")
optt = pickle.load(pickle_off)
pickle_off.close()
for k, v in optt.__dict__.items():
flag = False
if k in opt.__dict__:
try:
if type(v).__module__ == 'numpy' and (
v != opt.__dict__[k]).any():
flag = True
except:
pass
try:
if any(v != opt.__dict__[k]):
flag = True
except:
pass
try:
if v != opt.__dict__[k]:
flag = True
except:
pass
if flag:
optt.__dict__[k] = opt.__dict__[k]
for k, v in opt.__dict__.items():
flag = False
if k not in optt.__dict__:
flag = True
if flag:
optt.__dict__[k] = opt.__dict__[k]
opt = optt
elif opt.dataset == 'deepbns':
src = osj(opt.dataroot, "Hybrid_Database.h5")
print('dataroots:')
print(src)
md = {
'dtm': 9.452707692307693e-06,
'cutoff': opt.cutoff,
'ntm': opt.imageSize
}
md['vTn'] = np.arange(0.0, 3.05, 0.05, dtype=np.float64)
md['nTn'] = md['vTn'].size
ths_trn,ths_tst,ths_vld,\
vtm,fsc = deepbns_dataset(src,opt.batchPercent,opt.imageSize,opt.latentSize,\
opt.nzd,opt.nzf,md=md,nsy=opt.nsy,device=device)
md['fsc'] = fsc
opt.ncls = md['fsc']['ncat']
# Create natural period vector
opt.vTn = np.arange(0.0, 3.05, 0.05, dtype=np.float64)
opt.nTn = md['vTn'].size
tsave(ths_trn, './ths_trn.pth')
tsave(ths_tst, './ths_tst.pth')
tsave(ths_vld, './ths_vld.pth')
tsave(vtm, './vtm.pth')
with open('md.p', 'wb') as handle:
pickle.dump(md, handle)
handle.close()
with open('opt.p', 'wb') as handle:
pickle.dump(opt, handle)
handle.close()
elif opt.dataset == 'stead':
src = opt.dataroot
print('dataroots:')
print(src)
md = {'dtm': 0.01, 'cutoff': opt.cutoff, 'ntm': opt.imageSize}
md['vTn'] = np.arange(0.0, 3.05, 0.05, dtype=np.float64)
md['nTn'] = md['vTn'].size
ths_trn,ths_tst,ths_vld,\
vtm,fsc = stead_dataset(src,opt.batchPercent,opt.imageSize,opt.latentSize,\
opt.nzd,opt.nzf,md=md,nsy=opt.nsy,device=device)
md['fsc'] = fsc
opt.ncls = md['fsc']['ncat']
# Create natural period vector
opt.vTn = np.arange(0.0, 3.05, 0.05, dtype=np.float64)
opt.nTn = md['vTn'].size
tsave(ths_trn, './ths_trn.pth')
tsave(ths_tst, './ths_tst.pth')
tsave(ths_vld, './ths_vld.pth')
tsave(vtm, './vtm.pth')
with open('md.p', 'wb') as handle:
pickle.dump(md, handle)
handle.close()
with open('opt.p', 'wb') as handle:
pickle.dump(opt, handle)
handle.close()
elif opt.dataset == 'ann2bb':
src = opt.dataroot
print('dataroots:')
print(src)
md = {'dtm': 0.005, 'cutoff': opt.cutoff, 'ntm': opt.imageSize}
md['vTn'] = np.arange(0.0, 3.05, 0.05, dtype=np.float64)
md['nTn'] = md['vTn'].size
ths_trn,ths_tst,ths_vld,\
vtm,fsc,md = ann2bb_dataset(src,opt.batchPercent,opt.imageSize,opt.latentSize,\
opt.nzd,opt.nzf,md=md,nsy=opt.nsy,device=device)
md['fsc'] = fsc
opt.ncls = md['fsc']['ncat']
# Create natural period vector
opt.vTn = np.arange(0.0, 3.05, 0.05, dtype=np.float64)
opt.nTn = md['vTn'].size
tsave(ths_trn, './ths_trn.pth')
tsave(ths_tst, './ths_tst.pth')
tsave(ths_vld, './ths_vld.pth')
tsave(vtm, './vtm.pth')
with open('md.p', 'wb') as handle:
pickle.dump(md, handle)
handle.close()
with open('opt.p', 'wb') as handle:
pickle.dump(opt, handle)
handle.close()
params = {'batch_size': opt.batchSize,\
'shuffle': True,'num_workers':int(opt.workers)}
trn_loader,tst_loader,vld_loader = \
dataset2loader(ths_trn,ths_tst,ths_vld,**params)
# dataset2loader(ths_trn,thf_trn,wnz_trn,\
# ths_tst,thf_tst,wnz_tst,\
# ths_vld,thf_vld,wnz_vld,\
# **params)
actions = pd.read_csv(filepath_or_buffer=opt.actions,
sep=',',
true_values='True',
false_values='False')
tract = {'tract': actions['tract'].to_dict()}
trplt = {'trplt': actions['trplt'].to_dict()}
trcmp = {'trcmp': actions['trcmp'].to_dict()}
trdis = {'trdis': actions['trdis'].to_dict()}
strategy = pd.read_csv(filepath_or_buffer=opt.strategy,
sep=',',
na_values='None')
strategy = strategy.where((pd.notnull(strategy)), None)
strategy['strategy'] = strategy.values.tolist()
strategy = dict(strategy['strategy'].to_dict(), **tract, **trplt, **trcmp,
**trdis)
opt.strategy = strategy
cv = {'parser':parser,'opt':opt,'vtm':vtm,\
'trn_loader':trn_loader,\
'tst_loader':tst_loader,\
'vld_loader':vld_loader,\
'params':params,\
'device':device,\
'FloatTensor':FloatTensor,\
'LongTensor':LongTensor,\
'md':md}
return cv
from torchvision.utils import save_image, make_grid
from os import path
from google.colab import drive
notebooks_dir_name = 'notebooks'
drive.mount('/content/gdrive')
notebooks_base_dir = path.join('./gdrive/My Drive/', notebooks_dir_name)
if not path.exists(notebooks_base_dir):
print('Check your google drive directory. See you file explorer')
# Settings
download_root = 'mnist'
stored_path = 'images'
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(mean=(0.5, ), std=(0.5, ))])
device = 'cuda' if cuda.is_available() else 'cpu'
leraing_rate = 0.0002
# Dataset
train_set = MNIST(download_root,
train=True,
transform=transform,
download=True)
# Dataloader
train_loader = DataLoader(train_set, batch_size=60, shuffle=True)
# Image_dir
import os
import imageio
# 데이터 경로 지정부분
datadir, traindir, validdir, testdir, image_transforms, data, dataloaders = train_util.init_dataset(
)
cat_df, image_df = train_util.category_dataframe(traindir, validdir, testdir)
# 학습된 데이터 저장시 이름을 정하는 부분
save_file_name = './ModelSave/' + model_choice + '-transfer.pt'
checkpoint_path = './ModelSave/' + model_choice + '-transfer_bts' + str(
batch_size) + "_ep" + str(
training_epoch) + "_" + train_util.get_date() + '.pth'
# 이미지 갯수 분포 그래프 이미지로 저장
# train_util.save_distribution_of_images(cat_df, model_choice)
# Whether to train on a gpu
train_on_gpu = cuda.is_available() # GPU를 사용할 수 있는지 없는지 판단한다.
if train_on_gpu:
print('학습 모드 : GPU\n')
else:
print('학습 모드 : CPU\n')
cat_df.sort_values('n_train', ascending=False, inplace=True)
# print(f'{cat_df.head()}\n')
# print(f'{cat_df.tail()}\n')
# iter() : 전달된 데이터의 반복자를 꺼내 반환한다.
# trainiter = iter(dataloaders['train'])
# next() : 반복자를 입력받아 그 반복자가 다음에 출력해야할 요소를 반환한다.
# features, labels = next(trainiter) # 1개만 꺼내기위해 넣은 코드인듯
# print(f'{features.shape} , {labels.shape}') # 그냥 단순히 어떤 데이터가 어떤 형태로 들어있는지 알려주기 위한 코드인듯.
s.collapse_unary(True, True)
# get the structure IDs from the dictionary keys
conditions = list(structures.keys())
# filter down to those conditions found in conditions
data = data[data.condition.isin(conditions)]
# build the vocab list up from the structures
vocab = list({word
for tree in structures.values()
for word in tree.leaves()})
# load the glove embedding
embeddings = load_glove_embedding(args.embeddings, vocab)
device_to_use = device("cuda:0" if is_available() else "cpu")
if args.rnntype == "tree":
x_raw = [structures[c] for c in data.condition.values]
x = [x_raw[i:i + args.batch] for i in range(0, len(x_raw), args.batch)]
y_raw = data.response.values
y = [y_raw[i:i + args.batch] for i in range(0, len(y_raw), args.batch)]
rnntype = ChildSumConstituencyTreeLSTM
args.batch = 1
elif args.rnntype == "linear":
# Implmenent mini-batching
x_raw = [structures[c].words() for c in data.condition.values]
x = [x_raw[i:i + args.batch] for i in range(0, len(x_raw), args.batch)]
y_raw = data.response.values
y = [y_raw[i:i + args.batch] for i in range(0, len(y_raw), args.batch)]
rnntype = LSTM
def train(config: Config):
"""学習処理の実行スクリプト."""
transforms = get_transforms(config.resize_image)
dataset_type = DatasetName.value_of(config.dataset_name)
dataset_train, dataset_valid = get_dataset(dataset_type, transforms)
dataloader_train = td.DataLoader(
dataset_train,
config.batch_size,
shuffle=True,
num_workers=config.num_workers,
pin_memory=True,
worker_init_fn=_worker_init_random,
)
dataloader_valid = td.DataLoader(
dataset_valid,
config.batch_size,
shuffle=False,
num_workers=config.num_workers,
pin_memory=True,
worker_init_fn=_worker_init_random,
)
params = dc.asdict(config)
pl.seed_everything(config.random_seed)
network = get_network(
NetworkName.value_of(config.network_name),
in_channels=config.in_channels,
out_channels=config.out_channels,
image_size=config.resize_image,
)
model = VAETrainer(network, params)
model.set_dataloader(dataloader_train, dataloader_valid)
cache_dir = directories.get_processed().joinpath(config.cache_dir)
cache_dir.mkdir(exist_ok=True)
model_checkpoint = pl_callbacks.ModelCheckpoint(
filepath=str(cache_dir),
monitor="val_loss",
save_last=True,
save_top_k=config.save_top_k,
save_weights_only=config.save_weights_only,
mode="min",
period=1,
)
experiment_dir = cache_dir.joinpath(
"default", f"version_{config.experiment_version}"
)
pl_logger = pl_logging.TensorBoardLogger(
save_dir=str(cache_dir), version=config.experiment_version
)
trainer_params = dict()
if config.resume:
trainer_params["resume_from_checkpoint"] = str(cache_dir.joinpath("last.ckpt"))
elif experiment_dir.exists():
shutil.rmtree(experiment_dir)
for filepath in cache_dir.glob("*.ckpt"):
filepath.unlink()
for filepath in cache_dir.glob("*.pth"):
filepath.unlink()
pl_trainer = pl.Trainer(
early_stop_callback=config.early_stop,
default_root_dir=str(cache_dir),
fast_dev_run=False,
min_epochs=config.min_epochs,
max_epochs=config.max_epochs,
gpus=[0] if config.use_gpu and tc.is_available() else None,
progress_bar_refresh_rate=config.progress_bar_refresh_rate,
profiler=config.profiler,
checkpoint_callback=model_checkpoint,
logger=pl_logger,
log_gpu_memory=True,
**trainer_params,
)
pl_trainer.fit(model)
for ckptfile in cache_dir.glob("*.ckpt"):
pthfile = cache_dir.joinpath(ckptfile.stem + ".pth")
model = model.load_from_checkpoint(str(ckptfile), network, params)
torch.save(model.network.state_dict(), pthfile)