for word in text:
if random.random() < p[word]:
subsampling.append(word)
vocab_count = dict(Counter(subsampling).most_common(MAX_VOCAB_SIZE - 1))
vocab_count['<UNK>'] = 1
idx2word = [word for word in vocab_count.keys()]
word2idx = {word: i for i, word in enumerate(idx2word)}
nc = np.array([count for count in vocab_count.values()],
dtype=np.float32)**(3. / 4.)
word_freqs = nc / np.sum(nc)
dataset = WordEmbeddingDataset(subsampling, word2idx, word_freqs)
dataloader = tud.DataLoader(dataset, BATCH_SIZE, shuffle=True)
model = EmbeddingModel(len(idx2word), EMBEDDING_SIZE)
model.to(device)
model.train()
optimizer = optim.Adam(model.parameters(), lr=LR)
for epoch in range(EPOCHS):
pbar = tqdm(dataloader)
pbar.set_description("[Epoch {}]".format(epoch))
for i, (input_labels, pos_labels, neg_labels) in enumerate(pbar):
input_labels = input_labels.to(device)
pos_labels = pos_labels.to(device)
neg_labels = neg_labels.to(device)
model.zero_grad()
loss = model(input_labels, pos_labels, neg_labels).mean()
round((real_size-crop_size)/2):round((real_size+crop_size)/2),round((real_size-crop_size)/2):round((real_size+crop_size)/2)]*\
precandidate['seg'][round((real_size-crop_size)/2):round((real_size+crop_size)/2),\
round((real_size-crop_size)/2):round((real_size+crop_size)/2),round((real_size-crop_size)/2):round((real_size+crop_size)/2)]
train_data = torch.from_numpy(train_data)
val_data = torch.from_numpy(val_data)
column_train_label = list(map(int, column_train_label))
train_label = torch.Tensor(train_label)
val_label = torch.Tensor(val_label)
BATCH_SIZE = 32
torch_dataset_train = data.TensorDataset(train_data, train_label)
torch_dataset_val = data.TensorDataset(val_data, val_label)
train_loader = data.DataLoader(dataset=torch_dataset_train,
batch_size=BATCH_SIZE,
shuffle=True)
val_loader = data.DataLoader(dataset=torch_dataset_val,
batch_size=BATCH_SIZE,
shuffle=False)
test_loader = data.DataLoader(test_data, batch_size=test_num, shuffle=False)
def mixup_data(x, y, alpha, device):
'''Returns mixed inputs, pairs of targets, and lambda'''
if alpha > 0:
lam = np.random.beta(alpha, alpha)
else:
lam = 1
batch_size = x.size()[0]
def train():
rpf.train()
lr = args.lr
epoch = disp_loss = 0
eval_loss = 10000.
start_time = time.time()
epoch_size = len(train_dataset) // args.batch_size
max_epoch = int(args.max_iter / epoch_size)
step_values = [10000, 50000, 100000]
step_index = 0
batch_iterator = data.DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
for iteration in range(args.max_iter):
if iteration % epoch_size == 0:
if epoch != 0:
print("Saving state, epoch:", epoch)
torch.save(rpf.state_dict(),
save_dir + '/epoch_{}.pth'.format(epoch))
batch_iterator = iter(
data.DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers))
epoch += 1
demo_im, demo_action_onehot, follower_im, follower_action = next(
batch_iterator)
if args.cuda:
demo_im = Variable(demo_im.cuda(async=True))
demo_action_onehot = Variable(demo_action_onehot.cuda(async=True))
follower_im = Variable(follower_im.cuda(async=True))
with torch.no_grad():
follower_action = Variable(
follower_action.cuda(async=True).long())
record, done = rpf(
demo_im, demo_action_onehot, follower_im=follower_im
) #Record: 'eta', 'h_ts', 'attention_t', 'in_rgb_feats_t', 'mu_t', 'action_pred_t'
optimizer.zero_grad()
loss = criterion(record['action_pred_t'].view(-1, args.action_dim),
follower_action.view(-1))
loss.backward()
optimizer.step()
disp_loss += loss.item()
end_time = time.time()
if iteration % args.print_step == 0 and iteration > 0:
disp_loss = disp_loss / args.print_step
print(
'Epoch [%d/%d] Iter [%d/%d] Loss: %.6f, lr: %.6f, Iter time: %.5fs'
% (epoch, max_epoch, iteration, args.max_iter, disp_loss, lr,
(end_time - start_time)))
writer.add_scalar('train_loss', disp_loss, iteration)
disp_loss = 0.
start_time = time.time()
if iteration in step_values:
step_index += 1
lr = adjust_learning_rate(optimizer, step_index, args.lr_decay)
if iteration % args.test_step == 0: # and iteration > 1:
rpf.eval()
tot_start_time = time.time()
disp_loss = total_disp_loss = 0.
print("Evaluating RPF networks...")
valid_iterator = iter(
data.DataLoader(dataset=valid_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers))
with torch.no_grad():
for valid_iteration in range(valid_num_batches):
demo_im, demo_action_onehot, follower_im, follower_action = next(
valid_iterator)
if args.cuda:
with torch.no_grad():
demo_im = Variable(demo_im.cuda(async=True))
demo_action_onehot = Variable(
demo_action_onehot.cuda(async=True))
follower_im = Variable(
follower_im.cuda(async=True))
follower_action = Variable(
follower_action.cuda(async=True).long())
record, done = rpf(demo_im,
demo_action_onehot,
follower_im=follower_im)
loss = criterion(
record['action_pred_t'].view(-1, args.action_dim),
follower_action.view(-1))
disp_loss += loss.item()
total_disp_loss += loss.item()
if valid_iteration % args.print_step == 0:
disp_loss = disp_loss / args.print_step
end_time = time.time()
print('Iter [%d/%d] Eval loss: %.6f, Time: %.5fs' %
(valid_iteration, valid_num_batches, disp_loss,
(end_time - start_time)))
disp_loss = 0.
start_time = time.time()
total_disp_loss = total_disp_loss / (valid_num_batches - 1)
tot_end_time = time.time()
print('[Epoch : %d] Loss : %.6f, Eval_time : %.5fs' %
(epoch, total_disp_loss,
(tot_end_time - tot_start_time)))
writer.add_scalar('valid_loss', total_disp_loss, epoch)
if eval_loss > total_disp_loss:
eval_loss = total_disp_loss
torch.save(rpf.state_dict(), save_dir + '/best.pth')
print('Updated the best model')
rpf.train()
])
transform_test = transforms.Compose([
transforms.Resize(32),
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, )),
])
dataloader = datasets.MNIST
trainset = dataloader(root='./data',
train=True,
download=True,
transform=transform_train)
trainloader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
testset = dataloader(root='./data',
train=False,
download=True,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# In[11]:
# Model
print("==> creating model ")
# (doc[0], indx_tokens, output_film_size1, output_film_size2, output_film_size3, output_film_size4, attention_mask, spmed, (n,doc,Entdic, Reldic))
n_Entdics_Reldics = [a[0] for a in corpus]
word_input = torch.LongTensor([a[1] for a in corpus]).to(device)
attention_mask = torch.LongTensor([a[2] for a in corpus]).to(device)
sentencepieced = [a[3] for a in corpus]
doc_correspnd_info_dict = {} #document毎にシンボリックな値をdocument名と辞書に変えるための辞書
n_doc = []
for unit in n_Entdics_Reldics:
doc_correspnd_info_dict[unit[0]] = [(unit[1],unit[1][:-4]+'.ann'),{},{}]
n_doc.append([unit[0]])
n_doc = torch.LongTensor(n_doc).to(device)
test_dataset = D.TensorDataset(n_doc, word_input, attention_mask)
test_loader = D.DataLoader(test_dataset , batch_size=int(config.get('main', 'BATCH_SIZE_TEST' )), shuffle = strtobool(config.get('main', 'BATCH_SHUFFLE_TEST')))
print('finish', end='\n')
# pdb.set_trace()
print('Create Model...')
tokenizer = BERT_PRETRAINED_MODEL_JAPANESE(config, vocab).return_tokenizer()
model = SPAN_CNN(config, vocab, REL_DIC).to(device)
# relation_model = BRAN(config, vocab, REL_DIC).to(device)
relation_model = RELATION(config, vocab, REL_DIC).to(device)
model.load_state_dict(torch.load(NER_model_save_path, map_location=device))
relation_model.load_state_dict(torch.load(RE_model_save_path, map_location=device))
# pretrained_BERT_model = bert_pretrained_model_japanese.return_model()
# tokenizer.convert_ids_to_tokens()というメソッドでindexを用語に直せる
import torch.utils.data as utils
# a list of numpy arrays
labels_nonmitosis = torch.ones(
len(path2)) # another list of numpy arrays (targets)
labels_mitosis = torch.zeros(len(path3))
train_labels = torch.cat((labels_nonmitosis, labels_mitosis))
y1_tensor = torch.tensor(train_labels, dtype=torch.long)
tensor_x1 = torch.stack([transform1(i)
for i in lbp_train]) # transform to torch tensors
tensor_x4 = torch.stack([TRANSFORM_IMG(i) for i in tensor_x1])
train_data = utils.TensorDataset(tensor_x4, y1_tensor)
train_data_loader = utils.DataLoader(train_data,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=4) # create your dataloader
labels_nonmitosis_val = torch.ones(
len(path4)) # another list of numpy arrays (targets)
labels_mitosis_val = torch.zeros(len(path5))
val_labels = torch.cat((labels_nonmitosis_val, labels_mitosis_val))
y2_tensor = torch.tensor(val_labels, dtype=torch.long)
tensor_x2 = torch.stack([transform1(i)
for i in lbp_val]) # transform to torch tensors
tensor_x3 = torch.stack([TRANSFORM_IMG(i) for i in tensor_x2])
test_data = utils.TensorDataset(tensor_x3, y2_tensor) # create your datset
test_data_loader = utils.DataLoader(test_data,
batch_size=BATCH_SIZE,
shuffle=True,
def objective(SCI_RELU, SCI_BIAS, SCI_loss_type,
SCI_optimizer, SCI_LR, SCI_MM,
SCI_REGULARIZATION, SCI_EPOCHS, SCI_BATCH_SIZE,
SCI_DROPOUT, SCI_L_SECOND, SCI_BN_MOMENTUM, SCI_SGD_MOMENTUM,
SCI_BN_EPS, SCI_BN_STATS, SCI_LAST_LAYER, SCI_ACT_LAYER):
global count, PercentVector, PercentVec, device, MaxCredit
SCI_BATCH_SIZE = int(SCI_BATCH_SIZE)
SCI_LAST_LAYER = int(SCI_LAST_LAYER)
SCI_ACT_LAYER =int(SCI_ACT_LAYER)
SCI_MM = round(SCI_MM,3) # real with three decimals between (0.001, 0.999)
SCI_LR = round(SCI_LR,5) # real with five decimals between(1e-4, 7e-1)
SCI_DROPOUT = round(SCI_DROPOUT,2) # real with two decimals between (0, 0.4)
SCI_L_SECOND = int(SCI_L_SECOND) # integer between 2 and 64
SCI_EPOCHS = int(SCI_EPOCHS) # integer between (100, 500)
SCI_BN_MOMENTUM = round(SCI_BN_MOMENTUM,2) # real with two decimals between (0, 0.99)
SCI_SGD_MOMENTUM = round(SCI_SGD_MOMENTUM,2) # real with two decimals between (0, 0.99)
SCI_loss_type = int(SCI_loss_type) # integer between 1 and 3 ('CrossEntropyLoss', 'MultiMarginLoss','NLLLoss')
SCI_BN_EPS = int(SCI_BN_EPS)
if int(SCI_RELU) == 1 : # integer between 1 and 2 ('True', 'False')
SCI_RELU = True
else:
SCI_RELU = False
if int(SCI_BIAS) == 1 : # integer between 1 and 2 ('True', 'False')
SCI_BIAS = True
else:
SCI_BIAS = False
SCI_REGULARIZATION = float(str(SCI_REGULARIZATION))
if SCI_BN_EPS == 0:
BN_EPS = 5e-6
if SCI_BN_EPS == 1:
BN_EPS = 1e-5
if SCI_BN_EPS == 2:
BN_EPS = 5e-6
if SCI_BN_EPS == 3:
BN_EPS = 1e-6
if SCI_BN_EPS == 4:
BN_EPS = 5e-7
if SCI_BN_EPS == 5:
BN_EPS = 1e-7
if SCI_BN_EPS == 6:
BN_EPS = 5e-8
if SCI_BN_EPS == 7:
BN_EPS = 1e-8
if SCI_BN_EPS == 8:
BN_EPS = 3e-7
if SCI_BN_EPS == 9:
BN_EPS = 8e-7
if SCI_BN_EPS == 10:
BN_EPS = 1e-4
if SCI_BN_EPS == 11:
BN_EPS = 5e-4
if SCI_BN_EPS == 12:
BN_EPS = 8e-6
if SCI_BN_EPS == 13:
BN_EPS = 1e-6
if SCI_BN_EPS == 14:
BN_EPS = 8e-5
print('BN Batch EPS: ', BN_EPS)
SCI_BN_STATS = int(SCI_BN_STATS)
if SCI_BN_STATS == 0:
BN_STATS = True
if SCI_BN_STATS == 1:
BN_STATS = False
print('BN Batch STATS: ', BN_STATS)
cnn = CNN6(L_FIRST, SCI_L_SECOND, KERNEL_X,
SCI_BIAS, SCI_BN_MOMENTUM, SCI_RELU,
SCI_DROPOUT, dataset.CLASSES,
BN_EPS, BN_STATS, SCI_LAST_LAYER, SCI_ACT_LAYER)
optimizer = Utillities.optimization_algorithms(SCI_optimizer,cnn, SCI_LR, SCI_SGD_MOMENTUM,
SCI_REGULARIZATION)
if GPU_SELECT == 2:
if torch.cuda.device_count() > 1:
cnn = nn.DataParallel(cnn,device_ids=[0, 1], dim = 0)
cnn = cnn.cuda()
if GPU_SELECT == 1:
cnn.to(device)
if GPU_SELECT == 0:
cnn.to(device)
cnn.apply(CNN6.weights_init2)
#cnn.apply(CNN6.weights_reset)
cnn.share_memory()
loss_func = nn.CrossEntropyLoss()
def create_loss(LOSS):
print('*** LOSS ******:', LOSS)
if LOSS == 1:
loss_func = nn.BCELoss()
print('********* BCELoss')
if LOSS == 2:
loss_func = nn.MultiMarginLoss()
print('********* MMLoss')
if LOSS == 4:
loss_func = nn.CrossEntropyLoss()
print('********* CrossEntropyLoss ')
if LOSS == 3:
loss_func = nn.TripletMarginLoss()
print('********* TripletMarginLoss ')
return loss_func
MM = float(str(SCI_MM))
LR = float(str(SCI_LR))
train_losses = [] # to track the training loss as the model trains
output = 0
loss = 0
accuracy = 0
early_stopping.counter = 0
early_stopping.best_score = None
early_stopping.early_stop = False
early_stopping.verbose = False
TEST_RESULTS = torch.zeros(1,2)
loss_type = create_loss(SCI_loss_type)
#cnn, optimizer = amp.initialize(
# cnn, optimizer, opt_level=BITS,
# keep_batchnorm_fp32=True, loss_scale="dynamic"
#)
Utillities.listing(optimizer, SCI_SGD_MOMENTUM, SCI_BN_MOMENTUM,
SCI_L_SECOND, SCI_LR, SCI_RELU,
SCI_BIAS, SCI_loss_type, SCI_REGULARIZATION,
SCI_BATCH_SIZE, SCI_DROPOUT, SCI_LAST_LAYER, SCI_ACT_LAYER)
# Data Loader for easy mini-batch return in training
SCI_BATCH_SIZE = int(SCI_BATCH_SIZE)
train_loader = Data.DataLoader(dataset = dataset.train_dataset, batch_size = SCI_BATCH_SIZE, shuffle = True, num_workers = 0, drop_last=True, pin_memory=True)
validation_loader = Data.DataLoader(dataset = dataset.validation_dataset, batch_size = 30, shuffle = True, num_workers = 0, drop_last=True, pin_memory=True)
test_loader = Data.DataLoader(dataset = dataset.test_dataset, batch_size = 300, shuffle = True, num_workers = 0, drop_last=True, pin_memory=True)
flag = True;
for epoch in range(SCI_EPOCHS):
loss = None
cnn.train().cuda()
for step, (train_data, train_target) in enumerate(train_loader):
train_data, train_target = train_data.to(device), train_target.to(device)
output = cnn(train_data) # forward pass: compute predicted outputs by passing inputs to the model
loss = loss_func(output, train_target)
train_losses.append(loss.item())
#batch_loss.backward()
loss.backward() # backward pass: compute gradient of the loss with respect to model parameters
optimizer.zero_grad()
optimizer.step() # perform a single optimization step (parameter update)
cnn.eval().cuda() # switch to evaluation (no change) mode
valid_loss = 0
accuracy = 0
running_loss = 0.0
with torch.no_grad():
for step, (validation_data, validation_target) in enumerate(validation_loader):
validation_data, validation_target = validation_data.to(device), validation_target.to(device)
output = cnn(validation_data)
valid_loss = loss_func(output, validation_target)
running_loss += valid_loss.item()
epoch_val_loss = running_loss / len(validation_loader)
if epoch % 3 == 0:
SCI_LR, flag = Utillities.variable_learning_rate(SCI_LR, LR_MIN, LR_MAX, 2, flag)
SCI_DROPOUT = SCI_DROPOUT / 1.02
early_stopping(epoch_val_loss, cnn) #print('validation loss:',epoch_val_loss)
train_losses = []
if early_stopping.early_stop:
if os.path.exists('checkpoint.pt'):
#cnn = TheModelClass(*args, **kwargs)
print("Loaded the model with the lowest Validation Loss!")
cnn.load_state_dict(torch.load('checkpoint.pt')) # Choose whatever GPU device number you want
cnn.to(device)
break
running_loss = 0.0
cnn.eval()
class_correct = list(0. for i in range(1000))
class_total = list(0. for i in range(1000))
with torch.no_grad():
for (test_data, test_target) in test_loader:
test_data, test_target = test_data.to(device), test_target.to(device)
outputs = cnn(test_data)
_, predicted = torch.max(outputs, 1)
c = (predicted == test_target).squeeze()
dx = ((c.cpu()).numpy()).astype(int)
#dx = 600
for i in range(test_target.size(0)):
label = test_target[i]
class_correct[label] += c[i].item()
class_total[label] += 1
for i in range(dataset.CLASSES):
TEST_RESULTS[0,i] = class_correct[i] / dataset.TESTED_ELEMENTS[i]
print('Class: ',i,' accuracy: ', TEST_RESULTS[0,i])
print('Class: ',i,' correct: ', class_correct[i],' of ',dataset.TESTED_ELEMENTS[i])
#mp.matshow(dx.reshape((20, 30)))
#mp.ylabel('Correct Results')
#mp.colorbar()
#mp.show()
percent = (TEST_RESULTS[0,0]+TEST_RESULTS[0,1])/2
print('Final percentage: ',percent)
CreditCost = 0
CreditCost = int((1 - TEST_RESULTS[0,0]) * dataset.TESTED_ELEMENTS[0] + (1 - TEST_RESULTS[0,1]) * dataset.TESTED_ELEMENTS[1] * 5)
#if TEST_RESULTS[0,0] < 0.05 or TEST_RESULTS[0,1] < 0.05 :
# CreditCost = CreditCost + 300
print('Last epoch: ', epoch)
if os.path.exists('checkpoint.pt'):
os.remove('checkpoint.pt')
#print('Credit Cost: ',CreditCost)
#CreditCost = CreditCost + (SCI_SGD_MOMENTUM + SCI_DROPOUT + SCI_BATCH_SIZE + SCI_L_SECOND + SCI_optimizer + SCI_loss_type+ SCI_LR+ SCI_BN_EPS+SCI_BN_STATS+SCI_LAST_LAYER+SCI_ACT_LAYER)/10000
print('Credit Cost: ',CreditCost)
if -CreditCost > MaxCredit :
MaxCredit = -CreditCost
print('Best Score So Far: ',MaxCredit)
print()
print()
#CreditVector[count] = MaxCredit
#CreditVec[count] = count
# plot the data
#fig = mp.figure()
#ax = fig.add_subplot(1, 1, 1)
#ax.plot(CreditVec, -CreditVector, color='tab:orange')
#print(CreditVec, -CreditVector)
#count = count + 1
# display the plot
#mp.show()
# return function (with unknown internals) we wish to maximize.
return -CreditCost
def main(dataset='opencv_video_seq_dataset.VideoSeqDataset',
np_transforms=None,
tensor_transforms=(
'img_landmarks_transforms.ToTensor()',
'transforms.Normalize(mean=[0.5,0.5,0.5],std=[0.5,0.5,0.5])'),
workers=4,
batch_size=4):
import time
from fsgan.utils.obj_factory import obj_factory
from fsgan.utils.img_utils import tensor2bgr
np_transforms = obj_factory(
np_transforms) if np_transforms is not None else []
tensor_transforms = obj_factory(
tensor_transforms) if tensor_transforms is not None else []
img_transforms = img_landmarks_transforms.Compose(np_transforms +
tensor_transforms)
dataset = obj_factory(dataset, transform=img_transforms)
# dataset = VideoSeqDataset(root_path, img_list_path, transform=img_transforms, frame_window=frame_window)
dataloader = data.DataLoader(dataset,
batch_size=batch_size,
num_workers=workers,
pin_memory=True,
drop_last=True,
shuffle=True)
start = time.time()
for frame_window, landmarks_window in dataloader:
# print(frame_window.shape)
if isinstance(frame_window, (list, tuple)):
# For each batch
for b in range(frame_window[0].shape[0]):
# For each frame window in the list
for p in range(len(frame_window)):
# For each frame in the window
for f in range(frame_window[p].shape[2]):
print(frame_window[p][b, :, f, :, :].shape)
# Render
render_img = tensor2bgr(
frame_window[p][b, :, f, :, :]).copy()
landmarks = landmarks_window[p][b, f, :, :].numpy()
# for point in np.round(landmarks).astype(int):
for point in landmarks:
cv2.circle(render_img, (point[0], point[1]), 2,
(0, 0, 255), -1)
cv2.imshow('render_img', render_img)
if cv2.waitKey(0) & 0xFF == ord('q'):
break
else:
# For each batch
for b in range(frame_window.shape[0]):
# For each frame in the window
for f in range(frame_window.shape[2]):
print(frame_window[b, :, f, :, :].shape)
# Render
render_img = tensor2bgr(frame_window[b, :, f, :, :]).copy()
landmarks = landmarks_window[b, f, :, :].numpy()
# for point in np.round(landmarks).astype(int):
for point in landmarks:
cv2.circle(render_img, (point[0], point[1]), 2,
(0, 0, 255), -1)
cv2.imshow('render_img', render_img)
if cv2.waitKey(0) & 0xFF == ord('q'):
break
end = time.time()
print('elapsed time: %f[s]' % (end - start))
parser.add_argument('--epochs', type=int, default=10, metavar='N',help='number of epochs to train (default: 10)')
parser.add_argument('--lr', type=float, default=0.01, metavar='LR',help='learning rate (default: 0.01)')
parser.add_argument('--momentum', type=float, default=0.5, metavar='M',help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda', action='store_true', default=False,help='disables CUDA training')
parser.add_argument('--seed', type=int, default=1, metavar='S',help='random seed,default=1)')
parser.add_argument('--eps', type=float, default=1e-5, metavar='LR',help='learning rate,default=1e-5')
parser.add_argument('--log-interval', type=int, default=100, metavar='N',help='for printing training data is log interval')
args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
#train_loader
train_loader = D.DataLoader(datasets.MNIST('./data', train=True, download=True,
transform=transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),batch_size=args.batch_size, shuffle=True)
#test_loaer
test_loader = D.DataLoader(datasets.MNIST('./data', train=False, transform=transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),batch_size=args.test_batch_size, shuffle=True)
################################################################
#MODEL
class Model(nn.Module):
def __init__(self):
super(Model, self).__init__()
self.layer1 = nn.Sequential(
BinaryConv2d(1, 16, kernel_size=5, padding=2),
def train():
if cfg.dataset is None:
print("Missing dataset in config!")
exit(-1)
save_folder = Path(args.save_folder)
save_folder.mkdir(exist_ok=True, parents=True)
epoch_status_file_path = Path(args.epoch_status_file)
dataset = COCODetection(
image_path=cfg.dataset.train_images,
info_file=cfg.dataset.train_info,
transform=SSDAugmentation(cfg, MEANS),
label_map=cfg.dataset.get_valid_label_map(),
)
if args.validation_epoch > 0:
setup_eval()
val_dataset = COCODetection(
image_path=cfg.dataset.valid_images,
info_file=cfg.dataset.valid_info,
transform=BaseTransform(cfg, MEANS),
label_map=cfg.dataset.get_valid_label_map(),
)
# Parallel wraps the underlying module, but when saving and loading we don't want that
yolact_net = Yolact(cfg)
net = yolact_net
net.train()
if args.log:
log = Log(
cfg.name,
args.log_folder,
dict(args._get_kwargs()),
overwrite=(args.resume is None),
log_gpu_stats=args.log_gpu,
)
# I don't use the timer during training (I use a different timing method).
# Apparently there's a race condition with multiple GPUs, so disable it just to be safe.
timer.disable_all()
# Both of these can set args.resume to None, so do them before the check
if args.resume == "interrupt":
args.resume = SavePath.get_interrupt(args.save_folder)
elif args.resume == "latest":
args.resume = SavePath.get_latest(args.save_folder, cfg.name)
if args.resume is not None:
print("Resuming training, loading {}...".format(args.resume))
yolact_net.load_weights(args.resume)
if args.start_iter == -1:
args.start_iter = SavePath.from_str(args.resume).iteration
else:
print("Initializing weights...")
yolact_net.init_weights(backbone_path=cfg.backbone.path)
optimizer = optim.SGD(
net.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.decay
)
criterion = MultiBoxLoss(
num_classes=cfg.num_classes,
pos_threshold=cfg.positive_iou_threshold,
neg_threshold=cfg.negative_iou_threshold,
negpos_ratio=cfg.ohem_negpos_ratio,
cfg=cfg,
)
if args.batch_alloc is not None:
args.batch_alloc = [int(x) for x in args.batch_alloc.split(",")]
if sum(args.batch_alloc) != args.batch_size:
print(
"Error: Batch allocation (%s) does not sum to batch size (%s)."
% (args.batch_alloc, args.batch_size)
)
exit(-1)
net = CustomDataParallel(NetLoss(net, criterion))
if args.cuda:
net = net.cuda()
# Initialize everything
if not cfg.freeze_bn:
yolact_net.freeze_bn() # Freeze bn so we don't kill our means
yolact_net(torch.zeros(1, 3, cfg.max_size, cfg.max_size).cuda())
if not cfg.freeze_bn:
yolact_net.freeze_bn(True)
# loss counters
loc_loss = 0
conf_loss = 0
iteration = max(args.start_iter, 0)
last_time = time.time()
epoch_size = math.ceil(len(dataset) / args.batch_size)
print(f"\n\t ==> Number of iterations per epoch: {epoch_size}")
num_epochs = min(math.ceil(cfg.max_iter / epoch_size), cfg.max_num_epochs)
print(f"\t ==> Number of epochs: {num_epochs}\n")
# Which learning rate adjustment step are we on? lr' = lr * gamma ^ step_index
step_index = 0
data_loader = data.DataLoader(
dataset,
args.batch_size,
num_workers=args.num_workers,
shuffle=True,
collate_fn=detection_collate,
pin_memory=True,
)
save_path = lambda epoch, iteration: SavePath(cfg.name, epoch, iteration).get_path(
root=args.save_folder
)
time_avg = MovingAverage()
global loss_types # Forms the print order
loss_avgs = {k: MovingAverage(100) for k in loss_types}
print("Begin training!")
print()
# try-except so you can use ctrl+c to save early and stop training
try:
for epoch in range(num_epochs):
with epoch_status_file_path.open(
"w", encoding="utf-8"
) as epoch_status_file:
json.dump({"cur_epoch": epoch}, epoch_status_file)
# Resume from start_iter
if (epoch + 1) * epoch_size < iteration:
continue
for datum in data_loader:
# Stop if we've reached an epoch if we're resuming from start_iter
if iteration == (epoch + 1) * epoch_size:
break
# Stop at the configured number of iterations even if mid-epoch
if iteration == cfg.max_iter:
break
# Change a config setting if we've reached the specified iteration
changed = False
for change in cfg.delayed_settings:
if iteration >= change[0]:
changed = True
cfg.replace(change[1])
# Reset the loss averages because things might have changed
for avg in loss_avgs:
avg.reset()
# If a config setting was changed, remove it from the list so we don't keep checking
if changed:
cfg.delayed_settings = [
x for x in cfg.delayed_settings if x[0] > iteration
]
# Warm up by linearly interpolating the learning rate from some smaller value
if cfg.lr_warmup_until > 0 and iteration <= cfg.lr_warmup_until:
set_lr(
optimizer,
(args.lr - cfg.lr_warmup_init)
* (iteration / cfg.lr_warmup_until)
+ cfg.lr_warmup_init,
)
# Adjust the learning rate at the given iterations, but also if we resume from past that iteration
while (
step_index < len(cfg.lr_steps)
and iteration >= cfg.lr_steps[step_index]
):
step_index += 1
set_lr(optimizer, args.lr * (args.gamma ** step_index))
# Zero the grad to get ready to compute gradients
optimizer.zero_grad()
# Forward Pass + Compute loss at the same time (see CustomDataParallel and NetLoss)
losses = net(datum)
losses = {
k: (v).mean() for k, v in losses.items()
} # Mean here because Dataparallel
loss = sum([losses[k] for k in losses])
# no_inf_mean removes some components from the loss, so make sure to backward through all of it
# all_loss = sum([v.mean() for v in losses.values()])
# Backprop
loss.backward() # Do this to free up vram even if loss is not finite
if torch.isfinite(loss).item():
optimizer.step()
# Add the loss to the moving average for bookkeeping
for k in losses:
loss_avgs[k].add(losses[k].item())
cur_time = time.time()
elapsed = cur_time - last_time
last_time = cur_time
# Exclude graph setup from the timing information
if iteration != args.start_iter:
time_avg.add(elapsed)
if iteration % 10 == 0:
eta_str = str(
datetime.timedelta(
seconds=(cfg.max_iter - iteration) * time_avg.get_avg()
)
).split(".")[0]
total = sum([loss_avgs[k].get_avg() for k in losses])
loss_labels = sum(
[
[k, loss_avgs[k].get_avg()]
for k in loss_types
if k in losses
],
[],
)
print(
(
"[%3d] %7d ||"
+ (" %s: %.3f |" * len(losses))
+ " T: %.3f || ETA: %s || timer: %.3f"
)
% tuple(
[epoch, iteration] + loss_labels + [total, eta_str, elapsed]
),
flush=True,
)
if args.log:
precision = 5
loss_info = {k: round(losses[k].item(), precision) for k in losses}
loss_info["T"] = round(loss.item(), precision)
if args.log_gpu:
log.log_gpu_stats = iteration % 10 == 0 # nvidia-smi is sloooow
log.log(
"train",
loss=loss_info,
epoch=epoch,
iter=iteration,
lr=round(cur_lr, 10),
elapsed=elapsed,
)
log.log_gpu_stats = args.log_gpu
iteration += 1
if iteration % args.save_interval == 0 and iteration != args.start_iter:
if args.keep_latest:
latest = SavePath.get_latest(args.save_folder, cfg.name)
print("Saving state, iter:", iteration)
yolact_net.save_weights(save_path(epoch, iteration))
if args.keep_latest and latest is not None:
if (
args.keep_latest_interval <= 0
or iteration % args.keep_latest_interval
!= args.save_interval
):
print("Deleting old save...")
os.remove(latest)
# This is done per epoch
if args.validation_epoch > 0:
if epoch % args.validation_epoch == 0 and epoch > 0:
compute_validation_map(
epoch,
iteration,
yolact_net,
val_dataset,
log if args.log else None,
)
# Compute validation mAP after training is finished
compute_validation_map(
epoch, iteration, yolact_net, val_dataset, log if args.log else None
)
except KeyboardInterrupt:
if args.interrupt:
print("Stopping early. Saving network...")
# Delete previous copy of the interrupted network so we don't spam the weights folder
SavePath.remove_interrupt(args.save_folder)
# Wait for all torch processes to finish their task
time.sleep(1)
yolact_net.save_weights(save_path(epoch, repr(iteration) + "_interrupt"))
exit()
print("Saving weights...")
yolact_net.save_weights(save_path(epoch, repr(iteration) + "_end"))
self.fc = nn.Sequential(nn.Linear(3 * 3 * 128, 256), nn.ReLU(),
nn.BatchNorm1d(256), nn.Linear(256, 7))
def forward(self, x):
#image size (48,48)
x = self.conv1(x) #(24,24)
x = self.conv2(x) #(12,12)
x = self.conv3(x) #(6,6)
x = self.conv4(x) #(3,3)
x = x.view(-1, 3 * 3 * 128)
x = self.fc(x)
return x
if __name__ == '__main__':
model = Net()
model.load_state_dict(torch.load('model_70.pth'))
model.eval()
transform = transforms.Compose([transforms.ToTensor()])
with torch.no_grad():
test_image = sorted(glob.glob(os.path.join(sys.argv[1], '*.jpg')))
test_set = test_Dataset(test_image, transform)
test_loader = Data.DataLoader(test_set, batch_size=128, shuffle=False)
test_preds = get_all_preds(model, test_loader)
predict = torch.max(test_preds, 1)[1]
end = pd.DataFrame({'id': range(0, len(test_image)), 'label': predict})
end.to_csv(sys.argv[2], index=False)
p = (weight.t() / torch.sum(weight, dim=1)).t()
log_q = torch.log(q)
loss = F.kl_div(log_q, p)
return loss, p
loss, p = loss_func(feat_init)
train_data = torchvision.datasets.MNIST(
'../mnist',
train=True,
transform=torchvision.transforms.ToTensor(),
download=False)
train_loader = Data.DataLoader(dataset=train_data,
batch_size=256,
shuffle=True,
num_workers=2)
optimizer = torch.optim.SGD(list(vae.encoder.parameters()) +
list(vae.fc1.parameters()) + [cluster_centers],
lr=2.0)
def dist_2_label(q_t):
_, label = torch.max(q_t, dim=1)
return label.data.cpu().numpy()
for epoch in range(20):
for step, (batch_x, batch_y) in enumerate(train_loader):
batch_x = Variable(batch_x.view(-1, 784)).cuda()
batch_feat, _ = vae.encode(batch_x)
import torch
import torch.utils.data as Data
torch.manual_seed(1) # reproducible
BATCH_SIZE = 5
# BATCH_SIZE = 8
x = torch.linspace(1, 10, 10) # this is x data (torch tensor)
y = torch.linspace(10, 1, 10) # this is y data (torch tensor)
torch_dataset = Data.TensorDataset(x, y)
loader = Data.DataLoader(
dataset=torch_dataset, # torch TensorDataset format
batch_size=BATCH_SIZE, # mini batch size
shuffle=True, # random shuffle for training
num_workers=2, # subprocesses for loading
)
def show_batch():
for epoch in range(3): # train entire dataset 3 times
for step, (batch_x, batch_y) in enumerate(loader): # for each training step
# train your data...
print('Epoch: ', epoch, '| Step: ', step, '| batch x: ',
batch_x.numpy(), '| batch y: ', batch_y.numpy())
if __name__ == '__main__':
show_batch()
parser.add_argument('--train_batch', type=int, default=16)
parser.add_argument('--val_batch', type=int, default=1)
parser.add_argument('--num_workers', type=int, default=8)
parser.add_argument('--pretrained', type=int, default=0)
parser.add_argument('--path', type=str, default=None)
parser.add_argument('--epoch', type=int, default=20)
parser.add_argument('--learning_rate', type=float, default=0.001)
parser.add_argument('--momentum', type=float, default=0.9)
parser.add_argument('--weight_decay', type=float, default=5e-4)
parser.add_argument('--stage', type=int, default=1)
parser.add_argument('--evaluate', type=int, default=0)
args = parser.parse_args()
trainset = AudioVisualData()
testset = TestData()
trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True,
collate_fn=DataAllocate, num_workers=args.num_workers)
testloader = data.DataLoader(testset, batch_size=args.val_batch, shuffle=False,
collate_fn=TestAllocate, num_workers=args.num_workers)
vision_net = resnet18(modal='vision', pretrained=True)
audio_net = resnet18(modal='audio')
if args.evaluate:
net = Location(vision_net, audio_net).cuda()
net.load_state_dict(torch.load(args.path))
test(net, testloader)
exit()
net = MTask(vision_net, audio_net).cuda() if args.stage == 1 \
else Align(vision_net, audio_net).cuda()
if args.pretrained:
return len(self.data_file_list)
model = smp.Unet(
encoder_name='resnet34',
encoder_weights='imagenet',
classes=1,
activation='sigmoid'
)
preproc_fn = smp.encoders.get_preprocessing_fn('resnet34', 'imagenet')
x_train_path = r"D:\liver2\liver2\train\imgs"
y_train_path = r"D:\liver2\liver2\train\masks"
x_test_path = r"D:\liver2\liver2\test\imgs"
y_test_path = r"D:\liver2\liver2\test\masks"
train_dataset = LvDataset(x_train_path, y_train_path, [1], dataset_augmentation(), dataset_preprocessing(preproc_fn))
test_dataset = LvDataset(x_test_path, y_test_path, [1], dataset_augmentation(), dataset_preprocessing(preproc_fn))
train_loader = torchdata.DataLoader(train_dataset, batch_size=5, shuffle=True, num_workers=0)
test_loader = torchdata.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=0)
best_model = torch.load('./model_est.pth')
n = np.random.choice(len(test_loader))
image, mask = test_dataset[n]
mask = mask.squeeze()
x_tensor = torch.from_numpy(image).to("cpu").unsqueeze(0)
pred_mask = best_model.predict(x_tensor)
pred_mask = pred_mask.squeeze().cpu().numpy().round()
image = image.transpose(1,2,0).astype("float32")
visualize_help(image=image,original_mask=mask,predicted_mask=pred_mask)
torch.manual_seed(1)
EPOCH = 1
BATCH_SIZE = 50
LR = 0.001
DOWNLOAD_MNIST = False
train_data = torchvision.datasets.MNIST(
root='./mnist/',
train=True,
transform=torchvision.transforms.ToTensor(),
download=DOWNLOAD_MNIST,
)
train_loader = Data.DataLoader(dataset=train_data,
batch_size=BATCH_SIZE,
shuffle=True)
test_data = torchvision.datasets.MNIST(root='./mnist/', train=False)
# !!!!!!!! Change in here !!!!!!!!! #
test_x = Variable(torch.unsqueeze(test_data.test_data, dim=1)).type(
torch.FloatTensor)[:2000].cuda() / 255. # Tensor on GPU
test_y = test_data.test_labels[:2000]
class CNN(nn.Module):
def __init__(self):
super(CNN, self).__init__()
self.conv1 = nn.Sequential(
nn.Conv2d(
feats_name = 'r9y9outputmel'
mel_train = float_datasource(
vox_dir + '/' + 'fnames.train',
vox_dir + '/' + 'etc/falcon_feats.desc', feats_name,
vox_dir + '/' + 'festival/falcon_' + feats_name)
mel_val = float_datasource(vox_dir + '/' + 'fnames.val',
vox_dir + '/' + 'etc/falcon_feats.desc',
feats_name,
vox_dir + '/' + 'festival/falcon_' + feats_name)
# Dataset and Dataloader setup X, mfcc, mel, mol, fnames)
trainset = LIDmfccmelmolDataset(X_train, mfcc_train, mel_train, mol_train,
fnames_train)
train_loader = data_utils.DataLoader(trainset,
batch_size=hparams.batch_size,
num_workers=hparams.num_workers,
shuffle=True,
collate_fn=collate_fn_lidmfccmelmol,
pin_memory=hparams.pin_memory)
valset = LIDmfccmelmolDataset(X_val, mfcc_val, mel_val, mol_val,
fnames_val)
val_loader = data_utils.DataLoader(valset,
batch_size=hparams.batch_size,
num_workers=hparams.num_workers,
shuffle=True,
collate_fn=collate_fn_lidmfccmelmol,
pin_memory=hparams.pin_memory)
# Model
model = LIDMixtureofExpertsmfccattention(39, 80)
model = model.cuda()
def main():
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
cudnn.enabled = True
gpu = args.gpu
# create network
model = Res_Deeplab(num_classes=args.num_classes)
# load pretrained parameters
if args.restore_from[:4] == 'http' :
saved_state_dict = model_zoo.load_url(args.restore_from)
else:
saved_state_dict = torch.load(args.restore_from)
# only copy the params that exist in current model (caffe-like)
new_params = model.state_dict().copy()
for name, param in new_params.items():
print name
if name in saved_state_dict and param.size() == saved_state_dict[name].size():
new_params[name].copy_(saved_state_dict[name])
print('copy {}'.format(name))
model.load_state_dict(new_params)
model.train()
model.cuda(args.gpu)
cudnn.benchmark = True
# init D
model_D = FCDiscriminator(num_classes=args.num_classes)
if args.restore_from_D is not None:
model_D.load_state_dict(torch.load(args.restore_from_D))
model_D.train()
model_D.cuda(args.gpu)
if not os.path.exists(args.snapshot_dir):
os.makedirs(args.snapshot_dir)
train_dataset = VOCDataSet(args.data_dir, args.data_list, crop_size=input_size,
scale=args.random_scale, mirror=args.random_mirror, mean=IMG_MEAN)
train_dataset_size = len(train_dataset)
train_gt_dataset = VOCGTDataSet(args.data_dir, args.data_list, crop_size=input_size,
scale=args.random_scale, mirror=args.random_mirror, mean=IMG_MEAN)
if args.partial_data is None:
trainloader = data.DataLoader(train_dataset,
batch_size=args.batch_size, shuffle=True, num_workers=5, pin_memory=True)
trainloader_gt = data.DataLoader(train_gt_dataset,
batch_size=args.batch_size, shuffle=True, num_workers=5, pin_memory=True)
else:
#sample partial data
partial_size = int(args.partial_data * train_dataset_size)
if args.partial_id is not None:
train_ids = pickle.load(open(args.partial_id))
print('loading train ids from {}'.format(args.partial_id))
else:
train_ids = range(train_dataset_size)
np.random.shuffle(train_ids)
pickle.dump(train_ids, open(osp.join(args.snapshot_dir, 'train_id.pkl'), 'wb'))
train_sampler = data.sampler.SubsetRandomSampler(train_ids[:partial_size])
train_remain_sampler = data.sampler.SubsetRandomSampler(train_ids[partial_size:])
train_gt_sampler = data.sampler.SubsetRandomSampler(train_ids[:partial_size])
trainloader = data.DataLoader(train_dataset,
batch_size=args.batch_size, sampler=train_sampler, num_workers=3, pin_memory=True)
trainloader_remain = data.DataLoader(train_dataset,
batch_size=args.batch_size, sampler=train_remain_sampler, num_workers=3, pin_memory=True)
trainloader_gt = data.DataLoader(train_gt_dataset,
batch_size=args.batch_size, sampler=train_gt_sampler, num_workers=3, pin_memory=True)
trainloader_remain_iter = enumerate(trainloader_remain)
trainloader_iter = enumerate(trainloader)
trainloader_gt_iter = enumerate(trainloader_gt)
# implement model.optim_parameters(args) to handle different models' lr setting
# optimizer for segmentation network
optimizer = optim.SGD(model.optim_parameters(args),
lr=args.learning_rate, momentum=args.momentum,weight_decay=args.weight_decay)
optimizer.zero_grad()
# optimizer for discriminator network
optimizer_D = optim.Adam(model_D.parameters(), lr=args.learning_rate_D, betas=(0.9,0.99))
optimizer_D.zero_grad()
# loss/ bilinear upsampling
bce_loss = BCEWithLogitsLoss2d()
interp = nn.Upsample(size=(input_size[1], input_size[0]), mode='bilinear')
# labels for adversarial training
pred_label = 0
gt_label = 1
for i_iter in range(args.num_steps):
loss_seg_value = 0
loss_adv_pred_value = 0
loss_D_value = 0
loss_semi_value = 0
optimizer.zero_grad()
adjust_learning_rate(optimizer, i_iter)
optimizer_D.zero_grad()
adjust_learning_rate_D(optimizer_D, i_iter)
for sub_i in range(args.iter_size):
# train G
# don't accumulate grads in D
for param in model_D.parameters():
param.requires_grad = False
# do semi first
if args.lambda_semi > 0 and i_iter >= args.semi_start :
try:
_, batch = trainloader_remain_iter.next()
except:
trainloader_remain_iter = enumerate(trainloader_remain)
_, batch = trainloader_remain_iter.next()
# only access to img
images, _, _, _ = batch
images = Variable(images).cuda(args.gpu)
pred = interp(model(images))
D_out = interp(model_D(F.softmax(pred)))
D_out_sigmoid = F.sigmoid(D_out).data.cpu().numpy().squeeze(axis=1)
# produce ignore mask
semi_ignore_mask = (D_out_sigmoid < args.mask_T)
semi_gt = pred.data.cpu().numpy().argmax(axis=1)
semi_gt[semi_ignore_mask] = 255
semi_ratio = 1.0 - float(semi_ignore_mask.sum())/semi_ignore_mask.size
print('semi ratio: {:.4f}'.format(semi_ratio))
if semi_ratio == 0.0:
loss_semi_value += 0
else:
semi_gt = torch.FloatTensor(semi_gt)
loss_semi = args.lambda_semi * loss_calc(pred, semi_gt, args.gpu)
loss_semi = loss_semi/args.iter_size
loss_semi.backward()
loss_semi_value += loss_semi.data.cpu().numpy()[0]/args.lambda_semi
else:
loss_semi = None
# train with source
try:
_, batch = trainloader_iter.next()
except:
trainloader_iter = enumerate(trainloader)
_, batch = trainloader_iter.next()
images, labels, _, _ = batch
images = Variable(images).cuda(args.gpu)
ignore_mask = (labels.numpy() == 255)
pred = interp(model(images))
loss_seg = loss_calc(pred, labels, args.gpu)
D_out = interp(model_D(F.softmax(pred)))
loss_adv_pred = bce_loss(D_out, make_D_label(gt_label, ignore_mask))
loss = loss_seg + args.lambda_adv_pred * loss_adv_pred
# proper normalization
loss = loss/args.iter_size
loss.backward()
loss_seg_value += loss_seg.data.cpu().numpy()[0]/args.iter_size
loss_adv_pred_value += loss_adv_pred.data.cpu().numpy()[0]/args.iter_size
# train D
# bring back requires_grad
for param in model_D.parameters():
param.requires_grad = True
# train with pred
pred = pred.detach()
D_out = interp(model_D(F.softmax(pred)))
loss_D = bce_loss(D_out, make_D_label(pred_label, ignore_mask))
loss_D = loss_D/args.iter_size/2
loss_D.backward()
loss_D_value += loss_D.data.cpu().numpy()[0]
# train with gt
# get gt labels
try:
_, batch = trainloader_gt_iter.next()
except:
trainloader_gt_iter = enumerate(trainloader_gt)
_, batch = trainloader_gt_iter.next()
_, labels_gt, _, _ = batch
D_gt_v = Variable(one_hot(labels_gt)).cuda(args.gpu)
ignore_mask_gt = (labels_gt.numpy() == 255)
D_out = interp(model_D(D_gt_v))
loss_D = bce_loss(D_out, make_D_label(gt_label, ignore_mask_gt))
loss_D = loss_D/args.iter_size/2
loss_D.backward()
loss_D_value += loss_D.data.cpu().numpy()[0]
optimizer.step()
optimizer_D.step()
print('exp = {}'.format(args.snapshot_dir))
print('iter = {0:8d}/{1:8d}, loss_seg = {2:.3f}, loss_adv_p = {3:.3f}, loss_D = {4:.3f}, loss_semi = {5:.3f}'.format(i_iter, args.num_steps, loss_seg_value, loss_adv_pred_value, loss_D_value, loss_semi_value))
if i_iter >= args.num_steps-1:
print 'save model ...'
torch.save(model.state_dict(),osp.join(args.snapshot_dir, 'VOC_'+str(args.num_steps)+'.pth'))
torch.save(model_D.state_dict(),osp.join(args.snapshot_dir, 'VOC_'+str(args.num_steps)+'_D.pth'))
break
if i_iter % args.save_pred_every == 0 and i_iter!=0:
print 'taking snapshot ...'
torch.save(model.state_dict(),osp.join(args.snapshot_dir, 'VOC_'+str(i_iter)+'.pth'))
torch.save(model_D.state_dict(),osp.join(args.snapshot_dir, 'VOC_'+str(i_iter)+'_D.pth'))
end = timeit.default_timer()
print end-start,'seconds'
def custom_nmf2(V, Sw, Se, params, hyperparams, title, initE, initW):
# decide where to draw negative samples from - entities (= for every (i,j) positive, find (i, *) neg) or words
'''density_column = np.max(V.sum(0).A1)/V.shape[0]
density_row = np.max(V.sum(1).A1)/V.shape[1]
from_entities = True if density_row <= density_column else False
'''
from_entities = check_densities(V)
n_words = Sw.shape[0]
n_entities = Se.shape[0]
print('V shape: ', V.shape)
print('n entities - ', n_entities)
print('n words - ', n_words)
losses = []
# define model
'''initE, initW = init_values(V, hyperparams.n_features)
np.save('initE.npy', initE)
np.save('initW.npy', initW)
initW = np.load('initW.npy')
initE = np.load('initE.npy')
'''
initW = np.abs(initW)
initE = np.abs(initE)
print('Computed initial values')
print('initW min = {}, mean = {}'.format(np.min(initW), np.mean(initW)))
print('initE min = {}, mean = {}'.format(np.min(initE), np.mean(initE)))
model = NMF(n_entities, n_words, hyperparams.n_features, initE, initW,
params)
initW = None
initE = None
print('Created NMF model')
loss = CustomLoss(params.device)
optimizer = hyperparams.optim(model.parameters(),
**hyperparams.optim_settings)
# construct tensors
V_tensor = torch.Tensor(V.todense().astype(np.float32))
V_var = Variable(V_tensor, requires_grad=False)
MatrixDataset = DS(V, hyperparams.n_negatives, from_entities)
trainloader = data.DataLoader(MatrixDataset, batch_size=5, shuffle=True)
losses = []
differences = []
for epoch in range(hyperparams.n_epochs):
# construct batch = one row, n positive samples, 3n negative samples
total_loss = 0
bcount = -1
for batch2 in trainloader:
# transform batch
batch = []
for row, col in batch2:
batch.extend(list(zip(row, col)))
bcount += 1
optimizer.zero_grad()
#print('On batch ', bcount)
if print_time:
# construct sample
t_start = timeit.default_timer()
target = get_target_values(batch, V)
if print_time:
t_diff = (timeit.default_timer() - t_start) * 100
print('Time to collect target - ', t_diff)
if print_time: t_start = timeit.default_timer()
prediction = model(batch)
if print_time:
t_diff = (timeit.default_timer() - t_start) * 100
print('Time to compute prediction - ', t_diff)
if print_time: t_start = timeit.default_timer()
l = loss(target, prediction, hyperparams.lambdas, batch, model, Sw,
Se)
if print_time:
t_diff = (timeit.default_timer() - t_start) * 100
print('Time to compute loss - ', t_diff)
l.backward(retain_graph=True)
optimizer.step()
model.positivise()
total_loss += l.item()
# test the model after each epoch
#total_loss /= hyperparams.n_batches
losses.append(total_loss)
print('Epoch - {}: loss - {}'.format(epoch, total_loss))
if epoch % 50 == 0 or epoch == hyperparams.n_epochs - 1:
t_start = timeit.default_timer()
V_prediction = torch.mm(model.E, model.W)
t_diff = timeit.default_timer() - t_start
#print('time to compute difference - ', t_diff * 100)
diff = (V_tensor - V_prediction).norm(2).item()
differences.append(diff)
print('\t\t \t \t difference - {}'.format(diff))
MatrixDataset.reset()
plot_results(losses, differences, hyperparams.n_epochs, model, title)
return model.E, model.W
num_crop=8
scores_pred = np.zeros((num_crop, len(data_set['test']),4),dtype=np.float32)
q = np.linspace(0, 2600-int(2600*0.8), num_crop,dtype=int)
for aug_cnt, crop_start in enumerate(q):
crop_end = crop_start+int(2600*0.8)
data_set = {}
data_set['test'] = TWdata(index_pd=test_index,
data_root=test_data_root,
classes=le.classes_,
transform=TWAugVal(start_point=crop_start,end_point=crop_end),
)
data_loader = {}
data_loader['test'] = torchdata.DataLoader(data_set['test'], batch_size=bs, num_workers=4,
shuffle=False, pin_memory=True)
idx = 0
for batch_cnt, data in enumerate(data_loader['test']):
print 'aug %d, %d/%d' % (aug_cnt, batch_cnt, len(data_set['test']) // bs)
inputs, labels = data
if usecuda:
inputs = Variable(inputs.cuda())
else:
inputs = Variable(inputs)
# forward
gb,outputs = model(inputs)
# score = F.softmax(outputs)
score = outputs
print(f'Training MNIST Model on {device}\n{"=" * 44}')
# MNIST Data Loading
train_dataset = datasets.MNIST(root='./mnist_data/',
train=True,
transform=transforms.ToTensor(),
download=True)
test_dataset = datasets.MNIST(root='./mnist_data/',
train=False,
transform=transforms.ToTensor())
# Data Loader (Input Pipeline)
train_loader = data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
test_loader = data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False)
class MNISTClassifier(nn.Module):
def __init__(self):
super(MNISTClassifier, self).__init__()
self.l1 = nn.Linear(784, 520)
self.l2 = nn.Linear(520, 320)
self.l3 = nn.Linear(320, 240)
self.l4 = nn.Linear(240, 120)
self.l5 = nn.Linear(120, 10)
def main():
#####################
# Generate data
#####################
# data loader -
if isServerRun:
path = '/home/[email protected]/thesisML/'
else:
path = '/Users/chanaross/dev/Thesis/UberData/'
fileName = '3D_allDataLatLonCorrected_20MultiClass_500gridpickle_30min.p'
dataInput = np.load(path + fileName)
flag_save_network = True
xmin = 0
xmax = dataInput.shape[0]
ymin = 0
ymax = dataInput.shape[1]
zmin = 48
zmax = np.floor(dataInput.shape[2]*0.7).astype(int)
dataInput = dataInput[xmin:xmax, ymin:ymax, zmin:zmax] # shrink matrix size for fast training in order to test model
dataInput = dataInput[5:6, 10:11, :]
smoothParam = [10, 30, 40] #[10, 20, 30, 40] #[10, 15, 30]
testSize = 0.2
# define hyper parameters -
hidden_sizeVec = [64, 128, 256] # [20, 64, 256, 512] #[20, 64, 264, 512] # [20, 40, 64, 128]
sequence_sizeVec = [50, 60] # [5, 10, 20] # [5, 20, 30, 40] # [5, 10, 15] # length of sequence for lstm network
batch_sizeVec = [40]
num_epochs = 500
# optimizer parameters -
lrVec = [0.05, 0.01] #[0.05, 0.2, 0.5] # [0.1, 0.5, 0.9] #[0.1, 0.5, 0.9] # [0.1, 0.01, 0.001]
otVec = [1] # [1, 2]
dmp = 0
mm = 0.9
eps = 1e-08
wdVec = [2e-3]
# create case vectors
networksDict = {}
itr = itertools.product(smoothParam, sequence_sizeVec, batch_sizeVec, hidden_sizeVec, lrVec, otVec, wdVec)
for i in itr:
networkStr = 'smooth_{0}_seq_{1}_bs_{2}_hs_{3}_lr_{4}_ot_{5}_wd_{6}'.format(i[0], i[1], i[2], i[3], i[4], i[5], i[6])
networksDict[networkStr] = {'seq': i[1], 'bs': i[2], 'hs': i[3], 'lr': i[4], 'ot': i[5], 'wd': i[6], 'sm': i[0]}
for netConfig in networksDict:
dataInputSmooth = moving_average(dataInput, networksDict[netConfig]['sm']) # smoothing data so that results are more clear to network
# dataInput[dataInput>1] = 1 # limit all events larger than 10 to be 10
# define important sizes for network -
x_size = dataInputSmooth.shape[0]
y_size = dataInputSmooth.shape[1]
dataSize = dataInputSmooth.shape[2]
class_size = (np.max(np.unique(dataInputSmooth)) + 1).astype(int)
num_train = int((1 - testSize) * dataSize)
grid_size = x_size * y_size
# output file
outFile = open('LSTM_networksOutput.csv', 'w')
outFile.write('Name;finalAcc;finalLoss;trainTime;numWeights;NumEpochs\n')
print('Net Parameters: ' + netConfig)
# create network based on input parameter's -
hidden_size = networksDict[netConfig]['hs']
batch_size = networksDict[netConfig]['bs']
sequence_size = networksDict[netConfig]['seq']
lr = networksDict[netConfig]['lr']
ot = networksDict[netConfig]['ot']
wd = networksDict[netConfig]['wd']
my_net = Model(grid_size, hidden_size, batch_size, sequence_size, class_size)
my_net.lstm = my_net.create_lstm(grid_size) # lstm receives all grid points and seq length of
my_net.fc_after_lstm = my_net.create_fc_after_lstm(my_net.hiddenSize, grid_size*class_size)
my_net.to(device)
print("model device is:")
print(next(my_net.parameters()).device)
numWeights = sum(param.numel() for param in my_net.parameters())
print('number of parameters: ', numWeights)
my_net.optimizer = CreateOptimizer(my_net.parameters(), ot, lr, dmp, mm, eps, wd)
my_net.lossCrit = nn.NLLLoss(size_average=True) # nn.BCELoss(size_average=True)
my_net.maxEpochs = num_epochs
my_net.lr = lr
my_net.wd = wd
my_net.smoothingParam = networksDict[netConfig]['sm']
# network_path = '/Users/chanaross/dev/Thesis/MachineLearning/forGPU/GPU_results/limitedZero_500grid/'
# network_name = 'gridSize11_epoch4_batch5_torch.pkl'
# my_net = torch.load(network_path + network_name, map_location=lambda storage, loc: storage)
# load data from data loader and create train and test sets
data_train = dataInputSmooth[:, :, 0:num_train]
data_test = dataInputSmooth[:, :, num_train:]
dataset_uber_train = DataSet_oneLSTM_allGrid(data_train, sequence_size)
dataset_uber_test = DataSet_oneLSTM_allGrid(data_test , sequence_size)
# creating data loader
dataloader_uber_train = data.DataLoader(dataset=dataset_uber_train, batch_size=batch_size, shuffle=False)
dataloader_uber_test = data.DataLoader(dataset=dataset_uber_test , batch_size=batch_size, shuffle=False)
netOutDict = {}
labelsOutDict = {}
for numEpoch in range(num_epochs):
my_net.loss = None
# for each epoch, calculate loss for each batch -
my_net.train()
localLoss = [4]
accTrain = [0]
rmseTrain = [1]
trainCorr = 0.0
trainTot = 0.0
if (1+numEpoch)%40 == 0:
if my_net.optimizer.param_groups[0]['lr'] > 0.001:
my_net.optimizer.param_groups[0]['lr'] = my_net.optimizer.param_groups[0]['lr']/2
else:
my_net.optimizer.param_groups[0]['lr'] = 0.001
print('lr is: %.6f' % my_net.optimizer.param_groups[0]['lr'])
netOutList = []
labelOutList = []
for i, (input, labels) in enumerate(dataloader_uber_train):
inputD = input.to(device)
labelsD = labels.to(device)
my_net.loss = None
# create torch variables
# input is of size [batch_size, grid_id, seq_size]
inputVar = Variable(inputD).to(device)
labVar = Variable(labelsD).to(device)
# if isServerRun:
# labVar = labVar.type(torch.cuda.FloatTensor)
# else:
# labVar = labVar.type(torch.FloatTensor)
# reset gradient
my_net.optimizer.zero_grad()
# forward
grid_size = labels.shape[1]
local_batch_size = input.shape[0]
# input to LSTM is [seq_size, batch_size, grid_size] , will be transferred as part of the forward
netOut = my_net.forward(inputVar)
netOut = netOut.view(local_batch_size, class_size, grid_size)
_, labTrain = torch.max(torch.exp(netOut.data), 1)
my_net.calcLoss(netOut, labVar)
# backwards
my_net.backward()
# optimizer step
my_net.optimizer.step()
# local loss function list
localLoss.append(my_net.loss.item())
# if isServerRun:
# labTrain = labTrain.cpu()
if isServerRun:
labTrainNp = labTrain.type(torch.cuda.LongTensor).cpu().detach().numpy()
# print("number of net labels different from 0 is:" + str(np.sum(labTrainNp > 0)))
# print("number of net labels 0 is:"+str(np.sum(labTrainNp == 0)))
labelsNp = labels.cpu().detach().numpy()
# print("number of real labels different from 0 is:" + str(np.sum(labelsNp > 0)))
# print("number of real labels 0 is:" + str(np.sum(labelsNp == 0)))
trainCorr = torch.sum(labTrain.type(torch.cuda.LongTensor) == labels).cpu().detach().numpy() + trainCorr
else:
labTrainNp = labTrain.long().detach().numpy()
labelsNp = labels.detach().numpy()
trainCorr = torch.sum(labTrain.long() == labels).detach().numpy() + trainCorr
netOutList.append(labTrainNp)
labelOutList.append(labelsNp)
trainTot = labels.size(0) * labels.size(1) + trainTot
rmse = sqrt(metrics.mean_squared_error(labTrainNp.reshape(-1), labelsNp.reshape(-1)))
accTrain.append(100 * trainCorr / trainTot)
rmseTrain.append(rmse)
# output current state
if (i + 1) % 2 == 0:
print('Epoch: [%d/%d1 ], Step: [%d/%d], Loss: %.4f, Acc: %.4f, RMSE: %.4f'
% (numEpoch + 1, my_net.maxEpochs, i + 1,
dataloader_uber_train.batch_size,
my_net.loss.item(), accTrain[-1], rmseTrain[-1]))
# if (i+1) % 20 == 0:
# if ((localLoss[-1] < np.max(np.array(localLoss[0:-1]))) or (accTrain[-1] > np.max(np.array(accTrain[0:-1])))) and flag_save_network:
# # pickle.dump(my_net, open("gridSize" + str(xmax - xmin) + "_epoch" + str(numEpoch+1) + "_batch" + str(i+1) + ".pkl", 'wb'))
# my_net.saveModel("gridSize" + str(xmax - xmin) + "_epoch" + str(numEpoch+1) + "_batch" + str(i+1) + "_torch.pkl")
# # networkStr = "gridSize" + str(xmax - xmin) + "_epoch" + str(numEpoch+1) + "_batch" + str(i+1)
# # outArray = np.stack([np.array(localLoss), np.array(accTrain)])
# # np.save(networkStr + "_oArrBatch.npy", outArray)
my_net.lossVecTrain.append(np.average(localLoss))
my_net.accVecTrain.append(np.average(accTrain))
my_net.rmseVecTrain.append(np.average(rmseTrain))
# test network for each epoch stage
accEpochTest, lossEpochTest, rmseEpochTest = my_net.test_spesific(testLoader=dataloader_uber_test)
my_net.accVecTest.append(accEpochTest)
my_net.lossVecTest.append(lossEpochTest)
my_net.rmseVecTest.append(rmseEpochTest)
netOutDict[numEpoch] = netOutList
labelsOutDict[numEpoch] = labelOutList
if (flag_save_network):
my_net.saveModel(netConfig + "_torch.pkl")
# outArray = np.stack([np.array(my_net.lossVecTest), np.array(my_net.lossVecTrain),
# np.array(my_net.accVecTest), np.array(my_net.accVecTrain)])
# np.save("gridSize" + str(xmax - xmin) + "_epoch" + str(numEpoch) + "_oArrBatch.npy", outArray)
my_net.finalAcc = accEpochTest
my_net.finalLoss = lossEpochTest
my_net.finalRmse = rmseEpochTest
# name, HyperPerams, accur, num total weights
# err vs epoch, loss vs epoch,
saveFile(netOutDict, 'netDict')
saveFile(labelsOutDict, 'labelsDict')
strWrite = '{0};{1};{2};{3};{4}\n'.format(netConfig, my_net.finalAcc, my_net.finalLoss, numWeights, my_net.maxEpochs)
outFile.write(strWrite)
outFile.close()
return
flat_list.append(item)
"""
flatten = lambda l: [item for sublist in l for item in sublist]
x_train_flat = flatten(x_train.values.tolist())
x_test_flat = flatten(x_test.values.tolist())
# Datasets
partition = {'train': x_train_flat, 'validation': x_test_flat}
labels = data.set_index('name').to_dict()
"""
partition={'train': ['id-1', 'id-2', 'id-3'], 'validation': ['id-4']}
labels={'id-1': 0, 'id-2': 1, 'id-3': 2, 'id-4': 1}
"""
# Generators
training_set = Dataset(partition['train'], labels)
training_generator = Data.DataLoader(training_set, **params)
#inputs, labels,area=next(iter(training_generator))
#plt.imshow(moveaxis(a[0,:,:,:],2, 0))
validation_set = Dataset(partition['validation'], labels)
validation_generator = Data.DataLoader(validation_set, **params)
dataloaders_dict = {'train': training_generator, 'val': validation_generator}
MINIBATCH_SIZE = 8
"""
train_inputs, train_labels = torch.FloatTensor(normalizedImage(x_train_image)), torch.FloatTensor(y_train[:])
test_inputs, test_labels = torch.FloatTensor(normalizedImage(x_test_image)), torch.FloatTensor(y_test[:])
loader_train = Data.DataLoader(dataset=Data.TensorDataset(train_inputs, train_labels),
batch_size=MINIBATCH_SIZE,shuffle=True,
num_workers=0)
def train_single_domain_transfer(args, wf, src, repeat_seed):
# tb_dir = 'runs/{}_sup_base_{}_source_{}_train_num_{}_tune_{}_{}'.format(
# args.test, args.base_model, src, args.train_num, args.n_tune, repeat_seed)
# if os.path.exists(tb_dir) and os.path.isdir(tb_dir):
# shutil.rmtree(tb_dir)
# writer = SummaryWriter(tb_dir)
args.cuda = not args.no_cuda and torch.cuda.is_available()
say('cuda is available %s\n' % args.cuda)
np.random.seed(args.seed + repeat_seed)
torch.manual_seed(args.seed + repeat_seed)
if args.cuda:
torch.cuda.manual_seed(args.seed + repeat_seed)
pretrain_emb = torch.load(
os.path.join(settings.OUT_DIR, "rnn_init_word_emb.emb"))
src_model_dir = os.path.join(settings.OUT_DIR, src)
if args.base_model == "cnn":
encoder_src = CNNMatchModel(input_matrix_size1=args.matrix_size1,
input_matrix_size2=args.matrix_size2,
mat1_channel1=args.mat1_channel1,
mat1_kernel_size1=args.mat1_kernel_size1,
mat1_channel2=args.mat1_channel2,
mat1_kernel_size2=args.mat1_kernel_size2,
mat1_hidden=args.mat1_hidden,
mat2_channel1=args.mat2_channel1,
mat2_kernel_size1=args.mat2_kernel_size1,
mat2_hidden=args.mat2_hidden)
elif args.base_model == "rnn":
encoder_src = BiLSTM(pretrain_emb=pretrain_emb,
vocab_size=args.max_vocab_size,
embedding_size=args.embedding_size,
hidden_size=args.hidden_size,
dropout=args.dropout)
else:
raise NotImplementedError
if args.cuda:
encoder_src.load_state_dict(
torch.load(
os.path.join(
src_model_dir,
"{}-match-best-now-train-num-{}-try-{}.mdl".format(
args.base_model, args.train_num, repeat_seed))))
else:
encoder_src.load_state_dict(
torch.load(os.path.join(
src_model_dir,
"{}-match-best-now-train-num-{}-try-{}.mdl".format(
args.base_model, args.train_num, repeat_seed)),
map_location=torch.device('cpu')))
dst_model_dir = os.path.join(settings.OUT_DIR, args.test)
if args.base_model == "cnn":
encoder_dst_pretrain = CNNMatchModel(
input_matrix_size1=args.matrix_size1,
input_matrix_size2=args.matrix_size2,
mat1_channel1=args.mat1_channel1,
mat1_kernel_size1=args.mat1_kernel_size1,
mat1_channel2=args.mat1_channel2,
mat1_kernel_size2=args.mat1_kernel_size2,
mat1_hidden=args.mat1_hidden,
mat2_channel1=args.mat2_channel1,
mat2_kernel_size1=args.mat2_kernel_size1,
mat2_hidden=args.mat2_hidden)
elif args.base_model == "rnn":
encoder_dst_pretrain = BiLSTM(pretrain_emb=pretrain_emb,
vocab_size=args.max_vocab_size,
embedding_size=args.embedding_size,
hidden_size=args.hidden_size,
dropout=args.dropout)
else:
raise NotImplementedError
encoder_dst_pretrain.load_state_dict(
torch.load(
os.path.join(
dst_model_dir,
"{}-match-best-now-train-num-{}-try-{}.mdl".format(
args.base_model, args.train_num, repeat_seed))))
# args = argparser.parse_args()
say(args)
print()
say("Transferring from %s to %s\n" % (src, args.test))
if args.base_model == "cnn":
train_dataset_dst = ProcessedCNNInputDataset(args.test, "train",
args.train_num)
valid_dataset = ProcessedCNNInputDataset(args.test, "valid")
test_dataset = ProcessedCNNInputDataset(args.test, "test")
elif args.base_model == "rnn":
train_dataset_dst = ProcessedRNNInputDataset(args.test, "train",
args.train_num)
valid_dataset = ProcessedRNNInputDataset(args.test, "valid")
test_dataset = ProcessedRNNInputDataset(args.test, "test")
else:
raise NotImplementedError
print("train num", len(train_dataset_dst))
train_loader_dst = data.DataLoader(train_dataset_dst,
batch_size=args.batch_size,
shuffle=False,
num_workers=0)
valid_loader = data.DataLoader(valid_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=0)
test_loader = data.DataLoader(test_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=0)
say("Corpus loaded.\n")
if args.cuda:
encoder_dst_pretrain.cuda()
encoder_src.cuda()
# classifier.cuda()
requires_grad = lambda x: x.requires_grad
# task_params = list(classifier.parameters())
# if args.base_model == "cnn":
# optim_model = optim.Adagrad(
# filter(requires_grad, task_params),
# lr=args.lr,
# weight_decay=args.weight_decay
# )
# elif args.base_model == "rnn":
# optim_model = optim.Adam(
# filter(requires_grad, task_params),
# lr=args.lr,
# weight_decay=args.weight_decay
# )
# else:
# raise NotImplementedError
say("Training will begin from scratch\n")
iter_cnt = 0
min_loss_val = None
max_auc_val = None
best_test_results = None
weights_sources = None
model_dir = os.path.join(settings.OUT_DIR, args.test)
cur_src_idx = source_to_idx[src]
for epoch in range(args.max_epoch):
print("training epoch", epoch)
# iter_cnt = train_epoch(
# iter_cnt,
# [encoder_src, encoder_dst_pretrain], classifier,
# train_loader_dst,
# args,
# optim_model,
# epoch,
# writer
# )
thr, metrics_val = evaluate(
epoch,
[encoder_src, encoder_dst_pretrain],
valid_loader,
True,
args,
# writer
)
_, metrics_test = evaluate(
epoch,
[encoder_src, encoder_dst_pretrain],
test_loader,
False,
args,
# writer,
thr=thr)
if min_loss_val is None or min_loss_val > metrics_val[0]:
print("change val loss from {} to {}".format(
min_loss_val, metrics_val[0]))
min_loss_val = metrics_val[0]
best_test_results = metrics_test
# torch.save(classifier,
# os.path.join(model_dir, "{}_{}_classifier_from_src_{}_train_num_{}_try_{}.mdl".format(
# args.test, args.base_model, cur_src_idx, args.train_num, repeat_seed
# )))
print()
print(
"src: {}, min valid loss {:.4f}, best test metrics: AUC: {:.2f}, Prec: {:.2f}, Rec: {:.2f}, F1: {:.2f}\n"
.format(src, min_loss_val, best_test_results[1] * 100,
best_test_results[2] * 100, best_test_results[3] * 100,
best_test_results[4] * 100))
wf.write(
"from src {}, min valid loss {:.4f}, best test metrics: AUC: {:.2f}, Prec: {:.2f}, Rec: {:.2f}, F1: {:.2f}\n"
.format(src, min_loss_val, best_test_results[1] * 100,
best_test_results[2] * 100, best_test_results[3] * 100,
best_test_results[4] * 100))
# writer.close()
return min_loss_val, best_test_results[1:]
parser.add_argument("--out", default=True, help="save model or not")
parser.add_argument("--gpu", type=str, default="0", help="gpu card ID")
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
cudnn.benchmark = True
############################## PREPARE DATASET ##########################
train_data, test_data, user_num, item_num, train_mat = data_utils.load_all(
)
# construct the train and test datasets
train_dataset = data_utils.BPRData(train_data, item_num, train_mat,
args.num_ng, True)
test_dataset = data_utils.BPRData(test_data, item_num, train_mat, 0, False)
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
test_loader = data.DataLoader(test_dataset,
batch_size=args.test_num_ng + 1,
shuffle=False,
num_workers=0)
########################### CREATE MODEL #################################
model = model.BPR(user_num, item_num, args.factor_num)
#model.cuda()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
weight_decay=args.lamda)
# writer = SummaryWriter() # for visualization
desc="",
removeHs=args.removeHs,
labelspath=args.vscreening,
)
amap = utils.load_amap(args.amap)
testdata.atomicnums_to_idxs(amap)
n_species = len(amap)
mlflow.log_param("n_species", n_species)
testloader = data.DataLoader(
testdata,
batch_size=args.batchsize,
shuffle=False,
collate_fn=loaders.pad_collate,
)
AEVC = utils.loadAEVC(args.aev)
models = [utils.loadmodel(m) for m in args.models]
evaluate(
models,
testloader,
AEVC,
args.outpath,
stage="predict",
baseline=None,
plt=args.plot,
l.backward()
optimizer.step()
train_l_sum += l.item()
train_acc_sum += (outputs.argmax(dim=1) == y).sum().item()
n += y.shape[0]
print('epoch %d loss %f train acc %f' % (epoch + 1, train_l_sum / n, train_acc_sum / n))
if __name__ == '__main__':
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
raw_data, cat_cols, num_cols = load_data()
train_data = preprocess_data(raw_data, cat_cols, num_cols)
# train_data['label'] = raw_data['label']
train_data['label'] = raw_data['Label']
cat_tuple_list = get_cat_tuple_list(train_data, cat_cols)
X = torch.tensor(train_data.drop(['label'], axis=1).values, dtype=torch.float)
y = torch.tensor(train_data.label.values, dtype=torch.long)
dataset = Data.TensorDataset(X, y)
data_iter = Data.DataLoader(dataset=dataset, batch_size=128, shuffle=True)
model = DeepFM(num_cols, cat_cols, cat_tuple_list)
# print(model)
loss = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
# print(model)
train(model, data_iter, device, optimizer, loss, epochs=100)
def main():
logging.basicConfig(level=logging.INFO,
format="%(asctime)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
filename="log.txt",
filemode='w')
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
model = Dronet(PreActBlock, [1, 1, 1], True)
ModelManager.Read('../PyTorch/Models/DronetGray.pt', model)
DATA_PATH = "/Users/usi/PycharmProjects/data/"
picklename = "HimaxDynamic_12_03_20.pickle"
[x_test, y_test,
z_test] = DataProcessor.ProcessTestData(DATA_PATH + picklename, True)
t_test = DataProcessor.GetTimeStampsFromTestData(DATA_PATH + picklename)
if picklename.find(".pickle"):
picklename = picklename.replace(".pickle", '')
x_test2 = []
y_test2 = []
z_test2 = []
# for i in range(len(x_test)):
# gt = y_test[i]
# if ((gt[0] > 1.0) and (gt[0] < 2.0)):
# x_test2.append(x_test[i])
# y_test2.append(y_test[i])
# z_test2.append(z_test[i])
#
# x_test = np.asarray(x_test2)
# y_test = np.asarray(y_test2)
# z_test = np.asarray(z_test2)
test_set = Dataset(x_test, y_test)
params = {'batch_size': 1, 'shuffle': False, 'num_workers': 1}
test_generator = data.DataLoader(test_set, **params)
trainer = ModelTrainer(model)
MSE, MAE, r2_score, outputs, gt_labels = trainer.Test(test_generator)
# utils.SaveResultsToCSV(gt_labels, outputs, t_test, "wow.csv")
model2 = Dronet(PreActBlock, [1, 1, 1], True)
ModelManager.Read('../PyTorch/Models/DronetGrayAug120.pt', model2)
trainer2 = ModelTrainer(model2)
MSE, MAE, r2_score, outputs2, gt_labels2 = trainer2.Test(test_generator)
utils.SaveResultsToCSVWithCamPoses(gt_labels, outputs2, t_test, z_test,
picklename + ".csv")
outputs2 = np.reshape(outputs2, (-1, 4))
h = x_test.shape[2]
w = x_test.shape[3]
x_test = np.reshape(x_test, (-1, h, w))
VizWorldTopView(x_test, y_test, z_test, outputs, outputs2, True,
picklename)
def train(config):
## set pre-process
prep_dict = {}
prep_config = config["prep"]
prep_dict["train_set1"] = prep.image_train( \
resize_size=prep_config["resize_size"], \
crop_size=prep_config["crop_size"])
prep_dict["train_set2"] = prep.image_train( \
resize_size=prep_config["resize_size"], \
crop_size=prep_config["crop_size"])
## prepare data
dsets = {}
dset_loaders = {}
data_config = config["data"]
dsets["train_set1"] = ImageList(open(data_config["train_set1"]["list_path"]).readlines(),
transform=prep_dict["train_set1"])
dset_loaders["train_set1"] = util_data.DataLoader(dsets["train_set1"],
batch_size=data_config["train_set1"]["batch_size"],
shuffle=True, num_workers=data_config["train_set2"]['workers'],
pin_memory=True)
dsets["train_set2"] = ImageList(open(data_config["train_set2"]["list_path"]).readlines(),
transform=prep_dict["train_set2"])
dset_loaders["train_set2"] = util_data.DataLoader(dsets["train_set2"],
batch_size=data_config["train_set2"]["batch_size"],
shuffle=True, num_workers=data_config["train_set2"]['workers'],
pin_memory=True)
hash_bit = config["hash_bit"]
## set base network
net_config = config["network"]
if net_config["net_snapshot"] is None:
base_network = net_config["type"](**net_config["params"])
else:
base_network = torch.load(config["snapshot_path"])
use_gpu = torch.cuda.is_available()
if use_gpu:
base_network = base_network.cuda()
## collect parameters
parameter_list = [{"params": base_network.feature_layers.parameters(), "lr": 1}, \
{"params": base_network.hash_layer.weight, "lr": 10},
{"params": base_network.hash_layer.bias, "lr": 20}]
## set optimizer
optimizer_config = config["optimizer"]
optimizer = optim_dict[optimizer_config["type"]](parameter_list, **(optimizer_config["optim_params"]))
## set loss function
loss_config = config["loss"]
loss_f = loss_dict[loss_config["type"]]
param_lr = []
for param_group in optimizer.param_groups:
param_lr.append(param_group["lr"])
schedule_param = optimizer_config["lr_param"]
lr_scheduler = lr_schedule.schedule_dict[optimizer_config["lr_type"]]
## train
len_train1 = len(dset_loaders["train_set1"]) - 1
len_train2 = len(dset_loaders["train_set2"]) - 1
transfer_loss_value = classifier_loss_value = total_loss_value = 0.0
best_acc = 0.0
t_range = trange(config["num_iterations"], desc="Loss Value", leave=True)
for i in t_range:
if i % config["snapshot_interval"] == 0 or i == config["num_iterations"]-1:
if i == config["num_iterations"]-1: j = i+1
else: j = i
torch.save(base_network, osp.join(config["output_path"], \
"iter_{:05d}_model.pth.tar".format(j)))
## train one iter
base_network.train(True)
optimizer = lr_scheduler(param_lr, optimizer, i, **schedule_param)
optimizer.zero_grad()
if i % len_train1 == 0:
iter1 = iter(dset_loaders["train_set1"])
inputs1, labels1 = iter1.next()
if use_gpu:
inputs1, labels1, = Variable(inputs1).cuda(), Variable(labels1).cuda()
else:
inputs1, labels1 = Variable(inputs1), Variable(labels1)
if config["dual_batch"]:
if i % len_train2 == 0:
iter2 = iter(dset_loaders["train_set2"])
inputs2, labels2 = iter2.next()
if use_gpu:
inputs2, labels2 = Variable(inputs2).cuda(), Variable(labels2).cuda()
else:
inputs2, labels2 = Variable(inputs2), Variable(labels2)
else:
inputs2 = inputs1
labels2 = labels1
inputs = torch.cat((inputs1, inputs2), dim=0)
outputs = base_network(inputs)
similarity_loss = loss_f(outputs.narrow(0, 0, inputs1.size(0)), labels1,
outputs.narrow(0, inputs1.size(0), inputs2.size(0)), labels2,
config_loss=loss_config["loss_param"])
similarity_loss.backward()
if i > 0:
t_range.set_description("Loss Value: %f" % similarity_loss.float())
t_range.refresh()
# print("Iter: {:05d}, loss: {:.3f}".format(i, similarity_loss.float().data[0]))
config["out_file"].write("Iter: {:05d}, loss: {:.3f}".format(i, \
similarity_loss.float()))
optimizer.step()
LR = 0.01
BATCH_SIZE = 32
EPOCH = 12
# fake dataset
x = torch.unsqueeze(torch.linspace(-1, 1, 1000), dim=1)
y = x.pow(2) + 0.1*torch.normal(torch.zeros(*x.size()))
# plot dataset
plt.scatter(x.numpy(), y.numpy())
plt.show()
# 使用上节内容提到的 data loader
torch_dataset = Data.TensorDataset(data_tensor=x, target_tensor=y)
loader = Data.DataLoader(dataset=torch_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2,)
# 默认的 network 形式
class Net(torch.nn.Module):
def __init__(self):
super(Net, self).__init__()
self.hidden = torch.nn.Linear(1, 20) # hidden layer
self.predict = torch.nn.Linear(20, 1) # output layer
def forward(self, x):
x = F.relu(self.hidden(x)) # activation function for hidden layer
x = self.predict(x) # linear output
return x
# 为每个优化器创建一个 net
net_SGD = Net()