def __init__(self):
self.train_dataset = CustomDataset(TRAIN_PATH)
self.train_loader = data_utils.DataLoader(dataset=self.train_dataset,
batch_size=BATCH_SIZE,
shuffle=True)
self.valid_dataset = CustomDataset(VALID_PATH)
self.valid_loader = data_utils.DataLoader(dataset=self.valid_dataset,
batch_size=BATCH_SIZE,
shuffle=True)
self.build_model()
self.train_writer = SummaryWriter('./logs/train')
self.valid_writer = SummaryWriter('./logs/valid')
self.log_params()
return
def load_data(opt,
root_folder,
fpath_label,
batch_size,
shuffle=True,
num_workers=16,
train=False,
num_frames=32):
if train:
transform = transforms.Compose([
# transforms.ToPILImage(),#Converts a Tensor or a numpy of shape H x W x C to a PIL Image C x H x W
transforms.Resize((128, 171)),
transforms.CenterCrop((112, 112)), # Center
transforms.RandomHorizontalFlip(), # 训练集才需要做这一步处理，获得更多的随机化数据
transforms.ToTensor(),
transforms.Normalize(mean=(0.434, 0.405, 0.378),
std=(0.152, 0.149, 0.157))
])
else:
transform = transforms.Compose([
# transforms.ToPILImage(),#Converts a Tensor or a numpy of shape H x W x C to a PIL Image C x H x W
transforms.Resize((128, 171)),
transforms.CenterCrop((112, 112)), # Center
transforms.ToTensor(),
transforms.Normalize(mean=(0.434, 0.405, 0.378),
std=(0.152, 0.149, 0.157))
])
data_ = CustomDataset(opt,
root_folder=root_folder,
fpath_label=fpath_label,
transform=transform,
num_frames=num_frames)
# torch.utils.data.DataLoader
loader_ = data.DataLoader(
dataset=data_, # torch TensorDataset format
batch_size=batch_size, # mini batch size
shuffle=shuffle, # shuffle
num_workers=num_workers) # multi thread
return loader_
from pytorch_lightning.metrics.functional import accuracy
from sklearn.metrics import roc_auc_score, accuracy_score
import warnings
warnings.filterwarnings("ignore")
TRAIN_PATH = '/home/denis/repos/sber_risk_DL/week12/TabNet/data/train_adult.pickle'
VALID_PATH = '/home/denis/repos/sber_risk_DL/week12/TabNet/data/valid_adult.pickle'
BATCH_SIZE = 1200
EPOCHS = 30
#train_writer = SummaryWriter('./logs/train')
#valid_writer = SummaryWriter('./logs/valid')
print('Run train ...')
train_dataset = CustomDataset(TRAIN_PATH)
train_loader = data_utils.DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
shuffle=True)
vall_dataset = CustomDataset(VALID_PATH)
vall_loader = data_utils.DataLoader(dataset=vall_dataset,
batch_size=BATCH_SIZE,
shuffle=False)
emb_dim = 5
input_size = len(train_dataset.numeric_columns) + len(
train_dataset.embedding_columns) * emb_dim
tabnet = TabNet(nd_dim=64,
na_dim=64,
relax_factor=1,
input_size=input_size,
parser.add_argument('--model_save_path', type=str, default='models')
parser.add_argument('--sample_path', type=str, default='./stargan/samples')
parser.add_argument('--result_path', type=str, default='./stargan/results')
# Step size
parser.add_argument('--log_step', type=int, default=10)
parser.add_argument('--sample_step', type=int, default=500)
parser.add_argument('--model_save_step', type=int, default=1000)
return parser
crop_size = 178
image_size = 128
path = '/home/ubuntu/recognition/data/UTKFace/aligned_split_cls/'
mode_train = 'train'
mode_val = 'val'
bsize = 4
active_mode = mode_train
transform = get_transform_by(active_mode, crop_size, image_size)
ds = CustomDataset(path, active_mode, transform)
loader = DataLoader(dataset=ds, batch_size=bsize, shuffle=False)
solver = CustomSolver(loader, get_parser().parse_args())
solver.train()
def main(args):
# Instance of the class that will preprocess and generate proper images for training
train_set = CustomDataset(args.image_dir, original_transform)
# Data loader that will generate the proper batches of images for training
data_loader = torch.utils.data.DataLoader(train_set, batch_size = args.batch_size, shuffle = True, num_workers = args.num_workers)
# Model instance whose architecture was configured in the 'model.py' file
model = Color_model().cuda()
# model.load_state_dict(torch.load(args.load_model))
# Loss function used
criterion = nn.CrossEntropyLoss().cuda()
# Model parameters and optimizer used during the training step
params = list(model.parameters())
optimizer = torch.optim.Adam(params, lr = args.learning_rate)
# Instance of 'NNEncLayer' class that is responsable for to return a probability distribution for each pixel of the (a,b) channels. This class is in 'training_layers.py' file
encode_ab_layer = NNEncLayer()
# Instance of 'NonGrayMaskLayer'. Return 1 for not color images and 0 for grayscale images.
nongraymask_layer = NonGrayMaskLayer()
# Instance of 'PriorBoostLayer'. Returns the weights for every color.
priorboost_layer = PriorBoostLayer()
# Instance of 'ClassRebalance'. Ponders the colors with weights trying to make rare colors to contribute with the model.
class_rebalance_layer = ClassRebalance.apply
#####################################################################
#----------------------->> TRAINING STEP <<-------------------------#
#####################################################################
print("====>> Training step started!! <<====")
# Number of batches
total_batch = len(data_loader)
# Store the loss of every epoch for training dataset.
running_loss_history = []
# Start time to measure time of training
start_time = time.time()
# Main loop, loop for each epoch
for epoch in range(args.num_epochs):
# Every loss per batch is summed to get the final loss for each epoch for training dataset.
running_loss = 0.0
# Loop for each batch of images
for i, (images, img_ab, filename) in enumerate(data_loader):
#print(filename)
# Grayscale images represented by L channel
images = images.unsqueeze(1).float().cuda() # Unsqueeze(1) add one more dimension to the tensor in position 1, than converted to float and loaded to the GPU
# Ground truth represented by (a,b) channels
img_ab = img_ab.float()
# 'encode_ab' -> represents a probability distribution for each pixel of the (a,b) channels
# 'max_encode_ab' -> represents the indexes that have the highest values of probability along each pixel layers
encode_ab, max_encode_ab = encode_ab_layer.forward(img_ab)
#encode_ab = torch.from_numpy(encode_ab).long().cuda()
# 'max_encode_ab' is used as targets. So it is converted to long data type and then loaded to the GPU
targets = torch.Tensor(max_encode_ab).long().cuda()
nongray_mask = torch.Tensor(nongraymask_layer.forward(img_ab)).float().cuda()
prior_boost = torch.Tensor(priorboost_layer.forward(encode_ab)).float().cuda()
prior_boost_nongray = prior_boost * nongray_mask
# The input grayscale images are submitted to the model and the result tensor with shape [Bx313xWxH] is stored in 'output'
outputs = model(images)
# Class Rebalance execution, pondering the gradients.
outputs = class_rebalance_layer(outputs, prior_boost_nongray)
# loss = (criterion(outputs,targets)*(prior_boost_nongray.squeeze(1))).mean()
# The loss is performed for each batch(image)
loss = criterion(outputs,targets)
# Every loss per batch is summed to get the final loss for each epoch.
running_loss += loss.item()
model.zero_grad()
loss.backward()
optimizer.step()
# Print info about the training according to the log_step value
if (i) % args.log_step == 0:
print('Epoch [{}/{}], Batch [{}/{}]'.format(epoch+1, args.num_epochs, i+1, total_batch))
# Save the model according to the checkpoints configured
if epoch in args.checkpoint_step and i == (args.trainDataset_length/args.batch_size)-1:
torch.save(model.state_dict(), os.path.join(args.model_path, 'model-{}-{}.ckpt'.format(epoch + 1, i + 1)))
# Average Loss of an epoch for training dataset.
epoch_loss = running_loss/len(data_loader) # Acumulated Loss divided by number of images batches on training dataset.
running_loss_history.append(epoch_loss)
#print("{:.2f} minutes".format((time.time() - start_time)/60))
print('--------->>> Epoch [{}/{}], Epoch Loss: {:.4f}'.format(epoch+1, args.num_epochs, epoch_loss))
print('Loss History: {}'.format(running_loss_history))
print("{:.2f} minutes".format((time.time() - start_time)/60))
print(" ")
plt.plot(np.arange(0,args.num_epochs), running_loss_history, label='Training Loss')
ax = plt.gca()
ax.set_facecolor((0.85, 0.85, 0.85))
plt.grid(color='w', linestyle='solid')
ax.set_axisbelow(True)
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.savefig(args.save_lossCurve)
def infer_bert(args):
import pandas as pd
assert os.path.exists(
str(args.infer_data_file
)), "The argument --infer_data_file should be a valid file"
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
# load model
cc = ConcatenatedClassifier(
classifier_config_dir=args.classifier_config_dir,
device=device,
task_type='infer',
n_clf_layers=args.n_clf_layers,
use_dm=args.use_dm,
use_pm=args.use_pm,
use_rt=args.use_rt,
use_bio=args.use_bio,
use_name=args.use_name,
use_network=args.use_network,
use_count=args.use_count)
cc.load_state_dict(torch.load(args.model_dir))
cc.to(device)
# load data
data = []
with open(args.infer_data_file) as f:
for line in f:
data.append(json.loads(line))
dataset = CustomDataset(data_list=data,
task_type='infer',
use_pm=args.use_pm,
use_dm=args.use_dm,
use_rt=args.use_rt,
use_bio=args.use_bio,
use_network=args.use_network,
use_count=args.use_count,
use_name=args.use_name,
max_samples=args.max_samples)
data_loader = DataLoader(dataset,
batch_size=args.batch_size * 4,
shuffle=False,
num_workers=4,
collate_fn=dataset.collate_fn)
print("Starting evaluation")
save_dir = args.output_dir # create save directory to store models and training loss
if not os.path.exists(save_dir):
os.makedirs(save_dir)
pbar = tqdm(data_loader)
y_true, y_pred = [], []
with torch.no_grad():
cc.eval()
for batch in pbar:
logits = cc(batch)
y_pred.extend(logits.argmax(1).tolist())
# save
data_name = args.infer_data_file.split('/')[-1]
out = []
for (aid, bid), pred in zip(dataset.dyad_ids, y_pred):
out.append(
(aid, bid,
['social', 'romance', 'family', 'organizational',
'parasocial'][pred]))
df = pd.DataFrame(
out, columns=['user_id_a', 'user_id_b', 'predicted-relationship'])
save_file = os.path.join(args.output_dir, '%s-predictions.tsv' % data_name)
df.to_csv(save_file, sep='\t', index=False)
print("Saved inferred outputs to %s" % save_file)
return
def train_bert(args):
from transformers import AdamW
import gc
from transformers import get_linear_schedule_with_warmup
from sklearn.metrics import f1_score, precision_score, recall_score
assert os.path.exists(
str(args.train_data_file
)), "The argument --train_data_file should be a valid file"
assert os.path.exists(
str(args.val_data_file
)), "The argument --val_train_data_file should be a valid file"
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
# load model
cc = ConcatenatedClassifier(
classifier_config_dir=args.classifier_config_dir,
device=device,
task_type='train',
n_clf_layers=args.n_clf_layers,
use_dm=args.use_dm,
use_pm=args.use_pm,
use_rt=args.use_rt,
use_bio=args.use_bio,
use_name=args.use_name,
use_network=args.use_network,
use_count=args.use_count)
cc.to(device)
# load data
train_data = []
with open(args.train_data_file) as f:
for line in f:
train_data.append(json.loads(line))
val_data = []
with open(args.val_data_file) as f:
for line in f:
val_data.append(json.loads(line))
train_dataset = CustomDataset(data_list=train_data,
task_type='train',
use_pm=args.use_pm,
use_dm=args.use_dm,
use_rt=args.use_rt,
use_bio=args.use_bio,
use_network=args.use_network,
use_count=args.use_count,
use_name=args.use_name,
max_samples=args.max_samples)
train_data_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4,
collate_fn=train_dataset.collate_fn)
val_dataset = CustomDataset(data_list=val_data,
task_type='test',
use_pm=args.use_pm,
use_dm=args.use_dm,
use_rt=args.use_rt,
use_bio=args.use_bio,
use_network=args.use_network,
use_count=args.use_count,
use_name=args.use_name,
max_samples=args.max_samples)
val_data_loader = DataLoader(val_dataset,
batch_size=args.batch_size * 4,
shuffle=False,
num_workers=4,
collate_fn=train_dataset.collate_fn)
print("Starting evaluation")
save_dir = args.output_dir # create save directory to store models and training loss
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# hyperparameters
eps = args.eps
max_grad_norm = args.max_grad_norm
num_training_steps = len(train_data_loader) * args.n_epochs
num_warmup_steps = args.num_warmup_steps
optimizer = AdamW(
cc.parameters(), lr=args.lr, eps=eps
) # To reproduce BertAdam specific behavior set correct_bias=False
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=num_warmup_steps,
num_training_steps=num_training_steps) # PyTorch scheduler
# training progress
total_steps = 0
for epoch in range(args.n_epochs):
# Reset the total loss for this epoch.
total_loss = 0
# For each batch of training data...
pbar = tqdm(train_data_loader)
for step, batch in enumerate(pbar):
# validate
if total_steps % args.valid_interval == 0 and not total_steps == 0:
gc.collect()
torch.cuda.empty_cache()
y_true, y_pred = [], []
with torch.no_grad():
cc.eval()
valid_loss = 0
total_samples = 0
for batch in val_data_loader:
y_true.extend(batch['text'][0].tolist())
loss, logits = cc(batch)
y_pred.extend(logits.argmax(1).tolist())
valid_loss += loss.item() * len(logits)
total_samples += len(logits)
valid_loss /= total_samples
# save valid loss
with open(os.path.join(save_dir, 'valid-loss.txt'), 'a') as f:
f.write('\t'.join(
[str(x) for x in [total_steps,
round(valid_loss, 4)]]) + '\n')
# compute metrics and save them separately
for name, fun in [('f1', f1_score),
('precision', precision_score),
('recall', recall_score)]:
with open(os.path.join(save_dir, 'valid-%s.txt' % name),
'a') as f:
f.write('Step %d\n' % total_steps)
macro = fun(y_true=y_true,
y_pred=y_pred,
average='macro')
classwise = fun(y_true=y_true,
y_pred=y_pred,
average=None)
f.write('%1.3f\n' % macro)
f.write(
'\t'.join([str(round(x, 3))
for x in classwise]) + '\n')
# clear any memory and gpu
gc.collect()
torch.cuda.empty_cache()
cc.train()
if total_steps % args.checkpoint_interval == 0 and not total_steps == 0:
# save models
torch.save(obj=cc.state_dict(),
f=os.path.join(save_dir,
'%d-steps-cc.pth' % (total_steps)))
# collect garbage
torch.cuda.empty_cache()
gc.collect()
optimizer.zero_grad()
# compute for one step
loss, logits = cc(batch)
# gradient descent and update
loss.backward()
torch.nn.utils.clip_grad_norm_(cc.parameters(), max_grad_norm)
optimizer.step()
scheduler.step()
total_loss += loss.item()
pbar.set_description("[%d] Loss at %d/%dth batch: %1.3f" % (int(
os.getpid()), step + 1, len(train_data_loader), loss.item()))
with open(os.path.join(save_dir, 'training-loss.txt'), 'a') as f:
f.write(
'\t'.join([str(x)
for x in [step, round(loss.item(), 4)]]) + '\n')
total_steps += 1
print("Epoch %d complete! %d steps" % (epoch, total_steps))
return
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from cleopatra import LanguageModeler
from data_loader import CustomDataset
dataset = CustomDataset(
data_root=r"/home/vova/PycharmProjects/Deep_Exe2/Ass2DL/data/ner/train")
dataloader = DataLoader(dataset, batch_size=50, shuffle=True, num_workers=2)
obj = CustomDataset(
data_root=r"/home/vova/PycharmProjects/Deep_Exe2/Ass2DL/data/ner/train")
vocab = obj.vocab
CONTEXT_SIZE = 5
EMBEDDING_DIM = 10
losses = []
loss_function = nn.CrossEntropyLoss()
model = LanguageModeler(len(vocab),
EMBEDDING_DIM,
hidden_layer=64,
num_of_labels=5,
context_size=CONTEXT_SIZE)
optimizer = optim.SGD(model.parameters(), lr=0.001)
for epoch in range(30):
total_loss = 0
for i, data in enumerate(dataloader):
context, target = data