def main():
path = os.getcwd() + '/dataset/cleaned_lyrics.csv'
csv_reader = dataset.load_data(path)
new_path = os.getcwd() + '/dataset/nl_features_stratified2.csv'
f = open(new_path, 'w', encoding="utf8")
fieldnames = [
'song', 'genre', 'annotations', 'syllables', 'syll_per_line', 'words',
'verb', 'adj', 'noun', 'pre-det', 'det', 'prep', 'pronoun', 'pos',
'conj', 'cardinal', 'adverb', 'particle', 'exist', 'inj', 'aux', 'aa',
'abab', 'abba', 'freq'
]
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
counter = 0
data, labels = dataset.get_stratified_data(2000, True, False)
# for reading from the stratified dataset
for i in range(len(data)):
res_dict = {}
res_dict['song'] = "Test"
res_dict['genre'] = labels[i]
gen_dict(data[i], res_dict)
#if counter == 10001:
#break
writer.writerow(res_dict)
counter = counter + 1
'''
def __init__(self):
csv_reader = dataset.load_data("dataset/cleaned_lyrics.csv")
# skip past column headers
next(csv_reader)
for i in range(0): #TEMPORARY
next(csv_reader)
self.data = []
self.labels = []
datasize = 1000 #Just make this arbitrarily large when you want to use the whole dataset
print("Loading data...")
for i in range(datasize):
try:
song = next(csv_reader)
self.data.append(nltk.word_tokenize(song[2]))
self.labels.append(song[1])
except StopIteration:
break;
print("Building encoder...")
self.data_encoder = OHencoder.encode(j for i in self.data for j in i)
self.label_encoder = OHencoder.encode(self.labels)
self.data_decoder = list(self.data_encoder.keys()) #Gives you word/genre from vector index
self.label_decoder = list(self.label_encoder.keys())
self.model = MyRNN(len(self.data_encoder))
def main():
path = os.getcwd() + '/dataset/lyrics.csv'
new_path = os.getcwd() + '/dataset/cleaned_lyrics.csv'
csv_reader = dataset.load_data(path)
f=open(new_path, 'w', encoding="utf8")
fieldnames = ['song', 'genre','lyrics']
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
for line in csv_reader:
res = clean(line)
if(res!=None):
writer.writerow({'song': line[1], 'genre': line[4], 'lyrics':res})
def main(args):
# read super parameters
with open(args.cfg_path, 'r', encoding='utf-8') as f:
param_dict = yaml.load(f, Loader=yaml.FullLoader)
# creat save folder path
save_dir = increment_path(args.project) # str
check_path(save_dir)
param_dict['save_dir'] = save_dir # update to param_dict
param_dict['model_name'] = args.model_name # update to param_dict
# tensorboard
tb_writer = SummaryWriter(save_dir)
# set gpu
os.environ['CUDA_VISIBLE_DEVICES'] = param_dict['device']
# data loader
data_loader = load_data(params=param_dict)
init_seeds(seed=1) # activation cudnn
model = deeplab_v3_plus(class_number=param_dict['class_number'], fine_tune=True, backbone='resnet50').cuda()
continue_epoch = 0
if args.resume:
model_dict = model.state_dict()
pretrained_file = torch.load(args.resume)
pretrained_dict = pretrained_file['model'].float().state_dict()
continue_epoch = pretrained_file['epoch'] if 'epoch' in pretrained_file else 0
pretrained_dict = {k[7:]: v for k, v in pretrained_dict.items() if k[7:] in model_dict and v.size()==model_dict[k[7:]].size()}
assert len(pretrained_dict) == len(model_dict), "Unsuccessful import weight"
model_dict.update(pretrained_dict)
model.load_state_dict(model_dict)
model = nn.DataParallel(model) # keys add '.module', and has .module attribute
# TODO 许多要增加的，先完成，再改善！先成v1版本，再弄v2版本
if args.adam:
optimizer = optim.Adam(model.module.parameters(), lr=param_dict['lr0'], betas=(param_dict['momentum'], 0.999), weight_decay=5e-4)
else:
optimizer = optim.SGD(model.module.parameters(), lr=param_dict['lr0'], momentum=param_dict['momentum'], weight_decay=5e-4)
# set lr_scheduler Cosine Annealing
lf = lambda x: ((1 + math.cos(x * math.pi / param_dict['epoches'])) / 2) * (1 - param_dict['lrf']) + param_dict['lrf'] # cosine
scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lf) # no save scheduler params
# train stage
train(data_loader, model, optimizer, scheduler, tb_writer, param_dict, continue_epoch)
return
def main():
global x_train
global x_test
global y_train
global y_test
global x_scaled
global training_size
global testing_size
global matrix
global overall_size
global x_validation
global y_validation
path = os.getcwd() + '/dataset/nl_features_stratified2.csv'
#path = os.getcwd() + '/dataset/nl_features.csv'
csv_reader = dataset.load_data(path)
matrix = generate_full_matrix(csv_reader)
x = []
y = []
# Separate the matrix into the feature matrix and the classification vector
for line in matrix:
'''
Ordering found in the CSV - song name has already been removed at this time, and genre is either Rock (1) or Not rock (0)
0. Song Name - include song name so that we can go backwards
1. genre
2. annotations
3. syllables
4. syll_per_line
5. verb
6. adj
7. noun
8. pre-det
9. det
10. prep
11. pronoun
12. pos
13. conj
14. cardinal
15. adverb
16. particle
17. exist
18. inj
19. aux
20. aa - rhyme scheme
21. abab - rhyme scheme
22. abba - rhyme scheme
23. freq
'''
#TODO: Allow for different permutations of above to be ran, if we have to retrain the model each time
# we can just manually modify this
#Include all of the available features
x.append([
float(line[2]),
float(line[3]),
float(line[4]),
float(line[5]),
float(line[6]),
float(line[7]),
float(line[8]),
float(line[9]),
float(line[10]),
float(line[11]),
float(line[12]),
float(line[13]),
float(line[14]),
float(line[15]),
float(line[16]),
float(line[17]),
float(line[18]),
float(line[19]),
float(line[20]),
float(line[21]),
float(line[22]),
float(line[23])
])
y.append(int(line[1]))
x = np.asarray(x)
y = np.asarray(y)
# Scale the data matrix
# Note: support vector machine algorithms are not scale invariant, so it is highly recommended to scale your data
# For example, scale each attribute on the inmput vector X to [0,1] or [-1,1]
x_scaled = preprocessing.scale(x)
# x_scaled = x
overall_size = len(x_scaled)
# Training data should be 0 - training_size
training_size = math.ceil(len(x_scaled) * .80)
validation_size = math.ceil(len(x_scaled) * 90)
# Testing data should be testing_size (training_size + 1) - len(matrix)
# I guess because of the inclusive, exclusive testing_size should just start where training_size ends
testing_size = training_size
# to be used for training - x_train should be used with y_train (they should correspond with eachother)
x_train = x_scaled[:training_size]
y_train = y[:training_size]
x_validation = x_scaled[training_size:validation_size]
y_validation = y[training_size:validation_size]
#x_test = x_scaled[validation_size:]
#y_test = y[validation_size:]
# to be used for testing - x_test should be used with y_test (they should correspond with eachother)
# old code that was being used when there was only training and test
x_test = x_scaled[training_size:]
y_test = y[training_size:]
# manual_test()
automated_test()
def train():
if not os.path.exists('train_model/'):
os.makedirs('train_model/')
if not os.path.exists('result/'):
os.makedirs('result/')
train_data, dev_data, word2id, char2id, opts = load_data(vars(args))
model = FusionNet(opts)
if args.use_cuda:
model = model.cuda()
dev_batches = get_batches(dev_data, args.batch_size)
if args.eval:
print('load model...')
model.load_state_dict(torch.load(args.model_dir))
model.eval()
model.Evaluate(dev_batches,
args.data_path + 'dev_eval.json',
answer_file='result/' + args.model_dir.split('/')[-1] +
'.answers')
exit()
train_batches = get_batches(train_data, args.batch_size)
total_size = len(train_batches)
parameters = filter(lambda p: p.requires_grad, model.parameters())
optimizer = torch.optim.Adamax(parameters, lr=args.lrate)
lrate = args.lrate
best_score = 0.0
f1_scores = []
em_scores = []
for epoch in range(1, args.epochs + 1):
model.train()
for i, train_batch in enumerate(train_batches):
with torch.enable_grad():
loss = model(train_batch)
model.zero_grad()
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(parameters,
opts['grad_clipping'])
optimizer.step()
model.reset_parameters()
if i % 100 == 0:
print(
'Epoch = %d, step = %d / %d, loss = %.5f, lrate = %.5f best_score = %.3f'
% (epoch, i, total_size, loss, lrate, best_score))
with torch.no_grad():
model.eval()
exact_match_score, F1 = model.Evaluate(
dev_batches,
args.data_path + 'dev_eval.json',
answer_file='result/' + args.model_dir.split('/')[-1] +
'.answers')
f1_scores.append(F1)
em_scores.append(exact_match_score)
with open(args.model_dir + '_f1_scores.pkl', 'wb') as f:
pkl.dump(f1_scores, f)
with open(args.model_dir + '_em_scores.pkl', 'wb') as f:
pkl.dump(em_scores, f)
if best_score < F1:
best_score = F1
print('saving %s ...' % args.model_dir)
torch.save(model.state_dict(), args.model_dir)
if epoch > 0 and epoch % args.decay_period == 0:
lrate *= args.decay
for param_group in optimizer.param_groups:
param_group['lr'] = lrate
import numpy as np
from numpy import newaxis
from utils.dataset import load_data
x, y, w, m = load_data(391, 5) # 241
#
splitPoint = int(x.shape[0] * 0.8)
x_train = x[:splitPoint]
y_train = y[:splitPoint]
w_train = w[:splitPoint]
m_train = m[:splitPoint]
x_val = x[splitPoint:]
y_val = y[splitPoint:]
w_val = w[splitPoint:]
m_val = m[splitPoint:]
np.save("./utils/train_data.npy", x_train)
np.save("./utils/train_label.npy", y_train)
np.save("./utils/train_weather.npy", w_train)
np.save("./utils/train_meta.npy", m_train)
np.save("./utils/val_data.npy", x_val)
np.save("./utils/val_label.npy", y_val)
np.save("./utils/val_weather.npy", w_val)
np.save("./utils/val_meta.npy", m_val)
print("Done.")
def main():
# path = os.getcwd() + "/dataset/cleaned_lyrics.csv"
path = os.getcwd() + "/dataset/nl_features_subset.csv"
csv_reader = dataset.load_data(path)
rock_count = 0
not_rock_count = 0
NA_count = 0
other_count = 0
total_count = 0
country_count = 0
electronic_count = 0
folk_count = 0
indie_count = 0
jazz_count = 0
metal_count = 0
pop_count = 0
rb_count = 0
hiphop_count = 0
for line in csv_reader:
if len(line) == 0:
continue
if "genre" in line or "annotations" in line:
continue
total_count += 1
if line[1] == "Rock":
rock_count += 1
elif line[1] == "Country":
country_count += 1
elif line[1] == "Electronic":
electronic_count += 1
elif line[1] == "Hip-Hop":
hiphop_count += 1
elif line[1] == "Folk":
folk_count += 1
elif line[1] == "Indie":
indie_count += 1
elif line[1] == "Jazz":
jazz_count += 1
elif line[1] == "Metal":
metal_count += 1
elif line[1] == "Pop":
pop_count += 1
elif line[1] == "R&B":
rb_count += 1
elif line[1] == "Not Available":
NA_count += 1
elif line[1] == "Other":
other_count += 1
else:
not_rock_count += 1
print("Rock count is : " + str(rock_count) + "\toverall % " + str(100* rock_count / total_count))
print("Country count is : " + str(country_count) + "\toverall % " + str(100* country_count / total_count))
print("Electronic count is : " + str(electronic_count) + "\toverall % " + str(100* electronic_count / total_count))
print("Folk count is : " + str(folk_count) + "\toverall % " + str(100* folk_count / total_count))
print("Hip-Hop count is: " + str(hiphop_count) + "\toverall % " + str(100* hiphop_count / total_count))
print("Indie count is : " + str(indie_count) + "\toverall % " + str(100* indie_count / total_count))
print("Jazz count is : " + str(jazz_count) + "\toverall % " + str(100* jazz_count / total_count))
print("Metal count is : " + str(metal_count) + "\toverall % " + str(100* metal_count / total_count))
print("Pop count is : " + str(pop_count) + "\toverall % " + str(100* pop_count / total_count))
print("R&B count is : " + str(rb_count) + "\toverall % " + str(100* rb_count / total_count))
print()
print("Total number of songs: " + str(total_count))
set_random_seed(config.run.random_seed)
root_path = set_logger(config.run.mode)
if config.run.mode in ['train', 'continue'] and rank == 0:
writer = get_writer(folder=os.path.join(root_path, "tensorboard"))
else:
writer = None
config.tensorboard = writer
bert_config = get_bert_config(bert_path=config.model.nlp.bert_weight)
config.model.nlp.bert_config = bert_config
logging.info('Running begin!')
assert config.model.name in model_dict
model = model_dict[config.model.name](config).to(torch.device(config.run.device))
model = DistModule(model, config.linklink.sync)
loss_fn = getattr(loss, config.loss.func)(config)
eval_func = getattr(eval, config.eval.func)(config)
logging.info('Running mode: {}'.format(config.run.mode))
if config.run.mode in ['train', 'continue', 'test']:
train_loader, train_val_loader, val_loader = load_data(config)
train(model, loss_fn, eval_func, train_loader, train_val_loader, val_loader, config, root_path)
if writer is not None:
writer.close()
link.finalize()
# save args to JSON
saveArgsToJSON(args)
# Logger
log_format = '%(asctime)s %(message)s'
logging.basicConfig(filename=os.path.join(args.save, 'log.txt'),
level=logging.INFO,
format=log_format,
datefmt='%m/%d %I:%M:%S %p')
th = TqdmLoggingHandler()
th.setFormatter(logging.Formatter(log_format))
log = logging.getLogger()
log.addHandler(th)
# Data
testLoader, trainLoader, statsLoader = load_data(args, logging)
# Model
logging.info('{}\n'.format(' '.join(sys.argv)))
logging.info('==> Building model..')
modelClass = Models.__dict__[args.model]
model = modelClass(args)
if args.gradual:
args.gradEpochs = args.stepSize * (model.depth)
else:
args.gradEpochs = 0
assert args.epochs > args.gradEpochs # ensure if gradual will pass in all layers
# Load preTrained weights.
def train(model, lr, batch_size, seq_len, pre_train, weights, DEMODEL):
odmax = 391
use_tensorboard = True
gpu_count = len(os.environ["CUDA_VISIBLE_DEVICES"].split(','))
parallel = True if gpu_count != 1 else False
nb_epoch = 200 # number of epoch at training stage
nb_epoch_cont = 500 # number of epoch at continued training stage
T = 48 # number of time intervals in one day
m_patience = 20 # number of epoch to train
timestep = seq_len
map_height, map_width = 10, 5 # grid size
days_test = 36
pt = datetime.now().strftime('%m_%d_%H_%M_%S')
path_model = 'TRAIN/' + pt
if os.path.isdir(path_model) is False:
os.makedirs(path_model)
print("Exp: " + path_model)
# load data
print("loading data...")
'''
expect:
X = (sample, timestep, map_height * map_width, map_height, map_width)
Y = (sample, map_height * map_width, map_height, map_width)
weather = (sample, timestep, ?)
meta = (sample, timestep, ?)
The meta data is not used in this work, but we can explore its effect in future works.
'''
X, Y, weather, meta = load_data(odmax, timestep)
len_test = T * days_test
print("nb_epoch: " + str(nb_epoch) + " nb_epoch_cont: " + str(nb_epoch_cont) + " batch_size: " + str(batch_size))
print("patience: " + str(m_patience) + " lr: " + str(lr) + " seq_len: " + str(
timestep)) # + '-' + str(len_period) + '-' + str(len_trend))
print("odmax: " + str(odmax))
print("{} sample totally. {} for train, {} for test".format(X.shape[0], X.shape[0] - len_test, len_test))
X_train, X_test = X[:-len_test], X[-len_test:]
Y_train, Y_test = Y[:-len_test], Y[-len_test:]
weather_train, weather_test = weather[:-len_test], weather[-len_test:]
meta_train, meta_test = meta[:-len_test], meta[-len_test:]
# X_train = [X_train, weather_train, meta_train]
# X_test = [X_test, weather_test, meta_test]
"""********************************************************************************************"""
""" Frist, we train our model with fixed learning rate """
"""********************************************************************************************"""
model_para = {
"timestep": timestep,
"map_height": map_height,
"map_width": map_width,
"weather_dim": 0, # weather.shape[2],
"meta_dim": 0, # meta.shape[2],
}
# Build the model to train in parallel with multi-GPUs or only on GPU
if parallel:
model = DEMODEL.build_model(**model_para)
plot(model, to_file=os.path.join(path_model, 'networks.png'), show_shapes=True)
model.summary()
train_model = multi_gpu_model(model, gpu_count)
else:
model = DEMODEL.build_model(**model_para)
# plot(model, to_file=os.path.join(path_model, 'networks.png'), show_shapes=True)
model.summary()
train_model = model
# use the loss define in the model
loss = DEMODEL.get_loss()
optimizer = Adam(lr=lr)
metrics = [rmse, Metrics.mape, o_rmse, Metrics.o_mape,]
train_model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
# load weights to the pre_train model after model compiled
if pre_train:
model.load_weights(weights, by_name=True, skip_mismatch=True)
# define callbacks on training
callbacks = []
hyperparams_name = 'timestep{}.lr{}'.format(timestep, lr)
fname_param = os.path.join(path_model, hyperparams_name + '.best.h5')
lr_logger = SGDLearningRateTracker() # log out the learning rate after a epoch trained
callbacks.append(lr_logger)
callbacks.append(EarlyStopping(monitor='val_rmse', patience=m_patience, mode='min'))
callbacks.append(ModelCheckpoint(
fname_param, monitor='val_mape', verbose=0, save_best_only=True, mode='min'))
if use_tensorboard:
callbacks.append(get_tensorboard(path_model + "/tensorboard-1/"))
print('=' * 10)
print("training model...")
history = train_model.fit(X_train, Y_train,
nb_epoch=nb_epoch,
batch_size=batch_size,
validation_data=(X_test, Y_test),
callbacks=callbacks,
verbose=1)
model.save_weights(os.path.join(
path_model, '{}.h5'.format(hyperparams_name)), overwrite=True)
train_model.load_weights(fname_param)
model.save_weights(fname_param, overwrite=True)
pickle.dump((history.history), open(os.path.join(
path_model, '{}.history.pkl'.format(hyperparams_name)), 'wb'))
print('evaluating using the model that has the best model on the valid set')
model.load_weights(fname_param)
score = train_model.evaluate(X_train, Y_train, batch_size=batch_size, verbose=0)
show_score(odmax, score, "train")
score = train_model.evaluate(
X_test, Y_test, batch_size=batch_size, verbose=0)
show_score(odmax, score, "Test")
print('=' * 10)
"""********************************************************************************************"""
""" Second, we train our model with step_decay learning rate """
"""********************************************************************************************"""
# clear session to rebuild the model, in order to switch optimizor
K.clear_session()
DEMODEL.clear_graph()
# rebuild the model
if parallel:
model = DEMODEL.build_model(**model_para)
train_model = multi_gpu_model(model, gpu_count)
else:
model = DEMODEL.build_model(**model_para)
train_model = model
loss = DEMODEL.get_loss()
optimizer = Adam(lr=lr)
metrics = [rmse, Metrics.mape, o_rmse, Metrics.o_mape,]
train_model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
model.load_weights(fname_param)
fname_param_step = os.path.join(
path_model, hyperparams_name + '.cont.best.h5.{epoch:03d}-{val_mape:.4f}-{val_rmse:.4f}-{val_o_mape:.4f}-{val_o_rmse:.4f}')
callbacks_cont = []
# lr_logger = SGDLearningRateTracker()
# callbacks_cont.append(lr_logger)
callbacks_cont.append(LearningRateScheduler(get_decay(lr)))
callbacks_cont.append(ModelCheckpoint(
fname_param_step, monitor='val_mape', verbose=0, save_best_only=False, period=1, save_weights_only=True,
mode='min'))
if use_tensorboard:
callbacks_cont.append(get_tensorboard(path_model + "/tensorboard-2/"))
history = train_model.fit(X_train, Y_train,
nb_epoch=nb_epoch_cont,
batch_size=batch_size,
callbacks=callbacks_cont,
validation_data=(X_test, Y_test),
verbose=1)
pickle.dump((history.history), open(os.path.join(
path_model, '{}.cont.history.pkl'.format(hyperparams_name)), 'wb'))
model.save_weights(os.path.join(
path_model, '{}_cont.h5'.format(hyperparams_name)), overwrite=True)
model.load_weights(fname_param)
model.save_weights(fname_param, overwrite=True) # save the origin model weights instead of the paralleled one
print('=' * 10)
print('evaluating using the final model')
score = train_model.evaluate(X_train, Y_train, batch_size=32, verbose=0)
show_score(odmax, score, "train")
score = train_model.evaluate(
X_test, Y_test, batch_size=32, verbose=0)
show_score(odmax, score, "test")
def main(config):
csv_reader = dataset.load_data(config.dataset_path)
feature_names = next(csv_reader)
print(f"Found features={feature_names}")
def train(self):
# Parameters from online network stream
params = list(self.online_net.parameters()) + list(
self.predictor.parameters())
# Set optimizer
optimizer = self.optimizer(params,
lr=self.args.learning_rate,
momentum=0.9,
weight_decay=0.0004)
# Scheduling optimizer
cosine_scheduler = CosineAnnealingLR(optimizer=optimizer,
T_max=100,
eta_min=0,
last_epoch=-1)
# Warm-up wo/ restart
scheduler = GradualWarmupScheduler(optimizer,
multiplier=8,
total_epoch=10,
after_scheduler=cosine_scheduler)
# Set data_loader (train)
augmentation = augmentation_dictionary['covid']
data_set = load_data(args=self.args,
split='train',
transform=TwoCropsTransform(
base_transform=augmentation)).load_dataset()
train_loader = DataLoader(data_set,
batch_size=self.args.batch_size,
shuffle=True,
drop_last=False,
num_workers=self.args.num_workers,
pin_memory=True)
def net_switch(view1: augmented_image, view2: augmented_image):
# online network stream
out_online = self.online_net(view1)
out_online = self.predictor(out_online)
# target network stream
with torch.no_grad():
out_target = self.target_net(view2)
# get loss
loss = self.regression_loss(out_online, out_target)
# return loss
return loss
# set epoch number to start(+resume)
start_epoch = 0
if self.args.resume:
checkpoint = torch.load(path_ckpt)
start_epoch = checkpoint['epoch'] + 1
print(f'Resume training ... EPOCH({start_epoch})')
print(f'Get model from : {path_ckpt}')
self.online_net.load_state_dict(
checkpoint['online_network_state_dict'])
self.target_net.load_state_dict(
checkpoint['target_network_state_dict'])
self.predictor.load_state_dict(checkpoint['predictor_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
print('Loading Parameters Complete!')
else:
print("Start New Training!")
# optimizer.zero_grad()
# optimizer.step()
for epoch in range(start_epoch, self.max_epoch):
# Tensorboard loss per epoch
epoch_loss = 0
# pbar _ loss desc.
loss = 0
min_loss = 9999
# steps for one epoch
pbar = tqdm(enumerate(train_loader),
total=len(train_loader),
desc='loss_description')
for i, (pos_pair, _) in pbar:
# pbar _ dynamic desc
pbar.set_description(
f'| epoch: {epoch} | loss: {loss:.3f} | min_loss: {min_loss:.3f} | epoch_loss: {epoch_loss/(i+1):.3f} |'
)
# get mini_batch with augmentation
pos_pair[0] = pos_pair[0].to(self.args.device,
non_blocking=True)
pos_pair[1] = pos_pair[1].to(self.args.device,
non_blocking=True)
# get total losss
loss1 = net_switch(pos_pair[0], pos_pair[1])
loss2 = net_switch(pos_pair[1], pos_pair[0])
loss = (loss1 + loss2).mean()
epoch_loss += loss
if loss < min_loss:
min_loss = loss
# update parameters
# scheduler.step()
optimizer.zero_grad()
loss.backward()
optimizer.step()
self.writer.add_scalar('loss',
epoch_loss / (i + 1),
global_step=epoch)
torch.save(
{
'epoch': epoch + 1,
'online_network_state_dict': self.online_net.state_dict(),
'target_network_state_dict': self.target_net.state_dict(),
'predictor_state_dict': self.predictor.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
}, path_save +
f'/byol_epoch_{epoch}_loss_{epoch_loss/(i+1):.2f}.ckpt')
if loss_type == 'epoch':
# val_acc
plt.plot(iters, self.val_acc[loss_type], 'b', label='val acc')
# val_loss
plt.plot(iters, self.val_loss[loss_type], 'k', label='val loss')
plt.grid(True) # 设置网格形式
plt.xlabel(loss_type)
plt.ylabel('acc-loss') # 给x，y轴加注释
plt.legend(loc="upper right") # 设置图例显示位置
# plt.show()
plt.savefig('./output/acc-loss.jpg')
if __name__ == '__main__':
train_path = './data/val/'
images, labels = load_data(train_path)
# 主要是为了获得验证数据，验证数据以元组形式输入
# 之所以不用生成器作为验证数据，是为了能够显示出直方图信息，以及梯度的相关信息
#设置为0.95增大验证数据集数量
x_train, x_test, y_train, y_test = train_test_split(images,
labels,
test_size=0.95,
random_state=0)
# 编译模型来配置学习过程
train_img_path = './data/train/'
# val_img_path = './data/train/'
history = LossHistory()
sgd = SGD(lr=0.0001, momentum=0.98, decay=0.99, nesterov=False)
# adam = Adam(lr=0.0003)
# 运行构建的模型图
def train():
if not os.path.exists("train_model/"):
os.makedirs("train_model/")
if not os.path.exists("result/"):
os.makedirs("result/")
train_data, dev_data, word2id, id2word, char2id, opts = load_data(
vars(args))
model = UNet(opts)
if args.use_cuda:
model = model.cuda()
dev_batches = get_batches(dev_data, args.batch_size, evaluation=True)
if args.eval:
print("load model...")
model.load_state_dict(torch.load(args.model_dir))
model.eval()
model.Evaluate(
dev_batches,
os.path.join(args.prepro_dir, "dev_eval.json"),
answer_file="result/" + args.model_dir.split("/")[-1] + ".answers",
drop_file=os.path.join(args.prepro_dir, "drop.json"),
dev=args.dev_file,
)
exit()
if args.load_model:
print("load model...")
model.load_state_dict(torch.load(args.model_dir))
model.eval()
_, F1 = model.Evaluate(
dev_batches,
os.path.join(args.prepro_dir, "dev_eval.json"),
answer_file=os.path.join("result/",
args.model_dir.split("/")[-1],
".answers"),
drop_file=os.path.join(args.prepro_dir, "drop.json"),
dev=args.dev_file,
)
best_score = F1
with open(args.model_dir + "_f1_scores.pkl", "rb") as f:
f1_scores = pkl.load(f)
with open(args.model_dir + "_em_scores.pkl", "rb") as f:
em_scores = pkl.load(f)
else:
best_score = 0.0
f1_scores = []
em_scores = []
parameters = filter(lambda p: p.requires_grad, model.parameters())
optimizer = torch.optim.Adamax(parameters, lr=args.lrate)
lrate = args.lrate
for epoch in range(1, args.epochs + 1):
train_batches = get_batches(train_data, args.batch_size)
dev_batches = get_batches(dev_data, args.batch_size, evaluation=True)
total_size = len(train_data) // args.batch_size
model.train()
for i, train_batch in enumerate(train_batches):
loss = model(train_batch)
model.zero_grad()
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(parameters, opts["grad_clipping"])
optimizer.step()
model.reset_parameters()
if i % 100 == 0:
print(
"Epoch = %d, step = %d / %d, loss = %.5f, lrate = %.5f best_score = %.3f"
% (epoch, i, total_size, model.train_loss.value, lrate,
best_score))
sys.stdout.flush()
model.eval()
exact_match_score, F1 = model.Evaluate(
dev_batches,
os.path.join(args.prepro_dir, "dev_eval.json"),
answer_file=os.path.join("result/",
args.model_dir.split("/")[-1],
".answers"),
drop_file=os.path.join(args.prepro_dir, "drop.json"),
dev=args.dev_file,
)
f1_scores.append(F1)
em_scores.append(exact_match_score)
with open(args.model_dir + "_f1_scores.pkl", "wb") as f:
pkl.dump(f1_scores, f)
with open(args.model_dir + "_em_scores.pkl", "wb") as f:
pkl.dump(em_scores, f)
if best_score < F1:
best_score = F1
print("saving %s ..." % args.model_dir)
torch.save(model.state_dict(), args.model_dir)
if epoch > 0 and epoch % args.decay_period == 0:
lrate *= args.decay
for param_group in optimizer.param_groups:
param_group["lr"] = lrate
