def save_rev_ids_for_predictions(y_pred, y_test, z, model_name):
correct_minority = []
incorrect_minority = []
total = 0
correct = 0
incorrect = 0
for i in range(0, len(y_test)):
# if minority class in reality
if y_test[i] == 0:
total += 1
# if correctly predicted minority class
if y_pred[i] == y_test[i]:
correct += 1
correct_minority.append(z[i])
else:
incorrect += 1
incorrect_minority.append(z[i])
# save it in a file
with open('correct_' + model_name + '.txt', 'w') as f:
for item in correct_minority:
item = item.encode('utf8')
f.write("%s\n" % item)
f.close()
with open('incorrect_' + model_name + '.txt', 'w') as f:
for item in incorrect_minority:
item = item.encode('utf8')
f.write("%s\n" % item)
f.close()
return
def store_embeddings(path='MoRTy_embedding.', vocab=None, embs=None, store_as='.vec'):
print(path)
# vocab and embeddings are list and array, with the same sequence order
# vocab[2] => embs[2]
if vocab == None or embs == None:
raise Exception("empty data dump")
if store_as == '.pickle':
with open(path+store_as,'wb') as f:
pickle.dump({'vocab':vocab, 'embeddings':embs}, f)
elif store_as == '.vec':
with open(path+store_as,'w') as f:
f.write(str(len(embs)) + ' ' + str(len(embs[vocab[0]])) + '\n') # header
for token, emb in embs.items():
f.write(token + ' ' + ' '.join(str(x_i) for x_i in emb) + '\n')
else:
raise Exception('embedding output format not recognized')
def save_bert_file(dict,
output,
dataset_name,
model_name,
hyper_num,
oov_num,
f_info_out,
include_oov=True):
logger.info("save info...")
f_info_out.write(
f'{model_name}\t{dataset_name}\t{len(dict)}\t{oov_num}\t{hyper_num}\t{include_oov}\n'
)
logger.info("save json...")
dname = os.path.splitext(dataset_name)[0]
fjson = json.dumps(dict, ensure_ascii=False)
f = open(os.path.join(output, model_name.replace("/", "-"),
dname + ".json"),
mode="w",
encoding="utf-8")
f.write(fjson)
f.close()
def test(net, testset, testloader, criterian, batch_size, n_class, log_file):
'''Testing the network
'''
net.eval()
testloss, testacc = 0., 0.
class_correct = list(0. for i in range(10))
class_total = list(0. for i in range(10))
for (img, label) in testloader:
img = var(img).cuda()
label = var(label).cuda()
#forward pass
output = net(img)
#loss
loss = criterian(output, label)
testloss += loss.data[0]
#prediction
_, predict = torch.max(output, 1)
num_correct = (predict == label).sum()
testacc += num_correct.data[0]
#
c = (predict == label).squeeze()
for i in range(batch_size):
l = label[i].data[0]
class_correct[l] += c[i].data[0]
class_total[l] += 1
#end_for
#end_for
testloss /= len(testset)
testacc /= len(testset)
f = open(log_file, 'a')
f.write('\n-------------------\n')
print("Test: Loss: %.5f, Acc: %.2f %%" % (testloss, 100 * testacc))
f.write("Test: Loss: %.5f, Acc: %.2f %%\n" % (testloss, 100 * testacc))
for i in range(10):
print('Accuracy of %5d : %2d %%' %
(i, 100 * class_correct[i] / class_total[i]))
f.write('Accuracy of %5d : %2d %%\n' %
(i, 100 * class_correct[i] / class_total[i]))
#end_for
f.close()
def preprocess_training():
# Create connection to database
conn = sqlite3.connect('news.db')
c = conn.cursor()
query_severity_command = '''
SELECT * FROM severity
'''
query_keyword_frequency_command = '''
SELECT * FROM keywords
WHERE frequency > 1
'''
query_num_articles_command = '''
SELECT DISTINCT(keywords.date), num_articles FROM keywords
'''
article_volume = [] # date, num_articles
date_map = {} # str(date) -> idx
idx = 0
c.execute(query_num_articles_command)
for row in c:
date = row[0]
article_volume.append([date,row[1]])
if date not in set(date_map.keys()):
date_map[date] = idx
idx += 1
conn.commit()
with open('date_map.txt', 'w') as f:
f.write(json.dumps(date_map))
with open('article_volume.txt', 'w') as f:
for item in article_volume:
f.write("%s\n" % item)
severity = [] # date, confirmed, death
c.execute(query_severity_command)
for row in c:
date = row[0]
confirmed = row[1]
death = row[2]
if date in set(date_map.keys()):
severity.append([date, confirmed, death])
with open('severity.txt', 'w') as f:
for item in severity:
f.write("%s\n" % item)
keyword_occurance = {} # keyword: # occurence
c.execute(query_keyword_frequency_command)
for row in c:
keyword = row[1]
if keyword not in set(keyword_occurance.keys()):
keyword_occurance[keyword] = 0
keyword_occurance[keyword] += 1
conn.commit()
keyword_map = {} # str(keyword) -> idx
idx = 0
for keyword in list(keyword_occurance):
if keyword_occurance[keyword] < 30:
keyword_occurance.pop(keyword)
else:
keyword_map[keyword] = idx
idx += 1
with open('keyword_map.txt', 'w') as f:
f.write(json.dumps(keyword_map))
keyword_frequency = [] # date, keyword, frequency
c.execute(query_keyword_frequency_command)
for row in c:
date = row[0]
keyword = row[1]
frequency = row[2]
if keyword in keyword_occurance:
keyword_frequency.append([date, keyword, frequency])
conn.commit()
with open('keyword_frequency.txt', 'w') as f:
for item in keyword_frequency:
f.write("%s\n" % item)
num_date = len(article_volume) # 2020.1.7 - 2020.3.8
num_attr_x = len(keyword_map) + 1 # #DISTINCT(keyword) + 1 num_articles
num_attr_y = 2 # #confirmed, #death
inputs = torch.zeros((num_date, num_attr_x))
labels = torch.zeros((num_date, num_attr_y))
lr_labels = torch.zeros((num_date, num_attr_y))
for row in keyword_frequency:
date_idx = date_map[row[0]]
keyword_idx = keyword_map[row[1]]
frequency = row[2]
inputs[date_idx][keyword_idx] = frequency
for row in article_volume:
date_idx = date_map[row[0]]
num_articles = row[1]
inputs[date_idx][num_attr_x-1] = num_articles
# normalize inputs
row_sums = inputs.sum(axis=1)
inputs = inputs / row_sums[:, np.newaxis]
for row in severity:
date_idx = date_map[row[0]]
labels[date_idx][0] = row[1] # confirmed
labels[date_idx][1] = row[2] # death
# remove first
temp1 = labels[1:]
# remove last
temp2 = labels[:-1]
# calculate increase rate of confirmed and deaths cases
inc = (temp1 - temp2)/temp1
count = 0
for row in severity:
date_idx = date_map[row[0]]
if count == 0:
labels[date_idx][0] = 0 # confirmed
labels[date_idx][1] = 0 # death
if count != 0:
if inc[count-1][0] > 0.60:
labels[date_idx][0] = 1
else:
labels[date_idx][0] = 0
if inc[count-1][1] > 0.60:
labels[date_idx][1] = 1
else:
labels[date_idx][1] = 0
count += 1
torch.save(inputs, 'inputs.pt')
torch.save(labels, 'labels.pt')
def load_data(config,
filename='../dataset/lm_lengthsorted.txt',
max_sample_size=None):
samples = []
skipped = 0
input_vocab = Counter()
output_vocab = Counter()
bloom_filter = Counter()
try:
log.info('processing file: {}'.format(filename))
text_file = open(filename).readlines()
log.info('building input_vocabulary...')
sentences = set()
for i, l in tqdm(enumerate(text_file[:config.HPCONFIG.max_samples]),
desc='processing {}'.format(filename)):
sentence = remove_punct_symbols(l)
sentence = sentence.strip().split()
if len(sentence):
input_vocab.update(sentence)
sentences.add(tuple(sentence))
freq_threshold = (config.HPCONFIG.freq_threshold * (float(config.HPCONFIG.max_samples)
/len(text_file)))
log.info('freq_threhold: {}'.format(freq_threshold))
vocab = Vocab(input_vocab,
special_tokens = VOCAB,
freq_threshold = int(freq_threshold))
if config.CONFIG.write_vocab_to_file:
vocab.write_to_file(config.ROOT_DIR + '/vocab.csv')
for i, sentence in tqdm(enumerate(sentences),
desc='processing sentences'):
if len(sentence) < 2:
continue
unk_ratio = float(count_UNKS(sentence, vocab))/len(sentence)
log.debug('===')
log.debug(pformat(sentence))
sentence = [i if vocab[i] != vocab['UNK'] else 'UNK' for i in sentence ]
log.debug(pformat(sentence))
if unk_ratio > 0.7:
log.debug('unk ratio is heavy: {}'.format(unk_ratio))
continue
for center_word_pos, center_word in enumerate(sentence):
for w in range(-config.HPCONFIG.window_size,
config.HPCONFIG.window_size + 1):
context_word_pos = center_word_pos + w
# make soure not jump out sentence
if (context_word_pos < 0
or context_word_pos >= len(sentence)
or center_word_pos == context_word_pos):
continue
pair = (center_word, sentence[context_word_pos])
if pair[0] != 'UNK' and pair[1] != 'UNK':
if not pair in bloom_filter:
pass
samples.append(
Sample('{}.{}'.format(i, center_word_pos),
#sentence,
center_word,
sentence[context_word_pos]
)
)
bloom_filter.update([pair])
if max_sample_size and len(samples) > max_sample_size:
break
except:
skipped += 1
log.exception('{}'.format(l))
print('skipped {} samples'.format(skipped))
if config.CONFIG.dump_bloom_filter:
with open('word_pair.csv', 'w') as F:
for k,v in bloom_filter.items():
F.write('|'.join(list(k) + [str(v)]) + '\n')
#pivot = int(len(samples) * config.CONFIG.split_ratio)
#train_samples, test_samples = samples[:pivot], samples[pivot:]
train_samples, test_samples = samples, []
return Dataset(filename,
(train_samples, test_samples),
input_vocab = vocab,
output_vocab = vocab)
labels[i] = 0
else:
labels[i] = 1
for i in range(size):
for j in range(attributes):
point = data[i][j]
if point == 'y\\' or point == 'y':
data[i][j] = 1
elif point == 'n\\' or point == 'n' or point == 'n}':
data[i][j] = 0
medians = []
for j in range(attributes):
acceptable = []
for i in range(size):
if ((data[i][j] == 1) or (data[i][j] == 0)):
acceptable.append(data[i][j])
med = np.median(acceptable)
medians.append(int(med))
for i in range(size):
for j in range(attributes):
if data[i][j] != 1 and data[i][j] != 0:
data[i][j] = medians[j]
F = open('pol_data.csv', 'w+')
for i in range(size):
for j in range(attributes):
F.write(str(data[i][j]) + ',')
F.write('\n')
def main():
# Make directories if they don't already exist
util.make_directories()
# Load model options
model_options = constants.MAIN_MODEL_OPTIONS
########## DATA ##########
if constants.PRINT_MODEL_STATUS: print("Loading data")
dataset_map = util.load_dataset_map()
train_captions, val_captions, test_captions = util.load_text_vec(
'Data', constants.VEC_OUTPUT_FILE_NAME, dataset_map)
train_image_dict, val_image_dict, test_image_dict = util.get_images(
'Data', constants.DIRECTORY_PATH, constants.FLOWERS_DICTS_PATH)
########## MODEL ##########
generator = CondBeganGenerator(model_options)
discriminator = CondBeganDiscriminator(model_options)
# Put G and D on cuda if GPU available
if torch.cuda.is_available():
if constants.PRINT_MODEL_STATUS: print("CUDA is available")
generator = generator.cuda()
discriminator = discriminator.cuda()
if constants.PRINT_MODEL_STATUS: print("Moved models to GPU")
# Initialize weights
generator.apply(util.weights_init)
discriminator.apply(util.weights_init)
########## SAVED VARIABLES #########
new_epoch = 0
began_k = 0
train_losses = {"generator": [], "discriminator": [], "converge": []}
val_losses = {"generator": [], "discriminator": [], "converge": []}
losses = {'train': train_losses, 'val': val_losses}
########## OPTIMIZER ##########
g_optimizer = optim.Adam(generator.parameters(),
lr=constants.LR,
betas=constants.BETAS)
# Changes the optimizer to SGD if declared in constants
if constants.D_OPTIMIZER_SGD:
d_optimizer = optim.SGD(discriminator.parameters(), lr=constants.LR)
else:
d_optimizer = optim.Adam(discriminator.parameters(),
lr=constants.LR,
betas=constants.BETAS)
if constants.PRINT_MODEL_STATUS: print("Added optimizers")
########## RESUME OPTION ##########
if args.resume:
print("Resuming from epoch " + args.resume)
checkpoint = torch.load(constants.SAVE_PATH + 'weights/epoch' +
str(args.resume))
new_epoch = checkpoint['epoch'] + 1
generator.load_state_dict(checkpoint['g_dict'])
discriminator.load_state_dict(checkpoint['d_dict'])
began_k = checkpoint['began_k']
g_optimizer.load_state_dict(checkpoint['g_optimizer'])
d_optimizer.load_state_dict(checkpoint['d_optimizer'])
losses = torch.load(constants.SAVE_PATH + 'losses')
########## VARIABLES ##########
noise_vec = torch.FloatTensor(constants.BATCH_SIZE, model_options['z_dim'])
text_vec = torch.FloatTensor(constants.BATCH_SIZE,
model_options['caption_vec_len'])
real_img = torch.FloatTensor(constants.BATCH_SIZE,
model_options['image_channels'],
constants.IMAGE_SIZE, constants.IMAGE_SIZE)
real_caption = torch.FloatTensor(constants.BATCH_SIZE,
model_options['caption_vec_len'])
if constants.USE_CLS:
wrong_img = torch.FloatTensor(constants.BATCH_SIZE,
model_options['image_channels'],
constants.IMAGE_SIZE,
constants.IMAGE_SIZE)
wrong_caption = torch.FloatTensor(constants.BATCH_SIZE,
model_options['caption_vec_len'])
# Add cuda GPU option
if torch.cuda.is_available():
noise_vec = noise_vec.cuda()
text_vec = text_vec.cuda()
real_img = real_img.cuda()
real_caption = real_caption.cuda()
if constants.USE_CLS: wrong_img = wrong_img.cuda()
########## Training ##########
num_iterations = 0
for epoch in range(new_epoch, constants.NUM_EPOCHS):
print("Epoch %d" % (epoch))
st = time.time()
for i, batch_iter in enumerate(
util.grouper(train_captions.keys(), constants.BATCH_SIZE)):
batch_keys = [x for x in batch_iter if x is not None]
curr_batch_size = len(batch_keys)
discriminator.train()
generator.train()
discriminator.zero_grad() # Zero out gradient
# Save computations for gradient calculations
for p in discriminator.parameters():
p.requires_grad = True # Need this to be true to update generator as well
########## BATCH DATA #########
noise_batch = torch.randn(curr_batch_size, model_options['z_dim'])
text_vec_batch = torch.Tensor(
util.get_text_description(train_captions, batch_keys))
real_caption_batch = torch.Tensor(
util.get_text_description(train_captions, batch_keys))
real_img_batch = torch.Tensor(
util.choose_real_image(train_image_dict, batch_keys))
if constants.USE_CLS:
wrong_img_batch = torch.Tensor(
util.choose_wrong_image(train_image_dict, batch_keys))
if torch.cuda.is_available():
noise_batch = noise_batch.cuda()
text_vec_batch = text_vec_batch.cuda()
real_caption_batch = real_caption_batch.cuda()
real_img_batch = real_img_batch.cuda()
if constants.USE_CLS: wrong_img_batch = wrong_img_batch.cuda()
# Fill in tensors with batch data
noise_vec.resize_as_(noise_batch).copy_(noise_batch)
text_vec.resize_as_(text_vec_batch).copy_(text_vec_batch)
real_caption.resize_as_(text_vec_batch).copy_(text_vec_batch)
real_img.resize_as_(real_img_batch).copy_(real_img_batch)
if constants.USE_CLS:
wrong_img.resize_as_(wrong_img_batch).copy_(wrong_img_batch)
########## RUN THROUGH GAN ##########
gen_image = generator.forward(Variable(text_vec),
Variable(noise_vec))
real_img_passed = discriminator.forward(Variable(real_img),
Variable(real_caption))
fake_img_passed = discriminator.forward(gen_image.detach(),
Variable(real_caption))
if constants.USE_CLS:
wrong_img_passed = discriminator.forward(
Variable(wrong_img), Variable(real_caption))
########## TRAIN DISCRIMINATOR ##########
if constants.USE_REAL_LS:
# Real loss sensitivity
# L_D = L(y_r) - k * (L(y_f) + L(y_f, r))
# L_G = L(y_f) + L(y_f, r)
# k = k + lambda_k * (gamma * L(y_r) + L(y_f) + L(y_f, r))
d_real_loss = torch.mean(
torch.abs(real_img_passed - Variable(real_img)))
d_fake_loss = torch.mean(torch.abs(fake_img_passed -
gen_image))
d_real_sensitivity_loss = torch.mean(
torch.abs(fake_img_passed - Variable(real_img)))
d_loss = d_real_loss - began_k * (
0.5 * d_fake_loss + 0.5 * d_real_sensitivity_loss)
# Update began k value
balance = (model_options['began_gamma'] * d_real_loss -
0.5 * d_fake_loss -
0.5 * d_real_sensitivity_loss).data[0]
began_k = min(
max(began_k + model_options['began_lambda_k'] * balance,
0), 1)
elif constants.USE_CLS:
# Cond BEGAN Discrminator Loss with CLS
# L(y_w) is the caption loss sensitivity CLS (makes sure that captions match the image)
# L_D = L(y_r) + L(y_f, w) - k * L(y_f)
# L_G = L(y_f)
# k = k + lambda_k * (gamma * (L(y_r) + L(y_f, w)) - L(y_f))
d_real_loss = torch.mean(
torch.abs(real_img_passed - Variable(real_img)))
d_wrong_loss = torch.mean(
torch.abs(fake_img_passed - Variable(wrong_img)))
d_fake_loss = torch.mean(torch.abs(fake_img_passed -
gen_image))
d_loss = 0.5 * d_real_loss + 0.5 * d_wrong_loss - began_k * d_fake_loss
# Update began k value
balance = (model_options['began_gamma'] *
(0.5 * d_real_loss + 0.5 * d_wrong_loss) -
d_fake_loss).data[0]
began_k = min(
max(began_k + model_options['began_lambda_k'] * balance,
0), 1)
# No CLS option
else:
# Cond BEGAN Discriminator Loss
# L_D = L(y_r) - k * L(y_f)
# k = k + lambda_k * (gamma * L(y_r) + L(y_f))
d_real_loss = torch.mean(
torch.abs(real_img_passed - Variable(real_img)))
d_fake_loss = torch.mean(torch.abs(fake_img_passed -
gen_image))
d_loss = d_real_loss - began_k * d_fake_loss
# Update began k value
balance = (model_options['began_gamma'] * d_real_loss -
d_fake_loss).data[0]
began_k = min(
max(began_k + model_options['began_lambda_k'] * balance,
0), 1)
d_loss.backward()
d_optimizer.step()
########## TRAIN GENERATOR ##########
generator.zero_grad()
for p in discriminator.parameters():
p.requires_grad = False
# Generate image again if you want to
if constants.REGEN_IMAGE:
noise_batch = torch.randn(curr_batch_size,
model_options['z_dim'])
if torch.cuda.is_available():
noise_batch = noise_batch.cuda()
noise_vec.resize_as_(noise_batch).copy_(noise_batch)
gen_image = generator.forward(Variable(text_vec),
Variable(noise_vec))
new_fake_img_passed = discriminator.forward(
gen_image, Variable(real_caption))
# Generator Loss
# L_G = L(y_f)
g_loss = torch.mean(torch.abs(new_fake_img_passed - gen_image))
if constants.USE_REAL_LS:
g_loss += torch.mean(
torch.abs(new_fake_img_passed - Variable(real_img)))
elif constants.USE_CLS:
g_loss -= torch.mean(
torch.abs(new_fake_img_passed - Variable(wrong_img)))
g_loss.backward()
g_optimizer.step()
# M = L(y_r) + |gamma * L(y_r) - L(y_f)|
convergence_val = d_real_loss + abs(balance)
# learning rate decay
g_optimizer = util.adjust_learning_rate(g_optimizer,
num_iterations)
d_optimizer = util.adjust_learning_rate(d_optimizer,
num_iterations)
if i % constants.LOSS_SAVE_IDX == 0:
losses['train']['generator'].append((g_loss.data[0], epoch, i))
losses['train']['discriminator'].append(
(d_loss.data[0], epoch, i))
losses['train']['converge'].append(
(convergence_val.data[0], epoch, i))
num_iterations += 1
print('Total number of iterations: ', num_iterations)
print('Training G Loss: ', g_loss.data[0])
print('Training D Loss: ', d_loss.data[0])
print('Training Convergence: ', convergence_val.data[0])
print('K value: ', began_k)
epoch_time = time.time() - st
print("Time: ", epoch_time)
if epoch == constants.REPORT_EPOCH:
with open(constants.SAVE_PATH + 'report.txt', 'w') as f:
f.write(constants.EXP_REPORT)
f.write("Time per epoch: " + str(epoch_time))
print("Saved report")
########## DEV SET #########
# Calculate dev set loss
# Volatile is true because we are running in inference mode (no need to calculate gradients)
generator.eval()
discriminator.eval()
for i, batch_iter in enumerate(
util.grouper(val_captions.keys(), constants.BATCH_SIZE)):
batch_keys = [x for x in batch_iter if x is not None]
curr_batch_size = len(batch_keys)
# Gather batch data
noise_batch = torch.randn(curr_batch_size, model_options['z_dim'])
text_vec_batch = torch.Tensor(
util.get_text_description(val_captions, batch_keys))
real_caption_batch = torch.Tensor(
util.get_text_description(val_captions, batch_keys))
real_img_batch = torch.Tensor(
util.choose_real_image(val_image_dict, batch_keys))
if constants.USE_CLS:
wrong_img_batch = torch.Tensor(
util.choose_wrong_image(val_image_dict, batch_keys))
if torch.cuda.is_available():
noise_batch = noise_batch.cuda()
text_vec_batch = text_vec_batch.cuda()
real_caption_batch = real_caption_batch.cuda()
real_img_batch = real_img_batch.cuda()
if constants.USE_CLS:
wrong_img_batch = wrong_img_batch.cuda()
# Fill in tensors with batch data
noise_vec.resize_as_(noise_batch).copy_(noise_batch)
text_vec.resize_as_(text_vec_batch).copy_(text_vec_batch)
real_caption.resize_as_(text_vec_batch).copy_(text_vec_batch)
real_img.resize_as_(real_img_batch).copy_(real_img_batch)
if constants.USE_CLS:
wrong_img.resize_as_(wrong_img_batch).copy_(wrong_img_batch)
# Run through generator
gen_image = generator.forward(Variable(
text_vec, volatile=True), Variable(
noise_vec,
volatile=True)) # Returns tensor variable holding image
# Run through discriminator
real_img_passed = discriminator.forward(
Variable(real_img, volatile=True),
Variable(real_caption, volatile=True))
fake_img_passed = discriminator.forward(
gen_image.detach(), Variable(real_caption, volatile=True))
if constants.USE_CLS:
wrong_img_passed = discriminator.forward(
Variable(wrong_img, volatile=True),
Variable(real_caption, volatile=True))
# Calculate D loss
# D LOSS
if constants.USE_REAL_LS:
d_real_loss = torch.mean(
torch.abs(real_img_passed - Variable(real_img)))
d_fake_loss = torch.mean(torch.abs(fake_img_passed -
gen_image))
d_real_sensitivity_loss = torch.mean(
torch.abs(fake_img_passed - Variable(real_img)))
d_loss = d_real_loss - began_k * (
0.5 * d_fake_loss + 0.5 * d_real_sensitivity_loss)
balance = (model_options['began_gamma'] * d_real_loss -
0.5 * d_fake_loss -
0.5 * d_real_sensitivity_loss).data[0]
elif constants.USE_CLS:
d_real_loss = torch.mean(
torch.abs(real_img_passed - Variable(real_img)))
d_wrong_loss = torch.mean(
torch.abs(fake_img_passed - Variable(wrong_img)))
d_fake_loss = torch.mean(torch.abs(fake_img_passed -
gen_image))
d_loss = 0.5 * d_real_loss + 0.5 * d_wrong_loss - began_k * d_fake_loss
balance = (model_options['began_gamma'] *
(0.5 * d_real_loss + 0.5 * d_wrong_loss) -
d_fake_loss).data[0]
# No CLS option
else:
d_real_loss = torch.mean(
torch.abs(real_img_passed - Variable(real_img)))
d_fake_loss = torch.mean(torch.abs(fake_img_passed -
gen_image))
d_loss = d_real_loss - began_k * d_fake_loss
# Update began k value
balance = (model_options['began_gamma'] * d_real_loss -
d_fake_loss).data[0]
# Calculate G loss
if constants.USE_REAL_LS:
g_loss = 0.5 * torch.mean(
torch.abs(fake_img_passed - gen_image))
g_loss += 0.5 * torch.mean(
torch.abs(fake_img_passed - Variable(real_img)))
elif constants.USE_CLS:
g_loss = torch.mean(torch.abs(fake_img_passed - gen_image))
g_loss -= 0.5 * torch.mean(
torch.abs(fake_img_passed - Variable(wrong_img)))
else:
# L_G = L(y_f)
g_loss = torch.mean(torch.abs(fake_img_passed - gen_image))
# M = L(y_r) + |gamma * L(y_r) - L(y_f)|
convergence_val = d_real_loss + abs(balance)
if i % constants.LOSS_SAVE_IDX == 0:
losses['val']['generator'].append((g_loss.data[0], epoch, i))
losses['val']['discriminator'].append(
(d_loss.data[0], epoch, i))
losses['val']['converge'].append(
(convergence_val.data[0], epoch, i))
print('Val G Loss: ', g_loss.data[0])
print('Val D Loss: ', d_loss.data[0])
print('Val Convergence: ', convergence_val.data[0])
# Save losses
torch.save(losses, constants.SAVE_PATH + 'losses')
# Save images
vutils.save_image(gen_image[0].data.cpu(),
constants.SAVE_PATH + 'images/gen0_epoch' +
str(epoch) + '.png',
normalize=True)
vutils.save_image(gen_image[1].data.cpu(),
constants.SAVE_PATH + 'images/gen1_epoch' +
str(epoch) + '.png',
normalize=True)
vutils.save_image(fake_img_passed[0].data.cpu(),
constants.SAVE_PATH + 'images/gen_recon0_epoch' +
str(epoch) + '.png',
normalize=True)
vutils.save_image(fake_img_passed[1].data.cpu(),
constants.SAVE_PATH + 'images/gen_recon1_epoch' +
str(epoch) + '.png',
normalize=True)
# vutils.save_image(real_img_passed[0].data.cpu(),
# constants.SAVE_PATH + 'images/real_recon0_epoch' + str(epoch) + '.png',
# normalize=True)
# vutils.save_image(real_img_passed[1].data.cpu(),
# constants.SAVE_PATH + 'images/real_recon1_epoch' + str(epoch) + '.png',
# normalize=True)
# Save model
if epoch % constants.CHECKPOINT_FREQUENCY == 0 and epoch != 0 or epoch == constants.NUM_EPOCHS - 1:
save_checkpoint = {
'epoch': epoch,
'g_dict': generator.state_dict(),
'd_dict': discriminator.state_dict(),
'g_optimizer': g_optimizer.state_dict(),
'd_optimizer': d_optimizer.state_dict(),
'began_k': began_k
}
torch.save(save_checkpoint,
constants.SAVE_PATH + 'weights/epoch' + str(epoch))
plt.show()
plt.plot(l_hist_test)
plt.show()
plt.plot(acc_hist_test)
plt.show()
#output loss history to csv file
with open("net_dnn3_loss_train.csv", "w") as F:
for t in l_hist_train:
F.write(str(t) + ",")
F.write("0")
with open("net_dnn3_acc_train.csv", "w") as F:
for t in acc_hist_train:
F.write(str(t) + ",")
F.write("0")
with open("net_dnn3_loss_test.csv", "w") as F:
for t in l_hist_test:
F.write(str(t) + ",")
F.write("0")
with open("net_dnn3_acc_test.csv", "w") as F:
for t in acc_hist_test:
F.write(str(t) + ",")
def saveRegressionLoss(self,lossList,lossSavePath):
with open(lossSavePath, "wb") as f:
for loss in lossList:
f.write(str(loss) +"\n")
def evaluate(self, loader):
print('Evaluating at {} epochs...'.format(self.epoch))
torch.set_grad_enabled(False)
# remove previous viz results
makedirs(self.args.vis, remove=True)
self.netwrapper.eval()
# initialize meters
loss_meter = AverageMeter()
sdr_mix_meter = AverageMeter()
sdr_meter = AverageMeter()
sir_meter = AverageMeter()
sar_meter = AverageMeter()
# initialize HTML header
visualizer = HTMLVisualizer(os.path.join(self.args.vis, 'index.html'))
header = ['Filename', 'Input Mixed Audio']
for n in range(1, self.args.num_mix + 1):
header += [
'Video {:d}'.format(n), 'Predicted Audio {:d}'.format(n),
'GroundTruth Audio {}'.format(n),
'Predicted Mask {}'.format(n), 'GroundTruth Mask {}'.format(n)
]
header += ['Loss weighting']
visualizer.add_header(header)
vis_rows = []
eval_num = 0
valid_num = 0
#for i, batch_data in enumerate(self.loader['eval']):
for i, batch_data in enumerate(loader):
# forward pass
eval_num += batch_data['mag_mix'].shape[0]
with torch.no_grad():
err, outputs = self.netwrapper.forward(batch_data, args)
err = err.mean()
if self.mode == 'train':
self.writer.add_scalar('data/val_loss', err,
self.args.epoch_iters * self.epoch + i)
loss_meter.update(err.item())
print('[Eval] iter {}, loss: {:.4f}'.format(i, err.item()))
# calculate metrics
sdr_mix, sdr, sir, sar, cur_valid_num = calc_metrics(
batch_data, outputs, self.args)
print("sdr_mix, sdr, sir, sar: ", sdr_mix, sdr, sir, sar)
sdr_mix_meter.update(sdr_mix)
sdr_meter.update(sdr)
sir_meter.update(sir)
sar_meter.update(sar)
valid_num += cur_valid_num
'''
# output visualization
if len(vis_rows) < self.args.num_vis:
output_visuals(vis_rows, batch_data, outputs, self.args)
'''
metric_output = '[Eval Summary] Epoch: {}, Loss: {:.4f}, ' \
'SDR_mixture: {:.4f}, SDR: {:.4f}, SIR: {:.4f}, SAR: {:.4f}'.format(
self.epoch, loss_meter.average(),
sdr_mix_meter.sum_value()/eval_num,
sdr_meter.sum_value()/eval_num,
sir_meter.sum_value()/eval_num,
sar_meter.sum_value()/eval_num
)
if valid_num / eval_num < 0.8:
metric_output += ' ---- Invalid ---- '
print(metric_output)
learning_rate = ' lr_sound: {}, lr_frame: {}'.format(
self.args.lr_sound, self.args.lr_frame)
with open(self.args.log, 'a') as F:
F.write(metric_output + learning_rate + '\n')
self.history['val']['epoch'].append(self.epoch)
self.history['val']['err'].append(loss_meter.average())
self.history['val']['sdr'].append(sdr_meter.sum_value() / eval_num)
self.history['val']['sir'].append(sir_meter.sum_value() / eval_num)
self.history['val']['sar'].append(sar_meter.sum_value() / eval_num)
'''
print('Plotting html for visualization...')
visualizer.add_rows(vis_rows)
visualizer.write_html()
'''
# Plot figure
if self.epoch > 0:
print('Plotting figures...')
plot_loss_metrics(self.args.ckpt, self.history)
py_field = Field(tokenize=tokenize_py, init_token='<sos>', eos_token='<eos>', lower=True, batch_first=True)
han_field = Field(tokenize=tokenize_ch, init_token='<sos>', eos_token='<eos>', lower=True, batch_first=True)
# In[3]:
train_data = TranslationDataset('./data/ai_shell_train_sd',('.pinyin','.han'),(py_field,han_field))
valid_data = TranslationDataset('./data/ai_shell_dev_sd',('.pinyin','.han'),(py_field,han_field))
test_data = TranslationDataset('./data/ai_shell_test_sd',('.pinyin','.han'),(py_field,han_field))
py_field.build_vocab(train_data, min_freq=2)
han_field.build_vocab(train_data, min_freq=2)
py_stoi = dict(py_field.vocab.stoi)
with open('./data/py_vocab_sd.txt','wt') as F:
F.write(str(py_stoi))
han_stoi = dict(han_field.vocab.stoi)
with open('./data/han_vocab_sd.txt','wt') as F:
F.write(str(han_stoi))
# In[4]:
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# In[5]:
def save_word_embeddings(directory, file, embeddings_index):
f = open(os.path.join(directory, file), 'w')
for word, vec in embeddings_index.items():
f.write(word + ' ' + ' '.join(str(x) for x in vec) + '\n')
f.close()
plt.savefig('poke_img/type_acc.png')
plt.close()
# plt.plot(range(1, n_epochs + 1), history['train_acc'], label='train_acc')
plt.plot(range(1, n_epochs + 1), history['test2_acc'], label='test2_acc')
plt.title('Accuracies [pokemon_type]')
plt.xlabel('epoch')
plt.ylabel('accuracy')
plt.legend()
plt.savefig('poke_img/type_acc2.png')
plt.close()
text_name = 'result.txt'
print(best_pred)
with open(text_name, mode='w') as f:
f.write('------text------\n')
with open(text_name, mode='a') as f:
f.write(str(best_pred))
testsets = Creat_Datasets('trim_pokemon_image_151', data_transform)
test_loader = DataLoader(testsets, batch_size=1, shuffle=False)
net = torch.load('model')
correct = 0
with torch.no_grad():
for i, (images, labels) in enumerate(tqdm(test_loader)):
outputs = net(images)
_, predicted = torch.max(outputs.data, 1)
pred_type[i] = predicted[0].numpy()
correct += (predicted == labels).sum().item()
def main():
print("Starting..")
output_path = constants.SAVE_PATH
if not os.path.exists(output_path):
os.makedirs(output_path)
print("Made output directory")
else:
print("WARNING: starting training with an existing outputs directory")
if not os.path.exists(output_path + 'weights/'):
os.makedirs(output_path + 'weights/')
print("Made weights directory")
if not os.path.exists(output_path + 'images/'):
os.makedirs(output_path + 'images/')
print("Made images directory")
model_options = constants.MAIN_MODEL_OPTIONS
# Load map mapping examples to their train/dev/test split
dataset_map = util.load_dataset_map()
print("Loading data")
# Load the caption text vectors
train_captions, val_captions, test_captions = util.load_text_vec(
'Data', constants.VEC_OUTPUT_FILE_NAME, dataset_map)
# Loads and separates images into train, dev, and test sets
if os.path.exists(constants.FLOWERS_DICTS_PATH):
image_dicts = torch.load(constants.FLOWERS_DICTS_PATH)
train_image_dict, val_image_dict, test_image_dict = image_dicts
print("Loaded images")
else:
print("Loading images and separating into train/val/test sets")
filenames = train_captions.keys() + val_captions.keys(
) + test_captions.keys()
train_image_dict, val_image_dict, test_image_dict = util.load_images(
'Data/' + constants.DIRECTORY_PATH, filenames, dataset_map)
image_dicts = [train_image_dict, val_image_dict, test_image_dict]
torch.save(image_dicts, "Data/flowers_dicts.torch")
# Creates the model
if constants.USE_MODEL == 'began':
generator = CondBeganGenerator(model_options)
discriminator = CondBeganDiscriminator(model_options)
elif constants.USE_MODEL == 'wgan':
generator = WGanGenerator(model_options)
discriminator = WGanDiscriminator(model_options)
else:
generator = Generator(model_options)
discriminator = Discriminator(model_options)
# Put G and D on cuda if GPU available
if torch.cuda.is_available():
print("CUDA is available")
generator = generator.cuda()
discriminator = discriminator.cuda()
print("Moved models to GPU")
# Initialize weights
generator.apply(util.weights_init)
discriminator.apply(util.weights_init)
g_optimizer = optim.Adam(generator.parameters(),
lr=constants.LR,
betas=constants.BETAS)
# Changes the optimizer to SGD if declared in constants
if constants.D_OPTIMIZER_SGD:
d_optimizer = optim.SGD(discriminator.parameters(), lr=constants.LR)
else:
d_optimizer = optim.Adam(discriminator.parameters(),
lr=constants.LR,
betas=constants.BETAS)
print("Added optimizers")
new_epoch = 0
train_losses = {"generator": [], "discriminator": []}
val_losses = {"generator": [], "discriminator": []}
losses = {'train': train_losses, 'val': val_losses}
if args.resume:
print("Resuming from epoch " + args.resume)
checkpoint = torch.load(constants.SAVE_PATH + 'weights/epoch' +
str(args.resume))
new_epoch = checkpoint['epoch'] + 1
generator.load_state_dict(checkpoint['g_dict'])
discriminator.load_state_dict(checkpoint['d_dict'])
if constants.USE_MODEL == 'began':
discriminator.began_k = checkpoint['began_k']
g_optimizer.load_state_dict(checkpoint['g_optimizer'])
d_optimizer.load_state_dict(checkpoint['d_optimizer'])
losses = torch.load(constants.SAVE_PATH + 'losses')
# TODO: MAKE SURE IMAGES ARE OF DIMENSIONS (BATCHSIZE, CHANNELS, H, W)
# TODO: ADD L1/L2 Regularizaiton
# TODO: USE DATALOADER FROM TORCH UTILS!!!!!!!!!
# data_loader = DataLoader(self.dataset, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers)
# TODO: ADD PARALLELIZATION
# TODO: ADD IMAGE PREPROCESSING? DO WE NEED TO SUBTRACT/ADD ANYTHING TO IMAGES
# TODO: Add image aug
# NOTE: CREATING VARIABLES EARLY, THEN FILL IN LATER
noise_vec = torch.FloatTensor(constants.BATCH_SIZE, model_options['z_dim'],
1, 1)
g_text_des = torch.FloatTensor(constants.BATCH_SIZE,
model_options['caption_vec_len'])
real_img = torch.FloatTensor(constants.BATCH_SIZE, constants.IMAGE_SIZE,
constants.IMAGE_SIZE)
real_caption = torch.FloatTensor(constants.BATCH_SIZE,
model_options['caption_vec_len'])
if constants.USE_CLS:
wrong_img = torch.FloatTensor(constants.BATCH_SIZE,
constants.IMAGE_SIZE,
constants.IMAGE_SIZE)
wrong_caption = torch.FloatTensor(constants.BATCH_SIZE,
model_options['caption_vec_len'])
# Add cuda GPU option
if torch.cuda.is_available():
noise_vec = noise_vec.cuda()
g_text_des = g_text_des.cuda()
real_img = real_img.cuda()
real_caption = real_caption.cuda()
if constants.USE_CLS: wrong_img = wrong_img.cuda()
# Number of total iterations
num_iterations = 0
# Loop over dataset N times
for epoch in range(new_epoch, constants.NUM_EPOCHS):
print("Epoch %d" % (epoch))
st = time.time()
# WGAN trains D number of times more than G
curr_count = 0
if constants.USE_MODEL == 'wgan':
if num_iterations < 25 or num_iterations % 500 == 0:
d_iters = 100
else:
d_iters = model_options['wgan_d_iter']
for i, batch_iter in enumerate(
grouper(train_captions.keys(), constants.BATCH_SIZE)):
batch_keys = [x for x in batch_iter if x is not None]
curr_batch_size = len(batch_keys)
discriminator.train()
generator.train()
# Zero out gradient
discriminator.zero_grad()
# Save computations for gradient calculations
for p in discriminator.parameters():
p.requires_grad = True # Need this to be true to update generator as well
# Gather batch data
noise_batch = torch.randn(curr_batch_size, model_options['z_dim'],
1, 1)
g_text_des_batch = torch.Tensor(
get_text_description(train_captions, batch_keys))
real_caption_batch = torch.Tensor(
get_text_description(train_captions, batch_keys))
real_img_batch = torch.Tensor(
choose_real_image(train_image_dict, batch_keys))
if constants.USE_CLS:
wrong_img_batch = torch.Tensor(
choose_wrong_image(train_image_dict, batch_keys))
if torch.cuda.is_available():
noise_batch = noise_batch.cuda()
g_text_des_batch = g_text_des_batch.cuda()
real_caption_batch = real_caption_batch.cuda()
real_img_batch = real_img_batch.cuda()
if constants.USE_CLS:
wrong_img_batch = wrong_img_batch.cuda()
# Fill in tensors with batch data
noise_vec.resize_as_(noise_batch).copy_(noise_batch)
g_text_des.resize_as_(g_text_des_batch).copy_(g_text_des_batch)
real_caption.resize_as_(g_text_des_batch).copy_(g_text_des_batch)
real_img.resize_as_(real_img_batch).copy_(real_img_batch)
if constants.USE_CLS:
wrong_img.resize_as_(wrong_img_batch).copy_(wrong_img_batch)
# Run through generator
gen_image = generator.forward(
Variable(g_text_des),
Variable(noise_vec)) # Returns tensor variable holding image
# Run through discriminator
real_img_passed = discriminator.forward(Variable(real_img),
Variable(real_caption))
fake_img_passed = discriminator.forward(gen_image.detach(),
Variable(real_caption))
if constants.USE_CLS:
wrong_img_passed = discriminator.forward(
Variable(wrong_img), Variable(real_caption))
##### Train Discriminator #####
# calc_grad_d calcs gradients and steps backward
if constants.USE_MODEL == 'began':
if constants.USE_CLS:
d_loss = discriminator.calc_grad_d(Variable(real_img),
real_img_passed,
gen_image,
fake_img_passed,
Variable(wrong_img),
wrong_img_passed)
else:
d_loss = discriminator.calc_grad_d(Variable(real_img),
real_img_passed,
gen_image,
fake_img_passed)
else:
if constants.USE_CLS:
d_loss = discriminator.calc_grad_d(real_img_passed,
fake_img_passed,
wrong_img_passed)
else:
d_loss = discriminator.calc_grad_d(real_img_passed,
fake_img_passed)
d_optimizer.step()
# WGAN trains D number of times more than G
if constants.USE_MODEL == 'wgan':
if curr_count < d_iters and i < (len(train_captions) /
constants.BATCH_SIZE) - 1:
curr_count += 1
num_iterations += 1
continue
else:
# Update G after d iterations or after reaching end of epoch
curr_count = 0
##### Train Generator #####
for p in discriminator.parameters():
p.requires_grad = False
generator.zero_grad()
# Generate image again if you want to
if constants.REGEN_IMAGE:
noise_batch = torch.randn(curr_batch_size,
model_options['z_dim'], 1, 1)
if torch.cuda.is_available():
noise_batch = noise_batch.cuda()
noise_vec.resize_as_(noise_batch).copy_(noise_batch)
gen_image = generator.forward(Variable(g_text_des),
Variable(noise_vec))
new_fake_img_passed = discriminator.forward(
gen_image, Variable(real_caption))
if constants.USE_MODEL == 'began':
g_loss = generator.calc_grad_g(gen_image, new_fake_img_passed)
else:
g_loss = generator.calc_grad_g(new_fake_img_passed)
g_optimizer.step()
# learning rate decay
if constants.USE_MODEL == 'began':
g_optimizer = adjust_learning_rate(g_optimizer, num_iterations)
d_optimizer = adjust_learning_rate(d_optimizer, num_iterations)
if i % constants.LOSS_SAVE_IDX == 0:
losses['train']['generator'].append((g_loss.data[0], epoch, i))
losses['train']['discriminator'].append(
(d_loss.data[0], epoch, i))
num_iterations += 1
print('Total number of iterations: ', num_iterations)
print('Training G Loss: ', g_loss.data[0])
print('Training D Loss: ', d_loss.data[0])
epoch_time = time.time() - st
print("Time: ", epoch_time)
if epoch == constants.REPORT_EPOCH:
with open(constants.SAVE_PATH + 'report.txt', 'w') as f:
f.write(constants.EXP_REPORT)
f.write("Time per epoch: " + str(epoch_time))
copyfile("constants.py", constants.SAVE_PATH + 'constants.py')
print("Saved report")
'''
DEV SET
'''
# Calculate dev set loss
# Volatile is true because we are running in inference mode (no need to calculate gradients)
generator.eval()
discriminator.eval()
for i, batch_iter in enumerate(
grouper(val_captions.keys(), constants.BATCH_SIZE)):
batch_keys = [x for x in batch_iter if x is not None]
curr_batch_size = len(batch_keys)
# Gather batch data
noise_batch = torch.randn(curr_batch_size, model_options['z_dim'],
1, 1)
g_text_des_batch = torch.Tensor(
get_text_description(val_captions, batch_keys))
real_caption_batch = torch.Tensor(
get_text_description(val_captions, batch_keys))
real_img_batch = torch.Tensor(
choose_real_image(val_image_dict, batch_keys))
if constants.USE_CLS:
wrong_img_batch = torch.Tensor(
choose_wrong_image(val_image_dict, batch_keys))
if torch.cuda.is_available():
noise_batch = noise_batch.cuda()
g_text_des_batch = g_text_des_batch.cuda()
real_caption_batch = real_caption_batch.cuda()
real_img_batch = real_img_batch.cuda()
if constants.USE_CLS:
wrong_img_batch = wrong_img_batch.cuda()
# Fill in tensors with batch data
noise_vec.resize_as_(noise_batch).copy_(noise_batch)
g_text_des.resize_as_(g_text_des_batch).copy_(g_text_des_batch)
real_caption.resize_as_(g_text_des_batch).copy_(g_text_des_batch)
real_img.resize_as_(real_img_batch).copy_(real_img_batch)
if constants.USE_CLS:
wrong_img.resize_as_(wrong_img_batch).copy_(wrong_img_batch)
# Run through generator
gen_image = generator.forward(Variable(
g_text_des, volatile=True), Variable(
noise_vec,
volatile=True)) # Returns tensor variable holding image
# Run through discriminator
real_img_passed = discriminator.forward(
Variable(real_img, volatile=True),
Variable(real_caption, volatile=True))
fake_img_passed = discriminator.forward(
gen_image.detach(), Variable(real_caption, volatile=True))
if constants.USE_CLS:
wrong_img_passed = discriminator.forward(
Variable(wrong_img, volatile=True),
Variable(real_caption, volatile=True))
# Calculate D loss
if constants.USE_MODEL == 'began':
if constants.USE_CLS:
d_loss = discriminator.loss(Variable(real_img),
real_img_passed, gen_image,
fake_img_passed,
Variable(wrong_img),
wrong_img_passed)
else:
d_loss = discriminator.loss(Variable(real_img),
real_img_passed, gen_image,
fake_img_passed)
elif constants.USE_MODEL == 'wgan':
if constants.USE_CLS:
d_loss, d_real_loss, d_fake_loss, d_wrong_loss = discriminator.loss(
real_img_passed, fake_img_passed, wrong_img_passed)
else:
d_loss, d_real_loss, d_fake_loss = discriminator.loss(
real_img_passed, fake_img_passed)
# Vanilla Model
else:
if constants.USE_CLS:
d_loss = discriminator.loss(real_img_passed,
fake_img_passed,
wrong_img_passed)
else:
d_loss = discriminator.loss(real_img_passed,
fake_img_passed)
# Calculate G loss
if constants.USE_MODEL == 'began':
g_loss = generator.loss(gen_image, fake_img_passed)
else:
g_loss = generator.loss(fake_img_passed)
if i % constants.LOSS_SAVE_IDX == 0:
losses['val']['generator'].append((g_loss.data[0], epoch, i))
losses['val']['discriminator'].append(
(d_loss.data[0], epoch, i))
print('Val G Loss: ', g_loss.data[0])
print('Val D Loss: ', d_loss.data[0])
# Save losses
torch.save(losses, constants.SAVE_PATH + 'losses')
# Save images
vutils.save_image(gen_image[0].data.cpu(),
constants.SAVE_PATH + 'images/gen0_epoch' +
str(epoch) + '.png',
normalize=True)
vutils.save_image(gen_image[1].data.cpu(),
constants.SAVE_PATH + 'images/gen1_epoch' +
str(epoch) + '.png',
normalize=True)
if constants.USE_MODEL == 'began':
vutils.save_image(real_img_passed[0].data.cpu(),
constants.SAVE_PATH +
'images/real_recon0_epoch' + str(epoch) + '.png',
normalize=True)
vutils.save_image(real_img_passed[1].data.cpu(),
constants.SAVE_PATH +
'images/real_recon1_epoch' + str(epoch) + '.png',
normalize=True)
# Save model
if epoch % 20 == 0 or epoch == constants.NUM_EPOCHS - 1:
save_checkpoint = {
'epoch': epoch,
'g_dict': generator.state_dict(),
'd_dict': discriminator.state_dict(),
'g_optimizer': g_optimizer.state_dict(),
'd_optimizer': d_optimizer.state_dict(),
}
if constants.USE_MODEL == 'began':
save_checkpoint['began_k'] = discriminator.began_k
torch.save(save_checkpoint,
constants.SAVE_PATH + 'weights/epoch' + str(epoch))
def Dict2File(Dict, filename):
F = open(filename, 'a+')
F.write(str(Dict))
F.close()
def write_to_file(predictions):
with open('test.pred', 'w') as f:
for p in predictions:
f.write(str(p) + '\n')
f.close()
def saveTestLoss(self,lossList,lossSavePath):
with open(lossSavePath, "wb") as f:
for data in lossList:
f.write(str(data[0])+" "+ str(data[1]) +"\n")
def run(self):
low_dev_perf = 0
for e in range(self.epoch):
if self.config['use_entire_example_epoch'] > e:
train_examples = self.data_io.build_entire_examples()
else:
train_examples = self.data_io.build_examples(1.0,
split='train')
train_iter = self.data_io.build_iter_idx(train_examples, True)
for i, batch_idx in enumerate(train_iter):
loss = self.get_loss(batch_idx, train_examples,
self.train_loss)
loss.backward()
# plot_grad_flow(self.model.named_parameters())
if self.global_step % self.config['update_iter'] == 0:
clip_grad_norm_(self.params, self.config['grad_clip'])
self.sgd.step()
self.sgd.zero_grad()
self.global_step += 1
self.get_loss(
random.sample(self.high_dev_iter, 1)[0],
self.high_dev_examples, self.dev_loss)
if self.global_step % (self.config['check_step'] *
self.config['update_iter']) == 0:
train_log = self.train_log.format(
e, 100.0 * i / len(train_iter),
np.mean(self.train_loss['vader']
[-self.config['check_step']:]),
np.mean(self.train_loss['flair']
[-self.config['check_step']:]),
np.mean(self.train_loss['sent']
[-self.config['check_step']:]),
np.mean(self.train_loss['subj']
[-self.config['check_step']:]),
np.mean(self.train_loss['emotion']
[-self.config['check_step']:]),
np.mean(self.train_loss['author']
[-self.config['check_step']:]),
np.mean(self.train_loss['topic']
[-self.config['check_step']:]))
print(train_log)
self.train_procedure.append(train_log)
high_dev_perf, _ = self.get_perf(self.high_dev_iter,
self.high_dev_examples)
# print('HIGH DEV: ', high_dev_perf)
if high_dev_perf['mean'] > self.dev_perf['mean']:
self.dev_perf = high_dev_perf
low_dev_perf, _ = self.get_perf(
self.low_dev_iter, self.low_dev_examples)
torch.save(
{
'model': self.model.state_dict(),
'adam': self.sgd.state_dict()
},
os.path.join(self.config['root_folder'],
self.config['outlet'],
'best_model.pt'))
test_perf, test_preds = self.get_perf(
self.test_iter, self.test_examples)
self.test_perf = test_perf
json.dump(
test_perf,
open(
os.path.join(self.config['root_folder'],
self.config['outlet'],
'test_perf.json'), 'w'))
with open(
os.path.join(self.config['root_folder'],
self.config['outlet'],
'test_pred.jsonl'), 'w') as f:
for pred in test_preds:
data = json.dumps(pred)
f.write(data)
print('BEST DEV: ', self.dev_perf)
print('LOw DEV: ', low_dev_perf)
print('BEST TEST: ', self.test_perf)
# train_examples = self.data_io.build_examples(prob_to_full=self.config['prob_to_full'], split='train')
# train_iter = self.data_io.build_iter_idx(train_examples, True)
json.dump(
self.train_loss,
open(
os.path.join(self.config['root_folder'], self.config['outlet'],
'train_loss.json'), 'w'))
json.dump(
self.dev_loss,
open(
os.path.join(self.config['root_folder'], self.config['outlet'],
'dev_loss.json'), 'w'))
with open(
os.path.join(self.config['root_folder'], self.config['outlet'],
'train_log.txt'), 'w') as f:
for line in self.train_procedure:
f.write(line + '\n')
def saveValidLoss(self,lossList,lossSavePath):
with open(lossSavePath, "wb") as f:
for data in lossList:
f.write(str(data) +"\n")
def saveTXT(self, lossList, lossSavePath):
with open(lossSavePath, "wb") as f:
for i in xrange(lossList.shape[0]):
f.write(str(lossList[i]) + "\n")
loss = criterion(outputs, labels)
val_loss += loss.item()
for j, o in enumerate(outputs):
if torch.argmax(o) == labels[j]:
val_acc += 1
print("\rValidating. Epoch: {}\t|\ti: {}".format(epoch, i), end="")
res_text = "[{:3d}/{:3d}] {:.2f} sec(s) Train Acc: {:3.6f} Loss: {:3.6f} | Val Acc: {:3.6f} loss: {:3.6f}".format(
epoch, num_epoch,
time.time() - epoch_start_time, train_acc / train_size,
train_loss / t_num, val_acc / val_size, val_loss / val_size)
# 输出训练日志和写入日志文件
print(res_text)
with open(r"/path/to/log", "a") as f:
f.write(res_text)
# 保存最佳模型和固定间隔的模型
# 这里可以多保存一个优化器的中间状态
if (val_acc / val_size) > best_val_acc:
best_val_acc = (val_acc / val_size)
model_save_file_name = "best_model_save.pth"
torch.save({
'net': model.state_dict(),
'epoch': epoch
}, checkpoint_path / Path(model_save_file_name))
if epoch % checkpoint_flag == 0:
model_save_file_name = "resnet_model_save_epoch_{}.pt".format(epoch)
torch.save({
'net': model.state_dict(),
