def __init__(self, args):
self.args = args
# Define the Network
# self.netG_A2B = Generator(input_channels=self.args.input_nc, output_channels=self.args.output_nc, n_residual_blocks=self.args.n_Rk).to(device=self.args.device)
self.netG_A2B = Generator(
input_shape=(3, 256, 256),
num_residual_blocks=self.args.n_Rk).to(device=self.args.device)
# self.netG_B2A = Generator(input_channels=self.args.output_nc, output_channels=self.args.input_nc, n_residual_blocks=self.args.n_Rk).to(device=self.args.device)
self.netG_B2A = Generator(
input_shape=(3, 256, 256),
num_residual_blocks=self.args.n_Rk).to(device=self.args.device)
# self.netD_A = Discriminator(input_nc=self.args.input_nc).to(device=self.args.device)
# self.netD_B = Discriminator(input_nc=self.args.output_nc).to(device=self.args.device)
self.netD_A = Discriminator(input_shape=(3, 256, 256)).to(
device=self.args.device)
self.netD_B = Discriminator(input_shape=(3, 256, 256)).to(
device=self.args.device)
init_weight(self.netD_B, init_type=args.init_weight)
init_weight(self.netD_A, init_type=args.init_weight)
init_weight(self.netG_A2B, init_type=args.init_weight)
init_weight(self.netG_B2A, init_type=args.init_weight)
# Define Loss function
self.criterion_GAN = torch.nn.MSELoss()
self.criterion_cycle = torch.nn.L1Loss()
self.criterion_identity = torch.nn.L1Loss()
# Optimizers
self.optimizerG = torch.optim.Adam(itertools.chain(
self.netG_A2B.parameters(), self.netG_B2A.parameters()),
lr=self.args.lr,
betas=(self.args.b1, self.args.b2))
self.optimizerD_A = torch.optim.Adam(self.netD_A.parameters(),
lr=self.args.lr,
betas=(self.args.b1,
self.args.b2))
self.optimizerD_B = torch.optim.Adam(self.netD_B.parameters(),
lr=self.args.lr,
betas=(self.args.b1,
self.args.b2))
# Learning rate scheduler
self.lr_scheduler_G = torch.optim.lr_scheduler.LambdaLR(
self.optimizerG,
lr_lambda=LambdaLR(self.args.num_epochs, self.args.epoch,
self.args.decay_epoch).step)
self.lr_scheduler_D_A = torch.optim.lr_scheduler.LambdaLR(
self.optimizerD_A,
lr_lambda=LambdaLR(self.args.num_epochs, self.args.epoch,
self.args.decay_epoch).step)
self.lr_scheduler_D_B = torch.optim.lr_scheduler.LambdaLR(
self.optimizerD_B,
lr_lambda=LambdaLR(self.args.num_epochs, self.args.epoch,
self.args.decay_epoch).step)
#dataset
self.train_dataloader = data_loader(self.args)
self.val_dataloader = data_loader(self.args, mode='test')
def __init__(self, root='data/edges2shoes', result_dir='result', weight_dir='weight', load_weight=False,
batch_size=2, test_size=20, test_img_num=5, img_size=128, num_epoch=100, save_every=1000,
lr=0.0002, beta_1=0.5, beta_2=0.999, lambda_kl=0.01, lambda_img=10, lambda_z=0.5, z_dim=8):
# Data type(Can use GPU or not?)
self.dtype = torch.cuda.FloatTensor
if torch.cuda.is_available() is False:
self.dtype = torch.FloatTensor
# Data loader for training
self.dloader, dlen = data_loader(root=root, batch_size=batch_size, shuffle=True,
img_size=img_size, mode='train')
# Data loader for test
self.t_dloader, _ = data_loader(root=root, batch_size=test_size, shuffle=False,
img_size=img_size, mode='val')
# Models
# D_cVAE is discriminator for cVAE-GAN(encoded vector z).
# D_cLR is discriminator for cLR-GAN(random vector z).
# Both of D_cVAE and D_cLR has two discriminators which have different output size((14x14) and (30x30)).
# Totally, we have for discriminators now.
self.D_cVAE = model.Discriminator().type(self.dtype)
self.D_cLR = model.Discriminator().type(self.dtype)
self.G = model.Generator(z_dim=z_dim).type(self.dtype)
self.E = model.Encoder(z_dim=z_dim).type(self.dtype)
# Optimizers
self.optim_D_cVAE = optim.Adam(self.D_cVAE.parameters(), lr=lr, betas=(beta_1, beta_2))
self.optim_D_cLR = optim.Adam(self.D_cLR.parameters(), lr=lr, betas=(beta_1, beta_2))
self.optim_G = optim.Adam(self.G.parameters(), lr=lr, betas=(beta_1, beta_2))
self.optim_E = optim.Adam(self.E.parameters(), lr=lr, betas=(beta_1, beta_2))
# Optiminzer lr scheduler
#self.optim_D_scheduler = optim.lr_scheduler.LambdaLR(self.optim_D, lr_lambda=lr_decay_rule)
#self.optim_G_scheduler = optim.lr_scheduler.LambdaLR(self.optim_G, lr_lambda=lr_decay_rule)
#self.optim_E_scheduler = optim.lr_scheduler.LambdaLR(self.optim_E, lr_lambda=lr_decay_rule)
# fixed random_z for test
self.fixed_z = util.var(torch.randn(test_size, test_img_num, z_dim))
# Some hyperparameters
self.z_dim = z_dim
self.lambda_kl = lambda_kl
self.lambda_img = lambda_img
self.lambda_z = lambda_z
# Extra things
self.result_dir = result_dir
self.weight_dir = weight_dir
self.load_weight = load_weight
self.test_img_num = test_img_num
self.img_size = img_size
self.start_epoch = 0
self.num_epoch = num_epoch
self.save_every = save_every
def main():
# mode argument
args = argparse.ArgumentParser()
args.add_argument("--lr", type=float, default=0.0001)
args.add_argument("--cuda", type=bool, default=True)
args.add_argument("--num_epochs", type=int, default=50000)
args.add_argument("--model_name", type=str, default="5")
args.add_argument("--batch", type=int, default=2)
args.add_argument("--mode", type=str, default="train")
args.add_argument("--prediction_dir", type=str, default="prediction")
args.add_argument("--print_iter", type=int, default=20000)
config = args.parse_args()
lr = config.lr
cuda = config.cuda
num_epochs = config.num_epochs
model_name = config.model_name
batch = config.batch
mode = config.mode
prediction_dir = config.prediction_dir
print_iter = config.print_iter
nIn = 12
nOut = 1
model = LinearRegression(nIn, nOut)
device = torch.device('cuda') if cuda else torch.device('cpu')
#check parameter of model
print("------------------------------------------------------------")
total_params = sum(p.numel() for p in model.parameters())
print("num of parameter : ", total_params)
trainable_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print("num of trainable_ parameter :", trainable_params)
print("------------------------------------------------------------")
if mode == 'train':
print('train start')
train_loader = dataloader.data_loader(DATASET_PATH,
batch,
phase='train')
val_loader = dataloader.data_loader(DATASET_PATH, 1, phase='val')
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.Adam(params=params, lr=lr)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=40,
gamma=0.1)
train(num_epochs, model, device, train_loader, val_loader, optimizer,
lr_scheduler, prediction_dir, print_iter)
elif mode == 'test':
print('test start')
test_loader = dataloader.data_loader(DATASET_PATH, 1, phase='test')
load_model(model_name, model)
test(model, device, test_loader, prediction_dir)
def __init__(self, trn_dir, style_path, result_dir, weight_dir, num_epoch=2, batch_size=4,
content_loss_pos=2, lr=1e-3, lambda_c=1, lambda_s=5e+5, show_every=1000, save_every=5000):
if torch.cuda.is_available():
self.dtype = torch.cuda.FloatTensor
else:
self.dtype = torch.FloatTensor
self.style_path = style_path
self.result_dir = result_dir
self.weight_dir = weight_dir
# Models
self.trans_net = model.ImageTransformNet().type(self.dtype)
self.vgg16 = model.VGG16().type(self.dtype)
# Dataloader
self.dloader, total_num = dataloader.data_loader(root=trn_dir, batch_size = batch_size)
self.total_iter = int(total_num / batch_size) + 1
# Loss function and optimizer
self.mse_loss = nn.MSELoss()
self.optimizer = optim.Adam(self.trans_net.parameters(), lr=lr)
# Hyperparameters
self.content_loss_pos = content_loss_pos
self.lambda_c = lambda_c
self.lambda_s = lambda_s
self.show_every = show_every
self.save_every = save_every
self.num_epoch = num_epoch
def main(args):
dloader, dlen = data_loader(root=args.root, batch_size='all', shuffle=False,
img_size=128, mode='val')
if torch.cuda.is_available() is True:
dtype = torch.cuda.FloatTensor
else:
dtype = torch.FloatTensor
if args.epoch is not None:
weight_name = '{epoch}-G.pkl'.format(epoch=args.epoch)
else:
weight_name = 'G.pkl'
weight_path = os.path.join(args.weight_dir, weight_name)
G = model.Generator(z_dim=8).type(dtype)
G.load_state_dict(torch.load(weight_path))
G.eval()
if os.path.exists(args.result_dir) is False:
os.makedirs(args.result_dir)
# For example, img_name = random_55.png
if args.epoch is None:
args.epoch = 'latest'
img_name = '{type}_{epoch}.png'.format(type=args.sample_type, epoch=args.epoch)
img_path = os.path.join(args.result_dir, img_name)
# Make latent code and images
z = make_z(n=dlen, img_num=args.img_num, z_dim=8, sample_type=args.sample_type)
result_img = make_img(dloader, G, z, img_size=128)
torchvision.utils.save_image(result_img, img_path, nrow=args.img_num + 1, padding=4)
def create_thresholds():
df_manhattan = data_loader()
print(df_manhattan.columns)
df_total_avg = df_manhattan[[
'dropoff_latitude', 'dropoff_longitude', 'passenger_count'
]].groupby(['dropoff_longitude', 'dropoff_latitude'],
as_index=False).sum()
df_total_avg['passenger_count'] /= 90
df_total_avg.to_csv('threshold_overall.csv', index=False)
df_day_avg = df_manhattan[[
'dropoff_latitude', 'dropoff_longitude', 'passenger_count', 'week_day'
]].groupby(['dropoff_longitude', 'dropoff_latitude', 'week_day'],
as_index=False).sum()
df_day_avg['passenger_count'] /= 12
df_day_avg.to_csv('threshold_day_avg.csv', index=False)
choice = input('Do you want to create plots.Enter y/n ')
if choice == 'y':
lat, lon = [
round(float(i), 3) for i in (input(
'Enter latitude,longitude (separate by comma or enter 0,0 for Goldmna Sachs) '
)).split(',')
]
if lat == 0 and lon == 0:
plotter(df_manhattan)
else:
plotter(df_manhattan, lat, lon)
files.download('threshold_day_avg.csv')
files.download('threshold_overall.csv')
print('Data Load Succesfull')
return ('Data Load successful')
def main(alpha=None, gamma=None):
config = Config(args.config_path)
if args.mode:
config.mode = args.mode
if args.train_id:
config.train_id = args.train_id
if args.num_epochs:
config.num_epochs = args.num_epochs
if args.base_dir:
config.base_dir = args.base_dir
config.use_bayes_opt = args.use_bayes_opt
config.use_preprocess = args.use_preprocess
config.use_swa = args.use_swa
train_path = os.path.join(config.base_dir, config.train_dir, config.train_id)
result_path = os.path.join(config.base_dir, config.result_dir, config.train_id)
data_path = os.path.join(config.base_dir, config.data_dir)
if not os.path.isdir(train_path):
os.mkdir(train_path)
if not os.path.isdir(result_path):
os.mkdir(result_path)
init_logger(os.path.join(result_path, 'log.txt'))
set_seed(config)
# get data loader
tokenizer = AutoTokenizer.from_pretrained(config.bert_model_name)
param = {"root": data_path, "batch_size": config.batch_size, "tokenizer": tokenizer, "config": config}
train_dataloader = data_loader(**param, phase='train')
validate_dataloader = data_loader(**param, phase='validate')
test_dataloader = data_loader(**param, phase='test')
# create model config 확인
model = Trainer(config, train_dataloader, validate_dataloader, test_dataloader)
if config.mode == 'train':
result = model.train(alpha=alpha, gamma=gamma)
elif config.mode == 'test':
model.load_model(config.model_weight_file)
result = model.evaluate('test')
del model
return result
def main(args):
dloader, dlen = data_loader(root=args.root,
batch_size=1,
shuffle=False,
img_size=128,
mode='test')
if torch.cuda.is_available() is True:
dtype = torch.cuda.FloatTensor
else:
dtype = torch.FloatTensor
if args.epoch is not None:
weight_name = '{epoch}-G.pkl'.format(epoch=args.epoch)
else:
weight_name = 'G.pkl'
weight_path = os.path.join(args.weight_dir, weight_name)
G = model.Generator(z_dim=8).type(dtype)
G.load_state_dict(torch.load(weight_path))
G.eval()
if os.path.exists(args.result_dir) is False:
os.makedirs(args.result_dir)
# For example, img_name = random_55.png
if args.epoch is None:
args.epoch = 'latest'
filenames = get_files_name()
for iters, (img, ground_truth, mask) in enumerate(dloader):
img = util.var(img.type(dtype))
#mask = util.var(mask.type(dtype))
one = torch.ones([1, 3, 128, 128])
one = util.var(one.type(dtype))
for i in range(0, dlen):
# img_ = img.unsqueeze(dim=0)
#mask_ = mask[i].unsqueeze(dim=0)
#mask_ = one - mask_
# Make latent code and images
z = make_z(img_num=4, z_dim=8)
for j in range(4):
z_ = z[j, :].unsqueeze(dim=0)
out_img = G(img, z_)
outs_img = out_img / 2 + 0.5
img_name = '{filenames}_{style}.png'.format(
filenames=filenames[i], style=j)
print(img_name)
#mask_name = '{filenames}_{style}.png'.format(filenames = filenames[i], style = j)
img_path = os.path.join(args.result_dir, img_name)
#mask_path = os.path.join(args.mask_dir, mask_name)
torchvision.utils.save_image(outs_img, img_path)
def test2(modelfile):
paper_list = data_loader('asfd')
with open(TSV_DIR, 'wt', encoding='UTF8') as out_file:
tsv_writer = csv.writer(out_file, delimiter='\t')
for paper in paper_list[1344:]:
title = paper[2].lower()
abstract = paper[4].lower()
tokenized_words = tokenize_string(abstract)
abstract = ' '.join(tokenized_words)
tsv_writer.writerow([title, abstract, 'positive', 0])
def __init__(self, args):
self.args = args
self.G = Generator(3).to(device=self.args.device)
self.D = Discriminator(3).to(device=self.args.device)
init_weight(self.G, init_type=self.args.init_weight, init_gain=0.02)
init_weight(self.D, init_type=self.args.init_weight, init_gain=0.02)
self.optimizerG = torch.optim.Adam(self.G.parameters(),
lr=args.lr,
betas=(args.b1, args.b2))
self.optimizerD = torch.optim.Adam(self.D.parameters(),
lr=args.lr,
betas=(args.b1, args.b2))
self.criterion_GAN = torch.nn.MSELoss().to(device=args.device)
self.criterion_L1 = torch.nn.L1Loss().to(device=args.device)
self.dataloader = data_loader(self.args, mode=args.mode)
self.val_dataloader = data_loader(args, mode='test')
def main(args):
print("Begin to load data!")
EMBEDDING_DIM = glove_size
if args.is_yelp == "yelp":
root = '../data/yelp.full.cleaned'
# save_name = 'Yelp'
else:
root = '../data/amazon.cleaned.datasets'
# save_name = 'Amazon'
check_point = True if args.model_path else False
vocab, train_loader, val_loader = data_loader(root, args.max_size)
print("Loading Finished")
INPUT_DIM = len(vocab)
print("INPUT_DIM: ", INPUT_DIM)
pretrained_embeddings = vocab.vectors
model = CNN(INPUT_DIM, EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT)
# load model
if check_point:
model.load_state_dict(torch.load(args.model_path, map_location=device))
else:
model.embedding.weight.data.copy_(pretrained_embeddings)
train_iterator, valid_iterator = data.BucketIterator.splits(
(train_loader, val_loader),
batch_size=BATCH_SIZE,
device=device,
sort_key=lambda x: len(x.text),
# BucketIterator 依据什么对数据分组
sort_within_batch=True)
N_EPOCHS = 15
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
# criterion = torch.nn.BCEWithLogitsLoss()
criterion = torch.nn.CrossEntropyLoss()
model = model.to(device)
criterion = criterion.to(device)
print("Start training!")
for epoch in range(N_EPOCHS):
train_loss, train_acc = train(model, train_iterator, optimizer, criterion)
valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)
print(
f'| Epoch: {epoch + 1:02} | Train Loss: {train_loss:.3f} | Train Acc: {train_acc :.2f}% | Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc :.2f}% |')
def test(args):
ckpt_path = os.path.join(args.root_path, args.ckpt_path)
G = Generator(3).to(device=args.device)
try:
ckpt = pix2pix_load(ckpt_path, args.device)
G.load_state_dict(ckpt['G'])
except:
print('Failed to load checkpoint')
dataloader = data_loader(args, mode='test')
for i in range(len(dataloader)):
sample_images(args, i, G, dataloader)
def test(args):
# network init
netG_A2B = Generator(args.input_nc, args.output_nc,
args.n_Rk).to(device=args.device)
netG_B2A = Generator(args.output_nc, args.input_nc,
args.n_Rk).to(device=args.device)
args.result_path = args.result_path + "_test"
try:
ckpt = load_checkpoint(args.ckpt_path, args.device)
netG_B2A.load_state_dict(ckpt['netG_B2A'])
netG_A2B.load_state_dict(ckpt['netG_A2B'])
except:
print('Failed to load checkpoint')
dataloader = data_loader(args, mode='test')
netG_B2A.eval()
netG_A2B.eval()
for i in range(len(dataloader)):
sample_images(args, i, netG_A2B, netG_B2A, dataloader)
def main(args):
dloader, dlen = data_loader(root=args.root,
batch_size='all',
shuffle=False,
img_size=128,
mode='val')
if torch.cuda.is_available() is True:
dtype = torch.cuda.FloatTensor
else:
dtype = torch.FloatTensor
if args.epoch is not None:
weight_name = '{epoch}-G.pkl'.format(epoch=args.epoch)
else:
weight_name = 'G.pkl'
weight_path = os.path.join(args.weight_dir, weight_name)
G = model.Generator(z_dim=8).type(dtype)
G.load_state_dict(torch.load(weight_path))
G.eval()
if os.path.exists(args.result_dir) is False:
os.makedirs(args.result_dir)
# For example, img_name = random_55.png
if args.epoch is None:
args.epoch = 'latest'
img_name = '{type}_{epoch}.png'.format(type=args.sample_type,
epoch=args.epoch)
img_path = os.path.join(args.result_dir, img_name)
# Make latent code and images
z = make_z(n=dlen,
img_num=args.img_num,
z_dim=8,
sample_type=args.sample_type)
result_img = make_img(dloader, G, z, img_size=128)
torchvision.utils.save_image(result_img,
img_path,
nrow=args.img_num + 1,
padding=4)
def test(modelfile):
paper_list = data_loader('asfd')
paper_dic = {}
for paper in paper_list:
a = 3
b = 5
if i > 1343:
a = 2
b = 4
title = paper[a].lower()
abstract = paper[b].lower()
tokenized_words = tokenize_string(abstract)
paper_dic[title] = tokenized_words
print("dic complete")
start = time.time()
model = Doc2Vec.load(modelfile)
print(model.docvecs.index_to_doctag(1344))
print(len(model.docvecs))
similarities = {}
for s in range(len(model.docvecs)):
similarities[model.docvecs.index_to_doctag(s)] = 0
all_papers = [model.docvecs.index_to_doctag(s) for s in range(1344)]
ethics = [
model.docvecs.index_to_doctag(s)
for s in range(1344, len(model.docvecs))
]
for ethic_paper in ethics:
for paper in all_papers:
similarities[paper] += model.docvecs.similarity(ethic_paper, paper)
sorted_papers = sorted(similarities.items(), key=operator.itemgetter(1))
print("sort complete")
with open(TSV_DIR, 'wt') as out_file:
tsv_writer = csv.writer(out_file, delimiter='\t')
for paper in sorted_papers:
tsv_writer.writerow([
'2019', paper_section, paper_title, paper_authors,
paper_abstract, pdf_url, '', paper_publish_date
])
def test(modelfile):
paper_list = data_loader('asfd')
paper_dic = {}
for i, paper in enumerate(paper_list):
print(i)
a = 3
b = 5
if i > 1343:
a = 2
b = 4
title = paper[a].lower()
abstract = paper[b].lower()
tokenized_words = tokenize_string(abstract)
paper_dic[title] = ' '.join(tokenized_words)
print("dic complete")
start = time.time()
model = Doc2Vec.load(modelfile)
print(model.docvecs.index_to_doctag(1344))
print(len(model.docvecs))
similarities = {}
for s in range(1344):
similarities[model.docvecs.index_to_doctag(s)] = 0
all_papers = [model.docvecs.index_to_doctag(s) for s in range(1344)]
ethics = [
model.docvecs.index_to_doctag(s)
for s in range(1344, len(model.docvecs))
]
for ethic_paper in ethics:
for paper in all_papers:
similarities[paper] += model.docvecs.similarity(ethic_paper, paper)
sorted_papers = sorted(similarities.items(), key=operator.itemgetter(1))
print("sort complete")
print(sorted_papers)
with open(TSV_DIR, 'wt', encoding='UTF8') as out_file:
tsv_writer = csv.writer(out_file, delimiter='\t')
for paper in sorted_papers:
tsv_writer.writerow(
[paper[0], paper_dic[paper[0]], 'negative', paper[1]])
def main(args):
print("Start!")
model = CNN(INPUT_DIM, EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT, is_train=False)
model.load_state_dict(torch.load(args.model_path, map_location=device))
model = model.to(device)
print("Load model success!")
vocab, _, _ = data_loader(amazon_root, load_val=False, load_train=False)
print("Load vocab success!")
label_list = []
context_list = []
model.eval()
def get_vec(sentence, min_len=5):
tokenized = [tok.text for tok in spacy_en.tokenizer(sentence)]
if len(tokenized) < min_len:
tokenized += ['<pad>'] * (min_len - len(tokenized))
indexed = [vocab.stoi[t] for t in tokenized]
tensor = torch.LongTensor(indexed).to(device)
tensor = tensor.unsqueeze(1)
with torch.no_grad():
res = model(tensor)
return res
print("Start transfer to vector!")
with open(args.input_yelp_file, 'rt', encoding='utf-8') as fin:
csv_header = csv.reader(fin, delimiter=',')
for i, row in enumerate(csv_header):
label_list.append(row[0])
context_list.append(get_vec(row[1]).cpu())
label_list = label_list[1:]
context_list = context_list[1:]
"Start to write to json!"
with open(args.output_json_file, 'wt') as fout:
for i, context in enumerate(context_list):
average_vec_dict = {}
average_vec_dict['label'] = str(label_list[i])
average_vec_dict['avg_vec'] = context_list[i].squeeze(0).numpy().tolist()
json.dump(average_vec_dict, fout)
fout.write('\n')
if mode == 'train':
# define loss function
loss_fn = nn.L1Loss()
if cuda:
loss_fn = loss_fn.cuda()
# set optimizer
optimizer = Adam(
[param for param in model.parameters() if param.requires_grad],
lr=base_lr,
weight_decay=1e-4)
scheduler = StepLR(optimizer, step_size=40, gamma=0.1)
# get data loader
train_dataloader, _ = data_loader(root=DATASET_PATH,
phase='train',
batch_size=batch)
validate_dataloader, validate_label_file = data_loader(
root=DATASET_PATH, phase='validate', batch_size=batch)
time_ = datetime.datetime.now()
num_batches = len(train_dataloader)
#check parameter of model
print("------------------------------------------------------------")
total_params = sum(p.numel() for p in model.parameters())
print("num of parameter :", total_params)
trainable_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print("num of trainable_ parameter :", trainable_params)
print("------------------------------------------------------------")
def train(args):
if args.max_length < 0:
args.max_length = 9999
# initialize the logger
# create logger
logger_name = "mylog"
logger = logging.getLogger(logger_name)
logger.setLevel(logging.INFO)
# file handler
fh = logging.FileHandler(args.log_dir + args.log_fname)
fh.setLevel(logging.INFO)
logger.addHandler(fh)
# stream handler
console = logging.StreamHandler()
console.setLevel(logging.INFO)
logger.addHandler(console)
# torch.cuda.set_device(args.gpu_id)
for arg in vars(args):
logger.info(str(arg) + ' ' + str(getattr(args, arg)))
# load train/dev/test data
# train data
logger.info('loading data...')
train_loader, validation_loader, test_loader, len_chars, train_batch_num, test_batch_num = \
data_loader(args.train_file, args.dev_file, args.test_file, \
args.w2v_file, args.batch_size, args.max_char, args.max_sequence_length)
logger.info('train size # sent ' + str(len(train_loader)))
logger.info('dev size # sent ' + str(len(validation_loader)))
logger.info('test size # sent ' + str(len(test_loader)))
best_dev = [] # (epoch, dev_acc)
# build the model
bimpm = model.BiMPM(args.word_embedding_dim + args.char_rnn_dim, args.max_sequence_length,
len_chars + 2, args.max_char, args.char_embedding_dim, args.char_rnn_dim, args.rnn_layers, args.perspective)
# through to GPU
bimpm.cuda()
para = bimpm.parameters()
if args.optimizer == 'Adam':
optimizer = optim.Adam(para, lr=args.lr)
elif args.optimizer == 'Adadelta':
optimizer = optim.Adadelta(para, lr=args.lr)
else:
logger.info('No Optimizer.')
sys.exit()
criterion = nn.CrossEntropyLoss()
logger.info('start to train...')
total = 0.
correct = 0.
loss_data = 0.
timer = time.time()
########################################3
for epoch in range(args.epoch):
for i, (data1, data2, lengths1, lengths2, char1, char2, char_lengths1, char_lengths2, labels) in enumerate(train_loader):
data_batch1, data_batch2, word_length_batch1, word_length_batch2, char_batch1, char_batch2, \
char_length_batch1, char_length_batch2, label_batch = Variable(data1.float().cuda()), Variable(data2.float().cuda()), \
Variable(lengths1.cuda()), Variable(lengths2.cuda()), Variable(char1.cuda()), Variable(char2.cuda()), Variable(char_lengths1.cuda()), Variable(char_lengths2).cuda(), Variable(labels.cuda())
optimizer.zero_grad()
out = bimpm(data_batch1, data_batch2, char_batch1, char_batch2, args.hidden_size)
loss = criterion(out,label_batch)
loss.backward()
gradClamp(para, clip=args.max_grad_norm)
optimizer.step()
_, predict = out.data.max(dim=1)
total += label_batch.size(0)
correct += torch.sum(predict == label_batch.data)
loss_data += (loss.data[0] * args.batch_size) # / train_lbl_batch.data.size()[0])
if (i + 1) % args.display_interval == 0:
logger.info('epoch %d, batches %d|%d, train-acc %.3f, loss %.3f, time %.2fs, ' %
(epoch, i + 1, len(data_batch1), correct / total,
loss_data / label_batch.size(0), time.time() - timer))
train_sents = 0.
timer = time.time()
loss_data = 0.
correct = 0.
total = 0.
if i == len(data_batch1) - 1:
logger.info('epoch %d, batches %d|%d, train-acc %.3f, loss %.3f, time %.2fs, ' %
(epoch, i + 1, len(data_batch1), correct / total,
loss_data / label_batch.size(0), time.time() - timer))
train_sents = 0.
timer = time.time()
loss_data = 0.
correct = 0.
total = 0.
# evaluate
if (epoch + 1) % args.dev_interval == 0:
bimpm.eval()
correct = 0.
total = 0.
for j, (dev_data1, dev_data2, dev_lengths1, dev_lengths2, \
dev_char1, dev_char2, dev_char_lengths1, dev_char_lengths2, dev_labels) in enumerate(validation_loader):
dev_data_batch1, dev_data_batch2, dev_word_length_batch1, \
dev_word_length_batch2, dev_char_batch1, dev_char_batch2, \
dev_char_length_batch1, dev_char_length_batch2, dev_label_batch \
= Variable(dev_data1.float().cuda()), Variable(dev_data2.float().cuda()), Variable(dev_lengths1.cuda()), \
Variable(dev_lengths2.cuda()), Variable(dev_char1.cuda()), Variable(dev_char2.cuda()), \
Variable(dev_char_lengths1.cuda()), Variable(dev_char_lengths2.cuda()), Variable(dev_labels.cuda())
# if dev_lbl_batch.data.size(0) == 1:
# # simple sample batch
# dev_src_batch=torch.unsqueeze(dev_src_batch, 0)
# dev_tgt_batch=torch.unsqueeze(dev_tgt_batch, 0)
dev_out = bimpm(dev_data_batch1, dev_data_batch2, dev_char_batch1, dev_char_batch2, args.hidden_size)
_, predict=dev_out.data.max(dim=1)
total += dev_label_batch.size(0)
correct += torch.sum(predict == dev_label_batch.data)
dev_acc = correct / total
logger.info('dev-acc %.3f' % (dev_acc))
if (epoch + 1) / args.dev_interval == 1:
model_fname = '%s%s_epoch-%d_dev-acc-%.3f' %(args.model_path, args.log_fname.split('.')[0], epoch, dev_acc)
torch.save(bimpm.state_dict(), model_fname + 'bimpm.pt')
best_dev.append((epoch, dev_acc, model_fname))
logger.info('current best-dev:')
for t in best_dev:
logger.info('\t%d %.3f' %(t[0], t[1]))
logger.info('save model!')
else:
if dev_acc > best_dev[-1][1]:
model_fname = '%s%s_epoch-%d_dev-acc-%.3f' %(args.model_path, args.log_fname.split('.')[0], epoch, dev_acc)
torch.save(bimpm.state_dict(), model_fname + '_bimpm.pt')
best_dev.append((epoch, dev_acc, model_fname))
logger.info('current best-dev:')
for t in best_dev:
logger.info('\t%d %.3f' %(t[0], t[1]))
logger.info('save model!')
bimpm.train()
logger.info('training end!')
def main_work():
# TODO: 0003 시작 함수 main
# mode argument
args = argparse.ArgumentParser()
args.add_argument("--lr", type=int, default=0.001)
args.add_argument("--cuda", type=bool, default=True)
args.add_argument("--num_epochs", type=int, default=100)
args.add_argument("--print_iter", type=int, default=10)
args.add_argument("--model_name", type=str, default="model.pth")
args.add_argument("--prediction_file", type=str, default="prediction.txt")
args.add_argument("--batch", type=int, default=4)
args.add_argument("--mode", type=str, default="train")
config = args.parse_args()
base_lr = config.lr
cuda = config.cuda
num_epochs = config.num_epochs
print_iter = config.print_iter
model_name = config.model_name
prediction_file = config.prediction_file
batch = config.batch
mode = config.mode
# create model
model = Net()
if mode == 'test':
load_model(model_name, model)
pass
if cuda:
model = model.cuda()
pass
# -- train
if mode == 'train':
# define loss function
loss_fn = nn.L1Loss()
if cuda:
loss_fn = loss_fn.cuda()
pass
# set optimizer
optimizer = Adam(
[param for param in model.parameters() if param.requires_grad],
lr=base_lr,
weight_decay=1e-4)
scheduler = StepLR(optimizer, step_size=40, gamma=0.1)
# get data loader
#TODO 0004 데이터 로딩
log.info("Data loading now ......")
then = time.time()
log.info("Loading training data ....")
train_dataloader, _ = data_loader(root=DATASET_PATH,
phase='train',
batch_size=batch)
log.info("Done. Loading training data ....")
now = time.time()
log.info("// Training data loading. Duration %d sec(s)" % (now - then))
time_ = datetime.datetime.now()
num_batches = len(train_dataloader)
#check parameter of model
log.info(
"----------------- check parameter of model -------------------------------------------"
)
total_params = sum(p.numel() for p in model.parameters())
log.info("num of parameter : %s" % total_params)
trainable_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
log.info("num of trainable_ parameter : %s" % trainable_params)
log.info(
"------------------ // check parameter of model ------------------------------------------"
)
# train
for epoch in range(num_epochs):
model.train()
for iter_, (input_data,
output_data) in enumerate(train_dataloader):
# fetch train data
if cuda:
input_data = input_data.cuda()
output_data = output_data.cuda()
pass
# update weight
pred = model(input_data)
loss = loss_fn(pred, output_data)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (iter_ + 1) % print_iter == 0:
elapsed = datetime.datetime.now() - time_
expected = elapsed * (num_batches / print_iter)
_epoch = epoch + ((iter_ + 1) / num_batches)
print(
'[{:.3f}/{:d}] loss({}) elapsed {} expected per epoch {}'
.format(_epoch, num_epochs, loss.item(), elapsed,
expected))
time_ = datetime.datetime.now()
pass
pass
# scheduler update
scheduler.step()
# save model
save_model(str(epoch + 1), model, optimizer, scheduler)
elapsed = datetime.datetime.now() - time_
print('[epoch {}] elapsed: {} \n'.format(epoch + 1, elapsed))
pass # // train
# validate
if 1:
log.info("Validating ....")
then = time.time()
log.info("Loading validation data ....")
validate_dataloader, validate_label_file = data_loader(
root=DATASET_PATH, phase='validate', batch_size=batch)
now = time.time()
log.info("Done. Validation data loading. Duration %d sec(s) \n" %
(now - then))
validate(prediction_file, model, validate_dataloader,
validate_label_file, cuda)
log.info("Done. Validating. \n")
pass
# // -- train
elif mode == 'test':
model.eval()
# get data loader
test_dataloader, test_file = data_loader(root=DATASET_PATH,
phase='test',
batch_size=batch)
test(prediction_file, model, test_dataloader, cuda)
# submit test result
pass
# Save config in yaml file
if config.save_config:
import yaml
args_yaml = yaml.dump(vars(config))
config_yaml_path = os.path.join(output_path, 'config.yml')
with open(config_yaml_path, 'w') as fp:
yaml.dump(args_yaml, fp, default_flow_style=True)
''' ######################## < Step 2 > Create instances ######################## '''
# Build dataloader
print(
'\n[1 / 3]. Build data loader. Depending on your environment, this may take several minutes..'
)
dloader, dlen = data_loader(dataset_root=config.dataset_root,
resize=config.resize,
crop=config.crop,
batch_size=config.batch_size,
num_workers=config.num_workers,
type='encoder_train')
# Build models
print('\n[2 / 3]. Build models.. ')
encoder = nn.DataParallel(model.Resnet50(dim=config.feature_dim)).to(dev)
momentum_encoder = nn.DataParallel(
model.Resnet50(dim=config.feature_dim)).to(dev)
# loss history
loss_hist = []
# If resume, load ckpt and loss history
if config.resume:
file_name = 'ckpt_' + str(config.start_epoch) + '.pkl'
def mode_def(mode, device, PATH, class_names):
#Loading transformed dataset
image_datasets, dataloaders, dataset_sizes = data_loader()
if mode == 'train' or mode == 'test':
if config.loaded_model == None:
# Loading pretrained Resnet151 model
model = transfer_learn()
# If a loaded model exists
elif config.model_ft_mode == None:
model = config.loaded_model
if mode == 'train':
config.model_ft_mode, config.optimizer_ft_mode = train(
dataloaders, dataset_sizes, model, device)
print(
"Training completed.... Do you want to save the model now. Y to save N to skip"
)
choice = str(input("Enter your Choice: Y or N\t "))
if choice == 'Y' or 'y':
#Mapping dataset classes from our dataset to model
model.category_index = image_datasets['train'].class_to_idx
# Saving Model
save_model(PATH, config.model_ft_mode,
config.optimizer_ft_mode)
config.flag = 1
print("Model Saved")
elif mode == 'test':
if config.model_ft_mode:
test_model(config.model_ft_mode, dataloaders, device)
print("Testing is complete")
elif config.loaded_model:
test_model(config.loaded_model, dataloaders, device)
print("Testing is complete")
elif mode == 'save':
if config.flag == 1:
print("The model is already saved... Do you want to save again?")
choice = str(
input("Enter your choice Y to save again or N to skip"))
if choice == 'Y' or 'y':
model.category_index = image_datasets['train'].class_to_idx
save_model(PATH, config.model_ft_mode,
config.optimizer_ft_mode)
config.flag = 1
print("Model Saved")
elif path.exists(PATH):
print("A saved model already exists")
else:
model.category_index = image_datasets['train'].class_to_idx
save_model(PATH, config.model_ft_mode, config.optimizer_ft_mode)
config.flag = 1
print("Model Saved")
elif mode == 'load':
#Loading the saved model
config.loaded_model, config.category_index = load_checkpoint(
PATH, device)
if config.loaded_model == None and config.category_index == None:
print("Model not found, so cannot be loaded")
else:
#Fetching classification
config.classifications = {
values: key
for key, values in config.category_index.items()
} #swapping key and values for the classes
print("Loading model is complete")
elif mode == 'inference':
errorinf = {
"loaded_model": "None",
"class_names": "None",
"config.classifications": "None"
}
for error in errorinf.keys():
if errorinf[error] == None:
print(error + " Not Declared properly")
if ((config.loaded_model != None) and (class_names != None)
and (config.classifications != None)):
inference(config.loaded_model, class_names, config.classifications,
device)
else:
print("Wrong Choice")
def __init__(self,
root='dataset/edges2shoes',
result_dir='result',
weight_dir='weight',
load_weight=False,
batch_size=2,
test_size=20,
test_img_num=5,
img_size=128,
num_epoch=100,
save_every=1000,
lr=0.0002,
beta_1=0.5,
beta_2=0.999,
lambda_kl=0.01,
lambda_img=10,
lambda_z=0.5,
z_dim=8):
# Data type(Can use GPU or not?)
self.dtype = torch.cuda.FloatTensor
if torch.cuda.is_available() is False:
self.dtype = torch.FloatTensor
# Data loader for training
self.dloader, dlen = data_loader(root=root,
batch_size=batch_size,
shuffle=True,
img_size=img_size,
mode='train')
# Data loader for test
self.t_dloader, _ = data_loader(root=root,
batch_size=test_size,
shuffle=False,
img_size=img_size,
mode='val')
# Both of D_cVAE and D_cLR has two discriminators which have different output size((14x14) and (30x30)).
# Totally, we have for discriminators now.
self.D_cVAE = model.Discriminator().type(self.dtype)
self.D_cLR = model.Discriminator().type(self.dtype)
self.G = model.Generator(z_dim=z_dim).type(self.dtype)
self.E = model.Encoder(z_dim=z_dim).type(self.dtype)
# Optimizers
self.optim_D_cVAE = optim.Adam(self.D_cVAE.parameters(),
lr=lr,
betas=(beta_1, beta_2))
self.optim_D_cLR = optim.Adam(self.D_cLR.parameters(),
lr=lr,
betas=(beta_1, beta_2))
self.optim_G = optim.Adam(self.G.parameters(),
lr=lr,
betas=(beta_1, beta_2))
self.optim_E = optim.Adam(self.E.parameters(),
lr=lr,
betas=(beta_1, beta_2))
# fixed random_z for intermediate test
self.fixed_z = util.var(torch.randn(test_size, test_img_num, z_dim))
# Some hyper-parameters
self.z_dim = z_dim
self.lambda_kl = lambda_kl
self.lambda_img = lambda_img
self.lambda_z = lambda_z
# Extra things
self.result_dir = result_dir
self.weight_dir = weight_dir
self.load_weight = load_weight
self.test_img_num = test_img_num
self.img_size = img_size
self.start_epoch = 0
self.num_epoch = num_epoch
self.save_every = save_every
def train(args):
try:
os.makedirs(args.save_img_path)
except OSError:
pass
try:
os.makedirs(args.weight_path)
except OSError:
pass
lera.log_hyperparams(
{
"title": "hw2",
"batch_size": args.bs,
"epochs": args.epochs,
"g_lr": args.g_lr,
"d_lr": args.d_lr,
"z_size": args.z_size,
}
)
# dataset
dataloader = data_loader(
args.data_path, args.imgsize, args.bs, shuffle=True
)
# model
generator = Generator(args.bs, args.imgsize, z_dim=args.z_size).cuda()
discriminator = Discriminator(args.bs, args.imgsize).cuda()
if args.pre_epochs != 0:
generator.load_state_dict(
torch.load(
join(f"{args.weight_path}", f"generator_{args.pre_epochs}.pth")
)
)
discriminator.load_state_dict(
torch.load(
join(
f"{args.weight_path}",
f"discriminator_{args.pre_epochs}.pth",
)
)
)
# optimizer
g_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, generator.parameters()), lr=args.g_lr
)
d_optimizer = torch.optim.SGD(
filter(lambda p: p.requires_grad, discriminator.parameters()),
lr=args.d_lr,
)
# validate noise
fixed_noise = torch.randn(9, args.z_size)
fixed_noise = torch.tensor(fixed_noise).cuda()
# train
for epoch in range(args.pre_epochs, args.epochs):
for i, data in enumerate(dataloader):
discriminator.train()
generator.train()
# train discriminator
if i % 5 == 0:
d_optimizer.zero_grad()
real_img = torch.tensor(data[0]).cuda() * 2 - 1 # (-1, 1)
d__real, _, _ = discriminator(real_img)
z = torch.randn(args.bs, args.z_size)
z = torch.tensor(z).cuda()
fake_img, _, _ = generator(z)
d_fake, _, _ = discriminator(fake_img)
# hinge loss
d_loss_real = torch.nn.ReLU()(1.0 - d__real).mean()
d_loss_fake = torch.nn.ReLU()(1.0 + d_fake).mean()
d_loss = d_loss_real + d_loss_fake
d_loss.backward()
d_optimizer.step()
# train generator
g_optimizer.zero_grad()
z = torch.randn(args.bs, args.z_size)
z = torch.tensor(z).cuda()
fake_img, _, _ = generator(z)
g_fake, _, _ = discriminator(fake_img)
# hinge loss
g_loss = -g_fake.mean()
g_loss.backward()
g_optimizer.step()
if i % 100 == 0:
lera.log({"d_loss": d_loss.item(), "g_loss": g_loss.item()})
print(
"[epoch:%4d/%4d %3d/%3d] \t d_loss: %0.6f \t g_loss: %0.6f"
% (
epoch + 1,
args.epochs,
i,
len(dataloader),
d_loss.item(),
g_loss.item(),
)
)
if i % 300 == 0:
validate(
generator, i, epoch, args.save_img_path, fixed_noise
)
torch.save(
discriminator.state_dict(),
f"./{args.weight_path}/discriminator_{epoch+1}.pth",
)
torch.save(
generator.state_dict(),
f"./{args.weight_path}/generator_{epoch+1}.pth",
)
def train(self):
# build model
self.build_model()
loader = data_loader(self.data_root, self.batch_size, img_size=512)
loader = iter(cycle(loader))
mean_path_length = torch.tensor(0.0).to(dev)
average_path_length = torch.tensor(0.0).to(dev)
for iters in tqdm(range(self.max_iter + 1)):
real_img = next(loader)
real_img = real_img.to(dev)
# ===============================================================#
# 1. Train the discriminator #
# ===============================================================#
self.set_phase(mode="train")
self.reset_grad()
# Compute loss with real images.
d_real_out = self.D(real_img)
d_loss_real = F.softplus(-d_real_out).mean()
# Compute loss with face images.
z = torch.randn(2 * self.batch_size, self.z_dim).to(dev)
w = self.M(z)
dlatents_in = make_latents(w, self.batch_size,
len(self.channel_list))
fake_img, _ = self.G(dlatents_in)
d_fake_out = self.D(fake_img.detach())
d_loss_fake = F.softplus(d_fake_out).mean()
d_loss = d_loss_real + d_loss_fake
if iters % self.r1_iter == 0:
real_img.requires_grad = True
d_real_out = self.D(real_img)
r1_loss = self.r1_regularization(d_real_out, real_img)
r1_loss = self.r1_lambda / 2 * r1_loss * self.r1_iter
d_loss = d_loss + r1_loss
d_loss.backward()
self.d_optimizer.step()
# ===============================================================#
# 2. Train the Generator #
# ===============================================================#
if (iters + 1) % self.n_critic == 0:
self.reset_grad()
# Compute loss with fake images.
z = torch.randn(2 * self.batch_size, self.z_dim).to(dev)
w = self.M(z)
dlatents_in = make_latents(w, self.batch_size,
len(self.channel_list))
fake_img, _ = self.G(dlatents_in)
d_fake_out = self.D(fake_img)
g_loss = F.softplus(-d_fake_out).mean()
if iters % self.ppl_iter == 0:
path_loss, mean_path_length, path_length = self.path_length_regularization(
fake_img, dlatents_in, mean_path_length)
path_loss = path_loss * self.ppl_iter * self.ppl_lambda
g_loss = g_loss + path_loss
mean_path_length = mean_path_length.mean()
average_path_length += mean_path_length.mean()
# Backward and optimize.
g_loss.backward()
self.g_optimizer.step()
# ===============================================================#
# 3. Save parameters and images #
# ===============================================================#
# self.lr_update()
torch.cuda.synchronize()
self.set_phase(mode="test")
self.exponential_moving_average()
# Print total loss
if iters % self.print_loss_iter == 0:
print(
"Iter : [%d/%d], D_loss : [%.3f, %.3f, %.3f.], G_loss : %.3f, R1_reg : %.3f, "
"PPL_reg : %.3f, Path_length : %.3f" %
(iters, self.max_iter, d_loss.item(), d_loss_real.item(),
d_loss_fake.item(), g_loss.item(), r1_loss.item(),
path_loss.item(), mean_path_length.item()))
# Save generated images.
if iters % self.save_image_iter == 0:
fixed_w = self.M(self.fixed_z)
self.save_img(iters, fixed_w)
# Save the G and D parameters.
if iters % self.save_parameter_iter == 0:
self.save_model(iters)
# Save the logs on the tensorboard.
if iters % self.save_log_iter == 0:
self.writer.add_scalar('g_loss/g_loss', g_loss.item(), iters)
self.writer.add_scalar('d_loss/d_loss_total', d_loss.item(),
iters)
self.writer.add_scalar('d_loss/d_loss_real',
d_loss_real.item(), iters)
self.writer.add_scalar('d_loss/d_loss_fake',
d_loss_fake.item(), iters)
self.writer.add_scalar('reg/r1_regularization', r1_loss.item(),
iters)
self.writer.add_scalar('reg/ppl_regularization',
path_loss.item(), iters)
self.writer.add_scalar('length/path_length',
mean_path_length.item(), iters)
self.writer.add_scalar(
'length/avg_path_length',
average_path_length.item() / (iters // self.ppl_iter + 1),
iters)
# if train_loss/len(train_loader.dataset) < 100:
# break
torch.save(model.state_dict(), os.path.join(args.model_dir,'state_dict_final.pth'))
torch.save(model, os.path.join(args.model_dir,"model_final.pt"))
if __name__ == '__main__':
if not os.path.exists(args.model_dir):
os.makedirs(args.model_dir)
if not os.path.exists(args.cache_root):
os.makedirs(args.cache_root)
# load data
torchlist = data_loader(args, reshape_size)
train_loader = DataLoader(torchlist,batch_size=args.batch_size,shuffle=True,num_workers=4)
# set model
model = AE().to(device)
model.train()
if os.path.exists(args.checkpoint):
model.load_state_dict(torch.load(args.checkpoint))
print("==> Start training")
train(model, train_loader)
###
print("==> Computing all latent vectors using the trained model")
model.eval()
loss_fn = nn.CrossEntropyLoss()
# define loss function
if cuda:
loss_fn = loss_fn.cuda()
# set optimizer
optimizer = Adam(
[param for param in model.parameters() if param.requires_grad],
lr=base_lr, weight_decay=1e-4)
# scheduler = StepLR(optimizer, step_size=40, gamma=0.1)
# 원하는 에폭마다, 이전 학습률 대비 변경폭에 따라 학습률을 감소시켜주는 방식
# 스케쥴러 방식에 따라 적용방법이 다르다 : http://www.gisdeveloper.co.kr/?p=8443
scheduler = ReduceLROnPlateau(optimizer, factor=0.15, patience=10, verbose=1)
# get data loader
train_dataloader = data_loader(root=DATASET_PATH, phase='train', batch_size=batch)
time_ = datetime.datetime.now()
num_batches = len(train_dataloader)
#check parameter of model
print("------------------------------------------------------------")
total_params = sum(p.numel() for p in model.parameters())
print("num of parameter : ",total_params)
trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
print("num of trainable_ parameter :",trainable_params)
print("------------------------------------------------------------")
# train
mid_epochs = num_epochs//2
for epoch in range(num_epochs):
_lr = config.lr
_epochs = config.epochs
_batch_size = config.batch
_print_iter = config.print_iter
_model_type = config.model_type
_save_model_dir = config.save_model_dir
_seed = config.seed
_run_name = f"M-{_model_type}-step_size-{_step_size}-gamma-{_gamma}-BSZ-{_batch_size}-LR-{_lr}"
print(_run_name)
torch.manual_seed(_seed)
train_loader = data_loader(_root,
"train",
_context_max_len,
_context_word_len,
_query_max_len,
_query_word_len,
batch_size=_batch_size)
val_loader = data_loader(_root,
"validate",
_context_max_len,
_context_word_len,
_query_max_len,
_query_word_len,
batch_size=_batch_size)
model = get_model(_model_type)
loss_fn = nn.CrossEntropyLoss()
optimizer = Adam(
[param for param in model.parameters() if param.requires_grad],
num_channels = [args.n_hidden] * args.n_layers
in_channels = 1
kernel_size = args.kernel_size
dropout = args.dropout
out_features = 2 # Smart Wall features
sequence_length = 10 * 400 # 10*400 is the size of pictures in SmartWall dataset
# model
m = create_model(num_channels, in_channels, kernel_size, dropout,
out_features)
m = move_model_to_cuda(args, m)
if args.permute:
permute = torch.Tensor(np.random.permutation(sequence_length)).long()
permute = move_tensor_to_cuda(permute)
train_loader, test_loader = data_loader(args.batch_size)
optimizer = get_optimizer(args.optimizer, lr, m)
criterion = get_criterion()
# visualization
writer = SummaryWriter(args.tbx_log_dir)
for ep in range(1, args.epoch + 1):
train(args, m, train_loader, optimizer, criterion, in_channels,
sequence_length, ep, writer)
test(m, test_loader, writer, ep)
# learning rate annealing
if ep % 10 == 0:
lr /= 10
for g in optimizer.param_groups:
g['lr'] = lr
def main():
# mode argument
args = argparse.ArgumentParser()
args.add_argument(
"--letter",
type=str,
default=
" ,.()\'\"?!01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ가각간갇갈갉갊감갑값갓갔강갖갗같갚갛개객갠갤갬갭갯갰갱갸갹갼걀걋걍걔걘걜거걱건걷걸걺검겁것겄겅겆겉겊겋게겐겔겜겝겟겠겡겨격겪견겯결겸겹겻겼경곁계곈곌곕곗고곡곤곧골곪곬곯곰곱곳공곶과곽관괄괆괌괍괏광괘괜괠괩괬괭괴괵괸괼굄굅굇굉교굔굘굡굣구국군굳굴굵굶굻굼굽굿궁궂궈궉권궐궜궝궤궷귀귁귄귈귐귑귓규균귤그극근귿글긁금급긋긍긔기긱긴긷길긺김깁깃깅깆깊까깍깎깐깔깖깜깝깟깠깡깥깨깩깬깰깸깹깻깼깽꺄꺅꺌꺼꺽꺾껀껄껌껍껏껐껑께껙껜껨껫껭껴껸껼꼇꼈꼍꼐꼬꼭꼰꼲꼴꼼꼽꼿꽁꽂꽃꽈꽉꽐꽜꽝꽤꽥꽹꾀꾄꾈꾐꾑꾕꾜꾸꾹꾼꿀꿇꿈꿉꿋꿍꿎꿔꿜꿨꿩꿰꿱꿴꿸뀀뀁뀄뀌뀐뀔뀜뀝뀨끄끅끈끊끌끎끓끔끕끗끙끝끼끽낀낄낌낍낏낑나낙낚난낟날낡낢남납낫났낭낮낯낱낳내낵낸낼냄냅냇냈냉냐냑냔냘냠냥너넉넋넌널넒넓넘넙넛넜넝넣네넥넨넬넴넵넷넸넹녀녁년녈념녑녔녕녘녜녠노녹논놀놂놈놉놋농높놓놔놘놜놨뇌뇐뇔뇜뇝뇟뇨뇩뇬뇰뇹뇻뇽누눅눈눋눌눔눕눗눙눠눴눼뉘뉜뉠뉨뉩뉴뉵뉼늄늅늉느늑는늘늙늚늠늡늣능늦늪늬늰늴니닉닌닐닒님닙닛닝닢다닥닦단닫달닭닮닯닳담답닷닸당닺닻닿대댁댄댈댐댑댓댔댕댜더덕덖던덛덜덞덟덤덥덧덩덫덮데덱덴델뎀뎁뎃뎄뎅뎌뎐뎔뎠뎡뎨뎬도독돈돋돌돎돐돔돕돗동돛돝돠돤돨돼됐되된될됨됩됫됴두둑둔둘둠둡둣둥둬뒀뒈뒝뒤뒨뒬뒵뒷뒹듀듄듈듐듕드득든듣들듦듬듭듯등듸디딕딘딛딜딤딥딧딨딩딪따딱딴딸땀땁땃땄땅땋때땍땐땔땜땝땟땠땡떠떡떤떨떪떫떰떱떳떴떵떻떼떽뗀뗄뗌뗍뗏뗐뗑뗘뗬또똑똔똘똥똬똴뙈뙤뙨뚜뚝뚠뚤뚫뚬뚱뛔뛰뛴뛸뜀뜁뜅뜨뜩뜬뜯뜰뜸뜹뜻띄띈띌띔띕띠띤띨띰띱띳띵라락란랄람랍랏랐랑랒랖랗래랙랜랠램랩랫랬랭랴략랸럇량러럭런럴럼럽럿렀렁렇레렉렌렐렘렙렛렝려력련렬렴렵렷렸령례롄롑롓로록론롤롬롭롯롱롸롼뢍뢨뢰뢴뢸룀룁룃룅료룐룔룝룟룡루룩룬룰룸룹룻룽뤄뤘뤠뤼뤽륀륄륌륏륑류륙륜률륨륩륫륭르륵른를름릅릇릉릊릍릎리릭린릴림립릿링마막만많맏말맑맒맘맙맛망맞맡맣매맥맨맬맴맵맷맸맹맺먀먁먈먕머먹먼멀멂멈멉멋멍멎멓메멕멘멜멤멥멧멨멩며멱면멸몃몄명몇몌모목몫몬몰몲몸몹못몽뫄뫈뫘뫙뫼묀묄묍묏묑묘묜묠묩묫무묵묶문묻물묽묾뭄뭅뭇뭉뭍뭏뭐뭔뭘뭡뭣뭬뮈뮌뮐뮤뮨뮬뮴뮷므믄믈믐믓미믹민믿밀밂밈밉밋밌밍및밑바박밖밗반받발밝밞밟밤밥밧방밭배백밴밸뱀뱁뱃뱄뱅뱉뱌뱍뱐뱝버벅번벋벌벎범법벗벙벚베벡벤벧벨벰벱벳벴벵벼벽변별볍볏볐병볕볘볜보복볶본볼봄봅봇봉봐봔봤봬뵀뵈뵉뵌뵐뵘뵙뵤뵨부북분붇불붉붊붐붑붓붕붙붚붜붤붰붸뷔뷕뷘뷜뷩뷰뷴뷸븀븃븅브븍븐블븜븝븟비빅빈빌빎빔빕빗빙빚빛빠빡빤빨빪빰빱빳빴빵빻빼빽뺀뺄뺌뺍뺏뺐뺑뺘뺙뺨뻐뻑뻔뻗뻘뻠뻣뻤뻥뻬뼁뼈뼉뼘뼙뼛뼜뼝뽀뽁뽄뽈뽐뽑뽕뾔뾰뿅뿌뿍뿐뿔뿜뿟뿡쀼쁑쁘쁜쁠쁨쁩삐삑삔삘삠삡삣삥사삭삯산삳살삵삶삼삽삿샀상샅새색샌샐샘샙샛샜생샤샥샨샬샴샵샷샹섀섄섈섐섕서석섞섟선섣설섦섧섬섭섯섰성섶세섹센셀셈셉셋셌셍셔셕션셜셤셥셧셨셩셰셴셸솅소속솎손솔솖솜솝솟송솥솨솩솬솰솽쇄쇈쇌쇔쇗쇘쇠쇤쇨쇰쇱쇳쇼쇽숀숄숌숍숏숑수숙순숟술숨숩숫숭숯숱숲숴쉈쉐쉑쉔쉘쉠쉥쉬쉭쉰쉴쉼쉽쉿슁슈슉슐슘슛슝스슥슨슬슭슴습슷승시식신싣실싫심십싯싱싶싸싹싻싼쌀쌈쌉쌌쌍쌓쌔쌕쌘쌜쌤쌥쌨쌩썅써썩썬썰썲썸썹썼썽쎄쎈쎌쏀쏘쏙쏜쏟쏠쏢쏨쏩쏭쏴쏵쏸쐈쐐쐤쐬쐰쐴쐼쐽쑈쑤쑥쑨쑬쑴쑵쑹쒀쒔쒜쒸쒼쓩쓰쓱쓴쓸쓺쓿씀씁씌씐씔씜씨씩씬씰씸씹씻씽아악안앉않알앍앎앓암압앗았앙앝앞애액앤앨앰앱앳앴앵야약얀얄얇얌얍얏양얕얗얘얜얠얩어억언얹얻얼얽얾엄업없엇었엉엊엌엎에엑엔엘엠엡엣엥여역엮연열엶엷염엽엾엿였영옅옆옇예옌옐옘옙옛옜오옥온올옭옮옰옳옴옵옷옹옻와왁완왈왐왑왓왔왕왜왝왠왬왯왱외왹왼욀욈욉욋욍요욕욘욜욤욥욧용우욱운울욹욺움웁웃웅워웍원월웜웝웠웡웨웩웬웰웸웹웽위윅윈윌윔윕윗윙유육윤율윰윱윳융윷으윽은을읊음읍읏응읒읓읔읕읖읗의읩읜읠읨읫이익인일읽읾잃임입잇있잉잊잎자작잔잖잗잘잚잠잡잣잤장잦재잭잰잴잼잽잿쟀쟁쟈쟉쟌쟎쟐쟘쟝쟤쟨쟬저적전절젊점접젓정젖제젝젠젤젬젭젯젱져젼졀졈졉졌졍졔조족존졸졺좀좁좃종좆좇좋좌좍좔좝좟좡좨좼좽죄죈죌죔죕죗죙죠죡죤죵주죽준줄줅줆줌줍줏중줘줬줴쥐쥑쥔쥘쥠쥡쥣쥬쥰쥴쥼즈즉즌즐즘즙즛증지직진짇질짊짐집짓징짖짙짚짜짝짠짢짤짧짬짭짯짰짱째짹짼쨀쨈쨉쨋쨌쨍쨔쨘쨩쩌쩍쩐쩔쩜쩝쩟쩠쩡쩨쩽쪄쪘쪼쪽쫀쫄쫌쫍쫏쫑쫓쫘쫙쫠쫬쫴쬈쬐쬔쬘쬠쬡쭁쭈쭉쭌쭐쭘쭙쭝쭤쭸쭹쮜쮸쯔쯤쯧쯩찌찍찐찔찜찝찡찢찧차착찬찮찰참찹찻찼창찾채책챈챌챔챕챗챘챙챠챤챦챨챰챵처척천철첨첩첫첬청체첵첸첼쳄쳅쳇쳉쳐쳔쳤쳬쳰촁초촉촌촐촘촙촛총촤촨촬촹최쵠쵤쵬쵭쵯쵱쵸춈추축춘출춤춥춧충춰췄췌췐취췬췰췸췹췻췽츄츈츌츔츙츠측츤츨츰츱츳층치칙친칟칠칡침칩칫칭카칵칸칼캄캅캇캉캐캑캔캘캠캡캣캤캥캬캭컁커컥컨컫컬컴컵컷컸컹케켁켄켈켐켑켓켕켜켠켤켬켭켯켰켱켸코콕콘콜콤콥콧콩콰콱콴콸쾀쾅쾌쾡쾨쾰쿄쿠쿡쿤쿨쿰쿱쿳쿵쿼퀀퀄퀑퀘퀭퀴퀵퀸퀼큄큅큇큉큐큔큘큠크큭큰클큼큽킁키킥킨킬킴킵킷킹타탁탄탈탉탐탑탓탔탕태택탠탤탬탭탯탰탱탸턍터턱턴털턺텀텁텃텄텅테텍텐텔템텝텟텡텨텬텼톄톈토톡톤톨톰톱톳통톺톼퇀퇘퇴퇸툇툉툐투툭툰툴툼툽툿퉁퉈퉜퉤튀튁튄튈튐튑튕튜튠튤튬튱트특튼튿틀틂틈틉틋틔틘틜틤틥티틱틴틸팀팁팃팅파팍팎판팔팖팜팝팟팠팡팥패팩팬팰팸팹팻팼팽퍄퍅퍼퍽펀펄펌펍펏펐펑페펙펜펠펨펩펫펭펴편펼폄폅폈평폐폘폡폣포폭폰폴폼폽폿퐁퐈퐝푀푄표푠푤푭푯푸푹푼푿풀풂품풉풋풍풔풩퓌퓐퓔퓜퓟퓨퓬퓰퓸퓻퓽프픈플픔픕픗피픽핀필핌핍핏핑하학한할핥함합핫항해핵핸핼햄햅햇했행햐향허헉헌헐헒험헙헛헝헤헥헨헬헴헵헷헹혀혁현혈혐협혓혔형혜혠혤혭호혹혼홀홅홈홉홋홍홑화확환활홧황홰홱홴횃횅회획횐횔횝횟횡효횬횰횹횻후훅훈훌훑훔훗훙훠훤훨훰훵훼훽휀휄휑휘휙휜휠휨휩휫휭휴휵휸휼흄흇흉흐흑흔흖흗흘흙흠흡흣흥흩희흰흴흼흽힁히힉힌힐힘힙힛힝"
)
args.add_argument("--lr", type=float, default=0.0001)
args.add_argument("--cuda", type=bool, default=True)
args.add_argument("--num_epochs", type=int, default=50000)
args.add_argument("--model_name", type=str, default="55_10")
args.add_argument("--batch", type=int, default=2)
args.add_argument("--mode", type=str, default="test")
args.add_argument("--prediction_dir", type=str, default="prediction")
args.add_argument("--print_iter", type=int, default=10)
config = args.parse_args()
letter = config.letter
lr = config.lr
cuda = config.cuda
num_epochs = config.num_epochs
model_name = config.model_name
batch = config.batch
mode = config.mode
prediction_dir = config.prediction_dir
print_iter = config.print_iter
imgH = 32
imgW = 200
nclass = len(letter) + 1
nc = 1
new_model = model.CRNN(imgH, nc, nclass, 256)
new_model.apply(model.weights_init)
device = torch.device('cuda') if cuda else torch.device('cpu')
converter = dataloader.strLabelConverter(letter)
images = torch.FloatTensor(batch, 1, imgH, imgW)
texts = torch.IntTensor(batch * 1000)
lengths = torch.IntTensor(batch)
images = Variable(images)
texts = Variable(texts)
lengths = Variable(lengths)
#check parameter of model
print("------------------------------------------------------------")
total_params = sum(p.numel() for p in new_model.parameters())
print("num of parameter : ", total_params)
trainable_params = sum(p.numel() for p in new_model.parameters()
if p.requires_grad)
print("num of trainable_ parameter :", trainable_params)
print("------------------------------------------------------------")
if mode == 'train':
print('trian start')
train_loader = data_loader(DATASET_PATH,
batch,
imgH,
imgW,
phase='train')
val_loader = data_loader(DATASET_PATH, batch, imgH, imgW, phase='val')
params = [p for p in new_model.parameters() if p.requires_grad]
optimizer = optim.Adam(params, lr=lr, betas=(0.5, 0.999))
lr_scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=40,
gamma=0.1)
train(num_epochs, new_model, device, train_loader, val_loader, images,
texts, lengths, converter, optimizer, lr_scheduler,
prediction_dir, print_iter)
elif mode == 'test':
print('test start')
test_loader = data_loader(DATASET_PATH, 1, imgH, imgW, phase='test')
load_model(model_name, new_model)
test(new_model, device, test_loader, images, texts, lengths, converter,
prediction_dir)
# args = argparse.ArgumentParser()
# args.add_argument("--model_path", type=str)
# config = args.parse_args()
# _model_path = config.model_path
_model_path = 'saved_model/M-BiDAF-step_size-2-gamma-0.8-BSZ-2-LR-0.005.pth'
model_info, model_type = parse_model_path(_model_path)
state = torch.load(_model_path)
model = get_model(model_type)
model.load_state_dict(state["model"])
test_loader = data_loader(_root, "test",
_context_max_len, _context_word_len,
_query_max_len, _query_word_len, batch_size=_batch_size)
pred_answers = {}
for iter_, data in tqdm(enumerate(test_loader)):
cw_ids = data['cw_ids'].to(device)
cc_ids = data['cc_ids'].to(device)
qw_ids = data['qw_ids'].to(device)
qc_ids = data['qc_ids'].to(device)
mask = torch.zeros_like(cw_ids) != cw_ids
mask = mask.type(torch.float32).to(device)
logits = model(cw_ids, cc_ids, qw_ids, qc_ids)
start_logits, end_logits = logits
start_logits, end_logits = start_logits.squeeze(), end_logits.squeeze()
def main(network_name):
num_gpu = torch.cuda.device_count()
args = parser.parse_args()
args.network = network_name
print(args)
save_path = args.save_path + '_' + args.network
if args.need_save_checkpoint:
if os.path.exists(save_path):
os.system('rm -rf %s/*' % (save_path))
else:
os.system('mkdir %s' % (save_path))
if args.need_save_accuracy_log:
log_file = open('log_%s.txt' % (args.network), 'w')
train_dataset = data_loader(args, train=True)
test_dataset = data_loader(args, train=False)
train_loader = data.DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
test_loader = data.DataLoader(dataset=test_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
model = get_model(args).cuda()
if args.need_multi_gpus:
model = DataParallel(model, device_ids=range(num_gpu))
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
loss_func = nn.CrossEntropyLoss()
loss_func = loss_func.cuda()
train_total_step = len(train_loader)
test_total_step = len(test_loader)
for epoch in range(args.epoch):
loss_sum = 0
model.train()
train_class_correct = list(0. for i in range(args.class_num))
train_class_total = list(0. for i in range(args.class_num))
for i, (images, label) in enumerate(train_loader):
images = Variable(images.cuda())
label = Variable(label.cuda())
output = model(images)
prediction = torch.argmax(output, 1)
res = prediction == label
for label_idx in range(len(label)):
label_single = label[label_idx]
train_class_correct[label_single] += res[label_idx].item()
train_class_total[label_single] += 1
loss = loss_func(output, label)
loss_sum += loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
if i % args.display_train_interval == 0:
sys.stdout.write(
'[TRAIN net] Epoch [%d/%d], Step[%d/%d], loss:%.6f\r' %
(epoch + 1, args.epoch, i, train_total_step,
loss.cpu().data.numpy()))
print()
loss_mean = loss_sum / train_total_step
if args.need_save_checkpoint:
if args.need_multi_gpus:
torch.save(
model.module.state_dict(),
"{}/net{}_{}.pth".format(save_path, epoch,
loss_mean.cpu().data.numpy()))
else:
torch.save(
model.state_dict(),
"{}/net{}_{}.pth".format(save_path, epoch,
loss_mean.cpu().data.numpy()))
print('[LOSS] %f' % (loss_mean.cpu().data.numpy()))
print()
model.eval()
test_class_correct = list(0. for i in range(args.class_num))
test_class_total = list(0. for i in range(args.class_num))
for i, (images, label) in enumerate(test_loader):
images = Variable(images.cuda())
label = Variable(label.cuda())
output = model(images)
prediction = torch.argmax(output, 1)
res = prediction == label
for label_idx in range(len(label)):
label_single = label[label_idx]
test_class_correct[label_single] += res[label_idx].item()
test_class_total[label_single] += 1
if i % args.display_test_interval == 0:
sys.stdout.write('[TEST] Step[%d/%d]\r' % (i, test_total_step))
print()
acc_str = '[NET %s][EPOCH %3d/%3d] Train Accuracy: %f\tTest Accuracy: %f' % (
args.network, epoch + 1, args.epoch,
sum(train_class_correct) / sum(train_class_total),
sum(test_class_correct) / sum(test_class_total))
acc_str += '\t[TRAIN]\t'
for acc_idx in range(len(train_class_correct)):
try:
acc = train_class_correct[acc_idx] / train_class_total[acc_idx]
except:
acc = 0
finally:
acc_str += '\taccID:%d\tacc:%f\t' % (acc_idx + 1, acc)
acc_str += '\t[TEST]\t'
for acc_idx in range(len(test_class_correct)):
try:
acc = test_class_correct[acc_idx] / test_class_total[acc_idx]
except:
acc = 0
finally:
acc_str += '\taccID:%d\tacc:%f\t' % (acc_idx + 1, acc)
print(acc_str)
if args.need_save_accuracy_log:
log_file.write(acc_str + '\n')
log_file.flush()
if args.need_save_accuracy_log:
log_file.close()
