def input_index_data(num_docs=None, batch_size=8, dataset='f30k'):
from dataset import get_data_loader
captions = 'dataset_flickr30k.json' if dataset == 'f30k' else 'captions.txt'
data_loader = get_data_loader(root=os.path.join(cur_dir,
f'data/{dataset}/images'),
captions=os.path.join(
cur_dir, f'data/{dataset}/{captions}'),
split='test',
batch_size=batch_size,
dataset_type=dataset)
for i, (images, captions) in enumerate(data_loader):
for image in images:
with Document() as document:
document.buffer = image
document.modality = 'image'
document.mime_type = 'image/jpeg'
yield document
for caption in captions:
with Document() as document:
document.text = caption
document.modality = 'text'
document.mime_type = 'text/plain'
yield document
if num_docs and (i + 1) * batch_size >= num_docs:
break
def run(self):
train_dl, valid_dl = dataset.get_data_loader(self.bs)
train_dl = WrappedDataLoader(train_dl, preprocess)
valid_dl = WrappedDataLoader(valid_dl, preprocess)
model, opt = self.get_model()
loss_func = F.cross_entropy
self.fit(self.epochs, model, loss_func,
opt, train_dl, valid_dl)
def test1():
# visualize spectrograms in a batch
root_dir = "data/LibriSpeech/dev-clean"
loader = get_data_loader(root_dir, batch_size=4, shuffle=True)
for i, batch in enumerate(loader):
spec = batch["specgram"]
break
fig = plt.figure()
for i in range(4):
ax = fig.add_subplot(4, 1, i + 1)
ax.imshow(spec.log2()[i, :, :].detach().numpy(), cmap='gray')
plt.savefig(visual_path + "specgram.png")
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
image_dir = "data/"
json_path = image_dir + "annotations/captions_train2014.json"
root_dir = image_dir + "train2014"
dataset = CocoDataset(json_path=json_path,
root_dir=root_dir,
transform=transform)
data_loader = get_data_loader(dataset, batch_size=32)
# Build models
encoder = FeatureExtractor(args.embed_size).eval(
) # eval mode (batchnorm uses moving mean/variance)
decoder = CaptionGenerator(args.embed_size, args.hidden_size,
len(dataset.vocabulary), args.num_layers)
encoder = encoder.to(device)
decoder = decoder.to(device)
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
# Prepare an image
image = load_image(args.image, transform)
image_tensor = image.to(device)
# Generate an caption from the image
feature = encoder(image_tensor)
sampled_ids = decoder.sample(feature)
sampled_ids = sampled_ids[0].cpu().numpy(
) # (1, max_seq_length) -> (max_seq_length)
# Convert word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = data_loader.dataset.id_to_word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
# Print out the image and the generated caption
print(sentence)
image = Image.open(args.image)
plt.imshow(np.asarray(image))
def main(args):
set_seed(args.seed)
# load data
loaders = (get_data_loader(args=args,
data_path=args.data_path,
bert_path=args.bert_path,
batch_size=args.batch_size,
num_workers=args.num_workers,
split=split) for split in ['train', 'dev'])
trn_loader, dev_loader = loaders
# initialize model
model = MultimodalTransformer(
n_layers=args.n_layers,
n_heads=args.n_heads,
n_classes=args.n_classes,
only_audio=args.only_audio,
only_text=args.only_text,
d_audio_orig=args.n_mfcc,
d_text_orig=768, # BERT hidden size
d_model=args.d_model,
attn_dropout=args.attn_dropout,
relu_dropout=args.relu_dropout,
emb_dropout=args.emb_dropout,
res_dropout=args.res_dropout,
out_dropout=args.out_dropout,
attn_mask=args.attn_mask).to(args.device)
# warmup scheduling
args.total_steps = round(len(trn_loader) * args.epochs)
args.warmup_steps = round(args.total_steps * args.warmup_percent)
# optimizer & scheduler
optimizer, scheduler = get_optimizer_and_scheduler(args, model)
logging.info('training starts')
model.zero_grad()
args.global_step = 0
for epoch in tqdm(range(1, args.epochs + 1), desc='epochs'):
# training and evaluation steps
train(args, model, trn_loader, optimizer, scheduler)
loss, f1 = evaluate(model, dev_loader, args.device)
# save model
model_name = "epoch{}-loss{:.4f}-f1{:.4f}.bin".format(epoch, loss, f1)
model_path = os.path.join(args.save_path, model_name)
torch.save(model.state_dict(), model_path)
logging.info('training ended')
def test2():
# LAS forward pass
root_dir = "data/LibriSpeech/dev-clean"
loader = get_data_loader(root_dir, batch_size=4, shuffle=True)
for i, batch in enumerate(loader):
break
spec = batch["specgram"]
print(spec.size())
h_dim = 512
listen = Listen(n_mels=40, h_dim=h_dim)
spell = AttendAndSpell(h_dim=h_dim, voc_size=voc_size())
h = listen(spec)
print(h.size())
spell(h)
def main():
mp.set_start_method('spawn')
args = parse_args()
torch.set_grad_enabled(True)
data_loader_train, data_loader_dev, data_loader_test = get_data_loader(
args)
print("Data loaded.")
model = Net(args).to(args.device)
print("Model set.")
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
print("Optimizer set.")
train(args, data_loader_train, data_loader_dev, data_loader_test, model,
loss_fn, optimizer)
def evaluation_generator(num_docs=None,
batch_size=8,
dataset_type='f8k',
mode='text2image'):
from dataset import get_data_loader
captions = 'dataset_flickr30k.json' if dataset_type == 'f30k' else 'captions.txt'
data_loader = get_data_loader(
root=os.path.join(cur_dir, f'data/{dataset_type}/images'),
captions=os.path.join(cur_dir, f'data/{dataset_type}/{captions}'),
split='test',
batch_size=batch_size,
dataset_type=dataset_type)
for i, (images, captions) in enumerate(data_loader):
for image, caption in zip(images, captions):
hashed = hashlib.sha1(image).hexdigest()
if mode == 'text2image':
with Document() as document:
document.text = caption
document.modality = 'text'
document.mime_type = 'text/plain'
with Document() as gt:
match = Document()
match.tags['id'] = hashed
gt.matches.append(match)
yield document, gt
elif mode == 'image2text':
with Document() as document:
document.buffer = image
document.modality = 'image'
document.mime_type = 'image/jpeg'
document.convert_buffer_to_uri()
with Document() as gt:
match = Document()
match.tags['id'] = caption
gt.matches.append(match)
yield document, gt
else:
msg = f'Not supported mode {mode}.'
msg += 'expected `image2text` or `text2image`'
raise ValueError(msg)
if num_docs and (i + 1) * batch_size >= num_docs:
break
def test(model_filename, img_type, dir_name):
print("Testing {}".format(model_filename))
model = torch.load(os.path.join(dir_name, model_filename))
base_dirs = ['KKI', 'NeuroIMAGE', 'OHSU', 'Peking_1', 'Peking_2', 'Peking_3', 'Pittsburgh','WashU']
model.eval()
total_attempts = 0
total_correct = 0
res = collections.defaultdict(dict)
for base_dir in base_dirs:
d = dataset.get_data_loader(base_dir, img_type, batch_size=util.BATCH_SIZE, train=False)
dataset_correct = 0
dataset_attempt = 0
for idx, (img, label, err) in enumerate(d):
err_exists = sum(err) != 0
if err_exists:
print("Error in loading {}".format(base_dir))
continue
label = label.cuda()
img = Variable(img.float()).cuda().contiguous()
img = img.view(img.shape[0], 1, img.shape[1], img.shape[2], img.shape[3])
# print(img.shape)
with torch.no_grad():
out = model(img)
prediction = out.data.max(1)[1]
correct = float(prediction.eq(label.data).sum())
dataset_attempt += len(label)
dataset_correct += correct
if dataset_attempt != 0:
accuracy = (dataset_correct / float(dataset_attempt)) * 100.0
res[base_dir]['correct'] = dataset_correct
res[base_dir]['attempt'] = dataset_attempt
res[base_dir]['accuracy'] = accuracy
total_correct += dataset_correct
total_attempts += dataset_attempt
print('Test Accuracy {}: {}/{}={}'.format(base_dir, dataset_correct, dataset_attempt, accuracy))
res['summary']['correct'] = total_correct
res['summary']['attempt'] = total_attempts
res['summary']['accuracy'] = float(total_correct)/total_attempts * 100.0
util.save_data(res, os.path.join(dir_name, 'test_result.pckl'))
print('Total Accuracy: {}'.format(res['summary']['accuracy']))
return res
def main(args):
data_loader = get_data_loader(args=args,
data_path=args.data_path,
bert_path=args.bert_path,
num_workers=args.num_workers,
batch_size=args.batch_size,
split=args.split)
model = MultimodalTransformer(
n_layers=args.n_layers,
n_heads=args.n_heads,
n_classes=args.n_classes,
only_audio=args.only_audio,
only_text=args.only_text,
d_audio_orig=args.n_mfcc,
d_text_orig=768, # BERT hidden size
d_model=args.d_model,
attn_mask=args.attn_mask).to(args.device)
model.load_state_dict(torch.load(args.model_path))
# evaluation
logging.info('evaluation starts')
model.zero_grad()
evaluate(model, data_loader, args.device)
input_lengths = torch.full(size=(QUERY_NUM*CLASS_NUM,), fill_value=WINDOW_NUM, dtype=torch.long).to(device)
target_lengths = torch.full(size=(QUERY_NUM*CLASS_NUM,), fill_value=1, dtype=torch.long).to(device)
blank_prob = torch.full(size=(QUERY_NUM*CLASS_NUM, WINDOW_NUM, 1), fill_value=1, dtype=torch.float).to(device)
# Training Loop
train_ep = 0
while train_ep < args.train_ep:
# Load Data
if train_ep % args.load_frq == 0:
try:
if dataset_info["name"] != "finegym":
the_dataset = dataset.StandardDataset(dataset_info["folders"], "train", dataset_info["split"], CLASS_NUM, INST_NUM, FRAME_NUM, CLIP_NUM, WINDOW_NUM)
else:
the_dataset = dataset.FinegymDataset(dataset_info["folder"], dataset_info["finegym_info"], "train", dataset_info["split"], CLASS_NUM, INST_NUM, FRAME_NUM, CLIP_NUM, WINDOW_NUM)
dataloader = dataset.get_data_loader(the_dataset, num_per_class=SAMPLE_NUM+QUERY_NUM, num_workers=0)
data, data_labels = dataloader.__iter__().next() # [class*(support+query), window*clip, RGB, frame, H, W]
except Exception:
continue
data = data.view(-1, 3, FRAME_NUM, 128, 128)
print("Train_Ep[{}] Current_Accuracy = {}".format(train_ep, max_accuracy), end="\t")
# Generate support & query split
query_index = []
support_index = []
for i in range(CLASS_NUM):
start = (SAMPLE_NUM+QUERY_NUM) * i
end = (SAMPLE_NUM+QUERY_NUM) * (i+1)
index = list(range(start, end))
q = random.sample(index, QUERY_NUM)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--train-set',
type=str,
dest="train_set",
default="/store/slowmoyang/TopTagging/toptagging-training.root")
parser.add_argument(
'--valid-set',
type=str,
dest="valid_set",
default="/store/slowmoyang/TopTagging/toptagging-validation.root")
parser.add_argument(
'--test-set',
dest="test_set",
default="/store/slowmoyang/TopTagging/toptagging-test.root",
type=str)
parser.add_argument('--batch-size',
dest="batch_size",
default=128,
type=int,
help='batch size')
parser.add_argument('--test-batch-size',
type=int,
default=2048,
dest="test_batch_size",
help='batch size for test and validation')
parser.add_argument('--epoch',
dest="num_epochs",
default=2,
type=int,
help='number of epochs to train for')
parser.add_argument('--lr',
default=0.005,
type=float,
help='learning rate, default=0.005')
parser.add_argument("--logdir",
dest="log_dir",
default="./logs/untitled",
type=str,
help="the path to direactory")
parser.add_argument("--verbose", default=True, type=bool)
args = parser.parse_args()
#####################################
#
######################################
log_dir = Directory(args.log_dir, create=True)
log_dir.mkdir("state_dict")
log_dir.mkdir("validation")
log_dir.mkdir("roc_curve")
##################################################
# Logger
################################################
logger = logging.getLogger("TopTagging")
logger.setLevel(logging.INFO)
format_str = '[%(asctime)s] %(message)s'
date_format = '%Y-%m-%d %H:%M:%S'
formatter = logging.Formatter(format_str, date_format)
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
log_file_path = log_dir.concat("log.txt")
file_handler = logging.FileHandler(log_file_path)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
###############################
#
####################################
device = torch.device("cuda:0")
######################################
# Data Loader
########################################
train_loader = get_data_loader(path=args.train_set,
batch_size=args.batch_size)
valid_loader = get_data_loader(path=args.valid_set,
batch_size=args.test_batch_size)
test_loader = get_data_loader(path=args.test_set,
batch_size=args.test_batch_size)
#####################
# Model
######################
model = Classifier()
model.apply(init_weights)
model.cuda(device)
if args.verbose:
logger.info(model)
##################################
# Objective, optimizer,
##################################
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
################################
# Callbacks
################################
scheduler = ReduceLROnPlateau(optimizer, 'min', verbose=args.verbose)
########################################
# NOTE
#######################################
for epoch in xrange(1, args.num_epochs + 1):
logger.info("Epoch: [{:d}/{:d}]".format(epoch, args.num_epochs))
train(model=model,
data_loader=train_loader,
optimizer=optimizer,
criterion=criterion,
device=device,
logger=logger)
results = validate(model, valid_loader, device, logger)
# Callbacks
scheduler.step(results["loss"])
save_model(model, log_dir, epoch, results)
good_states = find_good_state(log_dir.state_dict.path)
for each in good_states:
model.load_state_dict(torch.load(each))
# evaluate(model, test_loader, log_dir)
logger.info("END")
default='shufflenetv2x0.5',
help='shufflenetv2x0.5 supported')
parser.add_argument('--load-model',
type=str,
default='None',
help='the path of the model to be loaded')
parser.add_argument('--preprocess', type=str, default='randaffine')
args = parser.parse_args()
if __name__ == '__main__':
batch_size = args.batch_size
preprocess = args.preprocess
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
testloader = get_data_loader(load_set='val',
preprocess=preprocess,
batch_size=batch_size)
model = get_model(model_name=args.model,
load_model=args.load_model,
batch_size=batch_size,
device=device)
best_acc = 0
t = datetime.datetime.now().strftime("%Y%m%d_%H%M")
model_name = args.model + '_' + t + '.pth'
print('model_name: ', model_name)
print('Number of parameters: {}'.format(
sum([p.numel() for p in model.parameters()])))
print('testing')
# test
def main(num_epochs=10, embedding_dim=256, data_dir="data/"):
""" Function to train the model.
Args:
num_epochs: int
Number of full dataset iterations to train the model.
embedding_dim: int
Output of the CNN model and input of the LSTM embedding size.
data_dir: str
Path to the folder of the data.
"""
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(f"WORKING WITH: {device}")
# Define the paths for train and validation
train_json_path = data_dir + "annotations/captions_train2014.json"
train_root_dir = data_dir + "train2014"
valid_json_path = data_dir + "annotations/captions_val2014.json"
valid_root_dir = data_dir + "val2014"
transform = transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
train_dataset = CocoDataset(json_path=train_json_path,
root_dir=train_root_dir,
transform=transform)
train_coco_dataset = get_data_loader(train_dataset, batch_size=128)
valid_dataset = CocoDataset(json_path=valid_json_path,
root_dir=valid_root_dir,
transform=transform)
valid_coco_dataset = get_data_loader(valid_dataset, batch_size=1)
encoder = FeatureExtractor(embedding_dim).to(device)
decoder = CaptionGenerator(embedding_dim, 512,
len(train_dataset.vocabulary), 1).to(device)
criterion = nn.CrossEntropyLoss()
# params = list(decoder.parameters()) + list(encoder.linear.parameters()) + list(encoder.bn.parameters())
params = list(decoder.parameters()) + list(
encoder.linear.parameters()) + list(encoder.bn.parameters())
optimizer = optim.Adam(params, lr=0.01)
print(f"TRAIN DATASET: {len(train_coco_dataset)}")
print(f"VALID DATASET: {len(valid_coco_dataset)}")
total_step = len(train_coco_dataset)
for epoch in range(num_epochs):
encoder.train()
decoder.train()
train_loss = 0.0
valid_loss = 0.0
for i, (images, captions,
descriptions) in enumerate(train_coco_dataset):
# targets = pack_padded_sequence(caption, 0, batch_first=True)[0]
images = images.to(device)
captions = captions.to(device)
# targets = pack_padded_sequence(captions, lengths, batch_first=True)[0]
features = encoder(images)
outputs = decoder(features, captions)
loss = criterion(outputs.view(-1, len(train_dataset.vocabulary)),
captions.view(-1))
# bleu = calculate_bleu(decoder, features, descriptions, coco_dataset)
# print(bleu)
encoder.zero_grad()
decoder.zero_grad()
loss.backward()
optimizer.step()
# Print log info
train_loss += loss.item()
'''
if i % 10 == 0:
print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}, Perplexity: {:5.4f}'
.format(epoch, num_epochs, i, total_step, loss.item(), np.exp(loss.item())))
'''
# Save the model checkpoints
if (i + 1) % 1000 == 0:
torch.save(
decoder.state_dict(),
os.path.join("models",
'decoder-{}-{}.ckpt'.format(epoch + 1,
i + 1)))
torch.save(
encoder.state_dict(),
os.path.join("models",
'encoder-{}-{}.ckpt'.format(epoch + 1,
i + 1)))
encoder.eval()
decoder.eval()
bleu = 0.0
for i, (images, captions,
descriptions) in enumerate(valid_coco_dataset):
if (i > 80000):
break
images = images.to(device)
captions = captions.to(device)
features = encoder(images)
outputs = decoder(features, captions)
loss = criterion(outputs.view(-1, len(train_dataset.vocabulary)),
captions.view(-1))
valid_loss += loss.item()
bleu += calculate_bleu(decoder, features, descriptions,
train_coco_dataset)
# print(f"BLEU: {bleu / 10000}")
print(
"Epoch: {}, Train Loss: {:.4f}, Valid Loss: {:.4f}, BLEU: {:.4f}".
format(epoch, train_loss / len(train_coco_dataset),
valid_loss / 80000, bleu / 80000))
def run(optimizer_type, img_type, lr = 0.005, large=False):
if large:
_model = model.StructuralModel3DFullImageLarge()
_model_dir = 'model_large'
else:
_model = model.StructuralModel3DFullImage()
_model_dir = 'model'
_model = _model.cuda()
if optimizer_type == 'adam':
optimizer = optim.Adam(_model.parameters(), lr=lr)
save_dir = '{}/{}/{}/{}'.format(_model_dir, str(img_type), optimizer_type, lr)
util.mkdir(save_dir)
base_dirs = ['KKI', 'NeuroIMAGE', 'OHSU', 'Peking_1', 'Peking_2', 'Peking_3', 'Pittsburgh','WashU']
criterion = nn.CrossEntropyLoss()
count = 0
begin_time = time.time()
train_acc_dict = {}
for epoch in range(0, num_of_epochs):
###### TRAIN
train_accu = []
train_loss = []
_model.train()
start_time = time.time()
print("Epoch: {}".format(epoch))
random.shuffle(base_dirs)
for base_dir in base_dirs:
print('Using {}'.format(base_dir))
d = dataset.get_data_loader(base_dir, img_type, batch_size=util.BATCH_SIZE, train=True)
if optimizer_type == 'adam':
for group in optimizer.param_groups:
for p in group['params']:
state = optimizer.state[p]
if ('step' in state and state['step'] >= 1024):
state['step'] = 1000
num_correct= 0
num_total = 0
for idx, (img, label, err) in enumerate(d):
if(sum(err) != 0):
print("Error occured")
continue
# ipdb.set_trace()
label = label.cuda()
img = Variable(img.float()).cuda().contiguous()
img = img.view(img.shape[0], 1, img.shape[1], img.shape[2], img.shape[3])
# print(img.shape)
out = _model(img)
# print(out)
optimizer.zero_grad()
loss = criterion(out, label)
loss.backward()
optimizer.step()
prediction = out.data.max(1)[1]
correct = float(prediction.eq(label.data).sum())
num_correct += correct
num_total += len(label)
accuracy = (correct / float(len(label))) * 100.0
train_loss.append(loss.item())
train_accu.append(accuracy)
# if count % 100 == 0:
print("Epoch {} iter {}: Training accuracy = {}/{}={} Loss = [{}]".format(epoch, idx, correct, len(label), accuracy, loss.item()))
count+= 1
if num_total != 0:
base_train_acc = float(num_correct)/num_total * 100.0
print("Epoch {} {}: Training accuracy = {}/{}={}".format(epoch, base_dir, num_correct, num_total, base_train_acc))
train_acc_dict[base_dir] = base_train_acc
torch.save(_model, './{}/{}'.format(save_dir, '{}.ckpt'.format(epoch)))
train_accu = np.asarray(train_accu)
train_loss = np.asarray(train_loss)
np.save(file=os.path.join(save_dir, 'loss.npy'), arr=train_loss)
np.save(file=os.path.join(save_dir, 'train_acc.npy'), arr=train_accu)
print("Ran for {}s".format(time.time()-begin_time))
util.save_data(filename=os.path.join(save_dir, 'train_dict.pckl'), d=train_acc_dict)
help='shufflenetv2x0.5 supported')
parser.add_argument('--load-model',
type=str,
default='None',
help='the path of the model to be loaded')
parser.add_argument('--preprocess', type=str, default='randaffine')
args = parser.parse_args()
if __name__ == '__main__':
batch_size = args.batch_size
preprocess = args.preprocess
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
trainloader = get_data_loader(load_set='train',
preprocess=preprocess,
batch_size=batch_size)
valloader = get_data_loader(load_set='val',
preprocess=preprocess,
batch_size=batch_size)
model = get_model(model_name=args.model,
load_model=args.load_model,
batch_size=batch_size,
device=device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9)
best_acc = 0
t = datetime.datetime.now().strftime("%Y%m%d_%H%M")
model_name = args.model + '_' + t + '.pth'
def train(dataset_path: str):
device = torch.device(config["train"]["device"])
print("Device: {}".format(device))
# device = torch.device("cpu") # gpu not enough memory :(
model = OpenAIGPTDoubleHeadsModel.from_pretrained("openai-gpt")
model.to(device)
tokenizer = OpenAIGPTTokenizer.from_pretrained("openai-gpt")
orig_num_tokens = len(tokenizer.encoder)
num_added_tokens = tokenizer.add_special_tokens(SPECIAL_TOKENS)
model.resize_token_embeddings(new_num_tokens=orig_num_tokens +
num_added_tokens)
# dataloader = get_data_loader(dataset_path, tokenizer, batch_size=4, shuffle=False, num_workers=1)
full_dataset = get_dataset(dataset_path, tokenizer)
assert len(full_dataset) > 0
train_size = int(
len(full_dataset) * config["train"]["train_dataset_proportion"] + 1)
test_size = len(full_dataset) - train_size
print("Full dataset has {} dialogs. Splitting into train: {} and test: {}".
format(len(full_dataset), train_size, test_size))
train_dataset, test_dataset = random_split(
full_dataset, [train_size, test_size],
torch.Generator().manual_seed(42))
print(len(train_dataset), len(test_dataset))
train_loader = get_data_loader(train_dataset, tokenizer,
config["train"]["batch_size"], True, 0)
test_loader = get_data_loader(test_dataset, tokenizer, 1, False, 0)
lr = config["train"]["learning_rate"]
print("lr: {}".format(lr))
optimizer = AdamW(model.parameters(), lr=lr)
# init logging
start_time = datetime.datetime.now()
save_path = os.path.join(
os.path.dirname(__file__),
"log/log-{}.txt".format(start_time.strftime("%y-%m-%d-%H-%M-%S")))
print(os.path.dirname(__file__), save_path)
f = open(save_path, "w+")
f.close()
epochs = config["train"]["num_epochs"]
eval_every = config["train"]["evaluate_interval_iters"]
num_tests = config["train"]["num_tests"]
last_model_save = datetime.datetime.now()
iteration = 0
for epoch in range(epochs):
print("Starting epoch {}/{}".format(epoch, epochs))
for batch in train_loader:
if iteration % eval_every == 0:
results = evaluate_model(model, test_loader, device, num_tests)
add_log(
save_path,
"test,{0},{1},{2[mc_correct]},{2[num_tests]},{2[lm_correct]},{2[lm_tested]}\n"
.format(iteration, epoch, results))
model.train()
input_ids = batch["input_ids"].to(device)
mc_token_ids = batch["mc_token_ids"].to(device)
token_type_ids = batch["token_type_ids"].to(device)
lm_labels = batch["lm_labels"].to(device)
mc_labels = batch["correct"].to(device)
try:
model_output = model(input_ids,
token_type_ids=token_type_ids,
mc_token_ids=mc_token_ids,
mc_labels=mc_labels,
labels=lm_labels)
except Exception as e:
print(input_ids,
token_type_ids,
mc_token_ids,
lm_labels,
mc_labels,
sep="\n")
raise e
# print("input_ids: {}\ntoken_type_ids: {}\nmc_token_ids: {}\nlm_labels: {}\nmc_labels: {}"
# .format(input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels))
# print(model_output.loss.item(), model_output.mc_loss.item())
lm_loss = model_output.loss
mc_loss = model_output.mc_loss
loss = lm_loss * config["train"]["lm_coeff"] + mc_loss * config[
"train"]["mc_coeff"]
add_log(
save_path,
"train,{},{},{},{},{}\n".format(iteration, epoch, loss,
lm_loss, mc_loss))
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(),
config["train"]["max_norm"])
optimizer.step()
optimizer.zero_grad()
# TODO: evaluation
if iteration % 50 == 0:
print(
"Time: {} Epoch: {}/{} Iteration: {}/{} Loss: {} ({} {})".
format(
datetime.datetime.now() - start_time, epoch, epochs,
iteration,
epochs *
(len(train_dataset) // config["train"]["batch_size"]),
loss.item(), lm_loss.item(), mc_loss.item()))
if datetime.datetime.now() - last_model_save > datetime.timedelta(
minutes=config["train"]["save_interval_mins"]):
print("Saving model...")
torch.save(
model.state_dict(),
os.path.join(os.path.dirname(__file__),
"checkpoints/model-{}-iter{}.pt").format(
start_time.strftime("%y-%m-%d-%H-%M-%S"),
iteration))
last_model_save = datetime.datetime.now()
iteration += 1
print("Saving model...")
torch.save(
model.state_dict(),
os.path.join(os.path.dirname(__file__),
"checkpoints/model-{}-iter{}.pt").format(
start_time.strftime("%y-%m-%d-%H-%M-%S"), iteration))
test_ep = 0
while test_ep < args.test_ep:
# Data Loading
try:
if dataset_info["name"] != "finegym":
the_dataset = dataset.StandardDataset(
dataset_info["folders"], "test", dataset_info["split"],
CLASS_NUM, INST_NUM, FRAME_NUM, CLIP_NUM, WINDOW_NUM)
else:
the_dataset = dataset.FinegymDataset(
dataset_info["folder"], dataset_info["finegym_info"],
"test", dataset_info["split"], CLASS_NUM, INST_NUM,
FRAME_NUM, CLIP_NUM, WINDOW_NUM)
dataloader = dataset.get_data_loader(the_dataset,
num_per_class=SAMPLE_NUM +
QUERY_NUM,
num_workers=0)
data, data_labels = dataloader.__iter__().next(
) # [class*(support+query), window*clip, RGB, frame, H, W]
except Exception:
continue
data = data.view(-1, 3, FRAME_NUM, 128, 128)
print("Test_Epi[{}]".format(test_ep), end="\t")
# Generate support & query split
query_index = []
support_index = []
for i in range(CLASS_NUM):
start = (SAMPLE_NUM + QUERY_NUM) * i
end = (SAMPLE_NUM + QUERY_NUM) * (i + 1)
def train_model(args):
bn_training = tf.placeholder(dtype=tf.bool, shape=[], name='bn_training')
x = tf.placeholder(dtype=tf.float32, shape=[None, 3, 32, 32], name='x')
y = tf.placeholder(dtype=tf.float32,
shape=[None, args.num_classes],
name='y')
weight_decay = args.weight_decay
with tf.name_scope('resnet18'):
pred = resnet18(x, args.num_classes, bn_training)
with tf.variable_scope('train'):
global_step = tf.Variable(0, name='global_step', trainable=False)
learning_rate = tf.placeholder(dtype=tf.float32,
shape=[],
name='learning_rate')
cross_entropy_loss = tf.reduce_mean(
slim.nn.softmax_cross_entropy_with_logits_v2(labels=y,
logits=pred))
weight_decay_loss = weight_decay * tf.add_n([
tf.nn.l2_loss(tf.cast(v, tf.float32))
for v in tf.trainable_variables()
if weight_decay_param_filter(v.name)
])
loss = cross_entropy_loss + weight_decay_loss
train_optimizer = slim.train.MomentumOptimizer(
learning_rate=learning_rate, momentum=0.9)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = train_optimizer.minimize(loss, global_step)
tf.summary.scalar('learning_rate', learning_rate)
tf.summary.scalar('loss', loss)
with tf.variable_scope('test'):
correct = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
tf.summary.scalar('accuracy', accuracy)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
train_loader, test_loader = get_data_loader(args.data_dir,
batch_size=args.batch_size,
num_workers=args.num_workers)
base_lr = args.lr
best_acc = 0.0
merged = tf.summary.merge_all()
i = 0
accuracy_list = []
with tf.Session(config=config) as sess:
summary_writer = tf.summary.FileWriter(args.log_dir, sess.graph)
sess.run(tf.global_variables_initializer())
pbar = tqdm.tqdm(range(args.epoch))
for epoch in pbar:
lr = get_learning_rate(base_lr, epoch)
_losses = []
for idx, batch_data in enumerate(train_loader):
i += 1
images, labels = batch_data
_loss, _, rs = sess.run([loss, train_op, merged],
feed_dict={
x: images,
y: labels,
learning_rate: lr,
bn_training: True
})
_losses.append(_loss)
summary_writer.add_summary(rs, i)
_test_accuracy = []
for _, batch_data in enumerate(test_loader):
images, labels = batch_data
acc = sess.run(accuracy,
feed_dict={
x: images,
y: labels,
bn_training: True
})
_test_accuracy.append(acc)
cur_acc = 100 * np.mean(_test_accuracy)
accuracy_list.append(cur_acc)
best_acc = max(cur_acc, best_acc)
pbar.set_description(
"e:{} loss:{:.3f} acc:{:.2f}% best:{:.2f}% lr:{:.5f}".format(
epoch, np.mean(_losses), cur_acc, best_acc, lr))
with open(os.path.join(args.log_dir, 'acc.pkl'), 'wb') as f:
pickle.dump(accuracy_list, f)
def main(arg):
##################################
for key in arg:
parse[key] = arg[key]
global args
args = SimpleNamespace(**parse)
'''
print('seed:{:}'.format(args.seed))
print('dataset:{:}'.format(args.dataset))
print('hidden_layers:{:}'.format(args.hidden_layers))
print('first_neurons:{:}'.format(args.first_neurons))
print('cross_link:{:}'.format(args.cross_link))
print('fully_cross:{:}'.format(args.fully_cross))
print()
exit(0)
'''
##################################
seed = util.prepare(args)
if not cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
cuda.manual_seed(seed)
cuda.set_device(args.gpu)
cudnn.benchmark = False
cudnn.deterministic = True
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
logging.info('hidden_layers:{:}'.format(args.hidden_layers))
logging.info('first_neurons:{:}'.format(args.first_neurons))
logging.info('change:{:}'.format(args.change))
logging.info('activate_func:{:}'.format(args.activate_func))
logging.info('opt:{:}'.format(args.opt))
logging.info('cross_link:{:}'.format(args.cross_link))
logging.info('fully_cross:{:}'.format(args.fully_cross))
model = Network(args)
model = model.cuda()
logging.info("param size = %fMB", util.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
# scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, args.epochs)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=30,
gamma=0.7)
train_data, valid_data = dataset.get_dataset(args.data, args.dataset)
train_queue, valid_queue = dataset.get_data_loader(train_data, valid_data,
2)
early_stop = util.EarlyStop(patience=10,
delta=0.0001,
save_path=args.save + '/best.pt')
for epoch in range(args.epochs):
logging.info('epoch %d lr %.6f', epoch, scheduler.get_lr()[0])
epoch_str = '[{:03d}/{:03d}]'.format(epoch, args.epochs)
train_acc, train_obj = train(train_queue, model, criterion, optimizer,
epoch_str)
logging.info('train_acc %.2f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion, epoch_str)
logging.info('valid_acc %.2f', valid_acc)
if early_stop.check(train_obj, valid_acc, model):
logging.info('Early stopping at {:}'.format(epoch))
break
scheduler.step()
decay_lr = args.decay_lr
decay_equilibrium = args.decay_equilibrium
slurm = args.slurm
writer = SummaryWriter(comment="_CELEBA_ALL")
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
net = VaeGan(z_size=z_size, recon_level=recon_level).to(device)
net = nn.DataParallel(net)
# DATASET
# dataloader = torch.utils.data.DataLoader(CELEBA(train_folder), batch_size=64,
# shuffle=True, num_workers=4)
# dataloader_test = torch.utils.data.DataLoader(CELEBA(test_folder), batch_size=100,
# shuffle=False, num_workers=1)
dataloader, dataloader_test = get_data_loader(img_size, train_folder,
test_folder, n_batch_train,
n_batch_test,
num_workers_train,
num_workers_test)
#margin and equilibirum
margin = 0.35
equilibrium = 0.68
#mse_lambda = 1.0
# OPTIM-LOSS
# an optimizer for each of the sub-networks, so we can selectively backprop
optimizer_encoder = RMSprop(params=net.module.encoder.parameters(),
lr=lr,
alpha=0.9,
eps=1e-8,
weight_decay=0,
momentum=0,
centered=False)
parser.add_argument('--resume_training', type=str2bool, default=False)
parser.add_argument('--to_train', type=str2bool, default=True)
parser.add_argument('--conditional', type=str2bool, default=False)
opt = parser.parse_args()
print(opt)
manual_seed = random.randint(1, 10000)
random.seed(manual_seed)
T.manual_seed(manual_seed)
if T.cuda.is_available():
T.cuda.manual_seed_all(manual_seed)
train_loader = get_data_loader(opt, True)
test_loader = get_data_loader(opt, False)
E = get_cuda(Encoder(opt))
G = get_cuda(Generator(opt)).apply(weights_init)
D = get_cuda(Discriminator(opt)).apply(weights_init)
device_ids = range(T.cuda.device_count())
E = nn.DataParallel(E, device_ids)
G = nn.DataParallel(G, device_ids)
D = nn.DataParallel(D, device_ids)
E_trainer = T.optim.Adam(E.parameters(), lr=opt.lr_e)
G_trainer = T.optim.Adam(G.parameters(), lr=opt.lr_g, betas=(0.5, 0.999))
D_trainer = T.optim.Adam(D.parameters(), lr=opt.lr_d, betas=(0.5, 0.999))
train_epoch = 80
resume_training = False
z_size = 512
def load_model_from_checkpoint():
global ae_model
checkpoint = torch.load(save_path)
vae.load_state_dict(checkpoint['ae_model'])
return checkpoint['epoch']
# load dataSet
train_loader = get_data_loader(img_size=im_size,
img_path=img_path,
shuffle=True,
batch_size=batch_size,
num_workers=8)
loss_function = pytorch_ssim.SSIM()
def main():
#build Neural Network
ae_model = ResNet_autoencoder(Bottleneck, DeconvBottleneck, [3, 4, 6, 3],
3)
ae_model.apply(weights_init_kaiming)
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
ae_model.cuda()
ae_model.train()
def main():
args = parser.parse_args()
# define args more
args.train_meta = './meta/CARS196/train.txt'
args.test_meta = './meta/CARS196/test.txt'
args.lr_decay_epochs = [
int(epoch) for epoch in args.lr_decay_epochs.split(',')
]
args.recallk = [int(k) for k in args.recallk.split(',')]
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu_idx)
args.ctx = [mx.gpu(0)]
print(args)
# Set random seed
mx.random.seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
# Load image transform
train_transform, test_transform = T.get_transform(
image_size=args.image_size)
# Load data loader
train_loader, test_loader = D.get_data_loader(
args.data_dir, args.train_meta, args.test_meta, train_transform,
test_transform, args.batch_size, args.num_instances, args.num_workers)
# Load model
model = Model(args.embed_dim, args.ctx)
model.hybridize()
# Load loss
loss = HPHNTripletLoss(margin=args.margin,
soft_margin=False,
num_instances=args.num_instances,
n_inner_pts=args.n_inner_pts,
l2_norm=args.ee_l2norm)
# Load logger and saver
summary_writer = SummaryWriter(
os.path.join(args.save_dir, 'tensorboard_log'))
print("steps in epoch:", args.lr_decay_epochs)
steps = list(map(lambda x: x * len(train_loader), args.lr_decay_epochs))
print("steps in iter:", steps)
lr_schedule = mx.lr_scheduler.MultiFactorScheduler(
step=steps, factor=args.lr_decay_factor)
lr_schedule.base_lr = args.lr
# Load optimizer for training
optimizer = mx.gluon.Trainer(model.collect_params(),
'adam', {
'learning_rate': args.lr,
'wd': args.wd
},
kvstore=args.kvstore)
# Load trainer & evaluator
trainer = Trainer(model,
loss,
optimizer,
train_loader,
summary_writer,
args.ctx,
summary_step=args.summary_step,
lr_schedule=lr_schedule)
evaluator = Evaluator(model, test_loader, args.ctx)
best_metrics = [0.0] # all query
global_step = args.start_epoch * len(train_loader)
# Enter to training loop
print("base lr mult:", args.base_lr_mult)
for epoch in range(args.start_epoch, args.epochs):
model.backbone.collect_params().setattr('lr_mult', args.base_lr_mult)
trainer.train(epoch)
global_step = (epoch + 1) * len(train_loader)
if (epoch + 1) % args.eval_epoch_term == 0:
old_best_metric = best_metrics[0]
# evaluate_and_log(summary_writer, evaluator, ranks, step, epoch, best_metrics)
best_metrics = evaluate_and_log(summary_writer,
evaluator,
args.recallk,
global_step,
epoch + 1,
best_metrics=best_metrics)
if best_metrics[0] != old_best_metric:
save_path = os.path.join(
args.save_dir, 'model_epoch_%05d.params' % (epoch + 1))
model.save_parameters(save_path)
sys.stdout.flush()
return False
parser.add_argument('--resume_training', type=str2bool, default=False)
parser.add_argument('--to_train', type=str2bool, default=True)
opt = parser.parse_args()
print(opt)
manual_seed = random.randint(1, 10000)
random.seed(manual_seed)
T.manual_seed(manual_seed)
if T.cuda.is_available():
T.cuda.manual_seed_all(manual_seed)
train_loader = get_data_loader(opt)
E = get_cuda(Encoder(opt))
G = get_cuda(Generator(opt)).apply(weights_init)
D = get_cuda(Discriminator()).apply(weights_init)
device_ids = range(T.cuda.device_count())
E = nn.DataParallel(E, device_ids)
G = nn.DataParallel(G, device_ids)
D = nn.DataParallel(D, device_ids)
E_trainer = T.optim.Adam(E.parameters(), lr=opt.lr_e)
G_trainer = T.optim.Adam(G.parameters(), lr=opt.lr_g, betas=(0.5, 0.999))
D_trainer = T.optim.Adam(D.parameters(), lr=opt.lr_d, betas=(0.5, 0.999))
