elif args.dataset == 'mnist':
Container = MNISTContainer
elif args.dataset == 'stl10':
Container = STL10Container
else:
raise NotImplementedError
train_set = Container('train', args.batch_size, args.augmentations, args.rotation_range, args.scale_range,
args.translation_range, args.gaussian_noise_std, args.snp_noise_probability,
image_size=[64, 64], n_generated_images=args.n_generated_images,
generated_data_name_suffix=args.generated_data_name_suffix)
test_set = Container('test', args.batch_size, image_size=[64, 64])
logging.info('Constructing model...')
network = Classifier([64, 64, 3], [10])
ground_truth_placeholder = tf.placeholder(tf.int64, shape=[None])
base_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(labels=ground_truth_placeholder, logits=network.outputs)
)
weights = sum([layer.weights for layer in network.layers], [])
weight_decay_loss = args.weight_decay * tf.add_n([tf.nn.l2_loss(w) for w in weights])
total_loss = base_loss + weight_decay_loss
optimizer = tf.train.AdamOptimizer(args.learning_rate)
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
train_step = optimizer.minimize(total_loss)
import os
import faust
from models import EventModel, Classifier
KAFKA_BROKER_URL = os.environ.get('KAFKA_BROKER_URL')
COMMENTS_TOPIC = os.environ.get('COMMENTS_TOPIC')
app = faust.App('comment-classification', broker=KAFKA_BROKER_URL)
comments_topic = app.topic(COMMENTS_TOPIC, value_type=EventModel)
classifer = Classifier()
@app.agent(comments_topic)
async def classify_comments(events):
async for event in events:
pred = classifer.predict(event.comment)
result = {
'user_id': event.user_id,
'comment': event.comment,
'datetime':event.datetime,
'data': pred
}
print(result)
if __name__ == '__main__':
app.main()
print(opt)
cuda = True if torch.cuda.is_available() else False
os.environ["CUDA_VISIBLE_DEVICES"] = '1, 2'
lambda_adv = 1
lambda_task = 0.1
# Calculate output of image discriminator (PatchGAN)
patch = int(opt.img_size / (2**4))
patch = (1, patch, patch)
generator = Generator(opt.latent_dim, opt.channels, opt.img_size,
opt.n_residual_blocks)
discriminator = Discriminator(opt.channels)
classifier = Classifier(opt.channels, opt.img_size, opt.n_classes)
generator = nn.DataParallel(generator)
generator.cuda()
discriminator = nn.DataParallel(discriminator)
discriminator.cuda()
classifier = nn.DataParallel(classifier)
classifier.cuda()
adversarial_loss = torch.nn.MSELoss().cuda()
task_loss = torch.nn.CrossEntropyLoss().cuda()
generator.apply(weights_init_normal)
discriminator.apply(weights_init_normal)
classifier.apply(weights_init_normal)
from misc import params
from misc.utils import get_data_loader, init_model, init_random_seed
from models import Classifier, Discriminator, Generator
if __name__ == '__main__':
# init random seed
init_random_seed(params.manual_seed)
# load dataset
src_data_loader = get_data_loader(params.src_dataset)
src_data_loader_test = get_data_loader(params.src_dataset, train=False)
tgt_data_loader = get_data_loader(params.tgt_dataset)
tgt_data_loader_test = get_data_loader(params.tgt_dataset, train=False)
# init models
classifier = init_model(net=Classifier(), restore=params.c_model_restore)
generator = init_model(net=Generator(), restore=params.g_model_restore)
critic = init_model(net=Discriminator(input_dims=params.d_input_dims,
hidden_dims=params.d_hidden_dims,
output_dims=params.d_output_dims),
restore=params.d_model_restore)
# train models
print("=== Training models ===")
print(">>> Classifier <<<")
print(classifier)
print(">>> Generator <<<")
print(generator)
print(">>> Critic <<<")
print(critic)
def sample_learner(self,
input_shape,
device,
allow_nas=False,
learner_type="base",
iteration_maps_seed=False,
iteration=None,
deterministic=False,
iterations_depth_schedule=100,
randomize_width=False):
if iteration_maps_seed:
iteration = iteration - 1
encoding = [iteration % 6, iteration // 6]
else:
encoding = None
if learner_type == "sampled":
layers = min(4, max(0, iteration // iterations_depth_schedule))
model, encoding = sample_model(
input_shape,
layers=layers,
encoding=encoding,
blocks=2,
seed=iteration if deterministic else None,
batch_norm_momentum=0)
tlogger.record_tabular("encoding", encoding)
elif learner_type == "sampled4":
model, encoding = sample_model(
input_shape,
layers=4,
encoding=encoding,
seed=iteration if deterministic else None,
batch_norm_momentum=0)
tlogger.record_tabular("encoding", encoding)
elif learner_type == "base":
model = Classifier(input_shape,
batch_norm_momentum=0.0,
randomize_width=randomize_width)
elif learner_type == "base_fc":
model = Classifier(input_shape,
batch_norm_momentum=0.0,
randomize_width=randomize_width,
use_global_pooling=False)
elif learner_type == "linear":
model = models.LinearClassifier(input_shape)
elif learner_type == "base_larger":
model = models.ClassifierLarger(input_shape,
batch_norm_momentum=0.0,
randomize_width=randomize_width)
elif learner_type == "base_larger2":
model = models.ClassifierLarger2(input_shape,
batch_norm_momentum=0.0,
randomize_width=randomize_width)
elif learner_type == "base_larger3":
model = models.ClassifierLarger3(input_shape,
batch_norm_momentum=0.0,
randomize_width=randomize_width)
elif learner_type == "base_larger3_global_pooling":
model = models.ClassifierLarger3(input_shape,
batch_norm_momentum=0.0,
randomize_width=randomize_width,
use_global_pooling=True)
elif learner_type == "base_larger4_global_pooling":
model = models.ClassifierLarger4(input_shape,
batch_norm_momentum=0.0,
randomize_width=randomize_width,
use_global_pooling=True)
elif learner_type == "base_larger4":
model = models.ClassifierLarger4(input_shape,
batch_norm_momentum=0.0,
randomize_width=randomize_width,
use_global_pooling=False)
else:
raise NotImplementedError()
return Learner(model=model.to(device),
optimizer=np.random.choice(self.optimizers)), encoding
args = parser.parse_args()
# load data
transform = transforms.Compose(
[transforms.CenterCrop(args.image_size),
transforms.ToTensor()])
testset = datasets.MNIST('./data',
train=False,
download=True,
transform=transform)
dataloader = torch.utils.data.DataLoader(testset,
batch_size=args.batch_size,
shuffle=True,
num_workers=1)
model = Classifier(args)
model_pt = torch.load(args.save_path)
model.load_state_dict(model_pt)
model.eval()
correct = 0
total = 0
for data, label in dataloader:
if torch.cuda.is_available():
model = model.cuda()
data = data.cuda()
label = label.cuda()
outputs = model(data)
_, predicted = torch.max(outputs.data, 1)
total += label.size(0)
def main():
global args
# Parse commands from ArgumentParser
args = parser.parse_args()
# Our text field for imdb data
TEXT = torchtext.data.Field(lower=True)
# Our label field for imdb data
LABEL = torchtext.data.Field(sequential=False)
# Load GloVE embeddings
orig_embeddings = torch.load(args.data_folder + 'all_orig_emb.pt')
total_words = len(orig_embeddings)
# Load shared words and all GloVE words
with open(args.data_folder + "shared_words.txt", "r") as file:
shared_words = file.read().split('\n')
with open(args.data_folder + "glove_words.txt", "r") as file:
glove_words = file.read().split('\n')
# Recreate GloVE_dict
glove_dict = {}
for i, word in enumerate(glove_words):
glove_dict[word] = orig_embeddings[i]
# Load IMDB dataset with standard splits and restrictions identical to paper
train, test = torchtext.datasets.IMDB.splits(
TEXT,
LABEL,
filter_pred=lambda ex: ex.label != 'neutral' and len(ex.text) <= 400)
# Both loops go through the words of train and test dataset, finds words without glove vectors, and replaces them with <unk>
for i in range(len(train)):
review = train.examples[i].text
for i, word in enumerate(review):
if word not in glove_dict:
review[i] = '<unk>'
for i in range(len(test)):
review = test.examples[i].text
for i, word in enumerate(review):
if word not in glove_dict:
review[i] = '<unk>'
# Build modified vocabulary
TEXT.build_vocab(train)
LABEL.build_vocab(train)
# Create iterators over train and test set
train_iter, test_iter = torchtext.data.BucketIterator.splits(
(train, test), batch_size=args.batch_size, repeat=False, device=-1)
# If we want to use baseline GloVE embeddings
if args.embedding_type == 'baseline':
# Initialize embedding
comp_embedding = np.random.uniform(
-0.25, 0.25, (len(TEXT.vocab), args.embedding_size))
# For each vocab word, replace embedding vector with GloVE vector
for word in shared_words:
comp_embedding[TEXT.vocab.stoi[word]] = glove_dict[word]
# Initialize Classifer with our GloVE embedding
base_c = Classifier(torch.FloatTensor(comp_embedding), args.batch_size)
# Put model into CUDA memory if using GPU
if use_gpu:
base_c = base_c.cuda()
# Initialize Optimizer
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
base_c.parameters()),
lr=args.lr)
# Define Loss function
loss_func = nn.NLLLoss()
else:
'''
Note- the model in the paper is different because they only store the source dictionaries,
making their model smaller than normal classifiers which is a major purpose of the paper.
By my formulation, my model actually has the same size. However, they are fundamentally equivalent,
except that the authors would have to preprocess the data (convert words into codes) whereas I
simply make an embedding layer of size Vocab like GloVE vectors. Either way, I should get the same
levels of accuracy, which is the primary importance of the sentiment classification task- to check
whether the coding embeddings still give the same level of accuracy.
'''
# Initialize embedding
code_embedding = torch.FloatTensor(
np.random.uniform(-0.25, 0.25,
(len(TEXT.vocab), args.embedding_size)))
# Load best model for code embedding generation
model = Code_Learner(args.embedding_size, args.M, args.K)
model = torch.load(args.model_file)
# Put model into CUDA memory if using GPU
if use_gpu:
code_embedding = code_embedding.cuda()
model = model.cuda()
# For all words in vocab
for i in range(len(TEXT.vocab)):
# Try to see if it has a corresponding glove_vector
try:
glove_vec = glove_dict[TEXT.vocab.itos[i]]
if use_gpu:
glove_vec = glove_vec.cuda()
# If so, then generate our own embedding for the word using our model
code_embedding[i] = model(glove_vec, training=False)
# The word doesn't have a GloVE vector, keep it randomly initialized
except KeyError:
pass
base_c = Classifier(torch.FloatTensor(code_embedding.cpu()),
args.batch_size)
# Put model into CUDA memory if using GPU
if use_gpu:
base_c = base_c.cuda()
# Initialize Optimizer
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
base_c.parameters()),
lr=args.lr)
# Define Loss function
loss_func = nn.NLLLoss()
classifier_train(args.epochs, base_c, optimizer, loss_func, train_iter,
test_iter, args.embedding_type)
MetaDataset.Initialize(mnli_train_support,
config["support_k"]))
val_datasets.append(
MetaDataset.Initialize(mnli_train_query, config["query_k"]))
test_datasets.append(
MetaDataset.Initialize(mnli_test,
config["support_k"],
test=True))
task_classes.append(3)
task_classes = {k: v for k, v in enumerate(task_classes)}
return train_datasets, val_datasets, test_datasets, task_classes
config = parser.parse_args().__dict__
model = Classifier(config)
train_datasets, val_datasets, test_datasets, task_classes = get_tasks(
config["tasks"])
meta_trainer = MetaTrainer(model=model,
train_datasets=train_datasets,
val_datasets=val_datasets,
test_datasets=test_datasets,
task_classes=task_classes,
epochs=config["epochs"],
inner_lr=config['inner_lr'],
outer_lr=config['outer_lr'],
n_inner_steps=config["n_inner_steps"],
num_episodes=len(config["tasks"]),
model_save_path=config["model_save_path"],
type=bool,
default=True,
help='If use GPU for training')
parser.add_argument('--gpu_num',
type=int,
default=2,
choices=range(0, 5),
help='GPU numbers available for parallel training')
parser.add_argument('--method', type=int, default=0, choices=range(2))
args = parser.parse_args()
methods = ['combined', 'vanilla']
method = methods[args.method]
ROBUSTNESS_NORM = 3.5
classifier = Classifier(args)
classifier_pt = torch.load('classifier.pt')
classifier.load_state_dict(classifier_pt)
classifier.eval()
transform = transforms.Compose(
[transforms.CenterCrop(args.image_size),
transforms.ToTensor()])
testset = datasets.MNIST('./data',
train=False,
download=True,
transform=transform)
dataloader = torch.utils.data.DataLoader(testset,
batch_size=LARGE_BATCH,
shuffle=False,
num_workers=1)
def train():
seal = tf.placeholder(shape=(None, 224, 224, 3), dtype=tf.float32)
noseal = tf.placeholder(shape=(None, 224, 224, 3), dtype=tf.float32)
seal_label = tf.placeholder(shape=(None, 2), dtype=tf.float32)
noseal_label = tf.placeholder(shape=(None, 2), dtype=tf.float32)
z = Encoder(seal)
fake = Generative(z)
feature_fake, logits_fake = Discriminative(fake)
feature_real_seal, logits_real_seal = Discriminative(seal, reuse=True)
feature_real_noseal, logits_real_noseal = Discriminative(noseal,
reuse=True)
feature_seal, _ = Classifier(fake)
feature_noseal, logits_noseal = Classifier(noseal, reuse=True)
_, logits_seal = Classifier(seal, reuse=True)
loss_D = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=seal_label, logits=logits_real_seal) + \
tf.nn.softmax_cross_entropy_with_logits(labels=seal_label, logits=logits_real_noseal) + \
tf.nn.softmax_cross_entropy_with_logits(labels=noseal_label, logits=logits_fake))
loss_C = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=seal_label, logits=logits_seal) + \
tf.nn.softmax_cross_entropy_with_logits(labels=noseal_label, logits=logits_noseal))
loss_GD = tools.get_loss_GD(feature_real_noseal, feature_real_seal,
feature_fake)
loss_GC = tools.get_loss_GC(feature_noseal, feature_seal)
loss_G = tools.get_loss_G(seal, fake)
loss_KL = tools.get_loss_KL(z)
all_var = tf.trainable_variables()
var_C = [var for var in all_var if var.name.startswith('Classifier')]
var_E = [var for var in all_var if var.name.startswith('Encoder')]
var_G = [var for var in all_var if var.name.startswith('Generative')]
var_D = [var for var in all_var if var.name.startswith('Discriminative')]
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
opt_C = optimizer.minimize(loss_C, var_list=var_C)
opt_E = optimizer.minimize(3 * loss_KL + loss_G, var_list=var_E)
opt_G = optimizer.minimize(loss_G + 0.01 * loss_GC + 0.01 * loss_GD,
var_list=var_G)
opt_D = optimizer.minimize(loss_D, var_list=var_D)
accuracy = 0.5*(tf.reduce_mean(tf.cast(tf.equal(tf.argmax(tf.nn.softmax(logits_seal), 1),
tf.argmax(tf.nn.softmax(seal_label), 1)), tf.float32)) + \
tf.reduce_mean(tf.cast(tf.equal(tf.argmax(tf.nn.softmax(logits_noseal), 1),
tf.argmax(tf.nn.softmax(noseal_label), 1)), tf.float32)))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
saver = tf.train.Saver(max_to_keep=1)
ckpt = tf.train.get_checkpoint_state(model_path)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
pass
for i in range(1, epoch + 1):
for j in range(iter):
seal_img, seal_labels = tools.get_seal_set(
batch_size / 2, i * iter + j)
noseal_img, noseal_labels = tools.get_noseal_set(
batch_size / 2, i * iter + j)
sess.run(opt_E, feed_dict={seal: seal_img})
sess.run(opt_G,
feed_dict={
seal: seal_img,
seal_label: seal_labels,
noseal: noseal_img,
noseal_label: noseal_labels
})
sess.run(opt_D,
feed_dict={
seal: seal_img,
seal_label: seal_labels,
noseal: noseal_img,
noseal_label: noseal_labels
})
sess.run(opt_C,
feed_dict={
seal: seal_img,
seal_label: seal_labels,
noseal: noseal_img,
noseal_label: noseal_labels
})
l_kl, l_c, l_g, l_gc, l_gd, l_d, a = sess.run(
[loss_KL, loss_C, loss_G, loss_GC, loss_GD, loss_D, accuracy],
feed_dict={
seal: seal_img,
seal_label: seal_labels,
noseal: noseal_img,
noseal_label: noseal_labels
})
print 'Epoch ' + str(i) + ' : L_KL = ' + str(l_kl) + ' ; L_C = ' + str(l_c) + ' ; L_G = ' + str(l_g) + \
' ; L_GC = ' + str(l_gc) + ' ; L_GD = ' + str(l_gd) + ' ; L_D = ' + str(l_d) + ' ; acc = ' + str(a)
if i % 10 == 0:
saver.save(sess, os.path.join(model_path, 'model.ckpt'))
seal_img = tools.get_test()
fake_img = sess.run(fake, feed_dict={seal: seal_img})
tools.save(fake_img[0, :, :, :], i)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--word-dim', type=int, default=300, help='size of word embeddings')
parser.add_argument('--hidden-dim', type=int, default=300, help='number of hidden units per layer')
parser.add_argument('--num-layers', type=int, default=1, help='number of layers in BiLSTM')
parser.add_argument('--att-dim', type=int, default=350, help='number of attention unit')
parser.add_argument('--att-hops', type=int, default=4, help='number of attention hops, for multi-hop attention model')
parser.add_argument('--clf-hidden-dim', type=int, default=512, help='hidden (fully connected) layer size for classifier MLP')
parser.add_argument('--clip', type=float, default=0.5, help='clip to prevent the too large grad in LSTM')
parser.add_argument('--lr', type=float, default=.001, help='initial learning rate')
parser.add_argument('--weight-decay', type=float, default=1e-5, help='weight decay rate per batch')
parser.add_argument('--dropout', type=float, default=0.3)
parser.add_argument('--max-epoch', type=int, default=8)
parser.add_argument('--seed', type=int, default=666)
parser.add_argument('--cuda', action='store_true', default=True)
parser.add_argument('--optimizer', default='adam', choices=['adam', 'sgd'])
parser.add_argument('--batch-size', type=int, default=32, help='batch size for training')
parser.add_argument('--penalization-coeff', type=float, default=0.1, help='the penalization coefficient')
parser.add_argument('--fix-word-embedding', action='store_true')
parser.add_argument('--model-type', required=True, choices=['sa', 'avgblock', 'hard'])
parser.add_argument('--data-type', required=True, choices=['age2', 'dbpedia', 'yahoo'])
parser.add_argument('--data', required=True, help='pickle file obtained by dataset dump')
parser.add_argument('--save-dir', type=str, required=True, help='path to save the final model')
parser.add_argument('--block-size', type=int, default=-1, help='block size only when model-type is avgblock')
args = parser.parse_args()
torch.manual_seed(args.seed)
random.seed(args.seed)
if torch.cuda.is_available():
if not args.cuda:
print("WARNING: You have a CUDA device, so you should probably run with --cuda")
else:
torch.cuda.manual_seed(args.seed)
#######################################
# a simple log file, the same content as stdout
if not os.path.exists(args.save_dir):
os.mkdir(args.save_dir)
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)-8s %(message)s')
logFormatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
rootLogger = logging.getLogger()
fileHandler = logging.FileHandler(os.path.join(args.save_dir, 'stdout.log'))
fileHandler.setFormatter(logFormatter)
rootLogger.addHandler(fileHandler)
########################################
for k, v in vars(args).items():
logging.info(k+':'+str(v))
#####################################################################
if args.data_type == 'age2':
data = AGE2(datapath=args.data, batch_size=args.batch_size)
num_classes = 5
elif args.data_type == 'dbpedia':
data = DBpedia(datapath=args.data, batch_size=args.batch_size)
num_classes = 14
elif args.data_type == 'yahoo':
data = Yahoo(datapath=args.data, batch_size=args.batch_size)
num_classes = 10
else:
raise Exception('Invalid argument data-type')
#####################################################################
if args.model_type == 'avgblock':
assert args.block_size > 0
#####################################################################
tic = time.time()
model = Classifier(
dictionary=data,
dropout=args.dropout,
num_words=data.num_words,
num_layers=args.num_layers,
hidden_dim=args.hidden_dim,
word_dim=args.word_dim,
att_dim=args.att_dim,
att_hops=args.att_hops,
clf_hidden_dim=args.clf_hidden_dim,
num_classes=num_classes,
model_type=args.model_type,
block_size=args.block_size,
)
print('It takes %.2f sec to build the model.' % (time.time() - tic))
logging.info(model)
model.word_embedding.weight.data.set_(data.weight)
if args.fix_word_embedding:
model.word_embedding.weight.requires_grad = False
if args.cuda:
model = model.cuda()
''' count parameters
num_params = sum(np.prod(p.size()) for p in model.parameters())
num_embedding_params = np.prod(model.word_embedding.weight.size())
print('# of parameters: %d' % num_params)
print('# of word embedding parameters: %d' % num_embedding_params)
print('# of parameters (excluding word embeddings): %d' % (num_params - num_embedding_params))
'''
if args.optimizer == 'adam':
optimizer_class = optim.Adam
elif args.optimizer == 'sgd':
optimizer_class = optim.SGD
else:
raise Exception('For other optimizers, please add it yourself. supported ones are: SGD and Adam.')
params = [p for p in model.parameters() if p.requires_grad]
optimizer = optimizer_class(params=params, lr=args.lr, weight_decay=args.weight_decay)
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='max', factor=0.5, patience=10, verbose=True)
criterion = nn.CrossEntropyLoss()
# Identity matrix for each batch
I = Variable(torch.eye(args.att_hops).unsqueeze(0).expand(args.batch_size, -1, -1))
if args.cuda:
I = I.cuda()
trpack = {
'model': model,
'params': params,
'criterion': criterion,
'optimizer': optimizer,
'I': I,
}
train_summary_writer = tensorboard.FileWriter(
logdir=os.path.join(args.save_dir, 'log', 'train'), flush_secs=10)
valid_summary_writer = tensorboard.FileWriter(
logdir=os.path.join(args.save_dir, 'log', 'valid'), flush_secs=10)
tsw, vsw = train_summary_writer, valid_summary_writer
logging.info('number of train batches: %d' % data.train_num_batch)
validate_every = data.train_num_batch // 10
best_vaild_accuacy = 0
iter_count = 0
tic = time.time()
for epoch_num in range(args.max_epoch):
for batch_iter, train_batch in enumerate(data.train_minibatch_generator()):
progress = epoch_num + batch_iter / data.train_num_batch
iter_count += 1
train_loss, train_accuracy = train_iter(args, train_batch, **trpack)
add_scalar_summary(tsw, 'loss', train_loss, iter_count)
add_scalar_summary(tsw, 'acc', train_accuracy, iter_count)
if (batch_iter + 1) % (data.train_num_batch // 100) == 0:
tac = (time.time() - tic) / 60
print(' %.2f minutes\tprogress: %.2f' % (tac, progress))
if (batch_iter + 1) % validate_every == 0:
correct_sum = 0
for valid_batch in data.dev_minibatch_generator():
correct, supplements = eval_iter(args, model, valid_batch)
correct_sum += unwrap_scalar_variable(correct)
valid_accuracy = correct_sum / data.dev_size
scheduler.step(valid_accuracy)
add_scalar_summary(vsw, 'acc', valid_accuracy, iter_count)
logging.info('Epoch %.2f: valid accuracy = %.4f' % (progress, valid_accuracy))
if valid_accuracy > best_vaild_accuacy:
correct_sum = 0
for test_batch in data.test_minibatch_generator():
correct, supplements = eval_iter(args, model, test_batch)
correct_sum += unwrap_scalar_variable(correct)
test_accuracy = correct_sum / data.test_size
best_vaild_accuacy = valid_accuracy
model_filename = ('model-%.2f-%.4f-%.4f.pkl' % (progress, valid_accuracy, test_accuracy))
model_path = os.path.join(args.save_dir, model_filename)
torch.save(model.state_dict(), model_path)
print('Saved the new best model to %s' % model_path)
import tensorflow as tf
from models import Classifier, BayesClassifier, PGDAttackClassifier, PGDAttackCombined
from eval_utils import BaseDetectorFactory, load_mnist_data
from eval_utils import get_tpr, get_fpr
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.logging.set_verbosity(tf.logging.ERROR)
np.random.seed(123)
(x_train, y_train), (x_test, y_test) = load_mnist_data()
if len(sys.argv) > 1:
num_eval_samples = int(sys.argv[1])
x_test, y_test = x_test[:num_eval_samples], y_test[:num_eval_samples]
classifier = Classifier(var_scope='classifier', dataset='MNIST')
classifier_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='classifier')
classifier_saver = tf.train.Saver(var_list=classifier_vars, max_to_keep=1)
factory = BaseDetectorFactory(eps=0.3)
attack_config = {
'max_distance': 0.3,
'num_steps': 100,
'step_size': 0.1,
'random_start': False,
'x_min': 0,
'x_max': 1.0,
'batch_size': x_test.shape[0] // 2,
'optimizer': 'adam',
'norm': 'Linf'
def load_cnn(self, args):
clf = Classifier(args, method='cnn2')
return clf
def load_softmax(self, args):
clf = Classifier(args, method='softmax')
return clf
import torch
from torch import nn
from dataset import GenerateIterator, GenerateIterator_eval
from myargs import args
import numpy as np
from tqdm import tqdm
from models import Classifier, Discriminator, EMA
from vat import VAT, ConditionalEntropyLoss
# discriminator network
feature_discriminator = Discriminator(large=args.large).cuda()
# classifier network.
classifier = Classifier(large=args.large).cuda()
# loss functions
cent = ConditionalEntropyLoss().cuda()
xent = nn.CrossEntropyLoss(reduction='mean').cuda()
sigmoid_xent = nn.BCEWithLogitsLoss(reduction='mean').cuda()
vat_loss = VAT(classifier).cuda()
# optimizer.
optimizer_cls = torch.optim.Adam(classifier.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2))
optimizer_disc = torch.optim.Adam(feature_discriminator.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2))
# datasets.
iterator_train = GenerateIterator(args)
def _prepare_training(self):
"""Prepare Training
Make tensorflow's graph.
To support Multi-GPU, divide mini-batch.
And this program has resume function.
If there is checkpoint file in FLAGS.classifier_dir/log, load checkpoint file and restart training.
"""
self.gpu_n = len(FLAGS.gpu_ids.split(','))
with tf.device('/cpu:0'):
self.imgs = tf.placeholder(tf.float32, (FLAGS.batch_size, FLAGS.img_width, FLAGS.img_height, FLAGS.img_dim), name='imgs')
self.labels = tf.placeholder(tf.float32, (FLAGS.batch_size, 26), name='labels')
self.is_train = tf.placeholder(tf.bool, name='is_train')
c_opt = tf.train.AdamOptimizer(learning_rate=0.001)
c_loss = [0] * self.gpu_n
c_acc = [0] * self.gpu_n
c_grads = [0] * self.gpu_n
label_corrects = [0] * self.gpu_n
label_ns = [0] * self.gpu_n
is_not_first = False
for i in range(self.gpu_n):
with tf.device('/gpu:{}'.format(i)):
classifier = Classifier(img_size=(FLAGS.img_width, FLAGS.img_height),
img_dim=FLAGS.img_dim,
k_size=3,
class_n=26,
smallest_unit_n=64)
classified = classifier(self.imgs, is_train=self.is_train, is_reuse=is_not_first)
c_loss[i] = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=classified))
label_indecies = tf.argmax(self.labels, 1)
correct_pred = tf.equal(tf.argmax(classified, 1), label_indecies)
c_acc[i] = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
if FLAGS.labelacc:
masked_label_indecies = tf.boolean_mask(label_indecies, correct_pred)
label_corrects[i] = tf.reduce_sum(tf.one_hot(masked_label_indecies, 26), axis=0)
label_ns[i] = tf.reduce_sum(tf.one_hot(label_indecies, 26), axis=0)
c_vars = [var for var in tf.trainable_variables() if 'classifier' in var.name]
c_grads[i] = c_opt.compute_gradients(c_loss[i], var_list=c_vars)
is_not_first = True
with tf.device('/cpu:0'):
self.c_loss = sum(c_loss) / len(c_loss)
self.c_acc = sum(c_acc) / len(c_acc)
if FLAGS.labelacc:
self.c_acc_by_labels = sum(label_corrects) / sum(label_ns)
avg_c_grads = average_gradients(c_grads)
self.c_train = c_opt.apply_gradients(avg_c_grads)
sess_config = tf.ConfigProto(
gpu_options=tf.GPUOptions(visible_device_list=FLAGS.gpu_ids),
allow_soft_placement=FLAGS.labelacc,
log_device_placement=FLAGS.labelacc
)
self.sess = tf.Session(config=sess_config)
self.saver = tf.train.Saver()
checkpoint = tf.train.get_checkpoint_state(self.dst_log)
if checkpoint:
saver_resume = tf.train.Saver()
saver_resume.restore(self.sess, checkpoint.model_checkpoint_path)
self.epoch_start = int(checkpoint.model_checkpoint_path.split('-')[-1])
print('restore ckpt')
else:
self.sess.run(tf.global_variables_initializer())
self.epoch_start = 0
import torch
import torch.optim as optim
from torch import nn
from torch.utils import data
from torchvision import datasets, transforms
from models import Classifier
cla = Classifier().cuda()
opt = optim.Adam(cla.parameters(), lr=1e-3)
## Step 1: train the classifier
trainloader = data.DataLoader(
datasets.MNIST('../data', download=True, train=True, transform=transforms.Compose([
transforms.ToTensor(),
])), batch_size=100, shuffle=True)
testloader = data.DataLoader(
datasets.MNIST('../data', download=True, train=False, transform=transforms.Compose([
transforms.ToTensor(),
])), batch_size=100, shuffle=True)
def accuracy():
total_correct = 0.0
total = 0.0
with torch.no_grad():
for images, targets in testloader:
out = cla(images.cuda())
preds = out.argmax(1)
total_correct += (preds.cpu()==targets).float().sum()
total += preds.shape[0]
return total_correct/total
with open("values.json", "r") as v:
values = json.load(v)
PATH = values["classifier"]
transform = transforms.Compose([transforms.RandomAffine(degrees=15, scale=(0.9, 1.0), fillcolor=256), transforms.Grayscale(), transforms.Resize(227), transforms.RandomHorizontalFlip(0.5), transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,))])
path = values["datasetFolder"]+"/SKETCHES_TRAINING"
trainset = torchvision.datasets.ImageFolder(root=path, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=4, shuffle=True, num_workers=4)
classes = set(values["classes"])
net = Classifier()
net.to(device)
if use_cuda:
net.cuda()
# Set up loss function and optimiser
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(net.parameters(), lr=0.0001, momentum=0.9)
epoch = 0
running_loss = 1.0
# Simple training for 500 epochs
def run_infer(opt):
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
n_classes = 6
weight_path = opt.weight_path
tst_preds = []
TEST_DIR = opt.test_dir
test = pd.DataFrame()
# test['image_id'] = sorted(list(os.listdir(TEST_DIR)))
content = pd.read_csv('/content/test_aug.csv')
test_image = []
for i in content["image_id"]:
test_image.append(i)
test['image_id'] = sorted(test_image)
if "vit" in opt.model_arch:
testset = ICDARDataset(test,
TEST_DIR,
transforms=get_inference_transforms(384))
else:
testset = ICDARDataset(test,
TEST_DIR,
transforms=get_inference_transforms(
opt.img_size))
tst_loader = DataLoader(testset,
batch_size=opt.valid_bs,
num_workers=opt.num_workers,
shuffle=False,
pin_memory=False)
print("[INFO] Found {} folds in weight path".format(
len(os.listdir(weight_path))))
print(os.listdir(weight_path))
print("[INFO] Start inference ...")
# for fold in os.listdir(weight_path):
# print(fold)
# model = Classifier(opt.model_arch, n_classes).to(device)
# model_path = os.path.join(weight_path, fold, "best.pt")
# model.load_state_dict(torch.load(model_path)['model'])
# with torch.no_grad():
# for _ in range(opt.tta):
# tst_preds += [inference_one_epoch(model, tst_loader, device)]
# del model
model = Classifier(opt.model_arch, n_classes).to(device)
model_path = '/content/drive/MyDrive/Emotion_Speech_Recognition/Models/Classify_image/resnet/best.pt'
model.load_state_dict(torch.load(model_path)['model'])
with torch.no_grad():
for _ in range(opt.tta):
tst_preds += [inference_one_epoch(model, tst_loader, device)]
del model
avg_tst_preds = np.mean(tst_preds, axis=0)
if not (os.path.isdir(opt.work_dir)):
os.mkdir(opt.work_dir)
# np.save(os.path.join(opt.work_dir, "{}.npy".format(opt.model_arch)), avg_tst_preds)
test['predict'] = np.argmax(avg_tst_preds, axis=1)
gt = []
for i in content["image_id"]:
gt.append(i.split('/')[0])
test['gt'] = gt
test.to_csv('submission.csv', index=False)
torch.cuda.empty_cache()
src_data_loader_eval = get_visda(root=params.data_root,
sub_dir='train',
split='test')
tgt_data_loader = get_visda(root=params.data_root,
sub_dir='validation',
split='train')
tgt_data_loader_eval = get_visda(root=params.data_root,
sub_dir='validation',
split='test')
# load models
src_encoder = init_model(net=ResNet34Encoder(),
restore=params.src_encoder_restore,
imageNet_train=True)
src_classifier = init_model(net=Classifier(),
restore=params.src_classifier_restore)
# train source model
# print("=== Training classifier for source domain ===")
# print(">>> Source Encoder <<<")
# print(src_encoder)
# print(">>> Source Classifier <<<")
# print(src_classifier)
if not (src_encoder.restored and src_classifier.restored
and params.src_model_trained):
src_encoder, src_classifier = train_src(src_encoder, src_classifier,
src_data_loader,
tgt_data_loader)
'''params'''
batch_size = 128
lr = 0.01
momentum = 0.9
total_epochs = 100
source_dataset_train, source_dataset_test = Data_Mnist()
target_dataset_train, target_dataser_test = Data_Mnist_M()
source_loader = torch.utils.data.DataLoader(source_dataset_train, batch_size = batch_size, shuffle = True)
target_loader = torch.utils.data.DataLoader(target_dataset_train, batch_size = batch_size, shuffle = True)
s_test_loader = torch.utils.data.DataLoader(source_dataset_test, batch_size = batch_size, shuffle = True)
t_test_loader = torch.utils.data.DataLoader(target_dataser_test, batch_size = batch_size, shuffle = True)
total_steps = total_epochs*len(source_loader)
'''定义网络框架'''
feature_extrator = Extractor()
class_classifier = Classifier()
class_criterion = nn.NLLLoss()
optimizer = optim.SGD([{'params': feature_extrator.parameters()},
{'params': class_classifier.parameters()}], lr= lr, momentum= momentum)
if torch.cuda.is_available():
feature_extrator = feature_extrator.cuda()
class_classifier = class_classifier.cuda()
class_criterion = class_criterion.cuda()
def train(f,c,source,target,optimizer,step):
result = []
source_data, source_label = source
target_data, target_label = target
# torchvision.utils.save_image(source_data,'mnist.png')
# torchvision.utils.save_image(target_data, 'mnist_M.png')
def train_sentiment(opts):
device = torch.device("cuda" if use_cuda else "cpu")
glove_loader = GloveLoader(os.path.join(opts.data_dir, 'glove', opts.glove_emb_file))
train_loader = DataLoader(RottenTomatoesReviewDataset(opts.data_dir, 'train', glove_loader, opts.maxlen), \
batch_size=opts.bsize, shuffle=True, num_workers=opts.nworkers)
valid_loader = DataLoader(RottenTomatoesReviewDataset(opts.data_dir, 'val', glove_loader, opts.maxlen), \
batch_size=opts.bsize, shuffle=False, num_workers=opts.nworkers)
model = Classifier(opts.hidden_size, opts.dropout_p, glove_loader, opts.enc_arch)
if opts.optim == 'adam':
optimizer = torch.optim.Adam(model.parameters(), lr=opts.lr, weight_decay=opts.wd)
else:
raise NotImplementedError("Unknown optim type")
criterion = nn.CrossEntropyLoss()
start_n_iter = 0
# for choosing the best model
best_val_acc = 0.0
model_path = os.path.join(opts.save_path, 'model_latest.net')
if opts.resume and os.path.exists(model_path):
# restoring training from save_state
print ('====> Resuming training from previous checkpoint')
save_state = torch.load(model_path, map_location='cpu')
model.load_state_dict(save_state['state_dict'])
start_n_iter = save_state['n_iter']
best_val_acc = save_state['best_val_acc']
opts = save_state['opts']
opts.start_epoch = save_state['epoch'] + 1
model = model.to(device)
# for logging
logger = TensorboardXLogger(opts.start_epoch, opts.log_iter, opts.log_dir)
logger.set(['acc', 'loss'])
logger.n_iter = start_n_iter
for epoch in range(opts.start_epoch, opts.epochs):
model.train()
logger.step()
for batch_idx, data in enumerate(train_loader):
acc, loss = run_iter(opts, data, model, criterion, device)
# optimizer step
optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), opts.max_norm)
optimizer.step()
logger.update(acc, loss)
val_loss, val_acc, time_taken = evaluate(opts, model, valid_loader, criterion, device)
# log the validation losses
logger.log_valid(time_taken, val_acc, val_loss)
print ('')
# Save the model to disk
if val_acc >= best_val_acc:
best_val_acc = val_acc
save_state = {
'epoch': epoch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'n_iter': logger.n_iter,
'opts': opts,
'val_acc': val_acc,
'best_val_acc': best_val_acc
}
model_path = os.path.join(opts.save_path, 'model_best.net')
torch.save(save_state, model_path)
save_state = {
'epoch': epoch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'n_iter': logger.n_iter,
'opts': opts,
'val_acc': val_acc,
'best_val_acc': best_val_acc
}
model_path = os.path.join(opts.save_path, 'model_latest.net')
torch.save(save_state, model_path)
def main():
args = parser.parse_args()
# model
model = Classifier(args.channels)
optimizer = optim.SGD(
model.parameters(), lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, args.epoch)
if args.gpu is not None:
model.cuda(args.gpu)
# dataset
raw_loader = torch.utils.data.DataLoader(
Dataset(os.path.join(DATA_DIR, 'raw')),
args.batch // 2, shuffle=True, drop_last=True)
noised_loader = torch.utils.data.DataLoader(
Dataset(os.path.join(DATA_DIR, 'noised_tgt')),
args.batch // 2, shuffle=True, drop_last=True)
# train
for epoch in range(args.epoch):
loss = 0
accuracy = 0
count = 0
for x0, x1 in zip(noised_loader, raw_loader):
if args.gpu is not None:
x0 = x0.cuda(args.gpu)
x1 = x1.cuda(args.gpu)
# train
model.train()
x = torch.cat((x0, x1), dim=0) # @UndefinedVariable
t = torch.zeros((x.shape[0], 2), device=x.device).float() # @UndefinedVariable
t[:x0.shape[0], 0] = 1
t[x0.shape[0]:, 1] = 1
x, t = mixup(x, t)
y = model(x)
e = (-1 * nn.functional.log_softmax(y, dim=1) * t).sum(dim=1).mean()
optimizer.zero_grad()
e.backward()
optimizer.step()
# validate
model.eval()
with torch.no_grad():
y0 = (model(x0).max(dim=1)[1] == 0).float()
y1 = (model(x1).max(dim=1)[1] == 1).float()
a = torch.cat((y0, y1), dim=0).mean() # @UndefinedVariable
loss += float(e) * len(x)
accuracy += float(a) * len(x)
count += len(x)
print('[{}] lr={:.7f}, loss={:.4f}, accuracy={:.4f}'.format(
epoch, float(optimizer.param_groups[0]['lr']), loss / count, accuracy / count),
flush=True)
scheduler.step()
snapshot = {'channels': args.channels, 'model': model.state_dict()}
torch.save(snapshot, '{}.tmp'.format(args.file))
os.rename('{}.tmp'.format(args.file), args.file)
batch_size = 10
n_frames = 30
n_vid_features = 3600
n_aud_features = 1
n_head = 3
n_layers = 3
dim_feedforward = 128
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
autoencoder = Autoencoder()
autoencoder.load_state_dict(torch.load(AUTOENCODER))
autoencoder.to(device)
autoencoder.eval()
model = Classifier(n_vid_features, n_aud_features, n_head, n_layers, dim_feedforward)
model.load_state_dict(torch.load(CLASSIFIER))
model = model.to(device)
model.eval()
start_time = datetime.datetime.now()
print(f'start time: {str(start_time)}')
print(f'using device: {device}')
count = 0
count_wrong = 0
val_dataset = EncodedDeepfakeDataset(VAL_FOLDERS, autoencoder.encoder, n_frames=n_frames, n_audio_reads=576, device=device, cache_folder="encode_cache")
dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=True)
for i, batch in enumerate(dataloader):
if i * batch_size >= test_size:
def main():
parser = argparse.ArgumentParser(description='Test SimCLR model')
parser.add_argument('--EPOCHS',
default=1,
type=int,
help='Number of epochs for training')
parser.add_argument('--BATCH_SIZE',
default=64,
type=int,
help='Batch size')
parser.add_argument(
'--LOG_INT',
type=int,
default=100,
help='how many batches to wait before logging training status')
parser.add_argument('--SAVED_MODEL', default='./ckpt/model.pth')
args = parser.parse_args()
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
saved_model = Encoder()
saved_model.load_state_dict(torch.load(args.SAVED_MODEL))
# Freeze weights in the pretrained model
for param in saved_model.parameters():
param.requires_grad = False
test_saved_model = Classifier(saved_model).to(device)
criterion = nn.CrossEntropyLoss()
optimizer_saved = optim.Adam(test_saved_model.fc.parameters())
standard_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])
])
trainset = torchvision.datasets.CIFAR10(root='./data',
train=True,
download=False,
transform=standard_transform)
train_loader = torch.utils.data.DataLoader(trainset,
batch_size=args.BATCH_SIZE,
shuffle=True)
testset = torchvision.datasets.CIFAR10(root='./data',
train=False,
download=False,
transform=standard_transform)
test_loader = torch.utils.data.DataLoader(testset,
batch_size=args.BATCH_SIZE,
shuffle=True)
for epoch in range(args.EPOCHS):
print("Performance on the saved model")
train(args, test_saved_model, device, train_loader, optimizer_saved,
epoch, criterion)
test(args, test_saved_model, device, test_loader)
from models import Classifier
import pickle
model_path = './models/model7.hd5'
# load dataset
with open('./datasets/80k_no_priorityroad_grey.pickle', 'rb') as f:
(x_train, y_train), (x_val, y_val), (x_test, y_test) = pickle.load(f)
if __name__ == '__main__':
# make model
model = Classifier()
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# train model
model.fit(x_train, y_train, epochs=5, batch_size=32, validation_data=(x_val, y_val))
# test model
model.evaluate(x_test, y_test)
# save model
model.save(model_path)
print('MODEL SAVED')
else:
# load
with open(config_path, 'r') as f:
config = json.load(f)
# split
img_gen_config = config['img_gen_config']
model_config = config['model_config']
train_config = config['train_config']
# warn
warn_message = f'Using config from {config_path_rel}'
warnings.warn(warn_message)
# build img data generators
train_generator, validation_generator, test_generator, class_names = build_set_generators(
**img_gen_config)
# define model
cl = Classifier(img_gen_config=img_gen_config,
model_config=model_config,
input_shape=train_generator.x.shape[1:])
cl.class_names = class_names
# cl.train(train_generator, validation_generator, train_config)
cl.train(train_generator, validation_generator, train_config)
# eval
cl.evaluate(test_generator)
# save
cl.save()
def main():
# Parse the arguments.
args = parse_arguments()
method = args.method
if args.data_name == 'suzuki':
datafile = 'data/suzuki_type_train_v2.csv'
class_num = 119
class_dict = {'M': 28, 'L': 23, 'B': 35, 'S': 10, 'A': 17}
dataset_filename = 'data.npz'
labels = ['Yield', 'M', 'L', 'B', 'S', 'A', 'id']
elif args.data_name == 'CN':
datafile = 'data/CN_coupling_train.csv'
class_num = 206
class_dict = {'M': 44, 'L': 47, 'B': 13, 'S': 22, 'A': 74}
dataset_filename = 'CN_data.npz'
labels = ['Yield', 'M', 'L', 'B', 'S', 'A', 'id']
elif args.data_name == 'Negishi':
datafile = 'data/Negishi_train.csv'
class_num = 106
class_dict = {'M': 32, 'L': 20, 'T': 8, 'S': 10, 'A': 30}
dataset_filename = 'Negishi_data.npz'
labels = ['Yield', 'M', 'L', 'T', 'S', 'A', 'id']
elif args.data_name == 'PKR':
datafile = 'data/PKR_train.csv'
class_num = 83
class_dict = {
'M': 18,
'L': 6,
'T': 7,
'S': 15,
'A': 11,
'G': 1,
'O': 13,
'P': 4,
'other': 1
}
dataset_filename = 'PKR_data.npz'
labels = [
'Yield', 'M', 'L', 'T', 'S', 'A', 'G', 'O', 'P', 'other', 'id'
]
else:
raise ValueError('Unexpected dataset name')
cache_dir = os.path.join('input', '{}_all'.format(method))
# Dataset preparation. Postprocessing is required for the regression task.
def postprocess_label(label_list):
return numpy.asarray(label_list, dtype=numpy.float32)
# Load the cached dataset.
dataset_cache_path = os.path.join(cache_dir, dataset_filename)
dataset = None
if os.path.exists(dataset_cache_path):
print('Loading cached dataset from {}.'.format(dataset_cache_path))
dataset = NumpyTupleDataset.load(dataset_cache_path)
if dataset is None:
print('Preprocessing dataset...')
if args.method == 'mpnn':
preprocessor = preprocess_method_dict['ggnn']()
else:
preprocessor = preprocess_method_dict[args.method]()
parser = CSVFileParser(
preprocessor,
postprocess_label=postprocess_label,
labels=labels,
smiles_col=['Reactant1', 'Reactant2', 'Product'],
label_dicts=class_dict)
# Load the entire dataset.
dataset = parser.parse(datafile)['dataset']
# Cache the laded dataset.
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
NumpyTupleDataset.save(dataset_cache_path, dataset)
# Scale the label values, if necessary.
if args.scale == 'standardize':
scaler = StandardScaler()
labels = dataset.get_datasets()[-2]
# yields = dataset.get_datasets()[-1]
yields = dataset.get_datasets()[-1][:, 0].reshape(-1,
1).astype('float32')
# Filter index here
# range_exp = (0.0 <= yields) & (yields <= 1.0)
range_exp = numpy.argsort(yields[:, 0]) # ascending
start_len = 0
end_len = len(yields) #int(len(yields) / 4)
range_exp = range_exp[start_len:end_len]
range_dataset = (dataset.get_datasets()[0][range_exp],
dataset.get_datasets()[1][range_exp],
dataset.get_datasets()[2][range_exp],
dataset.get_datasets()[3][range_exp],
dataset.get_datasets()[4][range_exp],
dataset.get_datasets()[5][range_exp])
yields = yields[range_exp]
labels = labels[range_exp]
dataset = NumpyTupleDataset(*(range_dataset + (
yields,
labels,
)))
else:
scaler = None
# Split the dataset into training and validation.
train_data_size = int(len(dataset) * args.train_data_ratio)
train, valid = split_dataset_random(dataset, train_data_size, args.seed)
# Set up the predictor.
predictor = set_up_predictor(args.method, args.unit_num, args.conv_layers,
class_num)
# Set up the iterator.
train_iter = iterators.SerialIterator(train, args.batchsize)
valid_iter = iterators.SerialIterator(valid,
args.batchsize,
repeat=False,
shuffle=False)
# Set up the regressor.
device = args.gpu
classifier = Classifier(predictor,
lossfun=F.sigmoid_cross_entropy,
metrics_fun=F.binary_accuracy,
device=args.gpu)
# Set up the optimizer.
optimizer = optimizers.Adam()
optimizer.setup(classifier)
# Set up the updater.
updater = training.StandardUpdater(train_iter,
optimizer,
device=args.gpu,
converter=concat_mols)
# Set up the trainer.
print('Training...')
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
trainer.extend(
E.Evaluator(valid_iter,
classifier,
device=device,
converter=concat_mols))
trainer.extend(E.snapshot(), trigger=(args.epoch, 'epoch'))
trainer.extend(
E.snapshot_object(
classifier,
filename='model_epoch-{.updater.epoch}')) # save every epoch
trainer.extend(E.LogReport())
trainer.extend(
E.PrintReport([
'epoch', 'main/loss', 'main/accuracy', 'validation/main/loss',
'validation/main/accuracy', 'elapsed_time'
]))
trainer.extend(E.ProgressBar())
trainer.run()
# Save the regressor's parameters.
model_path = os.path.join(args.out, args.model_filename)
print('Saving the trained model to {}...'.format(model_path))
classifier.save_pickle(model_path, protocol=args.protocol)
# Save the standard scaler's parameters.
if scaler is not None:
with open(os.path.join(args.out, 'scaler.pkl'), mode='wb') as f:
pickle.dump(scaler, f, protocol=args.protocol)
#bin3 = 16-40
#bin4 = 41 and more
bin1 = 0
bin2 = 0
bin3 = 0
bin4 = 0
tot1 = 0
tot2 = 0
tot3 = 0
tot4 = 0
# model = LSTM_main(config).to(device)
# PATH = bi_path
model = Classifier(config).to(device)
PATH = base_path
model.load_state_dict(torch.load(PATH, map_location=device))
model = model.to(device)
print(model)
for i in range(len(labels)):
s1_embed, s1_len = get_batch_from_idx(s1[i].split(), embeddings, config)
s2_embed, s2_len = get_batch_from_idx(s2[i].split(), embeddings, config)
u = torch.sum(s1_embed,0).to(device)
v = torch.sum(s1_embed,0).to(device)
feats = torch.cat((u, v, torch.abs(u- v), u*v), 0).to(device)
def train(seed=0,
dataset='grid',
samplers=(UniformDatasetSampler, UniformLatentSampler),
latent_dim=2,
model_dim=256,
device='cuda',
conditional=False,
learning_rate=2e-4,
betas=(0.5, 0.9),
batch_size=256,
iterations=400,
n_critic=5,
objective='gan',
gp_lambda=10,
output_dir='results',
plot=False,
spec_norm=True):
experiment_name = [
seed, dataset, samplers[0].__name__, samplers[1].__name__, latent_dim,
model_dim, device, conditional, learning_rate, betas[0], betas[1],
batch_size, iterations, n_critic, objective, gp_lambda, plot, spec_norm
]
experiment_name = '_'.join([str(p) for p in experiment_name])
results_dir = os.path.join(output_dir, experiment_name)
network_dir = os.path.join(results_dir, 'networks')
eval_log = os.path.join(results_dir, 'eval.log')
os.makedirs(results_dir, exist_ok=True)
os.makedirs(network_dir, exist_ok=True)
eval_file = open(eval_log, 'w')
if plot:
samples_dir = os.path.join(results_dir, 'samples')
os.makedirs(samples_dir, exist_ok=True)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
data, labels = load_data(dataset)
data_dim, num_classes = data.shape[1], len(set(labels))
data_sampler = samplers[0](
torch.tensor(data).float(),
torch.tensor(labels).long()) if conditional else samplers[0](
torch.tensor(data).float())
noise_sampler = samplers[1](
latent_dim, labels) if conditional else samplers[1](latent_dim)
if conditional:
test_data, test_labels = load_data(dataset, split='test')
test_dataset = TensorDataset(
torch.tensor(test_data).to(device).float(),
torch.tensor(test_labels).to(device).long())
test_dataloader = DataLoader(test_dataset, batch_size=4096)
G = Generator(latent_dim + num_classes, model_dim,
data_dim).to(device).train().train()
D = Discriminator(model_dim,
data_dim + num_classes,
spec_norm=spec_norm).to(device).train()
C_real = Classifier(model_dim, data_dim,
num_classes).to(device).train()
C_fake = Classifier(model_dim, data_dim,
num_classes).to(device).train()
C_fake.load_state_dict(deepcopy(C_real.state_dict()))
C_real_optimizer = optim.Adam(C_real.parameters(),
lr=2 * learning_rate)
C_fake_optimizer = optim.Adam(C_fake.parameters(),
lr=2 * learning_rate)
C_crit = nn.CrossEntropyLoss()
else:
G = Generator(latent_dim, model_dim, data_dim).to(device).train()
D = Discriminator(model_dim, data_dim,
spec_norm=spec_norm).to(device).train()
D_optimizer = optim.Adam(D.parameters(), lr=learning_rate, betas=betas)
G_optimizer = optim.Adam(G.parameters(), lr=learning_rate, betas=betas)
if objective == 'gan':
fake_target = torch.zeros(batch_size, 1).to(device)
real_target = torch.ones(batch_size, 1).to(device)
elif objective == 'wgan':
grad_target = torch.ones(batch_size, 1).to(device)
elif objective == 'hinge':
bound = torch.zeros(batch_size, 1).to(device)
sub = torch.ones(batch_size, 1).to(device)
stats = {'D': [], 'G': [], 'C_it': [], 'C_real': [], 'C_fake': []}
if plot:
fixed_latent_batch = noise_sampler.get_batch(20000)
sample_figure = plt.figure(num=0, figsize=(5, 5))
loss_figure = plt.figure(num=1, figsize=(10, 5))
if conditional:
accuracy_figure = plt.figure(num=2, figsize=(10, 5))
for it in range(iterations + 1):
# Train Discriminator
data_batch = data_sampler.get_batch(batch_size)
latent_batch = noise_sampler.get_batch(batch_size)
if conditional:
x_real, y_real = data_batch[0].to(device), data_batch[1].to(device)
real_sample = torch.cat([x_real, y_real], dim=1)
z_fake, y_fake = latent_batch[0].to(device), latent_batch[1].to(
device)
x_fake = G(torch.cat([z_fake, y_fake], dim=1)).detach()
fake_sample = torch.cat([x_fake, y_fake], dim=1)
else:
x_real = data_batch.to(device)
real_sample = x_real
z_fake = latent_batch.to(device)
x_fake = G(z_fake).detach()
fake_sample = x_fake
D.zero_grad()
real_pred = D(real_sample)
fake_pred = D(fake_sample)
if is_recorded(data_sampler):
data_sampler.record(real_pred.detach().cpu().numpy())
if is_weighted(data_sampler):
weights = torch.tensor(
data_sampler.get_weights()).to(device).float().view(
real_pred.shape)
else:
weights = torch.ones_like(real_pred).to(device)
if objective == 'gan':
D_loss = F.binary_cross_entropy(fake_pred, fake_target).mean() + (
weights *
F.binary_cross_entropy(real_pred, real_target)).mean()
stats['D'].append(D_loss.item())
elif objective == 'wgan':
alpha = torch.rand(batch_size,
1).expand(real_sample.size()).to(device)
interpolate = (alpha * real_sample +
(1 - alpha) * fake_sample).requires_grad_(True)
gradients = torch.autograd.grad(outputs=D(interpolate),
inputs=interpolate,
grad_outputs=grad_target,
create_graph=True,
retain_graph=True,
only_inputs=True)[0]
gradient_penalty = (gradients.norm(2, dim=1) -
1).pow(2).mean() * gp_lambda
D_loss = fake_pred.mean() - (real_pred * weights).mean()
stats['D'].append(-D_loss.item())
D_loss += gradient_penalty
elif objective == 'hinge':
D_loss = -(torch.min(real_pred - sub, bound) *
weights).mean() - torch.min(-fake_pred - sub,
bound).mean()
stats['D'].append(D_loss.item())
D_loss.backward()
D_optimizer.step()
# Train Generator
if it % n_critic == 0:
G.zero_grad()
latent_batch = noise_sampler.get_batch(batch_size)
if conditional:
z_fake, y_fake = latent_batch[0].to(
device), latent_batch[1].to(device)
x_fake = G(torch.cat([z_fake, y_fake], dim=1))
fake_pred = D(torch.cat([x_fake, y_fake], dim=1))
else:
z_fake = latent_batch.to(device)
x_fake = G(z_fake)
fake_pred = D(x_fake)
if objective == 'gan':
G_loss = F.binary_cross_entropy(fake_pred, real_target).mean()
stats['G'].extend([G_loss.item()] * n_critic)
elif objective == 'wgan':
G_loss = -fake_pred.mean()
stats['G'].extend([-G_loss.item()] * n_critic)
elif objective == 'hinge':
G_loss = -fake_pred.mean()
stats['G'].extend([-G_loss.item()] * n_critic)
G_loss.backward()
G_optimizer.step()
if conditional:
# Train fake classifier
C_fake.train()
C_fake.zero_grad()
C_fake_loss = C_crit(C_fake(x_fake.detach()), y_fake.argmax(1))
C_fake_loss.backward()
C_fake_optimizer.step()
# Train real classifier
C_real.train()
C_real.zero_grad()
C_real_loss = C_crit(C_real(x_real), y_real.argmax(1))
C_real_loss.backward()
C_real_optimizer.step()
if it % 5 == 0:
C_real.eval()
C_fake.eval()
real_correct, fake_correct, total = 0.0, 0.0, 0.0
for idx, (sample, label) in enumerate(test_dataloader):
real_correct += (
C_real(sample).argmax(1).view(-1) == label).sum()
fake_correct += (
C_fake(sample).argmax(1).view(-1) == label).sum()
total += sample.shape[0]
stats['C_it'].append(it)
stats['C_real'].append(real_correct.item() / total)
stats['C_fake'].append(fake_correct.item() / total)
line = f"{it}\t{stats['D'][-1]:.3f}\t{stats['G'][-1]:.3f}"
if conditional:
line += f"\t{stats['C_real'][-1]*100:.3f}\t{stats['C_fake'][-1]*100:.3f}"
print(line, eval_file)
if plot:
if conditional:
z_fake, y_fake = fixed_latent_batch[0].to(
device), fixed_latent_batch[1].to(device)
x_fake = G(torch.cat([z_fake, y_fake], dim=1))
else:
z_fake = fixed_latent_batch.to(device)
x_fake = G(z_fake)
generated = x_fake.detach().cpu().numpy()
plt.figure(0)
plt.clf()
plt.scatter(generated[:, 0],
generated[:, 1],
marker='.',
color=(0, 1, 0, 0.01))
plt.axis('equal')
plt.xlim(-1, 1)
plt.ylim(-1, 1)
plt.savefig(os.path.join(samples_dir, f'{it}.png'))
plt.figure(1)
plt.clf()
plt.plot(stats['G'], label='Generator')
plt.plot(stats['D'], label='Discriminator')
plt.legend()
plt.savefig(os.path.join(results_dir, 'loss.png'))
if conditional:
plt.figure(2)
plt.clf()
plt.plot(stats['C_it'], stats['C_real'], label='Real')
plt.plot(stats['C_it'], stats['C_fake'], label='Fake')
plt.legend()
plt.savefig(os.path.join(results_dir, 'accuracy.png'))
save_model(G, os.path.join(network_dir, 'G_trained.pth'))
save_model(D, os.path.join(network_dir, 'D_trained.pth'))
save_stats(stats, os.path.join(results_dir, 'stats.pth'))
if conditional:
save_model(C_real, os.path.join(network_dir, 'C_real_trained.pth'))
save_model(C_fake, os.path.join(network_dir, 'C_fake_trained.pth'))
eval_file.close()
