def __init__(self, config):
self.params = config
self.device = device
self.word2index, self.index2word, self.embeddings = pickle.load(open(config.data_pickle, 'rb'))
train_dataset = Loader(config, config.p_train_data)
val_dataset = Loader(config, config.p_val_data)
test_dataset = Loader(config, config.p_test_data)
self.model = Model(config, self.embeddings).to(self.device)
self.train_loader = DataLoader(dataset=train_dataset, batch_size=config.batch_size, shuffle=True)
self.val_loader = DataLoader(dataset=val_dataset, batch_size=config.batch_size, shuffle=False)
self.test_loader = DataLoader(dataset=test_dataset, batch_size=config.batch_size, shuffle=False)
params = filter(lambda param: param.requires_grad, self.model.parameters())
self.optimizer = torch.optim.Adam(lr=config.learning_rate, betas=(config.beta1, config.beta2), eps=1e-7, weight_decay=3e-7, params=params)
# lr = config.learning_rate
# base_lr = 1.0
# params = filter(lambda param: param.requires_grad, slef.model.parameters())
# optimizer = torch.optim.Adam(lr=base_lr, betas=(config.beta1, config.beta2), eps=1e-7, weight_decay=3e-7, params=params)
# cr = lr / math.log2(config.lr_warm_up_num)
# scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer,
# lr_lambda=lambda ee: cr * math.log2(ee + 1) if ee < config.lr_warm_up_num else lr)
self.model_path = os.path.join(self.params.cache_dir)
if not os.path.exists(self.model_path):
print('create path: ', self.model_path)
os.makedirs(self.model_path)
self.best_model = None
self.lr_epoch = 0
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--pmb")
parser.add_argument("--sick")
parser.add_argument("--sick2pd")
parser.add_argument("--out")
args = parser.parse_args()
# Final model
data = Loader.load_data("../NLI2FOLI/SICK/SICK_train.txt")
data = data.append(Loader.load_data("../NLI2FOLI/SICK/SICK_trial.txt"))
test = Loader.load_data(args.sick)
data["postags"] = FeatureExtractor.postag_tokenizer(data["tokens"])
test["postags"] = FeatureExtractor.postag_tokenizer(test["tokens"])
# The countvectorizer features with postagging appended to each token
bag_of_words_plus_pos = ColumnTransformer([
("POS", CountVectorizer(tokenizer=lambda x: x,
preprocessor=lambda x: x), "postags")
])
m = Model(data)
m.add_feature(bag_of_words_plus_pos, "Feature")
m.train_model(
RandomForestClassifier(n_estimators=900,
criterion="entropy",
max_depth=729))
with open('model.pkl', 'wb') as fid:
pickle.dump(m, fid)
with open('model.pkl', 'rb') as fid:
m = pickle.load(fid)
try:
m.test_model(test, test["entailment_judgment"])
labels = m.model.classes_
cm = confusion_matrix(test["entailment_judgment"], m.prediction)
print_cm(cm, labels)
except:
m.test_model(test)
with open(args.out, "w") as file:
for idx, pred in enumerate(m.prediction):
pid = test.iloc[idx]['pair_ID']
file.write("{}:{}\n".format(pid, pred))
print("Successfully generated prediction on test data.")
def makecsv(file, model, loadfile):
cuda = False
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
data_loader = Loader(c.FILE_TRAIN_LABELED_AUG, c.FILE_TRAIN_UNLABELED, c.FILE_TEST, 'data/test-labeled.p', kwargs)
test_loader = data_loader.getTest()
test_actual = data_loader.getValidation()
label_predict = np.array([])
mnist_model = model
if loadfile:
mnist_model = torch.load(model)
correct = 0
for data, target in test_loader:
mnist_model.eval()
data, target = Variable(data, volatile=True), Variable(target)
output = mnist_model(data)
temp = output.data.max(1)[1]
pred = output.data.max(1)[1]
correct += pred.eq(target.data).cpu().sum()
label_predict = np.concatenate((label_predict, temp.numpy().reshape(-1)))
print('\nTest set: Accuracy: {}/{} ({:.0f}%)\n'.format(correct, len(test_loader.dataset),
100. * correct / len(test_loader.dataset)))
predict_label = pd.DataFrame(label_predict, columns=['label'], dtype=int)
predict_label.reset_index(inplace=True)
predict_label.rename(columns={'index': 'ID'}, inplace=True)
filename = 'predictions/' + file + "-labeled.csv"
predict_label.to_csv(filename, index=False)
label_predict = np.array([])
correct = 0
for data, target in test_actual:
mnist_model.eval()
data, target = Variable(data, volatile=True), Variable(target)
output = mnist_model(data)
temp = output.data.max(1)[1]
pred = output.data.max(1)[1] # get the index of the max log-probability
correct += pred.eq(target.data).cpu().sum()
label_predict = np.concatenate((label_predict, temp.numpy().reshape(-1)))
print('\nTest set: Accuracy: {}/{} ({:.0f}%)\n'.format(correct, len(test_loader.dataset),
100. * correct / len(test_loader.dataset)))
predict_label = pd.DataFrame(label_predict, columns=['label'], dtype=int)
predict_label.reset_index(inplace=True)
predict_label.rename(columns={'index': 'ID'}, inplace=True)
filename = 'predictions/' + file + "-unlabeled.csv"
predict_label.to_csv(filename, index=False)
def getDataLoaderDict(idxModel): # idxModel is tuple
print("\nStarted Preprocessing")
trainDataLoaderDict = {}
testDataLoaderDict = {}
for idx in idxModel:
print("-Preprocessing model{} dataloader".format(idx))
loader = Loader(idx)
trainLoader, testLoader = loader.getLoader()
trainDataLoaderDict[idx] = trainLoader
testDataLoaderDict[idx] = testLoader
print("-Completed Preprocessing model{} dataloader\n".format(idx))
print("All Completed Preprocessing\n")
return trainDataLoaderDict, testDataLoaderDict
def main(_):
if config.mode == "train":
trainer = Trainer(config)
trainer.train()
elif config.mode == "preprocess":
dataloader.preprocess(_)
elif config.mode == "debug":
trainer = Trainer(config)
train_dataset = Loader(config, config.p_train_data)
train_loader = DataLoader(dataset=train_dataset, batch_size=2, shuffle=False)
data_iter = iter(train_loader)
frame_vecs, frame_n, ques, ques_n, start_frame, end_frame = data_iter.next()
frame_vecs = frame_vecs.to(trainer.device)
ques = ques.to(trainer.device)
# Forward pass
p1, p2 = trainer.model(frame_vecs, ques)
y1, y2 = start_frame.to(trainer.device), end_frame.to(trainer.device)
print(p1.shape,p2.shape,y1.shape,y2.shape)
loss1 = F.nll_loss(p1, y1, reduction='elementwise_mean')
loss2 = F.nll_loss(p2, y2, reduction='elementwise_mean')
loss = (loss1 + loss2) / 2
# Backward and optimize
trainer.optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_value_(trainer.model.parameters(), 1)
trainer.optimizer.step()
print(loss.item())
elif config.mode == "test":
trainer = Trainer(config)
trainer.evaluate()
else:
print("Unknown mode")
exit(0)
def main(_):
print 'begin loading data...'
loader = Loader.loader(FLAGS.mincount, FLAGS.maxlen)
print 'finished loading data\n'
print 'begin building model...'
model = Model(
loader,
embedding_size = FLAGS.embedding_size,
batch_size = FLAGS.batch_size,
max_epoch = FLAGS.max_epoch,
learning_rate = FLAGS.learning_rate,
keep_prob = FLAGS.keep_prob
)
print 'finished building model\n'
if FLAGS.retrain:
model.train()
test_accuracy = model.test()
else:
try:
model.load('save', 'best')
test_accuracy = model.test()
except tf.errors.NotFoundError:
model.train()
test_accuracy = model.test()
except Exception, e:
print 'Error may be caused by changing parameters without retraining. Try setting --retrain to True.'
raise e
def popularity_ratio(pop_model: np.ndarray, pop_model_user: np.ndarray,
dataset: Loader):
"""calculate the degree of the "long-tailness" for a distribution
Args:
pop_model (ndarray): the freq of items recommended by model
pop_model_user (ndarray): (user X topk) the recommend list of users
dataset (dataloader.Loader): the freq of items in dataset
Returns:
dict: {"I_ratio""float, "I_KL":float, "I_gini":float, "APT":[], "I_bin": float}
"""
pop_dataset, _ = dataset.popularity()
assert len(pop_model) == len(pop_dataset)
num_item = len(pop_dataset)
num_interaction = pop_model.sum()
metrics = {}
prop_model = pop_model / num_interaction
prop_uniform = 1. / num_item
prop_dataset = pop_dataset / pop_dataset.sum()
# print("dataset KL",
# np.sum(prop_dataset * np.log(prop_dataset / prop_uniform + 1e-7)))
metrics['I_KL'] = np.sum(prop_model *
np.log(prop_model / prop_uniform + 1e-7))
mapping = map_item_three(dataset.popularity()[0])
mapping_N = map_item_N((dataset.popularity()[0]),
[0.1, 0.1, 0.3, 0.3, 0.2])
metrics['APT'] = APT(pop_model_user, mapping)
metrics['APT5'] = APT(pop_model_user, mapping_N)
# print("dataset APT", APT(dataset.allPos, mapping))
# print("dataset bin", np.sum(pop_dataset/pop_dataset.max()))
metrics['I_bin'] = np.sum(pop_model / np.max(pop_model))
return metrics
def main(_):
print 'begin loading data...'
loader = Loader(FLAGS.mincount, FLAGS.maxlen, full_train=False)
print 'finished loading data\n'
print 'begin building model...'
# dataloader, embedding_size, max_epoch, learning_rate, keep_prob
model = Model(loader,
embedding_size=FLAGS.embedding_size,
max_epoch=FLAGS.max_epoch,
learning_rate=FLAGS.learning_rate,
keep_prob=FLAGS.keep_prob)
print 'finished building model\n'
# model.train(full_train=True)
# model.save_doc_vector()
model.many_test()
def main():
parser = Parser()
config = parser.config
for param, value in config.__dict__.items():
print(param + '.' * (50 - len(param) - len(str(value))) + str(value))
print()
# Load previous checkpoint if it exists
checkpoint = load_latest(config)
# Create model
model = load_model(config, checkpoint)
# print number of parameters in the model
n_params = sum([param.view(-1).size()[0] for param in model.parameters()])
print('Total number of parameters: \33[91m{}\033[0m'.format(n_params))
# Load train and test data
train_loader, valid_loader, test_loader = Loader(config)
n_batches = int(len(train_loader.dataset.train_data) / config.batch_size)
# save the configuration
with open(os.path.join(config.save, 'log.txt'), 'w') as file:
json.dump('json_stats: ' + str(config.__dict__), file)
# Instantiate the criterion, optimizer and learning rate scheduler
criterion = torch.nn.CrossEntropyLoss(size_average=True)
optimizer = torch.optim.SGD(model.parameters(),
lr=config.LR,
momentum=config.momentum,
weight_decay=config.weight_decay,
nesterov=config.nesterov)
start_time = 0
if checkpoint is not None:
start_epoch = checkpoint['time'] + 1
optimizer.load_state_dict(checkpoint['optimizer'])
if config.lr_shape == 'multistep':
scheduler = MultiStepLR(optimizer, milestones=[81, 122], gamma=0.1)
elif config.lr_shape == 'cosine':
if checkpoint is not None:
scheduler = checkpoint['scheduler']
else:
scheduler = CosineAnnealingRestartsLR(optimizer,
1,
config.T_e,
T_mul=config.T_mul)
# The trainer handles the training loop and evaluation on validation set
trainer = Trainer(model, criterion, config, optimizer, scheduler)
epoch = 1
while True:
# Train for a single epoch
train_top1, train_loss, stop_training = trainer.train(
epoch, train_loader)
# Run model on the validation and test set
valid_top1 = trainer.evaluate(epoch, valid_loader, 'valid')
test_top1 = trainer.evaluate(epoch, test_loader, 'test')
current_time = time.time()
results = {
'epoch': epoch,
'time': current_time,
'train_top1': train_top1,
'valid_top1': valid_top1,
'test_top1': test_top1,
'train_loss': float(train_loss.data),
}
with open(os.path.join(config.save, 'results.txt'), 'w') as file:
json.dump(str(results), file)
file.write('\n')
print(
'==> Finished epoch %d (budget %.3f): %7.3f (train) %7.3f (validation) %7.3f (test)'
% (epoch, config.budget, train_top1, valid_top1, test_top1))
if stop_training:
break
epoch += 1
if start_time >= config.budget:
trainer.evaluate(epoch, test_loader, 'test')
else:
save_checkpoint(int(config.budget), trainer.model, trainer.optimizer,
trainer.scheduler, config)
def test():
data = Loader.load_data("../NLI2FOLI/SICK/SICK_train.txt")
test = Loader.load_data("../NLI2FOLI/SICK/SICK_trial.txt")
encoder = FeatureExtractor.generate_postag_onehot(data["tokens"])
data["postags"] = FeatureExtractor.postag_tokenizer(data["tokens"])
test["postags"] = FeatureExtractor.postag_tokenizer(test["tokens"])
data["antons"] = FeatureExtractor.antonym_relations(data["pair_ID"])
test["antons"] = FeatureExtractor.antonym_relations(test["pair_ID"])
data["synons"] = FeatureExtractor.synonym_relations(
data["tokens"], data["pair_ID"])
test["synons"] = FeatureExtractor.synonym_relations(
test["tokens"], test["pair_ID"])
# Features
# You don't have to do anything here
# These are all the features we have
# The tfidf features with both sentences seperated
bag_of_words = ColumnTransformer([("A", TfidfVectorizer(), "sentence_A"),
("B", TfidfVectorizer(), "sentence_B")])
# The countvectorizer features with postagging appended to each token
bag_of_words_plus_pos = ColumnTransformer([
("POS", CountVectorizer(tokenizer=lambda x: x,
preprocessor=lambda x: x), "postags")
])
# The One-hot-encoded postag features for both sentences
postags_A = POSTAGTransformer(encoder, "sentence_A", maxlen=800)
postags_B = POSTAGTransformer(encoder, "sentence_B", maxlen=800)
# The negation features for both sentences
negation_A = NEGTransformer("sentence_A")
negation_B = NEGTransformer("sentence_B")
# The antonyms and synonyms features
antons = DumbTransfromer("antons")
synons = DumbTransfromer("synons")
# classifiers
# Every classifiers should be put in a tuple with its name on the right hand side
# You may tweak the hyperparameters
nb = (MultinomialNB(alpha=0.1), "Naive Bayes")
knn = (KNeighborsClassifier(), "KNN")
svm = (SVC(kernel="linear", C=0.7), "SVM")
forest = (RandomForestClassifier(n_estimators=1000,
max_depth=128), "Random Forest")
mlp = (MLPClassifier(1000), "Multi layer Perceptrons")
classifiers = [nb, knn, svm, forest, mlp]
# for n in range(5, 10):
# hyperforest = (RandomForestClassifier(
# n_estimators=n*100, max_depth=n**3), f"Random Forest with {n*100} of estimators and a max depth of {n**3}")
# classifiers.append(hyperforest)
# Feature_combs
# Feature combinations are a list of tuples
# Give the combination a name on the right hand side
combs = [([bag_of_words], "TFIDF"),
([bag_of_words_plus_pos], "Combined + Postagging"),
([bag_of_words, postags_A, postags_B], "TFIDF + One hot postags"),
([postags_A, postags_B], "OneHotPosTag only"),
([bag_of_words, negation_A, negation_B], "TFIDF + NEGATION"),
([negation_A, negation_B], "NEGATION_ONLY"),
([bag_of_words, antons, synons], "TFIDF + ANTONYMS + SYNONYMS"),
([bag_of_words, antons], "TFIDF + ANTONYMS"),
([bag_of_words, synons], "TFIDF + SYNONYMS"),
([
bag_of_words, postags_B, postags_A, negation_A, negation_B,
antons, synons
], "All features")]
# The seach function takes the combination, classifier list and the train test data.
# The result will be printed and exported to a csv file called search result
# It also returns the dictionary of all the accuracys
search(combs, classifiers, data, test)
# read word2vec
with open('/home/student/glove/glove.6B.100d.txt', 'r') as f:
lines = f.readlines()
dictionary = {}
weight_matrix = []
for i, line in enumerate(lines):
word = line.split()
dictionary[word[0]] = i
weight_matrix.append([float(word[i]) for i in range(1, len(word))])
weight_matrix = np.array(weight_matrix)
# ------------------------------------------------------------------------------- #
# read data
train_data = Loader(0, dictionary)
valid_data = Loader(1, dictionary)
test_data = Loader(2, dictionary)
train_data = DataLoader(train_data, batch_size=128, shuffle=True)
valid_data = DataLoader(valid_data, batch_size=128, shuffle=False)
test_data = DataLoader(test_data, batch_size=128, shuffle=False)
# ------------------------------------------------------------------------------ #
model = SentenceCompression(100, 200, 50, 2, weight_matrix,
len(dictionary)).cuda()
lr = 0.0005
num_epoch = 20
criterion = nn.CrossEntropyLoss().cuda()
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=lr,
batch_size = 2
device = '2'
os.environ['CUDA_VISIBLE_DEVICES'] = device
#main
if __name__ == '__main__':
#create model
model = Model()
model.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr)
loss = torch.nn.MSELoss()
visdom_server = Visdom(port=8097)
#create dataloade
train_set = Loader(mode='train')
val_set = Loader(mode='test')
data_loader = torch.utils.data.DataLoader(train_set,
batch_size=batch_size,
shuffle=True,
num_workers=1)
val_data_loader = torch.utils.data.DataLoader(val_set,
batch_size=batch_size,
shuffle=True,
num_workers=1)
#training
print(f'The number of training data {len(data_loader)}')
print(f'The number of testing data {len(val_data_loader)}')
for epoch in range(epochs):
for result in pre_results:
results['recall'] += result['recall']
results['precision'] += result['precision']
results['ndcg'] += result['ndcg']
results['recall'] /= float(len(users))
results['precision'] /= float(len(users))
results['ndcg'] /= float(len(users))
# results['auc'] = np.mean(auc_record)
if multicore == 1:
pool.close()
print(results)
return results
def cprint(words : str):
print(f"\033[0;30;43m{words}\033[0m")
if __name__ == '__main__':
train_dataset = Loader(path=world.dataset)
Recmodel = NGCF(world.config, train_dataset)
Neg_k = 1
bpr=BPRLoss(Recmodel, world.config)
f = open (f'train_logger_{world.dataset}.txt','w')
f_test = open (f'test_logger_{world.dataset}.txt','w')
for epoch in range(world.TRAIN_epochs):
if epoch %10 == 0:
cprint("[TEST]")
result = Test(train_dataset, Recmodel, epoch, world.config['multicore'])
print(epoch, result, file=f_test, flush=True)
output_information = BPR_train_original(train_dataset, Recmodel, bpr, epoch, neg_k=Neg_k,w=None)
log_output = f'EPOCH[{epoch+1}/{world.TRAIN_epochs}] {output_information}'
print(f'EPOCH[{epoch+1}/{world.TRAIN_epochs}] {output_information}')
print(log_output, file=f, flush=True)
f.close()
import scipy.sparse as sp
import numpy as np
from dataloader import Loader
from scipy.sparse import csr_matrix
loader = Loader()
R = loader.UserItemNet.tolil()
twohop = 0.004
adj_user = R.T.todok()
print("generating 2")
rowsum_user = np.array(adj_user.sum(axis=0))
D_user = np.power(rowsum_user, -0.5).flatten()
D_user[np.isinf(D_user)] = 0
Dmat_user = sp.diags(D_user)
print("generating 3")
adj_item = R.todok()
print("generating 4")
rowsum_item = np.array(adj_item.sum(axis=0))
D_item = np.power(rowsum_item, -0.5).flatten()
D_item[np.isinf(D_item)] = 0
Dmat_item = sp.diags(D_item)
print("generating 5")
norm_user = Dmat_item.dot(adj_user).dot(Dmat_user)
norm_item = Dmat_user.dot(adj_item).dot(Dmat_item)
def sparsify_propagation(adj, hop_thres):
adj_valid = (adj > hop_thres)
from yamlparams.utils import Hparam
import sys
from metrics import mean_average_presision_k, hitrate_k, novelty, coverage
from dataloader import Loader
from preprocessor import Preprocessor
from model import TopPopular
# Read config
if len(sys.argv) < 2:
raise AttributeError('Use config name to define model config')
cfg_path = sys.argv[1] #'books_big_setting.yml'
print('Using config ' + cfg_path)
config = Hparam(cfg_path)
loader = Loader(config.path)
preprocessor = Preprocessor()
print('Reading data')
df = loader.get_views()
print('Preprocessing')
# train_df = preprocessor.filter_zeros(train_df)
df = preprocessor.filter_lazy_users(df, 0)
train_df, test_df = loader.split_train_test(df, config.min_user_views,
config.testing.samples)
train_df, test_df = preprocessor.filter_not_in_test_items(train_df, test_df)
preprocessor.build_mappers(train_df.append(test_df))
train_df.user_id = train_df.user_id.apply(preprocessor.get_user_ix)
"word2vec": "data/word2vec.bin",
"is_origin_dataset": True,
"train_json": "data/activity-net/train.json",
"val_json": "data/activity-net/val_1.json",
"test_json": "data/activity-net/val_2.json",
"train_data": "data/activity-net/train_data.json",
"val_data": "data/activity-net/val_data.json",
"test_data": "data/activity-net/test_data.json",
"feature_path": "data/activity-c3d",
"feature_path_tsn": "data/tsn_score"
}
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
word2vec = KeyedVectors.load_word2vec_format(config["word2vec"],
binary=True)
train_dataset = Loader(config, config['train_data'], word2vec, flag=True)
val_dataset = Loader(config, config['val_data'], word2vec)
test_dataset = Loader(config, config['test_data'], word2vec)
train_loader = DataLoader(dataset=train_dataset,
batch_size=64,
shuffle=True)
val_loader = DataLoader(dataset=val_dataset, batch_size=64, shuffle=False)
test_loader = DataLoader(dataset=test_dataset,
batch_size=64,
shuffle=False)
d_model = config['d_model']
d_ff = config['d_ff']
num_heads = config['num_heads']
class cyclegan(object):
def __init__(self, sess):
self.sess = sess
def read_data(self):
self.loader = Loader()
self.loader.read_data()
def load_batch(self, num, type):
return self.loader.load_batch(num, type)
def load_data_tr(self):
return self.loader.load_dataset_tr()
def load_data_tst(self):
return self.loader.load_data_tst()
def set_hyperparam(self, args):
self.batch_size = args.batch_size
self.image_size = args.fine_size
self.input_c_dim = args.input_nc
self.output_c_dim = args.output_nc
self.L1_lambda = args.L1_lambda
self.dataset_dir = args.dataset_dir
if args.discrim_model == 'disc':
self.discriminator = disc
elif args.discrim_model == 'disc_my':
self.discriminator = disc_my
# if args.use_resnet:
# self.generator = generator_resnet
if args.gen_model == 'res':
self.generator = generator_res
elif args.gen_model == 'res_2':
self.generator = generator_res_2
else:
self.generator = generator_unet
if args.use_lsgan:
self.criterionGAN = mae_criterion
else:
self.criterionGAN = sce_criterion
OPTIONS = namedtuple(
'OPTIONS',
'batch_size image_size gf_dim df_dim output_c_dim is_training')
self.options = OPTIONS._make(
(args.batch_size, args.fine_size, args.ngf, args.ndf,
args.output_nc, args.phase == 'train'))
self._build_model()
self.saver = tf.train.Saver()
self.pool = ImagePool(args.max_size)
def _build_model(self):
self.real_data = tf.placeholder(tf.float32, [
None, self.image_size, self.image_size, self.image_size,
self.input_c_dim + self.input_c_dim
],
name='real_A_and_B_images')
print('real data shape : ', self.real_data.shape)
self.real_A = self.real_data[:, :, :, :, :self.input_c_dim]
self.real_B = self.real_data[:, :, :, :,
self.input_c_dim:self.input_c_dim +
self.output_c_dim]
print('real data A and B shape : ', self.real_A.shape,
self.real_B.shape)
self.fake_B = self.generator(self.real_A,
self.options,
False,
name="generatorA2B")
self.fake_A_ = self.generator(self.fake_B,
self.options,
False,
name="generatorB2A")
self.fake_A = self.generator(self.real_B,
self.options,
True,
name="generatorB2A")
self.fake_B_ = self.generator(self.fake_A,
self.options,
True,
name="generatorA2B")
print('fake data A and B shape : ', self.fake_A.shape,
self.fake_B.shape)
print('fake data A_ and B_ shape : ', self.fake_A_.shape,
self.fake_B_.shape)
self.DB_fake = self.discriminator(self.fake_B,
self.options,
reuse=False,
name="discriminatorB")
self.DA_fake = self.discriminator(self.fake_A,
self.options,
reuse=False,
name="discriminatorA")
self.g_loss_a2b = self.criterionGAN(self.DB_fake, tf.ones_like(self.DB_fake)) \
+ self.L1_lambda * abs_criterion(self.real_A, self.fake_A_) \
+ self.L1_lambda * abs_criterion(self.real_B, self.fake_B_)
self.g_loss_b2a = self.criterionGAN(self.DA_fake, tf.ones_like(self.DA_fake)) \
+ self.L1_lambda * abs_criterion(self.real_A, self.fake_A_) \
+ self.L1_lambda * abs_criterion(self.real_B, self.fake_B_)
self.g_loss = self.criterionGAN(self.DA_fake, tf.ones_like(self.DA_fake)) \
+ self.criterionGAN(self.DB_fake, tf.ones_like(self.DB_fake)) \
+ self.L1_lambda * abs_criterion(self.real_A, self.fake_A_) \
+ self.L1_lambda * abs_criterion(self.real_B, self.fake_B_)
print('g loss is builded')
# why sample is bellow ???
self.fake_A_sample = tf.placeholder(tf.float32, [
None, self.image_size, self.image_size, self.image_size,
self.input_c_dim
],
name='fake_A_sample')
self.fake_B_sample = tf.placeholder(tf.float32, [
None, self.image_size, self.image_size, self.image_size,
self.output_c_dim
],
name='fake_B_sample')
self.DB_real = self.discriminator(self.real_B,
self.options,
reuse=True,
name="discriminatorB")
self.DA_real = self.discriminator(self.real_A,
self.options,
reuse=True,
name="discriminatorA")
self.DB_fake_sample = self.discriminator(self.fake_B_sample,
self.options,
reuse=True,
name="discriminatorB")
self.DA_fake_sample = self.discriminator(self.fake_A_sample,
self.options,
reuse=True,
name="discriminatorA")
print('discriminating sample is builded')
self.db_loss_real = self.criterionGAN(self.DB_real,
tf.ones_like(self.DB_real))
self.db_loss_fake = self.criterionGAN(
self.DB_fake_sample, tf.zeros_like(self.DB_fake_sample))
self.db_loss = (self.db_loss_real + self.db_loss_fake) / 2
self.da_loss_real = self.criterionGAN(self.DA_real,
tf.ones_like(self.DA_real))
self.da_loss_fake = self.criterionGAN(
self.DA_fake_sample, tf.zeros_like(self.DA_fake_sample))
self.da_loss = (self.da_loss_real + self.da_loss_fake) / 2
self.d_loss = self.da_loss + self.db_loss
self.g_loss_a2b_sum = tf.summary.scalar("g_loss_a2b", self.g_loss_a2b)
self.g_loss_b2a_sum = tf.summary.scalar("g_loss_b2a", self.g_loss_b2a)
self.g_loss_sum = tf.summary.scalar("g_loss", self.g_loss)
self.g_sum = tf.summary.merge(
[self.g_loss_a2b_sum, self.g_loss_b2a_sum, self.g_loss_sum])
self.db_loss_sum = tf.summary.scalar("db_loss", self.db_loss)
self.da_loss_sum = tf.summary.scalar("da_loss", self.da_loss)
self.d_loss_sum = tf.summary.scalar("d_loss", self.d_loss)
self.db_loss_real_sum = tf.summary.scalar("db_loss_real",
self.db_loss_real)
self.db_loss_fake_sum = tf.summary.scalar("db_loss_fake",
self.db_loss_fake)
self.da_loss_real_sum = tf.summary.scalar("da_loss_real",
self.da_loss_real)
self.da_loss_fake_sum = tf.summary.scalar("da_loss_fake",
self.da_loss_fake)
self.d_sum = tf.summary.merge([
self.da_loss_sum, self.da_loss_real_sum, self.da_loss_fake_sum,
self.db_loss_sum, self.db_loss_real_sum, self.db_loss_fake_sum,
self.d_loss_sum
])
self.test_A = tf.placeholder(tf.float32, [
None, self.image_size, self.image_size, self.image_size,
self.input_c_dim
],
name='test_A')
self.test_B = tf.placeholder(tf.float32, [
None, self.image_size, self.image_size, self.image_size,
self.output_c_dim
],
name='test_B')
self.testB = self.generator(self.test_A,
self.options,
True,
name="generatorA2B")
self.testA = self.generator(self.test_B,
self.options,
True,
name="generatorB2A")
t_vars = tf.trainable_variables()
self.d_vars = [var for var in t_vars if 'discriminator' in var.name]
self.g_vars = [var for var in t_vars if 'generator' in var.name]
for var in t_vars:
print(var.name)
def train(self, args):
"""Train cyclegan"""
self.lr = tf.placeholder(tf.float32, None, name='learning_rate')
self.d_optim = tf.train.AdamOptimizer(
self.lr, beta1=args.beta1).minimize(self.d_loss,
var_list=self.d_vars)
self.g_optim = tf.train.AdamOptimizer(
self.lr, beta1=args.beta1).minimize(self.g_loss,
var_list=self.g_vars)
init_op = tf.global_variables_initializer()
self.sess.run(init_op)
self.writer = tf.summary.FileWriter("./logs", self.sess.graph)
counter = 1
start_time = time.time()
if args.continue_train:
if self.load(args.checkpoint_dir):
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
# dataset = self.load_data_tr()
for epoch in range(args.epoch):
# dataA, dataB = self.load_data_tr()
# dataset = [dataA, dataB]
# np.random.shuffle(dataset)
# batch_idxs = min(len(dataset), args.train_size) // self.batch_size
lr = args.lr if epoch < args.epoch_step else args.lr * (
args.epoch - epoch) / (args.epoch - args.epoch_step)
batch, meta = self.load_batch(self.batch_size, type='train')
batch_idxs = len(batch)
for idx in range(0, batch_idxs):
# batch_images = dataset[idx*self.batch_size:(idx+1)*self.batch_size]
batch_images = batch
# batch_files = list(zip(dataA[idx * self.batch_size:(idx + 1) * self.batch_size],
# dataB[idx * self.batch_size:(idx + 1) * self.batch_size]))
# batch_images = [load_train_data(batch_file, args.load_size, args.fine_size) for batch_file in batch_files]
# batch_images = np.array(batch_images).astype(np.float32)
# batch_images = batch_files[batch_idxs]
# batch_images = np.array(batch_images).astype(np.float32)
# Update G network and record fake outputs
fake_A, fake_B, _, summary_str, g_loss = self.sess.run(
[
self.fake_A, self.fake_B, self.g_optim, self.g_sum,
self.g_loss
],
feed_dict={
self.real_data: batch_images,
self.lr: lr
})
self.writer.add_summary(summary_str, counter)
[fake_A, fake_B] = self.pool([fake_A, fake_B])
# Update D network
_, summary_str, d_loss = self.sess.run(
[self.d_optim, self.d_sum, self.d_loss],
feed_dict={
self.real_data: batch_images,
self.fake_A_sample: fake_A,
self.fake_B_sample: fake_B,
self.lr: lr
})
self.writer.add_summary(summary_str, counter)
counter += 1
print(
"Epoch: [%2d] [%4d/%4d] time: %4.4f | g loss : %8d | d loss : %4d"
% (epoch, idx, batch_idxs, time.time() - start_time,
g_loss, d_loss))
if np.mod(counter, args.print_freq) == args.print_freq - 1:
print('save sample model at {}'.format((counter)))
self.sample_model(args.sample_dir, epoch, idx)
if np.mod(counter, args.save_freq) == args.save_freq - 1:
print('save in checkpoint directory at {}'.format(counter))
self.save(args.checkpoint_dir, counter)
def save(self, checkpoint_dir, step):
model_name = "cyclegan.model"
model_dir = "%s_%s" % (self.dataset_dir, self.image_size)
checkpoint_dir = os.path.join(checkpoint_dir, model_dir)
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
self.saver.save(self.sess,
os.path.join(checkpoint_dir, model_name),
global_step=step)
def load(self, checkpoint_dir):
print(" [*] Reading checkpoint...")
model_dir = "%s_%s" % (self.dataset_dir, self.image_size)
checkpoint_dir = os.path.join(checkpoint_dir, model_dir)
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
ckpt_name = os.path.basename(ckpt.model_checkpoint_path)
self.saver.restore(self.sess,
os.path.join(checkpoint_dir, ckpt_name))
return True
else:
return False
def sample_model(self, sample_dir, epoch, idx):
batch, meta = self.load_batch(self.batch_size, type='train')
sample_image = batch[0]
# batch_files = list(zip(dataA[:self.batch_size], dataB[:self.batch_size]))
# sample_images = [load_train_data(batch_file, is_testing=True) for batch_file in batch_files]
# sample_images = np.array(sample_images).astype(np.float32)
# sample_image = np.reshape(sample_image, [1,256,256,256,2])
sample_image = np.expand_dims(sample_image, axis=0) #[1,256,256,256,2]
print('sample image shape: {}'.format(sample_image.shape))
fake_A, fake_B = self.sess.run(
[self.fake_A, self.fake_B],
feed_dict={self.real_data: sample_image})
real_A = sample_image[:, :, :, :, :self.input_c_dim]
real_B = sample_image[:, :, :, :, self.input_c_dim:self.input_c_dim +
self.input_c_dim]
print('real image shape: {}'.format(real_A.shape))
print('fake image shape: {}'.format(fake_A.shape))
save_images(real_A,
'./{}/A_real{:02d}_{:04d}'.format(sample_dir, epoch, idx))
save_images(fake_A,
'./{}/A_fake{:02d}_{:04d}'.format(sample_dir, epoch, idx))
save_images(fake_B,
'./{}/B_fake{:02d}_{:04d}'.format(sample_dir, epoch, idx))
def test(self, args):
"""Test cyclegan"""
init_op = tf.global_variables_initializer()
self.sess.run(init_op)
batch, meta = self.load_batch(self.batch_size, type='test')
batch_idxs = len(batch)
if self.load(args.checkpoint_dir):
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
# write html for visual comparison
index_path = os.path.join(
args.test_dir, '{0}_index.html'.format(args.which_direction))
index = open(index_path, "w")
index.write("<html><body><table><tr>")
index.write("<th>name</th><th>input</th><th>output</th></tr>")
for idx in range(batch_idxs):
sample_file = meta[idx].get_name()
if args.which_direction == 'MtoP':
out_var, in_var = (self.testB, self.test_A)
sample_image = batch[idx][:, :, :, :self.input_c_dim]
else:
out_var, in_var = (self.testA, self.test_B)
sample_image = batch[idx][:, :, :, :self.
input_c_dim:self.input_c_dim +
self.input_c_dim]
sample_image = np.reshape(sample_image, [1, 256, 256, 256, 1])
print('Processing image: ' + sample_file)
# sample_image = [load_test_data(sample_file, args.fine_size)]
sample_image = np.array(sample_image).astype(np.float32)
image_path = os.path.join(
args.test_dir, '{0}_{1}'.format(args.which_direction,
os.path.basename(sample_file)))
fake_img = self.sess.run(out_var, feed_dict={in_var: sample_image})
save_images(fake_img, image_path)
index.write("<td>%s</td>" % os.path.basename(image_path))
index.write("<td><img src='%s'></td>" %
(sample_file if os.path.isabs(sample_file) else
('..' + os.path.sep + sample_file)))
index.write("<td><img src='%s'></td>" %
(image_path if os.path.isabs(image_path) else
('..' + os.path.sep + image_path)))
index.write("</tr>")
index.close()
histogram_freq=1,
write_graph=True,
write_images=False)
RP = ReduceLROnPlateau(
monitor='val_loss',
factor=0.1,
patience=3,
verbose=0,
mode='auto',
)
callbacks = [ES, TB, RP]
#create data
train_set = Loader(mode='train')
data_loader = torch.utils.data.DataLoader(train_set,
batch_size=300,
shuffle=False,
num_workers=1)
for i, (x, y) in enumerate(data_loader):
x = x.data.numpy()
y = y.data.numpy()
#main
if __name__ == '__main__':
model = simple_s2s()
model.fit(x,
y,
epochs=2000,
batch_size=1,
'num_workers': args.num_workers,
'pin_memory': True
} if args.cuda else {}
if args.cuda:
torch.cuda.manual_seed(args.seed)
captioning_dataset = json.load(open(args.captioning_dataset_path, "rb"))
all_arts = pickle.load(open(args.articles_metadata, 'rb'))
art2id = {}
# ipdb.set_trace()
for i, art in enumerate(all_arts):
art2id[art] = i
p = open(args.fake_articles, 'r')
fake_articles = [json.loads(l) for i, l in enumerate(p)]
train_loader = DataLoader(Loader(args, 'train', captioning_dataset, art2id,
fake_articles),
batch_size=args.batch_size,
shuffle=True,
collate_fn=collate)
val_loader = DataLoader(Loader(args, 'valid', captioning_dataset, art2id,
fake_articles),
batch_size=args.batch_size,
shuffle=False,
collate_fn=collate,
**kwargs)
test_loader = DataLoader(Loader(args, 'test', captioning_dataset, art2id,
fake_articles),
batch_size=args.batch_size,
shuffle=False,
collate_fn=collate,
**kwargs)
for line in ss_file:
line = line.split("|")
nr = line[0]
synsets = line[1].rstrip().split(",")[:-1]
wn_synsets = []
for ss in synsets:
ss = wn.synset(ss)
wn_synsets.append(ss)
wordnet_ss[nr] = wn_synsets
return wordnet_ss
if __name__ == "__main__":
data = Loader.load_data("../NLI2FOLI/SICK/SICK_train.txt")
test = Loader.load_data("../NLI2FOLI/SICK/SICK_trial.txt")
model = Model(data)
encoder = FeatureExtractor.generate_postag_onehot(data["tokens"])
data["postags"] = FeatureExtractor.postag_tokenizer(data["tokens"])
test["postags"] = FeatureExtractor.postag_tokenizer(test["tokens"])
data["antons"] = FeatureExtractor.antonym_relations(data["pair_ID"])
test["antons"] = FeatureExtractor.antonym_relations(test["pair_ID"])
data["synons"] = FeatureExtractor.synonym_relations(
data["tokens"], data["pair_ID"])
test["synons"] = FeatureExtractor.synonym_relations(
from train import *
from eval import *
from predict import predict
import paddle.distributed as dist
from paddle.distributed import fleet
def cprint(words: str):
print(f"\033[0;30;43m{words}\033[0m")
if __name__ == '__main__':
print(config.config)
train_dataset = Loader(path=config.dataset)
name_datasets = config.dataset.split('/')[-1]
Recmodel = NGCF(config.config, train_dataset)
if config.config['multigpu']:
print('using fleet multigpu training', Recmodel)
dist.init_parallel_env()
Recmodel = paddle.DataParallel(Recmodel)
if config.config['multicpu']:
fleet.init(is_collective=True)
optimizer = fleet.distributed_optimizer(optimizer)
Recmodel = fleet.distributed_model(Recmodel)
print('using fleet multicpu training', Recmodel)
Neg_k = 1
bpr = BPRLoss(Recmodel, config.config)
f = open(f'logger/train_logger_{name_datasets}.txt', 'w')
f_test = open(f'logger/test_logger_{name_datasets}.txt', 'w')
default=100,
metavar='N',
help='how many batches to wait before logging training status')
args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
print(args)
kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
data_loader = Loader('data/train_labeled_aug.p', c.FILE_TRAIN_UNLABELED,
c.FILE_VALIDATION, c.FILE_TEST, kwargs)
train_loader = data_loader.getLabeledtrain()
unlabeled_train_loader = data_loader.getUnlabeledtrain()
valid_loader = data_loader.getValidation()
model = Ladder()
if args.cuda:
model.cuda()
l2loss = torch.nn.BCELoss(
) #torch.nn.L1Loss() # BCELoss : Pass through sigmoid
#l2_2 = torch.nn.L1Loss()
nllloss = nn.NLLLoss()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
transforms = T.Compose([M.Numpy2Tensor()])
target_transforms = T.Compose([M.Numpy2Tensor()])
# image processing
RGBD2DivRGBD = IP.RGBD2DivRGBD(img_size=img_size,
div_num=output_size,
depth_max=depth_max,
device=device)
# 学習を開始した日付
dt_now = datetime.datetime.now()
dt_str = dt_now.strftime('%Y-%m-%d')
# Dataset object
dataset_full = Loader.ExYOLOMakeDatasetObject(train_data_dir + '/color',
train_data_dir + '/depth',
train_data_dir + '/AABB_3D',
img_size=img_size,
output_size=output_size,
output_channels=output_channels)
def train(dataset_train):
# __Initialize__ #
model.train()
running_loss = 0.0
iou_scores = 0.0
data_count = 0
dataloader_train = data.DataLoader(dataset=dataset_train,
batch_size=batch_size,
shuffle=True)
for step, (rgbds,
def setUp(self):
#Wipe DB
apiproxy_stub_map.apiproxy.GetStub('datastore_v3').Clear()
#load default data
l = Loader()
l.loadDatabase()
help="available datasets: [lastfm, gowalla, yelp2018, amazon-book]")
parser.add_argument(
'--path',
type=str,
default="./checkpoints",
help="path to save weights")
parser.add_argument(
'--topks', nargs='?', default="[20]", help="@k test list")
parser.add_argument('--epochs', type=int, default=1000)
parser.add_argument('--seed', type=int, default=2020, help='random seed')
return parser.parse_args()
if __name__ == '__main__':
args = parse_args()
dataset = Loader(args, path=args.dataset)
model = NGCF(args, dataset)
neg = 1
optim = paddle.optimizer.Adam(
parameters=model.parameters(), learning_rate=args.lr)
for epoch in range(args.epochs):
if epoch % 10 == 0:
results = test(dataset, model, epoch, args)
print(results)
log = train(dataset, model, epoch, optim, args, neg_k=neg, w=None)
print(f'EPOCH[{epoch+1}/{args.epochs}] {log}')
def read_data(self):
self.loader = Loader()
self.loader.read_data()
import torch
import pickle
import numpy as np
import pandas as pd
import torch.nn as nn
import constants as c
from dataloader import Loader
from torch.autograd import Variable
cuda = torch.cuda.is_available()
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
data_loader = Loader(c.FILE_TRAIN_LABELED_AUG, c.FILE_TRAIN_UNLABELED, c.FILE_TEST, 'data/test-labeled.p', kwargs)
test_loader = data_loader.getTest()
test_actual = data_loader.getValidation()
label_predict = np.array([])
def callval(mnist_model, test_loader, test_actual, model, file):
label_predict = np.array([])
loadfile = True
if loadfile:
mnist = torch.load(model)
mnist_model.load_state_dict(mnist)
correct = 0
if torch.cuda.is_available():
mnist_model.cuda()
