def main(_):
print("Parameters: ")
for k, v in FLAGS.__flags.items():
print("{} = {}".format(k, v))
if not os.path.exists("./prepro/"):
os.makedirs("./prepro/")
if FLAGS.prepro:
img_feat, tags_idx, a_tags_idx, vocab_processor = data_utils.load_train_data(FLAGS.train_dir, FLAGS.tag_path, FLAGS.prepro_dir, FLAGS.vocab)
else:
img_feat = cPickle.load(open(os.path.join(FLAGS.prepro_dir, "img_feat.dat"), 'rb'))
tags_idx = cPickle.load(open(os.path.join(FLAGS.prepro_dir, "tag_ids.dat"), 'rb'))
a_tags_idx = cPickle.load(open(os.path.join(FLAGS.prepro_dir, "a_tag_ids.dat"), 'rb'))
vocab_processor = VocabularyProcessor.restore(FLAGS.vocab)
img_feat = np.array(img_feat, dtype='float32')/127.5 - 1.
test_tags_idx = data_utils.load_test(FLAGS.test_path, vocab_processor)
print("Image feature shape: {}".format(img_feat.shape))
print("Tags index shape: {}".format(tags_idx.shape))
print("Attribute Tags index shape: {}".format(a_tags_idx.shape))
print("Vocab size: {}".format(len(vocab_processor._reverse_mapping)))
print("Vocab max length: {}".format(vocab_processor.max_document_length))
data = Data(img_feat, tags_idx, a_tags_idx, test_tags_idx, FLAGS.z_dim, vocab_processor)
Model = getattr(sys.modules[__name__], FLAGS.model)
print(Model)
model = Model(data, vocab_processor, FLAGS)
model.build_model()
model.train()
def find_cmaes_settings(start_sigma) -> Tuple[Any, Any]:
""" Find best settings for baseline image segmentation
using CMA-ES
"""
photos, annots = load_train_data()
photos = [p.astype(np.float32) for p in photos]
objective = partial(cmaes_metric_from_pool, photos=photos, annots=annots)
objective_wrapped = cmaes_utils.get_cmaes_params_warp(objective)
start_values = get_start_params()
return cma.fmin2(objective_wrapped, start_values, start_sigma) # xopt, es
def main(_):
print("Parameters: ")
log_writeln(outf, "Parameters: ")
for k, v in FLAGS.__flags.items():
print("{} = {}".format(k, v))
log_writeln(outf, "{} = {}".format(k, v))
if not os.path.exists("./prepro/"):
os.makedirs("./prepro/")
if FLAGS.prepro:
img_feat, tags_idx, a_tags_idx, vocab_processor = data_utils.load_train_data(
FLAGS.train_dir, FLAGS.tag_path, FLAGS.prepro_dir, FLAGS.vocab)
else:
img_feat = cPickle.load(
open(os.path.join(FLAGS.prepro_dir, "img_feat.dat"), 'rb'))
tags_idx = cPickle.load(
open(os.path.join(FLAGS.prepro_dir, "tag_ids.dat"), 'rb'))
a_tags_idx = cPickle.load(
open(os.path.join(FLAGS.prepro_dir, "a_tag_ids.dat"), 'rb'))
vocab_processor = VocabularyProcessor.restore(FLAGS.vocab)
img_feat = np.array(img_feat, dtype='float32') / 127.5 - 1.
test_tags_idx = data_utils.load_test(FLAGS.test_path, vocab_processor)
log_writeln(outf, "Image feature shape: {}".format(img_feat.shape))
log_writeln(outf, "Tags index shape: {}".format(tags_idx.shape))
log_writeln(outf,
"Attribute Tags index shape: {}".format(a_tags_idx.shape))
log_writeln(outf,
"Vocab size: {}".format(len(vocab_processor._reverse_mapping)))
log_writeln(
outf,
"Vocab max length: {}".format(vocab_processor.max_document_length))
data = Data(img_feat, tags_idx, a_tags_idx, test_tags_idx, FLAGS.z_dim,
vocab_processor)
Model = getattr(sys.modules[__name__], FLAGS.model)
print(Model)
log_writeln(outf, Model)
model = Model(data, vocab_processor, FLAGS)
model.build_model()
model.train()
def run(dataset, DF_layers, DI_layers, n_negs, alpha, gpu='0'):
import os
os.environ['CUDA_VISIBLE_DEVICES'] = gpu
print("##### {} Negative Samples experiment on {} DF: {} DI: {}".format(
n_negs, dataset, DF_layers, DI_layers))
learning_rate = 0.0001
batch_size = 256
#embed_dim = 256
#factor_dim = 64
if torch.cuda.is_available():
device = torch.device('cuda')
FloatTensor = torch.cuda.FloatTensor
else:
device = torch.device('cpu')
FloatTensor = torch.FloatTensor
manualSeed = 706
random.seed(manualSeed)
torch.manual_seed(manualSeed)
print('CUDA Available:', torch.cuda.is_available())
file_name = 'output/' + dataset + '_J-NCF_' + str(DF_layers) + '_' + str(
DI_layers) + '_n_' + str(n_negs) + '.txt'
output = open(file_name, 'w')
# Datasets
user_matrix, item_matrix, train_u, train_i, train_r, neg_candidates, u_cnt, user_rating_max = data_utils.load_train_data(
dataset)
if dataset == 'ml1m':
epochs = 100
eval_batch_size = 100 * 151
test_users, test_items = data_utils.load_test_ml1m()
elif dataset == 'ml100k':
epochs = 100
eval_batch_size = 100 * 41
test_users, test_items = data_utils.load_test_data(dataset)
elif dataset == 'yelp':
epochs = 50
eval_batch_size = 100 * 81
test_users, test_items = data_utils.load_test_data(dataset)
elif dataset == 'amusic':
epochs = 100
eval_batch_size = 100 * 3
test_users, test_items = data_utils.load_test_data(dataset)
elif dataset == 'agames':
epochs = 100
eval_batch_size = 100 * 34
test_users, test_items = data_utils.load_test_data(dataset)
n_users, n_items = user_matrix.shape[0], user_matrix.shape[1]
user_array = user_matrix.toarray()
item_array = item_matrix.toarray()
user_idxlist, item_idxlist = list(range(n_users)), list(range(n_items))
# Model
model = JNCF(DF_layers, DI_layers, n_users, n_items,
'concat').to(device) # 'multi' or 'concat'
pair_loss_function = TOP1 # TOP1 or BPR
point_loss_function = torch.nn.BCEWithLogitsLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
best_hr = 0.0
for epoch in range(epochs):
# Train
model.train() # Enable dropout (if have).
idxlist = np.array(range(len(train_u)))
np.random.shuffle(idxlist)
epoch_loss, epoch_pair_loss, epoch_point_loss, epoch_i_point_loss, epoch_j_point_loss = .0, .0, .0, .0, .0
start_time = time.time()
for batch_idx, start_idx in enumerate(
range(0, len(idxlist), batch_size)):
end_idx = min(start_idx + batch_size, len(idxlist))
idx = idxlist[start_idx:end_idx]
u_ids = train_u.take(idx)
i_ids = train_i.take(idx)
i_ratings = train_r.take(idx)
users = FloatTensor(user_array.take(u_ids, axis=0))
items = FloatTensor(item_array.take(i_ids, axis=0))
labels = FloatTensor(i_ratings)
rating_max = FloatTensor(user_rating_max.take(u_ids, axis=0))
Y_ui = labels / rating_max # for Normalized BCE
Y_uj = torch.zeros_like(
Y_ui) # for Negative samples point-wise loss
optimizer.zero_grad()
point_loss, pair_loss = 0., 0.
# Negative Sampling
neg_items_list = []
for _ in range(0, n_negs):
neg_items = one_negative_sampling(u_ids, neg_candidates)
neg_items_list.append(neg_items)
for ng_idx in range(0, n_negs):
neg_ids = neg_items_list[ng_idx]
items_j = FloatTensor(item_array.take(neg_ids, axis=0))
y_i, y_j = model(users, items, items_j)
i_point_loss = point_loss_function(y_i,
Y_ui) # positive items i
j_point_loss = point_loss_function(y_j,
Y_uj) # negative items j
point_loss = i_point_loss + j_point_loss
pair_loss = pair_loss_function(y_i, y_j, n_negs)
loss = alpha * pair_loss + (1 - alpha) * point_loss
epoch_loss += loss.item()
epoch_pair_loss += pair_loss.item()
epoch_point_loss += point_loss.item()
epoch_i_point_loss += i_point_loss.item()
epoch_j_point_loss += j_point_loss.item()
loss.backward()
optimizer.step()
train_time = time.time() - start_time
# Evaluate
model.eval()
HR, NDCG = [], []
time_E = time.time()
for start_idx in range(0, len(test_users), eval_batch_size):
end_idx = min(start_idx + eval_batch_size, len(test_users))
u_ids = test_users[start_idx:end_idx]
i_ids = test_items[start_idx:end_idx]
users = FloatTensor(user_array.take(u_ids, axis=0))
items = FloatTensor(item_array.take(i_ids, axis=0))
preds, _ = model(users, items, items)
e_batch_size = eval_batch_size // 100 # faster eval
preds = torch.chunk(preds.detach().cpu(), e_batch_size)
chunked_items = torch.chunk(torch.IntTensor(i_ids), e_batch_size)
for i, pred in enumerate(preds):
_, indices = torch.topk(pred, 10)
recommends = torch.take(chunked_items[i],
indices).numpy().tolist()
gt_item = chunked_items[i][0].item()
HR.append(hit(gt_item, recommends))
NDCG.append(ndcg(gt_item, recommends))
eval_time = time.time() - time_E
#if epoch % 10 == 0:
e_loss = epoch_loss / (batch_idx + 1)
e_pair = epoch_pair_loss / (batch_idx + 1)
e_point = epoch_point_loss / (batch_idx + 1)
e_i_point = epoch_i_point_loss / (batch_idx + 1)
e_j_point = epoch_j_point_loss / (batch_idx + 1)
text_1 = '[Epoch {:03d}]'.format(epoch) + '\ttrain: ' + time.strftime(
'%M: %S',
time.gmtime(train_time)) + '\tHR: {:.4f}\tNDCG: {:.4f}\n'.format(
np.mean(HR), np.mean(NDCG))
text_2 = 'Loss: {:.6f}\tPair: {:.4f}\tPoint: {:.4f}\ti_point: {:.4f}\tj_point: {:.4f}\n'.format(
e_loss, e_pair, e_point, e_i_point, e_j_point)
print(text_1[:-1])
print(text_2[:-1])
output.write(text_1)
output.write(text_2)
if np.mean(HR) > best_hr:
best_hr, best_ndcg, best_epoch = np.mean(HR), np.mean(NDCG), epoch
result = 'DF: {} DI: {}. Best epoch {:02d}: HR = {:.4f}, NDCG = {:.4f}\n'.format(
DF_layers, DI_layers, best_epoch, best_hr, best_ndcg)
print(result[:-1])
output.write(result)
output.close()
'train_img_dir' : args.train_img_dir
}
if not os.path.exists("./prepro/"):
os.makedirs("./prepro/")
if args.pre_parameter == True:
img_feat = cPickle.load(open(os.path.join(args.prepro_dir, "img_feat.dat"), 'rb'))
tags_idx = cPickle.load(open(os.path.join(args.prepro_dir, "tag_ids.dat"), 'rb'))
a_tags_idx = cPickle.load(open(os.path.join(args.prepro_dir, "a_tag_ids.dat"), 'rb'))
k_tmp_vocab = cPickle.load(open(os.path.join(args.prepro_dir, "k_tmp_vocab_ids.dat"), 'rb'))
vocab_processor = Vocab_Operator.restore(args.vocab)
else:
img_feat, tags_idx, a_tags_idx, vocab_processor, k_tmp_vocab = data_utils.load_train_data(args.train_dir,
args.tag_path, args.prepro_dir, args.vocab)
img_feat = np.array(img_feat, dtype='float32')/127.5 - 1.
test_tags_idx = data_utils.load_test(args.test_path, vocab_processor, k_tmp_vocab)
print("Image feature shape: {}".format(img_feat.shape))
print("Tags index shape: {}".format(tags_idx.shape))
print("Attribute Tags index shape: {}".format(a_tags_idx.shape))
print("Test Tags index shape: {}".format(test_tags_idx.shape))
data = Data(img_feat, tags_idx, a_tags_idx, test_tags_idx, args.z_dim, vocab_processor)
dcgan = dcgan.DCGAN(model_options, training_options, data, args.mode, args.resume, args.model_dir)
#embed_dim = 256
#factor_dim = 64
if torch.cuda.is_available():
device = torch.device('cuda')
FloatTensor = torch.cuda.FloatTensor
else:
device = torch.device('cpu')
FloatTensor = torch.FloatTensor
manualSeed = 706
random.seed(manualSeed)
torch.manual_seed(manualSeed)
print('CUDA Available:', torch.cuda.is_available())
# Prepare data
user_matrix, item_matrix, train_u, train_i, train_r, neg_candidates, u_cnt, user_rating_max = data_utils.load_train_data(
args.data)
test_users, test_items = data_utils.load_test_ml1m()
eval_batch_size = 100 * 151
n_users, n_items = user_matrix.shape[0], user_matrix.shape[1]
user_array = user_matrix.toarray()
item_array = item_matrix.toarray()
user_idxlist, item_idxlist = list(range(n_users)), list(range(n_items))
# Model
model = JNCF(DF_layers, DI_layers, n_users, n_items,
'concat').to(device) # 'multi' or 'concat'
# Optimize
pair_loss_function = TOP1
point_loss_function = torch.nn.BCEWithLogitsLoss()
import helpers
##############################################################################
# Settings
##############################################################################
CUDA = False
##############################################################################
# Load the dataset
##############################################################################
Data = namedtuple("Data", "corpus train dev test embeddings word_to_index")
data_utils.download_ask_ubuntu_dataset()
EMBEDDINGS, WORD_TO_INDEX = data_utils.load_part2_embeddings()
ASK_UBUNTU_CORPUS = data_utils.load_corpus(WORD_TO_INDEX)
ASK_UBUNTU_TRAIN_DATA = data_utils.load_train_data()
ASK_UBUNTU_DEV_DATA, ASK_UBUNTU_TEST_DATA = data_utils.load_eval_data()
ASK_UBUNTU_DATA = Data(ASK_UBUNTU_CORPUS, ASK_UBUNTU_TRAIN_DATA,\
ASK_UBUNTU_DEV_DATA, ASK_UBUNTU_TEST_DATA,\
EMBEDDINGS, WORD_TO_INDEX)
data_utils.download_android_dataset()
ANDROID_CORPUS = data_utils.load_android_corpus(WORD_TO_INDEX)
ANDROID_DEV_DATA, ANDROID_TEST_DATA = data_utils.load_android_eval_data()
ANDROID_DATA = Data(ANDROID_CORPUS, None,\
ANDROID_DEV_DATA, ANDROID_TEST_DATA,\
EMBEDDINGS, WORD_TO_INDEX)
##############################################################################
# Train and evaluate a baseline TFIDF model
##############################################################################
import helpers
##############################################################################
# Settings
##############################################################################
CUDA = False
##############################################################################
# Load the dataset
##############################################################################
Data = namedtuple("Data", \
"corpus train dev test embeddings word_to_index")
data_utils.download_ask_ubuntu_dataset()
EMBEDDINGS, WORD_TO_INDEX = data_utils.load_embeddings()
CORPUS = data_utils.load_corpus(WORD_TO_INDEX)
TRAIN_DATA = data_utils.load_train_data()
DEV_DATA, TEST_DATA = data_utils.load_eval_data()
DATA = Data(CORPUS, TRAIN_DATA, DEV_DATA, TEST_DATA,\
EMBEDDINGS, WORD_TO_INDEX)
##############################################################################
# Train and evaluate the models for Part 1
##############################################################################
RESULTS = []
MARGINS = [0.2]
MAX_EPOCHS = 50
BATCH_SIZE = 32
FILTER_WIDTHS = [3]
POOL_METHOD = "average"
FEATURE_DIMS = [600]
DROPOUT_PS = [0.3]
model = Dave_orig()
save_model_name = './Model1.h5'
elif model_name == '2':
# K.set_learning_phase(1)
model = Dave_norminit()
save_model_name = './Model2.h5'
elif model_name == '3':
# K.set_learning_phase(1)
model = Dave_dropout()
save_model_name = './Model3.h5'
else:
print(bcolors.FAIL + 'invalid model name, must one of 1, 2 or 3' +
bcolors.ENDC)
# the data, shuffled and split between train and test sets
train_generator, samples_per_epoch = load_train_data(batch_size=batch_size,
shape=(100, 100))
# trainig
model.fit_generator(train_generator,
steps_per_epoch=math.ceil(samples_per_epoch * 1. /
batch_size),
epochs=nb_epoch,
workers=8,
use_multiprocessing=True)
print(bcolors.OKGREEN + 'Model trained' + bcolors.ENDC)
# evaluation
K.set_learning_phase(0)
test_generator, samples_per_epoch = load_test_data(batch_size=batch_size,
shape=(100, 100))
model.evaluate_generator(test_generator,
_, _, _, submission_file = create_filenames(train_id)
# Make the submission
dsb.save_submission(dataset.ids, expit(predictions), test_img_sizes,
submission_file)
if __name__ == "__main__":
args = parser.parse_args()
# Load the train_config
train_config = load_train_setup(args.train_id)
trained_model = None
PLOT = args.plot
if args.train:
# Load the train data
train_ids, x_train, y_train = dsb.load_train_data(
path_to_train="../input/train/",
img_size=train_config["img_size"],
num_channels=3)
train_dataset = NpDataset(x=x_train, y=y_train, ids=train_ids)
# train the models
if not train_config["kfold"]:
raise NotImplementedError("Non-kfold training is not implemented")
trained_model = kfold(train_dataset,
train_config,
args.train_id,
num_completed=args.num_completed)
if args.test:
# Load the test data
test_ids, x_test, sizes_test = dsb.load_test_data(
path_to_test="../input/test/",
img_size=train_config["img_size"],
def main():
"""
Main Function for Training a Image classifier on a directory
containing sub folders train, valid , test with Images.
Inputs are passed using command line.
Type python train.py -h for help on usage.
"""
# Parse and validate the cmdline arguments passed for Training.
args, proceed = parse_args_train()
if proceed:
print('args in main:', args)
else:
print("\n Type python train.py -h for help on setting training "
"parameters. Thank you! \n")
return True
# Load the Training data
dataloaders, class_to_idx = load_train_data(args.data_directory,
args.batch_sizes)
# Build network
layers= args.hidden_units + [args.num_classes]
model = build_network(arch_name=args.arch, layers=layers,
dropout=args.dropout,
hidden_activation=args.hidden_activation,
output_activation=args.output_activation)
# Check if cuda is available
if args.gpu and torch.cuda.is_available():
device = torch.device("cuda")
elif not args.gpu:
device = torch.device("cpu")
else:
print("cuda is available? :", torch.cuda.is_available())
raise Exception('Error ! device CUDA is not available')
# load the model from checkpoint, checkpoint for losses and
# model params for training restart.
with active_session():
if args.checkpoint:
model, checkpoint = load_checkpoint(args.checkpoint)
else:
checkpoint = {}
trained_model, checkpoint = train_network(model, device,
criteria=args.criterion,
optimizr=args.optimizer,
learn_rate=args.learning_rate,
trainloader=dataloaders['train'],
validloader=dataloaders['valid'],
epochs=args.epochs,
checkpoint=checkpoint)
if args.save_dir:
checkpoint['class_to_idx'] = class_to_idx
checkpoint_filename = ""
checkpoint_filename += 'checkpoint'
checkpoint_filename += str(checkpoint['last_epoch'])
checkpoint_filename += '.pth'
checkpoint_filepath = os.path.join(args.save_dir, checkpoint_filename)
checkpoint_model(checkpoint, checkpoint_filepath)
# Load the clusters
train_cluster_ids = np.load("../clusters/train_clusters.npz")
test_cluster_ids = np.load("../clusters/test_clusters.npz")
in_set = np.vectorize(lambda a, s: a in s)
# In[5]:
# Load the training data
test_ids, X_test, sizes_test = dsb.load_test_data(path_to_test=PATH_TO_TEST,
img_size=None,
num_channels=NUM_CHANNELS,
mode='rgb')
train_ids, X_train, Y_train = dsb.load_train_data(path_to_train=PATH_TO_TRAIN,
img_size=None,
num_channels=NUM_CHANNELS,
mode='rgb')
print("Number of training samples: %s" % len(train_ids))
print("X-train shape: {}".format(X_train.shape))
print("Y-train shape: {}".format(Y_train.shape))
print("X-test shape: {}".format(X_test.shape))
# Get indexes from clusters
train_clusters = np.zeros(NUM_TRAIN, dtype=int)
test_clusters = np.zeros(NUM_TEST, dtype=int)
train_clusters[in_set(train_ids,
{a
for a in train_cluster_ids["cluster_0"]})] = 0
train_clusters[in_set(train_ids,
{a
for a in train_cluster_ids["cluster_1"]})] = 1