def __getitem__(self, idx):
'''
Abstract method. returns the image and label for a single input at index 'idx'.
'''
if torch.is_tensor(idx):
idx = idx.tolist()
image, label = generate_training_data(data=self.list_data[idx], resized_image_size=self.resized_image_size)
sample = {
'image':image,
'label':label
}
if self.transform:
sample = self.transform(sample)
return sample
def __getitem__(self, idx):
'''
Abstract method. Returns the image and label for a single input with the index of `idx`.
'''
if torch.is_tensor(idx):
idx = idx.tolist()
image, label_array = generate_training_data(
anchors_list=self.anchors_list,
xml_file_path=self.list_annotations[idx],
classes=self.classes,
resized_image_size=self.resized_image_size,
subsampled_ratio=self.subsampled_ratio,
excluded_classes=self.excluded_classes,
image_path=self.list_images[idx])
sample = {'image': image, 'label': label_array}
if self.transform:
sample = self.transform(sample)
return sample
def main(board_size=11,
tree_file=None,
pretrained_model=None,
game_file_saved_dict="game_record"):
if not os.path.exists(game_file_saved_dict):
os.mkdir(game_file_saved_dict)
if pretrained_model:
Net = torch.load(pretrained_model)
else:
Net = nn(input_layers=1, board_size=11)
stack = utils.random_stack()
if tree_file:
tree = utils.read_file(tree_file)
else:
tree = MCTS(board_size=board_size, neural_network=Net)
for game_time in range(100):
game_record = tree.game()
if len(game_record) % 2 == 1:
print(
"game {} completed, black win, this game length is {}".format(
game_time, len(game_record)))
else:
print("game {} completed, win win, this game length is {}".format(
game_time, len(game_record)))
utils.write_file(
game_record, game_file_saved_dict + "/" +
time.strftime("%Y%m%d-%H-%M-%S", time.localtime()))
train_data = utils.generate_training_data(game_record=game_record,
board_size=11)
for i in range(len(train_data)):
stack.push(train_data[i])
my_loader = utils.generate_data_loader(stack)
Net.train(my_loader, game_time)
torch.save(
Net,
"model_" + time.strftime("%Y%m%d-%H-%M", time.localtime()) + ".pkl")
print("Teacher forcing ratio must be between 0.0 and 1.0!")
exit(1)
print("Generating vocabulary and sentence pairs...")
vocabulary, sent_pairs = utils.prepare_training_data(dataset[:1000])
dev = torch.device(config.device if torch.cuda.is_available() else "cpu")
print("Performing train test split...")
train_sent_pairs, test_sent_pairs = train_test_split(sent_pairs,
shuffle=True,
test_size=0.2)
# deleting this to free up some memory
del sent_pairs
print("Generating training data...")
input_elems_train, output_elems_train = utils.generate_training_data(
train_sent_pairs, vocabulary)
if config.use_pretrained_embedding is False:
# initialize embedding -> this will be used in both encoder and decoder
print("Initializing embedding layer...")
embedding = nn.Embedding(vocabulary.num_words, config.embedding_size)
else:
print("Loading pretrained embeddings...")
pretrained_embeddings_train = utils.get_glove_embeddings(
glove_vectors,
input_elems_train[0],
vocabulary,
wv_dim=glove_vectors['the'].shape[0])
input_elems_train = (pretrained_embeddings_train, input_elems_train[1])
embedding = None
clf_str = "DeepFM"
elif dfm_params["use_fm"]:
clf_str = "FM"
elif dfm_params["use_deep"]:
clf_str = "DNN"
print("%s: %.5f (%.5f)"%(clf_str, gini_results_cv.mean(), gini_results_cv.std()))
filename = "%s_Mean%.5f_Std%.5f.csv"%(clf_str, gini_results_cv.mean(), gini_results_cv.std())
_plot_fig(gini_results_epoch_train, gini_results_epoch_valid, clf_str)
return y_train_meta, y_test_meta
'''
if __name__ == '__main__':
data = utils.generate_training_data()
data = data.rename(columns={'label':'target','user_id':'id'})
print('load data success')
data = shuffle(data)
data = data.iloc[:1000,:20]
print(data[:2])
train = data[:int(0.7*data.shape[0])]
test = data[int(0.7*data.shape[0]):]
y_train = train['target']
X_train = train.drop(['id','target'],axis=1)
numerical_cols = train.drop(['target','id','click_article_id']).columnt.tolist()
folds = list(StratifiedKFold(n_splits=3, shuffle=True,random_state=42).split(X_train, y_train))
_run_base_model_dfm(train,test,folds,dfm_params)
print("Loading models...")
encoder = torch.load(config.encoder_model)
decoder = torch.load(config.decoder_model)
vocabulary = joblib.load(config.vocabulary)
if config.use_pretrained_embedding is True and os.path.exists(config.glove_vectors):
glove_vectors = joblib.load(config.glove_vectors)
print("Pretrained embeddings loaded!")
else:
glove_vectors = None
print("No pretrained embeddings loaded.")
dev = torch.device(config.device if torch.cuda.is_available() else "cpu")
print("Chat service initiated!")
print()
while True:
text = input("Please enter a sentence: ")
text = text.lower().strip()
if text == 'q' or text == 'quit':
break
try:
input_elems, output_elems = generate_training_data([(text, text)], vocabulary)
except KeyError as ke:
print("Oops - seems like I don't know the following word: {}".format(str(ke)))
continue
response = test_model(encoder, decoder, input_elems, output_elems, vocabulary, dev,
config.use_pretrained_embedding, glove_vectors, chatting=True)
print("Response: {}".format(response))
print()
def main(tree_file=None,
pretrained_model=None,
game_file_saved_dict="game_record_2"):
if not os.path.exists(game_file_saved_dict):
os.mkdir(game_file_saved_dict)
if pretrained_model:
Net = torch.load(pretrained_model)
else:
Net = nn(input_layers=3,
board_size=utils.board_size,
learning_rate=utils.learning_rate)
stack = utils.random_stack()
if tree_file:
tree = utils.read_file(tree_file)
else:
tree = MCTS(board_size=utils.board_size, neural_network=Net)
Net.adjust_lr(1e-3)
record = []
game_time = 3600
while True:
game_record, eval, steps = tree.game()
if len(game_record) % 2 == 1:
print(
"game {} completed, black win, this game length is {}".format(
game_time, len(game_record)))
else:
print(
"game {} completed, white win, this game length is {}".format(
game_time, len(game_record)))
print("The average eval:{}, the average steps:{}".format(eval, steps))
utils.write_file(
game_record, game_file_saved_dict + "/" +
time.strftime("%Y%m%d_%H_%M_%S", time.localtime()) +
'_game_time:{}.pkl'.format(game_time))
train_data = utils.generate_training_data(game_record=game_record,
board_size=utils.board_size)
for i in range(len(train_data)):
stack.push(train_data[i])
my_loader = utils.generate_data_loader(stack)
utils.write_file(my_loader, "debug_loader.pkl")
if game_time % 100 == 0:
for _ in range(5):
record.extend(Net.train(my_loader, game_time))
print("train finished")
print(" ")
if game_time % 200 == 0:
torch.save(Net, "model_{}.pkl".format(game_time))
test_game_record, _, _ = tree.game(train=False)
utils.write_file(
test_game_record,
game_file_saved_dict + "/" + 'test_{}.pkl'.format(game_time))
print("We finished a test game at {} game time".format(game_time))
if game_time % 200 == 0:
plt.figure()
plt.plot(record)
plt.title("cross entropy loss")
plt.xlabel("step passed")
plt.ylabel("Loss")
plt.savefig("loss record_{}.svg".format(game_time))
plt.close()
game_time += 1