def test_train():
model_ = models.build_model()
# device = 'cuda' if torch.cuda.is_available() else 'cpu'
device = DEVICE
dataloader = data.read_data(mean=False,
in_range=False,
val=False,
start_random=True)
train_loader = data.read_data(mean=False,
in_range=False,
dataset="SODA",
val=False)
datas, label = data.read_test_data(mean=False, in_range=False)
datas = datas.to(device)
label = label.detach().numpy()
model_.to(device)
model_.eval()
y = model_(datas).cpu().detach().numpy()
print(tool.score(label, y))
model_.train()
train.train(model_, dataloader, train_loader)
model_.eval()
y = model_(datas).cpu().detach().numpy()
print(tool.score(label, y))
print(np.abs(y - label))
def test_model():
model_ = models.build_model()
datas, label = data.read_test_data()
label = label.detach().numpy()
y = model_(datas).detach().numpy()
# print(y.shape, label.shape)
print(tool.score(label, y))
def main(_):
count = []
with open('./processed/word2idx.pkl', 'rb') as f:
word2idx = pickle.load(f)
if not os.path.exists(FLAGS.checkpoint_dir):
os.makedirs(FLAGS.checkpoint_dir)
idx2word = dict(zip(word2idx.values(), word2idx.keys()))
FLAGS.nwords = len(word2idx)
pp.pprint(flags.FLAGS.__flags)
# train_data = read_data('%s/%s.train.txt' % (FLAGS.data_dir, FLAGS.data_name), count, word2idx)
# valid_data = read_data('%s/%s.valid.txt' % (FLAGS.data_dir, FLAGS.data_name), count, word2idx)
# test_data = read_data('%s/%s.test.txt' % (FLAGS.data_dir, FLAGS.data_name), count, word2idx)
# exit()
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.3 if FLAGS.inference else 0.6
with tf.Session(config=config) as sess:
model = MemN2N(FLAGS, sess)
model.build_model()
test_set_data = read_test_data(FLAGS.infer_set, word2idx)
if FLAGS.inference:
model.load()
answer = model.inference(test_set_data, word2idx)
import pandas as pd
answer = pd.DataFrame(answer, columns=['answer'])
answer.index += 1
answer.to_csv('./guess/guess.csv', index_label='id')
else:
if FLAGS.restore:
model.load()
with open('./processed/all_train.pkl', 'rb') as f:
train_data = pickle.load(f)
with open('./processed/all_valid.pkl', 'rb') as f:
valid_data = pickle.load(f)
test_data = read_our_data(
'./data/CBData/cbtest_CN_test_2500ex.txt', count, word2idx)
if FLAGS.is_test:
print('Do not use --is_test True')
exit()
model.run(valid_data, test_data, word2idx, test_set_data)
else:
model.run(train_data, valid_data, word2idx, test_set_data)
output_path = "/home/user/data/color-description/"
epochs = 20
batch_size = 128
hidden_size = 100
log_interval = 500
bidirectional = False
num_layers = 1
min_count = 1
gradient_clipping = 0.2
embedding_size = 100
cuda = True
print(str(datetime.now()), "Loading data")
train_colors = read_train_data(train_path)
test_colors = read_test_data(test_path)
random.shuffle(train_colors)
random.shuffle(test_colors)
print(str(datetime.now()), "Generating vocab")
vocab = Vocab(train_colors,
min_count=min_count,
add_padding=True,
add_bos=True,
add_eos=True)
embeddings = nn.Embedding(len(vocab.index2token),
embedding_size,
padding_idx=vocab.PAD.hash)
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from data import read_test_data
from train import read_train_data
from model import get_model
train = read_train_data()
x_train = train.iloc[:,
1:] # toutes les lignes et toutes les colonnes sauf la 1ere
y_train = train.iloc[:, :1]
y_train = np_utils.to_categorical(y_train, 10)
x_train = x_train / 255
test = read_test_data()
model = Sequential()
model.add(Dense(800, input_dim=784))
model.add(Dense(10, activation="softmax"))
model.compile(loss="categorical_crossentropy", optimizer="SGD")
model.summary()
model.fit(x_train, y_train)
test = test / 255
result = model.predict(test)
y_test = np.argmax(result, axis=1)
Sample_submissions_CNN = pd.DataFrame({
"ImageID": range(1,
def test_read_test_data():
datas, label = data.read_test_data()
print(datas.shape, label.shape)