data_loader = MultiClassDataLoader(tf.flags, WordDataProcessor())
data_loader.define_flags()
FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
print("\nParameters:")
for attr, value in sorted(FLAGS.__flags.items()):
print("{}={}".format(attr.upper(), value))
print("")
if FLAGS.eval_train:
x_raw, y_test = data_loader.load_data_and_labels()
y_test = np.argmax(y_test, axis=1)
else:
x_raw, y_test = data_loader.load_dev_data_and_labels()
y_test = np.argmax(y_test, axis=1)
# checkpoint_dir이 없다면 가장 최근 dir 추출하여 셋팅
if FLAGS.checkpoint_dir == "":
all_subdirs = [
"./runs/" + d for d in os.listdir('./runs/.')
if os.path.isdir("./runs/" + d)
]
latest_subdir = max(all_subdirs, key=os.path.getmtime)
FLAGS.checkpoint_dir = latest_subdir + "/checkpoints/"
# Map data into vocabulary
vocab_path = os.path.join(FLAGS.checkpoint_dir, "..", "vocab")
vocab_processor = data_loader.restore_vocab_processor(vocab_path)
x_test = np.array(list(vocab_processor.transform(x_raw)))
except Exception as e:
print("...failed load prev vocab\nNew vocab... & load train, dev data...")
x_train, y_train, x_dev, y_dev = data_loader.build_vocabulary()
vocab_processor = data_loader.vocab_processor
# restore data from npy or load data from file - train, dev
try:
print("{}\n restore train, dev data...".format(datetime.datetime.now().isoformat()))
#x_train, y_train = data_loader.load_train_data_and_labels()
print("file: {}".format(npy_t))
x_train = np.load(os.path.join('./npy/{}.npy'.format(npy_t)))
y_train = np.load(os.path.join('./npy/{}_y.npy'.format(npy_t)))
x_dev, y_dev = data_loader.load_dev_data_and_labels()
print("{}\n transform train, dev data by vocab...".format(datetime.datetime.now().isoformat()))
x_train = np.array(x_train)
#x_train = np.array(list(vocab_processor.transform(x_train)))
x_dev = np.array(list(vocab_processor.transform(x_dev)))
#save vocab in project root
vocab_processor.save(os.path.join("./", vocab_file))
print('save vocab')
except Exception as e:
print("{}\n ...failed restore data\n New data - transform train, dev data...".format(datetime.datetime.now().isoformat()))
try:
x_train
except NameError:
x_train, y_train, x_dev, y_dev = data_loader.prepare_data()