def create_xy_train(self,
tag_file,
embedding_file,
data_size=1,
window_size=5,
available_tags=[],
suffix=None,
mode="create",
load=None):
DataUtils.message("Prepearing Training Data...", new=True)
if mode == "create" or mode == "save":
x_train, y_train = self.__create_xy(tag_file, embedding_file,
data_size, window_size,
available_tags, suffix)
if mode == "save":
DataUtils.save_array(
DataUtils.get_filename("SFF",
"X_TRAIN" + "_" + str(window_size)),
x_train)
DataUtils.save_array(
DataUtils.get_filename("SFF",
"Y_TRAIN" + "_" + str(window_size)),
y_train)
if mode == "load" and load is not None:
x_train = DataUtils.load_array(load[0])
y_train = DataUtils.load_array(load[1])
self.x_train = np.array(x_train)
self.y_train = np.array(y_train)
self.INPUT_SHAPE = self.x_train.shape
self.OUTPUT_SHAPE = self.y_train.shape
def create_xy_test(self,
tag_file,
embedding_file,
data_size=1,
window_size=5,
available_tags=[],
suffix=None,
mode="create",
load=None):
DataUtils.message("Prepearing Test Data...", new=True)
if mode == "create" or mode == "save":
x_test, y_test = self.__create_xy(tag_file, embedding_file,
data_size, window_size,
available_tags, suffix)
if mode == "save":
DataUtils.save_array(
DataUtils.get_filename("SFF",
"X_TEST" + "_" + str(window_size)),
x_test)
DataUtils.save_array(
DataUtils.get_filename("SFF",
"Y_TEST" + "_" + str(window_size)),
y_test)
if mode == "load" and load is not None:
x_test = DataUtils.load_array(load[0])
y_test = DataUtils.load_array(load[1])
self.x_test = np.array(x_test)
self.y_test = np.array(y_test)
def create_xy_test(self,
embedding_file,
data_size=1,
look_back=0,
mode="create",
load=None):
DataUtils.message("Prepearing Test Data...", new=True)
if mode == "create" or mode == "save":
word_test, head_test, tag_test = self.__create_xy(embedding_file,
data_size,
look_back,
test=True)
if mode == "save":
DataUtils.save_array(
DataUtils.get_filename("DP_W", "TEST" + "_" + str(look_back)),
word_test)
DataUtils.save_array(
DataUtils.get_filename("DP_H", "TEST" + "_" + str(look_back)),
head_test)
DataUtils.save_array(
DataUtils.get_filename("DP_T", "TEST" + "_" + str(look_back)),
tag_test)
if mode == "load" and load is not None:
word_test = DataUtils.load_array(load[0])
head_test = DataUtils.load_array(load[1])
tag_test = DataUtils.load_array(load[2])
self.word_test = np.array(word_test)
self.head_test = np.array(head_test)
self.tag_test = np.array(tag_test)
def create_xy_train(self,
tag_file,
embedding_file,
data_size=1,
look_back=5,
threshold=0,
suffix=None,
mode="create",
load=None):
DataUtils.message("Prepearing Training Data...", new=True)
if mode == "create" or mode == "save":
x_train, y_train = self.__create_xy_train(tag_file, embedding_file,
data_size, look_back,
threshold, suffix)
if mode == "save":
DataUtils.save_array(
DataUtils.get_filename("ULSTM_X",
"TRAIN" + "_" + str(look_back)),
x_train)
DataUtils.save_array(
DataUtils.get_filename("ULSTM_Y",
"TRAIN" + "_" + str(look_back)),
y_train)
if mode == "load" and load is not None:
x_train = DataUtils.load_array(load[0])
y_train = DataUtils.load_array(load[1])
self.x_train = x_train
self.y_train = y_train
self.INPUT_SHAPE = x_train.shape
self.OUTPUT_SHAPE = y_train.shape
def create_xy_test(self,
tag_file,
embedding_file,
data_size=1,
look_back=5,
suffix=None,
mode="create",
load=None):
DataUtils.message("Prepearing Test Data...", new=True)
if mode == "create" or mode == "save":
x_test, y_test = self.__create_xy_test(tag_file, embedding_file,
data_size, look_back,
suffix)
if mode == "save":
DataUtils.save_array(
DataUtils.get_filename("ULSTM_X",
"TEST" + "_" + str(look_back)), x_test)
DataUtils.save_array(
DataUtils.get_filename("ULSTM_Y",
"TEST" + "_" + str(look_back)), y_test)
if mode == "load" and load is not None:
x_test = DataUtils.load_array(load[0])
y_test = DataUtils.load_array(load[1])
self.x_test = np.array(x_test)
self.y_test = np.array(y_test)
def create_xy_train(self,
dependency_tree,
embedding_file,
data_size=1,
look_back=0,
mode="create",
load=None):
DataUtils.message("Prepearing Training Data...", new=True)
if mode == "create" or mode == "save":
word_train, head_train, tag_train = self.__create_xy(
dependency_tree,
embedding_file,
data_size,
look_back,
test=False)
if mode == "save":
DataUtils.save_array(
DataUtils.get_filename("DP_W", "TRAIN" + "_" + str(look_back)),
word_train)
DataUtils.save_array(
DataUtils.get_filename("DP_H", "TRAIN" + "_" + str(look_back)),
head_train)
DataUtils.save_array(
DataUtils.get_filename("DP_T", "TRAIN" + "_" + str(look_back)),
tag_train)
if mode == "load" and load is not None:
word_train = DataUtils.load_array(load[0])
head_train = DataUtils.load_array(load[1])
tag_train = DataUtils.load_array(load[2])
self.word_train = np.array(word_train)
self.head_train = np.array(head_train)
self.tag_train = np.array(tag_train)
