}
return layer_output_last, timestep_outputs
else:
return layer_output_last
else:
layer_output_all = pd.DataFrame(layer_output_all)
return layer_output_all
# Load data
##============================================================================================
Xtrain, Ytrain = data_helpers.load_all_data(
config.train_path, config.validation_path, categories,
shuffle=False) # I changed this so it combines train and test
Xtest, Ytest = data_helpers.load_data(config.test_path, categories)
Xtest_raw, Ytest_raw = data_helpers.load_data_raw(config.test_path, categories)
X, y = data_helpers.load_whole_dataset(config.train_path,
config.validation_path,
config.test_path,
categories,
load_all=True,
shuffle=False,
one_hot=False)
import importlib
importlib.reload(data_helpers)
## Encode Ytrain
# =====================================================================================
#one hot encode and integer encode
Ytrain_encoded = np_utils.to_categorical(Ytrain)
#
# importlib.reload(data_helpers)
# Xtrain, Ytrain = data_helpers.load_all_data(config.train_path,config.validation_path, categories, shuffle=False) # I changed this so it combines train and validation
# Xvalidation, Yvalidation = data_helpers.load_data(config.validation_path, categories)
# Xvalidation_raw, Yvalidation_raw = data_helpers.load_data_raw(config.validation_path, categories)
# Xtrain, Ytrain = data_helpers.load_data(config.train_path, categories)
Xtrain, Ytrain = data_helpers.load_all_data(config.train_path,
config.validation_path,
categories,
shuffle=False)
Xvalidation, Yvalidation = data_helpers.load_data(config.validation_path,
categories)
Xvalidation_raw, Yvalidation_raw = data_helpers.load_data_raw(
config.validation_path, categories)
## Encode Ytrain
# =====================================================================================
#one hot encode and integer encode
Ytrain_encoded = np_utils.to_categorical(Ytrain)
Ytrain_integer = np.array(Ytrain)
Yvalidation_encoded = np_utils.to_categorical(Yvalidation)
Yvalidation_integer = np.array(Yvalidation)
# Zero pad (encode) Xtrain and Xvalidation
# ==================================================================================================
tokenizer = Tokenizer(filters='!"$%&()*+,-./:;<=>?@[\]^_`{|}~'
) #TODO depending on word embedding, set lower=False.
tokenizer.fit_on_texts(np.append(np.array(Xtrain), np.array(Xvalidation)))
# tokenizer.fit_on_texts(Xtrain)