} return layer_output_last, timestep_outputs else: return layer_output_last else: layer_output_all = pd.DataFrame(layer_output_all) return layer_output_all # Load data ##============================================================================================ Xtrain, Ytrain = data_helpers.load_all_data( config.train_path, config.validation_path, categories, shuffle=False) # I changed this so it combines train and test Xtest, Ytest = data_helpers.load_data(config.test_path, categories) Xtest_raw, Ytest_raw = data_helpers.load_data_raw(config.test_path, categories) X, y = data_helpers.load_whole_dataset(config.train_path, config.validation_path, config.test_path, categories, load_all=True, shuffle=False, one_hot=False) import importlib importlib.reload(data_helpers) ## Encode Ytrain # ===================================================================================== #one hot encode and integer encode Ytrain_encoded = np_utils.to_categorical(Ytrain)
# # importlib.reload(data_helpers) # Xtrain, Ytrain = data_helpers.load_all_data(config.train_path,config.validation_path, categories, shuffle=False) # I changed this so it combines train and validation # Xvalidation, Yvalidation = data_helpers.load_data(config.validation_path, categories) # Xvalidation_raw, Yvalidation_raw = data_helpers.load_data_raw(config.validation_path, categories) # Xtrain, Ytrain = data_helpers.load_data(config.train_path, categories) Xtrain, Ytrain = data_helpers.load_all_data(config.train_path, config.validation_path, categories, shuffle=False) Xvalidation, Yvalidation = data_helpers.load_data(config.validation_path, categories) Xvalidation_raw, Yvalidation_raw = data_helpers.load_data_raw( config.validation_path, categories) ## Encode Ytrain # ===================================================================================== #one hot encode and integer encode Ytrain_encoded = np_utils.to_categorical(Ytrain) Ytrain_integer = np.array(Ytrain) Yvalidation_encoded = np_utils.to_categorical(Yvalidation) Yvalidation_integer = np.array(Yvalidation) # Zero pad (encode) Xtrain and Xvalidation # ================================================================================================== tokenizer = Tokenizer(filters='!"$%&()*+,-./:;<=>?@[\]^_`{|}~' ) #TODO depending on word embedding, set lower=False. tokenizer.fit_on_texts(np.append(np.array(Xtrain), np.array(Xvalidation))) # tokenizer.fit_on_texts(Xtrain)