def get_data(config, results_dir):
config = load_config_from_path(config)
data_raw = Dataset(config)
data = SCAN_HS(
load_observations(data_raw.get_train_dataloader(shuffle=False)),
load_observations(data_raw.get_dev_dataloader(shuffle=False)),
load_observations(data_raw.get_test_dataloader(shuffle=False)),
)
hidden_states = SCAN_HS(np.load(f"{results_dir}/cache.train.npy"),
np.load(f"{results_dir}/cache.dev.npy"),
np.load(f"{results_dir}/cache.test.npy"))
sent_idx_slices = SCAN_HS(
get_sequence_cutoffs(hidden_states.train),
get_sequence_cutoffs(hidden_states.dev),
get_sequence_cutoffs(hidden_states.test),
)
token_idxs = SCAN_HS(None, get_token_idxs(data.dev),
get_token_idxs(data.test))
return data_raw, data, hidden_states, sent_idx_slices, token_idxs
