def get_data(config, results_dir): config = load_config_from_path(config) data_raw = Dataset(config) data = SCAN_HS( load_observations(data_raw.get_train_dataloader(shuffle=False)), load_observations(data_raw.get_dev_dataloader(shuffle=False)), load_observations(data_raw.get_test_dataloader(shuffle=False)), ) hidden_states = SCAN_HS(np.load(f"{results_dir}/cache.train.npy"), np.load(f"{results_dir}/cache.dev.npy"), np.load(f"{results_dir}/cache.test.npy")) sent_idx_slices = SCAN_HS( get_sequence_cutoffs(hidden_states.train), get_sequence_cutoffs(hidden_states.dev), get_sequence_cutoffs(hidden_states.test), ) token_idxs = SCAN_HS(None, get_token_idxs(data.dev), get_token_idxs(data.test)) return data_raw, data, hidden_states, sent_idx_slices, token_idxs