# Start by extracting the necessary parameters (we use a vectorized implementation).
state0_seq = []
# state1_seq = []
reward_seq = []
action_seq = []
terminal1_seq = []
for e in experiences:
state0_seq.append(e.state0)
# state1_seq.append(e.state1)
reward_seq.append(e.reward)
action_seq.append(e.action)
terminal1_seq.append(e.terminal1)
state0_seq = dqn.process_state_batch(state0_seq)
# state1_seq = dqn.process_state_batch(state1_seq)
reward_seq = np.array(reward_seq)
action_seq = np.array(action_seq, dtype=np.float32)
terminal1_seq = np.array(terminal1_seq)
hidden_states_seq = model_truncated.predict_on_batch(state0_seq)
hstates[jj, ...] = hidden_states_seq[np.newaxis, :-1, :]
actions[jj, ...] = action_seq[np.newaxis, :-1, np.newaxis]
next_hstate[jj, ...] = hidden_states_seq[np.newaxis, -1, :]
rewards[jj, ...] = reward_seq[np.newaxis, -1]
terminals[jj, ...] = terminal1_seq[np.newaxis, -1]
ml_model.fit([hstates, actions], [next_hstate, rewards, terminals],
verbose=1,
