def preprocess_samples_discrete(
self,
states: List[Dict[int, float]],
actions: List[str],
rewards: List[float],
next_states: List[Dict[int, float]],
next_actions: List[str],
is_terminals: List[bool],
possible_next_actions: List[List[str]],
reward_timelines: Optional[List[Dict[int, float]]],
minibatch_size: int,
) -> List[TrainingDataPage]:
# Shuffle
if reward_timelines is None:
merged = list(
zip(states, actions, rewards, next_states, next_actions,
is_terminals, possible_next_actions))
random.shuffle(merged)
states, actions, rewards, next_states, next_actions, \
is_terminals, possible_next_actions = zip(*merged)
else:
merged = list(
zip(states, actions, rewards, next_states, next_actions,
is_terminals, possible_next_actions, reward_timelines))
random.shuffle(merged)
states, actions, rewards, next_states, next_actions, \
is_terminals, possible_next_actions, reward_timelines = zip(*merged)
net = core.Net('gridworld_preprocessing')
C2.set_net(net)
preprocessor = PreprocessorNet(net, True)
saa = StackedAssociativeArray.from_dict_list(states, 'states')
state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
'state_norm',
)
saa = StackedAssociativeArray.from_dict_list(next_states,
'next_states')
next_state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
'next_state_norm',
)
workspace.RunNetOnce(net)
actions_one_hot = np.zeros(
[len(actions), len(self.ACTIONS)], dtype=np.float32)
for i, action in enumerate(actions):
actions_one_hot[i, self.ACTIONS.index(action)] = 1
rewards = np.array(rewards, dtype=np.float32).reshape(-1, 1)
next_actions_one_hot = np.zeros(
[len(next_actions), len(self.ACTIONS)], dtype=np.float32)
for i, action in enumerate(next_actions):
if action == '':
continue
next_actions_one_hot[i, self.ACTIONS.index(action)] = 1
possible_next_actions_mask = []
for pna in possible_next_actions:
pna_mask = [0] * self.num_actions
for action in pna:
pna_mask[self.ACTIONS.index(action)] = 1
possible_next_actions_mask.append(pna_mask)
possible_next_actions_mask = np.array(possible_next_actions_mask,
dtype=np.float32)
is_terminals = np.array(is_terminals, dtype=np.bool).reshape(-1, 1)
not_terminals = np.logical_not(is_terminals)
if reward_timelines is not None:
reward_timelines = np.array(reward_timelines, dtype=np.object)
states_ndarray = workspace.FetchBlob(state_matrix)
next_states_ndarray = workspace.FetchBlob(next_state_matrix)
tdps = []
for start in range(0, states_ndarray.shape[0], minibatch_size):
end = start + minibatch_size
if end > states_ndarray.shape[0]:
break
tdps.append(
TrainingDataPage(
states=states_ndarray[start:end],
actions=actions_one_hot[start:end],
rewards=rewards[start:end],
next_states=next_states_ndarray[start:end],
not_terminals=not_terminals[start:end],
next_actions=next_actions_one_hot[start:end],
possible_next_actions=possible_next_actions_mask[
start:end],
reward_timelines=reward_timelines[start:end]
if reward_timelines is not None else None,
))
return tdps
def preprocess_samples_discrete(
self,
samples: Samples,
minibatch_size: int,
one_hot_action: bool = True) -> List[TrainingDataPage]:
logger.info("Shuffling...")
samples.shuffle()
logger.info("Preprocessing...")
net = core.Net("gridworld_preprocessing")
C2.set_net(net)
preprocessor = PreprocessorNet(True)
saa = StackedAssociativeArray.from_dict_list(samples.states, "states")
state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
"state_norm",
False,
False,
False,
)
saa = StackedAssociativeArray.from_dict_list(samples.next_states,
"next_states")
next_state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
"next_state_norm",
False,
False,
False,
)
workspace.RunNetOnce(net)
logger.info("Converting to Torch...")
actions_one_hot = torch.tensor((np.array(samples.actions).reshape(
-1, 1) == np.array(self.ACTIONS)).astype(np.int64))
actions = actions_one_hot.argmax(dim=1, keepdim=True)
rewards = torch.tensor(samples.rewards,
dtype=torch.float32).reshape(-1, 1)
action_probabilities = torch.tensor(samples.action_probabilities,
dtype=torch.float32).reshape(
-1, 1)
next_actions_one_hot = torch.tensor(
(np.array(samples.next_actions).reshape(-1, 1) == np.array(
self.ACTIONS)).astype(np.int64))
logger.info("Converting PNA to Torch...")
possible_next_action_strings = np.array(
list(
itertools.zip_longest(*samples.possible_next_actions,
fillvalue=""))).T
possible_next_actions_mask = torch.zeros(
[len(samples.next_actions),
len(self.ACTIONS)])
for i, action in enumerate(self.ACTIONS):
possible_next_actions_mask[:, i] = torch.tensor(
np.max(possible_next_action_strings == action,
axis=1).astype(np.int64))
terminals = torch.tensor(samples.terminals,
dtype=torch.int32).reshape(-1, 1)
not_terminals = 1 - terminals
episode_values = None
logger.info("Converting RT to Torch...")
episode_values = torch.tensor(samples.episode_values,
dtype=torch.float32).reshape(-1, 1)
time_diffs = torch.ones([len(samples.states), 1])
logger.info("Preprocessing...")
preprocessor = Preprocessor(self.normalization, False)
states_ndarray = workspace.FetchBlob(state_matrix)
states_ndarray = preprocessor.forward(states_ndarray)
next_states_ndarray = workspace.FetchBlob(next_state_matrix)
next_states_ndarray = preprocessor.forward(next_states_ndarray)
logger.info("Batching...")
tdps = []
for start in range(0, states_ndarray.shape[0], minibatch_size):
end = start + minibatch_size
if end > states_ndarray.shape[0]:
break
tdp = TrainingDataPage(
states=states_ndarray[start:end],
actions=actions_one_hot[start:end]
if one_hot_action else actions[start:end],
propensities=action_probabilities[start:end],
rewards=rewards[start:end],
next_states=next_states_ndarray[start:end],
not_terminals=not_terminals[start:end],
next_actions=next_actions_one_hot[start:end],
possible_next_actions=possible_next_actions_mask[start:end],
episode_values=episode_values[start:end]
if episode_values is not None else None,
time_diffs=time_diffs[start:end],
)
tdp.set_type(torch.FloatTensor)
tdps.append(tdp)
return tdps
def preprocess_samples(
self,
samples: Samples,
minibatch_size: int,
use_gpu: bool = False,
one_hot_action: bool = True,
normalize_actions: bool = True,
) -> List[TrainingDataPage]:
logger.info("Shuffling...")
samples = shuffle_samples(samples)
logger.info("Sparse2Dense...")
net = core.Net("gridworld_preprocessing")
C2.set_net(net)
sorted_state_features, _ = sort_features_by_normalization(
self.normalization)
sorted_action_features, _ = sort_features_by_normalization(
self.normalization_action)
state_sparse_to_dense_processor = Caffe2SparseToDenseProcessor(
sorted_state_features)
action_sparse_to_dense_processor = Caffe2SparseToDenseProcessor(
sorted_action_features)
saa = StackedAssociativeArray.from_dict_list(samples.states, "states")
state_matrix, state_matrix_presence, _ = state_sparse_to_dense_processor(
saa)
saa = StackedAssociativeArray.from_dict_list(samples.next_states,
"next_states")
next_state_matrix, next_state_matrix_presence, _ = state_sparse_to_dense_processor(
saa)
saa = StackedAssociativeArray.from_dict_list( # type: ignore
samples.actions, "action")
action_matrix, action_matrix_presence, _ = action_sparse_to_dense_processor(
saa)
saa = StackedAssociativeArray.from_dict_list( # type: ignore
samples.next_actions, "next_action")
next_action_matrix, next_action_matrix_presence, _ = action_sparse_to_dense_processor(
saa)
action_probabilities = torch.tensor(samples.action_probabilities,
dtype=torch.float32).reshape(
-1, 1)
rewards = torch.tensor(samples.rewards,
dtype=torch.float32).reshape(-1, 1)
max_action_size = 4
pnas_mask_list: List[List[int]] = []
pnas_flat: List[Dict[str, float]] = []
for pnas in samples.possible_next_actions:
pnas_mask_list.append([1] * len(pnas) + [0] *
(max_action_size - len(pnas)))
pnas_flat.extend(pnas) # type: ignore
for _ in range(max_action_size - len(pnas)):
pnas_flat.append({}) # Filler
saa = StackedAssociativeArray.from_dict_list( # type: ignore
pnas_flat, "possible_next_actions")
pnas_mask = torch.Tensor(pnas_mask_list)
possible_next_actions_matrix, possible_next_actions_matrix_presence, _ = action_sparse_to_dense_processor(
saa)
workspace.RunNetOnce(net)
logger.info("Preprocessing...")
state_preprocessor = Preprocessor(self.normalization, False)
action_preprocessor = Preprocessor(self.normalization_action, False)
states_ndarray = state_preprocessor(
torch.from_numpy(workspace.FetchBlob(state_matrix)),
torch.from_numpy(
workspace.FetchBlob(state_matrix_presence)).float(),
)
if normalize_actions:
actions_ndarray = action_preprocessor(
torch.from_numpy(workspace.FetchBlob(action_matrix)),
torch.from_numpy(
workspace.FetchBlob(action_matrix_presence)).float(),
)
else:
actions_ndarray = torch.from_numpy(
workspace.FetchBlob(action_matrix))
next_states_ndarray = torch.from_numpy(
workspace.FetchBlob(next_state_matrix))
next_states_ndarray = state_preprocessor(
next_states_ndarray,
(next_states_ndarray != MISSING_VALUE).float())
state_pnas_tile = next_states_ndarray.repeat(
1, max_action_size).reshape(-1, next_states_ndarray.shape[1])
if normalize_actions:
next_actions_ndarray = action_preprocessor(
torch.from_numpy(workspace.FetchBlob(next_action_matrix)),
torch.from_numpy(
workspace.FetchBlob(next_action_matrix_presence)).float(),
)
else:
next_actions_ndarray = torch.from_numpy(
workspace.FetchBlob(next_action_matrix))
if normalize_actions:
logged_possible_next_actions = action_preprocessor(
torch.from_numpy(
workspace.FetchBlob(possible_next_actions_matrix)),
torch.from_numpy(
workspace.FetchBlob(
possible_next_actions_matrix_presence)).float(),
)
else:
logged_possible_next_actions = torch.from_numpy(
workspace.FetchBlob(possible_next_actions_matrix))
assert state_pnas_tile.shape[0] == logged_possible_next_actions.shape[
0], ("Invalid shapes: " + str(state_pnas_tile.shape) + " != " +
str(logged_possible_next_actions.shape))
logged_possible_next_state_actions = torch.cat(
(state_pnas_tile, logged_possible_next_actions), dim=1)
logger.info("Reward Timeline to Torch...")
time_diffs = torch.ones([len(samples.states), 1])
tdps = []
pnas_start = 0
logger.info("Batching...")
for start in range(0, states_ndarray.shape[0], minibatch_size):
end = start + minibatch_size
if end > states_ndarray.shape[0]:
break
pnas_end = pnas_start + (minibatch_size * max_action_size)
tdp = TrainingDataPage(
states=states_ndarray[start:end],
actions=actions_ndarray[start:end],
propensities=action_probabilities[start:end],
rewards=rewards[start:end],
next_states=next_states_ndarray[start:end],
next_actions=next_actions_ndarray[start:end],
not_terminal=(pnas_mask[start:end, :].sum(dim=1, keepdim=True)
> 0),
time_diffs=time_diffs[start:end],
possible_next_actions_mask=pnas_mask[start:end, :],
possible_next_actions_state_concat=
logged_possible_next_state_actions[pnas_start:pnas_end, :],
)
pnas_start = pnas_end
tdp.set_type(torch.cuda.FloatTensor if use_gpu else torch.
FloatTensor # type: ignore
)
tdps.append(tdp)
return tdps
def preprocess_samples(
self,
samples: Samples,
minibatch_size: int,
use_gpu: bool = False,
one_hot_action: bool = True,
normalize_actions: bool = True,
) -> List[TrainingDataPage]:
logger.info("Shuffling...")
samples.shuffle()
logger.info("Sparse2Dense...")
net = core.Net("gridworld_preprocessing")
C2.set_net(net)
saa = StackedAssociativeArray.from_dict_list(samples.states, "states")
sorted_state_features, _ = sort_features_by_normalization(self.normalization)
state_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_state_features
)
saa = StackedAssociativeArray.from_dict_list(samples.next_states, "next_states")
next_state_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_state_features
)
sorted_action_features, _ = sort_features_by_normalization(
self.normalization_action
)
saa = StackedAssociativeArray.from_dict_list(samples.actions, "action")
action_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_action_features
)
saa = StackedAssociativeArray.from_dict_list(
samples.next_actions, "next_action"
)
next_action_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_action_features
)
action_probabilities = torch.tensor(
samples.action_probabilities, dtype=torch.float32
).reshape(-1, 1)
rewards = torch.tensor(samples.rewards, dtype=torch.float32).reshape(-1, 1)
pnas_lengths_list = []
pnas_flat: List[List[str]] = []
for pnas in samples.possible_next_actions:
pnas_lengths_list.append(len(pnas))
pnas_flat.extend(pnas)
saa = StackedAssociativeArray.from_dict_list(pnas_flat, "possible_next_actions")
pnas_lengths = torch.tensor(pnas_lengths_list, dtype=torch.int32)
pna_lens_blob = "pna_lens_blob"
workspace.FeedBlob(pna_lens_blob, pnas_lengths.numpy())
possible_next_actions_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_action_features
)
state_pnas_tile_blob = C2.LengthsTile(next_state_matrix, pna_lens_blob)
workspace.RunNetOnce(net)
logger.info("Preprocessing...")
state_preprocessor = Preprocessor(self.normalization, False)
action_preprocessor = Preprocessor(self.normalization_action, False)
states_ndarray = workspace.FetchBlob(state_matrix)
states_ndarray = state_preprocessor.forward(states_ndarray)
actions_ndarray = torch.from_numpy(workspace.FetchBlob(action_matrix))
if normalize_actions:
actions_ndarray = action_preprocessor.forward(actions_ndarray)
next_states_ndarray = workspace.FetchBlob(next_state_matrix)
next_states_ndarray = state_preprocessor.forward(next_states_ndarray)
next_actions_ndarray = torch.from_numpy(workspace.FetchBlob(next_action_matrix))
if normalize_actions:
next_actions_ndarray = action_preprocessor.forward(next_actions_ndarray)
logged_possible_next_actions = action_preprocessor.forward(
workspace.FetchBlob(possible_next_actions_matrix)
)
state_pnas_tile = state_preprocessor.forward(
workspace.FetchBlob(state_pnas_tile_blob)
)
logged_possible_next_state_actions = torch.cat(
(state_pnas_tile, logged_possible_next_actions), dim=1
)
logger.info("Reward Timeline to Torch...")
possible_next_actions_ndarray = logged_possible_next_actions
possible_next_actions_state_concat = logged_possible_next_state_actions
time_diffs = torch.ones([len(samples.states), 1])
tdps = []
pnas_start = 0
logger.info("Batching...")
for start in range(0, states_ndarray.shape[0], minibatch_size):
end = start + minibatch_size
if end > states_ndarray.shape[0]:
break
pnas_end = pnas_start + torch.sum(pnas_lengths[start:end])
pnas = possible_next_actions_ndarray[pnas_start:pnas_end]
pnas_concat = possible_next_actions_state_concat[pnas_start:pnas_end]
pnas_start = pnas_end
tdp = TrainingDataPage(
states=states_ndarray[start:end],
actions=actions_ndarray[start:end],
propensities=action_probabilities[start:end],
rewards=rewards[start:end],
next_states=next_states_ndarray[start:end],
next_actions=next_actions_ndarray[start:end],
possible_next_actions=None,
not_terminals=(pnas_lengths[start:end] > 0).reshape(-1, 1),
time_diffs=time_diffs[start:end],
possible_next_actions_lengths=pnas_lengths[start:end],
possible_next_actions_state_concat=pnas_concat,
)
tdp.set_type(torch.cuda.FloatTensor if use_gpu else torch.FloatTensor)
tdps.append(tdp)
return tdps
def preprocess_samples(self, samples: Samples,
minibatch_size: int) -> List[TrainingDataPage]:
samples.shuffle()
net = core.Net("gridworld_preprocessing")
C2.set_net(net)
preprocessor = PreprocessorNet(True)
saa = StackedAssociativeArray.from_dict_list(samples.states, "states")
state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
"state_norm",
False,
False,
)
saa = StackedAssociativeArray.from_dict_list(samples.next_states,
"next_states")
next_state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
"next_state_norm",
False,
False,
)
saa = StackedAssociativeArray.from_dict_list(samples.actions, "action")
action_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization_action,
"action_norm",
False,
False,
)
saa = StackedAssociativeArray.from_dict_list(samples.next_actions,
"next_action")
next_action_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization_action,
"next_action_norm",
False,
False,
)
propensities = np.array(samples.propensities,
dtype=np.float32).reshape(-1, 1)
rewards = np.array(samples.rewards, dtype=np.float32).reshape(-1, 1)
pnas_lengths_list = []
pnas_flat: List[List[str]] = []
for pnas in samples.possible_next_actions:
pnas_lengths_list.append(len(pnas))
pnas_flat.extend(pnas)
saa = StackedAssociativeArray.from_dict_list(pnas_flat,
"possible_next_actions")
pnas_lengths = np.array(pnas_lengths_list, dtype=np.int32)
possible_next_actions_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization_action,
"possible_next_action_norm",
False,
False,
)
workspace.RunNetOnce(net)
states_ndarray = workspace.FetchBlob(state_matrix)
actions_ndarray = workspace.FetchBlob(action_matrix)
next_states_ndarray = workspace.FetchBlob(next_state_matrix)
next_actions_ndarray = workspace.FetchBlob(next_action_matrix)
possible_next_actions_ndarray = workspace.FetchBlob(
possible_next_actions_matrix)
tdps = []
pnas_start = 0
for start in range(0, states_ndarray.shape[0], minibatch_size):
end = start + minibatch_size
if end > states_ndarray.shape[0]:
break
pnas_end = pnas_start + np.sum(pnas_lengths[start:end])
pnas = possible_next_actions_ndarray[pnas_start:pnas_end]
pnas_start = pnas_end
tdps.append(
TrainingDataPage(
states=states_ndarray[start:end],
actions=actions_ndarray[start:end],
propensities=propensities[start:end],
rewards=rewards[start:end],
next_states=next_states_ndarray[start:end],
next_actions=next_actions_ndarray[start:end],
possible_next_actions=StackedArray(pnas_lengths[start:end],
pnas),
not_terminals=(pnas_lengths[start:end] > 0).reshape(-1, 1),
reward_timelines=samples.reward_timelines[start:end]
if samples.reward_timelines else None,
))
return tdps
def preprocess_samples_discrete(
self, samples: Samples,
minibatch_size: int) -> List[TrainingDataPage]:
samples.shuffle()
net = core.Net("gridworld_preprocessing")
C2.set_net(net)
preprocessor = PreprocessorNet(True)
saa = StackedAssociativeArray.from_dict_list(samples.states, "states")
state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
"state_norm",
False,
False,
)
saa = StackedAssociativeArray.from_dict_list(samples.next_states,
"next_states")
next_state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
"next_state_norm",
False,
False,
)
workspace.RunNetOnce(net)
actions_one_hot = np.zeros(
[len(samples.actions), len(self.ACTIONS)], dtype=np.float32)
for i, action in enumerate(samples.actions):
actions_one_hot[i, self.action_to_index(action)] = 1
rewards = np.array(samples.rewards, dtype=np.float32).reshape(-1, 1)
propensities = np.array(samples.propensities,
dtype=np.float32).reshape(-1, 1)
next_actions_one_hot = np.zeros(
[len(samples.next_actions),
len(self.ACTIONS)], dtype=np.float32)
for i, action in enumerate(samples.next_actions):
if action == "":
continue
next_actions_one_hot[i, self.action_to_index(action)] = 1
possible_next_actions_mask = []
for pna in samples.possible_next_actions:
pna_mask = [0] * self.num_actions
for action in pna:
pna_mask[self.action_to_index(action)] = 1
possible_next_actions_mask.append(pna_mask)
possible_next_actions_mask = np.array(possible_next_actions_mask,
dtype=np.float32)
is_terminals = np.array(samples.is_terminal,
dtype=np.bool).reshape(-1, 1)
not_terminals = np.logical_not(is_terminals)
if samples.reward_timelines is not None:
reward_timelines = np.array(samples.reward_timelines,
dtype=np.object)
states_ndarray = workspace.FetchBlob(state_matrix)
next_states_ndarray = workspace.FetchBlob(next_state_matrix)
tdps = []
for start in range(0, states_ndarray.shape[0], minibatch_size):
end = start + minibatch_size
if end > states_ndarray.shape[0]:
break
tdps.append(
TrainingDataPage(
states=states_ndarray[start:end],
actions=actions_one_hot[start:end],
propensities=propensities[start:end],
rewards=rewards[start:end],
next_states=next_states_ndarray[start:end],
not_terminals=not_terminals[start:end],
next_actions=next_actions_one_hot[start:end],
possible_next_actions=possible_next_actions_mask[
start:end],
reward_timelines=reward_timelines[start:end]
if reward_timelines is not None else None,
))
return tdps
def preprocess_samples_discrete(
self,
samples: Samples,
minibatch_size: int,
one_hot_action: bool = True,
use_gpu: bool = False,
do_shuffle: bool = True,
) -> List[TrainingDataPage]:
if do_shuffle:
logger.info("Shuffling...")
samples = shuffle_samples(samples)
logger.info("Preprocessing...")
sparse_to_dense_processor = Caffe2SparseToDenseProcessor()
if self.sparse_to_dense_net is None:
self.sparse_to_dense_net = core.Net("gridworld_sparse_to_dense")
C2.set_net(self.sparse_to_dense_net)
saa = StackedAssociativeArray.from_dict_list(samples.states, "states")
sorted_features, _ = sort_features_by_normalization(self.normalization)
self.state_matrix, _ = sparse_to_dense_processor(sorted_features, saa)
saa = StackedAssociativeArray.from_dict_list(
samples.next_states, "next_states"
)
self.next_state_matrix, _ = sparse_to_dense_processor(sorted_features, saa)
C2.set_net(None)
else:
StackedAssociativeArray.from_dict_list(samples.states, "states")
StackedAssociativeArray.from_dict_list(samples.next_states, "next_states")
workspace.RunNetOnce(self.sparse_to_dense_net)
logger.info("Converting to Torch...")
actions_one_hot = torch.tensor(
(np.array(samples.actions).reshape(-1, 1) == np.array(self.ACTIONS)).astype(
np.int64
)
)
actions = actions_one_hot.argmax(dim=1, keepdim=True)
rewards = torch.tensor(samples.rewards, dtype=torch.float32).reshape(-1, 1)
action_probabilities = torch.tensor(
samples.action_probabilities, dtype=torch.float32
).reshape(-1, 1)
next_actions_one_hot = torch.tensor(
(
np.array(samples.next_actions).reshape(-1, 1) == np.array(self.ACTIONS)
).astype(np.int64)
)
logger.info("Converting PA to Torch...")
possible_action_strings = np.array(
list(itertools.zip_longest(*samples.possible_actions, fillvalue=""))
).T
possible_actions_mask = torch.zeros([len(samples.actions), len(self.ACTIONS)])
for i, action in enumerate(self.ACTIONS):
possible_actions_mask[:, i] = torch.tensor(
np.max(possible_action_strings == action, axis=1).astype(np.int64)
)
logger.info("Converting PNA to Torch...")
possible_next_action_strings = np.array(
list(itertools.zip_longest(*samples.possible_next_actions, fillvalue=""))
).T
possible_next_actions_mask = torch.zeros(
[len(samples.next_actions), len(self.ACTIONS)]
)
for i, action in enumerate(self.ACTIONS):
possible_next_actions_mask[:, i] = torch.tensor(
np.max(possible_next_action_strings == action, axis=1).astype(np.int64)
)
terminals = torch.tensor(samples.terminals, dtype=torch.int32).reshape(-1, 1)
not_terminal = 1 - terminals
logger.info("Converting RT to Torch...")
time_diffs = torch.ones([len(samples.states), 1])
logger.info("Preprocessing...")
preprocessor = Preprocessor(self.normalization, False)
states_ndarray = workspace.FetchBlob(self.state_matrix)
states_ndarray = preprocessor.forward(states_ndarray)
next_states_ndarray = workspace.FetchBlob(self.next_state_matrix)
next_states_ndarray = preprocessor.forward(next_states_ndarray)
logger.info("Batching...")
tdps = []
for start in range(0, states_ndarray.shape[0], minibatch_size):
end = start + minibatch_size
if end > states_ndarray.shape[0]:
break
tdp = TrainingDataPage(
states=states_ndarray[start:end],
actions=actions_one_hot[start:end]
if one_hot_action
else actions[start:end],
propensities=action_probabilities[start:end],
rewards=rewards[start:end],
next_states=next_states_ndarray[start:end],
not_terminal=not_terminal[start:end],
next_actions=next_actions_one_hot[start:end],
possible_actions_mask=possible_actions_mask[start:end],
possible_next_actions_mask=possible_next_actions_mask[start:end],
time_diffs=time_diffs[start:end],
)
tdp.set_type(torch.cuda.FloatTensor if use_gpu else torch.FloatTensor)
tdps.append(tdp)
return tdps
def preprocess_samples(self, samples: Samples,
minibatch_size: int) -> List[TrainingDataPage]:
samples.shuffle()
net = core.Net("gridworld_preprocessing")
C2.set_net(net)
preprocessor = PreprocessorNet(True)
saa = StackedAssociativeArray.from_dict_list(samples.states, "states")
state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
"state_norm",
False,
False,
False,
)
saa = StackedAssociativeArray.from_dict_list(samples.next_states,
"next_states")
next_state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
"next_state_norm",
False,
False,
False,
)
saa = StackedAssociativeArray.from_dict_list(samples.actions, "action")
action_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization_action,
"action_norm",
False,
False,
False,
)
saa = StackedAssociativeArray.from_dict_list(samples.next_actions,
"next_action")
next_action_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization_action,
"next_action_norm",
False,
False,
False,
)
propensities = np.array(samples.propensities,
dtype=np.float32).reshape(-1, 1)
rewards = np.array(samples.rewards, dtype=np.float32).reshape(-1, 1)
pnas_lengths_list = []
pnas_flat: List[List[str]] = []
for pnas in samples.possible_next_actions:
pnas_lengths_list.append(len(pnas))
pnas_flat.extend(pnas)
saa = StackedAssociativeArray.from_dict_list(pnas_flat,
"possible_next_actions")
pnas_lengths = np.array(pnas_lengths_list, dtype=np.int32)
pna_lens_blob = "pna_lens_blob"
workspace.FeedBlob(pna_lens_blob, pnas_lengths)
possible_next_actions_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization_action,
"possible_next_action_norm",
False,
False,
False,
)
state_pnas_tile_blob = C2.LengthsTile(next_state_matrix, pna_lens_blob)
workspace.RunNetOnce(net)
state_preprocessor = Preprocessor(self.normalization, False)
action_preprocessor = Preprocessor(self.normalization_action, False)
states_ndarray = workspace.FetchBlob(state_matrix)
states_ndarray = state_preprocessor.forward(states_ndarray).numpy()
actions_ndarray = workspace.FetchBlob(action_matrix)
actions_ndarray = action_preprocessor.forward(actions_ndarray).numpy()
next_states_ndarray = workspace.FetchBlob(next_state_matrix)
next_states_ndarray = state_preprocessor.forward(
next_states_ndarray).numpy()
next_actions_ndarray = workspace.FetchBlob(next_action_matrix)
next_actions_ndarray = action_preprocessor.forward(
next_actions_ndarray).numpy()
logged_possible_next_actions = action_preprocessor.forward(
workspace.FetchBlob(possible_next_actions_matrix))
state_pnas_tile = state_preprocessor.forward(
workspace.FetchBlob(state_pnas_tile_blob))
logged_possible_next_state_actions = torch.cat(
(state_pnas_tile, logged_possible_next_actions), dim=1)
possible_next_actions_ndarray = logged_possible_next_actions.cpu(
).numpy()
next_state_pnas_concat = logged_possible_next_state_actions.cpu(
).numpy()
time_diffs = np.ones(len(states_ndarray))
episode_values = None
if samples.reward_timelines is not None:
episode_values = np.zeros(rewards.shape, dtype=np.float32)
for i, reward_timeline in enumerate(samples.reward_timelines):
for time_diff, reward in reward_timeline.items():
episode_values[i, 0] += reward * (DISCOUNT**time_diff)
tdps = []
pnas_start = 0
for start in range(0, states_ndarray.shape[0], minibatch_size):
end = start + minibatch_size
if end > states_ndarray.shape[0]:
break
pnas_end = pnas_start + np.sum(pnas_lengths[start:end])
pnas = possible_next_actions_ndarray[pnas_start:pnas_end]
pnas_concat = next_state_pnas_concat[pnas_start:pnas_end]
pnas_start = pnas_end
tdps.append(
TrainingDataPage(
states=states_ndarray[start:end],
actions=actions_ndarray[start:end],
propensities=propensities[start:end],
rewards=rewards[start:end],
next_states=next_states_ndarray[start:end],
next_actions=next_actions_ndarray[start:end],
possible_next_actions=StackedArray(pnas_lengths[start:end],
pnas),
not_terminals=(pnas_lengths[start:end] > 0).reshape(-1, 1),
episode_values=episode_values[start:end]
if episode_values is not None else None,
time_diffs=time_diffs[start:end],
possible_next_actions_lengths=pnas_lengths[start:end],
next_state_pnas_concat=pnas_concat,
))
return tdps
def preprocess_samples(
self,
states: List[Dict[int, float]],
actions: List[Dict[int, float]],
rewards: List[float],
next_states: List[Dict[int, float]],
next_actions: List[Dict[int, float]],
is_terminals: List[bool],
possible_next_actions: List[List[Dict[int, float]]],
reward_timelines: List[Dict[int, float]],
minibatch_size: int,
) -> List[TrainingDataPage]:
# Shuffle
merged = list(
zip(states, actions, rewards, next_states, next_actions,
is_terminals, possible_next_actions, reward_timelines))
random.shuffle(merged)
states, actions, rewards, next_states, next_actions, is_terminals, \
possible_next_actions, reward_timelines = zip(*merged)
net = core.Net('gridworld_preprocessing')
C2.set_net(net)
preprocessor = PreprocessorNet(net, True)
saa = StackedAssociativeArray.from_dict_list(states, 'states')
state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
'state_norm',
)
saa = StackedAssociativeArray.from_dict_list(next_states,
'next_states')
next_state_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization,
'next_state_norm',
)
saa = StackedAssociativeArray.from_dict_list(actions, 'action')
action_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization_action,
'action_norm',
)
saa = StackedAssociativeArray.from_dict_list(next_actions,
'next_action')
next_action_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization_action,
'next_action_norm',
)
rewards = np.array(rewards, dtype=np.float32).reshape(-1, 1)
pnas_lengths_list = []
pnas_flat = []
for pnas in possible_next_actions:
pnas_lengths_list.append(len(pnas))
pnas_flat.extend(pnas)
saa = StackedAssociativeArray.from_dict_list(pnas_flat,
'possible_next_actions')
pnas_lengths = np.array(pnas_lengths_list, dtype=np.int32)
possible_next_actions_matrix, _ = preprocessor.normalize_sparse_matrix(
saa.lengths,
saa.keys,
saa.values,
self.normalization_action,
'possible_next_action_norm',
)
workspace.RunNetOnce(net)
states_ndarray = workspace.FetchBlob(state_matrix)
actions_ndarray = workspace.FetchBlob(action_matrix)
next_states_ndarray = workspace.FetchBlob(next_state_matrix)
next_actions_ndarray = workspace.FetchBlob(next_action_matrix)
possible_next_actions_ndarray = workspace.FetchBlob(
possible_next_actions_matrix)
tdps = []
pnas_start = 0
for start in range(0, states_ndarray.shape[0], minibatch_size):
end = start + minibatch_size
if end > states_ndarray.shape[0]:
break
pnas_end = pnas_start + np.sum(pnas_lengths[start:end])
pnas = possible_next_actions_ndarray[pnas_start:pnas_end]
pnas_start = pnas_end
tdps.append(
TrainingDataPage(
states=states_ndarray[start:end],
actions=actions_ndarray[start:end],
rewards=rewards[start:end],
next_states=next_states_ndarray[start:end],
next_actions=next_actions_ndarray[start:end],
possible_next_actions=StackedArray(pnas_lengths[start:end],
pnas),
not_terminals=(pnas_lengths[start:end] > 0).reshape(-1, 1),
reward_timelines=reward_timelines[start:end]
if reward_timelines else None,
))
return tdps
def preprocess_samples(
self,
samples: Samples,
minibatch_size: int,
use_gpu: bool = False,
one_hot_action: bool = True,
normalize_actions: bool = True,
) -> List[TrainingDataPage]:
logger.info("Shuffling...")
samples = shuffle_samples(samples)
logger.info("Sparse2Dense...")
net = core.Net("gridworld_preprocessing")
C2.set_net(net)
saa = StackedAssociativeArray.from_dict_list(samples.states, "states")
sorted_state_features, _ = sort_features_by_normalization(self.normalization)
state_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_state_features
)
saa = StackedAssociativeArray.from_dict_list(samples.next_states, "next_states")
next_state_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_state_features
)
sorted_action_features, _ = sort_features_by_normalization(
self.normalization_action
)
saa = StackedAssociativeArray.from_dict_list(samples.actions, "action")
action_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_action_features
)
saa = StackedAssociativeArray.from_dict_list(
samples.next_actions, "next_action"
)
next_action_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_action_features
)
action_probabilities = torch.tensor(
samples.action_probabilities, dtype=torch.float32
).reshape(-1, 1)
rewards = torch.tensor(samples.rewards, dtype=torch.float32).reshape(-1, 1)
max_action_size = 4
pnas_mask_list: List[List[int]] = []
pnas_flat: List[Dict[str, float]] = []
for pnas in samples.possible_next_actions:
pnas_mask_list.append([1] * len(pnas) + [0] * (max_action_size - len(pnas)))
pnas_flat.extend(pnas)
for _ in range(max_action_size - len(pnas)):
pnas_flat.append({}) # Filler
saa = StackedAssociativeArray.from_dict_list(pnas_flat, "possible_next_actions")
pnas_mask = torch.Tensor(pnas_mask_list)
possible_next_actions_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_action_features
)
workspace.RunNetOnce(net)
logger.info("Preprocessing...")
state_preprocessor = Preprocessor(self.normalization, False)
action_preprocessor = Preprocessor(self.normalization_action, False)
states_ndarray = workspace.FetchBlob(state_matrix)
states_ndarray = state_preprocessor.forward(states_ndarray)
actions_ndarray = torch.from_numpy(workspace.FetchBlob(action_matrix))
if normalize_actions:
actions_ndarray = action_preprocessor.forward(actions_ndarray)
next_states_ndarray = workspace.FetchBlob(next_state_matrix)
next_states_ndarray = state_preprocessor.forward(next_states_ndarray)
state_pnas_tile = next_states_ndarray.repeat(1, max_action_size).reshape(
-1, next_states_ndarray.shape[1]
)
next_actions_ndarray = torch.from_numpy(workspace.FetchBlob(next_action_matrix))
if normalize_actions:
next_actions_ndarray = action_preprocessor.forward(next_actions_ndarray)
logged_possible_next_actions = action_preprocessor.forward(
workspace.FetchBlob(possible_next_actions_matrix)
)
assert state_pnas_tile.shape[0] == logged_possible_next_actions.shape[0], (
"Invalid shapes: "
+ str(state_pnas_tile.shape)
+ " != "
+ str(logged_possible_next_actions.shape)
)
logged_possible_next_state_actions = torch.cat(
(state_pnas_tile, logged_possible_next_actions), dim=1
)
logger.info("Reward Timeline to Torch...")
time_diffs = torch.ones([len(samples.states), 1])
tdps = []
pnas_start = 0
logger.info("Batching...")
for start in range(0, states_ndarray.shape[0], minibatch_size):
end = start + minibatch_size
if end > states_ndarray.shape[0]:
break
pnas_end = pnas_start + (minibatch_size * max_action_size)
tdp = TrainingDataPage(
states=states_ndarray[start:end],
actions=actions_ndarray[start:end],
propensities=action_probabilities[start:end],
rewards=rewards[start:end],
next_states=next_states_ndarray[start:end],
next_actions=next_actions_ndarray[start:end],
not_terminal=(pnas_mask[start:end, :].sum(dim=1, keepdim=True) > 0),
time_diffs=time_diffs[start:end],
possible_next_actions_mask=pnas_mask[start:end, :],
possible_next_actions_state_concat=logged_possible_next_state_actions[
pnas_start:pnas_end, :
],
)
pnas_start = pnas_end
tdp.set_type(torch.cuda.FloatTensor if use_gpu else torch.FloatTensor)
tdps.append(tdp)
return tdps
def preprocess_samples_discrete(
self,
samples: Samples,
minibatch_size: int,
one_hot_action: bool = True,
use_gpu: bool = False,
) -> List[TrainingDataPage]:
logger.info("Shuffling...")
samples = shuffle_samples(samples)
logger.info("Preprocessing...")
if self.sparse_to_dense_net is None:
self.sparse_to_dense_net = core.Net("gridworld_sparse_to_dense")
C2.set_net(self.sparse_to_dense_net)
saa = StackedAssociativeArray.from_dict_list(samples.states, "states")
sorted_features, _ = sort_features_by_normalization(self.normalization)
self.state_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_features
)
saa = StackedAssociativeArray.from_dict_list(
samples.next_states, "next_states"
)
self.next_state_matrix, _ = sparse_to_dense(
saa.lengths, saa.keys, saa.values, sorted_features
)
C2.set_net(None)
else:
StackedAssociativeArray.from_dict_list(samples.states, "states")
StackedAssociativeArray.from_dict_list(samples.next_states, "next_states")
workspace.RunNetOnce(self.sparse_to_dense_net)
logger.info("Converting to Torch...")
actions_one_hot = torch.tensor(
(np.array(samples.actions).reshape(-1, 1) == np.array(self.ACTIONS)).astype(
np.int64
)
)
actions = actions_one_hot.argmax(dim=1, keepdim=True)
rewards = torch.tensor(samples.rewards, dtype=torch.float32).reshape(-1, 1)
action_probabilities = torch.tensor(
samples.action_probabilities, dtype=torch.float32
).reshape(-1, 1)
next_actions_one_hot = torch.tensor(
(
np.array(samples.next_actions).reshape(-1, 1) == np.array(self.ACTIONS)
).astype(np.int64)
)
logger.info("Converting PA to Torch...")
possible_action_strings = np.array(
list(itertools.zip_longest(*samples.possible_actions, fillvalue=""))
).T
possible_actions_mask = torch.zeros([len(samples.actions), len(self.ACTIONS)])
for i, action in enumerate(self.ACTIONS):
possible_actions_mask[:, i] = torch.tensor(
np.max(possible_action_strings == action, axis=1).astype(np.int64)
)
logger.info("Converting PNA to Torch...")
possible_next_action_strings = np.array(
list(itertools.zip_longest(*samples.possible_next_actions, fillvalue=""))
).T
possible_next_actions_mask = torch.zeros(
[len(samples.next_actions), len(self.ACTIONS)]
)
for i, action in enumerate(self.ACTIONS):
possible_next_actions_mask[:, i] = torch.tensor(
np.max(possible_next_action_strings == action, axis=1).astype(np.int64)
)
terminals = torch.tensor(samples.terminals, dtype=torch.int32).reshape(-1, 1)
not_terminal = 1 - terminals
logger.info("Converting RT to Torch...")
time_diffs = torch.ones([len(samples.states), 1])
logger.info("Preprocessing...")
preprocessor = Preprocessor(self.normalization, False)
states_ndarray = workspace.FetchBlob(self.state_matrix)
states_ndarray = preprocessor.forward(states_ndarray)
next_states_ndarray = workspace.FetchBlob(self.next_state_matrix)
next_states_ndarray = preprocessor.forward(next_states_ndarray)
logger.info("Batching...")
tdps = []
for start in range(0, states_ndarray.shape[0], minibatch_size):
end = start + minibatch_size
if end > states_ndarray.shape[0]:
break
tdp = TrainingDataPage(
states=states_ndarray[start:end],
actions=actions_one_hot[start:end]
if one_hot_action
else actions[start:end],
propensities=action_probabilities[start:end],
rewards=rewards[start:end],
next_states=next_states_ndarray[start:end],
not_terminal=not_terminal[start:end],
next_actions=next_actions_one_hot[start:end],
possible_actions_mask=possible_actions_mask[start:end],
possible_next_actions_mask=possible_next_actions_mask[start:end],
time_diffs=time_diffs[start:end],
)
tdp.set_type(torch.cuda.FloatTensor if use_gpu else torch.FloatTensor)
tdps.append(tdp)
return tdps