def _install_experts(self):
im_width = self.se_io.get_image_width()
im_height = self.se_io.get_image_height()
self.input_dims = torch.Size((im_height, im_width, 3))
self.parent_rf_dims = Size2D(im_height // self.EXPERTS_IN_X,
im_width // self.EXPERTS_IN_X)
lrf_node = ReceptiveFieldNode(input_dims=self.input_dims,
parent_rf_dims=self.parent_rf_dims)
self.add_node(lrf_node)
self._top_level_flock_node = SpatialPoolerFlockNode(self.parent_params)
self._mid_node = ExpertFlockNode(self.mid_expert_params)
self._conv_node = ConvExpertFlockNode(self.conv_flock_params)
self.add_node(self._top_level_flock_node)
self.add_node(self._mid_node)
self.add_node(self._conv_node)
def rescale(inputs, outputs):
if self.TRAINING:
outputs[0].copy_(
inputs[0] *
1000) # large constant to make the label more important
else:
outputs[0].copy_(inputs[0] * float('nan'))
self._rescale_node = LambdaNode(rescale, 1,
[(self.se_io.get_num_labels(), )])
self.add_node(self._rescale_node)
unsqueeze_node_0 = UnsqueezeNode(0)
self.add_node(unsqueeze_node_0)
unsqueeze_node_1 = UnsqueezeNode(0)
self.add_node(unsqueeze_node_1)
Connector.connect(self.se_io.outputs.image_output,
lrf_node.inputs.input)
Connector.connect(lrf_node.outputs.output,
self._conv_node.inputs.sp.data_input)
Connector.connect(self._conv_node.outputs.tp.projection_outputs,
unsqueeze_node_0.inputs.input)
Connector.connect(unsqueeze_node_0.outputs.output,
self._mid_node.inputs.sp.data_input)
Connector.connect(self._mid_node.outputs.tp.projection_outputs,
self._join_node.inputs[0])
Connector.connect(self.se_io.outputs.task_to_agent_label,
self._rescale_node.inputs[0])
Connector.connect(self._rescale_node.outputs[0],
self._join_node.inputs[1])
Connector.connect(self._join_node.outputs.output,
unsqueeze_node_1.inputs.input)
Connector.connect(unsqueeze_node_1.outputs.output,
self._top_level_flock_node.inputs.sp.data_input)
def _install_experts(self):
lrf_node = ReceptiveFieldNode(input_dims=self.input_dims,
parent_rf_dims=self.parent_rf_dims)
self.add_node(lrf_node)
self._top_level_flock_node = SpatialPoolerFlockNode(self.parent_params,
name="Parent 1 SP")
self._conv_node = ConvSpatialPoolerFlockNode(self.conv_flock_params,
name="Conv SP flock")
self.add_node(self._top_level_flock_node)
self.add_node(self._conv_node)
scale = 1000
def rescale_up(inputs, outputs):
outputs[0].copy_(inputs[0] * scale)
def rescale_down(inputs, outputs):
outputs[0].copy_(inputs[0] / scale)
self._rescale_up_node = LambdaNode(rescale_up,
1, [(20, )],
name="upscale 1000")
self.add_node(self._rescale_up_node)
self._rescale_down_node = LambdaNode(
rescale_down,
1, [(1, self._top_level_expert_output_size() + 20)],
name="downscale 1000")
self.add_node(self._rescale_down_node)
unsqueeze_node = UnsqueezeNode(0)
self.add_node(unsqueeze_node)
Connector.connect(self.se_io.outputs.image_output,
lrf_node.inputs.input)
Connector.connect(lrf_node.outputs.output,
self._conv_node.inputs.sp.data_input)
Connector.connect(self._conv_node.outputs.sp.forward_clusters,
self._join_node.inputs[0])
Connector.connect(self.se_io.outputs.task_to_agent_label,
self._rescale_up_node.inputs[0])
Connector.connect(self._rescale_up_node.outputs[0],
self._join_node.inputs[1])
Connector.connect(self._join_node.outputs.output,
unsqueeze_node.inputs.input)
Connector.connect(unsqueeze_node.outputs.output,
self._top_level_flock_node.inputs.sp.data_input)
Connector.connect(
self._top_level_flock_node.outputs.sp.current_reconstructed_input,
self._rescale_down_node.inputs[0])
def _create_and_connect_agent(self, join_node: JoinNode, fork_node: ForkNode):
params = ExpertParams()
params.flock_size = 1
params.n_cluster_centers = 28
params.compute_reconstruction = True
params.spatial.cluster_boost_threshold = 1000
params.spatial.buffer_size = 500
params.spatial.batch_size = 500
params.spatial.learning_rate = 0.3
params.spatial.learning_period = 50
# conv_expert = ConvExpertFlockNode(params, name="Conv. expert")
# conv_expert = SpatialPoolerFlockNode(params, name=" SP")
conv_expert = ExpertFlockNode(params, name=" expert")
self.add_node(conv_expert)
unsqueeze_node_0 = UnsqueezeNode(0)
self.add_node(unsqueeze_node_0)
Connector.connect(join_node.outputs.output, unsqueeze_node_0.inputs.input)
Connector.connect(unsqueeze_node_0.outputs.output, conv_expert.inputs.sp.data_input)
def squeeze(inputs, outputs):
outputs[0].copy_(inputs[0].squeeze(0))
squeeze_node = LambdaNode(squeeze, 1, [(sum(fork_node._split_sizes),)],
name="squeeze lambda node")
self.add_node(squeeze_node)
Connector.connect(conv_expert.outputs.sp.predicted_reconstructed_input, squeeze_node.inputs[0])
Connector.connect(squeeze_node.outputs[0], fork_node.inputs.input)
def create_predictions_gather_node(
predicted_indexes: List[int],
num_objects: int = 1,
name: str = "Gather predicted data") -> LambdaNode:
"""Gathers just the parts of the output requested by the indexes (ids of the positions in the vector)"""
def gather_pred(inputs: List[torch.Tensor], outputs: List[torch.Tensor]):
latent_input = inputs[0]
predicted_ids = torch.tensor(predicted_indexes,
dtype=torch.long).view(1, -1)
predicted_expanded = predicted_ids.expand(latent_input.shape[0], -1)
# possible improvement
# gathered = latent_input.index_select(1, predicted_indexes)
gathered = latent_input.gather(1, predicted_expanded)
# outputs[0].copy_(gathered)
# TODO only one object supported for now
outputs[0].copy_(gathered.view(outputs[0].shape))
gather_predictions = LambdaNode(
gather_pred,
n_inputs=1,
# output_shapes=[(num_objects, len(predicted_indexes))],
# TODO only one object supported for now
output_shapes=[(len(predicted_indexes), )],
name=name)
return gather_predictions
def create_dot_product_node(input_size: int, output_sizes: List[Tuple],
name: str) -> LambdaNode:
# dot
def dot_product(inputs: List[torch.Tensor], outputs: List[torch.Tensor]):
outputs[0].copy_(inputs[0].squeeze().dot(inputs[1].squeeze()))
dot_node = LambdaNode(dot_product, input_size, output_sizes, name=name)
return dot_node
def _install_rescale(self, scale: float):
# rescale label - bigger weight
def rescale(inputs, outputs):
outputs[0].copy_(inputs[0] * scale)
self._rescale_node = LambdaNode(rescale, 1,
[(20, )]) # TODO constant here
self.add_node(self._rescale_node)
def create_arg_min_node(num_predictors: int) -> LambdaNode:
# argmin
def argmin(inputs: List[torch.Tensor], outputs: List[torch.Tensor]):
outputs[0].copy_(id_to_one_hot(inputs[0].argmin(), num_predictors))
argmin_lambda_node = LambdaNode(argmin,
1, [(num_predictors, )],
name="Argmin")
return argmin_lambda_node
def create_delay_node(num_predictors: int) -> LambdaNode:
# delay the data 1 step
def delay(inputs: List[torch.Tensor], outputs: List[torch.Tensor],
memory: List[torch.Tensor]):
outputs[0].copy_(memory[0])
memory[0].copy_(inputs[0])
delay_node = LambdaNode(delay,
1, [(num_predictors, )],
memory_shapes=[(num_predictors, )],
name="Gate Output Delay")
return delay_node
def create_discount_node(gamma: float, num_predictors: int) -> LambdaNode:
"""Returns a node that implements discount by gamma parameter"""
def discount(inputs: List[torch.Tensor], outputs: List[torch.Tensor],
memory: List[torch.Tensor]):
cum_val = (1. - gamma) * inputs[0].squeeze() + gamma * memory[0]
memory[0].copy_(cum_val)
outputs[0].copy_(cum_val)
discount_node = LambdaNode(discount,
1, [(num_predictors, )],
memory_shapes=[(num_predictors, )],
name="Discount")
return discount_node
def __init__(self,
num_predictors,
input_data_size,
name: str = "NNPredictorNodeGroup"):
super().__init__(name,
inputs=PredictorInputs(self),
outputs=PredictorOutputs(self))
p_node_params = NetworkFlockNodeParams()
p_node_params.flock_size = num_predictors
p_node_params.do_delay_coefficients = False
p_node_params.do_delay_input = True
p_node_params.normalize_coefficients = False
p_node_params.negative_coefficients_removal = True
p_node_params.buffer_size = 500
p_node_params.batch_size = 400
p_node_params.learning_period = 20
p_network_params = NeuralNetworkFlockParams()
p_network_params.flock_size = p_node_params.flock_size
p_network_params.input_size = input_data_size
p_network_params.hidden_size = input_data_size * 2
p_network_params.output_size = input_data_size
p_network_params.output_activation = OutputActivation.SOFTMAX
p_network_params.learning_rate = 0.1
p_network_params.coefficients_minimum_max = 0.3
predictors = NetworkFlockNode(node_params=p_node_params,
network_params=p_network_params,
name="predictors")
self.predictors = predictors
self.add_node(predictors)
to_one_hot_lambda_node = LambdaNode(
partial(to_one_hot, input_data_size), 1, [(input_data_size, )])
self.add_node(to_one_hot_lambda_node)
Connector.connect(self.inputs.data.output,
to_one_hot_lambda_node.inputs[0])
Connector.connect(to_one_hot_lambda_node.outputs[0],
predictors.inputs.input_data)
Connector.connect(to_one_hot_lambda_node.outputs[0],
predictors.inputs.target_data)
# Connector.connect(self.inputs.prediction_target.output, predictors.inputs.target_data)
Connector.connect(self.inputs.temporal_class_distribution.output,
predictors.inputs.learning_coefficients)
Connector.connect(predictors.outputs.prediction_output,
self.outputs.predictors_activations.input)
Connector.connect(predictors.outputs.error_output,
self.outputs.predictors_activations_error.input)
def __init__(self, num_predictors, name: str = "FakeGateNodeGroup"):
super().__init__(name,
inputs=GateInputs(self),
outputs=GateOutputs(self))
to_one_hot_lambda_node = LambdaNode(
partial(to_one_hot, num_predictors), 1, [(num_predictors, )])
self.add_node(to_one_hot_lambda_node)
# connections
Connector.connect(self.inputs.data.output,
to_one_hot_lambda_node.inputs[0])
Connector.connect(to_one_hot_lambda_node.outputs[0],
self.outputs.gate_activations.input)
def create_node_context_enhance(self) -> NodeBase:
data_window_list = []
def context_enhance(data_window: List[torch.Tensor], inputs, outputs):
"""Multiply context by coef and average over n steps"""
context = inputs[0] * self._params.gate_input_context_multiplier
# Add value to sliding window
data_window.append(context)
if len(data_window
) > self._params.gate_input_context_avg_window_size - 1:
data_window[:] = data_window[
-self._params.gate_input_context_avg_window_size:]
# average values
outputs[0].copy_(torch.mean(torch.stack(data_window), 0))
return LambdaNode(partial(context_enhance, data_window_list),
1,
[(self._params.flock_size, self._params.seq_count)],
name="Gateway input context enhance")
def create_node_extract_predicted_output(self) -> NodeBase:
specialist_extract_predicted_output_last_value = torch.zeros(
(self._params.flock_size, self._params.n_symbols))
def specialist_extract_predicted_output(last_value, inputs, outputs):
# global context_enhance_avg
output = inputs[0]
next_seq_position = self._specialist_node.params.temporal.seq_lookbehind
clusters = output[:, next_seq_position, :]
result = clusters.view(self._params.flock_size,
self._params.n_symbols)
outputs[0].copy_(last_value)
last_value.copy_(result)
return LambdaNode(partial(
specialist_extract_predicted_output,
specialist_extract_predicted_output_last_value),
1,
[(self._params.flock_size, self._params.n_symbols)],
name="Specialist extract predicted output")
def __init__(self):
super().__init__(device='cuda')
actions_descriptor = GridWorldActionDescriptor()
node_action_monitor = ActionMonitorNode(actions_descriptor)
params = GridWorldParams(map_name='MapTwoRoom',
reset_strategy=ResetStrategy.ANYWHERE)
noise_params = RandomNoiseParams(amplitude=0.0001)
node_grid_world = GridWorldNode(params)
expert_params = ExpertParams()
unsqueeze_node = UnsqueezeNode(dim=0)
noise_node = RandomNoiseNode(noise_params)
one_hot_node = ToOneHotNode()
def f(inputs, outputs):
probs = inputs[0]
outputs[0].copy_(probs[0, -1, :4] + SMALL_CONSTANT)
action_parser = LambdaNode(func=f, n_inputs=1, output_shapes=[(4, )])
expert_params.flock_size = 1
expert_params.n_cluster_centers = 64
expert_params.produce_actions = True
expert_params.temporal.seq_length = 17
expert_params.temporal.seq_lookahead = 13
expert_params.temporal.n_frequent_seqs = 700
expert_params.temporal.max_encountered_seqs = 1000
expert_params.temporal.exploration_probability = 0.05
expert_params.temporal.batch_size = 200
expert_params.temporal.buffer_size = 1000
expert_params.temporal.own_rewards_weight = 20
expert_params.temporal.frustration_threshold = 2
expert_params.temporal.compute_backward_pass = True
expert_params.compute_reconstruction = True
expert_node = ConvExpertFlockNode(expert_params)
#expert_node = ExpertFlockNode(expert_params)
self.add_node(node_grid_world)
self.add_node(node_action_monitor)
self.add_node(expert_node)
self.add_node(unsqueeze_node)
self.add_node(action_parser)
self.add_node(noise_node)
self.add_node(one_hot_node)
Connector.connect(node_grid_world.outputs.egocentric_image_action,
noise_node.inputs.input)
Connector.connect(noise_node.outputs.output,
unsqueeze_node.inputs.input)
Connector.connect(unsqueeze_node.outputs.output,
expert_node.inputs.sp.data_input)
Connector.connect(node_grid_world.outputs.reward,
expert_node.inputs.tp.reward_input)
Connector.connect(expert_node.outputs.sp.predicted_reconstructed_input,
action_parser.inputs[0])
Connector.connect(action_parser.outputs[0], one_hot_node.inputs.input)
Connector.connect(one_hot_node.outputs.output,
node_action_monitor.inputs.action_in)
Connector.connect(node_action_monitor.outputs.action_out,
node_grid_world.inputs.agent_action,
is_backward=True)
def __init__(self,
action_count=4,
location_vector_size=100,
use_grayscale: bool = False):
super().__init__("Task 1 - Basic expert",
inputs=Task1BasicExpertGroupInputs(self),
outputs=Task1BasicExpertGroupOutputs(self))
base_expert_params = ExpertParams()
base_expert_params.flock_size = 1
base_expert_params.n_cluster_centers = 100
base_expert_params.compute_reconstruction = False
base_expert_params.spatial.cluster_boost_threshold = 1000
base_expert_params.spatial.learning_rate = 0.2
base_expert_params.spatial.batch_size = 1000
base_expert_params.spatial.buffer_size = 1010
base_expert_params.spatial.learning_period = 100
parent_expert_params = ExpertParams()
parent_expert_params.flock_size = 1
parent_expert_params.n_cluster_centers = 20
parent_expert_params.compute_reconstruction = True
parent_expert_params.temporal.exploration_probability = 0.9
parent_expert_params.spatial.cluster_boost_threshold = 1000
parent_expert_params.spatial.learning_rate = 0.2
parent_expert_params.spatial.batch_size = 1000
parent_expert_params.spatial.buffer_size = 1010
parent_expert_params.spatial.learning_period = 100
parent_expert_params.temporal.context_without_rewards_size = location_vector_size
# flock-related nodes
flock_node = ExpertFlockNode(base_expert_params)
parent_flock_node = ExpertFlockNode(parent_expert_params)
join_node = JoinNode(flatten=True)
unsqueeze_node_to_base_expert = UnsqueezeNode(0)
unsqueeze_node_to_parent_expert = UnsqueezeNode(0)
fork_node = ForkNode(
0, [base_expert_params.n_cluster_centers, action_count])
def squeeze(inputs, outputs):
outputs[0].copy_(inputs[0].squeeze())
squeeze_node = LambdaNode(
squeeze,
1, [(base_expert_params.n_cluster_centers + action_count, )],
name="squeeze lambda node")
def stack_and_unsqueeze(inputs, outputs):
outputs[0].copy_(torch.stack([inputs[0], inputs[1]]).unsqueeze(0))
stack_unsqueeze_node = LambdaNode(stack_and_unsqueeze,
2, [(1, 2, location_vector_size)],
name="stack and unsqueeze node")
# add nodes to the graph
self.add_node(flock_node)
self.add_node(unsqueeze_node_to_base_expert)
self.add_node(parent_flock_node)
self.add_node(unsqueeze_node_to_parent_expert)
self.add_node(join_node)
self.add_node(fork_node)
self.add_node(squeeze_node)
self.add_node(stack_unsqueeze_node)
Connector.connect(self.inputs.actions.output, join_node.inputs[1])
if use_grayscale:
grayscale_node = GrayscaleNode(squeeze_channel=True)
self.add_node(grayscale_node)
Connector.connect(self.inputs.image.output,
grayscale_node.inputs.input)
Connector.connect(grayscale_node.outputs.output,
unsqueeze_node_to_base_expert.inputs.input)
else:
Connector.connect(self.inputs.image.output,
unsqueeze_node_to_base_expert.inputs.input)
Connector.connect(unsqueeze_node_to_base_expert.outputs.output,
flock_node.inputs.sp.data_input)
Connector.connect(self.inputs.current_location.output,
stack_unsqueeze_node.inputs[0])
Connector.connect(self.inputs.target_location.output,
stack_unsqueeze_node.inputs[1])
Connector.connect(fork_node.outputs[1], self.outputs.actions.input)
# first layer
Connector.connect(flock_node.outputs.tp.projection_outputs,
join_node.inputs[0])
# second layer
Connector.connect(join_node.outputs.output,
unsqueeze_node_to_parent_expert.inputs.input)
# second layer
Connector.connect(unsqueeze_node_to_parent_expert.outputs.output,
parent_flock_node.inputs.sp.data_input)
Connector.connect(stack_unsqueeze_node.outputs[0],
parent_flock_node.inputs.tp.context_input)
# actions
Connector.connect(
parent_flock_node.outputs.sp.predicted_reconstructed_input,
squeeze_node.inputs[0])
Connector.connect(squeeze_node.outputs[0], fork_node.inputs.input)
def __init__(self, model: GoalDirectedExpertGroupBase,
params: GoalDirectedTemplateTopologyParams):
super().__init__(device='cuda')
self.n_parallel_runs = params.n_parallel_runs
self.worlds = MultiGridWorldNode(self.n_parallel_runs,
params.world_params)
self.model = model
self.one_hot_node = ToOneHotNode()
noise_params = RandomNoiseParams(amplitude=0.0001)
self.noise_node = RandomNoiseNode(noise_params)
def parse_action(inputs, outputs):
probs = inputs[0]
outputs[0].copy_(probs[:, -1, :4] + SMALL_CONSTANT)
def shape_reward(inputs, outputs):
rewards = inputs[0]
rewards = torch.cat([rewards, torch.zeros_like(rewards)], dim=-1)
outputs[0].copy_(rewards)
self.action_parser = LambdaNode(func=parse_action,
n_inputs=1,
output_shapes=[(self.n_parallel_runs,
4)],
name="Action parser")
self.reward_shaper = LambdaNode(func=shape_reward,
n_inputs=1,
output_shapes=[(self.n_parallel_runs,
2)],
name="Reward shaper")
self.add_node(self.worlds)
self.add_node(self.model)
self.add_node(self.action_parser)
self.add_node(self.reward_shaper)
self.add_node(self.noise_node)
self.add_node(self.one_hot_node)
if params.use_egocentric:
Connector.connect(self.worlds.outputs.egocentric_image_actions,
self.noise_node.inputs.input)
else:
Connector.connect(self.worlds.outputs.output_image_actions,
self.noise_node.inputs.input)
Connector.connect(self.noise_node.outputs.output,
self.model.inputs.data)
Connector.connect(self.model.outputs.predicted_reconstructed_input,
self.action_parser.inputs[0])
Connector.connect(self.worlds.outputs.rewards,
self.reward_shaper.inputs[0])
Connector.connect(self.reward_shaper.outputs[0],
self.model.inputs.reward)
Connector.connect(self.action_parser.outputs[0],
self.one_hot_node.inputs.input)
# If a multi-action-monitor node is created, add it here between the one hot and the worlds.
Connector.connect(self.one_hot_node.outputs.output,
self.worlds.inputs.agent_actions,
is_backward=True)
def __init__(self, curriculum: tuple = (1, -1)):
super().__init__()
se_config = SpaceEngineersConnectorConfig()
se_config.render_width = 16
se_config.render_height = 16
se_config.curriculum = list(curriculum)
base_expert_params = ExpertParams()
base_expert_params.flock_size = 1
base_expert_params.n_cluster_centers = 100
base_expert_params.compute_reconstruction = False
base_expert_params.spatial.cluster_boost_threshold = 1000
base_expert_params.spatial.learning_rate = 0.2
base_expert_params.spatial.batch_size = 1000
base_expert_params.spatial.buffer_size = 1010
base_expert_params.spatial.learning_period = 100
base_expert_params.temporal.batch_size = 1000
base_expert_params.temporal.buffer_size = 1010
base_expert_params.temporal.learning_period = 200
base_expert_params.temporal.forgetting_limit = 20000
# parent_expert_params = ExpertParams()
# parent_expert_params.flock_size = 1
# parent_expert_params.n_cluster_centers = 20
# parent_expert_params.compute_reconstruction = True
# parent_expert_params.temporal.exploration_probability = 0.9
# parent_expert_params.spatial.cluster_boost_threshold = 1000
# parent_expert_params.spatial.learning_rate = 0.2
# parent_expert_params.spatial.batch_size = 1000
# parent_expert_params.spatial.buffer_size = 1010
# parent_expert_params.spatial.learning_period = 100
# parent_expert_params.temporal.context_without_rewards_size = se_config.LOCATION_SIZE_ONE_HOT
# SE nodes
actions_descriptor = SpaceEngineersActionsDescriptor()
node_se_connector = SpaceEngineersConnectorNode(
actions_descriptor, se_config)
node_action_monitor = ActionMonitorNode(actions_descriptor)
# flock-related nodes
flock_node = ExpertFlockNode(base_expert_params)
blank_task_control = ConstantNode((se_config.TASK_CONTROL_SIZE, ))
blank_task_labels = ConstantNode((20, ))
# parent_flock_node = ExpertFlockNode(parent_expert_params)
join_node = JoinNode(flatten=True)
actions = ['FORWARD', 'BACKWARD', 'LEFT', 'RIGHT']
action_count = len(actions)
pass_actions_node = PassNode(output_shape=(action_count, ),
name="pass actions")
fork_node = ForkNode(
0, [base_expert_params.n_cluster_centers, action_count])
def squeeze(inputs, outputs):
outputs[0].copy_(inputs[0].squeeze())
squeeze_node = LambdaNode(
squeeze,
1, [(base_expert_params.n_cluster_centers + action_count, )],
name="squeeze lambda node")
def stack_and_unsqueeze(inputs, outputs):
outputs[0].copy_(torch.stack([inputs[0], inputs[1]]).unsqueeze(0))
stack_unsqueeze_node = LambdaNode(
stack_and_unsqueeze,
2, [(1, 2, se_config.LOCATION_SIZE_ONE_HOT)],
name="stack and unsqueeze node")
to_one_hot_node = ToOneHotNode()
action_parser_node = AgentActionsParserNode(actions_descriptor,
actions)
random_node = RandomNumberNode(0,
action_count,
name="random action generator",
generate_new_every_n=5,
randomize_intervals=True)
switch_node = SwitchNode(2)
# add nodes to the graph
self.add_node(flock_node)
# self.add_node(parent_flock_node)
self.add_node(node_se_connector)
self.add_node(node_action_monitor)
self.add_node(blank_task_control)
self.add_node(blank_task_labels)
# self.add_node(join_node)
# self.add_node(fork_node)
# self.add_node(pass_actions_node)
# self.add_node(squeeze_node)
# self.add_node(to_one_hot_node)
# self.add_node(stack_unsqueeze_node)
self.add_node(action_parser_node)
self.add_node(random_node)
# self.add_node(switch_node)
# first layer
Connector.connect(node_se_connector.outputs.image_output,
flock_node.inputs.sp.data_input)
# Connector.connect(
# flock_node.outputs.tp.projection_outputs,
# join_node.inputs[0]
# )
# Connector.connect(
# pass_actions_node.outputs.output,
# join_node.inputs[1]
# )
# # second layer
# Connector.connect(
# join_node.outputs.output,
# parent_flock_node.inputs.sp.data_input
# )
# Connector.connect(
# node_se_connector.outputs.task_to_agent_location_one_hot,
# stack_unsqueeze_node.inputs[0]
# )
# Connector.connect(
# node_se_connector.outputs.task_to_agent_location_target_one_hot,
# stack_unsqueeze_node.inputs[1]
# )
# Connector.connect(
# stack_unsqueeze_node.outputs[0],
# parent_flock_node.inputs.tp.context_input
# )
#
# # actions
# Connector.connect(
# parent_flock_node.outputs.sp.predicted_reconstructed_input,
# squeeze_node.inputs[0]
# )
# Connector.connect(
# squeeze_node.outputs[0],
# fork_node.inputs.input
# )
# Connector.connect(
# fork_node.outputs[1],
# to_one_hot_node.inputs.input
# )
# Connector.connect(
# random_node.outputs.one_hot_output,
# switch_node.inputs[0]
# )
# Connector.connect(
# to_one_hot_node.outputs.output,
# switch_node.inputs[1]
# )
# Connector.connect(
# switch_node.outputs.output,
# action_parser_node.inputs.input
# )
# directly use random exploration
Connector.connect(random_node.outputs.one_hot_output,
action_parser_node.inputs.input)
Connector.connect(action_parser_node.outputs.output,
node_action_monitor.inputs.action_in)
# Connector.connect(
# switch_node.outputs.output,
# pass_actions_node.inputs.input,
# is_low_priority=True
# )
Connector.connect(
node_action_monitor.outputs.action_out,
node_se_connector.inputs.agent_action,
# is_low_priority=True
is_backward=False)
# blank connection
Connector.connect(blank_task_control.outputs.output,
node_se_connector.inputs.task_control)
Connector.connect(blank_task_labels.outputs.output,
node_se_connector.inputs.agent_to_task_label)
# Save the SE connector so we can check testing/training phase.
# When the se_io interface has been added, this can be removed.
self._node_se_connector = node_se_connector
def __init__(self, sequence_generators: List[Any] = None, gamma=0.8):
super().__init__('cpu')
# environment
if sequence_generators is None:
sequence_generator_0 = SequenceGenerator(
[
[1, 2, 3, 1, 2, 3, 1, 2, 3],
[5, 4, 3, 5, 4, 3, 5, 4, 3],
[1, 2, 3, 1, 2, 3, 1, 2, 3],
[5, 4, 3, 5, 4, 3, 5, 4, 3],
]
, diagonal_transition_matrix(4, 0.8))
sequence_generator_1 = SequenceGenerator(
[
[1, 2, 3, 1, 2, 3, 1, 2, 3],
[5, 4, 3, 5, 4, 3, 5, 4, 3],
[1, 2, 3, 1, 2, 3, 1, 2, 3],
[5, 4, 3, 5, 4, 3, 5, 4, 3],
]
, diagonal_transition_matrix(4, 0.8))
sequence_generators = [sequence_generator_0, sequence_generator_1]
self.sequences_len = len(sequence_generators)
# switch
switch_node = SwitchableSequencesNodeGroup(sequence_generators)
self.switch_node = switch_node
self.add_node(switch_node)
# gate
self.add_node(self.get_gate())
# predictors
self.add_node(self.get_predictors())
# discount buffer
def discount(inputs: List[torch.Tensor], outputs: List[torch.Tensor], memory: List[torch.Tensor]):
cum_val = (1. - gamma) * inputs[0].squeeze() + gamma * memory[0]
memory[0].copy_(cum_val)
outputs[0].copy_(cum_val)
discount_node = LambdaNode(discount, 1, [(self.get_num_predictors(),)],
memory_shapes=[(self.get_num_predictors(),)],
name="Discount")
self.add_node(discount_node)
# argmin
def argmin(inputs: List[torch.Tensor], outputs: List[torch.Tensor]):
outputs[0].copy_(id_to_one_hot(inputs[0].argmin(), self.get_num_predictors()))
argmin_lambda_node = LambdaNode(argmin, 1, [(self.get_num_predictors(),)], name="Argmin")
self.add_node(argmin_lambda_node)
# dot
def dot_product(inputs: List[torch.Tensor], outputs: List[torch.Tensor]):
outputs[0].copy_(inputs[0].squeeze().dot(inputs[1].squeeze()))
dot_node = LambdaNode(dot_product, 2, [(1,)], name="predictors error dot gate activations")
self.dot_node = dot_node
self.add_node(dot_node)
# connections
Connector.connect(switch_node.outputs.output, self.get_predictors().inputs.data)
Connector.connect(switch_node.outputs.output, self.get_gate().inputs.data_predicted)
Connector.connect(switch_node.outputs.sequence_num, self.get_gate().inputs.data)
Connector.connect(self.get_predictors().outputs.predictors_activations_error, discount_node.inputs[0])
Connector.connect(discount_node.outputs[0], argmin_lambda_node.inputs[0])
Connector.connect(argmin_lambda_node.outputs[0], self.get_gate().inputs.predictor_activations_target)
Connector.connect(self.get_gate().outputs.gate_activations,
self.get_predictors().inputs.temporal_class_distribution,
is_backward=True)
Connector.connect(self.get_predictors().outputs.predictors_activations_error, dot_node.inputs[0])
Connector.connect(self.get_gate().outputs.gate_activations, dot_node.inputs[1])
def __init__(self,
num_predictors,
spatial_input_size,
predicted_input_size,
name: str = "NNGateNodeGroup"):
super().__init__(name,
inputs=GateInputs(self),
outputs=GateOutputs(self))
# join input and label
inputs_join_node = JoinNode(flatten=True)
self.add_node(inputs_join_node)
# gate
g_node_params = NetworkFlockNodeParams()
g_node_params.flock_size = 1
g_node_params.do_delay_coefficients = False
g_node_params.do_delay_input = True
g_node_params.learning_period = 20
g_node_params.buffer_size = 1000
g_node_params.batch_size = 900
g_network_params = NeuralNetworkFlockParams()
g_network_params.input_size = spatial_input_size + predicted_input_size
g_network_params.mini_batch_size = 100
g_network_params.hidden_size = g_network_params.input_size * 2
g_network_params.output_size = num_predictors
g_network_params.output_activation = OutputActivation.SOFTMAX
g_network_params.learning_rate = 0.05
gate = NetworkFlockNode(node_params=g_node_params,
network_params=g_network_params,
name="gate")
self.gate = gate
self.add_node(gate)
# const
learning_constant = ConstantNode([1], 1)
self.add_node(learning_constant)
# squeeze
squeeze_node = SqueezeNode(0)
self.add_node(squeeze_node)
to_one_hot_lambda_node_p = LambdaNode(
partial(to_one_hot, predicted_input_size), 1,
[(predicted_input_size, )])
self.add_node(to_one_hot_lambda_node_p)
to_one_hot_lambda_node_s = LambdaNode(
partial(to_one_hot, spatial_input_size), 1,
[(spatial_input_size, )])
self.add_node(to_one_hot_lambda_node_s)
# connections
Connector.connect(self.inputs.data.output,
to_one_hot_lambda_node_s.inputs[0])
Connector.connect(to_one_hot_lambda_node_s.outputs[0],
inputs_join_node.inputs[0])
Connector.connect(self.inputs.data_predicted.output,
to_one_hot_lambda_node_p.inputs[0])
Connector.connect(to_one_hot_lambda_node_p.outputs[0],
inputs_join_node.inputs[1])
Connector.connect(learning_constant.outputs.output,
gate.inputs.learning_coefficients)
Connector.connect(inputs_join_node.outputs.output,
gate.inputs.input_data)
Connector.connect(self.inputs.predictor_activations_target.output,
gate.inputs.target_data)
Connector.connect(gate.outputs.prediction_output,
squeeze_node.inputs[0])
Connector.connect(squeeze_node.outputs[0],
self.outputs.gate_activations.input)
class SeT0ConvTopology(SeT0TopologicalGraph):
EXPERTS_IN_X = 4
BATCH = 2000
TRAINING = True
parent_params = ExpertParams()
parent_params.flock_size = 1
parent_params.n_cluster_centers = 80
parent_params.compute_reconstruction = True
parent_params.spatial.learning_period = 400
parent_params.spatial.learning_rate = 0.5
if not TRAINING:
parent_params.spatial.learning_rate *= 0
parent_params.spatial.buffer_size = BATCH
parent_params.spatial.batch_size = BATCH
parent_params.spatial.cluster_boost_threshold = 200
mid_expert_params = ExpertParams()
mid_expert_params.flock_size = 1
mid_expert_params.n_cluster_centers = 100
mid_expert_params.compute_reconstruction = True
mid_expert_params.spatial.cluster_boost_threshold = 200
mid_expert_params.spatial.learning_rate = 0.5
if not TRAINING:
mid_expert_params.spatial.learning_rate *= 0
mid_expert_params.spatial.batch_size = BATCH
mid_expert_params.spatial.buffer_size = BATCH
mid_expert_params.spatial.learning_period = 400
conv_flock_params = ExpertParams()
conv_flock_params.flock_size = EXPERTS_IN_X**2
conv_flock_params.n_cluster_centers = 600
conv_flock_params.spatial.learning_period = 1000
conv_flock_params.spatial.learning_rate = 0.2
if not TRAINING:
conv_flock_params.spatial.learning_rate *= 0
conv_flock_params.spatial.buffer_size = BATCH
conv_flock_params.spatial.batch_size = BATCH
conv_flock_params.spatial.cluster_boost_threshold = 1000
# parent_rf_stride_dims = (16, 16)
def _install_experts(self):
im_width = self.se_io.get_image_width()
im_height = self.se_io.get_image_height()
self.input_dims = torch.Size((im_height, im_width, 3))
self.parent_rf_dims = Size2D(im_height // self.EXPERTS_IN_X,
im_width // self.EXPERTS_IN_X)
lrf_node = ReceptiveFieldNode(input_dims=self.input_dims,
parent_rf_dims=self.parent_rf_dims)
self.add_node(lrf_node)
self._top_level_flock_node = SpatialPoolerFlockNode(self.parent_params)
self._mid_node = ExpertFlockNode(self.mid_expert_params)
self._conv_node = ConvExpertFlockNode(self.conv_flock_params)
self.add_node(self._top_level_flock_node)
self.add_node(self._mid_node)
self.add_node(self._conv_node)
def rescale(inputs, outputs):
if self.TRAINING:
outputs[0].copy_(
inputs[0] *
1000) # large constant to make the label more important
else:
outputs[0].copy_(inputs[0] * float('nan'))
self._rescale_node = LambdaNode(rescale, 1,
[(self.se_io.get_num_labels(), )])
self.add_node(self._rescale_node)
unsqueeze_node_0 = UnsqueezeNode(0)
self.add_node(unsqueeze_node_0)
unsqueeze_node_1 = UnsqueezeNode(0)
self.add_node(unsqueeze_node_1)
Connector.connect(self.se_io.outputs.image_output,
lrf_node.inputs.input)
Connector.connect(lrf_node.outputs.output,
self._conv_node.inputs.sp.data_input)
Connector.connect(self._conv_node.outputs.tp.projection_outputs,
unsqueeze_node_0.inputs.input)
Connector.connect(unsqueeze_node_0.outputs.output,
self._mid_node.inputs.sp.data_input)
Connector.connect(self._mid_node.outputs.tp.projection_outputs,
self._join_node.inputs[0])
Connector.connect(self.se_io.outputs.task_to_agent_label,
self._rescale_node.inputs[0])
Connector.connect(self._rescale_node.outputs[0],
self._join_node.inputs[1])
Connector.connect(self._join_node.outputs.output,
unsqueeze_node_1.inputs.input)
Connector.connect(unsqueeze_node_1.outputs.output,
self._top_level_flock_node.inputs.sp.data_input)
def _top_level_expert_output_size(self):
return self.mid_expert_params.n_cluster_centers * self.mid_expert_params.flock_size
def switch_training_testing(self, training: bool):
if self._rescale_node._unit is None:
return
def rescale(inputs, outputs):
if training:
outputs[0].copy_(
inputs[0] *
1000) # large constant to make the label more important
else:
outputs[0].copy_(inputs[0] * float('nan'))
self._rescale_node.change_function(rescale)
_topology_test_checked: bool = False
def get_properties(self):
prop_list = super().get_properties()
def topology_test_checked(value):
self._topology_test_checked = value
self.switch_training_testing(not parse_bool(value))
return value
prop_list.append(
ObserverPropertiesItem("Test topology", 'checkbox',
self._topology_test_checked,
topology_test_checked))
return prop_list
def _get_agent_output(self) -> MemoryBlock:
return self._top_level_flock_node.outputs.sp.current_reconstructed_input
def __init__(self, step_callback):
super().__init__('cpu')
self.add_node(LambdaNode(lambda inputs, outputs: step_callback(), 0, []))
def __init__(self):
super().__init__(device='cuda')
actions_descriptor = GridWorldActionDescriptor()
node_action_monitor = ActionMonitorNode(actions_descriptor)
params = GridWorldParams(map_name='MapE')
noise_params = RandomNoiseParams(amplitude=0.0001)
node_grid_world = GridWorldNode(params)
expert_params = ExpertParams()
unsqueeze_node = UnsqueezeNode(dim=0)
noise_node = RandomNoiseNode(noise_params)
constant_node = ConstantNode(shape=(1, 1, 3, 48))
one_hot_node = ToOneHotNode()
def context(inputs, outputs):
con = inputs[0]
con[:, :, 1:, 24:] = float('nan')
outputs[0].copy_(con)
def f(inputs, outputs):
probs = inputs[0]
outputs[0].copy_(probs[0, -1, :4] + SMALL_CONSTANT)
action_parser = LambdaNode(func=f, n_inputs=1, output_shapes=[(4,)])
context_assembler = LambdaNode(func=context, n_inputs=1, output_shapes=[(1, 1, 3, 48)])
expert_params.flock_size = 1
expert_params.n_cluster_centers = 24
expert_params.produce_actions = True
expert_params.temporal.seq_length = 9
expert_params.temporal.seq_lookahead = 7
expert_params.temporal.n_frequent_seqs = 700
expert_params.temporal.max_encountered_seqs = 1000
expert_params.temporal.exploration_probability = 0.01
expert_params.temporal.batch_size = 200
expert_params.temporal.own_rewards_weight = 20
expert_params.temporal.incoming_context_size = 48
expert_params.compute_reconstruction = True
#expert_node = ConvExpertFlockNode(expert_params)
expert_node = ExpertFlockNode(expert_params)
self.add_node(node_grid_world)
self.add_node(node_action_monitor)
self.add_node(expert_node)
self.add_node(unsqueeze_node)
self.add_node(action_parser)
self.add_node(noise_node)
self.add_node(constant_node)
self.add_node(context_assembler)
self.add_node(one_hot_node)
Connector.connect(node_grid_world.outputs.egocentric_image_action, noise_node.inputs.input)
Connector.connect(noise_node.outputs.output, unsqueeze_node.inputs.input)
Connector.connect(unsqueeze_node.outputs.output, expert_node.inputs.sp.data_input)
Connector.connect(node_grid_world.outputs.reward, expert_node.inputs.tp.reward_input)
Connector.connect(constant_node.outputs.output, context_assembler.inputs[0])
Connector.connect(context_assembler.outputs[0], expert_node.inputs.tp.context_input)
Connector.connect(expert_node.outputs.sp.predicted_reconstructed_input, action_parser.inputs[0])
Connector.connect(action_parser.outputs[0], one_hot_node.inputs.input)
Connector.connect(one_hot_node.outputs.output, node_action_monitor.inputs.action_in)
Connector.connect(node_action_monitor.outputs.action_out, node_grid_world.inputs.agent_action, is_backward=True)
def __init__(self):
super().__init__(device='cuda')
actions_descriptor = GridWorldActionDescriptor()
node_action_monitor = ActionMonitorNode(actions_descriptor)
params = GridWorldParams(map_name='MapE')
noise_params = RandomNoiseParams(amplitude=0.0001)
node_grid_world = GridWorldNode(params)
expert_params1 = ExpertParams()
unsqueeze_node = UnsqueezeNode(dim=0)
noise_node = RandomNoiseNode(noise_params)
one_hot_node = ToOneHotNode()
def f(inputs, outputs):
probs = inputs[0]
outputs[0].copy_(probs[0, -1, :4] + SMALL_CONSTANT)
action_parser = LambdaNode(func=f, n_inputs=1, output_shapes=[(4, )])
expert_params1.flock_size = 1
expert_params1.n_cluster_centers = 24
expert_params1.produce_actions = True
expert_params1.temporal.seq_length = 4
expert_params1.temporal.seq_lookahead = 2
expert_params1.temporal.n_frequent_seqs = 700
expert_params1.temporal.max_encountered_seqs = 1000
expert_params1.temporal.exploration_probability = 0.05
expert_params1.temporal.batch_size = 200
expert_params1.temporal.frustration_threshold = 2
# expert_params.temporal.own_rewards_weight = 20
expert_params1.compute_reconstruction = True
expert_params2 = expert_params1.clone()
expert_params2.temporal.seq_length = 5
expert_params2.temporal.seq_lookahead = 4
expert_params2.n_cluster_centers = 8
expert_params2.produce_actions = False
expert_params2.temporal.frustration_threshold = 10
#expert_params1.temporal.incoming_context_size = 2 * expert_params2.n_cluster_centers
expert_node1 = ExpertFlockNode(expert_params1)
expert_node2 = ExpertFlockNode(expert_params2)
self.add_node(node_grid_world)
self.add_node(node_action_monitor)
self.add_node(expert_node1)
self.add_node(expert_node2)
self.add_node(unsqueeze_node)
self.add_node(action_parser)
self.add_node(noise_node)
self.add_node(one_hot_node)
Connector.connect(node_grid_world.outputs.output_image_action,
noise_node.inputs.input)
Connector.connect(noise_node.outputs.output,
unsqueeze_node.inputs.input)
Connector.connect(unsqueeze_node.outputs.output,
expert_node1.inputs.sp.data_input)
Connector.connect(expert_node1.outputs.tp.projection_outputs,
expert_node2.inputs.sp.data_input)
Connector.connect(expert_node2.outputs.output_context,
expert_node1.inputs.tp.context_input,
is_backward=True)
Connector.connect(
expert_node1.outputs.sp.predicted_reconstructed_input,
action_parser.inputs[0])
Connector.connect(node_grid_world.outputs.reward,
expert_node1.inputs.tp.reward_input)
Connector.connect(node_grid_world.outputs.reward,
expert_node2.inputs.tp.reward_input)
Connector.connect(action_parser.outputs[0], one_hot_node.inputs.input)
Connector.connect(one_hot_node.outputs.output,
node_action_monitor.inputs.action_in)
Connector.connect(node_action_monitor.outputs.action_out,
node_grid_world.inputs.agent_action,
is_backward=True)
