def _install_experts(self): im_width = self.se_io.get_image_width() im_height = self.se_io.get_image_height() self.input_dims = torch.Size((im_height, im_width, 3)) self.parent_rf_dims = Size2D(im_height // self.EXPERTS_IN_X, im_width // self.EXPERTS_IN_X) lrf_node = ReceptiveFieldNode(input_dims=self.input_dims, parent_rf_dims=self.parent_rf_dims) self.add_node(lrf_node) self._top_level_flock_node = SpatialPoolerFlockNode(self.parent_params) self._mid_node = ExpertFlockNode(self.mid_expert_params) self._conv_node = ConvExpertFlockNode(self.conv_flock_params) self.add_node(self._top_level_flock_node) self.add_node(self._mid_node) self.add_node(self._conv_node) def rescale(inputs, outputs): if self.TRAINING: outputs[0].copy_( inputs[0] * 1000) # large constant to make the label more important else: outputs[0].copy_(inputs[0] * float('nan')) self._rescale_node = LambdaNode(rescale, 1, [(self.se_io.get_num_labels(), )]) self.add_node(self._rescale_node) unsqueeze_node_0 = UnsqueezeNode(0) self.add_node(unsqueeze_node_0) unsqueeze_node_1 = UnsqueezeNode(0) self.add_node(unsqueeze_node_1) Connector.connect(self.se_io.outputs.image_output, lrf_node.inputs.input) Connector.connect(lrf_node.outputs.output, self._conv_node.inputs.sp.data_input) Connector.connect(self._conv_node.outputs.tp.projection_outputs, unsqueeze_node_0.inputs.input) Connector.connect(unsqueeze_node_0.outputs.output, self._mid_node.inputs.sp.data_input) Connector.connect(self._mid_node.outputs.tp.projection_outputs, self._join_node.inputs[0]) Connector.connect(self.se_io.outputs.task_to_agent_label, self._rescale_node.inputs[0]) Connector.connect(self._rescale_node.outputs[0], self._join_node.inputs[1]) Connector.connect(self._join_node.outputs.output, unsqueeze_node_1.inputs.input) Connector.connect(unsqueeze_node_1.outputs.output, self._top_level_flock_node.inputs.sp.data_input)
def _install_experts(self): lrf_node = ReceptiveFieldNode(input_dims=self.input_dims, parent_rf_dims=self.parent_rf_dims) self.add_node(lrf_node) self._top_level_flock_node = SpatialPoolerFlockNode(self.parent_params, name="Parent 1 SP") self._conv_node = ConvSpatialPoolerFlockNode(self.conv_flock_params, name="Conv SP flock") self.add_node(self._top_level_flock_node) self.add_node(self._conv_node) scale = 1000 def rescale_up(inputs, outputs): outputs[0].copy_(inputs[0] * scale) def rescale_down(inputs, outputs): outputs[0].copy_(inputs[0] / scale) self._rescale_up_node = LambdaNode(rescale_up, 1, [(20, )], name="upscale 1000") self.add_node(self._rescale_up_node) self._rescale_down_node = LambdaNode( rescale_down, 1, [(1, self._top_level_expert_output_size() + 20)], name="downscale 1000") self.add_node(self._rescale_down_node) unsqueeze_node = UnsqueezeNode(0) self.add_node(unsqueeze_node) Connector.connect(self.se_io.outputs.image_output, lrf_node.inputs.input) Connector.connect(lrf_node.outputs.output, self._conv_node.inputs.sp.data_input) Connector.connect(self._conv_node.outputs.sp.forward_clusters, self._join_node.inputs[0]) Connector.connect(self.se_io.outputs.task_to_agent_label, self._rescale_up_node.inputs[0]) Connector.connect(self._rescale_up_node.outputs[0], self._join_node.inputs[1]) Connector.connect(self._join_node.outputs.output, unsqueeze_node.inputs.input) Connector.connect(unsqueeze_node.outputs.output, self._top_level_flock_node.inputs.sp.data_input) Connector.connect( self._top_level_flock_node.outputs.sp.current_reconstructed_input, self._rescale_down_node.inputs[0])
def _create_and_connect_agent(self, join_node: JoinNode, fork_node: ForkNode): params = ExpertParams() params.flock_size = 1 params.n_cluster_centers = 28 params.compute_reconstruction = True params.spatial.cluster_boost_threshold = 1000 params.spatial.buffer_size = 500 params.spatial.batch_size = 500 params.spatial.learning_rate = 0.3 params.spatial.learning_period = 50 # conv_expert = ConvExpertFlockNode(params, name="Conv. expert") # conv_expert = SpatialPoolerFlockNode(params, name=" SP") conv_expert = ExpertFlockNode(params, name=" expert") self.add_node(conv_expert) unsqueeze_node_0 = UnsqueezeNode(0) self.add_node(unsqueeze_node_0) Connector.connect(join_node.outputs.output, unsqueeze_node_0.inputs.input) Connector.connect(unsqueeze_node_0.outputs.output, conv_expert.inputs.sp.data_input) def squeeze(inputs, outputs): outputs[0].copy_(inputs[0].squeeze(0)) squeeze_node = LambdaNode(squeeze, 1, [(sum(fork_node._split_sizes),)], name="squeeze lambda node") self.add_node(squeeze_node) Connector.connect(conv_expert.outputs.sp.predicted_reconstructed_input, squeeze_node.inputs[0]) Connector.connect(squeeze_node.outputs[0], fork_node.inputs.input)
def create_predictions_gather_node( predicted_indexes: List[int], num_objects: int = 1, name: str = "Gather predicted data") -> LambdaNode: """Gathers just the parts of the output requested by the indexes (ids of the positions in the vector)""" def gather_pred(inputs: List[torch.Tensor], outputs: List[torch.Tensor]): latent_input = inputs[0] predicted_ids = torch.tensor(predicted_indexes, dtype=torch.long).view(1, -1) predicted_expanded = predicted_ids.expand(latent_input.shape[0], -1) # possible improvement # gathered = latent_input.index_select(1, predicted_indexes) gathered = latent_input.gather(1, predicted_expanded) # outputs[0].copy_(gathered) # TODO only one object supported for now outputs[0].copy_(gathered.view(outputs[0].shape)) gather_predictions = LambdaNode( gather_pred, n_inputs=1, # output_shapes=[(num_objects, len(predicted_indexes))], # TODO only one object supported for now output_shapes=[(len(predicted_indexes), )], name=name) return gather_predictions
def create_dot_product_node(input_size: int, output_sizes: List[Tuple], name: str) -> LambdaNode: # dot def dot_product(inputs: List[torch.Tensor], outputs: List[torch.Tensor]): outputs[0].copy_(inputs[0].squeeze().dot(inputs[1].squeeze())) dot_node = LambdaNode(dot_product, input_size, output_sizes, name=name) return dot_node
def _install_rescale(self, scale: float): # rescale label - bigger weight def rescale(inputs, outputs): outputs[0].copy_(inputs[0] * scale) self._rescale_node = LambdaNode(rescale, 1, [(20, )]) # TODO constant here self.add_node(self._rescale_node)
def create_arg_min_node(num_predictors: int) -> LambdaNode: # argmin def argmin(inputs: List[torch.Tensor], outputs: List[torch.Tensor]): outputs[0].copy_(id_to_one_hot(inputs[0].argmin(), num_predictors)) argmin_lambda_node = LambdaNode(argmin, 1, [(num_predictors, )], name="Argmin") return argmin_lambda_node
def create_delay_node(num_predictors: int) -> LambdaNode: # delay the data 1 step def delay(inputs: List[torch.Tensor], outputs: List[torch.Tensor], memory: List[torch.Tensor]): outputs[0].copy_(memory[0]) memory[0].copy_(inputs[0]) delay_node = LambdaNode(delay, 1, [(num_predictors, )], memory_shapes=[(num_predictors, )], name="Gate Output Delay") return delay_node
def create_discount_node(gamma: float, num_predictors: int) -> LambdaNode: """Returns a node that implements discount by gamma parameter""" def discount(inputs: List[torch.Tensor], outputs: List[torch.Tensor], memory: List[torch.Tensor]): cum_val = (1. - gamma) * inputs[0].squeeze() + gamma * memory[0] memory[0].copy_(cum_val) outputs[0].copy_(cum_val) discount_node = LambdaNode(discount, 1, [(num_predictors, )], memory_shapes=[(num_predictors, )], name="Discount") return discount_node
def __init__(self, num_predictors, input_data_size, name: str = "NNPredictorNodeGroup"): super().__init__(name, inputs=PredictorInputs(self), outputs=PredictorOutputs(self)) p_node_params = NetworkFlockNodeParams() p_node_params.flock_size = num_predictors p_node_params.do_delay_coefficients = False p_node_params.do_delay_input = True p_node_params.normalize_coefficients = False p_node_params.negative_coefficients_removal = True p_node_params.buffer_size = 500 p_node_params.batch_size = 400 p_node_params.learning_period = 20 p_network_params = NeuralNetworkFlockParams() p_network_params.flock_size = p_node_params.flock_size p_network_params.input_size = input_data_size p_network_params.hidden_size = input_data_size * 2 p_network_params.output_size = input_data_size p_network_params.output_activation = OutputActivation.SOFTMAX p_network_params.learning_rate = 0.1 p_network_params.coefficients_minimum_max = 0.3 predictors = NetworkFlockNode(node_params=p_node_params, network_params=p_network_params, name="predictors") self.predictors = predictors self.add_node(predictors) to_one_hot_lambda_node = LambdaNode( partial(to_one_hot, input_data_size), 1, [(input_data_size, )]) self.add_node(to_one_hot_lambda_node) Connector.connect(self.inputs.data.output, to_one_hot_lambda_node.inputs[0]) Connector.connect(to_one_hot_lambda_node.outputs[0], predictors.inputs.input_data) Connector.connect(to_one_hot_lambda_node.outputs[0], predictors.inputs.target_data) # Connector.connect(self.inputs.prediction_target.output, predictors.inputs.target_data) Connector.connect(self.inputs.temporal_class_distribution.output, predictors.inputs.learning_coefficients) Connector.connect(predictors.outputs.prediction_output, self.outputs.predictors_activations.input) Connector.connect(predictors.outputs.error_output, self.outputs.predictors_activations_error.input)
def __init__(self, num_predictors, name: str = "FakeGateNodeGroup"): super().__init__(name, inputs=GateInputs(self), outputs=GateOutputs(self)) to_one_hot_lambda_node = LambdaNode( partial(to_one_hot, num_predictors), 1, [(num_predictors, )]) self.add_node(to_one_hot_lambda_node) # connections Connector.connect(self.inputs.data.output, to_one_hot_lambda_node.inputs[0]) Connector.connect(to_one_hot_lambda_node.outputs[0], self.outputs.gate_activations.input)
def create_node_context_enhance(self) -> NodeBase: data_window_list = [] def context_enhance(data_window: List[torch.Tensor], inputs, outputs): """Multiply context by coef and average over n steps""" context = inputs[0] * self._params.gate_input_context_multiplier # Add value to sliding window data_window.append(context) if len(data_window ) > self._params.gate_input_context_avg_window_size - 1: data_window[:] = data_window[ -self._params.gate_input_context_avg_window_size:] # average values outputs[0].copy_(torch.mean(torch.stack(data_window), 0)) return LambdaNode(partial(context_enhance, data_window_list), 1, [(self._params.flock_size, self._params.seq_count)], name="Gateway input context enhance")
def create_node_extract_predicted_output(self) -> NodeBase: specialist_extract_predicted_output_last_value = torch.zeros( (self._params.flock_size, self._params.n_symbols)) def specialist_extract_predicted_output(last_value, inputs, outputs): # global context_enhance_avg output = inputs[0] next_seq_position = self._specialist_node.params.temporal.seq_lookbehind clusters = output[:, next_seq_position, :] result = clusters.view(self._params.flock_size, self._params.n_symbols) outputs[0].copy_(last_value) last_value.copy_(result) return LambdaNode(partial( specialist_extract_predicted_output, specialist_extract_predicted_output_last_value), 1, [(self._params.flock_size, self._params.n_symbols)], name="Specialist extract predicted output")
def __init__(self): super().__init__(device='cuda') actions_descriptor = GridWorldActionDescriptor() node_action_monitor = ActionMonitorNode(actions_descriptor) params = GridWorldParams(map_name='MapTwoRoom', reset_strategy=ResetStrategy.ANYWHERE) noise_params = RandomNoiseParams(amplitude=0.0001) node_grid_world = GridWorldNode(params) expert_params = ExpertParams() unsqueeze_node = UnsqueezeNode(dim=0) noise_node = RandomNoiseNode(noise_params) one_hot_node = ToOneHotNode() def f(inputs, outputs): probs = inputs[0] outputs[0].copy_(probs[0, -1, :4] + SMALL_CONSTANT) action_parser = LambdaNode(func=f, n_inputs=1, output_shapes=[(4, )]) expert_params.flock_size = 1 expert_params.n_cluster_centers = 64 expert_params.produce_actions = True expert_params.temporal.seq_length = 17 expert_params.temporal.seq_lookahead = 13 expert_params.temporal.n_frequent_seqs = 700 expert_params.temporal.max_encountered_seqs = 1000 expert_params.temporal.exploration_probability = 0.05 expert_params.temporal.batch_size = 200 expert_params.temporal.buffer_size = 1000 expert_params.temporal.own_rewards_weight = 20 expert_params.temporal.frustration_threshold = 2 expert_params.temporal.compute_backward_pass = True expert_params.compute_reconstruction = True expert_node = ConvExpertFlockNode(expert_params) #expert_node = ExpertFlockNode(expert_params) self.add_node(node_grid_world) self.add_node(node_action_monitor) self.add_node(expert_node) self.add_node(unsqueeze_node) self.add_node(action_parser) self.add_node(noise_node) self.add_node(one_hot_node) Connector.connect(node_grid_world.outputs.egocentric_image_action, noise_node.inputs.input) Connector.connect(noise_node.outputs.output, unsqueeze_node.inputs.input) Connector.connect(unsqueeze_node.outputs.output, expert_node.inputs.sp.data_input) Connector.connect(node_grid_world.outputs.reward, expert_node.inputs.tp.reward_input) Connector.connect(expert_node.outputs.sp.predicted_reconstructed_input, action_parser.inputs[0]) Connector.connect(action_parser.outputs[0], one_hot_node.inputs.input) Connector.connect(one_hot_node.outputs.output, node_action_monitor.inputs.action_in) Connector.connect(node_action_monitor.outputs.action_out, node_grid_world.inputs.agent_action, is_backward=True)
def __init__(self, action_count=4, location_vector_size=100, use_grayscale: bool = False): super().__init__("Task 1 - Basic expert", inputs=Task1BasicExpertGroupInputs(self), outputs=Task1BasicExpertGroupOutputs(self)) base_expert_params = ExpertParams() base_expert_params.flock_size = 1 base_expert_params.n_cluster_centers = 100 base_expert_params.compute_reconstruction = False base_expert_params.spatial.cluster_boost_threshold = 1000 base_expert_params.spatial.learning_rate = 0.2 base_expert_params.spatial.batch_size = 1000 base_expert_params.spatial.buffer_size = 1010 base_expert_params.spatial.learning_period = 100 parent_expert_params = ExpertParams() parent_expert_params.flock_size = 1 parent_expert_params.n_cluster_centers = 20 parent_expert_params.compute_reconstruction = True parent_expert_params.temporal.exploration_probability = 0.9 parent_expert_params.spatial.cluster_boost_threshold = 1000 parent_expert_params.spatial.learning_rate = 0.2 parent_expert_params.spatial.batch_size = 1000 parent_expert_params.spatial.buffer_size = 1010 parent_expert_params.spatial.learning_period = 100 parent_expert_params.temporal.context_without_rewards_size = location_vector_size # flock-related nodes flock_node = ExpertFlockNode(base_expert_params) parent_flock_node = ExpertFlockNode(parent_expert_params) join_node = JoinNode(flatten=True) unsqueeze_node_to_base_expert = UnsqueezeNode(0) unsqueeze_node_to_parent_expert = UnsqueezeNode(0) fork_node = ForkNode( 0, [base_expert_params.n_cluster_centers, action_count]) def squeeze(inputs, outputs): outputs[0].copy_(inputs[0].squeeze()) squeeze_node = LambdaNode( squeeze, 1, [(base_expert_params.n_cluster_centers + action_count, )], name="squeeze lambda node") def stack_and_unsqueeze(inputs, outputs): outputs[0].copy_(torch.stack([inputs[0], inputs[1]]).unsqueeze(0)) stack_unsqueeze_node = LambdaNode(stack_and_unsqueeze, 2, [(1, 2, location_vector_size)], name="stack and unsqueeze node") # add nodes to the graph self.add_node(flock_node) self.add_node(unsqueeze_node_to_base_expert) self.add_node(parent_flock_node) self.add_node(unsqueeze_node_to_parent_expert) self.add_node(join_node) self.add_node(fork_node) self.add_node(squeeze_node) self.add_node(stack_unsqueeze_node) Connector.connect(self.inputs.actions.output, join_node.inputs[1]) if use_grayscale: grayscale_node = GrayscaleNode(squeeze_channel=True) self.add_node(grayscale_node) Connector.connect(self.inputs.image.output, grayscale_node.inputs.input) Connector.connect(grayscale_node.outputs.output, unsqueeze_node_to_base_expert.inputs.input) else: Connector.connect(self.inputs.image.output, unsqueeze_node_to_base_expert.inputs.input) Connector.connect(unsqueeze_node_to_base_expert.outputs.output, flock_node.inputs.sp.data_input) Connector.connect(self.inputs.current_location.output, stack_unsqueeze_node.inputs[0]) Connector.connect(self.inputs.target_location.output, stack_unsqueeze_node.inputs[1]) Connector.connect(fork_node.outputs[1], self.outputs.actions.input) # first layer Connector.connect(flock_node.outputs.tp.projection_outputs, join_node.inputs[0]) # second layer Connector.connect(join_node.outputs.output, unsqueeze_node_to_parent_expert.inputs.input) # second layer Connector.connect(unsqueeze_node_to_parent_expert.outputs.output, parent_flock_node.inputs.sp.data_input) Connector.connect(stack_unsqueeze_node.outputs[0], parent_flock_node.inputs.tp.context_input) # actions Connector.connect( parent_flock_node.outputs.sp.predicted_reconstructed_input, squeeze_node.inputs[0]) Connector.connect(squeeze_node.outputs[0], fork_node.inputs.input)
def __init__(self, model: GoalDirectedExpertGroupBase, params: GoalDirectedTemplateTopologyParams): super().__init__(device='cuda') self.n_parallel_runs = params.n_parallel_runs self.worlds = MultiGridWorldNode(self.n_parallel_runs, params.world_params) self.model = model self.one_hot_node = ToOneHotNode() noise_params = RandomNoiseParams(amplitude=0.0001) self.noise_node = RandomNoiseNode(noise_params) def parse_action(inputs, outputs): probs = inputs[0] outputs[0].copy_(probs[:, -1, :4] + SMALL_CONSTANT) def shape_reward(inputs, outputs): rewards = inputs[0] rewards = torch.cat([rewards, torch.zeros_like(rewards)], dim=-1) outputs[0].copy_(rewards) self.action_parser = LambdaNode(func=parse_action, n_inputs=1, output_shapes=[(self.n_parallel_runs, 4)], name="Action parser") self.reward_shaper = LambdaNode(func=shape_reward, n_inputs=1, output_shapes=[(self.n_parallel_runs, 2)], name="Reward shaper") self.add_node(self.worlds) self.add_node(self.model) self.add_node(self.action_parser) self.add_node(self.reward_shaper) self.add_node(self.noise_node) self.add_node(self.one_hot_node) if params.use_egocentric: Connector.connect(self.worlds.outputs.egocentric_image_actions, self.noise_node.inputs.input) else: Connector.connect(self.worlds.outputs.output_image_actions, self.noise_node.inputs.input) Connector.connect(self.noise_node.outputs.output, self.model.inputs.data) Connector.connect(self.model.outputs.predicted_reconstructed_input, self.action_parser.inputs[0]) Connector.connect(self.worlds.outputs.rewards, self.reward_shaper.inputs[0]) Connector.connect(self.reward_shaper.outputs[0], self.model.inputs.reward) Connector.connect(self.action_parser.outputs[0], self.one_hot_node.inputs.input) # If a multi-action-monitor node is created, add it here between the one hot and the worlds. Connector.connect(self.one_hot_node.outputs.output, self.worlds.inputs.agent_actions, is_backward=True)
def __init__(self, curriculum: tuple = (1, -1)): super().__init__() se_config = SpaceEngineersConnectorConfig() se_config.render_width = 16 se_config.render_height = 16 se_config.curriculum = list(curriculum) base_expert_params = ExpertParams() base_expert_params.flock_size = 1 base_expert_params.n_cluster_centers = 100 base_expert_params.compute_reconstruction = False base_expert_params.spatial.cluster_boost_threshold = 1000 base_expert_params.spatial.learning_rate = 0.2 base_expert_params.spatial.batch_size = 1000 base_expert_params.spatial.buffer_size = 1010 base_expert_params.spatial.learning_period = 100 base_expert_params.temporal.batch_size = 1000 base_expert_params.temporal.buffer_size = 1010 base_expert_params.temporal.learning_period = 200 base_expert_params.temporal.forgetting_limit = 20000 # parent_expert_params = ExpertParams() # parent_expert_params.flock_size = 1 # parent_expert_params.n_cluster_centers = 20 # parent_expert_params.compute_reconstruction = True # parent_expert_params.temporal.exploration_probability = 0.9 # parent_expert_params.spatial.cluster_boost_threshold = 1000 # parent_expert_params.spatial.learning_rate = 0.2 # parent_expert_params.spatial.batch_size = 1000 # parent_expert_params.spatial.buffer_size = 1010 # parent_expert_params.spatial.learning_period = 100 # parent_expert_params.temporal.context_without_rewards_size = se_config.LOCATION_SIZE_ONE_HOT # SE nodes actions_descriptor = SpaceEngineersActionsDescriptor() node_se_connector = SpaceEngineersConnectorNode( actions_descriptor, se_config) node_action_monitor = ActionMonitorNode(actions_descriptor) # flock-related nodes flock_node = ExpertFlockNode(base_expert_params) blank_task_control = ConstantNode((se_config.TASK_CONTROL_SIZE, )) blank_task_labels = ConstantNode((20, )) # parent_flock_node = ExpertFlockNode(parent_expert_params) join_node = JoinNode(flatten=True) actions = ['FORWARD', 'BACKWARD', 'LEFT', 'RIGHT'] action_count = len(actions) pass_actions_node = PassNode(output_shape=(action_count, ), name="pass actions") fork_node = ForkNode( 0, [base_expert_params.n_cluster_centers, action_count]) def squeeze(inputs, outputs): outputs[0].copy_(inputs[0].squeeze()) squeeze_node = LambdaNode( squeeze, 1, [(base_expert_params.n_cluster_centers + action_count, )], name="squeeze lambda node") def stack_and_unsqueeze(inputs, outputs): outputs[0].copy_(torch.stack([inputs[0], inputs[1]]).unsqueeze(0)) stack_unsqueeze_node = LambdaNode( stack_and_unsqueeze, 2, [(1, 2, se_config.LOCATION_SIZE_ONE_HOT)], name="stack and unsqueeze node") to_one_hot_node = ToOneHotNode() action_parser_node = AgentActionsParserNode(actions_descriptor, actions) random_node = RandomNumberNode(0, action_count, name="random action generator", generate_new_every_n=5, randomize_intervals=True) switch_node = SwitchNode(2) # add nodes to the graph self.add_node(flock_node) # self.add_node(parent_flock_node) self.add_node(node_se_connector) self.add_node(node_action_monitor) self.add_node(blank_task_control) self.add_node(blank_task_labels) # self.add_node(join_node) # self.add_node(fork_node) # self.add_node(pass_actions_node) # self.add_node(squeeze_node) # self.add_node(to_one_hot_node) # self.add_node(stack_unsqueeze_node) self.add_node(action_parser_node) self.add_node(random_node) # self.add_node(switch_node) # first layer Connector.connect(node_se_connector.outputs.image_output, flock_node.inputs.sp.data_input) # Connector.connect( # flock_node.outputs.tp.projection_outputs, # join_node.inputs[0] # ) # Connector.connect( # pass_actions_node.outputs.output, # join_node.inputs[1] # ) # # second layer # Connector.connect( # join_node.outputs.output, # parent_flock_node.inputs.sp.data_input # ) # Connector.connect( # node_se_connector.outputs.task_to_agent_location_one_hot, # stack_unsqueeze_node.inputs[0] # ) # Connector.connect( # node_se_connector.outputs.task_to_agent_location_target_one_hot, # stack_unsqueeze_node.inputs[1] # ) # Connector.connect( # stack_unsqueeze_node.outputs[0], # parent_flock_node.inputs.tp.context_input # ) # # # actions # Connector.connect( # parent_flock_node.outputs.sp.predicted_reconstructed_input, # squeeze_node.inputs[0] # ) # Connector.connect( # squeeze_node.outputs[0], # fork_node.inputs.input # ) # Connector.connect( # fork_node.outputs[1], # to_one_hot_node.inputs.input # ) # Connector.connect( # random_node.outputs.one_hot_output, # switch_node.inputs[0] # ) # Connector.connect( # to_one_hot_node.outputs.output, # switch_node.inputs[1] # ) # Connector.connect( # switch_node.outputs.output, # action_parser_node.inputs.input # ) # directly use random exploration Connector.connect(random_node.outputs.one_hot_output, action_parser_node.inputs.input) Connector.connect(action_parser_node.outputs.output, node_action_monitor.inputs.action_in) # Connector.connect( # switch_node.outputs.output, # pass_actions_node.inputs.input, # is_low_priority=True # ) Connector.connect( node_action_monitor.outputs.action_out, node_se_connector.inputs.agent_action, # is_low_priority=True is_backward=False) # blank connection Connector.connect(blank_task_control.outputs.output, node_se_connector.inputs.task_control) Connector.connect(blank_task_labels.outputs.output, node_se_connector.inputs.agent_to_task_label) # Save the SE connector so we can check testing/training phase. # When the se_io interface has been added, this can be removed. self._node_se_connector = node_se_connector
def __init__(self, sequence_generators: List[Any] = None, gamma=0.8): super().__init__('cpu') # environment if sequence_generators is None: sequence_generator_0 = SequenceGenerator( [ [1, 2, 3, 1, 2, 3, 1, 2, 3], [5, 4, 3, 5, 4, 3, 5, 4, 3], [1, 2, 3, 1, 2, 3, 1, 2, 3], [5, 4, 3, 5, 4, 3, 5, 4, 3], ] , diagonal_transition_matrix(4, 0.8)) sequence_generator_1 = SequenceGenerator( [ [1, 2, 3, 1, 2, 3, 1, 2, 3], [5, 4, 3, 5, 4, 3, 5, 4, 3], [1, 2, 3, 1, 2, 3, 1, 2, 3], [5, 4, 3, 5, 4, 3, 5, 4, 3], ] , diagonal_transition_matrix(4, 0.8)) sequence_generators = [sequence_generator_0, sequence_generator_1] self.sequences_len = len(sequence_generators) # switch switch_node = SwitchableSequencesNodeGroup(sequence_generators) self.switch_node = switch_node self.add_node(switch_node) # gate self.add_node(self.get_gate()) # predictors self.add_node(self.get_predictors()) # discount buffer def discount(inputs: List[torch.Tensor], outputs: List[torch.Tensor], memory: List[torch.Tensor]): cum_val = (1. - gamma) * inputs[0].squeeze() + gamma * memory[0] memory[0].copy_(cum_val) outputs[0].copy_(cum_val) discount_node = LambdaNode(discount, 1, [(self.get_num_predictors(),)], memory_shapes=[(self.get_num_predictors(),)], name="Discount") self.add_node(discount_node) # argmin def argmin(inputs: List[torch.Tensor], outputs: List[torch.Tensor]): outputs[0].copy_(id_to_one_hot(inputs[0].argmin(), self.get_num_predictors())) argmin_lambda_node = LambdaNode(argmin, 1, [(self.get_num_predictors(),)], name="Argmin") self.add_node(argmin_lambda_node) # dot def dot_product(inputs: List[torch.Tensor], outputs: List[torch.Tensor]): outputs[0].copy_(inputs[0].squeeze().dot(inputs[1].squeeze())) dot_node = LambdaNode(dot_product, 2, [(1,)], name="predictors error dot gate activations") self.dot_node = dot_node self.add_node(dot_node) # connections Connector.connect(switch_node.outputs.output, self.get_predictors().inputs.data) Connector.connect(switch_node.outputs.output, self.get_gate().inputs.data_predicted) Connector.connect(switch_node.outputs.sequence_num, self.get_gate().inputs.data) Connector.connect(self.get_predictors().outputs.predictors_activations_error, discount_node.inputs[0]) Connector.connect(discount_node.outputs[0], argmin_lambda_node.inputs[0]) Connector.connect(argmin_lambda_node.outputs[0], self.get_gate().inputs.predictor_activations_target) Connector.connect(self.get_gate().outputs.gate_activations, self.get_predictors().inputs.temporal_class_distribution, is_backward=True) Connector.connect(self.get_predictors().outputs.predictors_activations_error, dot_node.inputs[0]) Connector.connect(self.get_gate().outputs.gate_activations, dot_node.inputs[1])
def __init__(self, num_predictors, spatial_input_size, predicted_input_size, name: str = "NNGateNodeGroup"): super().__init__(name, inputs=GateInputs(self), outputs=GateOutputs(self)) # join input and label inputs_join_node = JoinNode(flatten=True) self.add_node(inputs_join_node) # gate g_node_params = NetworkFlockNodeParams() g_node_params.flock_size = 1 g_node_params.do_delay_coefficients = False g_node_params.do_delay_input = True g_node_params.learning_period = 20 g_node_params.buffer_size = 1000 g_node_params.batch_size = 900 g_network_params = NeuralNetworkFlockParams() g_network_params.input_size = spatial_input_size + predicted_input_size g_network_params.mini_batch_size = 100 g_network_params.hidden_size = g_network_params.input_size * 2 g_network_params.output_size = num_predictors g_network_params.output_activation = OutputActivation.SOFTMAX g_network_params.learning_rate = 0.05 gate = NetworkFlockNode(node_params=g_node_params, network_params=g_network_params, name="gate") self.gate = gate self.add_node(gate) # const learning_constant = ConstantNode([1], 1) self.add_node(learning_constant) # squeeze squeeze_node = SqueezeNode(0) self.add_node(squeeze_node) to_one_hot_lambda_node_p = LambdaNode( partial(to_one_hot, predicted_input_size), 1, [(predicted_input_size, )]) self.add_node(to_one_hot_lambda_node_p) to_one_hot_lambda_node_s = LambdaNode( partial(to_one_hot, spatial_input_size), 1, [(spatial_input_size, )]) self.add_node(to_one_hot_lambda_node_s) # connections Connector.connect(self.inputs.data.output, to_one_hot_lambda_node_s.inputs[0]) Connector.connect(to_one_hot_lambda_node_s.outputs[0], inputs_join_node.inputs[0]) Connector.connect(self.inputs.data_predicted.output, to_one_hot_lambda_node_p.inputs[0]) Connector.connect(to_one_hot_lambda_node_p.outputs[0], inputs_join_node.inputs[1]) Connector.connect(learning_constant.outputs.output, gate.inputs.learning_coefficients) Connector.connect(inputs_join_node.outputs.output, gate.inputs.input_data) Connector.connect(self.inputs.predictor_activations_target.output, gate.inputs.target_data) Connector.connect(gate.outputs.prediction_output, squeeze_node.inputs[0]) Connector.connect(squeeze_node.outputs[0], self.outputs.gate_activations.input)
class SeT0ConvTopology(SeT0TopologicalGraph): EXPERTS_IN_X = 4 BATCH = 2000 TRAINING = True parent_params = ExpertParams() parent_params.flock_size = 1 parent_params.n_cluster_centers = 80 parent_params.compute_reconstruction = True parent_params.spatial.learning_period = 400 parent_params.spatial.learning_rate = 0.5 if not TRAINING: parent_params.spatial.learning_rate *= 0 parent_params.spatial.buffer_size = BATCH parent_params.spatial.batch_size = BATCH parent_params.spatial.cluster_boost_threshold = 200 mid_expert_params = ExpertParams() mid_expert_params.flock_size = 1 mid_expert_params.n_cluster_centers = 100 mid_expert_params.compute_reconstruction = True mid_expert_params.spatial.cluster_boost_threshold = 200 mid_expert_params.spatial.learning_rate = 0.5 if not TRAINING: mid_expert_params.spatial.learning_rate *= 0 mid_expert_params.spatial.batch_size = BATCH mid_expert_params.spatial.buffer_size = BATCH mid_expert_params.spatial.learning_period = 400 conv_flock_params = ExpertParams() conv_flock_params.flock_size = EXPERTS_IN_X**2 conv_flock_params.n_cluster_centers = 600 conv_flock_params.spatial.learning_period = 1000 conv_flock_params.spatial.learning_rate = 0.2 if not TRAINING: conv_flock_params.spatial.learning_rate *= 0 conv_flock_params.spatial.buffer_size = BATCH conv_flock_params.spatial.batch_size = BATCH conv_flock_params.spatial.cluster_boost_threshold = 1000 # parent_rf_stride_dims = (16, 16) def _install_experts(self): im_width = self.se_io.get_image_width() im_height = self.se_io.get_image_height() self.input_dims = torch.Size((im_height, im_width, 3)) self.parent_rf_dims = Size2D(im_height // self.EXPERTS_IN_X, im_width // self.EXPERTS_IN_X) lrf_node = ReceptiveFieldNode(input_dims=self.input_dims, parent_rf_dims=self.parent_rf_dims) self.add_node(lrf_node) self._top_level_flock_node = SpatialPoolerFlockNode(self.parent_params) self._mid_node = ExpertFlockNode(self.mid_expert_params) self._conv_node = ConvExpertFlockNode(self.conv_flock_params) self.add_node(self._top_level_flock_node) self.add_node(self._mid_node) self.add_node(self._conv_node) def rescale(inputs, outputs): if self.TRAINING: outputs[0].copy_( inputs[0] * 1000) # large constant to make the label more important else: outputs[0].copy_(inputs[0] * float('nan')) self._rescale_node = LambdaNode(rescale, 1, [(self.se_io.get_num_labels(), )]) self.add_node(self._rescale_node) unsqueeze_node_0 = UnsqueezeNode(0) self.add_node(unsqueeze_node_0) unsqueeze_node_1 = UnsqueezeNode(0) self.add_node(unsqueeze_node_1) Connector.connect(self.se_io.outputs.image_output, lrf_node.inputs.input) Connector.connect(lrf_node.outputs.output, self._conv_node.inputs.sp.data_input) Connector.connect(self._conv_node.outputs.tp.projection_outputs, unsqueeze_node_0.inputs.input) Connector.connect(unsqueeze_node_0.outputs.output, self._mid_node.inputs.sp.data_input) Connector.connect(self._mid_node.outputs.tp.projection_outputs, self._join_node.inputs[0]) Connector.connect(self.se_io.outputs.task_to_agent_label, self._rescale_node.inputs[0]) Connector.connect(self._rescale_node.outputs[0], self._join_node.inputs[1]) Connector.connect(self._join_node.outputs.output, unsqueeze_node_1.inputs.input) Connector.connect(unsqueeze_node_1.outputs.output, self._top_level_flock_node.inputs.sp.data_input) def _top_level_expert_output_size(self): return self.mid_expert_params.n_cluster_centers * self.mid_expert_params.flock_size def switch_training_testing(self, training: bool): if self._rescale_node._unit is None: return def rescale(inputs, outputs): if training: outputs[0].copy_( inputs[0] * 1000) # large constant to make the label more important else: outputs[0].copy_(inputs[0] * float('nan')) self._rescale_node.change_function(rescale) _topology_test_checked: bool = False def get_properties(self): prop_list = super().get_properties() def topology_test_checked(value): self._topology_test_checked = value self.switch_training_testing(not parse_bool(value)) return value prop_list.append( ObserverPropertiesItem("Test topology", 'checkbox', self._topology_test_checked, topology_test_checked)) return prop_list def _get_agent_output(self) -> MemoryBlock: return self._top_level_flock_node.outputs.sp.current_reconstructed_input
def __init__(self, step_callback): super().__init__('cpu') self.add_node(LambdaNode(lambda inputs, outputs: step_callback(), 0, []))
def __init__(self): super().__init__(device='cuda') actions_descriptor = GridWorldActionDescriptor() node_action_monitor = ActionMonitorNode(actions_descriptor) params = GridWorldParams(map_name='MapE') noise_params = RandomNoiseParams(amplitude=0.0001) node_grid_world = GridWorldNode(params) expert_params = ExpertParams() unsqueeze_node = UnsqueezeNode(dim=0) noise_node = RandomNoiseNode(noise_params) constant_node = ConstantNode(shape=(1, 1, 3, 48)) one_hot_node = ToOneHotNode() def context(inputs, outputs): con = inputs[0] con[:, :, 1:, 24:] = float('nan') outputs[0].copy_(con) def f(inputs, outputs): probs = inputs[0] outputs[0].copy_(probs[0, -1, :4] + SMALL_CONSTANT) action_parser = LambdaNode(func=f, n_inputs=1, output_shapes=[(4,)]) context_assembler = LambdaNode(func=context, n_inputs=1, output_shapes=[(1, 1, 3, 48)]) expert_params.flock_size = 1 expert_params.n_cluster_centers = 24 expert_params.produce_actions = True expert_params.temporal.seq_length = 9 expert_params.temporal.seq_lookahead = 7 expert_params.temporal.n_frequent_seqs = 700 expert_params.temporal.max_encountered_seqs = 1000 expert_params.temporal.exploration_probability = 0.01 expert_params.temporal.batch_size = 200 expert_params.temporal.own_rewards_weight = 20 expert_params.temporal.incoming_context_size = 48 expert_params.compute_reconstruction = True #expert_node = ConvExpertFlockNode(expert_params) expert_node = ExpertFlockNode(expert_params) self.add_node(node_grid_world) self.add_node(node_action_monitor) self.add_node(expert_node) self.add_node(unsqueeze_node) self.add_node(action_parser) self.add_node(noise_node) self.add_node(constant_node) self.add_node(context_assembler) self.add_node(one_hot_node) Connector.connect(node_grid_world.outputs.egocentric_image_action, noise_node.inputs.input) Connector.connect(noise_node.outputs.output, unsqueeze_node.inputs.input) Connector.connect(unsqueeze_node.outputs.output, expert_node.inputs.sp.data_input) Connector.connect(node_grid_world.outputs.reward, expert_node.inputs.tp.reward_input) Connector.connect(constant_node.outputs.output, context_assembler.inputs[0]) Connector.connect(context_assembler.outputs[0], expert_node.inputs.tp.context_input) Connector.connect(expert_node.outputs.sp.predicted_reconstructed_input, action_parser.inputs[0]) Connector.connect(action_parser.outputs[0], one_hot_node.inputs.input) Connector.connect(one_hot_node.outputs.output, node_action_monitor.inputs.action_in) Connector.connect(node_action_monitor.outputs.action_out, node_grid_world.inputs.agent_action, is_backward=True)
def __init__(self): super().__init__(device='cuda') actions_descriptor = GridWorldActionDescriptor() node_action_monitor = ActionMonitorNode(actions_descriptor) params = GridWorldParams(map_name='MapE') noise_params = RandomNoiseParams(amplitude=0.0001) node_grid_world = GridWorldNode(params) expert_params1 = ExpertParams() unsqueeze_node = UnsqueezeNode(dim=0) noise_node = RandomNoiseNode(noise_params) one_hot_node = ToOneHotNode() def f(inputs, outputs): probs = inputs[0] outputs[0].copy_(probs[0, -1, :4] + SMALL_CONSTANT) action_parser = LambdaNode(func=f, n_inputs=1, output_shapes=[(4, )]) expert_params1.flock_size = 1 expert_params1.n_cluster_centers = 24 expert_params1.produce_actions = True expert_params1.temporal.seq_length = 4 expert_params1.temporal.seq_lookahead = 2 expert_params1.temporal.n_frequent_seqs = 700 expert_params1.temporal.max_encountered_seqs = 1000 expert_params1.temporal.exploration_probability = 0.05 expert_params1.temporal.batch_size = 200 expert_params1.temporal.frustration_threshold = 2 # expert_params.temporal.own_rewards_weight = 20 expert_params1.compute_reconstruction = True expert_params2 = expert_params1.clone() expert_params2.temporal.seq_length = 5 expert_params2.temporal.seq_lookahead = 4 expert_params2.n_cluster_centers = 8 expert_params2.produce_actions = False expert_params2.temporal.frustration_threshold = 10 #expert_params1.temporal.incoming_context_size = 2 * expert_params2.n_cluster_centers expert_node1 = ExpertFlockNode(expert_params1) expert_node2 = ExpertFlockNode(expert_params2) self.add_node(node_grid_world) self.add_node(node_action_monitor) self.add_node(expert_node1) self.add_node(expert_node2) self.add_node(unsqueeze_node) self.add_node(action_parser) self.add_node(noise_node) self.add_node(one_hot_node) Connector.connect(node_grid_world.outputs.output_image_action, noise_node.inputs.input) Connector.connect(noise_node.outputs.output, unsqueeze_node.inputs.input) Connector.connect(unsqueeze_node.outputs.output, expert_node1.inputs.sp.data_input) Connector.connect(expert_node1.outputs.tp.projection_outputs, expert_node2.inputs.sp.data_input) Connector.connect(expert_node2.outputs.output_context, expert_node1.inputs.tp.context_input, is_backward=True) Connector.connect( expert_node1.outputs.sp.predicted_reconstructed_input, action_parser.inputs[0]) Connector.connect(node_grid_world.outputs.reward, expert_node1.inputs.tp.reward_input) Connector.connect(node_grid_world.outputs.reward, expert_node2.inputs.tp.reward_input) Connector.connect(action_parser.outputs[0], one_hot_node.inputs.input) Connector.connect(one_hot_node.outputs.output, node_action_monitor.inputs.action_in) Connector.connect(node_action_monitor.outputs.action_out, node_grid_world.inputs.agent_action, is_backward=True)