def __init__(self):
super().__init__("Single Expert")
expert_params = ExpertParams()
expert_params.flock_size = 1
expert_params.n_cluster_centers = 24
expert_params.produce_actions = True
expert_params.temporal.seq_length = 9
expert_params.temporal.seq_lookahead = 7
expert_params.temporal.n_frequent_seqs = 700
expert_params.temporal.max_encountered_seqs = 1000
expert_params.temporal.exploration_probability = 0.01
expert_params.temporal.batch_size = 200
expert_params.temporal.own_rewards_weight = 20
expert_params.temporal.compute_backward_pass = True
expert_params.compute_reconstruction = True
expert_node = ExpertFlockNode(expert_params)
self.add_node(expert_node)
Connector.connect(self.inputs.data.output, expert_node.inputs.sp.data_input)
Connector.connect(self.inputs.reward.output, expert_node.inputs.tp.reward_input)
Connector.connect(expert_node.outputs.sp.predicted_reconstructed_input,
self.outputs.predicted_reconstructed_input.input)
def _create_and_connect_agent(self, join_node: JoinNode, fork_node: ForkNode):
params = ExpertParams()
params.flock_size = 1
params.n_cluster_centers = 28
params.compute_reconstruction = True
params.spatial.cluster_boost_threshold = 1000
params.spatial.buffer_size = 500
params.spatial.batch_size = 500
params.spatial.learning_rate = 0.3
params.spatial.learning_period = 50
# conv_expert = ConvExpertFlockNode(params, name="Conv. expert")
# conv_expert = SpatialPoolerFlockNode(params, name=" SP")
conv_expert = ExpertFlockNode(params, name=" expert")
self.add_node(conv_expert)
unsqueeze_node_0 = UnsqueezeNode(0)
self.add_node(unsqueeze_node_0)
Connector.connect(join_node.outputs.output, unsqueeze_node_0.inputs.input)
Connector.connect(unsqueeze_node_0.outputs.output, conv_expert.inputs.sp.data_input)
def squeeze(inputs, outputs):
outputs[0].copy_(inputs[0].squeeze(0))
squeeze_node = LambdaNode(squeeze, 1, [(sum(fork_node._split_sizes),)],
name="squeeze lambda node")
self.add_node(squeeze_node)
Connector.connect(conv_expert.outputs.sp.predicted_reconstructed_input, squeeze_node.inputs[0])
Connector.connect(squeeze_node.outputs[0], fork_node.inputs.input)
def _create_expert_params() -> ExpertParams:
expert_params = ExpertParams()
expert_params.flock_size = 1
expert_params.n_cluster_centers = 200
expert_params.compute_reconstruction = True
expert_params.spatial.batch_size = 1000
expert_params.spatial.buffer_size = 1010
expert_params.spatial.cluster_boost_threshold = 200
return expert_params
def _make_top_layer(
num_labels: int, input_data_size: int, n_cluster_centers: int
) -> (SpReconstructionLayer, InputSlot, OutputSlotBase):
sp_params = ExpertParams()
sp_params.flock_size = 1
sp_params.n_cluster_centers = n_cluster_centers
sp_params.compute_reconstruction = True
layer = SpReconstructionLayer(input_data_size=input_data_size,
labels_size=num_labels,
sp_params=sp_params,
name='TOP')
return layer, layer.inputs.data, None
def _create_expert_params(self) -> ExpertParams:
expert_params = ExpertParams()
expert_params.flock_size = 1
expert_params.n_cluster_centers = self._num_ccs
expert_params.compute_reconstruction = True
expert_params.spatial.batch_size = 990
expert_params.spatial.buffer_size = self._buffer_size
expert_params.spatial.cluster_boost_threshold = self._num_ccs * 2
expert_params.spatial.learning_rate = SeT0BasicTopologyRT211.LEARNING_RATE
expert_params.spatial.sampling_method = self._sampling_method
expert_params.spatial.learning_period = 10
expert_params.spatial.max_boost_time = 5000
return expert_params
def create_flock(input_size,
flock_size,
incoming_context_size,
seq_length=4,
n_cluster_centers=20,
max_encountered_seqs=1000,
n_frequent_seqs=500,
sp_buffer_size=3000,
sp_batch_size=300,
sp_learning_period=20,
convolutional: bool = False,
sp_max_boost_time=1000,
sp_boost_threshold=100,
sampling_method: SamplingMethod = SamplingMethod.BALANCED,
n_context_providers=1,
device='cuda'):
params = ExpertParams()
params.n_cluster_centers = n_cluster_centers
params.flock_size = flock_size
params.compute_reconstruction = True
sp_params = params.spatial
sp_params.input_size = input_size
sp_params.buffer_size = sp_buffer_size
sp_params.batch_size = sp_batch_size
sp_params.learning_rate = 0.1
sp_params.cluster_boost_threshold = sp_boost_threshold
sp_params.max_boost_time = sp_max_boost_time
sp_params.learning_period = sp_learning_period
sp_params.sampling_method = sampling_method
tp_params = params.temporal
tp_params.buffer_size = 100
tp_params.seq_length = seq_length
tp_params.batch_size = 50 + tp_params.seq_length - 1
tp_params.learning_period = 50
tp_params.seq_lookahead = 2
tp_params.n_frequent_seqs = n_frequent_seqs
tp_params.max_encountered_seqs = max_encountered_seqs
tp_params.forgetting_limit = 5000
tp_params.incoming_context_size = incoming_context_size
tp_params.n_providers = n_context_providers
if convolutional:
print("Created convolutional flock.")
return ConvExpertFlock(params, AllocatingCreator(device))
else:
print("Created normal flock.")
return ExpertFlock(params, AllocatingCreator(device))
def convert_to_expert_params(self):
"""Parse from the MultipleLayerParams to list of ExpertParams"""
self.validate_params_for_n_layers()
params_list = []
for layer_id in range(self.num_conv_layers):
params = ExpertParams()
params.flock_size = 1
# spatial
params.n_cluster_centers = self.read_param('n_cluster_centers',
layer_id)
params.spatial.buffer_size = self.read_param(
'sp_buffer_size', layer_id)
params.spatial.batch_size = self.read_param(
'sp_batch_size', layer_id)
params.spatial.cluster_boost_threshold = self.read_param(
'cluster_boost_threshold', layer_id)
params.spatial.max_boost_time = self.read_param(
'max_boost_time', layer_id)
params.spatial.learning_rate = self.read_param(
'learning_rate', layer_id)
params.spatial.sampling_method = self.read_param(
'sampling_method', layer_id)
params.compute_reconstruction = self.read_param(
'compute_reconstruction', layer_id)
# temporal
params.temporal.seq_length = self.read_param(
'seq_length', layer_id)
params.temporal.seq_lookahead = self.read_param(
'seq_lookahead', layer_id)
params.temporal.max_encountered_seqs = self.read_param(
'max_encountered_seqs', layer_id)
params.temporal.n_frequent_seqs = self.read_param(
'max_frequent_seqs', layer_id)
params.temporal.exploration_probability = self.read_param(
'exploration_probability', layer_id)
# done
params_list.append(params)
return params_list
def __init__(self, c_n_ccs, c_buffer_size, c_seq_length, c_seq_lookahead, p_seq_length, p_seq_lookahead, p_n_ccs,
flock_size):
super().__init__("Two Experts")
expert_params1 = ExpertParams()
expert_params1.flock_size = flock_size
expert_params1.n_cluster_centers = c_n_ccs
expert_params1.produce_actions = True
expert_params1.temporal.seq_length = c_seq_length
expert_params1.temporal.seq_lookahead = c_seq_lookahead
expert_params1.temporal.n_frequent_seqs = 700
expert_params1.temporal.max_encountered_seqs = 1000
expert_params1.temporal.exploration_probability = 0.05
expert_params1.temporal.batch_size = 200
expert_params1.temporal.compute_backward_pass = True
expert_params1.temporal.frustration_threshold = 2
expert_params2 = expert_params1.clone()
expert_params1.spatial.buffer_size = c_buffer_size
expert_params1.compute_reconstruction = True
expert_params2.temporal.seq_length = p_seq_length
expert_params2.temporal.seq_lookahead = p_seq_lookahead
expert_params2.n_cluster_centers = p_n_ccs
expert_params2.produce_actions = False
expert_params2.temporal.frustration_threshold = 10
expert_node1 = ExpertFlockNode(expert_params1)
expert_node2 = ExpertFlockNode(expert_params2)
self.add_node(expert_node1)
self.add_node(expert_node2)
Connector.connect(self.inputs.data.output, expert_node1.inputs.sp.data_input)
Connector.connect(self.inputs.reward.output, expert_node1.inputs.tp.reward_input)
Connector.connect(self.inputs.reward.output, expert_node2.inputs.tp.reward_input)
# Connect the experts to each other.
Connector.connect(expert_node1.outputs.tp.projection_outputs, expert_node2.inputs.sp.data_input)
Connector.connect(expert_node2.outputs.output_context, expert_node1.inputs.tp.context_input, is_backward=True)
# Connect the group output.
Connector.connect(expert_node1.outputs.sp.predicted_reconstructed_input,
self.outputs.predicted_reconstructed_input.input)
def prepare_flock_for_context(flock_size=5,
input_size=3,
n_cluster_centers=3,
seq_lookahead=1,
seq_length=3,
max_provider_context_size=1,
n_providers=2,
device='cuda',
compute_reconstruction=False):
params = ExpertParams()
params.compute_reconstruction = compute_reconstruction
params.flock_size = flock_size
params.n_cluster_centers = n_cluster_centers
params.spatial.input_size = input_size
params.temporal.seq_lookahead = seq_lookahead
params.temporal.seq_length = seq_length
params.temporal.incoming_context_size = max_provider_context_size
params.temporal.n_providers = n_providers
flock = ExpertFlock(params, AllocatingCreator(device))
return flock
def __init__(self):
super().__init__(device='cuda')
actions_descriptor = GridWorldActionDescriptor()
node_action_monitor = ActionMonitorNode(actions_descriptor)
params = GridWorldParams(map_name='MapE')
noise_params = RandomNoiseParams(amplitude=0.0001)
node_grid_world = GridWorldNode(params)
expert_params = ExpertParams()
unsqueeze_node = UnsqueezeNode(dim=0)
noise_node = RandomNoiseNode(noise_params)
constant_node = ConstantNode(shape=(1, 1, 3, 48))
one_hot_node = ToOneHotNode()
def context(inputs, outputs):
con = inputs[0]
con[:, :, 1:, 24:] = float('nan')
outputs[0].copy_(con)
def f(inputs, outputs):
probs = inputs[0]
outputs[0].copy_(probs[0, -1, :4] + SMALL_CONSTANT)
action_parser = LambdaNode(func=f, n_inputs=1, output_shapes=[(4,)])
context_assembler = LambdaNode(func=context, n_inputs=1, output_shapes=[(1, 1, 3, 48)])
expert_params.flock_size = 1
expert_params.n_cluster_centers = 24
expert_params.produce_actions = True
expert_params.temporal.seq_length = 9
expert_params.temporal.seq_lookahead = 7
expert_params.temporal.n_frequent_seqs = 700
expert_params.temporal.max_encountered_seqs = 1000
expert_params.temporal.exploration_probability = 0.01
expert_params.temporal.batch_size = 200
expert_params.temporal.own_rewards_weight = 20
expert_params.temporal.incoming_context_size = 48
expert_params.compute_reconstruction = True
#expert_node = ConvExpertFlockNode(expert_params)
expert_node = ExpertFlockNode(expert_params)
self.add_node(node_grid_world)
self.add_node(node_action_monitor)
self.add_node(expert_node)
self.add_node(unsqueeze_node)
self.add_node(action_parser)
self.add_node(noise_node)
self.add_node(constant_node)
self.add_node(context_assembler)
self.add_node(one_hot_node)
Connector.connect(node_grid_world.outputs.egocentric_image_action, noise_node.inputs.input)
Connector.connect(noise_node.outputs.output, unsqueeze_node.inputs.input)
Connector.connect(unsqueeze_node.outputs.output, expert_node.inputs.sp.data_input)
Connector.connect(node_grid_world.outputs.reward, expert_node.inputs.tp.reward_input)
Connector.connect(constant_node.outputs.output, context_assembler.inputs[0])
Connector.connect(context_assembler.outputs[0], expert_node.inputs.tp.context_input)
Connector.connect(expert_node.outputs.sp.predicted_reconstructed_input, action_parser.inputs[0])
Connector.connect(action_parser.outputs[0], one_hot_node.inputs.input)
Connector.connect(one_hot_node.outputs.output, node_action_monitor.inputs.action_in)
Connector.connect(node_action_monitor.outputs.action_out, node_grid_world.inputs.agent_action, is_backward=True)
def __init__(self):
super().__init__(device='cuda')
actions_descriptor = GridWorldActionDescriptor()
node_action_monitor = ActionMonitorNode(actions_descriptor)
params = GridWorldParams(map_name='MapTwoRoom',
reset_strategy=ResetStrategy.ANYWHERE)
noise_params = RandomNoiseParams(amplitude=0.0001)
node_grid_world = GridWorldNode(params)
expert_params = ExpertParams()
unsqueeze_node = UnsqueezeNode(dim=0)
noise_node = RandomNoiseNode(noise_params)
one_hot_node = ToOneHotNode()
def f(inputs, outputs):
probs = inputs[0]
outputs[0].copy_(probs[0, -1, :4] + SMALL_CONSTANT)
action_parser = LambdaNode(func=f, n_inputs=1, output_shapes=[(4, )])
expert_params.flock_size = 1
expert_params.n_cluster_centers = 64
expert_params.produce_actions = True
expert_params.temporal.seq_length = 17
expert_params.temporal.seq_lookahead = 13
expert_params.temporal.n_frequent_seqs = 700
expert_params.temporal.max_encountered_seqs = 1000
expert_params.temporal.exploration_probability = 0.05
expert_params.temporal.batch_size = 200
expert_params.temporal.buffer_size = 1000
expert_params.temporal.own_rewards_weight = 20
expert_params.temporal.frustration_threshold = 2
expert_params.temporal.compute_backward_pass = True
expert_params.compute_reconstruction = True
expert_node = ConvExpertFlockNode(expert_params)
#expert_node = ExpertFlockNode(expert_params)
self.add_node(node_grid_world)
self.add_node(node_action_monitor)
self.add_node(expert_node)
self.add_node(unsqueeze_node)
self.add_node(action_parser)
self.add_node(noise_node)
self.add_node(one_hot_node)
Connector.connect(node_grid_world.outputs.egocentric_image_action,
noise_node.inputs.input)
Connector.connect(noise_node.outputs.output,
unsqueeze_node.inputs.input)
Connector.connect(unsqueeze_node.outputs.output,
expert_node.inputs.sp.data_input)
Connector.connect(node_grid_world.outputs.reward,
expert_node.inputs.tp.reward_input)
Connector.connect(expert_node.outputs.sp.predicted_reconstructed_input,
action_parser.inputs[0])
Connector.connect(action_parser.outputs[0], one_hot_node.inputs.input)
Connector.connect(one_hot_node.outputs.output,
node_action_monitor.inputs.action_in)
Connector.connect(node_action_monitor.outputs.action_out,
node_grid_world.inputs.agent_action,
is_backward=True)
def __init__(self, curriculum: tuple = (1, -1)):
super().__init__()
se_config = SpaceEngineersConnectorConfig()
se_config.render_width = 16
se_config.render_height = 16
se_config.curriculum = list(curriculum)
base_expert_params = ExpertParams()
base_expert_params.flock_size = 1
base_expert_params.n_cluster_centers = 100
base_expert_params.compute_reconstruction = False
base_expert_params.spatial.cluster_boost_threshold = 1000
base_expert_params.spatial.learning_rate = 0.2
base_expert_params.spatial.batch_size = 1000
base_expert_params.spatial.buffer_size = 1010
base_expert_params.spatial.learning_period = 100
base_expert_params.temporal.batch_size = 1000
base_expert_params.temporal.buffer_size = 1010
base_expert_params.temporal.learning_period = 200
base_expert_params.temporal.forgetting_limit = 20000
# parent_expert_params = ExpertParams()
# parent_expert_params.flock_size = 1
# parent_expert_params.n_cluster_centers = 20
# parent_expert_params.compute_reconstruction = True
# parent_expert_params.temporal.exploration_probability = 0.9
# parent_expert_params.spatial.cluster_boost_threshold = 1000
# parent_expert_params.spatial.learning_rate = 0.2
# parent_expert_params.spatial.batch_size = 1000
# parent_expert_params.spatial.buffer_size = 1010
# parent_expert_params.spatial.learning_period = 100
# parent_expert_params.temporal.context_without_rewards_size = se_config.LOCATION_SIZE_ONE_HOT
# SE nodes
actions_descriptor = SpaceEngineersActionsDescriptor()
node_se_connector = SpaceEngineersConnectorNode(
actions_descriptor, se_config)
node_action_monitor = ActionMonitorNode(actions_descriptor)
# flock-related nodes
flock_node = ExpertFlockNode(base_expert_params)
blank_task_control = ConstantNode((se_config.TASK_CONTROL_SIZE, ))
blank_task_labels = ConstantNode((20, ))
# parent_flock_node = ExpertFlockNode(parent_expert_params)
join_node = JoinNode(flatten=True)
actions = ['FORWARD', 'BACKWARD', 'LEFT', 'RIGHT']
action_count = len(actions)
pass_actions_node = PassNode(output_shape=(action_count, ),
name="pass actions")
fork_node = ForkNode(
0, [base_expert_params.n_cluster_centers, action_count])
def squeeze(inputs, outputs):
outputs[0].copy_(inputs[0].squeeze())
squeeze_node = LambdaNode(
squeeze,
1, [(base_expert_params.n_cluster_centers + action_count, )],
name="squeeze lambda node")
def stack_and_unsqueeze(inputs, outputs):
outputs[0].copy_(torch.stack([inputs[0], inputs[1]]).unsqueeze(0))
stack_unsqueeze_node = LambdaNode(
stack_and_unsqueeze,
2, [(1, 2, se_config.LOCATION_SIZE_ONE_HOT)],
name="stack and unsqueeze node")
to_one_hot_node = ToOneHotNode()
action_parser_node = AgentActionsParserNode(actions_descriptor,
actions)
random_node = RandomNumberNode(0,
action_count,
name="random action generator",
generate_new_every_n=5,
randomize_intervals=True)
switch_node = SwitchNode(2)
# add nodes to the graph
self.add_node(flock_node)
# self.add_node(parent_flock_node)
self.add_node(node_se_connector)
self.add_node(node_action_monitor)
self.add_node(blank_task_control)
self.add_node(blank_task_labels)
# self.add_node(join_node)
# self.add_node(fork_node)
# self.add_node(pass_actions_node)
# self.add_node(squeeze_node)
# self.add_node(to_one_hot_node)
# self.add_node(stack_unsqueeze_node)
self.add_node(action_parser_node)
self.add_node(random_node)
# self.add_node(switch_node)
# first layer
Connector.connect(node_se_connector.outputs.image_output,
flock_node.inputs.sp.data_input)
# Connector.connect(
# flock_node.outputs.tp.projection_outputs,
# join_node.inputs[0]
# )
# Connector.connect(
# pass_actions_node.outputs.output,
# join_node.inputs[1]
# )
# # second layer
# Connector.connect(
# join_node.outputs.output,
# parent_flock_node.inputs.sp.data_input
# )
# Connector.connect(
# node_se_connector.outputs.task_to_agent_location_one_hot,
# stack_unsqueeze_node.inputs[0]
# )
# Connector.connect(
# node_se_connector.outputs.task_to_agent_location_target_one_hot,
# stack_unsqueeze_node.inputs[1]
# )
# Connector.connect(
# stack_unsqueeze_node.outputs[0],
# parent_flock_node.inputs.tp.context_input
# )
#
# # actions
# Connector.connect(
# parent_flock_node.outputs.sp.predicted_reconstructed_input,
# squeeze_node.inputs[0]
# )
# Connector.connect(
# squeeze_node.outputs[0],
# fork_node.inputs.input
# )
# Connector.connect(
# fork_node.outputs[1],
# to_one_hot_node.inputs.input
# )
# Connector.connect(
# random_node.outputs.one_hot_output,
# switch_node.inputs[0]
# )
# Connector.connect(
# to_one_hot_node.outputs.output,
# switch_node.inputs[1]
# )
# Connector.connect(
# switch_node.outputs.output,
# action_parser_node.inputs.input
# )
# directly use random exploration
Connector.connect(random_node.outputs.one_hot_output,
action_parser_node.inputs.input)
Connector.connect(action_parser_node.outputs.output,
node_action_monitor.inputs.action_in)
# Connector.connect(
# switch_node.outputs.output,
# pass_actions_node.inputs.input,
# is_low_priority=True
# )
Connector.connect(
node_action_monitor.outputs.action_out,
node_se_connector.inputs.agent_action,
# is_low_priority=True
is_backward=False)
# blank connection
Connector.connect(blank_task_control.outputs.output,
node_se_connector.inputs.task_control)
Connector.connect(blank_task_labels.outputs.output,
node_se_connector.inputs.agent_to_task_label)
# Save the SE connector so we can check testing/training phase.
# When the se_io interface has been added, this can be removed.
self._node_se_connector = node_se_connector
def __init__(self,
action_count=4,
location_vector_size=100,
use_grayscale: bool = False):
super().__init__("Task 1 - Basic expert",
inputs=Task1BasicExpertGroupInputs(self),
outputs=Task1BasicExpertGroupOutputs(self))
base_expert_params = ExpertParams()
base_expert_params.flock_size = 1
base_expert_params.n_cluster_centers = 100
base_expert_params.compute_reconstruction = False
base_expert_params.spatial.cluster_boost_threshold = 1000
base_expert_params.spatial.learning_rate = 0.2
base_expert_params.spatial.batch_size = 1000
base_expert_params.spatial.buffer_size = 1010
base_expert_params.spatial.learning_period = 100
parent_expert_params = ExpertParams()
parent_expert_params.flock_size = 1
parent_expert_params.n_cluster_centers = 20
parent_expert_params.compute_reconstruction = True
parent_expert_params.temporal.exploration_probability = 0.9
parent_expert_params.spatial.cluster_boost_threshold = 1000
parent_expert_params.spatial.learning_rate = 0.2
parent_expert_params.spatial.batch_size = 1000
parent_expert_params.spatial.buffer_size = 1010
parent_expert_params.spatial.learning_period = 100
parent_expert_params.temporal.context_without_rewards_size = location_vector_size
# flock-related nodes
flock_node = ExpertFlockNode(base_expert_params)
parent_flock_node = ExpertFlockNode(parent_expert_params)
join_node = JoinNode(flatten=True)
unsqueeze_node_to_base_expert = UnsqueezeNode(0)
unsqueeze_node_to_parent_expert = UnsqueezeNode(0)
fork_node = ForkNode(
0, [base_expert_params.n_cluster_centers, action_count])
def squeeze(inputs, outputs):
outputs[0].copy_(inputs[0].squeeze())
squeeze_node = LambdaNode(
squeeze,
1, [(base_expert_params.n_cluster_centers + action_count, )],
name="squeeze lambda node")
def stack_and_unsqueeze(inputs, outputs):
outputs[0].copy_(torch.stack([inputs[0], inputs[1]]).unsqueeze(0))
stack_unsqueeze_node = LambdaNode(stack_and_unsqueeze,
2, [(1, 2, location_vector_size)],
name="stack and unsqueeze node")
# add nodes to the graph
self.add_node(flock_node)
self.add_node(unsqueeze_node_to_base_expert)
self.add_node(parent_flock_node)
self.add_node(unsqueeze_node_to_parent_expert)
self.add_node(join_node)
self.add_node(fork_node)
self.add_node(squeeze_node)
self.add_node(stack_unsqueeze_node)
Connector.connect(self.inputs.actions.output, join_node.inputs[1])
if use_grayscale:
grayscale_node = GrayscaleNode(squeeze_channel=True)
self.add_node(grayscale_node)
Connector.connect(self.inputs.image.output,
grayscale_node.inputs.input)
Connector.connect(grayscale_node.outputs.output,
unsqueeze_node_to_base_expert.inputs.input)
else:
Connector.connect(self.inputs.image.output,
unsqueeze_node_to_base_expert.inputs.input)
Connector.connect(unsqueeze_node_to_base_expert.outputs.output,
flock_node.inputs.sp.data_input)
Connector.connect(self.inputs.current_location.output,
stack_unsqueeze_node.inputs[0])
Connector.connect(self.inputs.target_location.output,
stack_unsqueeze_node.inputs[1])
Connector.connect(fork_node.outputs[1], self.outputs.actions.input)
# first layer
Connector.connect(flock_node.outputs.tp.projection_outputs,
join_node.inputs[0])
# second layer
Connector.connect(join_node.outputs.output,
unsqueeze_node_to_parent_expert.inputs.input)
# second layer
Connector.connect(unsqueeze_node_to_parent_expert.outputs.output,
parent_flock_node.inputs.sp.data_input)
Connector.connect(stack_unsqueeze_node.outputs[0],
parent_flock_node.inputs.tp.context_input)
# actions
Connector.connect(
parent_flock_node.outputs.sp.predicted_reconstructed_input,
squeeze_node.inputs[0])
Connector.connect(squeeze_node.outputs[0], fork_node.inputs.input)
def __init__(self):
super().__init__(device='cuda')
actions_descriptor = GridWorldActionDescriptor()
node_action_monitor = ActionMonitorNode(actions_descriptor)
params = GridWorldParams(map_name='MapE')
noise_params = RandomNoiseParams(amplitude=0.0001)
node_grid_world = GridWorldNode(params)
expert_params1 = ExpertParams()
unsqueeze_node = UnsqueezeNode(dim=0)
noise_node = RandomNoiseNode(noise_params)
one_hot_node = ToOneHotNode()
def f(inputs, outputs):
probs = inputs[0]
outputs[0].copy_(probs[0, -1, :4] + SMALL_CONSTANT)
action_parser = LambdaNode(func=f, n_inputs=1, output_shapes=[(4, )])
expert_params1.flock_size = 1
expert_params1.n_cluster_centers = 24
expert_params1.produce_actions = True
expert_params1.temporal.seq_length = 4
expert_params1.temporal.seq_lookahead = 2
expert_params1.temporal.n_frequent_seqs = 700
expert_params1.temporal.max_encountered_seqs = 1000
expert_params1.temporal.exploration_probability = 0.05
expert_params1.temporal.batch_size = 200
expert_params1.temporal.frustration_threshold = 2
# expert_params.temporal.own_rewards_weight = 20
expert_params1.compute_reconstruction = True
expert_params2 = expert_params1.clone()
expert_params2.temporal.seq_length = 5
expert_params2.temporal.seq_lookahead = 4
expert_params2.n_cluster_centers = 8
expert_params2.produce_actions = False
expert_params2.temporal.frustration_threshold = 10
#expert_params1.temporal.incoming_context_size = 2 * expert_params2.n_cluster_centers
expert_node1 = ExpertFlockNode(expert_params1)
expert_node2 = ExpertFlockNode(expert_params2)
self.add_node(node_grid_world)
self.add_node(node_action_monitor)
self.add_node(expert_node1)
self.add_node(expert_node2)
self.add_node(unsqueeze_node)
self.add_node(action_parser)
self.add_node(noise_node)
self.add_node(one_hot_node)
Connector.connect(node_grid_world.outputs.output_image_action,
noise_node.inputs.input)
Connector.connect(noise_node.outputs.output,
unsqueeze_node.inputs.input)
Connector.connect(unsqueeze_node.outputs.output,
expert_node1.inputs.sp.data_input)
Connector.connect(expert_node1.outputs.tp.projection_outputs,
expert_node2.inputs.sp.data_input)
Connector.connect(expert_node2.outputs.output_context,
expert_node1.inputs.tp.context_input,
is_backward=True)
Connector.connect(
expert_node1.outputs.sp.predicted_reconstructed_input,
action_parser.inputs[0])
Connector.connect(node_grid_world.outputs.reward,
expert_node1.inputs.tp.reward_input)
Connector.connect(node_grid_world.outputs.reward,
expert_node2.inputs.tp.reward_input)
Connector.connect(action_parser.outputs[0], one_hot_node.inputs.input)
Connector.connect(one_hot_node.outputs.output,
node_action_monitor.inputs.action_in)
Connector.connect(node_action_monitor.outputs.action_out,
node_grid_world.inputs.agent_action,
is_backward=True)
def _create_expert_params() -> ExpertParams:
expert_params = ExpertParams()
expert_params.flock_size = 1
expert_params.n_cluster_centers = 20
expert_params.compute_reconstruction = True
return expert_params