def _inverse_projection(self, data: InversePassOutputPacket) -> List[InversePassInputPacket]:
# Join only has one output, no need to identify what actually came in.
result_tensors = self._unit.inverse_projection(data.tensor.view(self._unit.output.shape))
# Create inverse pass units based on the order of the inputs.
return [InversePassInputPacket(result_tensor.view(input_block.tensor.shape), input_block)
for result_tensor, input_block in zip(result_tensors, self.inputs)]
def _inverse_projection(
self,
data: InversePassOutputPacket) -> List[InversePassInputPacket]:
if self.reentry:
raise InfiniteLoopException
self.reentry = True
return [InversePassInputPacket(data, self.inputs.input1)]
def _inverse_projection(self, data: InversePassOutputPacket) -> List[InversePassInputPacket]:
if data.slot == self.outputs.tp.projection_outputs:
# Only calculate for the expert output
# projected = self._unit.inverse_projection(data.tensor)
projected = self._unit.inverse_projection(data.tensor.view(
self._unit.flock.sp_flock.forward_clusters.shape))
# Change the last dimension into the original input dimensions.
projected = projected.view(self.inputs.sp.data_input.tensor.shape)
return [InversePassInputPacket(projected, self.inputs.sp.data_input)]
return []
def _inverse_projection(
self,
data: InversePassOutputPacket) -> List[InversePassInputPacket]:
output_index = self.outputs.index(data.slot)
dtype = self.outputs[0].tensor.dtype
device = self.outputs[0].tensor.device
all_other_tensors = [
self._creator.zeros(output_block.tensor.shape,
dtype=dtype,
device=device) for output_block in self.outputs
if output_block != data.slot
]
all_tensors = all_other_tensors[:output_index] + [
data.tensor
] + all_other_tensors[output_index:]
result = self._unit.inverse_projection(all_tensors)
return [InversePassInputPacket(result, self.inputs.input)]
def get_inverse_projections_for_all_clusters(
node: HierarchicalObservableNode,
expert_no: int) -> List[List[torch.Tensor]]:
torch.cuda.synchronize()
_invalid_state_tensor = [[torch.full((1, 1), float('nan'))]]
# First get the inverse projection.
projected_values = node.projected_values
if projected_values is None:
return _invalid_state_tensor
if type(projected_values) is TensorSurrogate:
return _invalid_state_tensor
all_projections = []
for i in range(projected_values.shape[1]):
projected_values_of_expert = torch.zeros_like(projected_values[:, i])
projected_values_of_expert[expert_no].copy_(projected_values[expert_no,
i])
packet = InversePassInputPacket(projected_values_of_expert,
node.projection_input)
projections = node.recursive_inverse_projection_from_input(packet)
all_projections.append(projections)
if len(all_projections) == 0:
logger.warning('No projections were created.')
return _invalid_state_tensor
# The projections are (cluster_center, projection_type).
# We need (projection_type, cluster_center).
grouped_projections = []
for i in range(len(all_projections[0])):
same_input_projections = [
cc_projections[i].tensor for cc_projections in all_projections
]
grouped_projections.append(same_input_projections)
return grouped_projections
def _inverse_projection(self, data: InversePassOutputPacket) -> List[InversePassInputPacket]:
result = self._unit.inverse_projection(data.tensor)
return [InversePassInputPacket(result, self.inputs.input)]
def _inverse_projection(
self,
data: InversePassOutputPacket) -> List[InversePassInputPacket]:
output_index = self.outputs.index(data.slot)
result = self._unit.inverse_projection(data.tensor, output_index)
return [InversePassInputPacket(result, self.inputs.input)]
def _inverse_projection(
self,
data: InversePassOutputPacket) -> List[InversePassInputPacket]:
slot: GroupVirtualOutputSlot = data.slot
true_slot = slot.input
return [InversePassInputPacket(data.tensor, true_slot)]