def measure(self, protocol: Protocol) -> float: disentanglement_scores = [] non_constant_positions = 0 for j in range(self.max_message_length): symbols_j = [message[j] for message in protocol.values()] symbol_mutual_info = [] symbol_entropy = compute_entropy(symbols_j) for i in range(self.num_concept_slots): concepts_i = [ flatten_derivation(derivation)[i] for derivation in protocol.keys() ] mutual_info = compute_mutual_information(concepts_i, symbols_j) symbol_mutual_info.append(mutual_info) symbol_mutual_info.sort(reverse=True) if symbol_entropy > 0: disentanglement_score = ( symbol_mutual_info[0] - symbol_mutual_info[1]) / symbol_entropy disentanglement_scores.append(disentanglement_score) non_constant_positions += 1 if non_constant_positions > 0: return sum(disentanglement_scores) / non_constant_positions else: return np.nan
def measure(self, protocol: Protocol) -> float: distance_messages = self._compute_distances( sequence=list(protocol.values()), metric=self.messages_metric) distance_inputs = self._compute_distances( sequence=[flatten_derivation(derivation) for derivation in protocol.keys()], metric=self.input_metric) return spearmanr(distance_messages, distance_inputs).correlation
def _protocol_to_tensor( self, protocol: Protocol ) -> Dict[Tuple[torch.LongTensor, torch.LongTensor], torch.LongTensor]: vocab = get_vocab_from_protocol(protocol) concept_set = set(concept for derivation in protocol.keys() for concept in flatten_derivation(derivation)) concepts = {concept: idx for idx, concept in enumerate(concept_set)} tensorized_protocol = {} for derivation, message in protocol.items(): derivation = derivation_to_tensor(derivation, concepts) message = torch.LongTensor([vocab[char] for char in message]) tensorized_protocol[derivation] = torch.nn.functional.one_hot( message, num_classes=len(vocab)).reshape(-1) return tensorized_protocol
def _compute_concept_symbol_matrix(self, protocol: Protocol, vocab: Dict[str, int], concepts: Dict[str, int], epsilon: float = 10e-8) -> np.ndarray: concept_to_message = defaultdict(list) for derivation, message in protocol.items(): for concept in flatten_derivation(derivation): concept_to_message[concept] += list(message) concept_symbol_matrix = np.ndarray((self.num_concepts, len(vocab))) concept_symbol_matrix.fill(epsilon) for concept, symbols in concept_to_message.items(): for symbol in symbols: concept_symbol_matrix[concepts[concept], vocab[symbol]] += 1 return concept_symbol_matrix
def measure(self, protocol: Protocol) -> float: character_set = set(c for message in protocol.values() for c in message) vocab = {char: idx for idx, char in enumerate(character_set)} concept_set = set(concept for concepts in protocol.keys() for concept in flatten_derivation(concepts)) concepts = {concept: idx for idx, concept in enumerate(concept_set)} concept_symbol_matrix = self._compute_concept_symbol_matrix( protocol, vocab, concepts) v_cs = concept_symbol_matrix.argmax(axis=1) context_independence_scores = np.zeros(len(concept_set)) for concept in range(len(concept_set)): v_c = v_cs[concept] p_vc_c = concept_symbol_matrix[ concept, v_c] / concept_symbol_matrix[concept, :].sum(axis=0) p_c_vc = concept_symbol_matrix[ concept, v_c] / concept_symbol_matrix[:, v_c].sum(axis=0) context_independence_scores[concept] = p_vc_c * p_c_vc return context_independence_scores.mean(axis=0)
def measure(self, protocol: Protocol) -> float: vocab = get_vocab_from_protocol(protocol) sender = FixedProtocolSender(protocol, vocab) receiver = RnnReceiverDeterministic( agent=Receiver(NN_CONFIG['receiver_hidden'], NN_CONFIG['num_features']), vocab_size=len(vocab) + 1, embed_dim=NN_CONFIG['receiver_emb'], hidden_size=NN_CONFIG['receiver_hidden'], cell=NN_CONFIG['receiver_cell'], num_layers=NN_CONFIG['cell_layers'] ) game = SenderReceiverRnnReinforce(sender, receiver, loss_nll, sender_entropy_coeff=0, receiver_entropy_coeff=0.05) if self.context_sensitive: concept_set = set(concept for derivation in protocol.keys() for concept in flatten_derivation(derivation)[1:]) else: concept_set = set(concept for derivation in protocol.keys() for concept in flatten_derivation(derivation)) concepts = {concept: idx for idx, concept in enumerate(concept_set)} if self.context_sensitive: derivations = [(derivation, derivation_to_tensor(derivation[1], concepts)) for derivation in protocol.keys()] else: derivations = [(derivation, derivation_to_tensor(derivation, concepts)) for derivation in protocol.keys()] shuffle(derivations) split_idx = int(len(derivations)*0.8) train_derivations, test_derivations = derivations[:split_idx], derivations[split_idx:] test_accuracy = self._train_and_test(game, train_derivations, test_derivations) return test_accuracy