def batch_tensors( self, tensor_list: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]: batched_text = nn_util.batch_tensor_dicts( tensor["text"] for tensor in tensor_list) # type: ignore batched_linking = torch.stack( [tensor["linking"] for tensor in tensor_list]) return {"text": batched_text, "linking": batched_linking}
def batch_tensors( self, tensor_list: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]: # pylint: disable=no-self-use batched_text = nn_util.batch_tensor_dicts( tensor['text'] for tensor in tensor_list) # type: ignore batched_linking = torch.stack( [tensor['linking'] for tensor in tensor_list]) return {'text': batched_text, 'linking': batched_linking}
def batch_tensors( self, tensor_list: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]: result: Dict[str, torch.Tensor] = util.batch_tensor_dicts(tensor_list) # Transformer models need LongTensors for indices, just in case we have more than 2 billion # different tokens. To save space, we make the switch as late as possible, i.e., here. result = { name: t.to(torch.int64) if t.dtype == torch.int32 else t for name, t in result.items() } return result
def batch_tensors(self, tensor_list: List[TextFieldTensors]) -> TextFieldTensors: # This is creating a dict of {token_indexer_name: {token_indexer_outputs: batched_tensor}} # for each token indexer used to index this field. indexer_lists: Dict[str, List[Dict[str, torch.Tensor]]] = defaultdict(list) for tensor_dict in tensor_list: for indexer_name, indexer_output in tensor_dict.items(): indexer_lists[indexer_name].append(indexer_output) batched_tensors = { # NOTE(mattg): if an indexer has its own nested structure, rather than one tensor per # argument, then this will break. If that ever happens, we should move this to an # `indexer.batch_tensors` method, with this logic as the default implementation in the # base class. indexer_name: util.batch_tensor_dicts(indexer_outputs) for indexer_name, indexer_outputs in indexer_lists.items() } return batched_tensors
def batch_tensors(self, tensor_list: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]: # pylint: disable=no-self-use # This is creating a dict of {token_indexer_key: batch_tensor} for each token indexer used # to index this field. return util.batch_tensor_dicts(tensor_list)
def batch_tensors(self, tensor_list: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]: # pylint: disable=no-self-use # This is creating a dict of {token_indexer_key: batch_tensor} for each token indexer used # to index this field. return util.batch_tensor_dicts(tensor_list)
def batch_tensors(self, tensor_list: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]: # pylint: disable=no-self-use batched_text = nn_util.batch_tensor_dicts(tensor['text'] for tensor in tensor_list) # type: ignore batched_linking = torch.stack([tensor['linking'] for tensor in tensor_list]) return {'text': batched_text, 'linking': batched_linking}