def distributed_broadcast_scalars( scalars: List[Union[int, float]], num_total_examples: Optional[int] = None ) -> torch.Tensor: try: tensorized_scalar = torch.tensor(scalars).cuda() output_tensors = [tensorized_scalar.clone() for _ in range(dist.get_world_size())] dist.all_gather(output_tensors, tensorized_scalar) concat = torch.cat(output_tensors, dim=0) # truncate the dummy elements added by SequentialDistributedSampler if num_total_examples is not None: concat = concat[:num_total_examples] return concat except AssertionError: raise AssertionError("Not currently using distributed training")
def distributed_concat(tensor: Any, num_total_examples: Optional[int] = None) -> Any: try: if isinstance(tensor, (tuple, list)): return type(tensor)(distributed_concat(t, num_total_examples) for t in tensor) output_tensors = [tensor.clone() for _ in range(dist.get_world_size())] output_tensors = [t if len(t.shape) > 0 else t[None] for t in output_tensors] dist.all_gather(output_tensors, tensor) concat = torch.cat(output_tensors, dim=0) # truncate the dummy elements added by SequentialDistributedSampler if num_total_examples is not None: concat = concat[:num_total_examples] return concat except AssertionError: raise AssertionError("Not currently using distributed training")