def _batch_unshuffle_ddp(self, x, idx_unshuffle):
batch_size_this = x.shape[0]
x_gather = concat_all_gather(x)
batch_size_all = x_gather.shape[0]
num_gpus = batch_size_all // batch_size_this
gpu_idx = dist.get_rank()
idx_this = idx_unshuffle.view(num_gpus, -1)[gpu_idx]
return x_gather[idx_this]
def _deque_and_enqueue(self, keys):
keys = concat_all_gather(keys)
batch_size = keys.shape[0]
pointer = int(self.queue_pointer) # self.queue_pointer.item()
assert self.K % batch_size == 0
self.neg_queue[:, pointer: pointer + batch_size] = keys.t()
pointer = (pointer + batch_size) % self.K
self.queue_pointer[0] = pointer
def _batch_shuffle_ddp(self, x):
batch_size_this = x.shape[0]
x_gather = concat_all_gather(x)
batch_size_all = x_gather.shape[0]
num_gpus = batch_size_all // batch_size_this
idx_shuffle = torch.randperm(batch_size_all).cuda()
dist.broadcast(idx_shuffle, src=0)
idx_unshuffle = torch.argsort(idx_shuffle)
gpu_idx = dist.get_rank()
idx_this = idx_shuffle.view(num_gpus, -1)[gpu_idx]
return x_gather[idx_this], idx_unshuffle
def _dequeue_and_enqueue(self, keys):
# gather keys before updating queue
if self.hparams.gather_keys_for_queue:
keys = utils.concat_all_gather(keys)
batch_size = keys.shape[0]
ptr = int(self.queue_ptr)
assert self.hparams.K % batch_size == 0 # for simplicity
# replace the keys at ptr (dequeue and enqueue)
self.queue[:, ptr:ptr + batch_size] = keys.T
ptr = (ptr + batch_size) % self.hparams.K # move pointer
self.queue_ptr[0] = ptr