def benchmark_data(cfg: AttrDict, split: str = "train"):
split = split.upper()
total_images = MAX_ITERS * cfg["DATA"][split]["BATCHSIZE_PER_REPLICA"]
timer = Timer()
dataset = build_dataset(cfg, split)
try:
device = torch.device("cuda" if cfg.MACHINE.DEVICE == "gpu" else "cpu")
except AttributeError:
device = torch.device("cuda")
# Gives sampler same seed for entire distributed group as per pytorch documentation.
sampler_seed = cfg.SEED_VALUE
dataloader = get_loader(
dataset=dataset,
dataset_config=cfg["DATA"][split],
num_dataloader_workers=cfg.DATA.NUM_DATALOADER_WORKERS,
pin_memory=False,
multi_processing_method=cfg.MULTI_PROCESSING_METHOD,
device=device,
sampler_seed=sampler_seed,
)
# Fairstore data sampler would require setting the start iter before it can start.
if hasattr(dataloader.sampler, "set_start_iter"):
dataloader.sampler.set_start_iter(0)
# initial warmup measured as warmup time
timer.reset()
data_iterator = iter(dataloader)
for i in range(10): # warmup
next(data_iterator)
if i == 0:
# the total number of seconds since the start/reset of the timer
warmup_time = timer.seconds()
logging.info(f"Warmup time {WARMUP_ITERS} batches: {warmup_time} seconds")
# measure the number of images per sec in 1000 iterations.
timer = Timer()
for _ in tqdm.trange(MAX_ITERS):
next(data_iterator)
time_elapsed = timer.seconds()
logging.info(
f"iters: {MAX_ITERS}; images: {total_images}; time: {time_elapsed} seconds; "
f"images/sec: {round(float(total_images / time_elapsed), 4)}; "
f"ms/img: {round(float(1000 * time_elapsed / total_images), 4)} ")
# run benchmark for a few more rounds to catch fluctuations
for round_idx in range(BENCHMARK_ROUNDS):
timer = Timer()
for _ in tqdm.trange(MAX_ITERS):
next(data_iterator)
time_elapsed = timer.seconds()
logging.info(
f"round: {round_idx}: iters: {MAX_ITERS}; images: {total_images}; "
f"time: {time_elapsed} seconds; "
f"images/sec: {round(float(total_images / time_elapsed), 4)}; "
f"ms/img: {round(float(1000 * time_elapsed / total_images), 4)} ")
del data_iterator
del dataloader
def build_dataloaders(self,
pin_memory: bool) -> torch.utils.data.DataLoader:
"""
Build PyTorch dataloaders for all the available_splits. We construct the
standard PyTorch Dataloader and allow setting all dataloader options.
"""
self.datasets, self.data_and_label_keys = self.build_datasets()
# Gives sampler same seed for entire distributed group as per pytorch documentation.
sampler_seed = self.config["SEED_VALUE"]
loaders = {
split.lower(): get_loader(
dataset=self.datasets[split],
dataset_config=self.config["DATA"][split],
num_dataloader_workers=self.config.DATA.NUM_DATALOADER_WORKERS,
pin_memory=pin_memory,
multi_processing_method=self.config.MULTI_PROCESSING_METHOD,
device=self.device,
sampler_seed=sampler_seed,
)
for split in self.available_splits
}
return loaders
def build_dataloaders(self, pin_memory: bool) -> torch.utils.data.DataLoader:
"""
Build PyTorch dataloaders for all the available_splits. We construct the
standard PyTorch Dataloader and allow setting all dataloader options.
"""
self.datasets, self.data_and_label_keys = self.build_datasets()
loaders = {
split.lower(): get_loader(
dataset=self.datasets[split],
dataset_config=self.config["DATA"][split],
num_dataloader_workers=self.config.DATA.NUM_DATALOADER_WORKERS,
pin_memory=pin_memory,
multi_processing_method=self.config.MULTI_PROCESSING_METHOD,
device=self.device,
)
for split in self.available_splits
}
return loaders