def _prepare(self):
super()._prepare()
# select negative sampling implementation
self._implementation = self.config.check(
"negative_sampling.implementation",
["triple", "all", "batch", "auto"],
)
if self._implementation == "auto":
max_nr_of_negs = max(self._sampler.num_samples)
if self._sampler.shared:
self._implementation = "batch"
elif max_nr_of_negs <= 30:
self._implementation = "triple"
else:
self._implementation = "batch"
self.config.set("negative_sampling.implementation",
self._implementation,
log=True)
self.config.log("Preparing negative sampling training job with "
"'{}' scoring function ...".format(
self._implementation))
# construct dataloader
self.num_examples = self.dataset.split(self.train_split).size(0)
self.loader = torch.utils.data.DataLoader(
range(self.num_examples),
collate_fn=self._get_collate_fun(),
shuffle=True,
batch_size=self.batch_size,
num_workers=self.config.get("train.num_workers"),
worker_init_fn=_generate_worker_init_fn(self.config),
pin_memory=self.config.get("train.pin_memory"),
)
def _prepare(self):
"""Construct dataloader"""
super()._prepare()
self.num_examples = self.dataset.split(self.train_split).size(0)
self.loader = torch.utils.data.DataLoader(
range(self.num_examples),
collate_fn=self._get_collate_fun(),
shuffle=True,
batch_size=self.batch_size,
num_workers=self.config.get("train.num_workers"),
worker_init_fn=_generate_worker_init_fn(self.config),
pin_memory=self.config.get("train.pin_memory"),
)
def _init_dataloader(self):
mp_context = (
torch.multiprocessing.get_context("fork")
if self.config.get("train.num_workers") > 0
else None
)
self.loader = torch.utils.data.DataLoader(
self.dataloader_dataset,
sampler=InfiniteSequentialSampler(self.dataloader_dataset),
collate_fn=self._get_collate_fun(),
shuffle=False,
# shuffle needs to be False, since it is handled in the dataset object
# batch_size=self.batch_size, # batch size needs to be 1 since it is handled in the dataset object
num_workers=self.config.get("train.num_workers"),
worker_init_fn=_generate_worker_init_fn(self.config),
pin_memory=self.config.get("train.pin_memory"),
multiprocessing_context=mp_context,
)
def _prepare(self):
super()._prepare()
# determine enabled query types
self.query_types = [
key
for key, enabled in self.config.get("KvsAll.query_types").items()
if enabled
]
# corresponding indexes
self.query_indexes: List[KvsAllIndex] = []
#' for each query type (ordered as in self.query_types), index right after last
#' example of that type in the list of all examples (over all query types)
self.query_last_example = []
# construct relevant data structures
self.num_examples = 0
for query_type in self.query_types:
index_type = ("sp_to_o" if query_type == "sp_" else
("so_to_p" if query_type == "s_o" else "po_to_s"))
index = self.dataset.index(f"{self.train_split}_{index_type}")
self.query_indexes.append(index)
self.num_examples += len(index)
self.query_last_example.append(self.num_examples)
# create dataloader
self.loader = torch.utils.data.DataLoader(
range(self.num_examples),
collate_fn=self._get_collate_fun(),
shuffle=True,
batch_size=self.batch_size,
num_workers=self.config.get("train.num_workers"),
worker_init_fn=_generate_worker_init_fn(self.config),
pin_memory=self.config.get("train.pin_memory"),
)
