def test_init_processors(self):
path = os.path.join(
os.path.abspath(__file__),
"../../../pythia/common/defaults/configs/datasets/vqa/vqa2.yml")
configuration = Configuration(os.path.abspath(path))
self._fix_configuration(configuration)
configuration.freeze()
base_dataset = BaseDataset(
"vqa2",
"train",
configuration.get_config()["dataset_attributes"]["vqa2"],
)
expected_processors = [
"answer_processor",
"ocr_token_processor",
"bbox_processor",
]
# Check no processors are initialized before init_processors call
self.assertFalse(
any(hasattr(base_dataset, key) for key in expected_processors))
for processor in expected_processors:
self.assertIsNone(registry.get("{}_{}".format("vqa2", processor)))
# Check processors are initialized after init_processors
base_dataset.init_processors()
self.assertTrue(
all(hasattr(base_dataset, key) for key in expected_processors))
for processor in expected_processors:
self.assertIsNotNone(
registry.get("{}_{}".format("vqa2", processor)))
def build_trainer(args, *rest, **kwargs):
configuration = Configuration(args.config)
# Update with the config override if passed
configuration.override_with_cmd_config(args.config_override)
# Now, update with opts args that were passed
configuration.override_with_cmd_opts(args.opts)
# Finally, update with args that were specifically passed
# as arguments
configuration.update_with_args(args)
configuration.freeze()
config = configuration.get_config()
registry.register("config", config)
registry.register("configuration", configuration)
trainer_type = config.training_parameters.trainer
trainer_cls = registry.get_trainer_class(trainer_type)
trainer_obj = trainer_cls(config)
# Set args as an attribute for future use
setattr(trainer_obj, 'args', args)
return trainer_obj
def setUp(self):
torch.manual_seed(1234)
registry.register("clevr_text_vocab_size", 80)
registry.register("clevr_num_final_outputs", 32)
config_path = os.path.join(get_pythia_root(), "..", "configs", "vqa",
"clevr", "cnn_lstm.yml")
config_path = os.path.abspath(config_path)
configuration = Configuration(config_path)
configuration.config["datasets"] = "clevr"
configuration.freeze()
self.config = configuration.config
registry.register("config", self.config)
def setUp(self):
torch.manual_seed(1234)
config_path = os.path.join(get_pythia_root(), "..", "configs",
"captioning", "coco",
"butd_nucleus_sampling.yml")
config_path = os.path.abspath(config_path)
configuration = Configuration(config_path)
configuration.config["datasets"] = "coco"
configuration.config["model_attributes"]["butd"]["inference"][
"params"]["sum_threshold"] = 0.5
configuration.freeze()
self.config = configuration.config
registry.register("config", self.config)
def load_config(self):
# TODO: Review configuration update once again
# (remember clip_gradients case)
self.configuration = Configuration(self.args.config)
# Update with the config override if passed
self.configuration.override_with_cmd_config(self.args.config_override)
# Now, update with opts args that were passed
self.configuration.override_with_cmd_opts(self.args.opts)
# Finally, update with args that were specifically passed
# as arguments
self.configuration.update_with_args(self.args)
self.configuration.freeze()
self.config = self.configuration.get_config()
registry.register("config", self.config)
module_name = file_name[:file_name.find(".py")]
importlib.import_module("pythia.trainers." + module_name)
elif f.endswith("builder.py"):
splits = f.split(os.sep)
task_name = splits[-3]
dataset_name = splits[-2]
if task_name == "tasks" or dataset_name == "tasks":
continue
file_name = splits[-1]
module_name = file_name[:file_name.find(".py")]
importlib.import_module("pythia.tasks." + task_name + "." +
dataset_name + "." + module_name)
setup_imports()
configuration = Configuration(
'pythia/common/defaults/configs/tasks/captioning/imagenet.yml')
configuration.freeze()
config = configuration.get_config()
registry.register("config", config)
registry.register("configuration", configuration)
dataset_config = config.task_attributes.captioning.dataset_attributes.coco
builder = COCOBuilder()
dataset_train = builder._load('train', dataset_config)
dataset_train.init_processors()
dataset_val = builder._load('val', dataset_config)
dataset_val.init_processors()
print(dataset_train.load_item(6))
print('************************')
print(dataset_val.load_item(6))