def test_init_processors(self): path = os.path.join( os.path.abspath(__file__), "../../../pythia/common/defaults/configs/datasets/vqa/vqa2.yml") configuration = Configuration(os.path.abspath(path)) self._fix_configuration(configuration) configuration.freeze() base_dataset = BaseDataset( "vqa2", "train", configuration.get_config()["dataset_attributes"]["vqa2"], ) expected_processors = [ "answer_processor", "ocr_token_processor", "bbox_processor", ] # Check no processors are initialized before init_processors call self.assertFalse( any(hasattr(base_dataset, key) for key in expected_processors)) for processor in expected_processors: self.assertIsNone(registry.get("{}_{}".format("vqa2", processor))) # Check processors are initialized after init_processors base_dataset.init_processors() self.assertTrue( all(hasattr(base_dataset, key) for key in expected_processors)) for processor in expected_processors: self.assertIsNotNone( registry.get("{}_{}".format("vqa2", processor)))
def build_trainer(args, *rest, **kwargs): configuration = Configuration(args.config) # Update with the config override if passed configuration.override_with_cmd_config(args.config_override) # Now, update with opts args that were passed configuration.override_with_cmd_opts(args.opts) # Finally, update with args that were specifically passed # as arguments configuration.update_with_args(args) configuration.freeze() config = configuration.get_config() registry.register("config", config) registry.register("configuration", configuration) trainer_type = config.training_parameters.trainer trainer_cls = registry.get_trainer_class(trainer_type) trainer_obj = trainer_cls(config) # Set args as an attribute for future use setattr(trainer_obj, 'args', args) return trainer_obj
def setUp(self): torch.manual_seed(1234) registry.register("clevr_text_vocab_size", 80) registry.register("clevr_num_final_outputs", 32) config_path = os.path.join(get_pythia_root(), "..", "configs", "vqa", "clevr", "cnn_lstm.yml") config_path = os.path.abspath(config_path) configuration = Configuration(config_path) configuration.config["datasets"] = "clevr" configuration.freeze() self.config = configuration.config registry.register("config", self.config)
def setUp(self): torch.manual_seed(1234) config_path = os.path.join(get_pythia_root(), "..", "configs", "captioning", "coco", "butd_nucleus_sampling.yml") config_path = os.path.abspath(config_path) configuration = Configuration(config_path) configuration.config["datasets"] = "coco" configuration.config["model_attributes"]["butd"]["inference"][ "params"]["sum_threshold"] = 0.5 configuration.freeze() self.config = configuration.config registry.register("config", self.config)
def load_config(self): # TODO: Review configuration update once again # (remember clip_gradients case) self.configuration = Configuration(self.args.config) # Update with the config override if passed self.configuration.override_with_cmd_config(self.args.config_override) # Now, update with opts args that were passed self.configuration.override_with_cmd_opts(self.args.opts) # Finally, update with args that were specifically passed # as arguments self.configuration.update_with_args(self.args) self.configuration.freeze() self.config = self.configuration.get_config() registry.register("config", self.config)
module_name = file_name[:file_name.find(".py")] importlib.import_module("pythia.trainers." + module_name) elif f.endswith("builder.py"): splits = f.split(os.sep) task_name = splits[-3] dataset_name = splits[-2] if task_name == "tasks" or dataset_name == "tasks": continue file_name = splits[-1] module_name = file_name[:file_name.find(".py")] importlib.import_module("pythia.tasks." + task_name + "." + dataset_name + "." + module_name) setup_imports() configuration = Configuration( 'pythia/common/defaults/configs/tasks/captioning/imagenet.yml') configuration.freeze() config = configuration.get_config() registry.register("config", config) registry.register("configuration", configuration) dataset_config = config.task_attributes.captioning.dataset_attributes.coco builder = COCOBuilder() dataset_train = builder._load('train', dataset_config) dataset_train.init_processors() dataset_val = builder._load('val', dataset_config) dataset_val.init_processors() print(dataset_train.load_item(6)) print('************************') print(dataset_val.load_item(6))