def make_data_reader(self, config_dict): config = NestedNamespace() config.load_from_json(config_dict) config.tokenizers = self.tokenizers data_reader_factory = DataReaderFactory(config) return data_reader_factory.create()
def load_from_config(cls, config_path): with open(config_path, "r", encoding="utf-8") as in_file: machine_config = NestedNamespace() machine_config.load_from_json(json.load(in_file)) machine_name = machine_config.name config = getattr(machine_config, machine_name, {}) return cls(config)
def load_and_setting(config_path): config = NestedNamespace() with open(config_path, "r") as f: defined_config = json.load(f) config.load_from_json(defined_config) config.data_reader.wikisql = NestedNamespace() config.data_reader.wikisql.is_test = True config = optimize_config(config, is_test=True) set_gpu_env(config) return config
def load_and_setting(config_path): config = NestedNamespace() with open(config_path, "r") as f: defined_config = json.load(f) config.load_from_json(defined_config) config = optimize_config(config, is_test=True) set_gpu_env(config) config.data_reader.train_file_path = SYNTHETIC_DATA_PATH config.data_reader.valid_file_path = SYNTHETIC_DATA_PATH return config
def test_train_base_config_argparse(): train_config = args.config(argv=["--base_config", "test/bidaf"], mode=Mode.TRAIN) config = NestedNamespace() with open("base_config/test/bidaf.json", "r") as f: defined_config = json.load(f) config.load_from_json(defined_config) args.set_gpu_env(config) assert train_config == config
def load_and_setting(config_path): config = NestedNamespace() config_path = add_config_extension(config_path) defined_config = read_config(config_path) config.load_from_json(defined_config) config = optimize_config(config, is_test=True) set_gpu_env(config) config.data_reader.train_file_path = SYNTHETIC_DATA_PATH config.data_reader.valid_file_path = SYNTHETIC_DATA_PATH return config
def open_qa_config(request): config_path = request.param machine_config = NestedNamespace() with open(config_path, "r") as f: defined_config = json.load(f) machine_config.load_from_json(defined_config) claf_name = machine_config.name config = getattr(machine_config, claf_name, {}) config.knowledge_base.wiki = WIKI_SYNTHETIC_DATA_PATH config.reasoning.reading_comprehension.checkpoint_path = "./logs/test/bidaf/checkpoint/model_1.pkl" return machine_config
def train_config(request): config_path = request.param config = NestedNamespace() with open(config_path, "r") as f: defined_config = json.load(f) config.load_from_json(defined_config) config.nsml = NestedNamespace() config.nsml.pause = 0 config = optimize_config(config, is_test=True) set_gpu_env(config) config.data_reader.train_file_path = SQUAD_SYNTHETIC_DATA_PATH config.data_reader.valid_file_path = SQUAD_SYNTHETIC_DATA_PATH return config
def load_and_setting(config_path): config = NestedNamespace() with open(config_path, "r") as f: defined_config = json.load(f) config.load_from_json(defined_config) config = optimize_config(config, is_test=True) set_gpu_env(config) config.data_reader.multitask_bert.readers[0][ "train_file_path"] = SYNTHETIC_SEQ_CLS_DATA_PATH config.data_reader.multitask_bert.readers[0][ "valid_file_path"] = SYNTHETIC_SEQ_CLS_DATA_PATH config.data_reader.multitask_bert.readers[1][ "train_file_path"] = SYNTHETIC_REG_DATA_PATH config.data_reader.multitask_bert.readers[1][ "valid_file_path"] = SYNTHETIC_REG_DATA_PATH config.data_reader.multitask_bert.readers[2][ "train_file_path"] = SYNTHETIC_QA_DATA_PATH config.data_reader.multitask_bert.readers[2][ "valid_file_path"] = SYNTHETIC_QA_DATA_PATH return config
help=""" NSML mode setting """, ) parser.add_argument( "--iteration", type=int, default=0, help=""" NSML default setting """, ) parser.add_argument( "--pause", type=int, default=0, help=""" NSML default setting """, ) args = parser.parse_args() with open(args.base_config, "r") as f: defined_config = json.load(f) config = NestedNamespace() config.load_from_json(defined_config) config.nsml = args set_logging_config() if args.mode == "train_and_evaluate": re_train_and_evaluate(config) elif args.mode == "test" or args.mode == "infer": test(config) else: raise ValueError(f"Unrecognized mode. {config.mode}")