def make_data_reader(self, config_dict):
config = NestedNamespace()
config.load_from_json(config_dict)
config.tokenizers = self.tokenizers
data_reader_factory = DataReaderFactory(config)
return data_reader_factory.create()
def load_from_config(cls, config_path):
with open(config_path, "r", encoding="utf-8") as in_file:
machine_config = NestedNamespace()
machine_config.load_from_json(json.load(in_file))
machine_name = machine_config.name
config = getattr(machine_config, machine_name, {})
return cls(config)
def load_and_setting(config_path):
config = NestedNamespace()
with open(config_path, "r") as f:
defined_config = json.load(f)
config.load_from_json(defined_config)
config.data_reader.wikisql = NestedNamespace()
config.data_reader.wikisql.is_test = True
config = optimize_config(config, is_test=True)
set_gpu_env(config)
return config
def load_and_setting(config_path):
config = NestedNamespace()
with open(config_path, "r") as f:
defined_config = json.load(f)
config.load_from_json(defined_config)
config = optimize_config(config, is_test=True)
set_gpu_env(config)
config.data_reader.train_file_path = SYNTHETIC_DATA_PATH
config.data_reader.valid_file_path = SYNTHETIC_DATA_PATH
return config
def test_train_base_config_argparse():
train_config = args.config(argv=["--base_config", "test/bidaf"],
mode=Mode.TRAIN)
config = NestedNamespace()
with open("base_config/test/bidaf.json", "r") as f:
defined_config = json.load(f)
config.load_from_json(defined_config)
args.set_gpu_env(config)
assert train_config == config
def load_and_setting(config_path):
config = NestedNamespace()
config_path = add_config_extension(config_path)
defined_config = read_config(config_path)
config.load_from_json(defined_config)
config = optimize_config(config, is_test=True)
set_gpu_env(config)
config.data_reader.train_file_path = SYNTHETIC_DATA_PATH
config.data_reader.valid_file_path = SYNTHETIC_DATA_PATH
return config
def open_qa_config(request):
config_path = request.param
machine_config = NestedNamespace()
with open(config_path, "r") as f:
defined_config = json.load(f)
machine_config.load_from_json(defined_config)
claf_name = machine_config.name
config = getattr(machine_config, claf_name, {})
config.knowledge_base.wiki = WIKI_SYNTHETIC_DATA_PATH
config.reasoning.reading_comprehension.checkpoint_path = "./logs/test/bidaf/checkpoint/model_1.pkl"
return machine_config
def train_config(request):
config_path = request.param
config = NestedNamespace()
with open(config_path, "r") as f:
defined_config = json.load(f)
config.load_from_json(defined_config)
config.nsml = NestedNamespace()
config.nsml.pause = 0
config = optimize_config(config, is_test=True)
set_gpu_env(config)
config.data_reader.train_file_path = SQUAD_SYNTHETIC_DATA_PATH
config.data_reader.valid_file_path = SQUAD_SYNTHETIC_DATA_PATH
return config
def load_and_setting(config_path):
config = NestedNamespace()
with open(config_path, "r") as f:
defined_config = json.load(f)
config.load_from_json(defined_config)
config = optimize_config(config, is_test=True)
set_gpu_env(config)
config.data_reader.multitask_bert.readers[0][
"train_file_path"] = SYNTHETIC_SEQ_CLS_DATA_PATH
config.data_reader.multitask_bert.readers[0][
"valid_file_path"] = SYNTHETIC_SEQ_CLS_DATA_PATH
config.data_reader.multitask_bert.readers[1][
"train_file_path"] = SYNTHETIC_REG_DATA_PATH
config.data_reader.multitask_bert.readers[1][
"valid_file_path"] = SYNTHETIC_REG_DATA_PATH
config.data_reader.multitask_bert.readers[2][
"train_file_path"] = SYNTHETIC_QA_DATA_PATH
config.data_reader.multitask_bert.readers[2][
"valid_file_path"] = SYNTHETIC_QA_DATA_PATH
return config
help=""" NSML mode setting """,
)
parser.add_argument(
"--iteration",
type=int,
default=0,
help=""" NSML default setting """,
)
parser.add_argument(
"--pause",
type=int,
default=0,
help=""" NSML default setting """,
)
args = parser.parse_args()
with open(args.base_config, "r") as f:
defined_config = json.load(f)
config = NestedNamespace()
config.load_from_json(defined_config)
config.nsml = args
set_logging_config()
if args.mode == "train_and_evaluate":
re_train_and_evaluate(config)
elif args.mode == "test" or args.mode == "infer":
test(config)
else:
raise ValueError(f"Unrecognized mode. {config.mode}")