def test_config_to_json_for_nested_objectsboo2(self): """For a config that contains a dict inside it, verify that config can be correctly created from a json/dict, and config can be correctly serialized and de-serialized """ class Boo3(ConfigBase): ayeaye: str class Boo2(ConfigBase): aye: Boo3 class TestConfigContainer(ConfigBase): class TestConfig(ConfigBase): a_list: List[Boo2] test_config: TestConfig a_str = "abc" # a_boo = {"boo1": {"aye": a_str}} a_boo = {"ayeaye": a_str} # b_boo = {"aye": {"boo3": a_boo}} b_boo = {"aye": a_boo} # a_boo_list = {"boolist1": {"boolistitems": [a_boo]}} a_config_containing_union = {"test_config": {"a_list": [b_boo]}} pytext_config = serialize.config_from_json(TestConfigContainer, a_config_containing_union) # verify that a_dict was read correctly self.assertEqual(pytext_config.test_config.a_list[0].aye.ayeaye, a_str) # serialize config to json, and deserialize back to config # verify that nothing changed jsonified_config = serialize.config_to_json(TestConfigContainer, pytext_config) pytext_config_deserialized = serialize.config_from_json( TestConfigContainer, jsonified_config) self.assertEqual(pytext_config, pytext_config_deserialized)
def test_config_to_json_for_union_with_baa(self): """For a config that contains a dict inside it, verify that config can be correctly created from a json/dict, and config can be correctly serialized and de-serialized """ class Boo(ConfigBase): aye: str class Baa(ConfigBase): oye: int class TestConfigContainer(ConfigBase): class TestConfig(ConfigBase): a_union: Union[Boo, Baa] test_config: TestConfig an_int = 1 a_baa = {"baa": {"oye": an_int}} a_config_containing_union = {"test_config": {"a_union": a_baa}} pytext_config = serialize.config_from_json(TestConfigContainer, a_config_containing_union) # verify that a_dict was read correctly self.assertEqual(pytext_config.test_config.a_union.oye, an_int) # serialize config to json, and deserialize back to config # verify that nothing changed jsonified_config = serialize.config_to_json(TestConfigContainer, pytext_config) pytext_config_deserialized = serialize.config_from_json( TestConfigContainer, jsonified_config) self.assertEqual(pytext_config, pytext_config_deserialized)
def test_config_to_json_for_union(self): """For a config that contains a dict inside it, verify that config can be correctly created from a json/dict, and config can be correctly serialized and de-serialized """ class Boo(ConfigBase): aye: str class TestConfigContainer(ConfigBase): class TestConfig(ConfigBase): a_union: Union[Boo, List[Boo]] test_config: TestConfig a_str = "abc" a_config_containing_union = { "test_config": { "a_union": { "boo": { "aye": a_str } } } } pytext_config = serialize.config_from_json(TestConfigContainer, a_config_containing_union) # verify that a_dict was read correctly self.assertEqual(pytext_config.test_config.a_union.aye, a_str) # serialize config to json, and deserialize back to config # verify that nothing changed jsonified_config = serialize.config_to_json(TestConfigContainer, pytext_config) pytext_config_deserialized = serialize.config_from_json( TestConfigContainer, jsonified_config) self.assertEqual(pytext_config, pytext_config_deserialized)
def test_config_to_json_for_dict(self): """For a config that contains a dict inside it, verify that config can be correctly created from a json/dict, and config can be correctly serialized and de-serialized """ class TestConfigContainer(ConfigBase): class TestConfig(ConfigBase): a_dict: Dict[str, Any] test_config: TestConfig a_dict = { "param2": { "nested_param1": 10, "nested_param2": 2 }, "param1": "val1" } a_config_containing_dict = {"test_config": {"a_dict": a_dict}} pytext_config = serialize.config_from_json(TestConfigContainer, a_config_containing_dict) # verify that a_dict was read correctly self.assertEqual(pytext_config.test_config.a_dict, a_dict) # serialize config to json, and deserialize back to config # verify that nothing changed jsonified_config = serialize.config_to_json(TestConfigContainer, pytext_config) pytext_config_deserialized = serialize.config_from_json( TestConfigContainer, jsonified_config) self.assertEqual(pytext_config, pytext_config_deserialized)
def torchscript_export(context, export_json, model, output_path, quantize, target): """Convert a pytext model snapshot to a torchscript model.""" export_cfg = ExportConfig() # only populate from export_json if no export option is configured from the command line. if export_json: export_json_config = _load_and_validate_export_json_config(export_json) read_chunk_size = export_json_config.pop("read_chunk_size", None) if read_chunk_size is not None: print("Warning: Ignoring read_chunk_size.") if export_json_config.get("read_chunk_size", None) is not None: print( "Error: Do not know what to do with read_chunk_size. Ignoring." ) if "export" in export_json_config.keys(): export_cfgs = [export_json_config["export"]] else: export_cfgs = export_json_config["export_list"] if target: print( "A single export was specified in the command line. Filtering out all other export options" ) export_cfgs = [ cfg for cfg in export_cfgs if cfg["target"] == target ] if export_cfgs == []: print( "No ExportConfig matches the target name specified in the command line." ) for partial_export_cfg in export_cfgs: if not quantize and not output_path: export_cfg = config_from_json(ExportConfig, partial_export_cfg) else: print( "the export-json config is ignored because export options are found the command line" ) export_cfg = config_from_json( ExportConfig, partial_export_cfg, ("export_caffe2_path", "export_onnx_path"), ) export_cfg.torchscript_quantize = quantize # if config has export_torchscript_path, use export_torchscript_path from config, otherwise keep the default from CLI if export_cfg.export_torchscript_path is not None: output_path = export_cfg.export_torchscript_path if not model or not output_path: config = context.obj.load_config() model = model or config.save_snapshot_path output_path = output_path or f"{config.save_snapshot_path}.torchscript" print(f"Exporting {model} to torchscript file: {output_path}") print(export_cfg) export_saved_model_to_torchscript(model, output_path, export_cfg)
def test_multiple_values_with_ellipsis(self): config_json = json.loads(""" {"foo":[[1,2], [3,4], [5,6]]} """) config = config_from_json(TupleTestConfig, config_json) self.assertEqual(config.foo[0][0], 1) self.assertEqual(config.foo[2][1], 6)
def test_nested_tuple(self): config_json = json.loads(""" {} """) config = config_from_json(TupleTestConfig, config_json) self.assertEqual(config.foo[0][0], 32) self.assertEqual(config.foo[1][1], 2)
def test_component_subconfig_serialize(self): config_json = json.loads("""{ "foo": 5, "models": [{ "bar": 12, "m2s1s1": "thing" }, { "m2s1s1": "thing2" }] }""") config = config_from_json(JointModel.Config, config_json) serialized = config_to_json(JointModel.Config, config) again = config_from_json(JointModel.Config, serialized) self.assertEqual(again.foo, 5) self.assertEqual(again.models[0].m2s1s1, "thing") self.assertEqual(again.models[1].bar, 3)
def test_implicit_type_cast(self): config_json = json.loads( """ {"foo": "123"} """ ) config = config_from_json(Model1, config_json) self.assertEqual(config.foo, 123)
def test_use_default_value(self): config_json = json.loads( """ {} """ ) config = config_from_json(Model1, config_json) self.assertEqual(config.foo, 1)
def test_component(self): config_json = json.loads("""{ "bar": 13, "m2s1s1": "sub_foo" }""") config = config_from_json(RegisteredModel.Config, config_json) self.assertEqual(config.bar, 13) self.assertEqual(config.m2s1s1, "sub_foo") self.assertEqual(config.m2s1, 5)
def run_single( rank: int, config_json: str, world_size: int, dist_init_method: str, summary_writer: SummaryWriter, ): config = config_from_json(PyTextConfig, config_json) if rank != 0: summary_writer = None train_model(config, dist_init_method, rank, rank, world_size, summary_writer)
def test_component_subconfig_deserialize(self): config_json = json.loads("""{ "foo": 5, "models": [{ "bar": 12, "m2s1s1": "thing" }, { "m2s1s1": "thing2" }] }""") config = config_from_json(JointModel.Config, config_json) self.assertEqual(config.foo, 5) self.assertEqual(len(config.models), 2) self.assertEqual(config.models[1].m2s1s1, "thing2")
def test_auto_include_sub_component_in_config(self): config_json = json.loads(""" { "model": {"ModelFoo": {}}, "model_bar_x": {"ModelBar1": {}}, "model_union": {"ModelBar": {}}, "datahandler": {"SubDataHandler": {}} } """) config = config_from_json(TestConfig, config_json) self.assertEqual(config.model.foo, 2) self.assertEqual(config.model_bar_x.bar, "bar1") self.assertEqual(config.model_union.bar, "bar") self.assertEqual(config.datahandler.foo, 3)
def run_single( rank: int, config_json: str, world_size: int, dist_init_method: str, metadata: CommonMetadata, ): config = config_from_json(PyTextConfig, config_json) summary_writer = SummaryWriter( ) if rank != 0 and config.use_tensorboard else None try: train_model(config, dist_init_method, rank, rank, world_size, summary_writer, metadata) finally: if summary_writer is not None: summary_writer.close()
def test_default_union(self): config_json = json.loads(""" { "task": { "model": { "Model2": { "bar": "test" } } }, "output": "foo/bar.pt" } """) config = config_from_json(PyTextConfig, config_json) self.assertTrue(isinstance(config.task, Task1)) self.assertTrue(isinstance(config.task.model, Model2)) self.assertEqual(config.task.model.bar, "test")
def test_serialize_union_with_expansible_component(self): config = TestConfig( model=ModelFoo.Config(), model_bar_x=ModelBar.Config(), model_union=ModelBar1.Config(), datahandler=SubDataHandler.Config(), ) json = config_to_json(TestConfig, config) print(json) self.assertEqual(json["model"]["ModelFoo"]["foo"], 2) self.assertEqual(json["model_bar_x"]["ModelBar"]["bar"], "bar") self.assertEqual(json["model_union"]["ModelBar1"]["bar"], "bar1") self.assertEqual(json["datahandler"]["SubDataHandler"]["foo"], 3) config = config_from_json(TestConfig, json) self.assertEqual(config.model.foo, 2) self.assertEqual(config.model_bar_x.bar, "bar") self.assertEqual(config.model_union.bar, "bar1") self.assertEqual(config.datahandler.foo, 3)
def run_single( rank: int, config_json: str, world_size: int, dist_init_method: Optional[str], metadata: Optional[CommonMetadata], metric_channels: Optional[List[Channel]], ): config = config_from_json(PyTextConfig, config_json) if rank != 0: metric_channels = [] train_model( config=config, dist_init_url=dist_init_method, device_id=rank, rank=rank, world_size=world_size, metric_channels=metric_channels, metadata=metadata, )
def test_default_value_expansible_config(self): config_json = json.loads(""" { "task": { "Task3": { "model": { "foobar": 1.0 } } }, "output": "foo/bar.pt" } """) config = config_from_json(PyTextConfig, config_json) self.assertTrue(isinstance(config.task, Task3)) self.assertTrue(isinstance(config.task.model, ExpansibleModel.Config)) self.assertFalse(isinstance(config.task.model, AExpansibleModel.Config)) self.assertFalse(isinstance(config.task.model, BExpansibleModel.Config)) self.assertEqual(config.task.model.foobar, 1.0)
def run_single(rank, config_json: str, world_size: int, dist_init_method: str): config = config_from_json(PyTextConfig, config_json) train_model(config, dist_init_method, rank, rank, world_size)