def test_config_to_json_for_nested_objectsboo2(self):
"""For a config that contains a dict inside it, verify that config
can be correctly created from a json/dict, and config can be correctly
serialized and de-serialized
"""
class Boo3(ConfigBase):
ayeaye: str
class Boo2(ConfigBase):
aye: Boo3
class TestConfigContainer(ConfigBase):
class TestConfig(ConfigBase):
a_list: List[Boo2]
test_config: TestConfig
a_str = "abc"
# a_boo = {"boo1": {"aye": a_str}}
a_boo = {"ayeaye": a_str}
# b_boo = {"aye": {"boo3": a_boo}}
b_boo = {"aye": a_boo}
# a_boo_list = {"boolist1": {"boolistitems": [a_boo]}}
a_config_containing_union = {"test_config": {"a_list": [b_boo]}}
pytext_config = serialize.config_from_json(TestConfigContainer,
a_config_containing_union)
# verify that a_dict was read correctly
self.assertEqual(pytext_config.test_config.a_list[0].aye.ayeaye, a_str)
# serialize config to json, and deserialize back to config
# verify that nothing changed
jsonified_config = serialize.config_to_json(TestConfigContainer,
pytext_config)
pytext_config_deserialized = serialize.config_from_json(
TestConfigContainer, jsonified_config)
self.assertEqual(pytext_config, pytext_config_deserialized)
def test_config_to_json_for_union_with_baa(self):
"""For a config that contains a dict inside it, verify that config
can be correctly created from a json/dict, and config can be correctly
serialized and de-serialized
"""
class Boo(ConfigBase):
aye: str
class Baa(ConfigBase):
oye: int
class TestConfigContainer(ConfigBase):
class TestConfig(ConfigBase):
a_union: Union[Boo, Baa]
test_config: TestConfig
an_int = 1
a_baa = {"baa": {"oye": an_int}}
a_config_containing_union = {"test_config": {"a_union": a_baa}}
pytext_config = serialize.config_from_json(TestConfigContainer,
a_config_containing_union)
# verify that a_dict was read correctly
self.assertEqual(pytext_config.test_config.a_union.oye, an_int)
# serialize config to json, and deserialize back to config
# verify that nothing changed
jsonified_config = serialize.config_to_json(TestConfigContainer,
pytext_config)
pytext_config_deserialized = serialize.config_from_json(
TestConfigContainer, jsonified_config)
self.assertEqual(pytext_config, pytext_config_deserialized)
def test_config_to_json_for_union(self):
"""For a config that contains a dict inside it, verify that config
can be correctly created from a json/dict, and config can be correctly
serialized and de-serialized
"""
class Boo(ConfigBase):
aye: str
class TestConfigContainer(ConfigBase):
class TestConfig(ConfigBase):
a_union: Union[Boo, List[Boo]]
test_config: TestConfig
a_str = "abc"
a_config_containing_union = {
"test_config": {
"a_union": {
"boo": {
"aye": a_str
}
}
}
}
pytext_config = serialize.config_from_json(TestConfigContainer,
a_config_containing_union)
# verify that a_dict was read correctly
self.assertEqual(pytext_config.test_config.a_union.aye, a_str)
# serialize config to json, and deserialize back to config
# verify that nothing changed
jsonified_config = serialize.config_to_json(TestConfigContainer,
pytext_config)
pytext_config_deserialized = serialize.config_from_json(
TestConfigContainer, jsonified_config)
self.assertEqual(pytext_config, pytext_config_deserialized)
def test_config_to_json_for_dict(self):
"""For a config that contains a dict inside it, verify that config
can be correctly created from a json/dict, and config can be correctly
serialized and de-serialized
"""
class TestConfigContainer(ConfigBase):
class TestConfig(ConfigBase):
a_dict: Dict[str, Any]
test_config: TestConfig
a_dict = {
"param2": {
"nested_param1": 10,
"nested_param2": 2
},
"param1": "val1"
}
a_config_containing_dict = {"test_config": {"a_dict": a_dict}}
pytext_config = serialize.config_from_json(TestConfigContainer,
a_config_containing_dict)
# verify that a_dict was read correctly
self.assertEqual(pytext_config.test_config.a_dict, a_dict)
# serialize config to json, and deserialize back to config
# verify that nothing changed
jsonified_config = serialize.config_to_json(TestConfigContainer,
pytext_config)
pytext_config_deserialized = serialize.config_from_json(
TestConfigContainer, jsonified_config)
self.assertEqual(pytext_config, pytext_config_deserialized)
def torchscript_export(context, export_json, model, output_path, quantize,
target):
"""Convert a pytext model snapshot to a torchscript model."""
export_cfg = ExportConfig()
# only populate from export_json if no export option is configured from the command line.
if export_json:
export_json_config = _load_and_validate_export_json_config(export_json)
read_chunk_size = export_json_config.pop("read_chunk_size", None)
if read_chunk_size is not None:
print("Warning: Ignoring read_chunk_size.")
if export_json_config.get("read_chunk_size", None) is not None:
print(
"Error: Do not know what to do with read_chunk_size. Ignoring."
)
if "export" in export_json_config.keys():
export_cfgs = [export_json_config["export"]]
else:
export_cfgs = export_json_config["export_list"]
if target:
print(
"A single export was specified in the command line. Filtering out all other export options"
)
export_cfgs = [
cfg for cfg in export_cfgs if cfg["target"] == target
]
if export_cfgs == []:
print(
"No ExportConfig matches the target name specified in the command line."
)
for partial_export_cfg in export_cfgs:
if not quantize and not output_path:
export_cfg = config_from_json(ExportConfig, partial_export_cfg)
else:
print(
"the export-json config is ignored because export options are found the command line"
)
export_cfg = config_from_json(
ExportConfig,
partial_export_cfg,
("export_caffe2_path", "export_onnx_path"),
)
export_cfg.torchscript_quantize = quantize
# if config has export_torchscript_path, use export_torchscript_path from config, otherwise keep the default from CLI
if export_cfg.export_torchscript_path is not None:
output_path = export_cfg.export_torchscript_path
if not model or not output_path:
config = context.obj.load_config()
model = model or config.save_snapshot_path
output_path = output_path or f"{config.save_snapshot_path}.torchscript"
print(f"Exporting {model} to torchscript file: {output_path}")
print(export_cfg)
export_saved_model_to_torchscript(model, output_path, export_cfg)
def test_multiple_values_with_ellipsis(self):
config_json = json.loads("""
{"foo":[[1,2], [3,4], [5,6]]}
""")
config = config_from_json(TupleTestConfig, config_json)
self.assertEqual(config.foo[0][0], 1)
self.assertEqual(config.foo[2][1], 6)
def test_nested_tuple(self):
config_json = json.loads("""
{}
""")
config = config_from_json(TupleTestConfig, config_json)
self.assertEqual(config.foo[0][0], 32)
self.assertEqual(config.foo[1][1], 2)
def test_component_subconfig_serialize(self):
config_json = json.loads("""{
"foo": 5,
"models": [{
"bar": 12,
"m2s1s1": "thing"
}, {
"m2s1s1": "thing2"
}]
}""")
config = config_from_json(JointModel.Config, config_json)
serialized = config_to_json(JointModel.Config, config)
again = config_from_json(JointModel.Config, serialized)
self.assertEqual(again.foo, 5)
self.assertEqual(again.models[0].m2s1s1, "thing")
self.assertEqual(again.models[1].bar, 3)
def test_implicit_type_cast(self):
config_json = json.loads(
"""
{"foo": "123"}
"""
)
config = config_from_json(Model1, config_json)
self.assertEqual(config.foo, 123)
def test_use_default_value(self):
config_json = json.loads(
"""
{}
"""
)
config = config_from_json(Model1, config_json)
self.assertEqual(config.foo, 1)
def test_component(self):
config_json = json.loads("""{
"bar": 13,
"m2s1s1": "sub_foo"
}""")
config = config_from_json(RegisteredModel.Config, config_json)
self.assertEqual(config.bar, 13)
self.assertEqual(config.m2s1s1, "sub_foo")
self.assertEqual(config.m2s1, 5)
def run_single(
rank: int,
config_json: str,
world_size: int,
dist_init_method: str,
summary_writer: SummaryWriter,
):
config = config_from_json(PyTextConfig, config_json)
if rank != 0:
summary_writer = None
train_model(config, dist_init_method, rank, rank, world_size,
summary_writer)
def test_component_subconfig_deserialize(self):
config_json = json.loads("""{
"foo": 5,
"models": [{
"bar": 12,
"m2s1s1": "thing"
}, {
"m2s1s1": "thing2"
}]
}""")
config = config_from_json(JointModel.Config, config_json)
self.assertEqual(config.foo, 5)
self.assertEqual(len(config.models), 2)
self.assertEqual(config.models[1].m2s1s1, "thing2")
def test_auto_include_sub_component_in_config(self):
config_json = json.loads("""
{
"model": {"ModelFoo": {}},
"model_bar_x": {"ModelBar1": {}},
"model_union": {"ModelBar": {}},
"datahandler": {"SubDataHandler": {}}
}
""")
config = config_from_json(TestConfig, config_json)
self.assertEqual(config.model.foo, 2)
self.assertEqual(config.model_bar_x.bar, "bar1")
self.assertEqual(config.model_union.bar, "bar")
self.assertEqual(config.datahandler.foo, 3)
def run_single(
rank: int,
config_json: str,
world_size: int,
dist_init_method: str,
metadata: CommonMetadata,
):
config = config_from_json(PyTextConfig, config_json)
summary_writer = SummaryWriter(
) if rank != 0 and config.use_tensorboard else None
try:
train_model(config, dist_init_method, rank, rank, world_size,
summary_writer, metadata)
finally:
if summary_writer is not None:
summary_writer.close()
def test_default_union(self):
config_json = json.loads("""
{
"task": {
"model": {
"Model2": {
"bar": "test"
}
}
},
"output": "foo/bar.pt"
}
""")
config = config_from_json(PyTextConfig, config_json)
self.assertTrue(isinstance(config.task, Task1))
self.assertTrue(isinstance(config.task.model, Model2))
self.assertEqual(config.task.model.bar, "test")
def test_serialize_union_with_expansible_component(self):
config = TestConfig(
model=ModelFoo.Config(),
model_bar_x=ModelBar.Config(),
model_union=ModelBar1.Config(),
datahandler=SubDataHandler.Config(),
)
json = config_to_json(TestConfig, config)
print(json)
self.assertEqual(json["model"]["ModelFoo"]["foo"], 2)
self.assertEqual(json["model_bar_x"]["ModelBar"]["bar"], "bar")
self.assertEqual(json["model_union"]["ModelBar1"]["bar"], "bar1")
self.assertEqual(json["datahandler"]["SubDataHandler"]["foo"], 3)
config = config_from_json(TestConfig, json)
self.assertEqual(config.model.foo, 2)
self.assertEqual(config.model_bar_x.bar, "bar")
self.assertEqual(config.model_union.bar, "bar1")
self.assertEqual(config.datahandler.foo, 3)
def run_single(
rank: int,
config_json: str,
world_size: int,
dist_init_method: Optional[str],
metadata: Optional[CommonMetadata],
metric_channels: Optional[List[Channel]],
):
config = config_from_json(PyTextConfig, config_json)
if rank != 0:
metric_channels = []
train_model(
config=config,
dist_init_url=dist_init_method,
device_id=rank,
rank=rank,
world_size=world_size,
metric_channels=metric_channels,
metadata=metadata,
)
def test_default_value_expansible_config(self):
config_json = json.loads("""
{
"task": {
"Task3": {
"model": {
"foobar": 1.0
}
}
},
"output": "foo/bar.pt"
}
""")
config = config_from_json(PyTextConfig, config_json)
self.assertTrue(isinstance(config.task, Task3))
self.assertTrue(isinstance(config.task.model, ExpansibleModel.Config))
self.assertFalse(isinstance(config.task.model,
AExpansibleModel.Config))
self.assertFalse(isinstance(config.task.model,
BExpansibleModel.Config))
self.assertEqual(config.task.model.foobar, 1.0)
def run_single(rank, config_json: str, world_size: int, dist_init_method: str):
config = config_from_json(PyTextConfig, config_json)
train_model(config, dist_init_method, rank, rank, world_size)
