def generate_description(self, alfacase_config: AlfacaseTestConfig):
"""
Helper method to generate a "Description" from the given alfacase_config
"""
alfacase_string = convert_description_to_alfacase(
alfacase_config.description_expected
)
alfacase_content = strictyaml.dirty_load(
yaml_string=alfacase_string,
schema=alfacase_config.schema,
allow_flow_style=True,
)
# 'LoadPvtModelsDescription' is special case and the DescriptionDocument doesn't need a FakeKey
skip_dict = (
alfacase_config.load_function_name == "load_pvt_models_description"
)
if alfacase_config.is_sequence:
alfacase_content = [alfacase_content]
elif alfacase_config.is_dict and not skip_dict:
alfacase_content = YAML(
CommentedMap({YAML("FakeKey"): alfacase_content})
)
description_document = DescriptionDocument(
content=alfacase_content, file_path=self.tmp_path / "test_case.alfacase"
)
return getattr(alfacase_to_case, alfacase_config.load_function_name)(
description_document
)
def parse(yaml_string: str,
force_separate_thread: bool = False,
dataset_class: Type[Dataset] = Dataset):
"""
Parameters
----------
yaml_string : str
force_separate_thread : bool
dataset_class : class
Returns
-------
cube_settings: list of dict
regularizers: list
topic_model: TopicModel
dataset: Dataset
"""
parsed = dirty_load(yaml_string, base_schema, allow_flow_style=True)
specific_topic_names, background_topic_names = create_default_topics(
parsed.data["topics"]["specific_topics"],
parsed.data["topics"]["background_topics"])
revalidate_section(parsed, "stages")
revalidate_section(parsed, "regularizers")
if "scores" in parsed:
revalidate_section(parsed, "scores")
cube_settings = []
dataset = dataset_class(parsed.data["model"]["dataset_path"])
modalities_to_use = parse_modalities_data(parsed)
data_stats = count_vocab_size(dataset.get_dictionary(), modalities_to_use)
model = init_simple_default_model(
dataset=dataset,
modalities_to_use=modalities_to_use,
main_modality=parsed.data["model"]["main_modality"],
specific_topics=parsed.data["topics"]["specific_topics"],
background_topics=parsed.data["topics"]["background_topics"],
)
regularizers = _add_parsed_regularizers(parsed, model,
specific_topic_names,
background_topic_names, data_stats)
topic_model = TopicModel(model)
_add_parsed_scores(parsed, topic_model)
for stage in parsed['stages']:
for elemtype, elem_args in stage.items():
settings = build_cube_settings(elemtype.data, elem_args)
if force_separate_thread:
settings[elemtype]["separate_thread"] = False
cube_settings.append(settings)
return cube_settings, regularizers, topic_model, dataset
def test_update_multi_input_flags_behavior():
content = strictyaml.dirty_load(
yaml_string=textwrap.dedent("""\
# Just constant, use "constant" flag.
volumetric_flow_rates_std:
gas:
value: 0.0
unit: sm3/d
# Constant and curve but no flag, use default flag.
mass_flow_rates:
gas:
value: 0.0
unit: kg/s
mass_flow_rates_curve:
gas:
image:
values: [0.0, 1.0]
unit: kg/s
domain:
values: [0, 10]
unit: s
# Just flag, use value from yaml;
total_mass_flow_rate_input_type: curve
# Just curve, use "curve" flag.
water_cut_curve:
image:
values: [0.2, 0.3]
unit: "-"
domain:
values: [0, 20]
unit: s
"""),
schema=mass_source_node_properties_description_schema,
allow_flow_style=True,
)
document = DescriptionDocument(content, Path())
mass_source_node_properties = load_mass_source_node_properties_description(
document)
assert (mass_source_node_properties.volumetric_flow_rates_std_input_type ==
MultiInputType.Constant)
assert (mass_source_node_properties.mass_flow_rates_input_type ==
MultiInputType.Constant)
assert (mass_source_node_properties.total_mass_flow_rate_input_type ==
MultiInputType.Curve)
assert mass_source_node_properties.water_cut_input_type == MultiInputType.Curve
def parse(yaml_string):
"""
Parameters
----------
yaml_string : str
Returns
-------
cube_settings: list of dict
regularizers: list
topic_model: TopicModel
dataset: Dataset
"""
parsed = dirty_load(yaml_string, base_schema, allow_flow_style=True)
revalidate_section(parsed, "stages")
revalidate_section(parsed, "regularizers")
cube_settings = []
regularizers = []
dataset = Dataset(parsed.data["model"]["dataset_path"])
model = init_simple_default_model(
dataset=dataset,
modalities_to_use=parsed.data["model"]["modalities_to_use"],
main_modality=parsed.data["model"]["main_modality"],
specific_topics=parsed.data["topics"]["specific_topics"],
background_topics=parsed.data["topics"]["background_topics"],
)
for stage in parsed.data['regularizers']:
for elemtype, elem_args in stage.items():
regularizer_object = build_regularizer(elemtype, elem_args, parsed)
regularizers.append(regularizer_object)
model.regularizers.add(regularizer_object, overwrite=True)
topic_model = TopicModel(model)
for stage in parsed['stages']:
for elemtype, elem_args in stage.items():
settings = build_cube_settings(elemtype.data, elem_args)
cube_settings.append(settings)
return cube_settings, regularizers, topic_model, dataset
def Parse(yaml_file_obj):
"""Parse yaml config.
Args:
yaml_file_obj: yaml file obj.
Returns:
a lab_config_pb2.LabConfig proto.
Raises:
ConfigError: if the config is incorrect.
"""
content = six.ensure_str(yaml_file_obj.read())
try:
config_dict = syaml.dirty_load(
content, schema=_YAML_SCHEMA, allow_flow_style=True).data
except (syaml.YAMLError, TypeError) as e:
raise ConfigError(e)
try:
return json_format.ParseDict(config_dict, lab_config_pb2.LabConfig())
except json_format.ParseError as e:
raise ConfigError(e)
def test_convert_alfacase_to_description_restart_file_path(tmp_path):
"""
Round-trip test with a description that has a Path as type.
- YAML representation should be a Str()
- CaseDescription should be a pathlib.Path
"""
alfacase_file = tmp_path / "test_case.yaml"
some_folder = tmp_path / "some_folder"
some_folder.mkdir()
restart_file = some_folder / "restart.state"
restart_file.touch()
physics_with_restart_file = attr.evolve(
filled_case_descriptions.PHYSICS_DESCRIPTION, restart_filepath=restart_file
)
alfacase_string = convert_description_to_alfacase(physics_with_restart_file)
restart_file_relative_path = restart_file.relative_to(alfacase_file.parent)
assert f"restart_filepath: {restart_file.absolute()}" in alfacase_string
alfacase_string = alfacase_string.replace(
f"restart_filepath: {restart_file.absolute()}",
f"restart_filepath: {restart_file_relative_path}",
)
alfacase_content = strictyaml.dirty_load(
yaml_string=alfacase_string,
schema=schema.physics_description_schema,
allow_flow_style=True,
)
assert isinstance(alfacase_content["restart_filepath"].value, str)
assert alfacase_content["restart_filepath"].value == str(
Path("some_folder/restart.state")
)
description_document = DescriptionDocument(
content=alfacase_content, file_path=alfacase_file
)
physics_description = load_physics_description(description_document)
assert physics_description.restart_filepath == restart_file
def loads(string):
yaml = dirty_load(string, schema=_deme_graph_schema, allow_flow_style=True)
d = yaml.data # data dict
g = demes.DemeGraph(
description=d.get("description"),
time_units=d.get("time_units"),
generation_time=d.get("generation_time"),
doi=d.get("doi"),
default_Ne=d.get("default_Ne"),
)
for deme_id, deme_dict in d.get("demes", dict()).items():
if "epochs" in deme_dict:
deme_dict["epochs"] = [
demes.Epoch(**epoch_dict) for epoch_dict in deme_dict["epochs"]
]
g.deme(deme_id, **deme_dict)
for migration_dict in d.get("migrations", []):
g.migration(**migration_dict)
for pulse_dict in d.get("pulses", []):
g.pulse(**pulse_dict)
return g
def generate_description(
self,
alfacase_config: AlfacaseTestConfig,
remove_redundant_input_type_data: bool = False,
):
"""
Helper method to generate a "Description" from the given alfacase_config
"""
alfacase_string = convert_description_to_alfacase(
alfacase_config.description_expected,
remove_redundant_input_type_data=
remove_redundant_input_type_data,
)
alfacase_content = strictyaml.dirty_load(
yaml_string=alfacase_string,
schema=alfacase_config.schema,
allow_flow_style=True,
)
# 'LoadPvtModelsDescription' is special case and the DescriptionDocument doesn't need a FakeKey
skip_dict = (alfacase_config.load_function_name ==
"load_pvt_models_description")
if alfacase_config.is_sequence:
alfacase_content = [alfacase_content]
elif alfacase_config.is_dict and not skip_dict:
alfacase_content = YAML(
CommentedMap({YAML("FakeKey"): alfacase_content}))
description_document = DescriptionDocument(
content=alfacase_content,
file_path=self.tmp_path / "test_case.alfacase")
if hasattr(alfacase_to_case, alfacase_config.load_function_name):
loader = getattr(alfacase_to_case,
alfacase_config.load_function_name)
else:
loader = alfacase_to_case.get_instance_loader(
class_=alfacase_config.description_expected.__class__)
return loader(description_document)
def parse(yaml_string: str,
force_separate_thread: bool = False,
dataset_class: Type[Dataset] = Dataset):
"""
Parameters
----------
yaml_string : str
force_separate_thread : bool
dataset_class : class
Returns
-------
cube_settings: list of dict
regularizers: list
topic_model: TopicModel
dataset: Dataset
"""
parsed = dirty_load(yaml_string, BASE_SCHEMA, allow_flow_style=True)
specific_topic_names, background_topic_names = create_default_topics(
parsed.data["topics"]["specific_topics"],
parsed.data["topics"]["background_topics"])
revalidate_section(parsed, "stages")
revalidate_section(parsed, "regularizers")
if "scores" in parsed:
revalidate_section(parsed, "scores")
dataset = dataset_class(
data_path=parsed.data["model"]["dataset_path"],
keep_in_memory=parsed.data["model"].get("keep_in_memory", True),
internals_folder_path=parsed.data["model"].get("internals_folder_path",
None),
)
filter_parameters = parsed.data["model"].get(
KEY_DICTIONARY_FILTER_PARAMETERS, dict())
if len(filter_parameters) > 0:
filtered_dictionary = dataset.get_dictionary().filter(
**filter_parameters)
dataset._cached_dict = filtered_dictionary
modalities_to_use = parse_modalities_data(parsed)
data_stats = count_vocab_size(dataset.get_dictionary(), modalities_to_use)
model = init_simple_default_model(
dataset=dataset,
modalities_to_use=modalities_to_use,
main_modality=parsed.data["model"]["main_modality"],
specific_topics=parsed.data["topics"]["specific_topics"],
background_topics=parsed.data["topics"]["background_topics"],
)
regularizers = _add_parsed_regularizers(parsed, model,
specific_topic_names,
background_topic_names, data_stats)
topic_model = TopicModel(model)
_add_parsed_scores(parsed, topic_model)
cube_settings = list()
for stage in parsed['stages']:
for elemtype, elem_args in stage.items():
settings = build_cube_settings(elemtype.data, elem_args)
settings[elemtype]["separate_thread"] = force_separate_thread
cube_settings.append(settings)
return cube_settings, regularizers, topic_model, dataset
MultiInputType.Curve)
assert mass_source_node_properties.water_cut_input_type == MultiInputType.Curve
@pytest.fixture()
def description_document_for_pvt_tables_test(tmp_path):
case = case_description.PvtModelsDescription(tables={
"acme": "acme.tab",
"acme_2": "acme.tab"
})
alfacase_string = convert_description_to_alfacase(case)
alfacase_file_path = tmp_path / "test_case.alfacase"
shutil.copy2(get_acme_tab_file_path(), tmp_path)
alfacase_content = strictyaml.dirty_load(
alfacase_string,
schema=schema.pvt_models_description_schema,
allow_flow_style=True,
)
return DescriptionDocument(content=alfacase_content,
file_path=alfacase_file_path)
def test_load_pvt_tables_with_relative_file(
description_document_for_pvt_tables_test, tmp_path):
"""
PvtModelsDescription.tables should accept a path relative to a tab file
"""
document = description_document_for_pvt_tables_test
pvt_model_description = load_pvt_models_description(document=document)
assert pvt_model_description.tables == {
def _deserialized_from_string(self, source):
obj = strictyaml.dirty_load(source, allow_flow_style=True)
return obj.data
