def translate(source_schema, target_schema) -> Translate:
context: Context = Context("", "", "", "", "", "", "", False, 1, False,
True)
translate_immutable: Translator = Translator(
target_schema.immutable, Translate.create_document_value_provider)
translate_temporal: Translator = Translator(
target_schema.temporal, Translate.create_document_value_provider)
return Translate(context, target_schema, translate_immutable,
translate_temporal)
def do_test(s_doc, s_spec, t_doc, t_spec, create_document_value_provider):
source_track: Track = Track.build(s_spec, None, "Source")
target_track: Track = Track.build(t_spec, source_track, "Target")
translate: Translator = Translator(target_track,
create_document_value_provider)
actual: OrderedDict[str, Any] = translate("composite_id", "period", s_doc)
assert actual == t_doc
def test_translate_no_target_variables(source_track, target_track):
target_track.values.return_value = []
document = {"a": 1}
translator = Translator(target_track)
translated = translator(document)
assert translated == {}
def test_use_same_source_twice(source_spec: Dict, source_doc: Dict,
create_document_value_provider, expected):
"""Two targets can use the same source."""
target_spec: Dict = {
"target_var_1": {
"name": "first_target",
"data_type": "Integer",
"sources": ["source_var_1"],
"sort_order": 0
},
"target_var_2": {
"name": "second_target",
"data_type": "Integer",
"sources": ["source_var_1"],
"sort_order": 1
}
}
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track,
create_document_value_provider)
actual: OrderedDict[str, Any] = translate("composite_id", "period",
source_doc)
assert actual == expected
def test_list_in_folder(source, target, index, create_document_value_provider):
source_spec, source_doc = source
target_spec, expected = target()
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track, create_document_value_provider)
actual: OrderedDict[str, Any] = translate("composite_id", "period", source_doc)
assert actual == expected[index]
def test_translate_no_target_variables(source_track, target_track,
create_document_value_provider):
target_track.values.return_value = []
document = {"a": 1}
translator = Translator(target_track, create_document_value_provider)
translated = translator("composite_id", "period", document)
assert translated == OrderedDict()
def test_list_in_folder(source, target):
source_spec, source_doc = source
target_spec, expected = target()
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track)
actual: Dict = translate(source_doc)
assert actual == expected
def build(cls, path_locator: "PathLocator", schema: Schema,
target_schema: str):
"""
:param path_locator:
:param schema: The source schema, already instantiated.
:param target_schema: The path to the definition of the target schema.
:return:
"""
target_schema_instance: Schema = Schema.load(path_locator,
target_schema,
source_schema=schema)
translate_immutable: Translator = Translator(
target_schema_instance.immutable)
translate_temporal: Translator = Translator(
target_schema_instance.temporal)
return cls(target_schema_instance, translate_immutable,
translate_temporal)
def test_translate_exception(monkeypatch, source_track, target_track, type_translator_class):
monkeypatch.setattr(TypeTranslatorRegistry, "get_translator_class", type_translator_class)
target_track.values.return_value = [create_variable("id_a", "a", None), create_variable("id_b", "b", None)]
document = {"a": 1, "b": AttributeError}
translator = Translator(target_track)
with pytest.raises(AttributeError):
_ = translator(document)
def test_translate_missing_source(monkeypatch, source_track, target_track, type_translator_class):
monkeypatch.setattr(TypeTranslatorRegistry, "get_translator_class", type_translator_class)
target_track.values.return_value = [create_variable("id_a", "a", None), create_variable("id_b", "b", None), create_variable("id_c", "c", None)]
document = {"a": 1, "b": 2}
translator = Translator(target_track)
translated = translator(document)
assert translated == {"a": 101, "b": 102}
def test_duplicate_name_raises(source_doc, source_spec, target_spec):
source_doc["list_source_1"]["Stacy"] = {
"Name": "Another Stacy"
}
with pytest.raises(ValueError):
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track)
translate(source_doc)
def test_remove_sources(source_doc: Dict, source_spec: Dict, target_spec: Dict):
"""Remove root sources at runtime, resulting in a cascade; no list is created."""
expected: Dict = {}
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
target_track["target_root"].sources = []
translate: Translator = Translator(target_track)
actual: Dict = translate(source_doc)
assert actual == expected
def test_rearrange(source_doc: Dict, source_spec: Dict, target_doc: Dict, target_spec: Dict):
"""Verify that translate respects the sort order property of the variables in the target spec, and ignores the
order in which the variables happen to be defined in the spec. """
shuffled_source_spec = shuffle(source_spec)
shuffled_target_spec = shuffle(target_spec)
source_track: Track = Track.build(shuffled_source_spec, None, "Source")
target_track: Track = Track.build(shuffled_target_spec, source_track, "Target")
translate: Translator = Translator(target_track)
actual: Dict = translate(source_doc)
assert actual == target_doc
def test_keyed_list_in_list(source, target, index, create_document_value_provider):
"""Reversing the order of the sources in the target list spec results in an equivalent change in the order of the
resulting list."""
source_spec, source_doc = source
target_spec, target_doc = target
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track, create_document_value_provider)
actual: OrderedDict[str, Any] = translate("composite_id", "period", source_doc)
assert actual == target_doc[index]
def test_translate_all_children_missing(source: Callable, target: Callable):
__, source_spec = source()
__, target_spec = target()
source_doc: Dict = {}
expected: Dict = {}
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track)
actual: Dict = translate(source_doc)
assert actual == expected
def test_named_list_in_list(source, target):
"""Reversing the order of the sources in the target list spec results in an equivalent change in the order of the
resulting list."""
source_spec, source_doc = source
target_spec, target_doc = target
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track)
actual: Dict = translate(source_doc)
assert actual == target_doc
def build(
cls, path_locator: "PathLocator", schema: Schema, target_schema: str
) -> "Translate": # type: ignore # Signature of "build" incompatible with supertype "Step"
"""
:param path_locator:
:param schema: The source schema, already instantiated.
:param target_schema: The path to the definition of the target schema.
:return:
"""
logging.info("Initializing Translate step.")
target_schema_instance: Optional[Schema] = Schema.load(
target_schema, source_schema=schema, path_locator=path_locator)
assert target_schema_instance is not None
translate_immutable: Translator = Translator(
target_schema_instance.immutable)
translate_temporal: Translator = Translator(
target_schema_instance.temporal)
return cls(target_schema_instance, translate_immutable,
translate_temporal)
def test_rearrange(source_doc: Dict, source_spec: Dict, target_docs: List[Dict], target_spec: Dict, index, create_document_value_provider):
"""Verify that translate respects the sort order property of the variables in the target spec, and ignores the
order in which the variables happen to be defined in the spec. """
shuffled_source_spec = shuffle(source_spec)
shuffled_target_spec = shuffle(target_spec)
source_track: Track = Track.build(shuffled_source_spec, None, "Source")
target_track: Track = Track.build(shuffled_target_spec, source_track, "Target")
translate: Translator = Translator(target_track, create_document_value_provider)
actual: OrderedDict[str, Any] = translate("composite_id", "period", source_doc)
assert actual == target_docs[index]
def test_translate_all_children_missing(source: Callable, target: Callable,
create_document_value_provider,
expected):
__, source_spec = source()
__, target_spec = target()
source_doc: Dict = {}
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track,
create_document_value_provider)
actual: OrderedDict[str, Any] = translate("composite_id", "period",
source_doc)
assert actual == expected
def test_folder_null_skipped(source, target, create_document_value_provider,
expected):
"""On occasion, e-files contain <EmptyElements/> that would normally contain list items. These are converted to
JSON as {"EmptyElement": null} and are not included as list items during translation."""
source_spec, source_doc = source
source_doc["second_source_folder"] = None
target_spec, _ = target
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track,
create_document_value_provider)
actual: OrderedDict[str, Any] = translate("composite_id", "period",
source_doc)
assert actual == expected
def test_translate_with_folders(source: Callable, target: Callable):
"""Summary: verify that source topology doesn't matter for a given target spec.
Long version: Try every combination of the above examples. Because a target variable's source is defined by ID,
the particular location of the source variable in the source hierarchy is irrelevant. That is, no matter how the
source hierarchy is arranged, a target spec should produce the same target hierarchy as long as all the source
variables exist."""
source_doc, source_spec = source()
expected, target_spec = target()
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track)
actual: Dict = translate(source_doc)
assert actual == expected
def test_translate_one_target_variable(monkeypatch, source_track, target_track,
type_translator_class,
create_document_value_provider):
monkeypatch.setattr(TypeTranslatorRegistry, "get_translator_class",
type_translator_class)
target_track.values.return_value = [
create_variable("id_a", "a", None, sort_order=1)
]
document = {"a": 1, "b": 2}
translator = Translator(target_track, create_document_value_provider)
translated = translator("composite_id", "period", document)
assert translated == OrderedDict([("a", 101)])
def test_translate_no_sources_listed(target_spec: Dict, source_spec: Dict,
source_doc: Dict):
"""If a primitive is supposed to be translated but it has no sources, it is always null."""
source_track: Track = Track.build(source_spec, None, "Source")
target_spec["target_var_id"]["sources"] = []
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track)
actual: Dict[str, Any] = translate(source_doc)
expected: Dict[str, Any] = {}
assert actual == expected
def test_translate_no_sources_listed(target_spec: Dict, source_spec: Dict,
source_doc: Dict,
create_document_value_provider, expected):
"""If a primitive is supposed to be translated but it has no sources, it is always null."""
source_track: Track = Track.build(source_spec, None, "Source")
target_spec["target_var_id"]["sources"] = []
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track,
create_document_value_provider)
actual: OrderedDict[str, Any] = translate("composite_id", "period",
source_doc)
assert actual == expected
def test_translate_exception(monkeypatch, source_track, target_track,
type_translator_class,
create_document_value_provider):
monkeypatch.setattr(TypeTranslatorRegistry, "get_translator_class",
type_translator_class)
target_track.values.return_value = [
create_variable("id_a", "a", None, sort_order=1),
create_variable("id_b", "b", None, sort_order=2)
]
document = {"a": 1, "b": AttributeError}
translator = Translator(target_track, create_document_value_provider)
with pytest.raises(AttributeError):
_ = translator("composite_id", "period", document)
def test_translate_all_children_none(create_document_value_provider, expected):
__, source_spec = source_one_folder()
__, target_spec = target_one_folder()
source_doc: Dict = {
"the_folder": {
"first_source": None,
"second_source": None
}
}
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track,
create_document_value_provider)
actual: OrderedDict[str, Any] = translate("composite_id", "period",
source_doc)
assert actual == expected
def test_translate_all_children_none():
__, source_spec = source_one_folder()
__, target_spec = target_one_folder()
source_doc: Dict = {
"the_folder": {
"first_source": None,
"second_source": None
}
}
expected = {
"the_folder": {
"first_target": None,
"second_target": None
}
}
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track)
actual: Dict = translate(source_doc)
assert actual == expected
def test_use_same_source_twice(source_spec: Dict, source_doc: Dict):
"""Two targets can use the same source."""
target_spec: Dict = {
"target_var_1": {
"name": "first_target",
"data_type": "Integer",
"sources": ["source_var_1"],
"sort_order": 0
},
"target_var_2": {
"name": "second_target",
"data_type": "Integer",
"sources": ["source_var_1"],
"sort_order": 1
}
}
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track)
actual: Dict[str, Any] = translate(source_doc)
expected: Dict[str, Any] = {"first_target": 75, "second_target": 75}
assert actual == expected
def do_test(s_doc, s_spec, t_doc, t_spec):
source_track: Track = Track.build(s_spec, None, "Source")
target_track: Track = Track.build(t_spec, source_track, "Target")
translate: Translator = Translator(target_track)
actual: Dict = translate(s_doc)
assert actual == t_doc
def translate(source_spec: Dict, target_spec: Dict) -> Translator:
source_track: Track = Track.build(source_spec, None, "Source")
target_track: Track = Track.build(target_spec, source_track, "Target")
translate: Translator = Translator(target_track)
return translate