def __call__(self, composite: Composite):
bmis = [
bmi for period, bmi in composite.get_all_observations(
self.annual_bmi_var)
]
mean_bmi = numpy.average(bmis)
composite.put_immutable(self.mean_bmi_var, mean_bmi)
def test_encode_list_multiple_empty(immutable_list_schema, encode_mapping):
"""Mappings present, no content."""
composite = Composite(immutable_list_schema)
content: List[Dict] = [{}, {}, {}]
expected: List = content.copy()
actual: List = list(composite.encode_list(encode_mapping, content))
assert actual == expected
def test_encode_unmapped_content_raises(immutable_list_schema, encode_mapping):
"""If a content list item contains a field without a mapping, raise."""
composite = Composite(immutable_list_schema)
content: List[Dict] = [{"unknown_field": "foo"}]
with pytest.raises(ValueError):
# Wrap in list to force lazy execution
list(composite.encode_list(encode_mapping, content))
def test_decode_list(immutable_list_schema, decode_mapping):
"""Both mappings and content present."""
composite = Composite(immutable_list_schema)
content: List = [{
"inner_folder": {
"nombre": "Al",
"edad": 25
},
"helado": "Rocky road"
}, {
"inner_folder": {
"nombre": "Joe",
"edad": 34
},
"helado": "Clam chowder"
}]
expected: List[Dict] = [
{
"name": "Al",
"age": 25,
"ice cream": "Rocky road"
},
{
"name": "Joe",
"age": 34,
"ice cream":
"Clam chowder" # https://now.tufts.edu/articles/dish-we-all-scream-for-ice-cream
}
]
actual: List = list(composite.decode_list(decode_mapping, content))
assert actual == expected
def test_decode_null_content(immutable_list_schema, decode_mapping):
"""Content that is explicitly null is decoded."""
composite = Composite(immutable_list_schema)
content: List = [{"inner_folder": {"nombre": None}}]
expected: List[Dict] = [{"name": None}]
actual: List = list(composite.decode_list(decode_mapping, content))
assert actual == expected
def alter(self, composite_id: str, composite: Composite) -> None:
overall_rank = self.ranked["overall"][composite_id]
composite.put_immutable(self.bmi_rank_overall_var, overall_rank)
gender: str = self._get_gender(composite_id)
gender_rank = self.ranked[gender][composite_id]
composite.put_immutable(self.bmi_rank_gender_var, gender_rank)
def test_pop_missing_observation_with_default_no_side_effect(
imperfect_composite: Composite):
expected: Dict = copy.deepcopy(imperfect_composite.content)
imperfect_composite.pop_observation("the_weight_var",
"2012",
treat_missing_as_null=True)
assert imperfect_composite.content == expected
def test_decode_null_content(immutable_named_list_schema, decode_mapping):
"""Content that is explicitly null is decoded."""
composite = Composite(immutable_named_list_schema)
content: Dict = {"x": {"inner_folder": {"nombre": None}}}
expected: Dict = {"x": {"name": None}}
actual: Dict = composite.decode_named_list(decode_mapping, content)
assert actual == expected
def test_encode_unmapped_content_raises(immutable_named_list_schema,
encode_mapping):
"""If a content list item contains a field without a mapping, raise."""
composite = Composite(immutable_named_list_schema)
content: Dict = {"x": {"unknown_field": "foo"}}
with pytest.raises(ValueError):
composite.encode_named_list(encode_mapping, content)
def test_encode_named_list(immutable_named_list_schema, encode_mapping):
"""Both mappings and content present."""
composite = Composite(immutable_named_list_schema)
content: Dict = {
"al": {
"name": "Al",
"age": 25,
"ice cream": "Rocky road"
},
"joe": {
"name": "Joe",
"age": 34,
"ice cream":
"Clam chowder" # https://now.tufts.edu/articles/dish-we-all-scream-for-ice-cream
}
}
expected: Dict = {
"al": {
"inner_folder": {
"nombre": "Al",
"edad": 25
},
"helado": "Rocky road"
},
"joe": {
"inner_folder": {
"nombre": "Joe",
"edad": 34
},
"helado": "Clam chowder"
}
}
actual: Dict = composite.encode_named_list(encode_mapping, content)
assert actual == expected
def __call__(self, composite: Composite):
zip_code: str = composite.get_immutable(self.zip_var)
city = self.lookups["zipcodes"][zip_code]["City"]
composite.put_immutable(self.city_var, city)
state = self.lookups["zipcodes"][zip_code]["State"]
composite.put_immutable(self.state_var, state)
def delete_immutable(self, var: Variable, composite: Composite) -> None:
if "immutable" not in composite.content:
return
if var.descends_from_list:
self.delete_multiple(var, composite.content["immutable"])
else:
composite.del_immutable(var.var_id)
def __call__(self, composite: Composite):
years = sorted([int(year) for year in composite.periods])
weights = [composite.get_observation(self.annual_weight_var, str(year)) for year in years]
slope, intercept, r_value, p_value, std_err = scipy.stats.linregress(
np.asarray(years), np.asarray(weights)
)
composite.put_immutable(self.weight_slope_var, slope)
composite.put_immutable(self.weight_pval_var, p_value)
def test_encode_named_list_multiple_empty(immutable_named_list_schema,
encode_mapping):
"""Mappings present, no content."""
composite = Composite(immutable_named_list_schema)
content: Dict = {"tom": {}, "dick": {}, "harry": {}}
expected: Dict = copy.deepcopy(content)
actual: Dict = composite.encode_named_list(encode_mapping, content)
assert actual == expected
def test_decode_named_list_multiple_empty(immutable_named_list_schema,
decode_mapping):
"""Mappings present, no content."""
composite = Composite(immutable_named_list_schema)
content: Dict = {"x": {}, "y": {}, "z": {}}
expected: Dict = content.copy()
actual: Dict = composite.decode_named_list(decode_mapping, content)
assert actual == expected
def __call__(self, composite: Composite):
periods: List[str] = list(composite.periods)
annual_prods = [
composite.get_observation(self.annual_prod_var, period)
for period in periods
]
mean_prod = numpy.average(annual_prods)
composite.put_immutable(self.mean_prod_var, mean_prod)
def alter(self, composite_id: str, composite: Composite) -> None:
value: Optional[int] = composite.get_immutable(
self.source, treat_missing_as_null=True)
if value is None:
return
assert value in self.value_to_quantile, "Value {} did not get recorded in value-to-quantile map".format(
value)
quantile: float = self.value_to_quantile[value]
composite.put_immutable(self.target, quantile)
def alter(self, composite_id: str, composite: Composite) -> None:
try:
value: Any = composite.get_immutable(self.source)
except MissingDataError:
return
assert value in self.ranks, "Value {} did not get recorded in value-to-rank map".format(
value)
rank: int = self.ranks[value]
composite.put_immutable(self.target, rank)
def test_pop_observation_deletes_value(imperfect_composite: Composite):
expected: Dict = {
"2010": {
"weight_in_pounds": None
},
"2011": {},
"2012": {}
}
imperfect_composite.pop_observation("the_weight_var", "2011")
assert imperfect_composite.content == expected
def test_del_immutable(simple_composite: Composite):
expected: Dict = {
"first_name":
"Steve",
"gender":
"male",
"total_weight_gain":
3.9,
"personal_summary":
"Steve's favorite color is orange (FFA000). Over the observation period, he gained"
" 3.9 lbs."
}
simple_composite.del_immutable("color_folder")
assert simple_composite.content["immutable"] == expected
def composite_2(schema) -> Composite:
content: Dict = {
"1": {"t": "x"},
"2": {"t": "y"},
"immutable": {"i": 1}
}
return Composite(schema, content)
def test_encode_parallel_structure_scenario(nested_list_composite,
nested_list_schema, encode_mapping,
parallel_repr):
"""Scenario 1: The structure of the schema list is identical to the structure of the internal list, but with
different labels."""
target: Composite = Composite(nested_list_schema, {})
# First application: encode the innermost items so that they match the schema. Notice that we hard-code the internal
# representation, which is the point of the encode/decode scheme--Polytropos actions can make use of any internal
# representation and still be agnostic to the final, external representation.
outer: List = []
for item in parallel_repr: # type: Dict
inner = item[
"inner"] # The calling method must be aware of its own internal structures
inner_elements_encoded = list(target.encode_list(
encode_mapping, inner))
inner_item_encoded = {"inner": inner_elements_encoded}
outer.append(inner_item_encoded)
# Second application: Treating the inner content as a black box, encode it into the outer represenation.
outer_encoded: List[Dict] = list(
nested_list_composite.encode_list(encode_mapping, outer))
target.put_immutable("outer_list_1_id", outer_encoded)
assert target.content == nested_list_composite.content
def composite_1(schema) -> Composite:
content: Dict = {
"period_1": {
"t_Text": "Not trivial",
"t_Folder": {
"t_Folder_Text": "Not trivial"
},
"t_List": [{
"t_List_Text": "Not trivial"
}],
"t_KeyedList": {
"a key": {
"t_KeyedList_Text": "Not trivial"
}
}
},
"period_2": {
"t_Text": None,
"t_Folder": {},
"t_List": [],
"t_KeyedList": {}
},
"immutable": {}
}
return Composite(schema, content, composite_id="composite_1")
def _do_test(content: Dict, expected: Dict) -> None:
composite: Composite = Composite(ah_schema, content)
subjects: List[VariableId] = [cast(VariableId, "ad_hoc_{}".format(idx)) for idx in range(4)]
target: VariableId = subjects[0]
change: CrossSectionalSum = CrossSectionalSum(ah_schema, {}, subjects, value_target=target)
change(composite)
assert composite.content == expected
def test_get_immutable_null(simple_schema: Schema):
composite: Composite = Composite(simple_schema,
{"immutable": {
"first_name": None
}})
actual: Optional[str] = composite.get_immutable("the_person_name_var")
assert actual is None
def composite_1(schema) -> Composite:
content: Dict = {
"1": {"t": "a"},
"2": {"t": "b"},
"immutable": {"i": 74}
}
return Composite(schema, content)
def emit(self) -> Iterator[Tuple[str, Composite]]:
for zip_code, transient in self.city_data.items():
city: Composite = Composite(self.target_schema, {})
transient_periods = set(transient.keys()) - {"immutable"}
for period in sorted(transient_periods):
n_companies = transient[period]["n_companies"]
tot_employees = transient[period]["tot_employees"]
tot_revenue = transient[period]["tot_revenue"]
mean_employees = tot_employees / n_companies
productivity = tot_revenue / tot_employees
city.put_observation(period, self.n_company_var, n_companies)
city.put_observation(period, self.annual_prod_var,
productivity)
city.put_observation(period, self.mean_employee_var,
mean_employees)
city.put_immutable(self.target_zip_var,
transient["immutable"]["zip"])
city.put_immutable(self.target_city_var,
transient["immutable"]["city"])
city.put_immutable(self.target_state_var,
transient["immutable"]["state"])
yield zip_code, city
def alter_and_write_composite(self, origin_dir, target_dir, filename):
with open(os.path.join(origin_dir, filename), 'r') as origin_file:
content: Dict = json.load(origin_file)
composite: Composite = Composite(self.schema, content)
self.alter(filename, composite)
with open(os.path.join(target_dir, filename), 'w') as target_file:
json.dump(composite.content, target_file, indent=2)
def test_underscore_folders_ignored():
spec: Dict = {
"binary_in_root": {
"name": "the_binary",
"data_type": "Binary",
"sort_order": 0
}
}
immutable: Track = Track.build(spec, None, "immutable")
temporal: Track = Track.build({}, None, "temporal")
schema: Schema = Schema(temporal, immutable)
content: Dict = {
"immutable": {
"the_binary": "true",
"_folder": {
"foo": "shouldn't matter",
"bar": "also shouldn't matter"
}
}
}
expected: Dict = {
"immutable": {
"the_binary": True,
"_folder": {
"foo": "shouldn't matter",
"bar": "also shouldn't matter"
}
}
}
composite: Composite = Composite(schema, content)
cast: Cast = Cast(schema, {})
cast(composite)
assert composite.content == expected
def process_composite(self, origin_dir: str, base_target_dir: str,
composite_id: str) -> None:
try:
relpath: str = relpath_for(composite_id)
origin_filename: str = os.path.join(origin_dir, relpath,
"%s.json" % composite_id)
logging.debug("Evolving %s." % origin_filename)
with open(origin_filename) as origin_file:
content: Dict = json.load(origin_file)
composite: Composite = Composite(self.schema,
content,
composite_id=composite_id)
for change in self.changes:
logging.debug('Applying change "%s" to %s.' %
(change.__class__.__name__, origin_filename))
change(composite)
target_dir: str = os.path.join(base_target_dir, relpath)
os.makedirs(target_dir, exist_ok=True)
target_filepath: str = os.path.join(target_dir,
"%s.json" % composite_id)
with open(target_filepath, 'w') as target_file:
json.dump(composite.content, target_file, indent=2)
except Exception as e:
logging.error("Error processing composite %s during evolve step." %
composite_id)
traceback.print_exc()
raise