def _run_task(scenario,
task_name,
expected_location,
output_dir: Optional[str] = None):
polytropos.actions.register_all()
conf = os.path.join(basepath, '../examples', scenario, 'conf')
data = os.path.join(basepath, '../examples', scenario, 'data')
with Context.build(conf, data, output_dir=output_dir) as context:
task = Task.build(context, task_name)
task.run()
actual_path = os.path.join(task.context.entities_output_dir,
task.target_data)
expected_path = os.path.join(task.context.entities_input_dir,
expected_location)
composite_ids: List = list(find_all_composites(expected_path))
assert list(find_all_composites(expected_path)) == composite_ids
for composite_id in composite_ids:
relpath: str = relpath_for(composite_id)
with open(
os.path.join(actual_path, relpath,
"%s.json" % composite_id)) as f:
with open(
os.path.join(expected_path, relpath,
"%s.json" % composite_id)) as g:
actual_data = json.load(f)
expected_data = json.load(g)
diff = Differ().compare(
json.dumps(actual_data, indent=4).split('\n'),
json.dumps(expected_data, indent=4).split('\n'))
assert compare(
actual_data,
expected_data), ('Diff: ' +
'\n'.join(line for line in diff))
def _do_copy_all(eins: Set[str], source_dir: str, target_dir: str) -> None:
for ein in eins:
relpath: str = relpath_for(ein)
source: str = "{}/{}/{}.json".format(source_dir, relpath, ein)
target: str = "{}/{}/{}.json".format(target_dir, relpath, ein)
os.makedirs(os.path.dirname(target), exist_ok=True)
shutil.copyfile(source, target)
def process_composite(self, origin_dir: str, base_target_dir: str,
composite_id: str) -> None:
try:
relpath: str = relpath_for(composite_id)
origin_filename: str = os.path.join(origin_dir, relpath,
"%s.json" % composite_id)
logging.debug("Evolving %s." % origin_filename)
with open(origin_filename) as origin_file:
content: Dict = json.load(origin_file)
composite: Composite = Composite(self.schema,
content,
composite_id=composite_id)
for change in self.changes:
logging.debug('Applying change "%s" to %s.' %
(change.__class__.__name__, origin_filename))
change(composite)
target_dir: str = os.path.join(base_target_dir, relpath)
os.makedirs(target_dir, exist_ok=True)
target_filepath: str = os.path.join(target_dir,
"%s.json" % composite_id)
with open(target_filepath, 'w') as target_file:
json.dump(composite.content, target_file, indent=2)
except Exception as e:
logging.error("Error processing composite %s during evolve step." %
composite_id)
traceback.print_exc()
raise
def process_composites(self, composite_ids: List[str], origin_dir: str) -> List[Tuple[str, Any]]:
result: List[Tuple[str, Any]] = []
for composite_id in composite_ids:
relpath: str = relpath_for(composite_id)
with open(os.path.join(origin_dir, relpath, "%s.json" % composite_id)) as origin_file:
content: Dict = json.load(origin_file)
composite: Composite = Composite(self.schema, content, composite_id=composite_id)
result.append((composite_id, self.extract(composite)))
return result
def write_composites(emissions: List[Tuple[str, Composite]],
target_base_dir: str) -> None:
for emission in emissions:
composite_id, composite = emission
relpath: str = relpath_for(composite_id)
target_dir: str = os.path.join(target_base_dir, relpath)
os.makedirs(target_dir, exist_ok=True)
with open(os.path.join(target_dir, composite_id + '.json'),
'w') as target_file:
json.dump(composite.content, target_file, indent=2)
def alter_and_write_composites(self, composite_ids: List[str], origin_dir: str, target_base_dir: str) -> None:
for composite_id in composite_ids:
relpath: str = relpath_for(composite_id)
with open(os.path.join(origin_dir, relpath, "%s.json" % composite_id)) as origin_file:
content: Dict = json.load(origin_file)
composite: Composite = Composite(self.schema, content, composite_id=composite_id)
self.alter(composite_id, composite)
target_dir: str = os.path.join(target_base_dir, relpath)
os.makedirs(target_dir, exist_ok=True)
with open(os.path.join(target_dir, "%s.json" % composite_id), 'w') as target_file:
json.dump(composite.content, target_file, indent=2)
def test_quantile_outcome(index: int):
composite_id: str = "{:09}".format(index)
relpath: str = relpath_for(composite_id)
e_path: str = os.path.join(base_path, "expected", relpath,
"%s.json" % composite_id)
a_path: str = os.path.join(working_path, "output", relpath,
"%s.json" % composite_id)
with open(e_path) as e_fh, open(a_path) as a_fh:
expected: Dict = json.load(e_fh)
actual: Dict = json.load(a_fh)
assert actual == expected
def _load_composite(self, base_dir: str, composite_id: str) -> Composite:
relpath: str = relpath_for(composite_id)
composite_path = os.path.join(base_dir, relpath,
"%s.json" % composite_id)
content: Dict
if not os.path.exists(composite_path):
content = {}
else:
with open(os.path.join(base_dir, relpath, "%s.json" %
composite_id)) as translate_file:
content = json.load(translate_file)
return Composite(self.schema, content, composite_id=composite_id)
def process_composites(self, composite_ids: List[str], origin_dir: str,
target_base_dir: str) -> None:
composites: Iterator[Composite] = self._as_composites(
composite_ids, origin_dir)
for filtered_composite in self.f_iter(composites):
c_id: str = filtered_composite.composite_id
relpath: str = relpath_for(c_id)
target_dir: str = os.path.join(target_base_dir, relpath)
os.makedirs(target_dir, exist_ok=True)
with open(os.path.join(target_dir, "%s.json" % c_id),
'w') as target_file:
json.dump(filtered_composite.content, target_file, indent=2)
def _as_composites(self, composite_ids: List[str],
origin_dir: str) -> Iterator[Composite]:
for composite_id in composite_ids:
relpath: str = relpath_for(composite_id)
with open(
os.path.join(origin_dir, relpath,
"%s.json" % composite_id)) as origin_file:
content: Dict = json.load(origin_file)
composite: Composite = Composite(self.schema,
content,
composite_id=composite_id)
yield composite
def extract(self, composite_ids: List[str]) -> CoverageFileExtractResult:
extract_result = CoverageFileExtractResult()
for composite_id in composite_ids:
logging.debug("Extracting data from composite %s", composite_id)
relpath: str = relpath_for(composite_id)
with open(os.path.join(self.origin_dir, relpath, "%s.json" % composite_id)) as origin_file:
content: Dict = json.load(origin_file)
composite: Composite = Composite(self.schema, content, composite_id=composite_id)
self._extract_temporal(composite, extract_result)
self._extract_immutable(composite, extract_result)
return extract_result
def extract(self, composite_ids: List[str]) -> List[List[Any]]:
result: List[List[Any]] = []
for composite_id in composite_ids:
relpath: str = relpath_for(composite_id)
with open(
os.path.join(self.origin_dir, relpath,
"%s.json" % composite_id)) as origin_file:
content: Dict = json.load(origin_file)
composite: Composite = Composite(self.schema,
content,
composite_id=composite_id)
result.extend(self._process_composite(composite))
return result
def process_composites(self, composite_ids: List[str],
origin_dir: str) -> List[Tuple[str, Optional[Any]]]:
"""For each composite_id: open a composite JSON file, deserialize it into a Composite object, then extract information to be used in
analysis."""
result: List[Tuple[str, Optional[Any]]] = []
for composite_id in composite_ids:
relpath: str = relpath_for(composite_id)
with open(
os.path.join(origin_dir, relpath,
"%s.json" % composite_id)) as origin_file:
content: Dict = json.load(origin_file)
composite: Composite = Composite(self.origin_schema,
content,
composite_id=composite_id)
result.append((composite_id, self.extract(composite)))
return result
def _merge_one(ein: str, primary_dir: str, secondary_dir: str,
target_dir: str) -> None:
relpath: str = relpath_for(ein)
primary_fn: str = os.path.join(primary_dir, relpath, "{}.json".format(ein))
secondary_fn: str = os.path.join(secondary_dir, relpath,
"{}.json".format(ein))
with open(primary_fn) as p_fh, open(secondary_fn) as s_fh:
primary_content: Dict = json.load(p_fh)
secondary_content: Dict = json.load(s_fh)
merged_content: Dict = merge_dicts(primary_content, secondary_content)
target_fn: str = os.path.join(target_dir, relpath, "{}.json".format(ein))
os.makedirs(os.path.dirname(target_fn), exist_ok=True)
with open(target_fn, "w") as t_fh:
json.dump(merged_content, t_fh, indent=2)
def test_relpath_for(composite_id: str, expected: str):
assert relpath_for(composite_id) == expected