def _run_task(scenario, task_name, expected_location, output_dir: Optional[str] = None): polytropos.actions.register_all() conf = os.path.join(basepath, '../examples', scenario, 'conf') data = os.path.join(basepath, '../examples', scenario, 'data') with Context.build(conf, data, output_dir=output_dir) as context: task = Task.build(context, task_name) task.run() actual_path = os.path.join(task.context.entities_output_dir, task.target_data) expected_path = os.path.join(task.context.entities_input_dir, expected_location) composite_ids: List = list(find_all_composites(expected_path)) assert list(find_all_composites(expected_path)) == composite_ids for composite_id in composite_ids: relpath: str = relpath_for(composite_id) with open( os.path.join(actual_path, relpath, "%s.json" % composite_id)) as f: with open( os.path.join(expected_path, relpath, "%s.json" % composite_id)) as g: actual_data = json.load(f) expected_data = json.load(g) diff = Differ().compare( json.dumps(actual_data, indent=4).split('\n'), json.dumps(expected_data, indent=4).split('\n')) assert compare( actual_data, expected_data), ('Diff: ' + '\n'.join(line for line in diff))
def _do_copy_all(eins: Set[str], source_dir: str, target_dir: str) -> None: for ein in eins: relpath: str = relpath_for(ein) source: str = "{}/{}/{}.json".format(source_dir, relpath, ein) target: str = "{}/{}/{}.json".format(target_dir, relpath, ein) os.makedirs(os.path.dirname(target), exist_ok=True) shutil.copyfile(source, target)
def process_composite(self, origin_dir: str, base_target_dir: str, composite_id: str) -> None: try: relpath: str = relpath_for(composite_id) origin_filename: str = os.path.join(origin_dir, relpath, "%s.json" % composite_id) logging.debug("Evolving %s." % origin_filename) with open(origin_filename) as origin_file: content: Dict = json.load(origin_file) composite: Composite = Composite(self.schema, content, composite_id=composite_id) for change in self.changes: logging.debug('Applying change "%s" to %s.' % (change.__class__.__name__, origin_filename)) change(composite) target_dir: str = os.path.join(base_target_dir, relpath) os.makedirs(target_dir, exist_ok=True) target_filepath: str = os.path.join(target_dir, "%s.json" % composite_id) with open(target_filepath, 'w') as target_file: json.dump(composite.content, target_file, indent=2) except Exception as e: logging.error("Error processing composite %s during evolve step." % composite_id) traceback.print_exc() raise
def process_composites(self, composite_ids: List[str], origin_dir: str) -> List[Tuple[str, Any]]: result: List[Tuple[str, Any]] = [] for composite_id in composite_ids: relpath: str = relpath_for(composite_id) with open(os.path.join(origin_dir, relpath, "%s.json" % composite_id)) as origin_file: content: Dict = json.load(origin_file) composite: Composite = Composite(self.schema, content, composite_id=composite_id) result.append((composite_id, self.extract(composite))) return result
def write_composites(emissions: List[Tuple[str, Composite]], target_base_dir: str) -> None: for emission in emissions: composite_id, composite = emission relpath: str = relpath_for(composite_id) target_dir: str = os.path.join(target_base_dir, relpath) os.makedirs(target_dir, exist_ok=True) with open(os.path.join(target_dir, composite_id + '.json'), 'w') as target_file: json.dump(composite.content, target_file, indent=2)
def alter_and_write_composites(self, composite_ids: List[str], origin_dir: str, target_base_dir: str) -> None: for composite_id in composite_ids: relpath: str = relpath_for(composite_id) with open(os.path.join(origin_dir, relpath, "%s.json" % composite_id)) as origin_file: content: Dict = json.load(origin_file) composite: Composite = Composite(self.schema, content, composite_id=composite_id) self.alter(composite_id, composite) target_dir: str = os.path.join(target_base_dir, relpath) os.makedirs(target_dir, exist_ok=True) with open(os.path.join(target_dir, "%s.json" % composite_id), 'w') as target_file: json.dump(composite.content, target_file, indent=2)
def test_quantile_outcome(index: int): composite_id: str = "{:09}".format(index) relpath: str = relpath_for(composite_id) e_path: str = os.path.join(base_path, "expected", relpath, "%s.json" % composite_id) a_path: str = os.path.join(working_path, "output", relpath, "%s.json" % composite_id) with open(e_path) as e_fh, open(a_path) as a_fh: expected: Dict = json.load(e_fh) actual: Dict = json.load(a_fh) assert actual == expected
def _load_composite(self, base_dir: str, composite_id: str) -> Composite: relpath: str = relpath_for(composite_id) composite_path = os.path.join(base_dir, relpath, "%s.json" % composite_id) content: Dict if not os.path.exists(composite_path): content = {} else: with open(os.path.join(base_dir, relpath, "%s.json" % composite_id)) as translate_file: content = json.load(translate_file) return Composite(self.schema, content, composite_id=composite_id)
def process_composites(self, composite_ids: List[str], origin_dir: str, target_base_dir: str) -> None: composites: Iterator[Composite] = self._as_composites( composite_ids, origin_dir) for filtered_composite in self.f_iter(composites): c_id: str = filtered_composite.composite_id relpath: str = relpath_for(c_id) target_dir: str = os.path.join(target_base_dir, relpath) os.makedirs(target_dir, exist_ok=True) with open(os.path.join(target_dir, "%s.json" % c_id), 'w') as target_file: json.dump(filtered_composite.content, target_file, indent=2)
def _as_composites(self, composite_ids: List[str], origin_dir: str) -> Iterator[Composite]: for composite_id in composite_ids: relpath: str = relpath_for(composite_id) with open( os.path.join(origin_dir, relpath, "%s.json" % composite_id)) as origin_file: content: Dict = json.load(origin_file) composite: Composite = Composite(self.schema, content, composite_id=composite_id) yield composite
def extract(self, composite_ids: List[str]) -> CoverageFileExtractResult: extract_result = CoverageFileExtractResult() for composite_id in composite_ids: logging.debug("Extracting data from composite %s", composite_id) relpath: str = relpath_for(composite_id) with open(os.path.join(self.origin_dir, relpath, "%s.json" % composite_id)) as origin_file: content: Dict = json.load(origin_file) composite: Composite = Composite(self.schema, content, composite_id=composite_id) self._extract_temporal(composite, extract_result) self._extract_immutable(composite, extract_result) return extract_result
def extract(self, composite_ids: List[str]) -> List[List[Any]]: result: List[List[Any]] = [] for composite_id in composite_ids: relpath: str = relpath_for(composite_id) with open( os.path.join(self.origin_dir, relpath, "%s.json" % composite_id)) as origin_file: content: Dict = json.load(origin_file) composite: Composite = Composite(self.schema, content, composite_id=composite_id) result.extend(self._process_composite(composite)) return result
def process_composites(self, composite_ids: List[str], origin_dir: str) -> List[Tuple[str, Optional[Any]]]: """For each composite_id: open a composite JSON file, deserialize it into a Composite object, then extract information to be used in analysis.""" result: List[Tuple[str, Optional[Any]]] = [] for composite_id in composite_ids: relpath: str = relpath_for(composite_id) with open( os.path.join(origin_dir, relpath, "%s.json" % composite_id)) as origin_file: content: Dict = json.load(origin_file) composite: Composite = Composite(self.origin_schema, content, composite_id=composite_id) result.append((composite_id, self.extract(composite))) return result
def _merge_one(ein: str, primary_dir: str, secondary_dir: str, target_dir: str) -> None: relpath: str = relpath_for(ein) primary_fn: str = os.path.join(primary_dir, relpath, "{}.json".format(ein)) secondary_fn: str = os.path.join(secondary_dir, relpath, "{}.json".format(ein)) with open(primary_fn) as p_fh, open(secondary_fn) as s_fh: primary_content: Dict = json.load(p_fh) secondary_content: Dict = json.load(s_fh) merged_content: Dict = merge_dicts(primary_content, secondary_content) target_fn: str = os.path.join(target_dir, relpath, "{}.json".format(ein)) os.makedirs(os.path.dirname(target_fn), exist_ok=True) with open(target_fn, "w") as t_fh: json.dump(merged_content, t_fh, indent=2)
def test_relpath_for(composite_id: str, expected: str): assert relpath_for(composite_id) == expected