def _run_task(scenario,
task_name,
expected_location,
output_dir: Optional[str] = None):
polytropos.actions.register_all()
conf = os.path.join(basepath, '../examples', scenario, 'conf')
data = os.path.join(basepath, '../examples', scenario, 'data')
with Context.build(conf, data, output_dir=output_dir) as context:
task = Task.build(context, task_name)
task.run()
actual_path = os.path.join(task.context.entities_output_dir,
task.target_data)
expected_path = os.path.join(task.context.entities_input_dir,
expected_location)
composite_ids: List = list(find_all_composites(expected_path))
assert list(find_all_composites(expected_path)) == composite_ids
for composite_id in composite_ids:
relpath: str = relpath_for(composite_id)
with open(
os.path.join(actual_path, relpath,
"%s.json" % composite_id)) as f:
with open(
os.path.join(expected_path, relpath,
"%s.json" % composite_id)) as g:
actual_data = json.load(f)
expected_data = json.load(g)
diff = Differ().compare(
json.dumps(actual_data, indent=4).split('\n'),
json.dumps(expected_data, indent=4).split('\n'))
assert compare(
actual_data,
expected_data), ('Diff: ' +
'\n'.join(line for line in diff))
def __call__(self, _origin_dir: str, _target_dir: Optional[str]) -> None:
self.before()
translate_composite_ids = set(find_all_composites(self.translate_dir))
trace_composite_ids = set(find_all_composites(self.trace_dir))
assert translate_composite_ids.issubset(trace_composite_ids)
coverage = SourceCoverageExtract(self.schema, self.translate_dir,
self.trace_dir, self.context.temp_dir)
per_composite_results: List[str] = list(
self.context.run_in_process_pool(coverage.extract,
list(trace_composite_ids)))
logging.info("Merging chunk results.")
self.consume(per_composite_results)
self.after()
def _get_ein_sets(self,
origin_dir: str) -> Tuple[Set[str], Set[str], Set[str]]:
primary_eins: Set[str] = {
ein
for ein in find_all_composites(origin_dir)
}
secondary_eins: Set[str] = {
ein
for ein in find_all_composites(self.secondary_dir)
}
primary_only = primary_eins - secondary_eins
secondary_only = secondary_eins - primary_eins
common = primary_eins.intersection(secondary_eins)
return primary_only, common, secondary_only
def __call__(self, origin_dir: str, target_dir: str) -> None:
composites_ids = list(find_all_composites(origin_dir))
logging.info(
"Spawning parallel processes to perform CompoundFilter operation.")
for _ in self.context.run_in_process_pool(self.process_composites,
composites_ids, origin_dir,
target_dir):
pass
def __call__(self, origin_dir: str, target_dir: str) -> None:
composites: List[str] = list(find_all_composites(origin_dir))
logging.info(
"Spawning parallel processes to perform Evolve process on all composites."
)
for _ in self.context.run_in_process_pool(self.process_composites,
composites, origin_dir,
target_dir):
pass
def __call__(self, origin_dir: str, target_dir: Optional[str]) -> None:
"""Generate the export file."""
self.before()
composite_ids: Iterable[str] = find_all_composites(origin_dir)
per_composite_results: Iterable[Any] = self.process_composites(
composite_ids, origin_dir)
self.consume(per_composite_results)
self.after()
def __call__(self, origin_dir: str, target_dir: str) -> None:
logging.info("Spawning parallel processes to extract data from each composite for global application.")
composite_ids: List[str] = list(find_all_composites(origin_dir))
self.analyze(
itertools.chain.from_iterable(self.context.run_in_process_pool(self.process_composites, composite_ids, origin_dir))
)
logging.info("Spawning parallel processes to apply global information to each composite.")
for _ in self.context.run_in_process_pool(self.alter_and_write_composites, composite_ids, origin_dir, target_dir):
pass
def __call__(self, origin_dir: str, target_dir: str) -> None:
composite_ids: List[str] = list(find_all_composites(origin_dir))
logging.info(
"Spawning parallel processes to extract data from each composite for aggregation."
)
per_composite_results: Iterable[Tuple[
str, Optional[Any]]] = itertools.chain.from_iterable(
self.context.run_in_process_pool(self.process_composites,
composite_ids, origin_dir))
self.analyze(per_composite_results)
logging.info(
"Spawning parallel processes to aggregate extracted data from each composite."
)
for _ in self.context.run_in_process_pool(write_composites,
list(self.emit()),
target_dir):
pass
def __call__(self, origin_dir: str, target_dir: str) -> None:
composites_ids = list(find_all_composites(origin_dir))
for _ in self.context.run_in_thread_pool(self.process_composites, composites_ids, origin_dir, target_dir):
pass
def test_find_all_composites(basepath):
fixture_basepath: str = os.path.join(basepath, "test_functional", "util", "path_fixtures")
expected: Set[str] = {"02164", "380476abcxyz", "person_3"}
actual: Set[str] = set(find_all_composites(fixture_basepath))
assert actual == expected