def _run_task(scenario,
task_name,
expected_location,
output_dir: Optional[str] = None):
polytropos.actions.register_all()
conf = os.path.join(basepath, '../examples', scenario, 'conf')
data = os.path.join(basepath, '../examples', scenario, 'data')
with Context.build(conf, data, output_dir=output_dir) as context:
task = Task.build(context, task_name)
task.run()
actual_path = os.path.join(task.context.entities_output_dir,
task.target_data)
expected_path = os.path.join(task.context.entities_input_dir,
expected_location)
composite_ids: List = list(find_all_composites(expected_path))
assert list(find_all_composites(expected_path)) == composite_ids
for composite_id in composite_ids:
relpath: str = relpath_for(composite_id)
with open(
os.path.join(actual_path, relpath,
"%s.json" % composite_id)) as f:
with open(
os.path.join(expected_path, relpath,
"%s.json" % composite_id)) as g:
actual_data = json.load(f)
expected_data = json.load(g)
diff = Differ().compare(
json.dumps(actual_data, indent=4).split('\n'),
json.dumps(expected_data, indent=4).split('\n'))
assert compare(
actual_data,
expected_data), ('Diff: ' +
'\n'.join(line for line in diff))
def _run_task(scenario, task_name, expected_location):
polytropos.actions.register_all()
conf = os.path.join(basepath, '../examples', scenario, 'conf')
data = os.path.join(basepath, '../examples', scenario, 'data')
task = Task.build(conf, data, task_name)
task.run()
actual_path = os.path.join(
task.path_locator.entities_dir, task.target_data
)
expected_path = os.path.join(
task.path_locator.entities_dir, expected_location
)
assert os.listdir(actual_path) == os.listdir(expected_path)
for filename in os.listdir(actual_path):
with open(os.path.join(actual_path, filename), 'r') as f:
with open(os.path.join(expected_path, filename), 'r') as g:
actual_data = json.load(f)
expected_data = json.load(g)
diff = Differ().compare(
json.dumps(actual_data, indent=4).split('\n'),
json.dumps(expected_data, indent=4).split('\n')
)
assert compare(actual_data, expected_data), (
'Diff: ' + '\n'.join(line for line in diff)
)
def setup_and_teardown():
shutil.rmtree(WORKING_PATH, ignore_errors=True)
polytropos.actions.register_all()
data_dir: str = os.path.join(FIXTURE_PATH, "data")
conf_dir: str = os.path.join(FIXTURE_PATH, "conf")
task_dir: str = os.path.join(conf_dir, "tasks")
with Context.build(conf_dir, data_dir, output_dir=WORKING_PATH) as context:
for file in os.scandir(task_dir):
task_name: str = file.name[:-5]
task = Task.build(context, task_name)
task.run()
yield
shutil.rmtree(WORKING_PATH, ignore_errors=True)
def task(data_path: str, config_path: str, task_name: str,
input_path: Optional[str], output_path: Optional[str],
temp_path: Optional[str], no_cleanup: bool,
chunk_size: Optional[int]) -> None:
"""Perform a Polytropos task."""
with Context.build(config_path,
data_path,
input_dir=input_path,
output_dir=output_path,
temp_dir=temp_path,
no_cleanup=no_cleanup,
process_pool_chunk_size=chunk_size) as context:
task = Task.build(context, task_name)
task.run()
def test_task_consume(scenario, task_name, expected_location):
conf = os.path.join(BASEPATH, '../../examples', scenario, 'conf')
data = os.path.join(BASEPATH, '../../examples', scenario, 'data')
task = Task.build(conf, data, task_name)
task.run()
actual_path = os.path.join(task.path_locator.conf_dir, '../')
expected_path = os.path.join(task.path_locator.conf_dir, '../',
expected_location)
filename = task.steps[-1].filename
with open(os.path.join(actual_path, filename), 'r') as f:
with open(os.path.join(expected_path, filename), 'r') as g:
actual_data = f.read()
expected_data = g.read()
assert actual_data == expected_data
def _do_export(task_name: str) -> Tuple[str, str]:
polytropos.actions.register_all()
conf = os.path.join(BASEPATH, '../../examples', "s_5_tr_export",
'conf')
data = os.path.join(BASEPATH, '../../examples', "s_5_tr_export",
'data')
task = Task.build(conf, data, task_name)
task.run()
actual_path: str = os.path.join(task.path_locator.conf_dir, '../')
expected_path: str = os.path.join(task.path_locator.conf_dir, '../',
"expected")
filename: str = task.steps[-1].filename
actual_fn: str = os.path.join(actual_path, filename)
expected_fn: str = os.path.join(expected_path, filename)
return actual_fn, expected_fn
import logging from polytropos.ontology.task import Task logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) conf_path = "/dmz/github/anr-polytropos-config/" data_path = "/dmz/output/polytropos_tmp/" task = "origin_to_logical" task = Task.build(conf_path, data_path, task) task.run()
def task(data_path: str, config_path: str, task_name: str) -> None:
"""Perform a Polytropos task."""
task = Task.build(config_path, data_path, task_name)
task.run()