def test_json_processing(self):
with ScratchDir('.'):
os.environ['BEEP_PROCESSING_DIR'] = os.getcwd()
os.mkdir("data-share")
os.mkdir(os.path.join("data-share", "structure"))
# Create dummy json obj
json_obj = {
"mode": self.events_mode,
"file_list": [self.arbin_file, "garbage_file"],
'run_list': [0, 1],
"validity": ['valid', 'invalid']
}
json_string = json.dumps(json_obj)
# Get json output from method
json_output = process_file_list_from_json(json_string)
reloaded = json.loads(json_output)
# Actual tests here
# Ensure garbage file doesn't have output string
self.assertEqual(reloaded['invalid_file_list'][0], 'garbage_file')
# Ensure first is correct
loaded_processed_cycler_run = loadfn(reloaded['file_list'][0])
loaded_from_raw = RawCyclerRun.from_file(json_obj['file_list'][0]).to_processed_cycler_run()
self.assertTrue(np.all(loaded_processed_cycler_run.summary == loaded_from_raw.summary),
"Loaded processed cycler_run is not equal to that loaded from raw file")
# Test same functionality with json file
with ScratchDir('.'):
os.environ['BEEP_PROCESSING_DIR'] = os.getcwd()
os.mkdir("data-share")
os.mkdir(os.path.join("data-share", "structure"))
json_obj = {
"mode": self.events_mode,
"file_list": [self.arbin_file, "garbage_file"],
'run_list': [0, 1],
"validity": ['valid', 'invalid']
}
dumpfn(json_obj, "test.json")
# Get json output from method
json_output = process_file_list_from_json("test.json")
reloaded = json.loads(json_output)
# Actual tests here
# Ensure garbage file doesn't have output string
self.assertEqual(reloaded['invalid_file_list'][0], 'garbage_file')
# Ensure first is correct
loaded_processed_cycler_run = loadfn(reloaded['file_list'][0])
loaded_from_raw = RawCyclerRun.from_file(json_obj['file_list'][0]).to_processed_cycler_run()
self.assertTrue(np.all(loaded_processed_cycler_run.summary == loaded_from_raw.summary),
"Loaded processed cycler_run is not equal to that loaded from raw file")
def test_python(self):
"""Python script for end to end test"""
# Copy
mapped = collate.process_files_json()
rename_output = json.loads(mapped)
rename_output['mode'] = self.events_mode # mode run|test|events_off
rename_output['run_list'] = list(range(len(
rename_output['file_list'])))
mapped = json.dumps(rename_output)
# Validation
validated = validate.validate_file_list_from_json(mapped)
validated_output = json.loads(validated)
validated_output['mode'] = self.events_mode # mode run|test|events_off
validated_output['run_list'] = list(
range(len(validated_output['file_list'])))
validated = json.dumps(validated_output)
# Data structuring
structured = structure.process_file_list_from_json(validated)
structured_output = json.loads(structured)
structured_output[
'mode'] = self.events_mode # mode run|test|events_off
structured_output['run_list'] = list(
range(len(structured_output['file_list'])))
structured = json.dumps(structured_output)
# Featurization
featurized = featurize.process_file_list_from_json(structured)
featurized_output = json.loads(featurized)
featurized_output[
'mode'] = self.events_mode # mode run|test|events_off
featurized_output['run_list'] = list(
range(len(featurized_output['file_list'])))
featurized = json.dumps(featurized_output)
# Prediction
predictions = run_model.process_file_list_from_json(
featurized, model_dir=MODEL_DIR)
# Validate output files
self._check_result_file_validity()
def test_json_processing(self):
with ScratchDir("."):
os.environ["BEEP_PROCESSING_DIR"] = os.getcwd()
os.mkdir("data-share")
os.mkdir(os.path.join("data-share", "structure"))
# Create dummy json obj
json_obj = {
"mode": self.events_mode,
"file_list": [self.arbin_file, "garbage_file"],
"run_list": [0, 1],
"validity": ["valid", "invalid"],
}
json_string = json.dumps(json_obj)
# Get json output from method
json_output = process_file_list_from_json(json_string)
reloaded = json.loads(json_output)
# Actual tests here
# Ensure garbage file doesn't have output string
self.assertEqual(reloaded["invalid_file_list"][0], "garbage_file")
# Ensure first is correct
loaded_processed_cycler_run = loadfn(reloaded["file_list"][0])
loaded_from_raw = RawCyclerRun.from_file(
json_obj["file_list"][0]).to_processed_cycler_run()
self.assertTrue(
np.all(loaded_processed_cycler_run.summary ==
loaded_from_raw.summary),
"Loaded processed cycler_run is not equal to that loaded from raw file",
)
# Workflow output
output_file_path = Path(tempfile.gettempdir()) / "results.json"
self.assertTrue(output_file_path.exists())
output_json = json.loads(output_file_path.read_text())
self.assertEqual(reloaded["file_list"][0], output_json["filename"])
self.assertEqual(os.path.getsize(output_json["filename"]),
output_json["size"])
self.assertEqual(0, output_json["run_id"])
self.assertEqual("structuring", output_json["action"])
self.assertEqual("success", output_json["status"])
# Test same functionality with json file
with ScratchDir("."):
os.environ["BEEP_PROCESSING_DIR"] = os.getcwd()
os.mkdir("data-share")
os.mkdir(os.path.join("data-share", "structure"))
json_obj = {
"mode": self.events_mode,
"file_list": [self.arbin_file, "garbage_file"],
"run_list": [0, 1],
"validity": ["valid", "invalid"],
}
dumpfn(json_obj, "test.json")
# Get json output from method
json_output = process_file_list_from_json("test.json")
reloaded = json.loads(json_output)
# Actual tests here
# Ensure garbage file doesn't have output string
self.assertEqual(reloaded["invalid_file_list"][0], "garbage_file")
# Ensure first is correct
loaded_processed_cycler_run = loadfn(reloaded["file_list"][0])
loaded_from_raw = RawCyclerRun.from_file(
json_obj["file_list"][0]).to_processed_cycler_run()
self.assertTrue(
np.all(loaded_processed_cycler_run.summary ==
loaded_from_raw.summary),
"Loaded processed cycler_run is not equal to that loaded from raw file",
)
# Workflow output
output_file_path = Path(tempfile.gettempdir()) / "results.json"
self.assertTrue(output_file_path.exists())
output_json = json.loads(output_file_path.read_text())
self.assertEqual(reloaded["file_list"][0], output_json["filename"])
self.assertEqual(os.path.getsize(output_json["filename"]),
output_json["size"])
self.assertEqual(0, output_json["run_id"])
self.assertEqual("structuring", output_json["action"])
self.assertEqual("success", output_json["status"])
