def test_json_processing(self): with ScratchDir('.'): os.environ['BEEP_PROCESSING_DIR'] = os.getcwd() os.mkdir("data-share") os.mkdir(os.path.join("data-share", "structure")) # Create dummy json obj json_obj = { "mode": self.events_mode, "file_list": [self.arbin_file, "garbage_file"], 'run_list': [0, 1], "validity": ['valid', 'invalid'] } json_string = json.dumps(json_obj) # Get json output from method json_output = process_file_list_from_json(json_string) reloaded = json.loads(json_output) # Actual tests here # Ensure garbage file doesn't have output string self.assertEqual(reloaded['invalid_file_list'][0], 'garbage_file') # Ensure first is correct loaded_processed_cycler_run = loadfn(reloaded['file_list'][0]) loaded_from_raw = RawCyclerRun.from_file(json_obj['file_list'][0]).to_processed_cycler_run() self.assertTrue(np.all(loaded_processed_cycler_run.summary == loaded_from_raw.summary), "Loaded processed cycler_run is not equal to that loaded from raw file") # Test same functionality with json file with ScratchDir('.'): os.environ['BEEP_PROCESSING_DIR'] = os.getcwd() os.mkdir("data-share") os.mkdir(os.path.join("data-share", "structure")) json_obj = { "mode": self.events_mode, "file_list": [self.arbin_file, "garbage_file"], 'run_list': [0, 1], "validity": ['valid', 'invalid'] } dumpfn(json_obj, "test.json") # Get json output from method json_output = process_file_list_from_json("test.json") reloaded = json.loads(json_output) # Actual tests here # Ensure garbage file doesn't have output string self.assertEqual(reloaded['invalid_file_list'][0], 'garbage_file') # Ensure first is correct loaded_processed_cycler_run = loadfn(reloaded['file_list'][0]) loaded_from_raw = RawCyclerRun.from_file(json_obj['file_list'][0]).to_processed_cycler_run() self.assertTrue(np.all(loaded_processed_cycler_run.summary == loaded_from_raw.summary), "Loaded processed cycler_run is not equal to that loaded from raw file")
def test_python(self): """Python script for end to end test""" # Copy mapped = collate.process_files_json() rename_output = json.loads(mapped) rename_output['mode'] = self.events_mode # mode run|test|events_off rename_output['run_list'] = list(range(len( rename_output['file_list']))) mapped = json.dumps(rename_output) # Validation validated = validate.validate_file_list_from_json(mapped) validated_output = json.loads(validated) validated_output['mode'] = self.events_mode # mode run|test|events_off validated_output['run_list'] = list( range(len(validated_output['file_list']))) validated = json.dumps(validated_output) # Data structuring structured = structure.process_file_list_from_json(validated) structured_output = json.loads(structured) structured_output[ 'mode'] = self.events_mode # mode run|test|events_off structured_output['run_list'] = list( range(len(structured_output['file_list']))) structured = json.dumps(structured_output) # Featurization featurized = featurize.process_file_list_from_json(structured) featurized_output = json.loads(featurized) featurized_output[ 'mode'] = self.events_mode # mode run|test|events_off featurized_output['run_list'] = list( range(len(featurized_output['file_list']))) featurized = json.dumps(featurized_output) # Prediction predictions = run_model.process_file_list_from_json( featurized, model_dir=MODEL_DIR) # Validate output files self._check_result_file_validity()
def test_json_processing(self): with ScratchDir("."): os.environ["BEEP_PROCESSING_DIR"] = os.getcwd() os.mkdir("data-share") os.mkdir(os.path.join("data-share", "structure")) # Create dummy json obj json_obj = { "mode": self.events_mode, "file_list": [self.arbin_file, "garbage_file"], "run_list": [0, 1], "validity": ["valid", "invalid"], } json_string = json.dumps(json_obj) # Get json output from method json_output = process_file_list_from_json(json_string) reloaded = json.loads(json_output) # Actual tests here # Ensure garbage file doesn't have output string self.assertEqual(reloaded["invalid_file_list"][0], "garbage_file") # Ensure first is correct loaded_processed_cycler_run = loadfn(reloaded["file_list"][0]) loaded_from_raw = RawCyclerRun.from_file( json_obj["file_list"][0]).to_processed_cycler_run() self.assertTrue( np.all(loaded_processed_cycler_run.summary == loaded_from_raw.summary), "Loaded processed cycler_run is not equal to that loaded from raw file", ) # Workflow output output_file_path = Path(tempfile.gettempdir()) / "results.json" self.assertTrue(output_file_path.exists()) output_json = json.loads(output_file_path.read_text()) self.assertEqual(reloaded["file_list"][0], output_json["filename"]) self.assertEqual(os.path.getsize(output_json["filename"]), output_json["size"]) self.assertEqual(0, output_json["run_id"]) self.assertEqual("structuring", output_json["action"]) self.assertEqual("success", output_json["status"]) # Test same functionality with json file with ScratchDir("."): os.environ["BEEP_PROCESSING_DIR"] = os.getcwd() os.mkdir("data-share") os.mkdir(os.path.join("data-share", "structure")) json_obj = { "mode": self.events_mode, "file_list": [self.arbin_file, "garbage_file"], "run_list": [0, 1], "validity": ["valid", "invalid"], } dumpfn(json_obj, "test.json") # Get json output from method json_output = process_file_list_from_json("test.json") reloaded = json.loads(json_output) # Actual tests here # Ensure garbage file doesn't have output string self.assertEqual(reloaded["invalid_file_list"][0], "garbage_file") # Ensure first is correct loaded_processed_cycler_run = loadfn(reloaded["file_list"][0]) loaded_from_raw = RawCyclerRun.from_file( json_obj["file_list"][0]).to_processed_cycler_run() self.assertTrue( np.all(loaded_processed_cycler_run.summary == loaded_from_raw.summary), "Loaded processed cycler_run is not equal to that loaded from raw file", ) # Workflow output output_file_path = Path(tempfile.gettempdir()) / "results.json" self.assertTrue(output_file_path.exists()) output_json = json.loads(output_file_path.read_text()) self.assertEqual(reloaded["file_list"][0], output_json["filename"]) self.assertEqual(os.path.getsize(output_json["filename"]), output_json["size"]) self.assertEqual(0, output_json["run_id"]) self.assertEqual("structuring", output_json["action"]) self.assertEqual("success", output_json["status"])