def test_feature_generation_list_to_json(self): processed_cycler_run_path = os.path.join(TEST_FILE_DIR, PROCESSED_CYCLER_FILE) with ScratchDir('.'): os.environ['BEEP_PROCESSING_DIR'] = os.getcwd() # Create dummy json obj json_obj = { "mode": self.events_mode, "file_list": [processed_cycler_run_path, processed_cycler_run_path], 'run_list': [0, 1] } json_string = json.dumps(json_obj) newjsonpaths = process_file_list_from_json( json_string, processed_dir=os.getcwd()) reloaded = json.loads(newjsonpaths) # Check that at least strings are output self.assertIsInstance(reloaded['file_list'][-1], str) # Ensure first is correct features_reloaded = loadfn(reloaded['file_list'][0]) self.assertIsInstance(features_reloaded, DeltaQFastCharge) self.assertEqual( features_reloaded.X.loc[0, 'nominal_capacity_by_median'], 1.0628421000000001)
def test_insufficient_data_file(self): processed_cycler_run_path = os.path.join( TEST_FILE_DIR, self.processed_cycler_file_insuf) with ScratchDir("."): os.environ["BEEP_PROCESSING_DIR"] = os.getcwd() json_obj = { "file_list": [processed_cycler_run_path], "run_list": [1], } json_string = json.dumps(json_obj) json_path = process_file_list_from_json(json_string, processed_dir=os.getcwd()) output_obj = json.loads(json_path) self.assertEqual(output_obj["result_list"][0], "incomplete") self.assertEqual( output_obj["message_list"][0]["comment"], "Insufficient or incorrect data for featurization", ) # Workflow output output_file_path = Path(tempfile.gettempdir()) / "results.json" self.assertTrue(output_file_path.exists()) output_data = json.loads(output_file_path.read_text()) output_json = output_data[0] self.assertEqual(output_obj["file_list"][0], output_json["filename"]) self.assertEqual(os.path.getsize(output_json["filename"]), output_json["size"]) self.assertEqual(1, output_json["run_id"]) self.assertEqual("featurizing", output_json["action"]) self.assertEqual("incomplete", output_json["status"])
def test_feature_generation_list_to_json(self): processed_cycler_run_path = os.path.join( TEST_FILE_DIR, 'PreDiag_000240_000227_truncated_structure.json') with ScratchDir('.'): os.environ['BEEP_PROCESSING_DIR'] = TEST_FILE_DIR #os.environ['BEEP_PROCESSING_DIR'] = os.getcwd() # Create dummy json obj json_obj = { "mode": self.events_mode, "file_list": [processed_cycler_run_path, processed_cycler_run_path], 'run_list': [0, 1] } json_string = json.dumps(json_obj) newjsonpaths = process_file_list_from_json( json_string, processed_dir=os.getcwd()) reloaded = json.loads(newjsonpaths) # Check that at least strings are output self.assertIsInstance(reloaded['file_list'][-1], str) # Ensure first is correct features_reloaded = loadfn(reloaded['file_list'][0]) self.assertIsInstance(features_reloaded, DeltaQFastCharge) self.assertEqual( features_reloaded.X.loc[0, 'nominal_capacity_by_median'], 0.07114775279999999) features_reloaded = loadfn(reloaded['file_list'][-1]) self.assertIsInstance(features_reloaded, DiagnosticProperties) self.assertListEqual( list(features_reloaded.X.iloc[2, :]), [143, 0.9753520623934744, 'rpt_0.2C', 'discharge_energy'])
def test_feature_generation_list_to_json(self): processed_cycler_run_path = os.path.join( TEST_FILE_DIR, "PreDiag_000240_000227_truncated_structure.json") with ScratchDir("."): os.environ["BEEP_PROCESSING_DIR"] = TEST_FILE_DIR # os.environ['BEEP_PROCESSING_DIR'] = os.getcwd() # Create dummy json obj json_obj = { "mode": self.events_mode, "file_list": [processed_cycler_run_path, processed_cycler_run_path], "run_list": [0, 1], } json_string = json.dumps(json_obj) newjsonpaths = process_file_list_from_json( json_string, processed_dir=os.getcwd()) reloaded = json.loads(newjsonpaths) # Check that at least strings are output self.assertIsInstance(reloaded["file_list"][-1], str) # Ensure first is correct features_reloaded = loadfn(reloaded["file_list"][4]) self.assertIsInstance(features_reloaded, DeltaQFastCharge) self.assertEqual( features_reloaded.X.loc[0, "nominal_capacity_by_median"], 0.07114775279999999, ) features_reloaded = loadfn(reloaded["file_list"][-1]) self.assertIsInstance(features_reloaded, DiagnosticProperties) self.assertListEqual( list(features_reloaded.X.iloc[2, :]), [ 141, 0.9859837086597274, 91.17758004259996, 2.578137278917377, 'reset', 'discharge_energy' ], ) # Workflow output output_file_path = Path(tempfile.gettempdir()) / "results.json" self.assertTrue(output_file_path.exists()) output_data = json.loads(output_file_path.read_text()) output_json = output_data[0] self.assertEqual(reloaded["file_list"][0], output_json["filename"]) self.assertEqual(os.path.getsize(output_json["filename"]), output_json["size"]) self.assertEqual(0, output_json["run_id"]) self.assertEqual("featurizing", output_json["action"]) self.assertEqual("success", output_json["status"])
def test_insufficient_data_file(self): processed_cycler_run_path = os.path.join(TEST_FILE_DIR, processed_cycler_file_insuf) json_obj = { "mode": self.events_mode, "file_list": [processed_cycler_run_path], 'run_list': [1] } json_string = json.dumps(json_obj) json_path = process_file_list_from_json(json_string, processed_dir=TEST_FILE_DIR) output_obj = json.loads(json_path) self.assertEqual(output_obj['result_list'][0], 'incomplete') self.assertEqual(output_obj['message_list'][0]['comment'], 'Insufficient data for featurization')
def test_insufficient_data_file(self): processed_cycler_run_path = os.path.join(TEST_FILE_DIR, PROCESSED_CYCLER_FILE_INSUF) with ScratchDir('.'): os.environ['BEEP_ROOT'] = os.getcwd() json_obj = { "mode": self.events_mode, "file_list": [processed_cycler_run_path], 'run_list': [1] } json_string = json.dumps(json_obj) json_path = process_file_list_from_json(json_string, processed_dir=os.getcwd()) output_obj = json.loads(json_path) self.assertEqual(output_obj['result_list'][0], 'incomplete') self.assertEqual(output_obj['message_list'][0]['comment'], 'Insufficient or incorrect data for featurization')
def test_python(self): """Python script for end to end test""" # Copy mapped = collate.process_files_json() rename_output = json.loads(mapped) rename_output['mode'] = self.events_mode # mode run|test|events_off rename_output['run_list'] = list(range(len( rename_output['file_list']))) mapped = json.dumps(rename_output) # Validation validated = validate.validate_file_list_from_json(mapped) validated_output = json.loads(validated) validated_output['mode'] = self.events_mode # mode run|test|events_off validated_output['run_list'] = list( range(len(validated_output['file_list']))) validated = json.dumps(validated_output) # Data structuring structured = structure.process_file_list_from_json(validated) structured_output = json.loads(structured) structured_output[ 'mode'] = self.events_mode # mode run|test|events_off structured_output['run_list'] = list( range(len(structured_output['file_list']))) structured = json.dumps(structured_output) # Featurization featurized = featurize.process_file_list_from_json(structured) featurized_output = json.loads(featurized) featurized_output[ 'mode'] = self.events_mode # mode run|test|events_off featurized_output['run_list'] = list( range(len(featurized_output['file_list']))) featurized = json.dumps(featurized_output) # Prediction predictions = run_model.process_file_list_from_json( featurized, model_dir=MODEL_DIR) # Validate output files self._check_result_file_validity()
def test_features_on_list(self): files = [ "PredictionDiagnostics_000102_0001B1_structure.json", "PredictionDiagnostics_000103_0001B3_structure.json", "PredictionDiagnostics_000114_00003C_structure.json", "PredictionDiagnostics_000117_00003E_structure.json", "PredictionDiagnostics_000120_000041_structure.json", "PredictionDiagnostics_000122_000043_structure.json", "PredictionDiagnostics_000124_000049_structure.json", "PredictionDiagnostics_000130_000044_structure.json", "PredictionDiagnostics_000133_00004D_structure (2).json", "PredictionDiagnostics_000136_00002D_structure (1).json", "PredictionDiagnostics_000139_000034_structure.json", "PredictionDiagnostics_000144_00002E_structure.json", "PredictionDiagnostics_000148_000038_structure.json", "PredictionDiagnostics_000150_00003B_structure.json", "PredictionDiagnostics_000156_000023_structure.json", "PredictionDiagnostics_000160_000251_structure.json", "PredictionDiagnostics_000163_000022_structure.json", "PredictionDiagnostics_000163_000022_structure.json", "PredictionDiagnostics_000164_000239_structure.json", "PredictionDiagnostics_000167_000255_structure.json", "PredictionDiagnostics_000168_000253_structure.json", "PredictionDiagnostics_000175_000247_structure.json", "PredictionDiagnostics_000178_00023B_structure.json", "PredictionDiagnostics_000181_00023A_structure.json", "PredictionDiagnostics_000184_000244_structure.json", "PredictionDiagnostics_000186_00024E_structure.json", "PredictionDiagnostics_000194_000242_structure.json", ] with ScratchDir("."): os.environ["BEEP_PROCESSING_DIR"] = TEST_FILE_DIR for file in files: pcycler_run_path = os.path.join(TEST_FILE_DIR, file) json_obj = { "file_list": [pcycler_run_path], "run_list": [0], } json_string = json.dumps(json_obj) newjsonpaths = process_file_list_from_json( json_string, processed_dir=os.getcwd())
def test_raw_to_features(self): os.environ["BEEP_PROCESSING_DIR"] = TEST_FILE_DIR download_s3_object(bucket=self.maccor_file_w_parameters_s3["bucket"], key=self.maccor_file_w_parameters_s3["key"], destination_path=self.maccor_file_w_parameters) with ScratchDir("."): os.environ["BEEP_PROCESSING_DIR"] = TEST_FILE_DIR # os.environ['BEEP_PROCESSING_DIR'] = os.getcwd() dp = MaccorDatapath.from_file(self.maccor_file_w_parameters) dp.autostructure() processed_run_path = os.path.join(TEST_FILE_DIR, "processed_diagnostic.json") # Dump to the structured file and check the file size dumpfn(dp, processed_run_path) # Create dummy json obj json_obj = { "file_list": [processed_run_path], "run_list": [0], } json_string = json.dumps(json_obj) newjsonpaths = process_file_list_from_json( json_string, processed_dir=os.getcwd()) reloaded = json.loads(newjsonpaths) import pprint pprint.pprint(reloaded) result_list = ['success'] * 7 self.assertEqual(reloaded['result_list'], result_list) rpt_df = loadfn(reloaded['file_list'][0]) self.assertEqual( np.round(rpt_df.X['m0_Amp_rpt_0.2C_1'].iloc[0], 6), 0.867371)
def test_feature_generation_list_to_json(self): processed_cycler_run_path = os.path.join(TEST_FILE_DIR, processed_cycler_file) # Create dummy json obj json_obj = { "mode": self.events_mode, "file_list": [processed_cycler_run_path, processed_cycler_run_path], 'run_list': [0, 1] } json_string = json.dumps(json_obj) newjsonpaths = process_file_list_from_json(json_string, processed_dir=TEST_FILE_DIR) reloaded = json.loads(newjsonpaths) # Check that at least strings are output self.assertIsInstance(reloaded['file_list'][-1], str) # Ensure first is correct predictor_reloaded = loadfn(reloaded['file_list'][0]) self.assertIsInstance(predictor_reloaded, DegradationPredictor) self.assertEqual(predictor_reloaded.nominal_capacity, 1.0628421000000001)