def test_kinesis_put_analyzing_event(self):
events = KinesisEvents(service='Testing', mode='test')
processed_paths_list = [os.path.join(TEST_FILE_DIR, "2017-12-04_4_65C-69per_6C_features.json")]
processed_run_list = [24]
processed_result_list = ["success"]
processed_message_list = [{'comment': '',
'error': ''}]
output_data = {"file_list": processed_paths_list,
"run_list": processed_run_list,
"result_list": processed_result_list,
"message_list": processed_message_list
}
response_valid = events.put_analyzing_event(output_data, 'featurizing', 'complete')
assert response_valid['ResponseMetadata']['HTTPStatusCode'] == 200
processed_paths_list = [os.path.join(TEST_FILE_DIR, "2017-12-04_4_65C-69per_6C_predictions.json")]
processed_run_list = [24]
processed_result_list = ["success"]
processed_message_list = [{'comment': '',
'error': ''}]
output_data = {"file_list": processed_paths_list,
"run_list": processed_run_list,
"result_list": processed_result_list,
"message_list": processed_message_list
}
response_valid = events.put_analyzing_event(output_data, 'predicting', 'complete')
assert response_valid['ResponseMetadata']['HTTPStatusCode'] == 200
def test_kinesis_put_analyzing_event(self):
events = KinesisEvents(service="Testing", mode="test")
processed_paths_list = [
os.path.join(TEST_FILE_DIR,
"2017-12-04_4_65C-69per_6C_features.json")
]
processed_run_list = [24]
processed_result_list = ["success"]
processed_message_list = [{"comment": "", "error": ""}]
output_data = {
"file_list": processed_paths_list,
"run_list": processed_run_list,
"result_list": processed_result_list,
"message_list": processed_message_list,
}
response_valid = events.put_analyzing_event(output_data, "featurizing",
"complete")
assert response_valid["ResponseMetadata"]["HTTPStatusCode"] == 200
processed_paths_list = [
os.path.join(TEST_FILE_DIR,
"2017-12-04_4_65C-69per_6C_predictions.json")
]
processed_run_list = [24]
processed_result_list = ["success"]
processed_message_list = [{"comment": "", "error": ""}]
output_data = {
"file_list": processed_paths_list,
"run_list": processed_run_list,
"result_list": processed_result_list,
"message_list": processed_message_list,
}
response_valid = events.put_analyzing_event(output_data, "predicting",
"complete")
assert response_valid["ResponseMetadata"]["HTTPStatusCode"] == 200
def process_file_list_from_json(file_list_json,
processed_dir='data-share/features/',
features_label='full_model',
predict_only=False,
prediction_type="multi",
predicted_quantity="cycle"):
"""
Function to take a json file containing processed cycler run file locations,
extract features, dump the processed file into a predetermined directory,
and return a jsonable dict of feature file locations.
Args:
file_list_json (str): json string or json filename corresponding
to a dictionary with a file_list attribute,
if this string ends with ".json", a json file is assumed
and loaded, otherwise interpreted as a json string.
processed_dir (str): location for processed cycler run output files
to be placed.
features_label (str): name of feature generation method.
predict_only (bool): whether to calculate predictions or not.
prediction_type (str): Single or multi-point predictions.
predicted_quantity (str): quantity being predicted - cycle or capacity.
Returns:
str: json string of feature files (with key "file_list").
"""
# Get file list and validity from json, if ends with .json,
# assume it's a file, if not assume it's a json string
if file_list_json.endswith(".json"):
file_list_data = loadfn(file_list_json)
else:
file_list_data = json.loads(file_list_json)
# Setup Events
events = KinesisEvents(service='DataAnalyzer', mode=file_list_data['mode'])
# Add root path to processed_dir
processed_dir = os.path.join(os.environ.get("BEEP_ROOT", "/"),
processed_dir)
file_list = file_list_data['file_list']
run_ids = file_list_data['run_list']
processed_run_list = []
processed_result_list = []
processed_message_list = []
processed_paths_list = []
required_cycle_num = 100 #for full model
for path, run_id in zip(file_list, run_ids):
logger.info('run_id=%s featurizing=%s', str(run_id), path, extra=s)
#check if there is enough data to try featurizing
if not len(loadfn(path).summary) > required_cycle_num:
logger.info("run_id=%s Insufficient data for featurization",
str(run_id),
extra=s)
processed_paths_list.append(path)
processed_run_list.append(run_id)
processed_result_list.append("incomplete")
processed_message_list.append({
'comment': 'Insufficient data for featurization',
'error': ''
})
else:
processed_data = DegradationPredictor.from_processed_cycler_run_file(
path,
features_label=features_label,
predict_only=predict_only,
prediction_type=prediction_type,
predicted_quantity=predicted_quantity)
new_filename = os.path.basename(path)
new_filename = scrub_underscore_suffix(new_filename)
# Append model_name along with "features" to demarcate
# different models when saving the feature vectors.
new_filename = add_suffix_to_filename(
new_filename,
"_" + features_label + "_" + prediction_type + "_features")
processed_path = os.path.join(processed_dir, new_filename)
processed_path = os.path.abspath(processed_path)
dumpfn(processed_data, processed_path)
processed_paths_list.append(processed_path)
processed_run_list.append(run_id)
processed_result_list.append("success")
processed_message_list.append({'comment': '', 'error': ''})
output_data = {
"file_list": processed_paths_list,
"run_list": processed_run_list,
"result_list": processed_result_list,
"message_list": processed_message_list
}
events.put_analyzing_event(output_data, 'featurizing', 'complete')
# Return jsonable file list
return json.dumps(output_data)
def process_file_list_from_json(file_list_json,
processed_dir='data-share/features/',
features_label='full_model',
predict_only=False,
prediction_type="multi",
predicted_quantity="cycle"):
"""
Function to take a json file containing processed cycler run file locations,
extract features, dump the processed file into a predetermined directory,
and return a jsonable dict of feature file locations.
Args:
file_list_json (str): json string or json filename corresponding
to a dictionary with a file_list attribute,
if this string ends with ".json", a json file is assumed
and loaded, otherwise interpreted as a json string.
processed_dir (str): location for processed cycler run output files
to be placed.
features_label (str): name of feature generation method.
predict_only (bool): whether to calculate predictions or not.
prediction_type (str): Single or multi-point predictions.
predicted_quantity (str): quantity being predicted - cycle or capacity.
Returns:
str: json string of feature files (with key "file_list").
"""
# Get file list and validity from json, if ends with .json,
# assume it's a file, if not assume it's a json string
if file_list_json.endswith(".json"):
file_list_data = loadfn(file_list_json)
else:
file_list_data = json.loads(file_list_json)
# Setup Events
events = KinesisEvents(service='DataAnalyzer', mode=file_list_data['mode'])
# Add root path to processed_dir
processed_dir = os.path.join(os.environ.get("BEEP_ROOT", "/"),
processed_dir)
file_list = file_list_data['file_list']
run_ids = file_list_data['run_list']
processed_run_list = []
processed_result_list = []
processed_message_list = []
processed_paths_list = []
for path, run_id in zip(file_list, run_ids):
logger.info('run_id=%s featurizing=%s', str(run_id), path, extra=s)
processed_cycler_run = loadfn(path)
featurizer_classes = [DeltaQFastCharge, TrajectoryFastCharge]
for featurizer_class in featurizer_classes:
featurizer = featurizer_class.from_run(path, processed_dir,
processed_cycler_run)
if featurizer:
dumpfn(featurizer, featurizer.name)
processed_paths_list.append(featurizer.name)
processed_run_list.append(run_id)
processed_result_list.append("success")
processed_message_list.append({'comment': '', 'error': ''})
logger.info('Successfully generated %s',
featurizer.name,
extra=s)
else:
processed_paths_list.append(path)
processed_run_list.append(run_id)
processed_result_list.append("incomplete")
processed_message_list.append({
'comment':
'Insufficient or incorrect data for featurization',
'error': ''
})
logger.info('Unable to featurize %s', path, extra=s)
output_data = {
"file_list": processed_paths_list,
"run_list": processed_run_list,
"result_list": processed_result_list,
"message_list": processed_message_list
}
events.put_analyzing_event(output_data, 'featurizing', 'complete')
# Return jsonable file list
return json.dumps(output_data)
def process_file_list_from_json(file_list_json, model_dir="/data-share/models/",
processed_dir='data-share/predictions/',
hyperparameters=None, model_name=None, predict_only=True):
"""
Function to take a json file containing featurized json locations,
train a new model if necessary, write files containing predictions into a
predetermined directory, and return a jsonable dict of prediction file locations
Args:
file_list_json (str): json string or json filename corresponding
to a dictionary with a file_list attribute,
if this string ends with ".json", a json file is assumed
and loaded, otherwise interpreted as a json string
model_dir (str): location where models are serialized and stored
processed_dir (str): location for processed cycler run output files
to be placed
hyperparameters (dict): dictionary of hyperparameters to optimize/use for training
model_name (str): name of feature generation method
predict_only (bool):
Returns:
str: json string of feature files (with key "feature_file_list").
"""
# Get file list and validity from json, if ends with .json,
# assume it's a file, if not assume it's a json string
if file_list_json.endswith(".json"):
file_list_data = loadfn(file_list_json)
else:
file_list_data = json.loads(file_list_json)
# Setup Events
events = KinesisEvents(service='DataAnalyzer', mode=file_list_data['mode'])
# Add BEEP_ROOT to processed_dir
processed_dir = os.path.join(os.environ.get("BEEP_ROOT", "/"),
processed_dir)
file_list = file_list_data['file_list']
run_ids = file_list_data['run_list']
processed_run_list = []
processed_result_list = []
processed_message_list = []
processed_paths_list = []
project_name = get_project_name_from_list(file_list)
if predict_only:
features = loadfn(file_list[0])
if model_name is None and project_name in DEFAULT_MODEL_PROJECTS:
if features.prediction_type == 'multi':
model = DegradationModel.from_serialized_model(model_dir=model_dir,
serialized_model='d3batt_multi_point.model')
else:
model = DegradationModel.from_serialized_model(model_dir=model_dir,
serialized_model='d3batt_single_point.model')
elif model_name is None and project_name not in DEFAULT_MODEL_PROJECTS:
output_data = {"file_list": [],
"run_list": [],
"result_list": [],
"message_list": []
}
events.put_analyzing_event(output_data, 'predicting', 'error')
# Return jsonable file list
return json.dumps(output_data)
else:
model = DegradationModel.from_serialized_model(model_dir=model_dir,
serialized_model=model_name)
else:
if hyperparameters is None:
hyperparameters = {'random_state': 1,
'test_size': .3,
'k_fold': 5,
'tol': 0.001,
'l1_ratio': [.1, .5, .7, .9, .95, .99, 1]
}
dataset_id = file_list_data.get("dataset_id")
model = DegradationModel.train(file_list_json, dataset_id=dataset_id,
model_type='linear', regularization_type='elasticnet',
model_name=model_name, hyperparameters=hyperparameters)
logger.warning('fitting=%s dataset=%s', model.name, str(dataset_id), extra=s)
for path, run_id in zip(file_list, run_ids):
logger.info('model=%s run_id=%s predicting=%s', model.name, str(run_id), path, extra=s)
features = loadfn(path)
prediction = model.predict(features)
prediction_dict = model.prediction_to_dict(prediction, features.nominal_capacity)
new_filename = os.path.basename(path)
new_filename = scrub_underscore_suffix(new_filename)
new_filename = add_suffix_to_filename(new_filename, "_predictions")
processed_path = os.path.join(processed_dir, new_filename)
processed_path = os.path.abspath(processed_path)
dumpfn(prediction_dict, processed_path)
# Append file loc to list to be returned
processed_paths_list.append(processed_path)
processed_run_list.append(run_id)
processed_result_list.append("success")
processed_message_list.append({'comment': '',
'error': ''})
output_data = {"file_list": processed_paths_list,
"run_list": processed_run_list,
"result_list": processed_result_list,
"message_list": processed_message_list
}
events.put_analyzing_event(output_data, 'predicting', 'complete')
# Return jsonable file list
return json.dumps(output_data)