def TECA(pred_list, ground_list, mains): #pred_list and ground_list will contain the meters for each appliace listSize = len(pred_list) total_diff_sum = 0.0 total_aggr_sum = 0.0 sum_samples = 0.0 for i in range(listSize): pred = pred_list[i] ground = ground_list[i] aligned_meters_pg = align_two_meters(pred, ground) for chunk in aligned_meters_pg: chunk.fillna(0, inplace=True) sum_samples += len(chunk) total_diff_sum += sum(abs((chunk.iloc[:, 0]) - chunk.iloc[:, 1])) if(i==0): #count the total timestamps sum_samples += len(chunk) #Aggregate sum aligned_meters_pm = align_two_meters(pred, mains) for chunk_mains in aligned_meters_pm: chunk_mains.fillna(0, inplace=True) total_aggr_sum += sum(chunk_mains.iloc[:, 1]) if sum_samples == 0: return None else: return 1 - (1/2)*(total_diff_sum/total_aggr_sum)
def recall_precision_accuracy_f1(pred, ground): aligned_meters = align_two_meters(pred, ground) threshold = ground.on_power_threshold() chunk_results = [] sum_samples = 0.0 for chunk in aligned_meters: sum_samples += len(chunk) pr = np.array([0 if (p) < threshold else 1 for p in chunk.iloc[:, 0]]) gr = np.array([0 if p < threshold else 1 for p in chunk.iloc[:, 1]]) tp, tn, fp, fn = tp_tn_fp_fn(pr, gr) p = sum(pr) n = len(pr) - p chunk_results.append([tp, tn, fp, fn, p, n]) if sum_samples == 0: return None else: [tp, tn, fp, fn, p, n] = np.sum(chunk_results, axis=0) res_recall = recall(tp, fn) res_precision = precision(tp, fp) res_f1 = f1(res_precision, res_recall) res_accuracy = accuracy(tp, tn, p, n) return (res_recall, res_precision, res_accuracy, res_f1)
def recall_precision_accuracy_f1(pred, ground, pr_threshold = None, gr_threshold = None): aligned_meters = align_two_meters(pred, ground) if pr_threshold == None: pr_threshold = ground.on_power_threshold() #If not TH was provided, both sets get TH from ground if gr_threshold == None: gr_threshold = ground.on_power_threshold() # If not TH was provided, both sets get TH from ground print('True threshold: ',gr_threshold) print('Pred threshold: ', pr_threshold) chunk_results = [] sum_samples = 0.0 for chunk in aligned_meters: sum_samples += len(chunk) pr = chunk.iloc[:,0].fillna(0) #method='bfill' gr = chunk.iloc[:,1].fillna(0) pr = np.array([0 if (p)<pr_threshold else 1 for p in pr]) gr = np.array([0 if p<gr_threshold else 1 for p in gr]) tp, tn, fp, fn = tp_tn_fp_fn(pr,gr) p = sum(pr) n = len(pr) - p chunk_results.append([tp,tn,fp,fn,p,n]) if sum_samples == 0: return None else: [tp,tn,fp,fn,p,n] = np.sum(chunk_results, axis=0) res_recall = recall(tp,fn) res_precision = precision(tp,fp) res_f1 = f1(res_precision,res_recall) res_accuracy = accuracy(tp,tn,p,n) return (res_recall,res_precision,res_accuracy,res_f1)
def rms_error_power(predictions, ground_truth): '''Compute RMS error in assigned power .. math:: error^{(n)} = \\sqrt{ \\frac{1}{T} \\sum_t{ \\left ( y_t - \\hat{y}_t \\right )^2 } } Parameters ---------- predictions, ground_truth : nilmtk.MeterGroup Returns ------- error : pd.Series Each index is an meter instance int (or tuple for MeterGroups). Each value is the RMS error in predicted power for that appliance. ''' error = {} both_sets_of_meters = iterate_through_submeters_of_two_metergroups( predictions, ground_truth) for pred_meter, ground_truth_meter in both_sets_of_meters: sum_of_squared_diff = 0.0 n_samples = 0 for aligned_meters_chunk in align_two_meters(pred_meter, ground_truth_meter): diff = aligned_meters_chunk.icol(0) - aligned_meters_chunk.icol(1) diff.dropna(inplace=True) sum_of_squared_diff += (diff ** 2).sum() n_samples += len(diff) error[pred_meter.instance()] = math.sqrt(sum_of_squared_diff / n_samples) return pd.Series(error)
def mean_normalized_error_power(predictions, ground_truth): '''Compute mean normalized error in assigned power .. math:: error^{(n)} = \\frac { \\sum_t {\\left | y_t^{(n)} - \\hat{y}_t^{(n)} \\right |} } { \\sum_t y_t^{(n)} } Parameters ---------- predictions, ground_truth : nilmtk.MeterGroup Returns ------- mne : pd.Series Each index is an meter instance int (or tuple for MeterGroups). Each value is the MNE for that appliance. ''' mne = {} both_sets_of_meters = iterate_through_submeters_of_two_metergroups( predictions, ground_truth) for pred_meter, ground_truth_meter in both_sets_of_meters: total_abs_diff = 0.0 sum_of_ground_truth_power = 0.0 for aligned_meters_chunk in align_two_meters(pred_meter, ground_truth_meter): diff = aligned_meters_chunk.icol(0) - aligned_meters_chunk.icol(1) total_abs_diff += sum(abs(diff.dropna())) sum_of_ground_truth_power += aligned_meters_chunk.icol(1).sum() mne[pred_meter.instance()] = total_abs_diff / sum_of_ground_truth_power return pd.Series(mne)
def FTE_func(predictions, ground_truth): both_sets_of_meters = iterate_through_submeters_of_two_metergroups( predictions, ground_truth) total_pred_list = [] total_gt_list = [] fraction_pred = [] fraction_gt = [] fraction_min = [] total_pred = 0.0 total_gt = 0.0 for pred_meter, ground_truth_meter in both_sets_of_meters: total_app_pred = 0.0 total_app_gt = 0.0 for aligned_meters_chunk in align_two_meters(pred_meter, ground_truth_meter): total_pred += aligned_meters_chunk.icol(0).sum() total_gt += aligned_meters_chunk.icol(1).sum() total_app_pred += aligned_meters_chunk.icol(0).sum() total_app_gt += aligned_meters_chunk.icol(1).sum() total_pred_list.append(total_app_pred) total_gt_list.append(total_app_gt) fraction_gt = np.array(total_gt_list) / total_gt fraction_pred = np.array(total_pred_list) / total_pred for i in range(len(fraction_pred)): fraction_min.append(min(fraction_pred[i], fraction_gt[i])) return np.array(fraction_min).sum()
def getStackTrainGenerators(b, train_meterRef): trainXGen_list = [] for path in dsPathsList[b]: train = DataSet(path) train_elec = train.buildings[b].elec train_meter = train_elec.submeters()[meter_key] # print('Stack train: ', train_meter.get_timeframe().start.date(), " - ", train_meter.get_timeframe().end.date()) # Align the 'train_meterRef' with the X file (smaller). it's also a way to read the X meters chunk-by-chunk aligned_meters = align_two_meters(train_meterRef, train_meter) trainXGen_list.append(aligned_meters) return trainXGen_list
def getMeterTargetGenerator(b, train_meterRef): trainYDS = DataSet(dsPathY) print('Stack train: ', train_meterRef.get_timeframe().start.date(), " - ", train_meterRef.get_timeframe().end.date()) # trainYDS.set_window(start=train_meter.get_timeframe().start.date(), end=train_meter.get_timeframe().end.date()) trainY_elec = trainYDS.buildings[b].elec trainY_meter = trainY_elec.submeters()[meter_key] # print(trainY_meter.sample_period()) trainYGen = align_two_meters(train_meterRef, trainY_meter) return trainYGen
def root_mean_squared_error(pred, ground): aligned_meters = align_two_meters(pred, ground) total_sum = 0.0 sum_samples = 0.0 for chunk in aligned_meters: chunk.fillna(0, inplace=True) sum_samples += len(chunk) total_sum += sum(pow((chunk.iloc[:,0] - chunk.iloc[:,1]),2)) if sum_samples == 0: return None else: return pow(total_sum / sum_samples, 0.5)
def mean_absolute_error(pred, ground): aligned_meters = align_two_meters(pred, ground) total_sum = 0.0 sum_samples = 0.0 for chunk in aligned_meters: chunk.fillna(0, inplace=True) sum_samples += len(chunk) total_sum += sum(abs((chunk.iloc[:, 0]) - chunk.iloc[:, 1])) if sum_samples == 0: return None else: return total_sum / sum_samples
def RMSE(pred, ground): aligned_meters = align_two_meters(pred, ground) total_sum = 0.0 sum_samples = 0.0 for chunk in aligned_meters: chunk.fillna(0, inplace=True) sum_samples += len(chunk) total_sum += sum(np.power((chunk.iloc[:,0]) - chunk.iloc[:,1],2)) if sum_samples == 0: return None else: return math.sqrt(total_sum / sum_samples)
def f1_score(predictions, ground_truth): '''Compute F1 scores. .. math:: F_{score}^{(n)} = \\frac {2 * Precision * Recall} {Precision + Recall} Parameters ---------- predictions, ground_truth : nilmtk.MeterGroup Returns ------- f1_scores : pd.Series Each index is an meter instance int (or tuple for MeterGroups). Each value is the F1 score for that appliance. If there are multiple chunks then the value is the weighted mean of the F1 score for each chunk. ''' # If we import sklearn at top of file then sphinx breaks. from sklearn.metrics import f1_score as sklearn_f1_score # sklearn produces lots of DepreciationWarnings with PyTables import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) # align_two_meters does not work!! f1_scores = {} both_sets_of_meters = iterate_through_submeters_of_two_metergroups( predictions, ground_truth) for pred_meter, ground_truth_meter in both_sets_of_meters: scores_for_meter = pd.DataFrame(columns=['score', 'n_samples']) aligned_states_chunks = align_two_meters(pred_meter, ground_truth_meter, 'when_on') for aligned_states_chunk in aligned_states_chunks: aligned_states_chunk.dropna(inplace=True) aligned_states_chunk = aligned_states_chunk.astype(int) score = sklearn_f1_score(aligned_states_chunk.icol(0), aligned_states_chunk.icol(1)) scores_for_meter = scores_for_meter.append( {'score': score, 'n_samples': len(aligned_states_chunk)}, ignore_index=True) # Calculate weighted mean tot_samples = scores_for_meter['n_samples'].sum() scores_for_meter['proportion'] = (scores_for_meter['n_samples'] / tot_samples) avg_score = (scores_for_meter['score'] * scores_for_meter['proportion']).sum() f1_scores[pred_meter.instance()] = avg_score return pd.Series(data=f1_scores.values(), index=f1_scores.keys(), dtype=np.float32)
def acc_score(predictions, ground_truth): '''Compute F1 scores. .. math:: F_{score}^{(n)} = \\frac {2 * Precision * Recall} {Precision + Recall} Parameters ---------- predictions, ground_truth : nilmtk.MeterGroup Returns ------- f1_scores : pd.Series Each index is an meter instance int (or tuple for MeterGroups). Each value is the F1 score for that appliance. If there are multiple chunks then the value is the weighted mean of the F1 score for each chunk. ''' # If we import sklearn at top of file then sphinx breaks. from sklearn.metrics import accuracy_score as sklearn_acc_score # sklearn produces lots of DepreciationWarnings with PyTables import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) f1_scores = {} both_sets_of_meters = iterate_through_submeters_of_two_metergroups( predictions, ground_truth) for pred_meter, ground_truth_meter in both_sets_of_meters: scores_for_meter = pd.DataFrame(columns=['score', 'n_samples']) for aligned_states_chunk in align_two_meters(pred_meter, ground_truth_meter, 'when_on'): aligned_states_chunk.dropna(inplace=True) aligned_states_chunk = aligned_states_chunk.astype(int) score = sklearn_acc_score(aligned_states_chunk.icol(0), aligned_states_chunk.icol(1)) scores_for_meter = scores_for_meter.append( {'score': score, 'n_samples': len(aligned_states_chunk)}, ignore_index=True) # Calculate weighted mean tot_samples = scores_for_meter['n_samples'].sum() scores_for_meter['proportion'] = (scores_for_meter['n_samples'] / tot_samples) avg_score = (scores_for_meter['score'] * scores_for_meter['proportion']).sum() f1_scores[pred_meter.instance()] = avg_score return pd.Series(f1_scores)
def nad(pred, ground): aligned_meters = align_two_meters(pred, ground) nominator = 0.0 denominator = 0.0 sum_samples = 0.0 for chunk in aligned_meters: chunk.fillna(0, inplace=True) sum_samples += len(chunk) nominator += sum(abs((chunk.iloc[:, 0]) - chunk.iloc[:, 1])) denominator += sum(abs(chunk.iloc[:, 1])) if sum_samples == 0: return None else: return np.sqrt(nominator / denominator)
def relative_error_total_energy(pred, ground): aligned_meters = align_two_meters(pred, ground) chunk_results = [] sum_samples = 0.0 for chunk in aligned_meters: chunk.fillna(0, inplace=True) sum_samples += len(chunk) E_pred = sum(chunk.iloc[:, 0]) E_ground = sum(chunk.iloc[:, 1]) chunk_results.append([E_pred, E_ground]) if sum_samples == 0: return None else: [E_pred, E_ground] = np.sum(chunk_results, axis=0) return abs(E_pred - E_ground) / float(max(E_pred, E_ground))
def total_disag_err(predictions, ground_truth): #only iterate for the instance in the prediction/ elecmeter with lesser instance both_sets_of_meters = iterate_through_submeters_of_two_metergroups( predictions, ground_truth) # additional of total variable total_diff = 0.0 total_pred = 0.0 total_gt = 0.0 for pred_meter, ground_truth_meter in both_sets_of_meters: for aligned_meters_chunk in align_two_meters(pred_meter, ground_truth_meter): diff = aligned_meters_chunk.icol(0) - aligned_meters_chunk.icol(1) total_pred += aligned_meters_chunk.icol(0).sum() total_diff += sum(abs(diff.dropna())) total_gt += aligned_meters_chunk.icol(1).sum() return float(total_diff) / float(total_gt)
def disaggregation_accuracy(pred, ground): aligned_meters = align_two_meters(pred, ground) nominator = 0.0 denominator = 0.0 sum_samples = 0.0 for chunk in aligned_meters: chunk.fillna(0, inplace=True) sum_samples += len(chunk) nominator += np.linalg.norm(chunk.iloc[:, 0] - chunk.iloc[:, 1], ord=1) denominator += np.linalg.norm(chunk.iloc[:, 0], ord=1) if sum_samples == 0: return None else: return 1 - (float(nominator) / (2 * denominator))
def create_trainset(meter, mains, train_size, window_size): all_x_train = np.empty((train_size,window_size,1)) all_y_train = np.empty((train_size,)) low_index = 0 gen = align_two_meters(meter, mains) for chunk in gen: if (chunk.shape[0]<3000): continue chunk.fillna(method='ffill', inplace=True) X_batch, Y_batch = gen_batch(chunk.iloc[:,1], chunk.iloc[:,0], chunk.shape[0]-window_size, 0, window_size) high_index = min(len(X_batch), train_size-low_index) all_x_train[low_index:high_index+low_index] = X_batch[:high_index] all_y_train[low_index:high_index+low_index] = Y_batch[:high_index] low_index = high_index+low_index if (low_index == train_size): break return all_x_train, all_y_train
def root_mean_squared_error(pred, ground): aligned_meters = align_two_meters(pred, ground) total_sum = 0.0 sum_samples = 0.0 for chunk in aligned_meters: chunk.fillna(0, inplace=True) sum_samples += len(chunk) total_sum += sum(pow((chunk.iloc[:,0] - chunk.iloc[:,1]),2)) if sum_samples == 0: return None else: return pow(total_sum / sum_samples, 0.5) def recall(tp,fn): return tp/float(tp+fn) def precision(tp,fp): return tp/float(tp+fp) def f1(prec,rec): return 2 * (prec*rec) / float(prec+rec) def accuracy(tp, tn, p, n): return (tp + tn) / float(p + n)
def runExperiment(experiment: experimentInfo, metricsResFileName, clearMetricsFile): dsPathsList_Test = experiment.dsList outFileName = experiment.outName test_building = experiment.building meter_key = experiment.meter_key pathOrigDS = experiment.pathOrigDS meterTH = experiment.meterTH print('House ', test_building) # Load a "complete" dataset to have the test's timerange test = DataSet(dsPathsList_Test[0]) test_elec = test.buildings[test_building].elec testRef_meter = test_elec.submeters( )[meter_key] # will be used as reference to align all meters based on this # Align every test meter with testRef_meter as master test_series_list = [] for path in dsPathsList_Test: test = DataSet(path) test_elec = test.buildings[test_building].elec test_meter = test_elec.submeters()[meter_key] # print('Stack test: ', test_meter.get_timeframe().start.date(), " - ", test_meter.get_timeframe().end.date()) aligned_meters = align_two_meters(testRef_meter, test_meter) test_series_list.append(aligned_meters) # Init vars for the output MIN_CHUNK_LENGTH = 300 # Depends on the basemodels of the ensemble timeframes = [] building_path = '/building{}'.format(test_meter.building()) mains_data_location = building_path + '/elec/meter1' data_is_available = False disag_filename = outFileName output_datastore = HDFDataStore(disag_filename, 'w') run = True chunkDataForOutput = None # -- Used to hold necessary data for saving the results using NILMTK (e.g. timeframes). # -- (in case where chunks have different size (not in current implementation), must use the chunk whose windowsSize is the least (to have all the data)) while run: try: testX = [] columnInd = 0 # Get Next chunk of each series for testXGen in test_series_list: chunkALL = next(testXGen) chunk = chunkALL[ 'slave'] # slave is the meter needed (master is only for aligning) chunk.fillna(0, inplace=True) if (columnInd == 0): chunkDataForOutput = chunk # Use 1st found chunk for it's metadata if (testX == []): testX = np.zeros( [len(chunk), len(test_series_list)] ) # Initialize the array that will hold all of the series as columns testX[:, columnInd] = chunk[:] columnInd += 1 testX = scaler.transform(testX) except: run = False break if len(chunkDataForOutput) < MIN_CHUNK_LENGTH: continue # print("New sensible chunk: {}".format(len(chunk))) startTime = chunkDataForOutput.index[0] endTime = chunkDataForOutput.index[ -1] # chunkDataForOutput.shape[0] - 1 # print('Start:',startTime,'End:',endTime) timeframes.append(TimeFrame( startTime, endTime)) #info needed for output for use with NILMTK measurement = ('power', 'active') pred = clf.predict(testX) column = pd.Series(pred, index=chunkDataForOutput.index, name=0) appliance_powers_dict = {} appliance_powers_dict[0] = column appliance_power = pd.DataFrame(appliance_powers_dict) appliance_power[appliance_power < 0] = 0 # Append prediction to output data_is_available = True cols = pd.MultiIndex.from_tuples([measurement]) meter_instance = test_meter.instance() df = pd.DataFrame(appliance_power.values, index=appliance_power.index, columns=cols, dtype="float32") key = '{}/elec/meter{}'.format(building_path, meter_instance) output_datastore.append(key, df) # Append aggregate data to output mains_df = pd.DataFrame(chunkDataForOutput, columns=cols, dtype="float32") # Note (For later): not 100% right. Should be mains. But it won't be used anywhere, so it doesn't matter in this case output_datastore.append(key=mains_data_location, value=mains_df) # Save metadata to output if data_is_available: disagr = Disaggregator() disagr.MODEL_NAME = 'Stacked model' disagr._save_metadata_for_disaggregation( output_datastore=output_datastore, sample_period=sample_period, measurement=measurement, timeframes=timeframes, building=test_meter.building(), meters=[test_meter]) #======================== Calculate Metrics ===================================== testYDS = DataSet(pathOrigDS) testYDS.set_window(start=test_meter.get_timeframe().start.date(), end=test_meter.get_timeframe().end.date()) testY_elec = testYDS.buildings[test_building].elec testY_meter = testY_elec.submeters()[meter_key] test_mains = testY_elec.mains() result = DataSet(disag_filename) res_elec = result.buildings[test_building].elec rpaf = metrics.recall_precision_accuracy_f1(res_elec[meter_key], testY_meter, meterTH, meterTH) relError = metrics.relative_error_total_energy(res_elec[meter_key], testY_meter) MAE = metrics.mean_absolute_error(res_elec[meter_key], testY_meter) RMSE = metrics.RMSE(res_elec[meter_key], testY_meter) print("============ Recall: {}".format(rpaf[0])) print("============ Precision: {}".format(rpaf[1])) print("============ Accuracy: {}".format(rpaf[2])) print("============ F1 Score: {}".format(rpaf[3])) print("============ Relative error in total energy: {}".format(relError)) print("============ Mean absolute error(in Watts): {}".format(MAE)) print("=== For docs: {:.4}\t{:.4}\t{:.4}\t{:.4}\t{:.4}\t{:.4}".format( rpaf[0], rpaf[1], rpaf[2], rpaf[3], relError, MAE)) # print("============ RMSE: {}".format(RMSE)) # print("============ TECA: {}".format(metrics.TECA([res_elec[meter_key]],[testY_meter],test_mains))) resDict = { 'model': 'TEST', 'building': test_building, 'Appliance': meter_key, 'Appliance_Type': 2, 'Recall': rpaf[0], 'Precision': rpaf[1], 'Accuracy': rpaf[2], 'F1': rpaf[3], 'relError': relError, 'MAE': MAE, 'RMSE': RMSE } metrics.writeResultsToCSV(resDict, metricsResFileName, clearMetricsFile)