def ScaledLoadsROIExtraction(self, df_power_nilm_exprmnts, d_synch_all,
d_synch, num_Nh_chunks):
dfs = []
var_name = 'NhId'
expIds = list(df_power_nilm_exprmnts["expId"].unique())
for expId in util.tqdm(expIds, ncols=100):
# print('expId: ', expId)
sourceId = {'expId': expId}
df_power_nilm_exprmnt = df_power_nilm_exprmnts[
df_power_nilm_exprmnts["expId"] == expId]
df_Nh_chunk = self.ExtractROISections(
df_power_nilm_exprmnt.copy(),
var_name=var_name,
expected_number_patterns=num_Nh_chunks,
d_synch_all=d_synch_all,
d_synch=d_synch)
num_extracted_sections = len(df_Nh_chunk[var_name].unique())
extracted_roi_durations = df_Nh_chunk[var_name].value_counts()
self._CheckNumExtractedSections(var_name, num_Nh_chunks,
num_extracted_sections,
extracted_roi_durations, sourceId)
dfs.append(df_Nh_chunk)
df_power_nilm_Nh_chunk = pd.concat(dfs, axis=0, ignore_index=False)
return df_power_nilm_Nh_chunk
def LoadSystemLogData(self):
print("Loading System Log Data... ")
DisagWindows = [self.disaggregation_window]
dfs = []
PO = self.MPOpts
# for expId in range(0, self.num_repeated_expt):
for expId in util.tqdm(range(self.num_repeated_expt), ncols=100):
# print("expId: ", expId)
for Nh in self.Nhs:
# print("NumberHousePerMp: ", Nh)
for Np in self.Nps:
# print("Parallel Processing Np: ", Np)
for dw in DisagWindows:
# print("DisagWindows: ", dw)
# df.head()
filename = util.resource_log_filename_format.format(
self.system_log_path, PO, dw, expId, Nh, Np)
df = pd.read_csv(filename + "_per_sys.csv")
df['expId'] = expId + 1
df['time_peronline_total_sec'] = df[[
'time_mpInit_sec', 'time_disagg_online_sec',
'time_mpClose_sec'
]].sum(axis=1)
df['time_mp_total_sec'] = df[[
'time_mpInit_sec', 'time_mpClose_sec'
]].sum(axis=1)
df_proc = pd.read_csv(filename + "_per_process.csv")
# df_proc['energy_wh'] = df_proc['energy_kwh'].apply(lambda x: 1000.0*x)
df['house_energy_min_kwh'] = df_proc['energy_kwh'].min(
axis=0)
df['house_energy_mean_kwh'] = df_proc[
'energy_kwh'].mean(axis=0)
df['house_energy_median_kwh'] = df_proc[
'energy_kwh'].median(axis=0)
df['house_energy_max_kwh'] = df_proc['energy_kwh'].max(
axis=0)
df['house_energy_total_kwh'] = df_proc[
'energy_kwh'].sum(axis=0)
dfs.append(df)
df_system_log_pt = pd.concat(dfs, axis=0, ignore_index=True)
df_system_log_pt.rename(columns={'numProcesses': 'num_processes'},
inplace=True)
# print(df_system_log_pt.shape)
# print(df_system_log_pt.head())
self.df_system_log_pt = df_system_log_pt
def OnlineLoadROIExtraction(self, df_power_nilm_Nps, d_synch_all, d_synch,
num_online_chunks):
dfs = []
var_name = 'onlineId'
expIds = list(df_power_nilm_Nps["expId"].unique())
for expId in util.tqdm(expIds, ncols=100):
# print('expId: ', expId)
df_wu_agg = df_power_nilm_Nps[df_power_nilm_Nps["expId"] == expId]
NhIds = list(df_wu_agg["NhId"].unique())
for NhId in NhIds:
# print('NhId: ', NhId)
df_wu_agg_sec_1 = df_wu_agg[df_wu_agg["NhId"] == NhId]
NpIds = list(df_wu_agg_sec_1["NpId"].unique())
for NpId in NpIds:
# print('NpId: ', NpId)
sourceId = {'expId': expId, 'NhId': NhId, 'NpId': NpId}
df_Np = df_wu_agg_sec_1[(df_wu_agg_sec_1["NpId"] == NpId)]
df_online = self.ExtractROISections(
df_power_agg=df_Np.copy(),
var_name=var_name,
expected_number_patterns=num_online_chunks,
d_synch_all=d_synch_all,
d_synch=d_synch)
num_extracted_sections = len(df_online[var_name].unique())
extracted_roi_durations = df_online[var_name].value_counts(
)
self._CheckNumExtractedSections(var_name,
num_online_chunks,
num_extracted_sections,
extracted_roi_durations,
sourceId)
dfs.append(df_online)
df_power_nilm_online = pd.concat(dfs, axis=0, ignore_index=False)
df_power_nilm_online = self._CleanNILMROI(df_power_nilm_online)
return df_power_nilm_online
def MultiProcessesROIExtraction(self, df_power_nilm_Nh_chunk, d_synch_all,
d_synch, num_Nps):
dfs = []
var_name = 'NpId'
expIds = list(df_power_nilm_Nh_chunk["expId"].unique())
for expId in util.tqdm(expIds, ncols=100):
# print('expId: ', expId)
df_wu_agg = df_power_nilm_Nh_chunk[df_power_nilm_Nh_chunk["expId"]
== expId]
NhIds = list(df_wu_agg["NhId"].unique())
for NhId in NhIds:
# print('NhId: ', NhId)
sourceId = {'expId': expId, 'NhId': NhId}
df_wu_agg_sec_1 = df_wu_agg[df_wu_agg["NhId"] == NhId]
df_Np = self.ExtractROISections(
df_wu_agg_sec_1.copy(),
var_name=var_name,
expected_number_patterns=num_Nps,
d_synch_all=d_synch_all,
d_synch=d_synch)
num_extracted_sections = len(df_Np[var_name].unique())
extracted_roi_durations = df_Np[var_name].value_counts()
self._CheckNumExtractedSections(var_name, num_Nps,
num_extracted_sections,
extracted_roi_durations,
sourceId)
dfs.append(df_Np)
df_power_nilm_Nps = pd.concat(dfs, axis=0, ignore_index=False)
return df_power_nilm_Nps
def main(result_path):
"""
Provides visualization tools to plot the cleaned ROIs extracted by the MeterReadingROIExtractor.
It also maps the generic labels of the ROIs into their actual values using the monitoring setting from CostMonitor.
"""
print('MeterROIExtractionPlotter...')
# %% create figure directory
figure_path = "{}/{}".format(result_path, setting_dict["meter_log"]["figure_dir"])
util.CreateDir(figure_path)
# %%
ObjSeqPattern = AlgCostSignalExtractor(result_path)
print('Loading experiment settings...')
ObjSeqPattern.LoadExprimentSetting()
exprmnt_setting_dict = ObjSeqPattern.exprmnt_setting_dict
# util.PrintDict(exprmnt_setting_dict)
isdays = exprmnt_setting_dict['isdays']
disaggtimewindow = exprmnt_setting_dict['disaggtimewindow']
num_repeated_expt = exprmnt_setting_dict['num_repeated_expt']
num_Nh_chunks = exprmnt_setting_dict['num_Nh_chunks']
num_Nps = exprmnt_setting_dict['num_Nps']
Nh_chunks = exprmnt_setting_dict['Nh_chunks']
Nps = exprmnt_setting_dict['Nps']
init_d = exprmnt_setting_dict['init_d']
exprmnt_d = exprmnt_setting_dict['exprmnt_d']
Nhchunk_d = exprmnt_setting_dict['Nhchunk_d']
Np_d = exprmnt_setting_dict['Np_d']
onlinedw_d = exprmnt_setting_dict['onlinedw_d']
synch_len = onlinedw_d
num_online_chunks = 1
# %%
print('Loading wattsup smart meter reading...')
df_power_log, fig = ObjSeqPattern.LoadMeterPowerReading()
df_power_log.head()
df_power_log.shape
plt.tight_layout(pad=0.3, w_pad=0.3, h_pad=0.3)
plt.legend(ncol = 2, loc='center right', bbox_to_anchor = (1., 1+0.1), borderaxespad=0, frameon=True )
util.SaveFigure("{}/meter_log_bg_removed_rec".format(figure_path), fig)
# %%
print('Reconstructing experiments...')
df_wupower_extracted = ReconstructSections(df_power_log, exprmnt_setting_dict, ObjSeqPattern.filename_meter_log_roi_uncleaned)
plt.plot(df_power_log.t, df_power_log.Watts + df_power_log['Watts_bg'], label= 'Total Power')
plt.plot(df_power_log.t, df_power_log.Watts, label= 'Dynamic Power')
plt.xlabel('t (sec)', fontsize=FIGURE_LBL_FONTSIZE_MAIN)
plt.ylabel('P (W)', fontsize=FIGURE_LBL_FONTSIZE_MAIN)
util.PlotGridSpacing(df_power_log.t.values, [0, df_power_log.Watts.max() + df_power_log['Watts_bg'].max()], x_gridno=6, y_gridno=6, issnscat = True)
plt.tight_layout(pad=0.3, w_pad=0.3, h_pad=0.3)
plt.legend(ncol = 2, loc='center right', bbox_to_anchor = (1., 1+0.1), borderaxespad=0, frameon=True )
util.SaveFigure("{}/meter_log_bg_removed_rec".format(figure_path), fig)
# %%
print('Extracting NILMROI...')
d_synch_all = init_d + exprmnt_d + Nhchunk_d + Np_d + onlinedw_d
try:
df_power_nilm, fig = ObjSeqPattern.NILMAlgROIExtraction(df_power_log.copy(), 5*d_synch_all, init_d)
except:
raise 'NILM section extraction error!'
finally:
plt.tight_layout(pad=0.3, w_pad=0.3, h_pad=0.3)
# plt.legend(bbox_to_anchor = (1, 1.21))
plt.legend(loc='center right', bbox_to_anchor = (1., 1+0.1), borderaxespad=0, frameon=True)
util.SaveFigure("{}/meter_log_alg_rec".format(figure_path), fig)
# %%
print('Extracting RepetitionROIs...')
ObjSeqPattern.PlotSelections(df_wupower_extracted, 'expId', figure_path = figure_path, figname = 'all_experiments'+'_rec')
# %%
print('Extracting LoadscaleROIs...')
df_wupower_extracted_cleaned = pd.read_csv(ObjSeqPattern.filename_meter_log_roi)
df_wupower_extracted_cleaned.rename(columns={'P_w':'Watts', 'P_bg_w':'Watts_bg', 'NhId':'Nh', 'NpId':'Np'}, inplace=True)
df_wupower_extracted_cleaned['Nh'] = df_wupower_extracted_cleaned['Nh'].apply(lambda x: exprmnt_setting_dict['Nh_chunks'][x])
df_wupower_extracted_cleaned['expId'] = df_wupower_extracted_cleaned['expId'] + 1
df_wupower_extracted = df_wupower_extracted[df_wupower_extracted.expId.isin(df_wupower_extracted_cleaned.expId.unique())]
expIds = list(df_wupower_extracted.expId.unique())
for expId in util.tqdm(expIds, ncols=100):
# print('plotting expId: ', expId)
df = df_wupower_extracted[(df_wupower_extracted.expId==expId)]
ObjSeqPattern.PlotSelections(df, 'Nh', figure_path = figure_path, figname = "expId_{}_rec".format(expId))
# %%
print('Extracting MultiprocessROIs...')
expIds = list(df_wupower_extracted.expId.unique())
for expId in util.tqdm(expIds, ncols=100):
# print('plotting expId: ', expId)
df_exprmnt = df_wupower_extracted[(df_wupower_extracted.expId==expId)]
Nhs = list(df_exprmnt.Nh.unique())
for Nh in Nhs:
# print('plotting Nhs: ', Nh)
df = df_exprmnt[(df_exprmnt.Nh==Nh)]
ObjSeqPattern.PlotSelections(df, 'Np', figure_path = figure_path, figname = "expId_{}_NhId_{}_rec".format(expId, Nh))
print('Done!')
def main(result_path):
"""
Extracts the region of interest (ROI) sections of the target active algorithm hierarchicaly and recursively from system-level power readings
"""
print('MeterROIExtraction...')
# %% create debug figure directory
figure_path = "{}/{}/{}".format(
result_path, setting_dict["meter_log"]["log_dir"],
setting_dict["meter_log"]["debug_figure_dir"])
util.CreateDir(figure_path)
# %%
ObjSeqPattern = AlgCostSignalExtractor(result_path)
ObjSeqPattern.ResetErrorLog()
# %%
print('Loading experiment settings...')
ObjSeqPattern.LoadExprimentSetting()
exprmnt_setting_dict = ObjSeqPattern.exprmnt_setting_dict
util.PrintDict(exprmnt_setting_dict)
isdays = exprmnt_setting_dict['isdays']
disaggtimewindow = exprmnt_setting_dict['disaggtimewindow']
num_repeated_expt = exprmnt_setting_dict['num_repeated_expt']
num_Nh_chunks = exprmnt_setting_dict['num_Nh_chunks']
num_Nps = exprmnt_setting_dict['num_Nps']
Nh_chunks = exprmnt_setting_dict['Nh_chunks']
Nps = exprmnt_setting_dict['Nps']
init_d = exprmnt_setting_dict['init_d']
exprmnt_d = exprmnt_setting_dict['exprmnt_d']
Nhchunk_d = exprmnt_setting_dict['Nhchunk_d']
Np_d = exprmnt_setting_dict['Np_d']
onlinedw_d = exprmnt_setting_dict['onlinedw_d']
synch_len = onlinedw_d
num_online_chunks = 1
# %%
print('Loading wattsup smart meter reading...')
df_power_log, fig = ObjSeqPattern.LoadMeterPowerReading()
util.SaveFigure("{}/meter_log_bg_removed".format(figure_path), fig)
# %%
print('Extracting NILMROI...')
d_synch_all = init_d + exprmnt_d + Nhchunk_d + Np_d + onlinedw_d
try:
df_power_nilm, fig = ObjSeqPattern.NILMAlgROIExtraction(
df_power_log.copy(), 5 * d_synch_all, init_d)
except:
raise 'NILM section extraction error!'
finally:
util.SaveFigure("{}/meter_log_alg".format(figure_path), fig)
# %%
print('Extracting RepetitionROIs...')
d_synch_all = d_synch_all - init_d
df_power_nilm_exprmnts = ObjSeqPattern.RepetitionROIExtraction(
df_power_nilm.copy(), 2 * 5 * d_synch_all, exprmnt_d,
num_repeated_expt)
ObjSeqPattern.PlotSelections(df_power_nilm_exprmnts,
'expId',
figure_path=figure_path,
figname='all_experiments')
# %%
print('Extracting LoadscaleROIs...')
d_synch_all = d_synch_all - exprmnt_d
df_power_nilm_Nh_chunks = ObjSeqPattern.ScaledLoadsROIExtraction(
df_power_nilm_exprmnts.copy(), 2 * 5 * d_synch_all, Nhchunk_d,
num_Nh_chunks)
expIds = list(df_power_nilm_Nh_chunks["expId"].unique())
for expId in util.tqdm(expIds, ncols=40):
# print('plotting expId: ', expId)
df = df_power_nilm_Nh_chunks[(
df_power_nilm_Nh_chunks["expId"] == expId)]
ObjSeqPattern.PlotSelections(df,
'NhId',
figure_path=figure_path,
figname="expId_{}".format(expId))
# %%
print('Extracting MultiprocessROIs...')
d_synch_all = d_synch_all - Nhchunk_d
df_power_nilm_Nps = ObjSeqPattern.MultiProcessesROIExtraction(
df_power_nilm_Nh_chunks, 2 * 5 * d_synch_all, Np_d, num_Nps)
expIds = list(df_power_nilm_Nps["expId"].unique())
for expId in util.tqdm(expIds, ncols=100):
# print('plotting expId: ', expId)
df_exprmnt = df_power_nilm_Nps[(df_power_nilm_Nps["expId"] == expId)]
NhIds = list(df_exprmnt["NhId"].unique())
for NhId in NhIds:
# print('plotting NhId: ', NhId)
df = df_exprmnt[(df_exprmnt["NhId"] == NhId)]
ObjSeqPattern.PlotSelections(df,
'NpId',
figure_path=figure_path,
figname="expId_{}_NhId_{}".format(
expId, Nh_chunks[NhId]))
# %%
print('Extracting consumptions of the algorithm...')
d_synch_all = d_synch_all - onlinedw_d
df_power_nilm_Nps_online = ObjSeqPattern.OnlineLoadROIExtraction(
df_power_nilm_Nps.copy(), d_synch_all, onlinedw_d, num_online_chunks)
ObjSeqPattern.SaveExtractedData(df_power_nilm_Nps_online.copy(),
iscleaned=0)
expIds = list(df_power_nilm_Nps_online["expId"].unique())
for expId in util.tqdm(expIds, ncols=100):
# print('plotting expId: ', expId)
df_exprmnt = df_power_nilm_Nps_online[(
df_power_nilm_Nps_online["expId"] == expId)]
NhIds = list(df_exprmnt["NhId"].unique())
for NhId in NhIds:
# print('plotting NhId: ', NhId)
df = df_exprmnt[(df_exprmnt["NhId"] == NhId)]
ObjSeqPattern.PlotSelections(df,
'NpId',
figure_path=figure_path,
figname="expId_{}_NhId_{}_alg".format(
expId, Nh_chunks[NhId]))
# %%
print('Discarding corrupted experiment sections...')
# corrupted_expIds = ObjSeqPattern.corrupted_expIds
df_power_nilm_Nps_online = ObjSeqPattern.DiscardCorruptedExpriments(
df_power_nilm_Nps_online)
# %%
print('Saving extracted and disaggregated data...')
ObjSeqPattern.SaveExtractedData(df_power_nilm_Nps_online)
# %%
print('Done!')