def __init__(self,
appliances,
filename,
windows,
sample_period,
start_date,
valid_range,
building_id,
validate_length,
fold=['unseen_appliances'],
rng_seed=None,
format='HDF'):
self.appliances = appliances
self.seq_length = validate_length
self.filename = filename
self.sample_period = sample_period
self.start_date = dt.datetime.strptime(start_date, "%Y/%m/%d")
self.valid_date = self.start_date
self.valid_range = valid_range
self.building_id = building_id
check_windows(windows)
self.windows = windows
self.fold = fold
self.format = format
super(ValidationSource, self).__init__(rng_seed=rng_seed)
self._load_mains_into_memory()
def __init__(self, activations, target_appliance,
seq_length, filename, windows, sample_period,
target_inclusion_prob=0.5,
uniform_prob_of_selecting_each_building=True,
allow_incomplete_target=True,
include_incomplete_target_in_output=True,
allow_multiple_target_activations_in_aggregate=False,
include_multiple_targets_in_output=False,
rng_seed=None):
self.activations = copy(activations)
self.target_appliance = target_appliance
self.seq_length = seq_length
self.filename = filename
check_windows(windows)
self.windows = windows
self.sample_period = sample_period
self.target_inclusion_prob = target_inclusion_prob
self.uniform_prob_of_selecting_each_building = (
uniform_prob_of_selecting_each_building)
self.allow_incomplete_target = allow_incomplete_target
self.include_incomplete_target_in_output = (
include_incomplete_target_in_output)
self.allow_multiple_target_activations_in_aggregate = (
allow_multiple_target_activations_in_aggregate)
self.include_multiple_targets_in_output = (
include_multiple_targets_in_output)
super(RealAggregateSource, self).__init__(rng_seed=rng_seed)
self._load_mains_into_memory()
self._remove_activations_with_no_mains()
self._find_sections_with_no_target()
self._compute_gap_probabilities()
def __init__(self,
target_appliance,
seq_length,
filename,
windows,
sample_period,
stride=None,
rng_seed=None,
sanity_check=1):
self.target_appliance = target_appliance
self.seq_length = seq_length
self.filename = filename
if sanity_check:
check_windows(windows)
self.windows = windows
self.sample_period = sample_period
self.stride = self.seq_length if stride is None else stride
self._reset()
super(StrideSource, self).__init__(rng_seed=rng_seed)
# stop validation only when we've gone through all validation data
self.num_batches_for_validation = None
if (stride > seq_length):
raise ValueError(
"`stride` should not be greater than `seq_length` ")
self._load_data_into_memory()
self._compute_num_sequences_per_building()
def __init__(self, activations, target_appliance,
seq_length, filename, windows, sample_period,
target_inclusion_prob=0.5,
uniform_prob_of_selecting_each_building=True,
allow_incomplete_target=True,
include_incomplete_target_in_output=True,
allow_multiple_target_activations_in_aggregate=False,
include_multiple_targets_in_output=False,
rng_seed=None):
self.activations = copy(activations)
self.target_appliance = target_appliance
self.seq_length = seq_length
self.filename = filename
check_windows(windows)
self.windows = windows
self.sample_period = sample_period
self.target_inclusion_prob = target_inclusion_prob
self.uniform_prob_of_selecting_each_building = (
uniform_prob_of_selecting_each_building)
self.allow_incomplete_target = allow_incomplete_target
self.include_incomplete_target_in_output = (
include_incomplete_target_in_output)
self.allow_multiple_target_activations_in_aggregate = (
allow_multiple_target_activations_in_aggregate)
self.include_multiple_targets_in_output = (
include_multiple_targets_in_output)
super(RealAggregateSource, self).__init__(rng_seed=rng_seed)
self._load_mains_into_memory()
self._remove_activations_with_no_mains()
self._find_sections_with_no_target()
self._compute_gap_probabilities()
def __init__(self, target_appliance,
seq_length, filename, windows, sample_period,
stride=None,
rng_seed=None):
self.target_appliance = target_appliance
self.seq_length = seq_length
self.filename = filename
check_windows(windows)
self.windows = windows
self.sample_period = sample_period
self.stride = self.seq_length if stride is None else stride
self._reset()
super(StrideSource, self).__init__(rng_seed=rng_seed)
# stop validation only when we've gone through all validation data
self.num_batches_for_validation = None
self._load_data_into_memory()
self._compute_num_sequences_per_building()
def __init__(self, target_appliance,
seq_length, filename, windows, sample_period,
stride=None,
rng_seed=None):
self.target_appliance = target_appliance
self.seq_length = seq_length
self.filename = filename
check_windows(windows)
self.windows = windows
self.sample_period = sample_period
self.stride = self.seq_length if stride is None else stride
self._reset()
super(StrideSource, self).__init__(rng_seed=rng_seed)
# stop validation only when we've gone through all validation data
self.num_batches_for_validation = None
self._load_data_into_memory()
self._compute_num_sequences_per_building()
def __init__(self, activations, target_appliance,
seq_length, filename, windows, sample_period,
uniform_prob_of_selecting_each_building=True,
allow_incomplete_target=True,
rng_seed=None):
self.activations = copy(activations)
self.target_appliance = target_appliance
self.seq_length = seq_length
self.filename = filename
check_windows(windows)
self.windows = windows
self.sample_period = sample_period
self.uniform_prob_of_selecting_each_building=(
uniform_prob_of_selecting_each_building)
self.allow_incomplete_target = allow_incomplete_target
super(Sample, self).__init__(rng_seed=rng_seed)
self._load_mains_into_memory()
self.target_inclusion_prob=0.5
def load_nilmtk_activations(appliances, filename, sample_period, windows):
"""
Parameters
----------
appliances : list of strings
filename : string
sample_period : int
windows : dict
Structure example:
{
'train': {<building_i>: <window>},
'unseen_activations_of_seen_appliances': {<building_i>: <window>},
'unseen_appliances': {<building_i>: <window>}
}
Returns
-------
all_activations : dict
Structure example:
{<train | unseen_appliances | unseen_activations_of_seen_appliances>: {
<appliance>: {
<building_name>: [<activations>]
}}}
Each activation is a pd.Series with DatetimeIndex and the following
metadata attributes: building, appliance, fold.
"""
logger.info("Loading NILMTK activations...")
# Sanity check
check_windows(windows)
# Load dataset
#dataset = nilmtk.DataSet(filename)
all_activations = {}
for fold, buildings_and_windows in windows.items():
activations_for_fold = {}
for building_i, window in buildings_and_windows.items():
dataset = load_appliance_dataset(int(building_i))
dataset = set_window(dataset, window[0], window[1])
#elec = dataset.buildings[building_i].elec
building_name = ('REDD' + '_building_{}'.format(building_i))
for appliance in appliances:
logger.info("Loading {} for {}...".format(
appliance, building_name))
# Get meter for appliance
#try:
meter = dataset
resample_kwargs = {}
resample_kwargs['rule'] = '{:d}S'.format(sample_period)
meter = meter['power']['apparent']
meter = safe_resample(meter, resample_kwargs)
meter = meter.agg(np.mean)
meter = meter.fillna(method='ffill')
#except KeyError as exception:
# logger.info(building_name + " has no " + appliance +
# ". Full exception: {}".format(exception))
# continue
# Get activations_for_fold and process them
meter_activations = get_app_activations(meter)
meter_activations = [
activation.astype(np.float32)
for activation in meter_activations
]
# Attach metadata
for activation in meter_activations:
activation._metadata = copy(activation._metadata)
activation._metadata.extend(
["building", "appliance", "fold"])
activation.building = building_name
activation.appliance = appliance
activation.fold = fold
# Save
if meter_activations:
activations_for_fold.setdefault(
appliance, {})[building_name] = meter_activations
logger.info("Loaded {} {} activations from {}.".format(
len(meter_activations), appliance, building_name))
all_activations[fold] = activations_for_fold
#dataset.store.close()
logger.info("Done loading NILMTK activations.")
return all_activations
def load_nilmtk_activations(appliances, filename, sample_period, windows):
"""
Parameters
----------
appliances : list of strings
filename : string
sample_period : int
windows : dict
Structure example:
{
'train': {<building_i>: <window>},
'unseen_activations_of_seen_appliances': {<building_i>: <window>},
'unseen_appliances': {<building_i>: <window>}
}
Returns
-------
all_activations : dict
Structure example:
{<train | unseen_appliances | unseen_activations_of_seen_appliances>: {
<appliance>: {
<building_name>: [<activations>]
}}}
Each activation is a pd.Series with DatetimeIndex and the following
metadata attributes: building, appliance, fold.
"""
logger.info("Loading NILMTK activations...")
# Sanity check
check_windows(windows)
# Load dataset
dataset = nilmtk.DataSet(filename)
all_activations = {}
for fold, buildings_and_windows in list(windows.items()):
activations_for_fold = {}
for building_i, window in list(buildings_and_windows.items()):
dataset.set_window(*window)
elec = dataset.buildings[building_i].elec
building_name = (dataset.metadata['name'] +
'_building_{}'.format(building_i))
for appliance in appliances:
logger.info("Loading {} for {}...".format(
appliance, building_name))
# Get meter for appliance
try:
meter = elec[appliance]
except KeyError as exception:
logger.info(building_name + " has no " + appliance +
". Full exception: {}".format(exception))
continue
# Get activations_for_fold and process them
meter_activations = meter.get_activations(
sample_period=sample_period)
meter_activations = [
activation.astype(np.float32)
for activation in meter_activations
]
# Attach metadata
for activation in meter_activations:
activation._metadata = copy(activation._metadata)
activation._metadata.extend(
["building", "appliance", "fold"])
activation.building = building_name
activation.appliance = appliance
activation.fold = fold
# Save
if meter_activations:
activations_for_fold.setdefault(
appliance, {})[building_name] = meter_activations
logger.info("Loaded {} {} activations from {}.".format(
len(meter_activations), appliance, building_name))
all_activations[fold] = activations_for_fold
dataset.store.close()
logger.info("Done loading NILMTK activations.")
return all_activations
def load_nilmtk_activations(appliances, filename, sample_period, windows):
"""
Parameters
----------
appliances : list of strings
filename : string
sample_period : int
windows : dict
Structure example:
{
'train': {<building_i>: <window>},
'unseen_activations_of_seen_appliances': {<building_i>: <window>},
'unseen_appliances': {<building_i>: <window>}
}
Returns
-------
all_activations : dict
Structure example:
{<train | unseen_appliances | unseen_activations_of_seen_appliances>: {
<appliance>: {
<building_name>: [<activations>]
}}}
Each activation is a pd.Series with DatetimeIndex and the following
metadata attributes: building, appliance, fold.
"""
logger.info("Loading NILMTK activations...")
# Sanity check
check_windows(windows)
# Load dataset
dataset = nilmtk.DataSet(filename)
all_activations = {}
for fold, buildings_and_windows in windows.iteritems():
activations_for_fold = {}
for building_i, window in buildings_and_windows.iteritems():
dataset.set_window(*window)
elec = dataset.buildings[building_i].elec
building_name = (
dataset.metadata['name'] + '_building_{}'.format(building_i))
for appliance in appliances:
logger.info(
"Loading {} for {}...".format(appliance, building_name))
# Get meter for appliance
try:
meter = elec[appliance]
except KeyError as exception:
logger.info(building_name + " has no " + appliance +
". Full exception: {}".format(exception))
continue
# Get activations_for_fold and process them
meter_activations = meter.get_activations(
sample_period=sample_period)
meter_activations = [activation.astype(np.float32)
for activation in meter_activations]
# Attach metadata
for activation in meter_activations:
activation._metadata = copy(activation._metadata)
activation._metadata.extend(
["building", "appliance", "fold"])
activation.building = building_name
activation.appliance = appliance
activation.fold = fold
# Save
if meter_activations:
activations_for_fold.setdefault(
appliance, {})[building_name] = meter_activations
logger.info(
"Loaded {} {} activations from {}."
.format(len(meter_activations), appliance, building_name))
all_activations[fold] = activations_for_fold
dataset.store.close()
logger.info("Done loading NILMTK activations.")
return all_activations
def load_nilmtk_activations( dataset_paths,
target_appliance_name,
appliance_names,
on_power_threshold,
min_on_duration,
min_off_duration,
sample_period,
windows,
sanity_check=1 ):
"""
Parameters
----------
windows : dict
Structure example:
{
'UKDALE': {
'train': {<building_i>: <window>},
'unseen_activations_of_seen_appliances': {<building_i>: <window>},
'unseen_appliances': {<building_i>: <window>}
}
}
Returns
-------
all_activations : dict
Structure example:
{<train | unseen_appliances | unseen_activations_of_seen_appliances>: {
<appliance>: {
<building_name>: [<activations>]
}}}
Each activation is a pd.Series with DatetimeIndex and the following
metadata attributes: building, appliance, fold.
"""
logger.info("Loading NILMTK activations...")
if sanity_check:
# Sanity check
for dataset in windows:
check_windows(windows[dataset])
all_activations = {}
for dataset_name, folds in windows.items():
# Load dataset
dataset = nilmtk.DataSet(dataset_paths[dataset_name])
appliance_aliases = appliance_names[dataset_name][target_appliance_name]
for fold, buildings_and_windows in folds.items():
logger.info(
"Loading activations for fold {}.....".format(fold))
for building_i, windows_for_building in buildings_and_windows.items():
#dataset.set_window(*window)
elec = dataset.buildings[building_i].elec
building_name = (
dataset.metadata['name'] + '_building_{}'.format(building_i))
appliance_meters = []
for meter in elec.meters:
if meter.is_site_meter():
continue
append_meter = False
for a in meter.appliances:
if a.type['type'] in appliance_aliases:
append_meter = True
if append_meter:
appliance_meters.append(meter)
print(meter.appliances)
if not appliance_meters:
logger.info(
"No {} found in {}".format(target_appliance_name, building_name))
continue
#if appliance_meters:
if len(appliance_meters) > 1:
meter = nilmtk.MeterGroup(meters=appliance_meters)
else:
meter = appliance_meters[0]
logger.info(
"Loading {} for {}...".format(target_appliance_name, building_name))
meter_activations = []
for window in windows_for_building:
if dataset_name == "ECO":
dataset.store.window = TimeFrame(start=window[0], end=window[1], tz='GMT')
else:
dataset.set_window(*window) # does not work for ECO
# Get activations_for_fold and process them
meter_activations_for_building = meter.get_activations(
sample_period=sample_period,
min_off_duration=min_off_duration,
min_on_duration=min_on_duration,
on_power_threshold=on_power_threshold,
resample_kwargs={'fill_method': 'ffill', 'how': 'mean', 'limit': 20})
#meter_activations_for_building = [activation.astype(np.float32)
# for activation in meter_activations_for_building]
meter_activations.extend(meter_activations_for_building)
# Attach metadata
#for activation in meter_activations:
# activation._metadata = copy(activation._metadata)
# activation._metadata.extend(
# ["building", "appliance", "fold"])
# activation.building = building_name
# activation.appliance = appliance
# activation.fold = fold
# Save
if meter_activations:
all_activations.setdefault(
fold, {}).setdefault(
target_appliance_name, {})[building_name] = meter_activations
logger.info(
"Loaded {} {} activations from {}."
.format(len(meter_activations), target_appliance_name, building_name))
dataset.store.close()
logger.info("Done loading NILMTK activations.")
return all_activations
def load_nilmtk_activations(appliances, filename, sample_period, windows,
on_power_thresholds=None,
min_on_durations=None,
min_off_durations=None,
sanity_check=1):
"""
Parameters
----------
appliances : list of strings
filename : string
sample_period : int
windows : dict
Structure example:
{
'train': {<building_i>: <window>},
'unseen_activations_of_seen_appliances': {<building_i>: <window>},
'unseen_appliances': {<building_i>: <window>}
}
Returns
-------
all_activations : dict
Structure example:
{<train | unseen_appliances | unseen_activations_of_seen_appliances>: {
<appliance>: {
<building_name>: [<activations>]
}}}
Each activation is a pd.Series with DatetimeIndex and the following
metadata attributes: building, appliance, fold.
"""
logger.info("Loading NILMTK activations...")
# check whether optional parameters are provided and if so, are of the same length as `appliances`
# if not provided build a list of the same size as `appliances` and fill it with None entries.
if (on_power_thresholds is not None) and (len(on_power_thresholds) != len(appliances)):
raise ValueError("`on_power_thresholds` must have the same size as `appliances` ")
elif on_power_thresholds is None:
on_power_thresholds = [None for i in range(len(appliances))]
if (min_on_durations is not None) and (len(min_on_durations) != len(appliances)):
raise ValueError("`min_on_durations` must have the same size as `appliances` ")
elif min_on_durations is None:
min_on_durations = [None for i in range(len(appliances))]
if (min_off_durations is not None) and (len(min_off_durations) != len(appliances)):
raise ValueError("`min_off_durations` must have the same size as `appliances` ")
elif min_off_durations is None:
min_off_durations = [None for i in range(len(appliances))]
if sanity_check:
# Sanity check
check_windows(windows)
# Load dataset
dataset = nilmtk.DataSet(filename)
all_activations = {}
for fold, buildings_and_windows in windows.items():
logger.info(
"Loading activations for fold {}.....".format(fold))
activations_for_fold = {}
for building_i, window in buildings_and_windows.items():
dataset.set_window(*window)
elec = dataset.buildings[building_i].elec
building_name = (
dataset.metadata['name'] + '_building_{}'.format(building_i))
for i, appliance in enumerate(appliances):
logger.info(
"Loading {} for {}...".format(appliance, building_name))
# Get meter for appliance
try:
meter = elec[appliance]
except KeyError as exception:
logger.info(building_name + " has no " + appliance +
". Full exception: {}".format(exception))
continue
# Get activations_for_fold and process them
meter_activations = meter.get_activations(
sample_period=sample_period,
min_off_duration=min_off_durations[i],
min_on_duration=min_on_durations[i],
on_power_threshold=on_power_thresholds[i])
meter_activations = [activation.astype(np.float32)
for activation in meter_activations]
# Attach metadata
for activation in meter_activations:
activation._metadata = copy(activation._metadata)
activation._metadata.extend(
["building", "appliance", "fold"])
activation.building = building_name
activation.appliance = appliance
activation.fold = fold
# Save
if meter_activations:
activations_for_fold.setdefault(
appliance, {})[building_name] = meter_activations
logger.info(
"Loaded {} {} activations from {}."
.format(len(meter_activations), appliance, building_name))
all_activations[fold] = activations_for_fold
dataset.store.close()
logger.info("Done loading NILMTK activations.")
return all_activations
