def __init__(self, appliances, filename, windows, sample_period, start_date, valid_range, building_id, validate_length, fold=['unseen_appliances'], rng_seed=None, format='HDF'): self.appliances = appliances self.seq_length = validate_length self.filename = filename self.sample_period = sample_period self.start_date = dt.datetime.strptime(start_date, "%Y/%m/%d") self.valid_date = self.start_date self.valid_range = valid_range self.building_id = building_id check_windows(windows) self.windows = windows self.fold = fold self.format = format super(ValidationSource, self).__init__(rng_seed=rng_seed) self._load_mains_into_memory()
def __init__(self, activations, target_appliance, seq_length, filename, windows, sample_period, target_inclusion_prob=0.5, uniform_prob_of_selecting_each_building=True, allow_incomplete_target=True, include_incomplete_target_in_output=True, allow_multiple_target_activations_in_aggregate=False, include_multiple_targets_in_output=False, rng_seed=None): self.activations = copy(activations) self.target_appliance = target_appliance self.seq_length = seq_length self.filename = filename check_windows(windows) self.windows = windows self.sample_period = sample_period self.target_inclusion_prob = target_inclusion_prob self.uniform_prob_of_selecting_each_building = ( uniform_prob_of_selecting_each_building) self.allow_incomplete_target = allow_incomplete_target self.include_incomplete_target_in_output = ( include_incomplete_target_in_output) self.allow_multiple_target_activations_in_aggregate = ( allow_multiple_target_activations_in_aggregate) self.include_multiple_targets_in_output = ( include_multiple_targets_in_output) super(RealAggregateSource, self).__init__(rng_seed=rng_seed) self._load_mains_into_memory() self._remove_activations_with_no_mains() self._find_sections_with_no_target() self._compute_gap_probabilities()
def __init__(self, target_appliance, seq_length, filename, windows, sample_period, stride=None, rng_seed=None, sanity_check=1): self.target_appliance = target_appliance self.seq_length = seq_length self.filename = filename if sanity_check: check_windows(windows) self.windows = windows self.sample_period = sample_period self.stride = self.seq_length if stride is None else stride self._reset() super(StrideSource, self).__init__(rng_seed=rng_seed) # stop validation only when we've gone through all validation data self.num_batches_for_validation = None if (stride > seq_length): raise ValueError( "`stride` should not be greater than `seq_length` ") self._load_data_into_memory() self._compute_num_sequences_per_building()
def __init__(self, target_appliance, seq_length, filename, windows, sample_period, stride=None, rng_seed=None): self.target_appliance = target_appliance self.seq_length = seq_length self.filename = filename check_windows(windows) self.windows = windows self.sample_period = sample_period self.stride = self.seq_length if stride is None else stride self._reset() super(StrideSource, self).__init__(rng_seed=rng_seed) # stop validation only when we've gone through all validation data self.num_batches_for_validation = None self._load_data_into_memory() self._compute_num_sequences_per_building()
def __init__(self, activations, target_appliance, seq_length, filename, windows, sample_period, uniform_prob_of_selecting_each_building=True, allow_incomplete_target=True, rng_seed=None): self.activations = copy(activations) self.target_appliance = target_appliance self.seq_length = seq_length self.filename = filename check_windows(windows) self.windows = windows self.sample_period = sample_period self.uniform_prob_of_selecting_each_building=( uniform_prob_of_selecting_each_building) self.allow_incomplete_target = allow_incomplete_target super(Sample, self).__init__(rng_seed=rng_seed) self._load_mains_into_memory() self.target_inclusion_prob=0.5
def load_nilmtk_activations(appliances, filename, sample_period, windows): """ Parameters ---------- appliances : list of strings filename : string sample_period : int windows : dict Structure example: { 'train': {<building_i>: <window>}, 'unseen_activations_of_seen_appliances': {<building_i>: <window>}, 'unseen_appliances': {<building_i>: <window>} } Returns ------- all_activations : dict Structure example: {<train | unseen_appliances | unseen_activations_of_seen_appliances>: { <appliance>: { <building_name>: [<activations>] }}} Each activation is a pd.Series with DatetimeIndex and the following metadata attributes: building, appliance, fold. """ logger.info("Loading NILMTK activations...") # Sanity check check_windows(windows) # Load dataset #dataset = nilmtk.DataSet(filename) all_activations = {} for fold, buildings_and_windows in windows.items(): activations_for_fold = {} for building_i, window in buildings_and_windows.items(): dataset = load_appliance_dataset(int(building_i)) dataset = set_window(dataset, window[0], window[1]) #elec = dataset.buildings[building_i].elec building_name = ('REDD' + '_building_{}'.format(building_i)) for appliance in appliances: logger.info("Loading {} for {}...".format( appliance, building_name)) # Get meter for appliance #try: meter = dataset resample_kwargs = {} resample_kwargs['rule'] = '{:d}S'.format(sample_period) meter = meter['power']['apparent'] meter = safe_resample(meter, resample_kwargs) meter = meter.agg(np.mean) meter = meter.fillna(method='ffill') #except KeyError as exception: # logger.info(building_name + " has no " + appliance + # ". Full exception: {}".format(exception)) # continue # Get activations_for_fold and process them meter_activations = get_app_activations(meter) meter_activations = [ activation.astype(np.float32) for activation in meter_activations ] # Attach metadata for activation in meter_activations: activation._metadata = copy(activation._metadata) activation._metadata.extend( ["building", "appliance", "fold"]) activation.building = building_name activation.appliance = appliance activation.fold = fold # Save if meter_activations: activations_for_fold.setdefault( appliance, {})[building_name] = meter_activations logger.info("Loaded {} {} activations from {}.".format( len(meter_activations), appliance, building_name)) all_activations[fold] = activations_for_fold #dataset.store.close() logger.info("Done loading NILMTK activations.") return all_activations
def load_nilmtk_activations(appliances, filename, sample_period, windows): """ Parameters ---------- appliances : list of strings filename : string sample_period : int windows : dict Structure example: { 'train': {<building_i>: <window>}, 'unseen_activations_of_seen_appliances': {<building_i>: <window>}, 'unseen_appliances': {<building_i>: <window>} } Returns ------- all_activations : dict Structure example: {<train | unseen_appliances | unseen_activations_of_seen_appliances>: { <appliance>: { <building_name>: [<activations>] }}} Each activation is a pd.Series with DatetimeIndex and the following metadata attributes: building, appliance, fold. """ logger.info("Loading NILMTK activations...") # Sanity check check_windows(windows) # Load dataset dataset = nilmtk.DataSet(filename) all_activations = {} for fold, buildings_and_windows in list(windows.items()): activations_for_fold = {} for building_i, window in list(buildings_and_windows.items()): dataset.set_window(*window) elec = dataset.buildings[building_i].elec building_name = (dataset.metadata['name'] + '_building_{}'.format(building_i)) for appliance in appliances: logger.info("Loading {} for {}...".format( appliance, building_name)) # Get meter for appliance try: meter = elec[appliance] except KeyError as exception: logger.info(building_name + " has no " + appliance + ". Full exception: {}".format(exception)) continue # Get activations_for_fold and process them meter_activations = meter.get_activations( sample_period=sample_period) meter_activations = [ activation.astype(np.float32) for activation in meter_activations ] # Attach metadata for activation in meter_activations: activation._metadata = copy(activation._metadata) activation._metadata.extend( ["building", "appliance", "fold"]) activation.building = building_name activation.appliance = appliance activation.fold = fold # Save if meter_activations: activations_for_fold.setdefault( appliance, {})[building_name] = meter_activations logger.info("Loaded {} {} activations from {}.".format( len(meter_activations), appliance, building_name)) all_activations[fold] = activations_for_fold dataset.store.close() logger.info("Done loading NILMTK activations.") return all_activations
def load_nilmtk_activations(appliances, filename, sample_period, windows): """ Parameters ---------- appliances : list of strings filename : string sample_period : int windows : dict Structure example: { 'train': {<building_i>: <window>}, 'unseen_activations_of_seen_appliances': {<building_i>: <window>}, 'unseen_appliances': {<building_i>: <window>} } Returns ------- all_activations : dict Structure example: {<train | unseen_appliances | unseen_activations_of_seen_appliances>: { <appliance>: { <building_name>: [<activations>] }}} Each activation is a pd.Series with DatetimeIndex and the following metadata attributes: building, appliance, fold. """ logger.info("Loading NILMTK activations...") # Sanity check check_windows(windows) # Load dataset dataset = nilmtk.DataSet(filename) all_activations = {} for fold, buildings_and_windows in windows.iteritems(): activations_for_fold = {} for building_i, window in buildings_and_windows.iteritems(): dataset.set_window(*window) elec = dataset.buildings[building_i].elec building_name = ( dataset.metadata['name'] + '_building_{}'.format(building_i)) for appliance in appliances: logger.info( "Loading {} for {}...".format(appliance, building_name)) # Get meter for appliance try: meter = elec[appliance] except KeyError as exception: logger.info(building_name + " has no " + appliance + ". Full exception: {}".format(exception)) continue # Get activations_for_fold and process them meter_activations = meter.get_activations( sample_period=sample_period) meter_activations = [activation.astype(np.float32) for activation in meter_activations] # Attach metadata for activation in meter_activations: activation._metadata = copy(activation._metadata) activation._metadata.extend( ["building", "appliance", "fold"]) activation.building = building_name activation.appliance = appliance activation.fold = fold # Save if meter_activations: activations_for_fold.setdefault( appliance, {})[building_name] = meter_activations logger.info( "Loaded {} {} activations from {}." .format(len(meter_activations), appliance, building_name)) all_activations[fold] = activations_for_fold dataset.store.close() logger.info("Done loading NILMTK activations.") return all_activations
def load_nilmtk_activations( dataset_paths, target_appliance_name, appliance_names, on_power_threshold, min_on_duration, min_off_duration, sample_period, windows, sanity_check=1 ): """ Parameters ---------- windows : dict Structure example: { 'UKDALE': { 'train': {<building_i>: <window>}, 'unseen_activations_of_seen_appliances': {<building_i>: <window>}, 'unseen_appliances': {<building_i>: <window>} } } Returns ------- all_activations : dict Structure example: {<train | unseen_appliances | unseen_activations_of_seen_appliances>: { <appliance>: { <building_name>: [<activations>] }}} Each activation is a pd.Series with DatetimeIndex and the following metadata attributes: building, appliance, fold. """ logger.info("Loading NILMTK activations...") if sanity_check: # Sanity check for dataset in windows: check_windows(windows[dataset]) all_activations = {} for dataset_name, folds in windows.items(): # Load dataset dataset = nilmtk.DataSet(dataset_paths[dataset_name]) appliance_aliases = appliance_names[dataset_name][target_appliance_name] for fold, buildings_and_windows in folds.items(): logger.info( "Loading activations for fold {}.....".format(fold)) for building_i, windows_for_building in buildings_and_windows.items(): #dataset.set_window(*window) elec = dataset.buildings[building_i].elec building_name = ( dataset.metadata['name'] + '_building_{}'.format(building_i)) appliance_meters = [] for meter in elec.meters: if meter.is_site_meter(): continue append_meter = False for a in meter.appliances: if a.type['type'] in appliance_aliases: append_meter = True if append_meter: appliance_meters.append(meter) print(meter.appliances) if not appliance_meters: logger.info( "No {} found in {}".format(target_appliance_name, building_name)) continue #if appliance_meters: if len(appliance_meters) > 1: meter = nilmtk.MeterGroup(meters=appliance_meters) else: meter = appliance_meters[0] logger.info( "Loading {} for {}...".format(target_appliance_name, building_name)) meter_activations = [] for window in windows_for_building: if dataset_name == "ECO": dataset.store.window = TimeFrame(start=window[0], end=window[1], tz='GMT') else: dataset.set_window(*window) # does not work for ECO # Get activations_for_fold and process them meter_activations_for_building = meter.get_activations( sample_period=sample_period, min_off_duration=min_off_duration, min_on_duration=min_on_duration, on_power_threshold=on_power_threshold, resample_kwargs={'fill_method': 'ffill', 'how': 'mean', 'limit': 20}) #meter_activations_for_building = [activation.astype(np.float32) # for activation in meter_activations_for_building] meter_activations.extend(meter_activations_for_building) # Attach metadata #for activation in meter_activations: # activation._metadata = copy(activation._metadata) # activation._metadata.extend( # ["building", "appliance", "fold"]) # activation.building = building_name # activation.appliance = appliance # activation.fold = fold # Save if meter_activations: all_activations.setdefault( fold, {}).setdefault( target_appliance_name, {})[building_name] = meter_activations logger.info( "Loaded {} {} activations from {}." .format(len(meter_activations), target_appliance_name, building_name)) dataset.store.close() logger.info("Done loading NILMTK activations.") return all_activations
def load_nilmtk_activations(appliances, filename, sample_period, windows, on_power_thresholds=None, min_on_durations=None, min_off_durations=None, sanity_check=1): """ Parameters ---------- appliances : list of strings filename : string sample_period : int windows : dict Structure example: { 'train': {<building_i>: <window>}, 'unseen_activations_of_seen_appliances': {<building_i>: <window>}, 'unseen_appliances': {<building_i>: <window>} } Returns ------- all_activations : dict Structure example: {<train | unseen_appliances | unseen_activations_of_seen_appliances>: { <appliance>: { <building_name>: [<activations>] }}} Each activation is a pd.Series with DatetimeIndex and the following metadata attributes: building, appliance, fold. """ logger.info("Loading NILMTK activations...") # check whether optional parameters are provided and if so, are of the same length as `appliances` # if not provided build a list of the same size as `appliances` and fill it with None entries. if (on_power_thresholds is not None) and (len(on_power_thresholds) != len(appliances)): raise ValueError("`on_power_thresholds` must have the same size as `appliances` ") elif on_power_thresholds is None: on_power_thresholds = [None for i in range(len(appliances))] if (min_on_durations is not None) and (len(min_on_durations) != len(appliances)): raise ValueError("`min_on_durations` must have the same size as `appliances` ") elif min_on_durations is None: min_on_durations = [None for i in range(len(appliances))] if (min_off_durations is not None) and (len(min_off_durations) != len(appliances)): raise ValueError("`min_off_durations` must have the same size as `appliances` ") elif min_off_durations is None: min_off_durations = [None for i in range(len(appliances))] if sanity_check: # Sanity check check_windows(windows) # Load dataset dataset = nilmtk.DataSet(filename) all_activations = {} for fold, buildings_and_windows in windows.items(): logger.info( "Loading activations for fold {}.....".format(fold)) activations_for_fold = {} for building_i, window in buildings_and_windows.items(): dataset.set_window(*window) elec = dataset.buildings[building_i].elec building_name = ( dataset.metadata['name'] + '_building_{}'.format(building_i)) for i, appliance in enumerate(appliances): logger.info( "Loading {} for {}...".format(appliance, building_name)) # Get meter for appliance try: meter = elec[appliance] except KeyError as exception: logger.info(building_name + " has no " + appliance + ". Full exception: {}".format(exception)) continue # Get activations_for_fold and process them meter_activations = meter.get_activations( sample_period=sample_period, min_off_duration=min_off_durations[i], min_on_duration=min_on_durations[i], on_power_threshold=on_power_thresholds[i]) meter_activations = [activation.astype(np.float32) for activation in meter_activations] # Attach metadata for activation in meter_activations: activation._metadata = copy(activation._metadata) activation._metadata.extend( ["building", "appliance", "fold"]) activation.building = building_name activation.appliance = appliance activation.fold = fold # Save if meter_activations: activations_for_fold.setdefault( appliance, {})[building_name] = meter_activations logger.info( "Loaded {} {} activations from {}." .format(len(meter_activations), appliance, building_name)) all_activations[fold] = activations_for_fold dataset.store.close() logger.info("Done loading NILMTK activations.") return all_activations