def prepare_data(self, configuration_json, debug=False):
assert configuration_json.get("beacons") is not None
config = ProcessingConfiguration.create_configuration(
dictionary=configuration_json)
print("configuration:\n{}".format(config))
raw_data_df = io_interface.load_raw_fingerprint_data(
configuration=config)
validation_raw_df = io_interface.load_raw_fingerprint_data(
configuration=config, suffix='validation/validation_fingerprints')
raw_data_beacons = raw_data_df['beacon'].unique().tolist()
assert set(raw_data_beacons) == set(
config.beacons
), "please add raw data beacons to config file: {}".format(
raw_data_beacons)
print("nodes fingerprinted: {}".format(
raw_data_df['node'].unique().size))
if validation_raw_df.size > 0:
print("found validation fingerprints: {}".format(
validation_raw_df['node'].unique().size))
if config.beacons is None:
config.beacons = raw_data_df.unique().tolist()
if debug:
print("using beacons from raw data:\n{}".format(
config.beacons))
channel_list = []
if configuration_json['doIOS']:
channel_list.append('0')
if configuration_json['doAndroid']:
channel_list.extend(['25', '26', '27'])
for channel in channel_list:
config.channel = channel
if config.check_processed_file_exists() and not debug:
print("file exists")
else:
processed_data = processing_interface.process_fingerprints(
raw_data_df=raw_data_df,
validation_raw_data=validation_raw_df,
configuration=config,
debug=debug)
io_interface.save_processed_fingerpint_data(processed_data,
config=config)
print("completed data wrangling")
def plot_finger_prints(configuration_json):
config = ProcessingConfiguration.create_configuration(configuration_json)
raw_data = io_interface.load_raw_fingerprint_data(config)
true_coordinates = get_coordinates(raw_data)
fig, ax = plt.subplots()
plots.plot_coordinates_on_map(
true_coordinates,
true_coordinates,
factor=configuration_json['factor'],
image_path=config.image_path,
ax=ax,
)
plt.savefig(
os.path.join(os.path.split(config.image_path)[0], "fingerprints.png"))
def prepare_validation_data(configuration,processed_data, workers=1):
configuration = copy.deepcopy(configuration)
beacon_data = io_interface.load_raw_fingerprint_data(
configuration=configuration,
suffix='validation/validation_fingerprints'
)
rolled_features = tdpr.create_features_from_raw_data(raw_data_frame=beacon_data,
configuration=configuration,
workers=workers)
validation_features = pr.transform_temporal_features(
rolled_features,
processed_data,
channel=configuration.channel,
missing_value=configuration.missing_value,
)
return validation_features
import processing.io_interface as io
import processing.processing_interface as pr
if __name__ == '__main__':
configuration_json = {
"method": 'csv',
"use_max": True, "data_path": "/sites", "site": "BELK", "channel": '0',
"split_point": 1.0, "use_mad": False,
"floor": "Floor1", "location": "FortWorth", "use_median": True,
"filter_beacons": ['2500E7E3000B', '2600E197000B', '2700D0D0000B']
}
data = io.load_raw_fingerprint_data(configuration_json=configuration_json)
configuration_json['method'] = 'mongo'
processed_data = pr.process_fingerprints(data,-1, 5,2)
# dataio.save_raw_fingerprint_data(data, configuration_json=configuration_json)
configuration_json['method'] = 'pickle'
configuration_json['filter_beacons'] = ["2700D0D0000B", "2600E197000B"]
configuration_json['simulations_count'] = 1
configuration_json['features'] = ['max', 'median']
configuration_json['window_size'] = 10
configuration_json['window_step'] = 1
configuration_json['filter_data'] = False
configuration_json['dropObs'] = True
configuration_json['missing_value'] = -105.0
configuration_json['plot_type'] = ['rms', 'cdf', 'coordinates', 'heatmap']
configuration_json['doAndroid'] = True
configuration_json['observed_value'] = 5.0
config = ProcessingConfiguration.create_configuration(configuration_json)
print(config)
raw_data = io_interface.load_raw_fingerprint_data(configuration=config,
suffix='groceries')
processed_data_groceries = processing_interface.process_fingerprints(
raw_data,
configuration=config,
workers=8,
)
raw_data_test = io_interface.load_raw_fingerprint_data(
configuration=config, suffix='validation')
processed_data_groceries_copy = copy.deepcopy(processed_data_groceries)
processed_data_validation = processing_interface.process_fingerprints(
raw_data_test,
configuration=config,
workers=1,
configuration_json = {
"method": 'csv',
"use_max": True,
"data_path": "/sites",
"site": "SMARTFOCUS",
"channel": '0',
"split_point": 1.0,
"use_mad": False,
"floor": "Floor8",
"location": "London",
"use_median": True
}
for channel in ['0', '25', '26', '27']:
configuration_json['channel'] = channel
data = io_interface.load_raw_fingerprint_data(
configuration_json=configuration_json)
unique_beacons = pandas.Series.unique(data[-1]['beacon'])
processed_data = processing_interface.process_static_data(
data=data, pcaComponentsCount=10, verbose=False)
processed_rolling_data = processing_interface.process_rolling_data(
data=data, window_size=5, window_step=5)
validation_features = processing_interface.process_validation_data(
rolled_data=processed_rolling_data,
processed_data=processed_data,
full_beacons=unique_beacons)
for direction in constants.DIRECTIONS:
processed_data[direction]['validation'] = validation_features[
def step_impl(context, method):
raw_data_frame = io.load_raw_fingerprint_data(configuration_json=context.configuration_json)
assert isinstance(raw_data_frame, pandas.DataFrame), 'raw data is dataframe'
assert all([x in raw_data_frame.columns for x in [u'timestamp', u'node', u'beacon', u'rssi', u'direction', u'set']])
assert not np.any(raw_data_frame.isnull().values), 'all data is valid'
context.raw_data = raw_data_frame