def sensor_failure_marker(attachment_marker_stream_id: uuid, mshrv_accel_id: uuid, mshrv_gyro_id: uuid, wrist: str, owner_id: uuid, dd_stream_name, CC: CerebralCortex, config: dict): """ Label a window as packet-loss if received packets are less than the expected packets. All the labeled data (st, et, label) with its metadata are then stored in a datastore. :param stream_id: :param CC_obj: :param config: """ # using stream_id, data-diagnostic-stream-id, and owner id to generate a unique stream ID for battery-marker sensor_failure_stream_id = uuid.uuid3(uuid.NAMESPACE_DNS, str( attachment_marker_stream_id + dd_stream_name + owner_id + "SENSOR FAILURE MARKER")) stream_days = get_stream_days(attachment_marker_stream_id, sensor_failure_stream_id, CC) try: for day in stream_days: # load stream data to be diagnosed attachment_marker_stream = CC.get_datastream(attachment_marker_stream_id, day, data_type=DataSet.COMPLETE) results = OrderedDict() if attachment_marker_stream.data: for marker_window in attachment_marker_stream.data: if "MOTIONSENSE-ON-BODY" in marker_window.sample: mshrv_accel_stream = CC.get_datastream(mshrv_accel_id, day, data_type=DataSet.ONLY_DATA, start_time=marker_window.start_time, end_time=marker_window.end_time) mshrv_gyro_stream = CC.get_datastream(mshrv_gyro_id, day, data_type=DataSet.ONLY_DATA, start_time=marker_window.start_time, end_time=marker_window.end_time) results_accel = process_windows(mshrv_accel_stream, config) results_gyro = process_windows(mshrv_gyro_stream, config) key = marker_window.start_time, marker_window.end_time # if sensor failure period is more than 12 hours then mark it as a sensor failure if results_accel > 0 and results_gyro < 1: sample = "MOTIONSENE-HRV-" + str(wrist) + "ACCELEROMETER-FAILURE" results[key].append(DataPoint(marker_window.start_time, marker_window.end_time, sample)) elif results_accel < 1 and results_gyro > 0: sample = "MOTIONSENE-HRV-" + str(wrist) + "GYRO-FAILURE" results[key].append(DataPoint(marker_window.start_time, marker_window.end_time, sample)) merged_windows = merge_consective_windows(results) if len(results) > 0: input_streams = [{"owner_id": owner_id, "id": str(attachment_marker_stream_id), "name": attachment_marker_stream.name}] output_stream = {"id": sensor_failure_stream_id, "name": dd_stream_name, "algo_type": config["algo_type"]["sensor_failure"]} metadata = get_metadata(dd_stream_name, input_streams, config) store(merged_windows, input_streams, output_stream, metadata, CC, config) except Exception as e: print(e)
def test_DataPoint(self): ts = datetime.datetime.now() dp = DataPoint(start_time=ts, end_time=ts, sample={'Foo': 123}, metadata={'label': 'good'}) self.assertDictEqual(dp.sample, {'Foo': 123}) self.assertEqual(dp.start_time, ts) self.assertEqual(dp.end_time, ts) self.assertEqual(dp.metadata, {'label': 'good'})
def setUpClass(cls): configuration_file = os.path.join(os.path.dirname(__file__), '../../../cerebralcortex.yml') cls.CC = CerebralCortex(configuration_file, master="local[*]", name="Data Diagnostic App", time_zone="US/Central") cls.config = Configuration( filepath="../data_diagnostic/data_diagnostic_config.yml").config cls.sample_battery_data = [] for row in range(1, 481): if row < 61: battery = 87.0 elif row > 60 and row < 120: battery = 0.0 elif row > 120 and row < 240: battery = 87.0 elif row > 240 and row < 300: battery = 7.0 elif row > 300 and row < 360: battery = 0.0 elif row > 360: battery = 60.0 tz = pytz.timezone("US/Central") start_time = tz.localize( datetime.fromtimestamp( int(round((time.time() + row) * 1000)) / 1e3)) dp = DataPoint(start_time=start_time, sample=battery) cls.sample_battery_data.append(dp) cls.window_size = 60
def data(self, value): result = [] for dp in value: result.append( DataPoint(self._identifier, dp.start_time, dp.end_time, dp.sample)) self._data = result
def data_quality_led(windowed_data): """ :param windowed_data: a datastream with a collection of windows :return: a list of window labels """ window_list = windowed_data dps = [] for key, window in window_list.items(): quality_results = compute_quality(window) dps.append(DataPoint(key[0], key[1], quality_results)) return dps
def test_06_store_stream(self): identifier = "6db98dfb-d6e8-4b27-8d55-95b20fa0f754" owner = "06634264-56bc-4c92-abd7-377dbbad79dd" name = "data-store-test" data_descriptor = {} execution_context = json.loads( '{"execution_context": {"algorithm": {"method": "cerebralcortex.data_processor.data_diagnostic.BatteryDataMarker"}}}' ) annotations = {} datapoints = [] stream_type = "datastream" start_time = datetime.datetime(2017, 4, 24, 0, 0, 1) end_time = datetime.datetime(2017, 4, 24, 0, 0, 2) localtz = timezone('US/Central') start_time = localtz.localize(start_time) end_time = localtz.localize(end_time) sample = {'Foo3': 123} dp1 = DataPoint(start_time=start_time, end_time=end_time, sample=sample) datapoints.append(dp1) ds = DataStream(identifier, owner, name, data_descriptor, execution_context, annotations, stream_type, start_time, end_time, datapoints) self.CC.save_datastream(ds) stream = self.CC.get_datastream(identifier, data_type=DataSet.COMPLETE) self.assertEqual(stream._identifier, identifier) self.assertEqual(stream._owner, owner) self.assertEqual(stream._name, name) self.assertEqual(stream._data_descriptor, data_descriptor) self.assertEqual(stream._execution_context, execution_context) self.assertEqual(stream._annotations, annotations) self.assertEqual(stream._datastream_type, stream_type) self.assertEqual(stream.data[0].start_time, start_time) self.assertEqual(stream.data[0].end_time, end_time) self.assertEqual(stream.data[0].sample, sample)
def test_07_stream_filter(self): identifier_anno = "6db98dfb-d6e8-4b27-8d55-95b20fa0f750" identifier_data = "6db98dfb-d6e8-4b27-8d55-95b20fa0f751" owner_id = "06634264-56bc-4c92-abd7-377dbbad79dd" name_anno = "data-store-test-annotation" name_data = "data-store-test-data" data_descriptor = {} execution_context_anno = json.loads( '{"execution_context": {"algorithm": {"method": "test.data_store.annotation.filter"}}}' ) execution_context_data = json.loads( '{"execution_context": {"algorithm": {"method": "test.data_store.data.filter"}}}' ) annotations_data = json.loads( '[{"name": "test-case","identifier": "6db98dfb-d6e8-4b27-8d55-95b20fa0f750"}]' ) annotations_anno = {} datapoints_anno = [] datapoints_data = [] result_data = Metadata(self.CC).is_id_created(owner_id, name_data, execution_context_data) if result_data["status"] != "new": identifier_data = result_data["id"] Metadata(self.CC).store_stream_info( identifier_anno, owner_id, name_anno, data_descriptor, execution_context_anno, annotations_anno, "annotations", datetime.datetime(2017, 4, 24, 0, 0, 1), datetime.datetime(2017, 4, 24, 0, 0, 5), result_data["status"]) result_anno = Metadata(self.CC).is_id_created(owner_id, name_data, execution_context_data) if result_anno["status"] != "new": identifier_anno = result_anno["id"] Metadata(self.CC).store_stream_info( identifier_data, owner_id, name_data, data_descriptor, execution_context_data, annotations_data, "datastream", datetime.datetime(2017, 4, 24, 0, 0, 1), datetime.datetime(2017, 4, 24, 0, 0, 5), result_anno["status"]) for i in range(0, 5): if (i % 2 == 0): sample_anno = 'good' else: sample_anno = 'bad' sample_data = i, i + 2, i + 3 start_time_anno = datetime.datetime(2017, 4, 24, 0, 0, i) end_time_anno = datetime.datetime(2017, 4, 24, 0, 0, (5 + i)) start_time_data = datetime.datetime(2017, 4, 24, 0, 0, i) end_time_data = datetime.datetime(2017, 4, 24, 0, 0, (3 + i)) localtz = timezone('US/Central') start_time_anno = localtz.localize(start_time_anno) end_time_anno = localtz.localize(end_time_anno) start_time_data = localtz.localize(start_time_data) end_time_data = localtz.localize(end_time_data) datapoints_anno.append( DataPoint(start_time=start_time_anno, end_time=end_time_anno, sample=sample_anno)) datapoints_data.append( DataPoint(start_time=start_time_data, end_time=end_time_data, sample=sample_data)) ds_anno = DataStream(uuid.UUID(identifier_anno), owner_id, name_anno, data_descriptor, execution_context_anno, annotations_data, "annotations", start_time_anno, end_time_anno, datapoints_anno) ds_data = DataStream(uuid.UUID(identifier_data), owner_id, name_data, data_descriptor, execution_context_data, annotations_anno, "datastream", start_time_anno, end_time_anno, datapoints_data) self.CC.save_datastream(ds_anno) self.CC.save_datastream(ds_data) filted_stream = self.CC.filter_stream(identifier_data, "test-case", "good") self.assertEqual(len(filted_stream), 5) for i in range(0, 5): sample_data = [i, i + 2, i + 3] start_time_data = datetime.datetime(2017, 4, 24, 0, 0, i) end_time_data = datetime.datetime(2017, 4, 24, 0, 0, (3 + i)) start_time_data = localtz.localize(start_time_data) end_time_data = localtz.localize(end_time_data) self.assertEqual(filted_stream[i].start_time, start_time_data) self.assertEqual(filted_stream[i].end_time, end_time_data) self.assertEqual(filted_stream[i].sample, sample_data)
def test_DataPoint_None(self): dp = DataPoint() self.assertIsNone(dp.start_time) self.assertIsNone(dp.end_time) self.assertIsNone(dp.sample) self.assertIsNone(dp.metadata)