def read_precleaned(subject_id):
psg_path = str(
utils.get_project_root().joinpath('data/labels/' + subject_id +
'_labeled_sleep.txt'))
data = []
with open(psg_path, 'rt') as csv_file:
file_reader = csv.reader(csv_file, delimiter=' ', quotechar='|')
count = 0
rows_per_epoch = 1
for row in file_reader:
if count == 0:
start_time = float(row[0])
start_score = int(row[1])
epoch = Epoch(timestamp=start_time, index=1)
data.append(
StageItem(epoch=epoch,
stage=PSGConverter.get_label_from_int(
start_score)))
else:
timestamp = start_time + count * 30
score = int(row[1])
epoch = Epoch(
timestamp=timestamp,
index=(1 + int(np.floor(count / rows_per_epoch))))
data.append(
StageItem(
epoch=epoch,
stage=PSGConverter.get_label_from_int(score)))
count = count + 1
return PSGRawDataCollection(subject_id=subject_id, data=data)
def parse(report_summary, psg_stage_path):
data = []
with open(psg_stage_path, 'rt') as csv_file:
file_reader = csv.reader(csv_file, delimiter=',', quotechar='|')
count = 0
rows_per_epoch = Epoch.DURATION / VitaportProcessor.DT_TXT_PSG
for row in file_reader:
if count == 0:
start_time = row[1]
start_score = int(row[0])
report_summary.start_time = start_time
start_time_seconds = TimeService.get_start_epoch_timestamp(report_summary)
epoch = Epoch(timestamp=start_time_seconds, index=1)
data.append(StageItem(epoch=epoch, stage=PSGConverter.get_label_from_int(start_score)))
if np.mod(count, rows_per_epoch) == 0 and count != 0:
timestamp = start_time_seconds + count * VitaportProcessor.DT_TXT_PSG
score = int(row[0])
epoch = Epoch(timestamp=timestamp,
index=(1 + int(np.floor(count / rows_per_epoch))))
data.append(StageItem(epoch=epoch, stage=PSGConverter.get_label_from_int(score)))
count = count + 1
return data
def test_build(self, mock_load_cropped_array):
subject_id = 'subjectA'
data = np.array([[1, 1], [10, 2], [20, 0], [40, 1], [70, 2], [90, 3],
[100, 1], [120, 2]])
valid_epochs = [
Epoch(timestamp=10, index=1),
Epoch(timestamp=40, index=2)
]
mock_load_cropped_array.return_value = data
expected_labels = np.array([2, 1])
returned_labels = PSGLabelService.build(subject_id, valid_epochs)
self.assertEqual(expected_labels.tolist(), returned_labels.tolist())
def test_constructor(self):
stage = SleepStage.rem
epoch_time = 3
epoch_index = 4
stage_item = StageItem(Epoch(timestamp=epoch_time, index=epoch_index),
stage=stage)
self.assertEqual(stage, stage_item.stage)
self.assertEqual(epoch_time, stage_item.epoch.timestamp)
self.assertEqual(epoch_index, stage_item.epoch.index)
def test_parse(self, mock_csv, mock_open, mock_time_service):
mock_csv.reader.return_value = ['W', '1', 'W', '2', 'R']
report_summary = ReportSummary(study_date="04/02/2019",
start_epoch=2,
start_time="10:30:13 PM",
file_type=PSGFileType.Compumedics)
psg_stage_path = 'path/to/file'
mock_time_service.get_start_epoch_timestamp.return_value = expected_start_timestamp = 1234567890
expected_data = [
StageItem(epoch=Epoch(timestamp=expected_start_timestamp, index=2),
stage=SleepStage.n1),
StageItem(epoch=Epoch(timestamp=expected_start_timestamp +
Epoch.DURATION,
index=3),
stage=SleepStage.wake),
StageItem(epoch=Epoch(timestamp=expected_start_timestamp +
Epoch.DURATION * 2,
index=4),
stage=SleepStage.n2),
StageItem(epoch=Epoch(timestamp=expected_start_timestamp +
Epoch.DURATION * 3,
index=5),
stage=SleepStage.rem)
]
data = CompumedicsProcessor.parse(report_summary, psg_stage_path)
TestHelper.assert_models_equal(self, expected_data[0].epoch,
data[0].epoch)
TestHelper.assert_models_equal(self, expected_data[0].stage,
data[0].stage)
TestHelper.assert_models_equal(self, expected_data[1].epoch,
data[1].epoch)
TestHelper.assert_models_equal(self, expected_data[1].stage,
data[1].stage)
TestHelper.assert_models_equal(self, expected_data[2].epoch,
data[2].epoch)
TestHelper.assert_models_equal(self, expected_data[2].stage,
data[2].stage)
TestHelper.assert_models_equal(self, expected_data[3].epoch,
data[3].epoch)
TestHelper.assert_models_equal(self, expected_data[3].stage,
data[3].stage)
def test_get_window(self):
timestamps = np.array([-1000, -500, 32, 50, 60, 800, 1000])
epoch = Epoch(timestamp=55, index=120)
expected_indices_in_range = np.array([2, 3, 4])
actual_indices_in_range = HeartRateFeatureService.get_window(
timestamps, epoch)
self.assertEqual(expected_indices_in_range.tolist(),
actual_indices_in_range.tolist())
def test_get_window(self):
timestamps = np.array([-2000, 22, 32, 50, 60, 800, 1000])
epoch = Epoch(timestamp=55, index=120)
expected_indices_in_range = np.array([1, 2, 3, 4])
actual_indices_in_range = ActivityCountFeatureService.get_window(
timestamps, epoch)
self.assertEqual(expected_indices_in_range.tolist(),
actual_indices_in_range.tolist())
def test_constructor_and_get_functions(self):
subject_id = "subject9000"
data = [
StageItem(Epoch(timestamp=2, index=2), stage=SleepStage.rem),
StageItem(Epoch(timestamp=200, index=4), stage=SleepStage.n1)
]
expected_interval = Interval(start_time=2, end_time=200)
expected_np_array = np.array([[2, 5], [200, 1]])
psg_raw_data_collection = PSGRawDataCollection(subject_id=subject_id,
data=data)
self.assertListEqual(data, psg_raw_data_collection.data)
self.assertEqual(subject_id, psg_raw_data_collection.subject_id)
TestHelper.assert_models_equal(self, expected_interval,
psg_raw_data_collection.get_interval())
self.assertListEqual(expected_np_array.tolist(),
psg_raw_data_collection.get_np_array().tolist())
def load_cropped(subject_id):
cropped_array = PSGService.load_cropped_array(subject_id)
stage_items = []
for row in range(np.shape(cropped_array)[0]):
value = cropped_array[row, 1]
stage_items.append(
StageItem(epoch=Epoch(timestamp=cropped_array[row, 0],
index=row),
stage=PSGConverter.get_label_from_int(value)))
return PSGRawDataCollection(subject_id=subject_id, data=stage_items)
def test_build_feature_array(self, mock_load_cropped, mock_get_feature):
subject_id = "subjectA"
data = np.array([[1, 10], [10, 220], [20, 0], [40, 500], [70, 200],
[90, 0], [100, 0], [120, 4]])
activity_count_collection = ActivityCountCollection(
subject_id=subject_id, data=data)
mock_load_cropped.return_value = activity_count_collection
expected_features = [np.array([0.1]), np.array([0.2])]
mock_get_feature.side_effect = expected_features
expected_feature_array = np.array(expected_features)
valid_epochs = [
Epoch(timestamp=4, index=1),
Epoch(timestamp=50, index=2)
]
returned_feature_array = ActivityCountFeatureService.build(
subject_id, valid_epochs)
self.assertEqual(expected_feature_array.tolist(),
returned_feature_array.tolist())
def test_parse(self, mock_csv, mock_open, mock_time_service):
mock_csv.reader.return_value = [['0', '23:20:55'], ['0', '23:21:05'], ['0', '23:21:15'], ['2', '23:21:25'],
['2', '23:21:35']]
report_summary = ReportSummary(study_date="04/02/2019",
start_epoch=1,
start_time="11:20:55 PM",
file_type=PSGFileType.Vitaport)
psg_stage_path = 'path/to/file'
mock_time_service.get_start_epoch_timestamp.return_value = expected_start_timestamp = 1234567890
expected_data = [StageItem(epoch=Epoch(timestamp=expected_start_timestamp,
index=1), stage=SleepStage.wake),
StageItem(epoch=Epoch(timestamp=expected_start_timestamp + Epoch.DURATION,
index=2), stage=SleepStage.n2)]
data = VitaportProcessor.parse(report_summary, psg_stage_path)
TestHelper.assert_models_equal(self, expected_data[0].epoch, data[0].epoch)
TestHelper.assert_models_equal(self, expected_data[0].stage, data[0].stage)
TestHelper.assert_models_equal(self, expected_data[1].epoch, data[1].epoch)
TestHelper.assert_models_equal(self, expected_data[1].stage, data[1].stage)
def test_crop(self):
subject_id = 'subjectA'
crop_interval = Interval(start_time=50, end_time=101)
data = [
StageItem(epoch=Epoch(timestamp=10, index=2), stage=SleepStage.n1),
StageItem(epoch=Epoch(timestamp=50, index=3), stage=SleepStage.n2),
StageItem(epoch=Epoch(timestamp=100, index=4),
stage=SleepStage.n3),
StageItem(epoch=Epoch(timestamp=170, index=5),
stage=SleepStage.rem)
]
expected_data = [
StageItem(epoch=Epoch(timestamp=50, index=3), stage=SleepStage.n2),
StageItem(epoch=Epoch(timestamp=100, index=4),
stage=SleepStage.n3),
]
input_raw_data_collection = PSGRawDataCollection(subject_id=subject_id,
data=data)
returned_psg_raw_collection = PSGService.crop(
input_raw_data_collection, crop_interval)
self.assertEqual(subject_id, returned_psg_raw_collection.subject_id)
TestHelper.assert_models_equal(self, expected_data[0],
returned_psg_raw_collection.data[0])
TestHelper.assert_models_equal(self, expected_data[1],
returned_psg_raw_collection.data[1])
def test_builds_features(self, mock_get_valid_epochs, mock_build_labels,
mock_build_from_wearables, mock_build_from_time):
subject_id = "subjectA"
mock_get_valid_epochs.return_value = valid_epochs = [
Epoch(timestamp=1, index=1000)
]
FeatureBuilder.build(subject_id)
mock_get_valid_epochs.assert_called_once_with(subject_id)
mock_build_labels.assert_called_once_with(subject_id, valid_epochs)
mock_build_from_wearables.assert_called_once_with(
subject_id, valid_epochs)
mock_build_from_time.assert_called_once_with(subject_id, valid_epochs)
def test_load_cropped_array(self, mock_pd):
subject_id = 'subject100'
cropped_psg_path = Constants.CROPPED_FILE_PATH.joinpath(
subject_id + "_cleaned_psg.out")
mock_pd.read_csv.return_value.values = np.array([[1, 2], [4, 5],
[10, 1]])
psg_collection = PSGService.load_cropped(subject_id)
mock_pd.read_csv.assert_called_once_with(str(cropped_psg_path),
delimiter=' ')
TestHelper.assert_models_equal(
self,
StageItem(epoch=Epoch(timestamp=1, index=0), stage=SleepStage.n2),
psg_collection.data[0])
TestHelper.assert_models_equal(
self,
StageItem(epoch=Epoch(timestamp=4, index=1), stage=SleepStage.rem),
psg_collection.data[1])
TestHelper.assert_models_equal(
self,
StageItem(epoch=Epoch(timestamp=10, index=2), stage=SleepStage.n1),
psg_collection.data[2])
def test_write(self, mock_np):
subject_id = 'subjectA'
mock_np.array.return_value = returned_array = np.array([[1, 2], [3,
4]])
list_data = [[10, 1], [50, 2], [100, 3], [170, 5]]
data = [
StageItem(epoch=Epoch(timestamp=10, index=2), stage=SleepStage.n1),
StageItem(epoch=Epoch(timestamp=50, index=3), stage=SleepStage.n2),
StageItem(epoch=Epoch(timestamp=100, index=4),
stage=SleepStage.n3),
StageItem(epoch=Epoch(timestamp=170, index=5),
stage=SleepStage.rem)
]
psg_raw_data_collection = PSGRawDataCollection(subject_id=subject_id,
data=data)
psg_output_path = Constants.CROPPED_FILE_PATH.joinpath(
"subjectA_cleaned_psg.out")
PSGService.write(psg_raw_data_collection)
mock_np.array.assert_called_once_with(list_data)
mock_np.savetxt.assert_called_once_with(psg_output_path,
returned_array,
fmt='%f')
def parse(report_summary, psg_stage_path):
data = []
score_strings = []
with open(psg_stage_path, 'rt') as csv_file:
file_reader = csv.reader(csv_file, delimiter=',', quotechar='|')
for row in file_reader:
score_strings.append(row[0])
start_epoch = report_summary.start_epoch
start_time_seconds = TimeService.get_start_epoch_timestamp(report_summary)
for epoch_index in range(start_epoch - 1, len(score_strings)):
timestamp = start_time_seconds + (epoch_index - start_epoch + 1) * CompumedicsProcessor.DT_COMPUMEDICS_PSG
epoch = Epoch(timestamp=timestamp, index=epoch_index + 1)
stage = PSGConverter.get_label_from_string(score_strings[epoch_index])
data.append(StageItem(epoch=epoch, stage=stage))
return data
def build(file_id):
if Constants.VERBOSE:
print('Building MESA subject ' + file_id + '...')
raw_labeled_sleep = MesaPSGService.load_raw(file_id)
heart_rate_collection = MesaHeartRateService.load_raw(file_id)
activity_count_collection = MesaActigraphyService.load_raw(file_id)
circadian_model = MesaTimeBasedService.load_circadian_model(file_id)
if activity_count_collection.data[0][0] != -1 and circadian_model is not None:
circadian_model = utils.remove_nans(circadian_model)
interval = Interval(start_time=0, end_time=np.shape(raw_labeled_sleep)[0])
activity_count_collection = ActivityCountService.crop(activity_count_collection, interval)
heart_rate_collection = HeartRateService.crop(heart_rate_collection, interval)
valid_epochs = []
for timestamp in range(interval.start_time, interval.end_time, Epoch.DURATION):
epoch = Epoch(timestamp=timestamp, index=len(valid_epochs))
activity_count_indices = ActivityCountFeatureService.get_window(activity_count_collection.timestamps,
epoch)
heart_rate_indices = HeartRateFeatureService.get_window(heart_rate_collection.timestamps, epoch)
if len(activity_count_indices) > 0 and 0 not in heart_rate_collection.values[heart_rate_indices]:
valid_epochs.append(epoch)
else:
pass
labeled_sleep = np.expand_dims(
MesaPSGService.crop(psg_labels=raw_labeled_sleep, valid_epochs=valid_epochs),
axis=1)
feature_count = ActivityCountFeatureService.build_from_collection(activity_count_collection,
valid_epochs)
feature_hr = HeartRateFeatureService.build_from_collection(heart_rate_collection, valid_epochs)
feature_time = np.expand_dims(TimeBasedFeatureService.build_time(valid_epochs), axis=1)
feature_cosine = np.expand_dims(TimeBasedFeatureService.build_cosine(valid_epochs), axis=1)
feature_circadian = TimeBasedFeatureService.build_circadian_model_from_raw(circadian_model,
valid_epochs)
feature_dictionary = {FeatureType.count: feature_count,
FeatureType.heart_rate: feature_hr,
FeatureType.time: feature_time,
FeatureType.circadian_model: feature_circadian,
FeatureType.cosine: feature_cosine}
subject = Subject(subject_id=file_id,
labeled_sleep=labeled_sleep,
feature_dictionary=feature_dictionary)
# Uncomment to save files for all subjects
# ax = plt.subplot(5, 1, 1)
# ax.plot(range(len(feature_hr)), feature_hr)
# ax = plt.subplot(5, 1, 2)
# ax.plot(range(len(feature_count)), feature_count)
# ax = plt.subplot(5, 1, 3)
# ax.plot(range(len(feature_cosine)), feature_cosine)
# ax = plt.subplot(5, 1, 4)
# ax.plot(range(len(feature_circadian)), feature_circadian)
# ax = plt.subplot(5, 1, 5)
# ax.plot(range(len(labeled_sleep)), labeled_sleep)
#
# plt.savefig(str(Constants.FIGURE_FILE_PATH.joinpath(file_id + '.png')))
# plt.close()
return subject