class XMLOptionsParser(object):
def __init__(self, optionsFilePath, langHelper, tempDbPath, projSrcPath,
stopWords):
self.optionsFilePath = optionsFilePath
self.algorithmFactory = AlgorithmFactory(langHelper, tempDbPath)
self.graphFactory = GraphFactory(langHelper, tempDbPath, projSrcPath,
stopWords)
def getAlgorithms(self, navPathType):
tree = ElementTree(file=self.optionsFilePath)
root = tree.getroot()
graphAlgorithmsMap = {}
for child in root:
if child.tag == 'algorithms' and self.__isNavPathType(
child, navPathType):
graph = self.__getGraph(child)
algorithms = self.__getAlgorithms(child)
graphAlgorithmsMap[graph] = algorithms
return graphAlgorithmsMap
def __isNavPathType(self, algorithmsNode, navPathType):
navPathTypeStr = None
if 'navPathType' in algorithmsNode.attrib:
navPathTypeAttrib = algorithmsNode.attrib['navPathType']
navPathTypeStr = str(navPathTypeAttrib)
else:
navPathTypeStr = 'Default'
if navPathTypeStr.lower() == navPathType.lower():
return True
else:
return False
def __getGraph(self, algorithmsNode):
graphType = None
variantsDb = None
if 'graphType' in algorithmsNode.attrib:
graphType = algorithmsNode.attrib["graphType"]
if 'variantsDb' in algorithmsNode.attrib:
variantsDb = algorithmsNode.attrib["variantsDb"]
return self.graphFactory.getGraph(graphType, variantsDb)
def __getAlgorithms(self, algorithmsNode):
algorithms = []
suffix = None
if "suffix" in algorithmsNode.attrib:
suffix = algorithmsNode.attrib["suffix"]
for child in algorithmsNode:
if child.tag == 'algorithm':
algorithm = self.algorithmFactory.getAlgorithm(child, suffix)
if algorithm != None:
algorithms.append(algorithm)
return algorithms
class XMLOptionsParser(object):
def __init__(self, optionsFilePath, langHelper, tempDbPath, projSrcPath, stopWords):
self.optionsFilePath = optionsFilePath
self.algorithmFactory = AlgorithmFactory(langHelper, tempDbPath)
self.graphFactory = GraphFactory(langHelper, tempDbPath, projSrcPath, stopWords)
def getAlgorithms(self, navPathType):
tree = ElementTree(file=self.optionsFilePath)
root = tree.getroot()
graphAlgorithmsMap = {}
for child in root:
if child.tag == 'algorithms' and self.__isNavPathType(child, navPathType):
graph = self.__getGraph(child)
algorithms = self.__getAlgorithms(child)
graphAlgorithmsMap[graph] = algorithms
return graphAlgorithmsMap
def __isNavPathType(self, algorithmsNode, navPathType):
navPathTypeStr = None
if 'navPathType' in algorithmsNode.attrib:
navPathTypeAttrib = algorithmsNode.attrib['navPathType']
navPathTypeStr = str(navPathTypeAttrib)
else:
navPathTypeStr = 'Default'
if navPathTypeStr.lower() == navPathType.lower():
return True
else:
return False
def __getGraph(self, algorithmsNode):
graphType = None
variantsDb = None
if 'graphType' in algorithmsNode.attrib:
graphType = algorithmsNode.attrib["graphType"]
if 'variantsDb' in algorithmsNode.attrib:
variantsDb = algorithmsNode.attrib["variantsDb"]
return self.graphFactory.getGraph(graphType, variantsDb)
def __getAlgorithms(self, algorithmsNode):
algorithms = []
suffix = None
if "suffix" in algorithmsNode.attrib:
suffix = algorithmsNode.attrib["suffix"]
for child in algorithmsNode:
if child.tag == 'algorithm':
algorithm = self.algorithmFactory.getAlgorithm(child, suffix)
if algorithm != None:
algorithms.append(algorithm)
return algorithms
def getFeatures(self,
eegSegment,
timeStampSegment=0,
time_step=0,
local_mu=0,
local_sigma=0):
extractorTypes = self.extractorType.split(',')
extractorCnt = 0
for extractorType in extractorTypes:
factory = AlgorithmFactory(extractorType.rstrip().lstrip())
extractor = factory.generateExtractor()
# print('eegSegment.shape = ' + str(eegSegment.shape))
features = extractor.getFeatures(eegSegment)
# print('****** features.shape = ' + str(features.shape))
if extractorCnt == 0:
merged = features
else:
merged = np.concatenate((merged, features), axis=0)
extractorCnt += 1
# print('****** merged.shape = ' + str(merged.shape))
# print('########### final merged.shape = ' + str(merged.shape))
mergedTransposed = merged.transpose()
return mergedTransposed
def classifySequentially(params, paramID, paramDir, fileIDpair):
# print('classifySequentially by ' + str(fileID))
pickledDir = params.pickledDir
extractorType = params.extractorType
classifierType = params.classifierType
classifierParams = params.classifierParams
samplingFreq = params.samplingFreq
windowSizeInSec = params.windowSizeInSec
wsizeInSamplePointNum = windowSizeInSec * samplingFreq # window size in sample points. data is sampled at 128 Hz, so 1280 sample points = 10 sec.
timeWindowStrideInSec = params.timeWindowStrideInSec
timeWindowStrideInSamplePointNum = timeWindowStrideInSec * samplingFreq
markovOrderForPrediction = params.markovOrderForPrediction
strideNumInTimeWindow = np.ceil(windowSizeInSec / timeWindowStrideInSec)
classifierType = params.classifierType
eegFilePrefix = params.eegFilePrefix
classifierFilePrefix = params.classifierFilePrefix
# replacesWWWRtoWWWW = params.replacesWWWRtoWWWW
numOfConsecutiveWsThatProhibitsR = params.numOfConsecutiveWsThatProhibitsR
predict_by_batch = params.predict_by_batch
testFileID = fileIDpair[0]
classifierID = fileIDpair[1]
print('$#$#$# in sequentialPrediction, classifierID =', classifierID)
predictionTargetDataFilePath = pickledDir + '/' + eegFilePrefix + '.' + testFileID + '.pkl'
print('predictionTargetDataFilePath =', predictionTargetDataFilePath)
dataFileHandler = open(predictionTargetDataFilePath, 'rb')
(eeg, ch2, stageSeq, timeStamps) = pickle.load(dataFileHandler)
dataFileHandler.close()
print('eeg.shape = ' + str(eeg.shape))
print('len(stageSeq) = ' + str(len(stageSeq)))
if params.useEMG:
label4EMG = params.label4withEMG
else:
label4EMG = params.label4withoutEMG
params.classifierName = classifierFilePrefix + '.' + classifierType + '.' + label4EMG + '.excludedFileID.' + testFileID + '.classifierID.' + classifierID
factory = AlgorithmFactory(extractorType)
extractor = factory.generateExtractor()
# if params.classifierType == 'deep':
# classifier = DeepClassifier(classifierID=testFileID)
# else:
classifierDir = params.classifierDir
pwd = dirname(abspath(__file__))
print('# classifierID =', classifierID)
# classLabels = list(params.labelCorrectionDict.keys())[:params.maximumStageNum]
classLabels = self.params.sampleClassLabels[:self.params.maximumStageNum]
paramFileName = 'params.' + classifierID + '.json'
params_for_network_structure = ParameterSetup(paramDir=paramDir,
paramFileName=paramFileName)
if params.classifierType == 'deep':
classifier = DeepClassifier(
classLabels,
classifierID=classifierID,
paramsForDirectorySetup=params,
paramsForNetworkStructure=params_for_network_structure)
# if classifierID == '':
# model_path = pwd + '/' + paramDir + '/model.pkl'
#else:
# model_path = pwd + '/' + params.deepParamsDir + '/' + testFileID + '/' + classifierID + '/model.pkl'
model_path = pwd + '/' + paramDir + '/weights.' + classifierID + '.pkl'
print('model_path = ', model_path)
classifier.load_weights(model_path)
else:
classifierFileName = params.classifierDir + '/' + params.classifierPrefix + '.' + classifierID + '.pkl'
classifierFileHandler = open(classifierFileName, 'rb')
classifier = pickle.load(classifierFileHandler)
classifierFileHandler.close()
stagePredictor = StagePredictor(params_for_network_structure, extractor,
classifier, classifierDir, classifierID,
markovOrderForPrediction)
### stagePredictor = StagePredictor(params, extractor, classifier, classifierDir, classifierID)
# stagePredictor = StagePredictor(params, extractor, classifier)
sLen = len(stageSeq)
samplePointNum = min(eeg.shape[0], sLen * wsizeInSamplePointNum)
print('%$%$%$ sLen =', sLen)
print('eeg.shape[0] =', eeg.shape[0])
print('sLen * wsizeInSamplePointNum =', sLen * wsizeInSamplePointNum)
print('%$%$%$ samplePointNum =', samplePointNum)
print('%$%$%$ timeWindowStrideInSamplePointNum =',
timeWindowStrideInSamplePointNum)
# reset eeg and emg statistical values
# eeg_mean = 0
# eeg_variance = 0
# emg_mean = 0
# emg_variance= 0
# oldSampleNum = 0
y_test_L = []
y_pred_L = []
wID = 0
# startSamplePoint = 0
# stagePredictions_L = []
replacedR = False
# while startSamplePoint + wsizeInSamplePointNum <= samplePointNum:
records_L = []
timeStampSegments_L = []
### all_past_eeg, all_past_ch2 = np.array([]), np.array([])
standardizer_eeg = standardizer(samplePointNum)
standardizer_ch2 = standardizer(samplePointNum)
# standardized_all_past_eeg = np.array([])
for startSamplePoint in range(0, samplePointNum,
timeWindowStrideInSamplePointNum):
endSamplePoint = startSamplePoint + wsizeInSamplePointNum
if endSamplePoint > samplePointNum:
break
timeStampSegment = timeStamps[startSamplePoint:endSamplePoint]
eegSegment = eeg[startSamplePoint:endSamplePoint]
### standardized_eegSegment, all_past_eeg = standardize(eegSegment, all_past_eeg)
standardized_eegSegment = standardizer_eeg.standardize(eegSegment)
if params.useEMG:
ch2Segment = ch2[startSamplePoint:endSamplePoint]
### standardized_ch2Segment, all_past_ch2 = standardize(ch2Segment, all_past_ch2)
standardized_ch2Segment = standardizer_ch2.standardize(ch2Segment)
one_record = np.r_[standardized_eegSegment,
standardized_ch2Segment]
else:
one_record = standardized_eegSegment
# standardized_all_past_eeg = np.r_[standardized_all_past_eeg, standardized_eegSegment]
# print('standardized_all_past_eeg.mean() =', standardized_all_past_eeg.mean(), ', standardized_all_past_eeg.std() =', standardized_all_past_eeg.std())
'''
eegSegment_orig_mean = np.mean(eegSegment)
eegSegment_orig_std = np.std(eegSegment)
eeg_old_mean = eeg_mean
eeg_mean = recompMean(eegSegment, eeg_mean, oldSampleNum)
eeg_variance = recompVariance(eegSegment, eeg_variance, eeg_old_mean, eeg_mean, oldSampleNum)
standardized_eegSegment = (eegSegment - eeg_mean) / np.sqrt(eeg_variance)
if params.usech2:
ch2Segment = ch2[startSamplePoint:endSamplePoint]
ch2_old_mean = ch2_mean
ch2_mean = recompMean(ch2Segment, ch2_mean, oldSampleNum)
ch2_variance = recompVariance(ch2Segment, ch2_variance, ch2_old_mean, ch2_mean, oldSampleNum)
standardized_ch2Segment = (ch2Segment - ch2_mean) / np.sqrt(ch2_variance)
one_record = np.r_[standardized_eegSegment, standardized_ch2Segment]
else:
one_record = standardized_eegSegment
oldSampleNum += eegSegment.shape[0]
'''
# local_mu = np.mean(standardized_eegSegment)
# local_sigma = np.std(standardized_eegSegment)
# print('local_mu =', local_mu)
# print(' ')
# print('in sequentialPredictionDecisionTree.classifySequentially():')
### print('one_record.shape = ' + str(one_record.shape))
### print('one_record[:] = ' + str(one_record[:]))
# print('one_record[:10] = ' + str(one_record[:10]))
# print('one_record[:,1] = ' + str(one_record[:,1]))
# stageEstimate is one of ['w', 'n', 'r']
# stagePrediction = params.reverseLabel(stagePredictor.predict(one_record, local_mu=eegSegment_orig_mean, local_sigma=eegSegment_orig_std))
# print('np.mean(one_record[:,0]) = ' + str(np.mean(one_record[:])))
# print('np.std(one_record[:,0]) = ' + str(np.std(one_record[:])))
# print('before stagePredictor.predict')
# stagePrediction = params.reverseLabel(stagePredictor.predict(one_record, timeStampSegment, local_mu=local_mu, local_sigma=local_sigma, wID=wID))
if predict_by_batch:
records_L.append(one_record)
timeStampSegments_L.append(timeStampSegment)
else:
orig_prediction = stagePredictor.predict(
one_record, timeStampSegment, params.stageLabels4evaluation,
params.stageLabel2stageID)
# print(orig_prediction, end='')
stagePrediction = params.reverseLabel(orig_prediction)
# print('after stagePredictor.predict')
# replaces R to W when W appears consecutively
'''
Ws = 'W' * numOfConsecutiveWsThatProhibitsR
for wCnt in range(1,numOfConsecutiveWsThatProhibitsR+1):
if len(y_pred_L) >= wCnt:
if y_pred_L[len(y_pred_L)-wCnt] != 'W':
break
if stagePrediction == 'R':
print(Ws + '->R changed to ' + Ws + '->W at wID = ' + str(wID) + ', startSamplePoint = ' + str(startSamplePoint))
stagePrediction = 'W'
replacedR = True
'''
#----
# if the prediction is P, then use the previous one
if stagePrediction == 'P':
# print('stagePrediction == P for wID = ' + str(wID))
if len(y_pred_L) > 0:
stagePrediction = y_pred_L[len(y_pred_L) - 1]
else:
stagePrediction = 'M'
#-----
# append to the lists of results
y_pred_L.append(stagePrediction)
if wID >= len(stageSeq):
print('len(stageSeq) =', len(stageSeq), ', wID =', wID)
print('startSamplePoint =', startSamplePoint,
', samplePointNum =', samplePointNum,
', timeWindowStrideInSamplePointNum =',
timeWindowStrideInSamplePointNum)
trueLabel = stageSeq[wID]
if replacedR:
print(' -> for wID = ' + str(wID) + ', trueLabel = ' +
trueLabel)
replacedR = False
y_test_L.append(trueLabel)
wID += 1
#-----
# vote for choosing the label for 10 second window
'''
stagePredictions_L.append(stagePrediction)
if len(stagePredictions_L) == strideNumInTimeWindow:
finalStagePrediction = voteForStagePrediction(stagePredictions_L[-(markovOrderForPrediction+1):])
#-----
# append to the lists of results
y_pred_L.append(finalStagePrediction)
stagePredictions_L = []
# print('stagePredictions_L[' + stagePrediction + '] = ' + )
# startSamplePoint = endSamplePoint
trueLabel = stageSeq[wID]
if replacedR:
print(' -> for wID = ' + str(wID) + ', trueLabel = ' + trueLabel)
replacedR = False
y_test_L.append(trueLabel)
### print('wID = ' + str(wID) + ', trueLabel = ' + trueLabel + ', stagePrediction = ' + stagePrediction)
wID += 1
'''
if predict_by_batch:
# print('predicting by batch:')
y_pred = np.array([
params.reverseLabel(y_pred_orig)
for y_pred_orig in stagePredictor.batch_predict(
np.array(records_L),
np.array(timeStampSegments_L),
local_mu=local_mu,
local_sigma=local_sigma,
stageLabels4evaluation=params.stageLabels4evaluation,
stageLabel2stageID=params.stageLabel2stageID)
])
y_test = np.array(stageSeq)
else:
y_test = np.array(y_test_L)
y_pred = np.array(y_pred_L)
return (y_test, y_pred)
from parameterSetup import ParameterSetup
from algorithmFactory import AlgorithmFactory
args = sys.argv
if len(args) > 1:
option = args[1]
else:
option = ''
# get params shared by programs
params = ParameterSetup()
extractorType = params.extractorType
waveletWidths = params.waveletWidths
time_step = 1 / params.samplingFreq
factory = AlgorithmFactory(extractorType)
extractor = factory.generateExtractor()
#--------
# load data
fileIDs = ['DBL-NO-D0793-HALO-BL-20161005']
if option == '':
fileID = fileIDs[0]
else:
fileID = option
# print('fileID = ' + str(fileID))
dataFileHandler = open(
params.pickledDir + '/' + params.eegFilePrefix + '.' + fileID + '.pkl',
'rb')
(eeg, emg, stageSeq, timeStamps) = pickle.load(dataFileHandler)
def __init__(self,
recordWaves,
extractorType,
classifierType,
classifierID,
inputFileID='',
offsetWindowID=0,
chamberID=-1,
samplingFreq=0,
epochTime=0):
self.recordWaves = recordWaves
self.inputFileID = inputFileID
self.chamberID = chamberID
chamberLabel = '_chamber' + str(
self.chamberID +
1) # adds 1 because in Sleep Sign Recorder, chambers start from 1.
self.params = ParameterSetup()
if samplingFreq == 0:
self.samplingFreq = self.params.samplingFreq
else:
self.samplingFreq = samplingFreq
if epochTime == 0:
self.samplePointNum = self.params.windowSizeInSec * self.samplingFreq # the number of sample points received at once
else:
self.samplePointNum = epochTime * self.samplingFreq # the number of sample points received at once
self.graphUpdateFreqInHz = self.params.graphUpdateFreqInHz # frequency of updating the graph (if set to 1, redraws graph every second)
assert self.samplingFreq / self.graphUpdateFreqInHz == np.floor(
self.samplingFreq /
self.graphUpdateFreqInHz) # should be an integer
self.updateGraph_samplePointNum = np.int(self.samplingFreq /
self.graphUpdateFreqInHz)
# print('self.updateGraph_samplePointNum =', self.updateGraph_samplePointNum)
self.hasGUI = True
self.graphColors = ['b', 'g']
self.ylim_max_eeg, self.ylim_max_ch2 = 2.0, 2.0
self.graph_ylim = [[-self.ylim_max_eeg, self.ylim_max_eeg],
[-self.ylim_max_ch2, self.ylim_max_ch2]]
self.lightPeriodStartTime = self.params.lightPeriodStartTime
self.sampleID = 0
self.segmentID = offsetWindowID
# makes a classifier class
label4EMG = self.params.label4withoutEMG
self.showCh2 = self.params.showCh2
self.useCh2ForReplace = self.params.useCh2ForReplace
self.minimumCh2Intensity = 0
self.maximumCh2Intensity = 0
self.past_eegSegment, self.past_ch2Segment = np.array([]), np.array([])
# self.previous_eeg, self.previous_ch2 = np.array([]), np.array([])
classifierFilePrefix = self.params.classifierFilePrefix
factory = AlgorithmFactory(extractorType)
print('generating extractor: ')
self.extractor = factory.generateExtractor()
# self.classLabels = list(self.params.labelCorrectionDict.keys())[:self.params.maximumStageNum]
self.classLabels = self.params.sampleClassLabels[:self.params.
maximumStageNum]
self.setStagePredictor(classifierID)
presentTime = timeFormatting.presentTimeEscaped()
logFileID = 'classifier.' + presentTime
if self.chamberID != -1:
logFileID += chamberLabel
logFileName = logFileID + '.csv'
self.logFile = open(self.params.logDir + '/' + logFileName, 'a')
# connect to an output device
self.connected2serialClient = False
# print('in __init__ of classifierClient, self.connected2serialClient = False')
self.serialClient, self.connected2serialClient = connect_laser_device()
self.eeg_till_now = np.zeros((0, ))
'''
# prepare for computing Kolmogorov-Smirnov test
standardMice_L, files_L = readStandardMice(self.params)
self.statisticalTester = StatisticalTester(standardMice_L)
self.dSeries_L = []
self.dMat = readdMat(self.params)
self.dTensor = readdTensor(self.params)
self.dTensorSegmentNum = self.dTensor.shape[1]
self.segmentMax4computingKS = self.dTensorSegmentNum * 4
self.segmentMax4computingKS = self.dTensorSegmentNum
print('Computes KS (Kolmogorov-Smirnov) till the input reaches segment', self.segmentMax4computingKS)
'''
# opens a file for recording waves and prediction results
if self.inputFileID == '':
outputFileID = timeFormatting.presentTimeEscaped()
else:
outputFileID = self.inputFileID
if self.chamberID != -1:
outputFileID += chamberLabel
waveFileName = outputFileID + '_wave.csv'
self.ch2_mode = "Video"
self.ch2_thresh_value = self.params.ch2_thresh_default
self.eeg_normalize_for_prediction = 1
self.ch2_normalize_for_prediction = 0
self.eeg_graph_normalize = 0
self.ch2_graph_normalize = 0
self.currentCh2Intensity = 0
if self.recordWaves:
self.waveOutputFile = open(
self.params.waveOutputDir + '/' + waveFileName, 'a')
self.waveOutputFile_standardized = open(
self.params.waveOutputDir + '/standardized_' + waveFileName,
'a')
self.predictionState = 0
self.one_record = np.zeros((self.samplePointNum, 2))
self.raw_one_record = np.zeros((self.samplePointNum, 2))
self.one_record_for_graph = np.zeros((self.samplePointNum, 2))
self.windowStartTime = ''
self.y_pred_L = []
'''
ksFileID = outputFileID
if self.chamberID != -1:
outputFileID += chamberLabel
ksFileName = ksFileID + '_ks.csv'
try:
self.ksOutputFile = open(self.params.ksDir + '/' + ksFileName, 'a')
outLine = 'segmentID, d, chi^2\n'
self.writeKS2file(outLine)
# self.windowStartTime = presentTime
except EnvironmentError:
pass
'''
# makes a file in params.predDir
if self.inputFileID == '':
self.predFileID = timeFormatting.presentTimeEscaped()
else:
self.predFileID = self.inputFileID
if self.chamberID != -1:
self.predFileID += chamberLabel
print('writes prediction results to ' + self.params.predDir + '/' +
self.predFileID + '_pred.txt')
self.predFile = open(
self.params.predDir + '/' + self.predFileID + '_pred.txt', 'w')
self.predFileBeforeOverwrite = open(
self.params.predDir + '/' + self.predFileID +
'_pred_before_overwrite.txt', 'w')
self.predFileWithTimeStamps = open(
self.params.predDir + '/' + self.predFileID +
'_pred_with_timestamps.txt', 'w')
self.max_storage_for_standardization = self.samplePointNum * self.params.standardization_max_storage_window_num
self.standardizer_eeg = standardizer(
self.max_storage_for_standardization)
self.standardizer_ch2 = standardizer(
self.max_storage_for_standardization)
def __init__(self, optionsFilePath, langHelper, tempDbPath, projSrcPath, stopWords):
self.optionsFilePath = optionsFilePath
self.algorithmFactory = AlgorithmFactory(langHelper, tempDbPath)
self.graphFactory = GraphFactory(langHelper, tempDbPath, projSrcPath, stopWords)
def __init__(self, optionsFilePath, langHelper, tempDbPath, projSrcPath,
stopWords):
self.optionsFilePath = optionsFilePath
self.algorithmFactory = AlgorithmFactory(langHelper, tempDbPath)
self.graphFactory = GraphFactory(langHelper, tempDbPath, projSrcPath,
stopWords)