def GetDetectMatchTrain(trainFile, detectDir, verbose=True):
trainFileSsn = TFC.getSession(trainFile)
trainFileSubject = TFC.getName(trainFile)
if(verbose == True):
print "Matching Training file:"
print "File: " + trainFile
#print matching criterions
print "-> Session: " + trainFileSsn
print "-> Subject: " + trainFileSubject
detFileList = []
for root, dirs, detFileNames in os.walk(str(detectDir)):
for fileName in detFileNames:
fullDetFileName = os.path.join(root, fileName)
detFileSsn = TFC.getSession(fullDetFileName)
detFileSubject = TFC.getName(fullDetFileName)
#check its a detection file to avoid matching training files in the same directory/subdirectories
if((detFileSsn == trainFileSsn) & (detFileSubject == trainFileSubject) & (TFC.isDetect(fullDetFileName))):
# Check f is a name
detFileList.append(fullDetFileName)
if(verbose == True):
print "-> Matched detection file: " + fullDetFileName + "\r\n"
if ((verbose == True) & (detFileList == [])):
print "-> No matching detection file in the detection directory"
print "-> Ensure we have consistent subject name/session in the training and detection files\r\n"
return detFileList
def GetDetectMatchTrain(trainFile, detectDir, verbose=True):
trainFileSsn = TFC.getSession(trainFile)
trainFileSubject = TFC.getName(trainFile)
if (verbose == True):
print "Matching Training file:"
print "File: " + trainFile
#print matching criterions
print "-> Session: " + trainFileSsn
print "-> Subject: " + trainFileSubject
detFileList = []
for root, dirs, detFileNames in os.walk(str(detectDir)):
for fileName in detFileNames:
fullDetFileName = os.path.join(root, fileName)
detFileSsn = TFC.getSession(fullDetFileName)
detFileSubject = TFC.getName(fullDetFileName)
#check its a detection file to avoid matching training files in the same directory/subdirectories
if ((detFileSsn == trainFileSsn) &
(detFileSubject == trainFileSubject) &
(TFC.isDetect(fullDetFileName))):
# Check f is a name
detFileList.append(fullDetFileName)
if (verbose == True):
print "-> Matched detection file: " + fullDetFileName + "\r\n"
if ((verbose == True) & (detFileList == [])):
print "-> No matching detection file in the detection directory"
print "-> Ensure we have consistent subject name/session in the training and detection files\r\n"
return detFileList
def getSessionDetails(self):
# Initializing
Session = {}
# Training File
Session["TrainingFile"] = self.getTrainingFile()
filepath = AppConfig.TrainingFolder + "/" + Session["TrainingFile"]
# Profile Name
Session["ProfileName"] = TrainingFileClass.getName(filepath)
# Classifier Name
Session["Classifiers"] = str(
self.MW.Classifiers_ComboBox.currentText())
# Classes Names
# Classes Names
ClassesNames = TrainingFileClass.getClasses(filepath)
Session["ClassesNames"] = []
for s in AppConfig.SortedClasses:
for c in ClassesNames:
if (c == s):
Session["ClassesNames"].append(c)
# Number of Trials & Classes
Session["nTrials"] = int(self.MW.NumberOfTrials_LineEdit.text())
Session["nClasses"] = len(Session["ClassesNames"])
Session["nTrials_Total"] = Session["nTrials"] * Session["nClasses"]
# Timing Details
Session["BlankTime"] = int(self.MW.BlankTime_LineEdit.text())
Session["SteadyTime"] = int(self.MW.SteadyTime_LineEdit.text())
Session["RecordingTime"] = int(self.MW.RecordingTime_LineEdit.text())
Session["PostRecordingTime"] = int(
self.MW.PostRecordingTime_LineEdit.text())
Session["TrialTime"] = Session["BlankTime"] + Session[
"SteadyTime"] + Session["RecordingTime"] + Session[
"PostRecordingTime"]
Session["TimeTotal"] = Session["TrialTime"] * Session["nTrials_Total"]
# Done Trials
Session["DoneTrials"] = {}
for c in Session["ClassesNames"]:
Session["DoneTrials"][c] = 0
Session["DoneTrials_Total"] = 0
print Session
return Session
def browseButtonClicked(self):
self.fileDialog = QtGui.QFileDialog()
self.trainFilePath = self.getFileName()
self.datafile.setText(self.trainFilePath)
# deprecated
self.XmlFileName = self.trainFilePath
#read CSV files GDF files will be supported later
Details = {}
TrainingFileClass.getClasses(self.trainFilePath)
Details["SubjectName"] = TrainingFileClass.getName(self.trainFilePath)
Details["Classes"] = TrainingFileClass.getClasses(self.trainFilePath)
self.subjectName.setText(Details["SubjectName"]) #subject name label
def getAutorunSessionDetails(self):
# Initializing
Session = {}
# Training File
Session["TrainingFile"] = AppConfig.Autorun_TrainingFile
filepath = AppConfig.TrainingFolder + "/" + Session["TrainingFile"]
# Profile Name
Session["ProfileName"] = TrainingFileClass.getName(filepath)
# Classifier Name
Session["Classifiers"] = str(AppConfig.Autorun_Classifier)
# Classes Names
ClassesNames = TrainingFileClass.getClasses(filepath)
Session["ClassesNames"] = []
for s in AppConfig.SortedClasses:
for c in ClassesNames:
if (c == s):
Session["ClassesNames"].append(c)
# Number of Trials & Classes
Session["nTrials"] = AppConfig.Autorun_nTrials
Session["nClasses"] = len(Session["ClassesNames"])
Session["nTrials_Total"] = Session["nTrials"] * Session["nClasses"]
# Timing Details
Session["BlankTime"] = AppConfig.Autorun_BlankTime
Session["SteadyTime"] = AppConfig.Autorun_SteadyTime
Session["RecordingTime"] = AppConfig.Autorun_RecordingTime
Session["PostRecordingTime"] = AppConfig.Autorun_PostRecordingTime
Session["TrialTime"] = Session["BlankTime"] + Session[
"SteadyTime"] + Session["RecordingTime"] + Session[
"PostRecordingTime"]
Session["TimeTotal"] = Session["TrialTime"] * Session["nTrials_Total"]
# Done Trials
Session["DoneTrials"] = {}
for c in Session["ClassesNames"]:
Session["DoneTrials"][c] = 0
Session["DoneTrials_Total"] = 0
print Session
return Session
def getSessionDetails(self):
# Initializing
Session = {}
# Training File
Session["TrainingFile"] = self.getTrainingFile()
filepath = AppConfig.TrainingFolder + "/" + Session["TrainingFile"]
# Profile Name
Session["ProfileName"] = TrainingFileClass.getName(filepath)
# Classifier Name
Session["Classifiers"] = str( self.MW.Classifiers_ComboBox.currentText() )
# Classes Names
# Classes Names
ClassesNames = TrainingFileClass.getClasses(filepath)
Session["ClassesNames"] = []
for s in AppConfig.SortedClasses:
for c in ClassesNames:
if(c==s):
Session["ClassesNames"].append(c)
# Number of Trials & Classes
Session["nTrials"] = int( self.MW.NumberOfTrials_LineEdit.text() )
Session["nClasses"] = len( Session["ClassesNames"] )
Session["nTrials_Total"] = Session["nTrials"] * Session["nClasses"]
# Timing Details
Session["BlankTime"] = int( self.MW.BlankTime_LineEdit.text() )
Session["SteadyTime"] = int( self.MW.SteadyTime_LineEdit.text() )
Session["RecordingTime"] = int( self.MW.RecordingTime_LineEdit.text() )
Session["PostRecordingTime"] = int( self.MW.PostRecordingTime_LineEdit.text() )
Session["TrialTime"] = Session["BlankTime"] + Session["SteadyTime"] + Session["RecordingTime"] + Session["PostRecordingTime"]
Session["TimeTotal"] = Session["TrialTime"] * Session["nTrials_Total"]
# Done Trials
Session["DoneTrials"] = {}
for c in Session["ClassesNames"]:
Session["DoneTrials"][c]= 0
Session["DoneTrials_Total"] = 0
print Session
return Session
def getAutorunSessionDetails(self):
# Initializing
Session = {}
# Training File
Session["TrainingFile"] = AppConfig.Autorun_TrainingFile
filepath = AppConfig.TrainingFolder + "/" + Session["TrainingFile"]
# Profile Name
Session["ProfileName"] = TrainingFileClass.getName(filepath)
# Classifier Name
Session["Classifiers"] = str( AppConfig.Autorun_Classifier )
# Classes Names
ClassesNames = TrainingFileClass.getClasses(filepath)
Session["ClassesNames"] = []
for s in AppConfig.SortedClasses:
for c in ClassesNames:
if(c==s):
Session["ClassesNames"].append(c)
# Number of Trials & Classes
Session["nTrials"] = AppConfig.Autorun_nTrials
Session["nClasses"] = len( Session["ClassesNames"] )
Session["nTrials_Total"] = Session["nTrials"] * Session["nClasses"]
# Timing Details
Session["BlankTime"] = AppConfig.Autorun_BlankTime
Session["SteadyTime"] = AppConfig.Autorun_SteadyTime
Session["RecordingTime"] = AppConfig.Autorun_RecordingTime
Session["PostRecordingTime"] = AppConfig.Autorun_PostRecordingTime
Session["TrialTime"] = Session["BlankTime"] + Session["SteadyTime"] + Session["RecordingTime"] + Session["PostRecordingTime"]
Session["TimeTotal"] = Session["TrialTime"] * Session["nTrials_Total"]
# Done Trials
Session["DoneTrials"] = {}
for c in Session["ClassesNames"]:
Session["DoneTrials"][c]= 0
Session["DoneTrials_Total"] = 0
print Session
return Session
def getFileName(self):
self.fileDialog = QtGui.QFileDialog()
self.path = self.fileDialog.getOpenFileName()
print(" File Selected: ", self.path)
if(self.csv_rb.isChecked()):
self.filePath_txt.setText(self.path)
self.XmlFileName = self.path
#read CSV files GDF files will be supported later
Details = {}
TrainingFileClass.getClasses(self.path)
Details["SubjectName"] = TrainingFileClass.getName(self.path)
Details["Classes"] = TrainingFileClass.getClasses(self.path)
self.subject_name.setText(Details["SubjectName"]) #subject name label
elif(self.gdf_rb.isChecked()):
pass
else:#el 7etta deh m7taga ttzbat shewaya !
self.subject_name.setText("please select file type")
self.filePath_txt.setText("please select file type")
def getFileName(self):
self.fileDialog = QtGui.QFileDialog()
self.path = self.fileDialog.getOpenFileName()
print(" File Selected: ", self.path)
if (self.csv_rb.isChecked()):
self.filePath_txt.setText(self.path)
self.XmlFileName = self.path
#read CSV files GDF files will be supported later
Details = {}
TrainingFileClass.getClasses(self.path)
Details["SubjectName"] = TrainingFileClass.getName(self.path)
Details["Classes"] = TrainingFileClass.getClasses(self.path)
self.subject_name.setText(
Details["SubjectName"]) #subject name label
elif (self.gdf_rb.isChecked()):
pass
else: #el 7etta deh m7taga ttzbat shewaya !
self.subject_name.setText("please select file type")
self.filePath_txt.setText("please select file type")
def MultiFileBulkDetect(filesDict, noiseDict, featDict, preprocDict, enhanceDict, classDict, \
sampleStart, sampleEnd, selectedData, sameFile, updateGSpread=False):
k = 0
threadList = []
detectFileRowIndexDict = {}
detectFileAccListDict = {}
if (updateGSpread == True):
##Connect to google spreadsheet
#gc = gspread.login(GSC.email, GSC.password)
json_key = json.load(
open('/tmp/mozilla_root0/BCIWheelChair2018-eef792bad9a5.json'))
scope = ['https://spreadsheets.google.com/feeds']
credentials = SignedJwtAssertionCredentials(
json_key['client_email'], json_key['private_key'].encode(),
scope)
gc = gspread.authorize(credentials)
sh = gc.open(GSC.title)
worksheet = sh.worksheet(GSC.sheet_title)
print "\r\nDebug"
print "------"
print filesDict
print filesDict.items()
print "\r\n"
for tfItem, tfValue in filesDict.items():
## given a train file, and dict methods, detect files, loop on each and do magic!
print "Training File: " + tfItem.split("/")[-1]
print "************"
# reset to the first column
i = 0
tfDesc = TFC.getDescription(tfItem)
tfName = TFC.getName(tfItem)
if (updateGSpread == True):
rowIndex = GSPU.getEmptyRowIndex(range(1, 10))
j = 0
worksheet.update_cell(rowIndex, 3, tfDesc)
worksheet.update_cell(rowIndex, 2, tfName)
accumelatedAcc = []
print "Creating Detection Paths:"
print "-----------------------"
for noiseItem, noiseValue in noiseDict.items():
#TODO: remove this hack! a hack till we propagate the noise removal to be an unified variable(enum or struct) throughout all the code!
wrappingNoiseValue = False
if (noiseItem == "Remove") & (noiseValue & True):
wrappingNoiseValue = True
elif (noiseItem == "Raw") & (noiseValue & True):
wrappingNoiseValue = False
for preprocItem, preprocValue in preprocDict.items():
#get feature dictionary
for featItem, featValue in featDict.items():
for enhanceItem, enhanceValue in enhanceDict.items():
for classItem, classValue in classDict.items():
# TODO: when we support separate training from the detection, do the training here and detect for each file by itself
# that would speed the process too much if we've multiple detection sessions for the same training session
if (BulkUtilities.CheckTrainingFile(
tfItem, tfValue, False) == False):
continue
#df are stored as lists
for dfItem in tfValue:
print "Detection File: " + dfItem.split(
"/")[-1]
print "************"
# use rowIndex
if (updateGSpread == True):
if dfItem + tfItem in detectFileRowIndexDict:
# a recurrent detection file
print "---- test ----"
print "---- recurrent ----"
print dfItem
print tfItem
print rowIndex
print "---- test ----"
rowIndex = detectFileRowIndexDict[
dfItem + tfItem]
else:
# a new detection file ^_^
rowIndex = GSPU.getEmptyRowIndex(
range(1, 10))
detectFileRowIndexDict[
dfItem + tfItem] = rowIndex
dfDesc = TFC.getDescription(dfItem)
dfName = TFC.getName(dfItem)
worksheet.update_cell(
rowIndex, 8, dfDesc)
worksheet.update_cell(
rowIndex, 7, dfName)
print "---- test ----"
print "---- first timer ----"
print dfItem
print tfItem
print rowIndex
print "---- test ----"
Path = "Path " + str(
i
) + ": " + noiseItem + ", " + featItem + ", " + preprocItem + ", " + enhanceItem + ", " + classItem
print Path
print classItem
print classValue
#note the program behaves strangely when threads aren't closed :|
# how: for changing second value ->> readDataThread won't "always" recieve the right dfItem, but rather a cached copy of it
# may be the reason we didn't wait for the thread after getAcc!
thread = readDataThread(tfItem, dfItem, wrappingNoiseValue, sampleStart, sampleEnd, \
featValue, preprocValue, enhanceValue, classValue, \
False, selectedData, sameFile, True)
thread.start()
thread.wait()
if (updateGSpread == True):
Acc = thread.getAcc()
thread.wait()
if dfItem + tfItem in detectFileAccListDict:
detectFileAccListDict[
dfItem + tfItem].append(Acc)
else:
detectFileAccListDict[dfItem +
tfItem] = []
detectFileAccListDict[
dfItem + tfItem].append(Acc)
#Write the path description and it's accuracy
worksheet.update_cell(
rowIndex + 1, 9 + j, Path)
worksheet.update_cell(
rowIndex + 1, 10 + j, Acc)
#Array to hold accurcies of all paths
#accumelatedAcc.append(Acc)
thread.exit()
if (updateGSpread):
# move to the next 2 columns once finished all the detection files
j += 2
# get into the next column
i += 1
# for each Training File
if ((updateGSpread == True) & (i > 0)):
#Get the Min, Max and avrg accuracy then write them
for tfItem, tfValue in filesDict.items():
for dfItem in tfValue:
accumelatedAcc = detectFileAccListDict[dfItem + tfItem]
rowIndex = detectFileRowIndexDict[dfItem + tfItem]
mySorted = sorted(accumelatedAcc)
worksheet.update_cell(rowIndex, 4, mySorted[0])
print "i is " + str(i)
print mySorted
print detectFileAccListDict
print detectFileAccListDict.items
print "Rania is here"
worksheet.update_cell(rowIndex, 5, mySorted[i - 1])
temp = 0
for k in range(0, i):
temp = temp + accumelatedAcc[k]
avrg = temp / len(accumelatedAcc)
worksheet.update_cell(rowIndex, 6, avrg)
print "-----------------------"
print "Finished bulk detection"
def BulkEightyTweenty(
filesDict,
noiseDict,
featDict,
preprocDict,
enhanceDict,
classDict,
sampleStart,
sampleEnd,
selectedData,
updateGSpread=False,
):
threadList = []
offsetDict = {}
# TODO: change to enums
offsetDict["All"] = False
offsetDict["off0"] = False
offsetDict["off1"] = False
offsetDict["off2"] = False
offsetDict["off3"] = False
offsetDict["off4"] = False
if updateGSpread == True:
##Connect to google spreadsheet
gc = gspread.login(GSC.email, GSC.password)
sh = gc.open(GSC.title)
worksheet = sh.worksheet(GSC.sheet_title)
for offsetItem, offsetValue in offsetDict.items():
print "Detection Offset: " + str(offsetItem)
print "************"
i = 0
if updateGSpread == True:
rowIndex = GSPU.getEmptyRowIndex(range(1, 10))
for tfItem, tfValue in filesDict.items():
## given a train file, and dict methods, detect files, loop on each and do magic!
print "Training File: " + tfItem
print "************"
desc = str(TrainingFileClass.getDescription(tfItem)) + " " + str(offsetItem)
name = TrainingFileClass.getName(tfItem)
if updateGSpread == True:
j = 0
worksheet.update_cell(rowIndex, 3, desc)
worksheet.update_cell(rowIndex, 2, name)
accumelatedAcc = []
print "Creating Detection Paths:"
print "-----------------------"
for noiseItem, noiseValue in noiseDict.items():
# TODO: remove this hack! a hack till we propagate the noise removal to be an unified variable(enum or struct) throughout all the code!
wrappingNoiseValue = False
if (noiseItem == "Remove") & (noiseValue & True):
wrappingNoiseValue = True
elif (noiseItem == "Raw") & (noiseValue & True):
wrappingNoiseValue = False
for preprocItem, preprocValue in preprocDict.items():
# get feature dictionary
for featItem, featValue in featDict.items():
for enhanceItem, enhanceValue in enhanceDict.items():
for classItem, classValue in classDict.items():
# if we can separate the training from the detection, do the training here and detect for each file by itself
# that would speed the process too much if we've multiple detection sessions for the same training session
offsetDict = CrossValidationUtilities.SelectSingleDataOffset(offsetDict, offsetItem)
Path = (
"Path "
+ str(i)
+ ": "
+ noiseItem
+ ", "
+ featItem
+ ", "
+ preprocItem
+ ", "
+ enhanceItem
+ ", "
+ classItem
+ ", "
+ str(offsetItem)
)
print Path
thread = readDataThread(
tfItem,
tfItem,
wrappingNoiseValue,
sampleStart,
sampleEnd,
featValue,
preprocValue,
enhanceValue,
classValue,
False,
offsetDict,
True,
True,
)
thread.start()
thread.wait()
if updateGSpread == True:
Acc = thread.getAcc()
thread.wait()
# Write the path description and it's accuracy
worksheet.update_cell(rowIndex, 7 + j, Path)
worksheet.update_cell(rowIndex, 8 + j, Acc)
# Array to hold accurcies of all paths
accumelatedAcc.append(Acc)
j += 2
thread.exit()
i += 1
if (updateGSpread == True) & (i > 0):
# Get the Min, Max and avrg accuracy then write them
mySorted = sorted(accumelatedAcc)
worksheet.update_cell(rowIndex, 4, mySorted[0])
worksheet.update_cell(rowIndex, 5, mySorted[i - 1])
temp = 0
for k in range(0, i):
temp = temp + accumelatedAcc[k]
avrg = temp / len(accumelatedAcc)
worksheet.update_cell(rowIndex, 6, avrg)
print "-----------------------"
print "Finished bulk detection"
def MultiFileBulkDetect(filesDict, noiseDict, featDict, preprocDict, enhanceDict, classDict, \
sampleStart, sampleEnd, selectedData, sameFile, updateGSpread=False):
k = 0
threadList = []
detectFileRowIndexDict = {}
detectFileAccListDict = {}
if (updateGSpread == True):
##Connect to google spreadsheet
gc = gspread.login(GSC.email, GSC.password)
sh = gc.open(GSC.title)
worksheet = sh.worksheet(GSC.sheet_title)
print "\r\nDebug"
print "------"
print filesDict
print filesDict.items()
print "\r\n"
for tfItem, tfValue in filesDict.items():
## given a train file, and dict methods, detect files, loop on each and do magic!
print "Training File: " + tfItem
print "************"
# reset to the first column
i = 0
tfDesc = TFC.getDescription(tfItem)
tfName = TFC.getName(tfItem)
if (updateGSpread == True):
rowIndex = GSPU.getEmptyRowIndex(range(1, 10))
j = 0
worksheet.update_cell(rowIndex, 3, tfDesc)
worksheet.update_cell(rowIndex, 2, tfName)
accumelatedAcc = []
print "Creating Detection Paths:"
print "-----------------------"
for noiseItem, noiseValue in noiseDict.items():
#TODO: remove this hack! a hack till we propagate the noise removal to be an unified variable(enum or struct) throughout all the code!
wrappingNoiseValue = False
if (noiseItem == "Remove") & (noiseValue & True):
wrappingNoiseValue = True
elif (noiseItem == "Raw") & (noiseValue & True):
wrappingNoiseValue = False
for preprocItem, preprocValue in preprocDict.items():
#get feature dictionary
for featItem, featValue in featDict.items():
for enhanceItem, enhanceValue in enhanceDict.items():
for classItem, classValue in classDict.items():
# TODO: when we support separate training from the detection, do the training here and detect for each file by itself
# that would speed the process too much if we've multiple detection sessions for the same training session
if (BulkUtilities.CheckTrainingFile(tfItem, tfValue, False) == False):
continue
#df are stored as lists
for dfItem in tfValue:
print "Detection File: " + dfItem
print "************"
# use rowIndex
if (updateGSpread == True):
if dfItem + tfItem in detectFileRowIndexDict:
# a recurrent detection file
print "---- test ----"
print "---- recurrent ----"
print dfItem
print tfItem
print rowIndex
print "---- test ----"
rowIndex = detectFileRowIndexDict[dfItem + tfItem]
else:
# a new detection file ^_^
rowIndex = GSPU.getEmptyRowIndex(range(1, 10))
detectFileRowIndexDict[dfItem + tfItem] = rowIndex
dfDesc = TFC.getDescription(dfItem)
dfName = TFC.getName(dfItem)
worksheet.update_cell(rowIndex, 8, dfDesc)
worksheet.update_cell(rowIndex, 7, dfName)
print "---- test ----"
print "---- first timer ----"
print dfItem
print tfItem
print rowIndex
print "---- test ----"
Path = "Path " + str(i) + ": " + noiseItem + ", " + featItem + ", " + preprocItem + ", " + enhanceItem + ", " + classItem
print Path
print classItem
print classValue
#note the program behaves strangely when threads aren't closed :|
# how: for changing second value ->> readDataThread won't "always" recieve the right dfItem, but rather a cached copy of it
# may be the reason we didn't wait for the thread after getAcc!
thread = readDataThread(tfItem, dfItem, wrappingNoiseValue, sampleStart, sampleEnd, \
featValue, preprocValue, enhanceValue, classValue, \
False, selectedData, sameFile, True)
thread.start()
thread.wait()
if (updateGSpread == True):
Acc = thread.getAcc()
thread.wait()
if dfItem + tfItem in detectFileAccListDict:
detectFileAccListDict[dfItem + tfItem].append(Acc)
else:
detectFileAccListDict[dfItem + tfItem] = []
detectFileAccListDict[dfItem + tfItem].append(Acc)
#Write the path description and it's accuracy
worksheet.update_cell(rowIndex + 1, 9 + j , Path)
worksheet.update_cell(rowIndex + 1, 10 + j , Acc)
#Array to hold accurcies of all paths
#accumelatedAcc.append(Acc)
thread.exit()
if (updateGSpread):
# move to the next 2 columns once finished all the detection files
j += 2
# get into the next column
i += 1
# for each Training File
if ((updateGSpread == True) & (i > 0)):
#Get the Min, Max and avrg accuracy then write them
for tfItem, tfValue in filesDict.items():
for dfItem in tfValue:
accumelatedAcc = detectFileAccListDict[dfItem + tfItem]
rowIndex = detectFileRowIndexDict[dfItem + tfItem]
mySorted = sorted(accumelatedAcc)
worksheet.update_cell(rowIndex, 4 , mySorted[0])
print "i is " + str(i)
print mySorted
print detectFileAccListDict
print detectFileAccListDict.items
print "Rania is here"
worksheet.update_cell(rowIndex, 5 , mySorted[i-1])
temp = 0
for k in range(0, i):
temp = temp + accumelatedAcc[k]
avrg = temp / len(accumelatedAcc)
worksheet.update_cell(rowIndex, 6 , avrg)
print "-----------------------"
print "Finished bulk detection"
def SingleFileBulkDetect(self):
#TODO: add assertions -> there must be at least a single selected checkbox for each column
#TODO: add time estimation, even static ones would be great for now! calc a trial p
noiseDict = self.DictOfNoiseCB()
featDict = self.DictOfFeaturesCB()
preprocDict = self.DictOfPreprocessingCB()
enhanceDict = self.DictOfEnhancementCB()
classDict = self.DictOfClassifiersCB()
updateGDocs = self.CBGetter(self.updateGDocsCB)
##Connect to google spreadsheet
#gc = gspread.login( GSC.email , GSC.password)
if (updateGDocs == True):
json_key = json.load(open('/tmp/mozilla_root0/BCIWheelChair2018-eef792bad9a5.json'))
scope = ['https://spreadsheets.google.com/feeds']
credentials = SignedJwtAssertionCredentials(json_key['client_email'], json_key['private_key'].encode(), scope)
gc = gspread.authorize(credentials)
sh = gc.open(GSC.title)
worksheet = sh.worksheet(GSC.sheet_title)
self.rowIndex = GSPU.getEmptyRowIndex(range(1, 10))
worksheet.update_cell(self.rowIndex, 3, desc)
worksheet.update_cell(self.rowIndex, 2, name)
self.threadList = []
i = 0
j = 0
print "Traing File: " + self.trainPath.split("/")[-1]
print "************"
print "Detection File: " + self.detectPath.split("/")[-1]
print "************"
desc = TrainingFileClass.getDescription(self.trainPath)
name = TrainingFileClass.getName(self.trainPath)
self.accumelatedAcc = []
print "Creating Detection Paths:"
print "-------------------------"
for noiseItem,noiseValue in noiseDict.items():
#TODO: remove this hack! a hack till we propagate the noise removal to be an unified variable(enum or struct) throughout all the code!
wrappingNoiseValue = False
if (noiseItem == "Remove") & (noiseValue & True):
wrappingNoiseValue = True
elif (noiseItem == "Raw") & (noiseValue & True):
wrappingNoiseValue = False
for preprocItem, preprocValue in preprocDict.items():
#get feature dictionary
for featItem, featValue in featDict.items():
for enhanceItem, enhanceValue in enhanceDict.items():
for classItem, classValue in classDict.items():
Path = "Path " + str(i) + ": " +noiseItem + ", " + featItem + ", " + preprocItem + ", " + enhanceItem + ", " + classItem
print Path
print classValue
thread = readDataThread(self.trainPath, self.detectPath, wrappingNoiseValue, self.sampleStart, self.sampleEnd, \
featValue, preprocValue, enhanceValue, classValue, \
False, self.selectedData, self.sameFile,True)
self.threadList.append(thread)
self.threadList[i].start()
self.threadList[i].wait()
Acc = self.threadList[i].getAcc()
#Write the path description and it's accuracy
if (updateGDocs == True):
worksheet.update_cell(self.rowIndex, 7+j , Path)
worksheet.update_cell(self.rowIndex, 8+j , Acc)
#Array to hold accurcies of all paths
self.accumelatedAcc.append(Acc)
i += 1
j += 2
#Get the Min, Max and avrg accuracy then write them
self.sorted = sorted(self.accumelatedAcc)
if (updateGDocs == True):
worksheet.update_cell(self.rowIndex, 4 , self.sorted[0])
worksheet.update_cell(self.rowIndex, 5 , self.sorted[i-1])
temp = 0
for o in range(0, i):
temp = temp + self.accumelatedAcc[o]
avrg = temp / len(self.accumelatedAcc)
if (updateGDocs == True):
worksheet.update_cell(self.rowIndex, 6 , avrg)
print "-----------------------"
print "Finished bulk detection"
def SingleFileBulkDetect(self):
#TODO: add assertions -> there must be at least a single selected checkbox for each column
#TODO: add time estimation, even static ones would be great for now! calc a trial p
noiseDict = self.DictOfNoiseCB()
featDict = self.DictOfFeaturesCB()
preprocDict = self.DictOfPreprocessingCB()
enhanceDict = self.DictOfEnhancementCB()
classDict = self.DictOfClassifiersCB()
##Connect to google spreadsheet
gc = gspread.login( GSC.email , GSC.password)
sh = gc.open(GSC.title)
worksheet = sh.worksheet(GSC.sheet_title)
self.rowIndex = GSPU.getEmptyRowIndex(range(1, 10))
self.threadList = []
i = 0
j = 0
print "Traing File: " + self.trainPath
print "************"
print "Detection File: " + self.detectPath
print "************"
desc = TrainingFileClass.getDescription(self.trainPath)
name = TrainingFileClass.getName(self.trainPath)
worksheet.update_cell(self.rowIndex, 3, desc)
worksheet.update_cell(self.rowIndex, 2, name)
self.accumelatedAcc = []
print "Creating Detection Paths:"
print "-------------------------"
for noiseItem,noiseValue in noiseDict.items():
#TODO: remove this hack! a hack till we propagate the noise removal to be an unified variable(enum or struct) throughout all the code!
wrappingNoiseValue = False
if (noiseItem == "Remove") & (noiseValue & True):
wrappingNoiseValue = True
elif (noiseItem == "Raw") & (noiseValue & True):
wrappingNoiseValue = False
for preprocItem, preprocValue in preprocDict.items():
#get feature dictionary
for featItem, featValue in featDict.items():
for enhanceItem, enhanceValue in enhanceDict.items():
for classItem, classValue in classDict.items():
Path = "Path " + str(i) + ": " +noiseItem + ", " + featItem + ", " + preprocItem + ", " + enhanceItem + ", " + classItem
print Path
print classValue
thread = readDataThread(self.trainPath, self.detectPath, wrappingNoiseValue, self.sampleStart, self.sampleEnd, \
featValue, preprocValue, enhanceValue, classValue, \
False, self.selectedData, self.sameFile,True)
self.threadList.append(thread)
self.threadList[i].start()
self.threadList[i].wait()
Acc = self.threadList[i].getAcc()
#Write the path description and it's accuracy
worksheet.update_cell(self.rowIndex, 7+j , Path)
worksheet.update_cell(self.rowIndex, 8+j , Acc)
#Array to hold accurcies of all paths
self.accumelatedAcc.append(Acc)
i += 1
j += 2
#Get the Min, Max and avrg accuracy then write them
self.sorted = sorted(self.accumelatedAcc)
worksheet.update_cell(self.rowIndex, 4 , self.sorted[0])
worksheet.update_cell(self.rowIndex, 5 , self.sorted[i-1])
temp = 0
for o in range(0, i):
temp = temp + self.accumelatedAcc[o]
avrg = temp / len(self.accumelatedAcc)
worksheet.update_cell(self.rowIndex, 6 , avrg)
print "-----------------------"
print "Finished bulk detection"
def BulkEightyTweenty(filesDict, noiseDict, featDict, preprocDict, enhanceDict, classDict, \
sampleStart, sampleEnd, selectedData, updateGSpread=False):
threadList = []
offsetDict = {}
#TODO: change to enums
offsetDict["All"] = False
offsetDict["off0"] = False
offsetDict["off1"] = False
offsetDict["off2"] = False
offsetDict["off3"] = False
offsetDict["off4"] = False
if (updateGSpread == True):
##Connect to google spreadsheet
gc = gspread.login(GSC.email, GSC.password)
sh = gc.open(GSC.title)
worksheet = sh.worksheet(GSC.sheet_title)
for offsetItem, offsetValue in offsetDict.items():
print "Detection Offset: " + str(offsetItem)
print "************"
i = 0
if (updateGSpread == True):
rowIndex = GSPU.getEmptyRowIndex(range(1, 10))
for tfItem, tfValue in filesDict.items():
## given a train file, and dict methods, detect files, loop on each and do magic!
print "Training File: " + tfItem
print "************"
desc = str(TrainingFileClass.getDescription(
tfItem)) + " " + str(offsetItem)
name = TrainingFileClass.getName(tfItem)
if (updateGSpread == True):
j = 0
worksheet.update_cell(rowIndex, 3, desc)
worksheet.update_cell(rowIndex, 2, name)
accumelatedAcc = []
print "Creating Detection Paths:"
print "-----------------------"
for noiseItem, noiseValue in noiseDict.items():
#TODO: remove this hack! a hack till we propagate the noise removal to be an unified variable(enum or struct) throughout all the code!
wrappingNoiseValue = False
if (noiseItem == "Remove") & (noiseValue & True):
wrappingNoiseValue = True
elif (noiseItem == "Raw") & (noiseValue & True):
wrappingNoiseValue = False
for preprocItem, preprocValue in preprocDict.items():
#get feature dictionary
for featItem, featValue in featDict.items():
for enhanceItem, enhanceValue in enhanceDict.items(
):
for classItem, classValue in classDict.items():
# if we can separate the training from the detection, do the training here and detect for each file by itself
# that would speed the process too much if we've multiple detection sessions for the same training session
offsetDict = CrossValidationUtilities.SelectSingleDataOffset(
offsetDict, offsetItem)
Path = "Path " + str(
i
) + ": " + noiseItem + ", " + featItem + ", " + preprocItem + ", " + enhanceItem + ", " + classItem + ", " + str(
offsetItem)
print Path
thread = readDataThread(tfItem, tfItem, wrappingNoiseValue, sampleStart, sampleEnd, \
featValue, preprocValue, enhanceValue, classValue, \
False, offsetDict, True, True)
thread.start()
thread.wait()
if (updateGSpread == True):
Acc = thread.getAcc()
thread.wait()
#Write the path description and it's accuracy
worksheet.update_cell(
rowIndex, 7 + j, Path)
worksheet.update_cell(
rowIndex, 8 + j, Acc)
#Array to hold accurcies of all paths
accumelatedAcc.append(Acc)
j += 2
thread.exit()
i += 1
if ((updateGSpread == True) & (i > 0)):
#Get the Min, Max and avrg accuracy then write them
mySorted = sorted(accumelatedAcc)
worksheet.update_cell(rowIndex, 4, mySorted[0])
worksheet.update_cell(rowIndex, 5, mySorted[i - 1])
temp = 0
for k in range(0, i):
temp = temp + accumelatedAcc[k]
avrg = temp / len(accumelatedAcc)
worksheet.update_cell(rowIndex, 6, avrg)
print "-----------------------"
print "Finished bulk detection"
