def reset(self):
"""Reset settings."""
util.loadSettings()
# block signals while resetting gui buttons to avoid double emission of signals
self.resetting = True
# self.blockSignals(True)
#reset slider, labels and checkboxes
for name in ["Red", "Green", "Blue", "Brigh"]:
self.ColorSliders[name].setValue(constants.settings["Color"][f"LED_{name}"])
self.ColorLabels [name].setText(f"{name}\n{constants.settings['Color'][f'LED_{name}']}")
self.ColorSliders["Exp"].setValue(constants.settings["Color"]["exposureTime"])
self.ColorLabels ["Exp"].setText(f"Exp\n{constants.settings['Color']['exposureTime']}")
self.UVSliders["Brigh"].setValue(constants.settings["UV"]["LED_Brigh"])
self.UVLabels ["Brigh"].setText(f"Brigh\n{constants.settings['UV']['LED_Brigh']}")
self.UVSliders["Exp"].setValue(constants.settings["UV"]["exposureTime"])
self.UVLabels ["Exp"].setText(f"Exp\n{constants.settings['UV']['exposureTime']}")
for name, checkbox in self.showColorCheckboxes.items():
checkbox.setChecked(constants.settings["show"][name])
# self.blockSignals(False)
self.resetting = False
self.resetSignal.emit()
def RemoveBoiler():
todayDate = util.getTodayDateFolder()
lastSuggNewsBoiled = util.loadSettings(
Constants.LAST_BOILER_SUGGGOOGLENEWS)
lastNewsBoiled = util.loadSettings(Constants.LAST_BOILER_GOOGLENEWS)
lastDataBoiled = util.loadSettings(Constants.LAST_BOILER_DATA_DIR)
if lastNewsBoiled:
util.logger.info("Google news last boiled for =" + lastNewsBoiled)
else:
util.logger.info("Google news last boiled for None")
if lastSuggNewsBoiled:
util.logger.info("Sugg Google news last boiled for =" +
lastSuggNewsBoiled)
else:
util.logger.info("Sugg Google news last boiled for None")
if lastDataBoiled:
util.logger.info("data last boiled for =" + lastDataBoiled)
else:
util.logger.info("data last boiled for = None")
boilerNewsStatus = True
boilerSuggNewsStatus = True
boilerDataStatus = True
#Check whether today links have been extracted or not
if lastNewsBoiled != todayDate:
boilerNewsStatus = BoilerNews(todayDate)
else:
util.logger.info("Boiler news already done for today :" + todayDate)
#Check whether today sugg links have been extracted or not
if lastSuggNewsBoiled != todayDate:
boilerSuggNewsStatus = BoilerSuggNews(todayDate)
else:
util.logger.info("Sugg Boiler news already done for today :" +
todayDate)
trainingFolders = util.findTrainingDays()
anyTrainingFolderBoiled = True
trainingDoneForDays = 0
if lastDataBoiled == trainingFolders[0]:
util.logger.info("Boiler data already done for today :" +
lastDataBoiled)
else:
anyTrainingFolderBoiled = False
folderIndex = 0
if lastDataBoiled != None:
try:
folderIndex = trainingFolders.index(lastDataBoiled)
anyTrainingFolderBoiled = True
except Exception, e:
folderIndex = Constants.MAX_PREVIOUS_DAYS
else:
def ConnectionClarity():
todayDate = util.getYesterdayDateFolder()
lastClarityDate = util.loadSettings(Constants.LAST_CLARITY_DIR)
lastSuggClarityDate = util.loadSettings(Constants.LAST_SUGG_CLARITY_DIR)
if lastClarityDate:
util.logger.info("Google Clarity done last for =" + lastClarityDate)
else:
util.logger.info("Google Clarity done last for none")
if lastSuggClarityDate:
util.logger.info("Sugg Clarity done last for =" + lastClarityDate)
else:
util.logger.info("Sugg Clarity done last for none")
if todayDate == lastClarityDate and todayDate == lastSuggClarityDate:
util.logger.info("Clarity signal done for today =" + todayDate)
return True
trainFiles = util.findTrainingFiles()
trainFiles = util.random_select(trainFiles)
trainCorpus, usedTrainFiles = util.findCorpus(trainFiles)
normalClarity = True
if todayDate != lastClarityDate:
testFiles = util.findTestFiles()
testCorpus, usedTestFiles = util.findCorpus(testFiles)
clarityobj = Clarity(trainCorpus, testCorpus)
clarityScore = clarityobj.ClarityScore()
normalClarity = printNormalRankedDocs(clarityScore, usedTestFiles)
if normalClarity == True:
util.saveSettings(Constants.LAST_CLARITY_DIR, todayDate)
util.logger.info("Google Clarity info just completed for =" +
todayDate)
suggClarity = True
if todayDate != lastClarityDate:
testFiles = util.findSuggTestFiles()
testCorpus, usedTestFiles = util.findCorpus(testFiles)
clarityobj = Clarity(trainCorpus, testCorpus)
clarityScore = clarityobj.ClarityScore()
suggClarity = printSuggRankedDocs(clarityScore, usedTestFiles)
if suggClarity == True:
util.saveSettings(Constants.LAST_SUGG_CLARITY_DIR, todayDate)
util.logger.info("SuggGoogle Clarity info just completed for =" +
todayDate)
return normalClarity or suggClarity
def RecommendationMetric():
todayDateFolder = util.getTodayDateFolder()
lastRecommended= util.loadSettings(Constants.LAST_RECOMMENDATION_DONE)
if todayDateFolder == lastRecommended:
return True
try:
relevance_json = {}
#Get Relevance json
readRelevanceDir = os.path.join(Constants.ROOT_FOLDER,Constants.RECOMMENDATION_DIR,Constants.ENGINE_DIR
,todayDateFolder,Constants.RELEVANCE_FILE)
if os.path.isfile(readRelevanceDir) is True:
with open(readRelevanceDir) as json_data:
json_text = json_data.read()
relevance_json = json.loads(json_text)
json_data.close()
#Get Smoothness json
smoothness_json = {}
readSmoothnessDir = os.path.join(Constants.ROOT_FOLDER,Constants.RECOMMENDATION_DIR,Constants.ENGINE_DIR
,todayDateFolder,Constants.SMOOTHNESS_FILE)
if os.path.isfile(readSmoothnessDir) is True:
with open(readSmoothnessDir) as json_data:
json_text = json_data.read()
smoothness_json = json.loads(json_text)
json_data.close()
#Get Clarity json
clarity_json = {}
readClarityDir = os.path.join(Constants.ROOT_FOLDER,Constants.RECOMMENDATION_DIR,Constants.ENGINE_DIR
,todayDateFolder,Constants.CLARITY_FILE)
if os.path.isfile(readClarityDir) is True:
with open(readClarityDir) as json_data:
json_text = json_data.read()
clarity_json = json.loads(json_text)
json_data.close()
#Lets not apply any linear weight formula for now
cou = Counter()
cou.update(relevance_json)
cou.update(smoothness_json)
cou.update(clarity_json)
#Convert above back to a dictionary
final_json = dict(cou)
result = printRecommendedDocs(final_json, todayDateFolder)
if result == True:
util.saveSettings(Constants.LAST_RECOMMENDATION_DONE, todayDateFolder)
util.logger.info("Recommended links done for ="+ todayDateFolder)
pass
except Exception, e:
util.logger.error( "Exception at recommending links for : %s Exception = %s"
% (todayDateFolder,traceback.print_exc))
def RecommendationMetric():
todayDateFolder = util.getTodayDateFolder()
lastRecommended= util.loadSettings(Constants.LAST_RECOMMENDATION_DONE)
if todayDateFolder == lastRecommended:
return True
try:
relevance_json = {}
#Get Relevance json
readRelevanceDir = os.path.join(Constants.ROOT_FOLDER,Constants.RECOMMENDATION_DIR,Constants.ENGINE_DIR
,todayDateFolder,Constants.GOOGLENEWS,Constants.RELEVANCE_FILE)
if os.path.isfile(readRelevanceDir) is True:
with open(readRelevanceDir) as json_data:
json_text = json_data.read()
relevance_json = json.loads(json_text)
json_data.close()
#Get Smoothness json
smoothness_json = {}
readSmoothnessDir = os.path.join(Constants.ROOT_FOLDER,Constants.RECOMMENDATION_DIR,Constants.ENGINE_DIR
,todayDateFolder,Constants.GOOGLENEWS,Constants.SMOOTHNESS_FILE)
if os.path.isfile(readSmoothnessDir) is True:
with open(readSmoothnessDir) as json_data:
json_text = json_data.read()
smoothness_json = json.loads(json_text)
json_data.close()
#Get Clarity json
clarity_json = {}
readClarityDir = os.path.join(Constants.ROOT_FOLDER,Constants.RECOMMENDATION_DIR,Constants.ENGINE_DIR
,todayDateFolder,Constants.GOOGLENEWS,Constants.CLARITY_FILE)
if os.path.isfile(readClarityDir) is True:
with open(readClarityDir) as json_data:
json_text = json_data.read()
clarity_json = json.loads(json_text)
json_data.close()
#Lets not apply any linear weight formula for now
cou = Counter()
cou.update(relevance_json)
cou.update(smoothness_json)
cou.update(clarity_json)
#Convert above back to a dictionary
final_json = dict(cou)
result = printRecommendedDocs(final_json, todayDateFolder)
if result == True:
util.saveSettings(Constants.LAST_RECOMMENDATION_DONE, todayDateFolder)
util.logger.info("Recommended Google links done for ="+ todayDateFolder)
pass
except Exception, e:
util.logger.error( "Exception at recommending google links for : %s Exception = %s"
% (todayDateFolder,traceback.print_exc))
def GoogleNews():
downloadedLinks = []
todayDate = util.getTodayDateFolder()
lastNewsDownloaded = util.loadSettings(Constants.LAST_GOOGLENEWS_DOWNLOAD)
lastLinksDownloaded = util.loadSettings(Constants.LAST_GOOGLELINKS_DOWNLOAD)
googleLinksStatus = True
googleNewsStatus = True
#Check whether today links have been extracted or not
if lastLinksDownloaded != todayDate:
googleLinksStatus = getGoogleLinks(todayDate)
else:
util.logger.info("Google links downloaded successfully for = "+todayDate)
#Check whether today news has been extracted or not
if todayDate != lastNewsDownloaded:
googleNewsStatus = downloadGoogleNews(todayDate)
else:
util.logger.info("Google news already downloaded successfully for = "+ todayDate)
return googleLinksStatus & googleNewsStatus
def RemoveBoiler():
todayDate = util.getTodayDateFolder()
lastNewsBoiled = util.loadSettings(Constants.LAST_BOILER_GOOGLENEWS)
lastDataBoiled = util.loadSettings(Constants.LAST_BOILER_DATA_DIR)
boilerNewsStatus = True
boilerDataStatus = True
#Check whether today links have been extracted or not
# if lastNewsBoiled != todayDate:
# boilerNewsStatus = BoilerNews(todayDate)
# else:
# util.logger.info("Boiler news already done for today :" + todayDate)
#
trainingFolders = util.findTrainingDays()
anyTrainingFolderBoiled = True
if lastDataBoiled == trainingFolders[0]:
util.logger.info("Boiler data already done for today :" + lastDataBoiled)
else:
anyTrainingFolderBoiled = False
folderIndex = 0
if lastDataBoiled != None:
folderIndex = trainingFolders.index(lastDataBoiled)
else:
folderIndex = Constants.TRAINING_DAY
if folderIndex < 0:
folderIndex = 0
util.logger.info("Boiler data for none of the last %d days have been downloaded" % Constants.TRAINING_DAY)
for folder in range(folderIndex - 1, -1, -1):
boilerDataStatus = BoilerData(trainingFolders[folder])
if boilerDataStatus == False:
util.logger.error("Boiler data not done for today :" + trainingFolders[folder])
else:
anyTrainingFolderBoiled = True
return boilerNewsStatus & anyTrainingFolderBoiled
def GoogleNews():
downloadedLinks = []
todayDate = util.getTodayDateFolder()
lastNewsDownloaded = util.loadSettings(Constants.LAST_GOOGLENEWS_DOWNLOAD)
lastLinksDownloaded = util.loadSettings(
Constants.LAST_GOOGLELINKS_DOWNLOAD)
googleLinksStatus = True
googleNewsStatus = True
#Check whether today links have been extracted or not
if lastLinksDownloaded != todayDate:
googleLinksStatus = getGoogleLinks(todayDate)
else:
util.logger.info("Google links downloaded successfully for = " +
todayDate)
#Check whether today news has been extracted or not
if todayDate != lastNewsDownloaded:
googleNewsStatus = downloadGoogleNews(todayDate)
else:
util.logger.info("Google news already downloaded successfully for = " +
todayDate)
return googleLinksStatus & googleNewsStatus
def ConnectionClarity():
todayDate = util.getTodayDateFolder()
lastClarityDate = util.loadSettings(Constants.LAST_CLARITY_DIR)
if todayDate == lastClarityDate :
util.logger.info("Clarity signal done for today =" + todayDate)
return True
trainFiles = util.findTrainingFiles()
testFiles = util.findTestFiles()
trainCorpus, usedTrainFiles = util.findCorpus(trainFiles)
testCorpus, usedTestFiles = util.findCorpus(testFiles)
clarityobj = Clarity(trainCorpus,testCorpus)
clarityScore = clarityobj.ClarityScore()
ret = printRankedDocs(clarityScore, usedTestFiles)
if ret == True:
util.saveSettings(Constants.LAST_CLARITY_DIR, todayDate)
util.logger.info("Clarity info just completed for ="+todayDate)
return ret
def Relevance():
todayDate = util.getTodayDateFolder()
lastRelevanceDate = util.loadSettings(Constants.LAST_RELEVANCE_DIR)
if todayDate == lastRelevanceDate :
util.logger.info("Relevance signal already done for today :" + todayDate)
return True
trainFiles = util.findTrainingFiles()
testFiles = util.findTestFiles()
trainCorpus, usedTrainFiles = util.findCorpus(trainFiles)
testCorpus, usedTestFiles = util.findCorpus(testFiles)
all_tokens = sum(trainCorpus, [])
tokens_once = set(word for word in set(all_tokens) if all_tokens.count(word) == 1)
texts = [[word for word in text if word not in tokens_once]
for text in trainCorpus]
pass
dictionary = corpora.Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]
tfidf = models.TfidfModel(corpus=corpus, id2word=dictionary,normalize=True)
index = similarities.SparseMatrixSimilarity(tfidf[corpus],num_features=len(dictionary))
count = 0
testJson = {}
for text in testCorpus:
vec=dictionary.doc2bow(text)
sims = index[tfidf[vec]]
score = sum(sims)
#print(list(enumerate(sims)))
testJson[usedTestFiles[count]] = score
count = count + 1
ret = printRankedDocs(testJson)
if ret == True:
util.saveSettings(Constants.LAST_RELEVANCE_DIR, todayDate)
util.logger.info("Relevance info just completed for ="+todayDate)
return ret
# ctr-c press event handler
# save model and quit
# this is for when i get tired of waiting
import signal
def signal_handler(sig, frame):
print('\nsaving')
model.save(settings['model']['production'])
print('quiting\n')
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
settings = util.loadSettings()
model = load_model(settings['model']['production'])
t = util.loadTokenizer(settings['tokenizer']['production'])
# load training data
# this does refit the tokenizer
(vocab, input_data, context_user, context_bot, next_data_categorical,
n) = util.load_and_process_data(settings['files']['training'], t)
num_steps = settings['tokenizer']['max_word_count']
batch_size = settings['training']['batch_size']
num_epochs = settings['training']['epochs']
model.fit([input_data, context_user, context_bot],
next_data_categorical,
def Relevance():
todayDate = util.getYesterdayDateFolder()
lastRelevanceDate = util.loadSettings(Constants.LAST_RELEVANCE_DIR)
lastSuggRelevanceDate = util.loadSettings(
Constants.LAST_SUGG_RELEVANCE_DIR)
if lastRelevanceDate:
util.logger.info("Google Relevance done last for =" +
lastRelevanceDate)
else:
util.logger.info("Google Relevance done last for None")
if lastSuggRelevanceDate:
util.logger.info("Sugg Relevance done last for =" + lastRelevanceDate)
else:
util.logger.info("Sugg Relevance done last for None")
if todayDate == lastRelevanceDate and todayDate == lastSuggRelevanceDate:
util.logger.info("Relevance signal already done for today :" +
todayDate)
return True
trainFiles = util.findTrainingFiles()
trainCorpus, usedTrainFiles = util.findCorpus(trainFiles)
all_tokens = sum(trainCorpus, [])
tokens_once = set(word for word in set(all_tokens)
if all_tokens.count(word) == 1)
texts = [[word for word in text if word not in tokens_once]
for text in trainCorpus]
dictionary = corpora.Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]
tfidf = models.TfidfModel(corpus=corpus,
id2word=dictionary,
normalize=True)
index = similarities.SparseMatrixSimilarity(tfidf[corpus],
num_features=len(dictionary))
normalRelevance = True
if todayDate != lastRelevanceDate:
testFiles = util.findTestFiles()
testCorpus, usedTestFiles = util.findCorpus(testFiles)
count = 0
testJson = {}
for text in testCorpus:
vec = dictionary.doc2bow(text)
sims = index[tfidf[vec]]
score = sum(sims)
#print(list(enumerate(sims)))
testJson[usedTestFiles[count]] = score
count = count + 1
normalRelevance = printNormalRankedDocs(testJson)
if normalRelevance == True:
util.saveSettings(Constants.LAST_RELEVANCE_DIR, todayDate)
util.logger.info("Google Relevance info just completed for =" +
todayDate)
suggRelevance = True
if todayDate != lastSuggRelevanceDate:
testFiles = util.findSuggTestFiles()
testCorpus, usedTestFiles = util.findCorpus(testFiles)
count = 0
testJson = {}
for text in testCorpus:
vec = dictionary.doc2bow(text)
sims = index[tfidf[vec]]
score = sum(sims)
#print(list(enumerate(sims)))
testJson[usedTestFiles[count]] = score
count = count + 1
suggRelevance = printSuggRankedDocs(testJson)
if suggRelevance == True:
util.saveSettings(Constants.LAST_SUGG_RELEVANCE_DIR, todayDate)
util.logger.info("Google Relevance info just completed for =" +
todayDate)
return normalRelevance or suggRelevance
def __init__(self):
super(MainWindow, self).__init__()
self.errorSignal .connect(self.openErrorMessage)
self.countingDoneSignal .connect(self.countingDone)
self.triggeringDoneSignal.connect(self.triggeringDone)
self.backToPreviewSignal .connect(self.backToPreview)
self.cellsQueue = Queue.Queue() # queue to pass cell coordinates found by counting algorithm
self.imageQueue = Queue.Queue() # Queue to hold images
self.mode = None # "Color" or "UV"
util.loadSettings()
self.hardwareHandler = HardwareHandler(self.imageQueue)
##### shortcuts ####
self.quitSC = QShortcut(QKeySequence('Ctrl+Q'), self)
self.quitSC.activated.connect(QApplication.instance().quit)
self.exitFS = QShortcut("ESC", self)
self.exitFS.activated.connect(self.showMaximized)
def switchMaxFS():
if self.isFullScreen():
self.showNormal() #workaround... if showNormal is not called in between, showMaxize does not work...
self.showMaximized()
else: self.showFullScreen()
self.switchMaxFS = QShortcut(QKeySequence('Ctrl+F'), self)
self.switchMaxFS.activated.connect(switchMaxFS)
############# layout #############
self.centralWidget = QWidget(self)
self.hlayout = QHBoxLayout()
##### image view #####
self.imageWidget = ImageWidget(self.imageQueue, self)
self.hlayout.addWidget(self.imageWidget)
##### control widget #####
self.controlWidget = QStackedWidget()
self.controlWidget.setMaximumWidth(320)
## page 1 - main page##
self.page1Widget = QWidget(self.controlWidget)
self.page1Layout = QVBoxLayout()
self.page1Widget.setLayout(self.page1Layout)
buttonTrigger = QPushButton("&Trigger")
buttonSettings = QPushButton("&Settings")
buttonTriggerAndSave = QPushButton("Trigger + Save")
self.buttonMode = QPushButton("Switch to ...")
buttonTrigger .clicked.connect(self.trigger)
buttonSettings .clicked.connect(lambda: self.controlWidget.setCurrentIndex(2))
buttonSettings .clicked.connect(lambda: self.infoTextBox.setText(""))
buttonTriggerAndSave.clicked.connect(self.triggerAndSave)
self.buttonMode .clicked.connect(lambda: self.changeMode())
self.page1Layout.addWidget(buttonTrigger)
self.page1Layout.addWidget(self.buttonMode)
self.page1Layout.addWidget(buttonSettings)
self.page1Layout.addWidget(buttonTriggerAndSave)
## page 2 - image captured##
self.page2Widget = QWidget(self.controlWidget)
self.page2Layout = QVBoxLayout()
self.page2Widget.setLayout(self.page2Layout)
buttonBackToPreview = QPushButton("&Back")
buttonSaveImage = QPushButton("&Save")
buttonCount = QPushButton("&Count")
buttonBackToPreview.clicked.connect(self.backToPreview)
buttonSaveImage .clicked.connect(lambda: self.saveImage())
buttonCount .clicked.connect(self.startCounting)
self.page2Layout.addWidget(buttonBackToPreview)
self.page2Layout.addWidget(buttonSaveImage)
self.page2Layout.addWidget(buttonCount)
## page 3 - settings ##
self.settingsWidget = SettingsWidget(self.controlWidget)
self.settingsWidget.OKButton.clicked.connect(lambda: self.controlWidget.setCurrentIndex(0))
self.settingsWidget.OKButton.clicked.connect(lambda: self.infoTextBox.setText("Live capturing"))
# signals emitted when settings change
self.settingsWidget.UVLEDSettingsUpdatedSignal .connect(self.hardwareHandler.updateLEDUV)
self.settingsWidget.ColorLEDSettingsUpdatedSignal .connect(self.hardwareHandler.updateLEDColors)
self.settingsWidget.captureSettingsUpdatedSignal .connect(lambda : self.hardwareHandler.updateCaptureSettings(mode = self.mode))
self.settingsWidget.resetSignal .connect(self.hardwareHandler.updateLEDUV)
self.settingsWidget.resetSignal .connect(self.hardwareHandler.updateLEDColors)
self.settingsWidget.resetSignal .connect(lambda : self.hardwareHandler.updateCaptureSettings(mode = self.mode))
#set mode if tab is changed in settings widget
def setModeFromTabIndex(tabIndex: int):
if tabIndex == 0: self.changeMode("Color")
elif tabIndex == 1: self.changeMode("UV")
self.settingsWidget.tabs.currentChanged.connect(setModeFromTabIndex)
## page 4 - counting ##
self.page4Widget = QWidget(self.controlWidget)
self.page4Layout = QVBoxLayout(self.page4Widget)
# buttonStopCounting = QPushButton("&Stop Counting")
# buttonStopCounting.clicked.connect(self.stopCounting)
# self.page4Layout.addWidget(buttonStopCounting)
countingLabel = QLabel("Counting..", alignment = Qt.AlignCenter)
self.page4Layout.addWidget(countingLabel)
## page 5 - trigger and save ##
self.page5Widget = QWidget(self.controlWidget)
self.page5Layout = QVBoxLayout(self.page5Widget)
self.triggerAndSaveLabel = QLabel("Capture color Image\t\n"
"Save color Image\t\t\n"
"Capture UV Image\t\n"
"Save UV Image\t\t", alignment = Qt.AlignVCenter | Qt.AlignLeft)
self.page5Layout.addWidget(self.triggerAndSaveLabel)
self.controlWidget.addWidget(self.page1Widget)
self.controlWidget.addWidget(self.page2Widget)
self.controlWidget.addWidget(self.settingsWidget)
self.controlWidget.addWidget(self.page4Widget)
self.controlWidget.addWidget(self.page5Widget)
self.hlayout.addWidget(self.controlWidget)
self.hlayout.setContentsMargins(0,0,0,0)
self.centralWidget.setLayout(self.hlayout)
self.setCentralWidget(self.centralWidget)
## info in right bottom corner
self.infoTextBox = QLabel(self.centralWidget)
self.infoTextBox.setText("TEST")
self.infoTextBox.setAlignment(Qt.AlignRight | Qt.AlignBottom)
self.infoTextBox.setAttribute(Qt.WidgetAttribute.WA_TransparentForMouseEvents) # pylint: disable=no-member
self.installEventFilter( util.ObjectResizer(self, self.infoTextBox))
logger.info("Gui started")
# start capture and led
self.changeMode("Color")
self.imageWidget.startShowLive()
self.infoTextBox.setText("Live capturing")
def Smoothness():
todayDate = util.getYesterdayDateFolder()
lastSmoothnessDate = util.loadSettings(Constants.LAST_SMOOTHNESS_DIR)
lastSuggSmoothnessDate = util.loadSettings(
Constants.LAST_SUGG_SMOOTHNESS_DIR)
if lastSmoothnessDate:
util.logger.info("Google Smoothness done last for =" +
lastSmoothnessDate)
else:
util.logger.info("Google Smoothness done last for none")
if lastSuggSmoothnessDate:
util.logger.info("Sugg Google Smoothness done last for =" +
lastSuggSmoothnessDate)
else:
util.logger.info("Sugg Google Smoothness done last for none")
if todayDate == lastSmoothnessDate and todayDate == lastSuggSmoothnessDate:
util.logger.info("Smoothness signal done for today" + todayDate)
return True
trainFiles = util.findTrainingFiles()
trainFiles = util.random_select(trainFiles)
trainCorpus, usedTrainFiles = util.findCorpus(trainFiles)
bm25obj = Bm25(trainCorpus)
trainUniqueWords = []
for trainText in trainCorpus:
trainUniqueWords.append(set(trainText))
normalSmoothness = True
if todayDate != lastSmoothnessDate:
testFiles = util.findTestFiles()
testCorpus, usedTestFiles = util.findCorpus(testFiles)
testJson = {}
testUniqueWords = []
smoothness = zeros((len(testCorpus), len(trainCorpus)))
for testText in testCorpus:
testUniqueWords.append(set(testText))
for testDoc in range(len(testCorpus)):
uniqueTest = testUniqueWords[testDoc]
SminusDcontext = zeros(bm25obj.N)
DminusScontext = zeros(bm25obj.N)
for trainDoc in range(len(trainCorpus)):
uniqueTrain = trainUniqueWords[trainDoc]
# t0 = time()
SminusD = [
word for word in trainCorpus[trainDoc]
if word not in uniqueTest
]
# t1 = time()
#print "time 1 = "+str(t1-t0)
DminusS = [
word for word in testCorpus[testDoc]
if word not in uniqueTrain
]
# t2 = time()
# print "time 2 = "+str(t2-t1)
SminusDcontext = bm25obj.BM25Score(SminusD)
# t3 = time()
# print "time 3 = "+str(t3-t2)
DminusScontext = bm25obj.BM25Score(DminusS)
# t4 = time()
# print "time 4 = "+str(t4-t3)
smoothness[testDoc][trainDoc] = np.dot(SminusDcontext,
DminusScontext)
# t5 = time()
#print "time 5 = "+str(t5-t4)
normalSmoothness = printNormalRankedDocs(smoothness, usedTestFiles)
if normalSmoothness == True:
util.saveSettings(Constants.LAST_SMOOTHNESS_DIR, todayDate)
util.logger.info("Google Smoothness info just completed for =" +
todayDate)
suggSmoothness = True
if todayDate != lastSuggSmoothnessDate:
testFiles = util.findSuggTestFiles()
testCorpus, usedTestFiles = util.findCorpus(testFiles)
testJson = {}
testUniqueWords = []
smoothness = zeros((len(testCorpus), len(trainCorpus)))
for testText in testCorpus:
testUniqueWords.append(set(testText))
for testDoc in range(len(testCorpus)):
uniqueTest = testUniqueWords[testDoc]
SminusDcontext = zeros(bm25obj.N)
DminusScontext = zeros(bm25obj.N)
for trainDoc in range(len(trainCorpus)):
uniqueTrain = trainUniqueWords[trainDoc]
SminusD = [
word for word in trainCorpus[trainDoc]
if word not in uniqueTest
]
DminusS = [
word for word in testCorpus[testDoc]
if word not in uniqueTrain
]
SminusDcontext = bm25obj.BM25Score(SminusD)
DminusScontext = bm25obj.BM25Score(DminusS)
smoothness[testDoc][trainDoc] = np.dot(SminusDcontext,
DminusScontext)
suggSmoothness = printSuggRankedDocs(smoothness, usedTestFiles)
if suggSmoothness == True:
util.saveSettings(Constants.LAST_SUGG_SMOOTHNESS_DIR, todayDate)
util.logger.info("Sugg Smoothness info just completed for =" +
todayDate)
return normalSmoothness or suggSmoothness
