def runPart(self):
try:
logger.debug("Fetch url... (qsize=" + str(self.__queue.qsize()) + ")")
url = self.__queue.get(True, 1)
cachedValue = self.__cache.get(url)
if cachedValue:
if url.getExpandedUrl() != cachedValue["url"]:
raise Exception("Different url " + url.getExpandedUrl() + " != " + cachedValue["url"])
elif "error" in cachedValue and cachedValue["error"]:
url.setError()
self.__mgr.afterResolveUrl(url)
return
else:
url.setTextAndHtmlAndUrl(cachedValue["text"], cachedValue["htm"], cachedValue["real_url"])
self.__mgr.afterResolveUrl(url)
return
resolver = UrlStoppableResolver(url, self.__mgr, self.__cache, self)
putInQueue = resolver.resolve()
if putInQueue:
logger.debug("Requeue url")
if self.correctInternetConnection():
logger.error(u"Restart model")
Publisher.sendMessage("model.start", data={"soft": True})
else:
logger.error(u"Pause model")
Publisher.sendMessage("model.pause", data={"soft": True})
self.__queue.put(url)
except Empty:
return
def _OnLinkClicked(self, event):
r = event.GetRow()
c = event.GetCol()
url = self.__urls[r][0]
if c == 5:
logger.info(simplejson.dumps(url, indent="\t"))
elif c == 4:
Publisher.sendMessage("model.prob_dist", data = url)
elif c == 3:
import webbrowser
webbrowser.open(url["url"])
def run(self):
self.running = True
for a in self.accounts:
if self.running:
try:
a = a.rsplit()[0]
a = a.split(':')
user = a[0]
passw = a[1]
self.like(user, passw, self.pageid)
Publisher.sendMessage('uBar', 1)
except:
pass
else:
Publisher().sendMessage(
"update", "Closing thread number {0}".format(str(self.id)))
return
def _doSmthElse(self):
if self.__refreshGui.isSet():
logger.info("Send data to GUI")
self.__refreshGui.clear()
data = {}
data["urls"] = self.__tweetResolvedListener.finalUrls()
Publisher.sendMessage("update.urls", data=data)
if self.__showProbDist.isSet():
url = self.__probDistUrl
self.__showProbDist.clear()
self.__probDistUrl = None
probDistI = TxtClassificatorWrapper.instance().probDist(url.getText())
if self.__refreshStatusBar.isSet():
self.__refreshStatusBar.clear()
data = {}
data["cache"] = self.__urlResolver.cacheHitRate()
data["position"] = self.__iter.position()
data["position_end"] = self.__iter.count()
data["current_file_c"] = self.__iter.currentFile()
data["last_file_c"] = self.__iter.filesCount()
Publisher.sendMessage("update.statusBar", data=data)
def onRefreshMenuClick(self, event):
logger.info("Request data for GUI")
Publisher.sendMessage("model.refreshGui")
def onStartPauseButtonClick(self, event):
if self.__paused:
Publisher.sendMessage("model.start")
else:
Publisher.sendMessage("model.pause")
def onShowTreeMap(self, event):
Publisher.sendMessage("model.showTreeMap")
def onTimerEvent(self, event):
logger.info("Request status for GUI")
Publisher.sendMessage("model.refreshStatusBar")
def doContinueJob(self):
self.continueJob()
self.__urlResolver.continueWorkers()
Publisher.sendMessage("model.started")
def doPauseJob(self):
self.pauseJob()
self.__urlResolver.pauseWorkers()
Publisher.sendMessage("model.paused")
def onSave(self, ev):
Publisher.sendMessage("model.save")
