class Server(QtNetwork.QTcpServer) : addNewThread = QtCore.Signal(MoniterObject) # basic initial # private object : # widget : TabWidget for WebView # monitor : A thread object for monite all connection and release finished connection # connections : # when main widget close, it will shutdown server instantly # a new connection handler # monitor add new object event def __init__(self) : super(Server, self).__init__() self.widget = TabWidget() self.monitor = MoniterThread() self.widget.closeSignal.connect(self.stopServer) self.newConnection.connect(self.newConnectionHandler) self.addNewThread.connect(self.monitor.addNewThread) pass # start monitor, show widget, start service def run(self) : self.monitor.start() self.widget.show() self.listen(QtNetwork.QHostAddress("127.0.0.1"), 5000) @QtCore.Slot() def newConnectionHandler(self) : newTcpSocket = self.nextPendingConnection() # get new connection socket connection = Connection(newTcpSocket) # create a new connection delegate object crossThread = CrossThread() # create connection thread self.widget.addTab(str(crossThread.currentThreadId()), connection.widget()) # insert connection attach widget to server main widget connection.closeConnection.connect(self.widget.deleteTab) # connect connection close to remove widget of server main widget crossThread.finishedSignal.connect(self.monitor.deleteThread) # when a thread dead or finish its job, it will tell monitor to delete it. crossThread.startedSignal.connect(connection.service) # inform new connection thread start service. connection.moveToThread(crossThread) # let connection run in new thread moniterObject = MoniterObject(crossThread, connection) # create a monitor object for monitor thread self.addNewThread.emit(moniterObject) # make monitor thread registe a new monitor object crossThread.start() # new connection thread start running # don't stop directly, it will wait all thread finished and stop safely @QtCore.Slot() def stopServer(self) : self.monitor.deleteAll() while not self.monitor.isFinished() : pass print("Is monitor thread is end ? : " + str(self.monitor.isFinished())) self.close() self.widget.deleteLater() self.deleteLater()
class Crawler(QtGui.QWidget): signalClickedToLoadUrl = QtCore.Signal(QtCore.QUrl) def __init__(self, width = 800, height = 600): super(Crawler, self).__init__() self.setGeometry(10, 10, width, height) self.setupUI() def setupUI(self) : self.mainHBoxLayout = QtGui.QHBoxLayout() self.mainRVBoxLayout = QtGui.QVBoxLayout() self.mainLVBoxLayout = QtGui.QVBoxLayout() self.executeCrawler = QtGui.QPushButton("Get Content!!!") self.urlBarLineEdit = QtGui.QLineEdit("http://www2.tpa.edu.tw/tpaedu/Home/login.asp") self.myWebView = WebView() self.myTabWidget = TabWidget() #self.myWebView.settings().setDefaultTextEncoding("big5") self.myWebView.show() self.myTabWidget.addTab(self.myWebView, "my Web View") self.mainLVBoxLayout.addWidget(self.urlBarLineEdit) self.mainLVBoxLayout.addWidget(self.myTabWidget) self.mainRVBoxLayout.addWidget(self.executeCrawler) self.mainHBoxLayout.addLayout(self.mainLVBoxLayout) self.mainHBoxLayout.addLayout(self.mainRVBoxLayout) self.setLayout(self.mainHBoxLayout) self.executeCrawler.clicked.connect(self.slotClickedToLoadUrl) self.signalClickedToLoadUrl.connect(self.slotLoadUrl) self.myWebView.loadFinished.connect(self.slotWebViewLoadFinished) self.show() @QtCore.Slot() def slotClickedToLoadUrl(self) : print(self.urlBarLineEdit.text()) self.signalClickedToLoadUrl.emit(QtCore.QUrl(self.urlBarLineEdit.text())) @QtCore.Slot(QtCore.QUrl) def slotLoadUrl(self, qUrl) : self.myWebView.load(qUrl) @QtCore.Slot(bool) def slotWebViewLoadFinished(self, isFinished) : if isFinished : self.urlBarLineEdit.setText(self.myWebView.url().toString()) pageFrames = self.myWebView.page().mainFrame().childFrames() print("base Url := " + self.myWebView.page().mainFrame().baseUrl().toString()) if self.myWebView.page().mainFrame().baseUrl().toString() == "http://www2.tpa.edu.tw/tpaedu/default.asp" : print("it is tpa local net") for frame in pageFrames : print(frame.frameName()) #print(frame.toHtml()) elements = frame.findAllElements("td") print(elements.count()) for index in range(elements.count()) : """print("elements" + str(index)) print(elements.at(index + 1).tagName()) print(elements.at(index + 1).firstChild().tagName()) print(elements.at(index + 1).firstChild().nextSibling().tagName()) print(elements.at(index + 1).firstChild().nextSibling().nextSibling().tagName()) print(elements.at(index + 1).firstChild().nextSibling().nextSibling().nextSibling().tagName()) print(str(type(elements.at(index + 1).firstChild().nextSibling().nextSibling().nextSibling().nextSibling())))""" findAllElementsChild(elements.at(index + 1).firstChild()) if frame.frameName() == "logo" : pass elif frame.frameName() == "menu" : # get menu for javascript robot action pass elif frame.frameName() == "main" : # get main content to crawler data pass elif self.myWebView.page().mainFrame().baseUrl().toString() == "http://www2.tpa.edu.tw/tpaedu/Home/login.asp" : pass pass
class Server(QtNetwork.QTcpServer): addNewThread = QtCore.Signal(MoniterObject) # basic initial # private object : # widget : TabWidget for WebView # monitor : A thread object for monite all connection and release finished connection # connections : # when main widget close, it will shutdown server instantly # a new connection handler # monitor add new object event def __init__(self): super(Server, self).__init__() self.widget = TabWidget() self.monitor = MoniterThread() self.widget.closeSignal.connect(self.stopServer) self.newConnection.connect(self.newConnectionHandler) self.addNewThread.connect(self.monitor.addNewThread) pass # start monitor, show widget, start service def run(self): self.monitor.start() self.widget.show() self.listen(QtNetwork.QHostAddress("127.0.0.1"), 5000) @QtCore.Slot() def newConnectionHandler(self): newTcpSocket = self.nextPendingConnection( ) # get new connection socket connection = Connection( newTcpSocket) # create a new connection delegate object crossThread = CrossThread() # create connection thread self.widget.addTab(str( crossThread.currentThreadId()), connection.widget( )) # insert connection attach widget to server main widget connection.closeConnection.connect( self.widget.deleteTab ) # connect connection close to remove widget of server main widget crossThread.finishedSignal.connect( self.monitor.deleteThread ) # when a thread dead or finish its job, it will tell monitor to delete it. crossThread.startedSignal.connect( connection.service) # inform new connection thread start service. connection.moveToThread( crossThread) # let connection run in new thread moniterObject = MoniterObject( crossThread, connection) # create a monitor object for monitor thread self.addNewThread.emit( moniterObject) # make monitor thread registe a new monitor object crossThread.start() # new connection thread start running # don't stop directly, it will wait all thread finished and stop safely @QtCore.Slot() def stopServer(self): self.monitor.deleteAll() while not self.monitor.isFinished(): pass print("Is monitor thread is end ? : " + str(self.monitor.isFinished())) self.close() self.widget.deleteLater() self.deleteLater()
class Crawler(QtGui.QWidget): signalClickedToLoadUrl = QtCore.Signal(QtCore.QUrl) def __init__(self, width=800, height=600): super(Crawler, self).__init__() self.setGeometry(10, 10, width, height) self.setupUI() def setupUI(self): self.mainHBoxLayout = QtGui.QHBoxLayout() self.mainRVBoxLayout = QtGui.QVBoxLayout() self.mainLVBoxLayout = QtGui.QVBoxLayout() self.executeCrawler = QtGui.QPushButton("Get Content!!!") self.urlBarLineEdit = QtGui.QLineEdit( "http://www2.tpa.edu.tw/tpaedu/Home/login.asp") self.myWebView = WebView() self.myTabWidget = TabWidget() #self.myWebView.settings().setDefaultTextEncoding("big5") self.myWebView.show() self.myTabWidget.addTab(self.myWebView, "my Web View") self.mainLVBoxLayout.addWidget(self.urlBarLineEdit) self.mainLVBoxLayout.addWidget(self.myTabWidget) self.mainRVBoxLayout.addWidget(self.executeCrawler) self.mainHBoxLayout.addLayout(self.mainLVBoxLayout) self.mainHBoxLayout.addLayout(self.mainRVBoxLayout) self.setLayout(self.mainHBoxLayout) self.executeCrawler.clicked.connect(self.slotClickedToLoadUrl) self.signalClickedToLoadUrl.connect(self.slotLoadUrl) self.myWebView.loadFinished.connect(self.slotWebViewLoadFinished) self.show() @QtCore.Slot() def slotClickedToLoadUrl(self): print(self.urlBarLineEdit.text()) self.signalClickedToLoadUrl.emit( QtCore.QUrl(self.urlBarLineEdit.text())) @QtCore.Slot(QtCore.QUrl) def slotLoadUrl(self, qUrl): self.myWebView.load(qUrl) @QtCore.Slot(bool) def slotWebViewLoadFinished(self, isFinished): if isFinished: self.urlBarLineEdit.setText(self.myWebView.url().toString()) pageFrames = self.myWebView.page().mainFrame().childFrames() print("base Url := " + self.myWebView.page().mainFrame().baseUrl().toString()) if self.myWebView.page().mainFrame().baseUrl().toString( ) == "http://www2.tpa.edu.tw/tpaedu/default.asp": print("it is tpa local net") for frame in pageFrames: print(frame.frameName()) #print(frame.toHtml()) elements = frame.findAllElements("td") print(elements.count()) for index in range(elements.count()): """print("elements" + str(index)) print(elements.at(index + 1).tagName()) print(elements.at(index + 1).firstChild().tagName()) print(elements.at(index + 1).firstChild().nextSibling().tagName()) print(elements.at(index + 1).firstChild().nextSibling().nextSibling().tagName()) print(elements.at(index + 1).firstChild().nextSibling().nextSibling().nextSibling().tagName()) print(str(type(elements.at(index + 1).firstChild().nextSibling().nextSibling().nextSibling().nextSibling())))""" findAllElementsChild( elements.at(index + 1).firstChild()) if frame.frameName() == "logo": pass elif frame.frameName() == "menu": # get menu for javascript robot action pass elif frame.frameName() == "main": # get main content to crawler data pass elif self.myWebView.page().mainFrame().baseUrl().toString( ) == "http://www2.tpa.edu.tw/tpaedu/Home/login.asp": pass pass