class Server(QtNetwork.QTcpServer) :
addNewThread = QtCore.Signal(MoniterObject)
# basic initial
# private object :
# widget : TabWidget for WebView
# monitor : A thread object for monite all connection and release finished connection
# connections :
# when main widget close, it will shutdown server instantly
# a new connection handler
# monitor add new object event
def __init__(self) :
super(Server, self).__init__()
self.widget = TabWidget()
self.monitor = MoniterThread()
self.widget.closeSignal.connect(self.stopServer)
self.newConnection.connect(self.newConnectionHandler)
self.addNewThread.connect(self.monitor.addNewThread)
pass
# start monitor, show widget, start service
def run(self) :
self.monitor.start()
self.widget.show()
self.listen(QtNetwork.QHostAddress("127.0.0.1"), 5000)
@QtCore.Slot()
def newConnectionHandler(self) :
newTcpSocket = self.nextPendingConnection() # get new connection socket
connection = Connection(newTcpSocket) # create a new connection delegate object
crossThread = CrossThread() # create connection thread
self.widget.addTab(str(crossThread.currentThreadId()), connection.widget()) # insert connection attach widget to server main widget
connection.closeConnection.connect(self.widget.deleteTab) # connect connection close to remove widget of server main widget
crossThread.finishedSignal.connect(self.monitor.deleteThread) # when a thread dead or finish its job, it will tell monitor to delete it.
crossThread.startedSignal.connect(connection.service) # inform new connection thread start service.
connection.moveToThread(crossThread) # let connection run in new thread
moniterObject = MoniterObject(crossThread, connection) # create a monitor object for monitor thread
self.addNewThread.emit(moniterObject) # make monitor thread registe a new monitor object
crossThread.start() # new connection thread start running
# don't stop directly, it will wait all thread finished and stop safely
@QtCore.Slot()
def stopServer(self) :
self.monitor.deleteAll()
while not self.monitor.isFinished() :
pass
print("Is monitor thread is end ? : " + str(self.monitor.isFinished()))
self.close()
self.widget.deleteLater()
self.deleteLater()
class Crawler(QtGui.QWidget):
signalClickedToLoadUrl = QtCore.Signal(QtCore.QUrl)
def __init__(self, width = 800, height = 600):
super(Crawler, self).__init__()
self.setGeometry(10, 10, width, height)
self.setupUI()
def setupUI(self) :
self.mainHBoxLayout = QtGui.QHBoxLayout()
self.mainRVBoxLayout = QtGui.QVBoxLayout()
self.mainLVBoxLayout = QtGui.QVBoxLayout()
self.executeCrawler = QtGui.QPushButton("Get Content!!!")
self.urlBarLineEdit = QtGui.QLineEdit("http://www2.tpa.edu.tw/tpaedu/Home/login.asp")
self.myWebView = WebView()
self.myTabWidget = TabWidget()
#self.myWebView.settings().setDefaultTextEncoding("big5")
self.myWebView.show()
self.myTabWidget.addTab(self.myWebView, "my Web View")
self.mainLVBoxLayout.addWidget(self.urlBarLineEdit)
self.mainLVBoxLayout.addWidget(self.myTabWidget)
self.mainRVBoxLayout.addWidget(self.executeCrawler)
self.mainHBoxLayout.addLayout(self.mainLVBoxLayout)
self.mainHBoxLayout.addLayout(self.mainRVBoxLayout)
self.setLayout(self.mainHBoxLayout)
self.executeCrawler.clicked.connect(self.slotClickedToLoadUrl)
self.signalClickedToLoadUrl.connect(self.slotLoadUrl)
self.myWebView.loadFinished.connect(self.slotWebViewLoadFinished)
self.show()
@QtCore.Slot()
def slotClickedToLoadUrl(self) :
print(self.urlBarLineEdit.text())
self.signalClickedToLoadUrl.emit(QtCore.QUrl(self.urlBarLineEdit.text()))
@QtCore.Slot(QtCore.QUrl)
def slotLoadUrl(self, qUrl) :
self.myWebView.load(qUrl)
@QtCore.Slot(bool)
def slotWebViewLoadFinished(self, isFinished) :
if isFinished :
self.urlBarLineEdit.setText(self.myWebView.url().toString())
pageFrames = self.myWebView.page().mainFrame().childFrames()
print("base Url := " + self.myWebView.page().mainFrame().baseUrl().toString())
if self.myWebView.page().mainFrame().baseUrl().toString() == "http://www2.tpa.edu.tw/tpaedu/default.asp" :
print("it is tpa local net")
for frame in pageFrames :
print(frame.frameName())
#print(frame.toHtml())
elements = frame.findAllElements("td")
print(elements.count())
for index in range(elements.count()) :
"""print("elements" + str(index))
print(elements.at(index + 1).tagName())
print(elements.at(index + 1).firstChild().tagName())
print(elements.at(index + 1).firstChild().nextSibling().tagName())
print(elements.at(index + 1).firstChild().nextSibling().nextSibling().tagName())
print(elements.at(index + 1).firstChild().nextSibling().nextSibling().nextSibling().tagName())
print(str(type(elements.at(index + 1).firstChild().nextSibling().nextSibling().nextSibling().nextSibling())))"""
findAllElementsChild(elements.at(index + 1).firstChild())
if frame.frameName() == "logo" :
pass
elif frame.frameName() == "menu" :
# get menu for javascript robot action
pass
elif frame.frameName() == "main" :
# get main content to crawler data
pass
elif self.myWebView.page().mainFrame().baseUrl().toString() == "http://www2.tpa.edu.tw/tpaedu/Home/login.asp" :
pass
pass
class Server(QtNetwork.QTcpServer):
addNewThread = QtCore.Signal(MoniterObject)
# basic initial
# private object :
# widget : TabWidget for WebView
# monitor : A thread object for monite all connection and release finished connection
# connections :
# when main widget close, it will shutdown server instantly
# a new connection handler
# monitor add new object event
def __init__(self):
super(Server, self).__init__()
self.widget = TabWidget()
self.monitor = MoniterThread()
self.widget.closeSignal.connect(self.stopServer)
self.newConnection.connect(self.newConnectionHandler)
self.addNewThread.connect(self.monitor.addNewThread)
pass
# start monitor, show widget, start service
def run(self):
self.monitor.start()
self.widget.show()
self.listen(QtNetwork.QHostAddress("127.0.0.1"), 5000)
@QtCore.Slot()
def newConnectionHandler(self):
newTcpSocket = self.nextPendingConnection(
) # get new connection socket
connection = Connection(
newTcpSocket) # create a new connection delegate object
crossThread = CrossThread() # create connection thread
self.widget.addTab(str(
crossThread.currentThreadId()), connection.widget(
)) # insert connection attach widget to server main widget
connection.closeConnection.connect(
self.widget.deleteTab
) # connect connection close to remove widget of server main widget
crossThread.finishedSignal.connect(
self.monitor.deleteThread
) # when a thread dead or finish its job, it will tell monitor to delete it.
crossThread.startedSignal.connect(
connection.service) # inform new connection thread start service.
connection.moveToThread(
crossThread) # let connection run in new thread
moniterObject = MoniterObject(
crossThread,
connection) # create a monitor object for monitor thread
self.addNewThread.emit(
moniterObject) # make monitor thread registe a new monitor object
crossThread.start() # new connection thread start running
# don't stop directly, it will wait all thread finished and stop safely
@QtCore.Slot()
def stopServer(self):
self.monitor.deleteAll()
while not self.monitor.isFinished():
pass
print("Is monitor thread is end ? : " + str(self.monitor.isFinished()))
self.close()
self.widget.deleteLater()
self.deleteLater()
class Crawler(QtGui.QWidget):
signalClickedToLoadUrl = QtCore.Signal(QtCore.QUrl)
def __init__(self, width=800, height=600):
super(Crawler, self).__init__()
self.setGeometry(10, 10, width, height)
self.setupUI()
def setupUI(self):
self.mainHBoxLayout = QtGui.QHBoxLayout()
self.mainRVBoxLayout = QtGui.QVBoxLayout()
self.mainLVBoxLayout = QtGui.QVBoxLayout()
self.executeCrawler = QtGui.QPushButton("Get Content!!!")
self.urlBarLineEdit = QtGui.QLineEdit(
"http://www2.tpa.edu.tw/tpaedu/Home/login.asp")
self.myWebView = WebView()
self.myTabWidget = TabWidget()
#self.myWebView.settings().setDefaultTextEncoding("big5")
self.myWebView.show()
self.myTabWidget.addTab(self.myWebView, "my Web View")
self.mainLVBoxLayout.addWidget(self.urlBarLineEdit)
self.mainLVBoxLayout.addWidget(self.myTabWidget)
self.mainRVBoxLayout.addWidget(self.executeCrawler)
self.mainHBoxLayout.addLayout(self.mainLVBoxLayout)
self.mainHBoxLayout.addLayout(self.mainRVBoxLayout)
self.setLayout(self.mainHBoxLayout)
self.executeCrawler.clicked.connect(self.slotClickedToLoadUrl)
self.signalClickedToLoadUrl.connect(self.slotLoadUrl)
self.myWebView.loadFinished.connect(self.slotWebViewLoadFinished)
self.show()
@QtCore.Slot()
def slotClickedToLoadUrl(self):
print(self.urlBarLineEdit.text())
self.signalClickedToLoadUrl.emit(
QtCore.QUrl(self.urlBarLineEdit.text()))
@QtCore.Slot(QtCore.QUrl)
def slotLoadUrl(self, qUrl):
self.myWebView.load(qUrl)
@QtCore.Slot(bool)
def slotWebViewLoadFinished(self, isFinished):
if isFinished:
self.urlBarLineEdit.setText(self.myWebView.url().toString())
pageFrames = self.myWebView.page().mainFrame().childFrames()
print("base Url := " +
self.myWebView.page().mainFrame().baseUrl().toString())
if self.myWebView.page().mainFrame().baseUrl().toString(
) == "http://www2.tpa.edu.tw/tpaedu/default.asp":
print("it is tpa local net")
for frame in pageFrames:
print(frame.frameName())
#print(frame.toHtml())
elements = frame.findAllElements("td")
print(elements.count())
for index in range(elements.count()):
"""print("elements" + str(index))
print(elements.at(index + 1).tagName())
print(elements.at(index + 1).firstChild().tagName())
print(elements.at(index + 1).firstChild().nextSibling().tagName())
print(elements.at(index + 1).firstChild().nextSibling().nextSibling().tagName())
print(elements.at(index + 1).firstChild().nextSibling().nextSibling().nextSibling().tagName())
print(str(type(elements.at(index + 1).firstChild().nextSibling().nextSibling().nextSibling().nextSibling())))"""
findAllElementsChild(
elements.at(index + 1).firstChild())
if frame.frameName() == "logo":
pass
elif frame.frameName() == "menu":
# get menu for javascript robot action
pass
elif frame.frameName() == "main":
# get main content to crawler data
pass
elif self.myWebView.page().mainFrame().baseUrl().toString(
) == "http://www2.tpa.edu.tw/tpaedu/Home/login.asp":
pass
pass
