class Parse(Proxylist):
def __init__(self):
super().__init__()
self.items = []
self.parse_obj = None
self.parse_thread = None
self.itemsModel = TableModel('Id', 'Dt', 'Name', 'Price', 'Author',
'Address', 'Phone')
self.sortItemsModel = SortFilterProxyModel(self.itemsModel)
self.rootContext().setContextProperty('sortItemsModel',
self.sortItemsModel)
def parse(self):
if self.parse_thread and self.parse_thread.isRunning():
return
self.log('Start parsing', 'Parse')
self.itemsModel.clear()
self.items = []
self.parse_obj = ParseWorker(self.proxylist)
self.parse_thread = QThread()
self.parse_obj.moveToThread(self.parse_thread)
self.parse_obj.message.connect(self.parse_on_message)
self.parse_obj.finished.connect(self.parse_on_finished)
self.parse_thread.started.connect(self.parse_obj.start)
self.parse_thread.start()
def parse_on_message(self, message):
if message[0]:
self.log(message[0], 'Parse')
if message[1]:
item = message[1]
self.itemsModel.addRow([
item['id'] + 1, item['dt'], item['name'], item['price'],
item['author'], item['address'], item['phone']
])
def parse_on_finished(self, message):
self.parse_thread.quit()
self.parse_thread.wait()
self.items = message[0]
self.proxylist = message[1]
self.log('Parsing END')
class Proxylist(Log):
def __init__(self):
super().__init__()
self.proxylist = []
self.proxylist_obj = None
self.proxylist_thread = None
self.proxyModel = TableModel('Id', 'Country', 'Url', 'Response_time', 'Used', 'Error')
self.sortProxyModel = SortFilterProxyModel(self.proxyModel)
self.rootContext().setContextProperty('sortProxyModel', self.sortProxyModel)
async def proxylist_init(self):
cursor = await self.db.execute('SELECT * FROM proxy')
proxylist = await cursor.fetchall()
if proxylist:
for p in proxylist:
self.proxylist.append({'id': p[0], 'added': p[1], 'country': p[2], 'host': p[3], 'port': p[4], 'url': p[5], 'response_time': p[6], 'used': p[7], 'error': p[8]})
self.proxyModel.addRow([p[0], p[2], p[5], f'{p[6]:.2f}', p[7], p[8]])
self.log(f'Load {len(self.proxylist)} proxy', 'Proxylist')
async def proxylist_close(self):
if self.proxylist:
await self.db.executemany('UPDATE proxy SET used=?, error=? WHERE id=?',
[(proxy['used'], proxy['error'], proxy['id']) for proxy in self.proxylist])
await self.db.commit()
def proxylist_update(self):
if self.proxylist_thread and self.proxylist_thread.isRunning():
return
self.log('Start update proxylist', 'Proxylist')
self.proxyModel.clear()
self.proxylist = []
self.proxylist_obj = ProxylistUpdate()
self.proxylist_thread = QThread()
self.proxylist_obj.moveToThread(self.proxylist_thread)
self.proxylist_obj.message.connect(self.proxylist_on_message)
self.proxylist_obj.finished.connect(self.proxylist_on_finished)
self.proxylist_thread.started.connect(self.proxylist_obj.start)
self.proxylist_thread.start()
async def proxylist_save(self):
await self.db.executemany('INSERT OR REPLACE INTO proxy (country,host,port,url,response_time) VALUES (?,?,?,?,?)',
[(proxy['country'], proxy['host'], proxy['port'], proxy['url'], proxy['response_time']) for proxy in self.proxylist])
await self.db.commit()
self.log(f'Save {len(self.proxylist)} proxy', 'Proxylist')
async def proxylist_clear(self):
self.proxyModel.clear()
self.proxylist = []
await self.db.execute(f'DELETE FROM proxy')
await self.db.execute(f'DELETE FROM sqlite_sequence WHERE name="proxy"')
await self.db.commit()
self.log('Clear proxylist', 'Proxylist')
def proxylist_on_message(self, message):
if message[0]:
self.log(message[0], 'Proxylist')
if message[1]:
proxy = message[1]
self.proxyModel.addRow([proxy['id']+1, proxy['country'], proxy['url'], f'{proxy["response_time"]:.2f}', proxy['used'], proxy['error']])
def proxylist_on_finished(self, message):
self.proxylist_thread.quit()
self.proxylist_thread.wait()
self.proxylist = message
self.loop.run_until_complete(self.proxylist_save())