class AccessManager(QNetworkAccessManager): Rule = namedtuple('Rule', ['rule_type', 'rule']) _allow = 'allow' _reject = 'reject' def __init__(self, parent): super().__init__(parent) self.control = parent.control self.rule_list = [] self.proxy = None self.proxy_factory = ProxyManager(self) self.setProxyFactory(self.proxy_factory) self.proxyAuthenticationRequired.connect(self.proxy_authenticate) self.authenticationRequired.connect(self.authenticate) self.cookie_manager = CookieManager() self.setCookieJar(self.cookie_manager) self.clear_cookie_timer = QTimer(self) self.clear_cookie_timer.setTimerType(Qt.VeryCoarseTimer) # Clear cookies every hour self.clear_cookie_timer.setInterval(3600 * 1000) self.clear_cookie_timer.timeout.connect(self.clear_cookies) self.clear_cookies() def clear_cookies(self): logger.error(self.control.prepend_id('clearing cookies')) self.cookie_manager.clear_all_cookies() def authenticate(self, network_proxy, authenticator): logger.error(self.control.prepend_id('Proxy Authenticate {}'.format(network_proxy.url()))) self.control.abort() def proxy_authenticate(self, network_proxy, authenticator): if self.proxy: authenticator.setUser(self.proxy.user()) authenticator.setPassword(self.proxy.password()) else: logger.error(self.control.prepend_id('Asked for proxy when no proxy is present: {}'.format(network_proxy.url()))) def set_page_proxy(self, proxy_string, auth_string): if not proxy_string: return pr = proxy_string.split(':', 1) if len(pr) != 2: logger.error(self.control.prepend_id('Invalid proxy string {}, auth {}'.format(proxy_string, auth_string))) return host = pr[0] port = int(pr[1]) if not (host and port): logger.error(self.control.prepend_id('Invalid proxy string {}, auth {}'.format(proxy_string, auth_string))) return if auth_string: aus = auth_string.split(':', 1) if len(aus) != 2: logger.error(self.control.prepend_id('Invalid proxy string {}, auth {}'.format(proxy_string, auth_string))) return user = aus[0] password = aus[1] self.proxy = QNetworkProxy(QNetworkProxy.HttpProxy, host, port, user, password) else: self.proxy = QNetworkProxy(QNetworkProxy.HttpProxy, host, port) def reset(self): self.rule_list = [] self.proxy = None def request_finished(self, network_reply): if not self.control.job(): return error = network_reply.error() url = network_reply.url() url_str = url.toString() retry_after_sec = 60 if error != 0: if error == QNetworkReply.OperationCanceledError: logger.debug(self.control.prepend_id('Operation cancelled for url {}'.format(url_str))) return request = network_reply.request() request_headers_string = 'Request:\n' for header in request.rawHeaderList(): request_headers_string += '{}: {}\n'.format(header.data().decode(encoding=HTTP_HEADER_CHARSET), request.rawHeader(header).data().decode(encoding=HTTP_HEADER_CHARSET)) response_headers_string = 'Response:\n' for header in network_reply.rawHeaderList(): header_key = header.data().decode(encoding=HTTP_HEADER_CHARSET) header_value = network_reply.rawHeader(header).data().decode(encoding=HTTP_HEADER_CHARSET) if 'Retry-After' == header and header_value.isdigit(): retry_after_sec = int(header_value) response_headers_string += '{}: {}\n'.format(header_key, header_value) logger.error(self.control.prepend_id('e_id="{eid};{estr}" url="{url}"\n{req_h}\n{res_h}'.format(eid=error, estr=network_reply.errorString(), url=url_str, req_h=request_headers_string, res_h=response_headers_string))) ############################################### # Throttle the retries as lazily as possible ############################################### # We go slightly above the recommended value, randomize it a bit and double check the values just in case if retry_after_sec < 1: retry_after_sec = 2 if retry_after_sec < 100: retry_after_sec = (retry_after_sec * 10) + randint(0, 30) # Restrict the retry sec to 1 hour if retry_after_sec > 3600: retry_after_sec = 3600 self.control.abort(retry_after_sec) def createRequest(self, operation, request, device=None): if not self.control.job(): return super().createRequest(operation, request, device) url = request.url() url_str = url.toString() if self.rule_list: for r in self.rule_list: if r.rule.search(url_str): if r.rule_type == self._reject: logger.debug(self.control.prepend_id('Blocking {}'.format(url_str))) return super().createRequest(operation, QNetworkRequest(QUrl()), device) else: break if self.proxy and self.control.job() and self.control.job().is_crawlera: scheme = url.scheme() if scheme == 'https': url.setScheme('http') request.setRawHeader(b'X-Crawlera-Use-HTTPS', b'1') # TODO Check if this is needed request.setSslConfiguration(QSslConfiguration()) request.setUrl(url) elif scheme == 'http': pass else: # We fail any request that is not http or https logger.warning(self.control.prepend_id('Unsupported Schema {}'.format(url_str))) return super().createRequest(operation, QNetworkRequest(QUrl()), device) key = bytes('{}:{}'.format(self.proxy.user(), self.proxy.password()), HTTP_HEADER_CHARSET) proxy_auth_value = b'Basic ' + base64.urlsafe_b64encode(key) request.setRawHeader(b'Proxy-Authorization', proxy_auth_value) request.setRawHeader(b'Proxy-Connection', b'Keep-Alive') request.setRawHeader(b'X-Crawlera-Cookies', b'disable') request.setRawHeader(b'X-Crawlera-UA', b'desktop') network_reply = super().createRequest(operation, request, device) network_reply.finished.connect(lambda: self.request_finished(network_reply)) return network_reply def set_filter(self, filter_str_list): for filter_str in filter_str_list: filter_conf = filter_str.split(':', 1) if len(filter_conf) != 2: logger.error(self.control.prepend_id('Invalid filter string {}'.format(filter_str))) return rule_type = filter_conf[0] if not (rule_type == self._allow or rule_type == self._reject): logger.error(self.control.prepend_id('Invalid filter string {}'.format(filter_str))) return rule_str = filter_conf[1] if not rule_str: logger.error(self.control.prepend_id('Invalid filter string {}'.format(filter_str))) return self.rule_list.append(self.Rule(rule_type, re.compile(rule_str)))
class MyWebView(QWebEngineView): def __init__(self, parent=None, acc=None): super().__init__() self.parent = parent self.session = acc.session self.user_id = acc.user_id self.chekLoadFinish = False self.progress = 0 # Connect sql and get data os.chdir('/root/PycharmProjects/HearSheep') con = sql.connect(os.getcwd() + '/db/db.sqlite3') cur = con.cursor() with con: cur.execute('SELECT login, pass, token, proxy, user_agent FROM accaunt where user_id=?', (self.user_id,)) row = cur.fetchone() self.login, self.password, self.token, proxy, user_agent = row con.close() # Path Cookies os.chdir('/root/PycharmProjects/HearSheep') pathCookies = '/root/PycharmProjects/HearSheep/cookies/' + str(self.user_id) if os.path.exists(pathCookies): pass else: os.chdir('/root/PycharmProjects/HearSheep/cookies') os.mkdir(str(self.user_id)) pathCookies = '/root/PycharmProjects/HearSheep/cookies/' + str(self.user_id) # settings profile = self.page().profile() profile.setPersistentStoragePath(pathCookies) profile.setHttpUserAgent(user_agent) profile.setHttpAcceptLanguage("ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4") urlinfo = parse.urlparse(proxy) self.proxy = QNetworkProxy() self.proxy.setType(QNetworkProxy.HttpProxy) self.proxy.setHostName(urlinfo.hostname) self.proxy.setPort(urlinfo.port) self.proxy.setUser(urlinfo.username) self.proxy.setPassword(urlinfo.password) QNetworkProxy.setApplicationProxy(self.proxy) self.page().proxyAuthenticationRequired.connect(self.handleProxyAuthReq) self.loadProgress.connect(self.setProgress) self.load(QtCore.QUrl('https://vk.com/')) self.auth() def handleProxyAuthReq(self, url, auth, proxyhost): auth.setUser(self.proxy.user()) auth.setPassword(self.proxy.password()) def setProgress(self, value): self.progress = value def waitForSignal(self): loop = QEventLoop() def work(value): print('loop', value) if value > 95: loop.quit() self.loadProgress.connect(work) loop.exec_() self.loadProgress.disconnect() time.sleep(2) return True def auth(self): self.waitForSignal() if self.page().url().url() == 'https://vk.com/' or self.page().url().url() == 'https://vk.com/index.php': page = self.page() page.runJavaScript( 'document.querySelector("#index_email").value = "{}"'.format(self.login)) page.runJavaScript( 'document.querySelector("#index_pass").value = "{}"'.format(self.password)) page.runJavaScript( 'document.querySelector("#index_login_button").click()') print('auth complete') def joinTheChat(self, chat): number_chat, count_users, invite_link = chat print('connect to', invite_link) self.load(QtCore.QUrl(invite_link)) print('waiting load inviting page') self.waitForSignal() page = self.page() page.runJavaScript( "document.querySelector('button.flat_button.round_button._im_join_chat.im-invitation--join').click()") print('waiting after clicking the button "connect"') self.waitForSignal() # Check the entry in the chat print('get dialogs') getUrl = UrlAPI(self.token) response = self.session.get(getUrl.messages.getDialogs(count=3), proxies=self.session.proxies) dialogs = json.loads(response.text)['response']['items'] for dialog in dialogs: print(dialog) if dialog['message']['title'] == 'Читаем стихи': print('Find') os.chdir('/root/PycharmProjects/HearSheep') con = sql.connect(os.getcwd() + '/db/db.sqlite3') cur = con.cursor() with con: cur.execute('INSERT INTO accaunt_chat(user_id,chat_id,number_chat) VALUES(?,?,?)', (self.user_id, dialog['message']['chat_id'], number_chat)) cur.execute('UPDATE chat SET count_users=? WHERE number_chat=?', (count_users+1, number_chat)) con.close() print('joined the chat') return True print("not joined the chat") return False def createChat(self): getUrl = UrlAPI(self.token) response = self.session.get(getUrl.messages.createChat(title='Читаем стихи'), proxies=self.session.proxies) jResponse = json.loads(response.text) if 'response' in jResponse: chat_id = jResponse['response'] peer_id = 2000000000 + chat_id res = self.session.get(getUrl.messages.getInviteLink(peer_id=peer_id), proxies=self.session.proxies) jRes = json.loads(res.text)['response'] invite_link = jRes['link'] os.chdir('/root/PycharmProjects/HearSheep') con = sql.connect(os.getcwd() + '/db/db.sqlite3') cur = con.cursor() with con: cur.execute('INSERT INTO chat(number_chat,count_users,invite_link) VALUES (NULL,1,?)', (invite_link, )) cur.execute('SELECT number_chat FROM chat WHERE invite_link=?', (invite_link, )) row = cur.fetchone() number_chat = row[0] cur.execute('INSERT INTO accaunt_chat(user_id,chat_id,number_chat) VALUES (?,?,?)', (self.user_id, chat_id, number_chat)) con.close() print('the chat created') return True print("the chat don't created") return False def sendMessage(self): page = self.page() page.runJavaScript( "document.getElementById('im_editable88230675').innerHTML = 'Йо!'") page.runJavaScript( "document.querySelector('button.im-send-btn.im-chat-input--send.im-send-btn_saudio._im_send.im-send-btn_audio').click()")
class MyWebView(QWebEngineView): def __init__(self, parent=None, user_id=None): super().__init__() self.parent = parent self.chekLoadFinish = False # Connect sql and get data os.chdir('/root/PycharmProjects/HearSheep') con = sql.connect(os.getcwd() + '/db/db.sqlite3') cur = con.cursor() with con: cur.execute( 'SELECT login, pass, proxy, user_agent FROM accaunt where user_id=?', (user_id, )) row = cur.fetchone() self.login, self.password, proxy, user_agent = row con.close() #Path Cookies os.chdir('/root/PycharmProjects/HearSheep') pathCookies = '/root/PycharmProjects/HearSheep/cookies/' + str(user_id) if os.path.exists(pathCookies): pass else: os.chdir('/root/PycharmProjects/HearSheep/cookies') os.mkdir(str(user_id)) pathCookies = '/root/PycharmProjects/HearSheep/cookies/' + str( user_id) # settings profile = self.page().profile() profile.setPersistentStoragePath(pathCookies) profile.setHttpUserAgent(user_agent) profile.setHttpAcceptLanguage("ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4") urlinfo = parse.urlparse(proxy) self.proxy = QNetworkProxy() self.proxy.setType(QNetworkProxy.HttpProxy) self.proxy.setHostName(urlinfo.hostname) self.proxy.setPort(urlinfo.port) self.proxy.setUser(urlinfo.username) self.proxy.setPassword(urlinfo.password) QNetworkProxy.setApplicationProxy(self.proxy) # self.page().proxyAuthenticationRequired.connect( self.handleProxyAuthReq) self.loadStarted.connect(self.loadStart) self.loadFinished.connect(self.loadFinish) self.load(QtCore.QUrl('https://vk.com')) def handleProxyAuthReq(self, url, auth, proxyhost): auth.setUser(self.proxy.user()) auth.setPassword(self.proxy.password()) def loadStart(self): self.chekLoadFinish = False def loadFinish(self): self.chekLoadFinish = True if self.page().url().url() == 'https://vk.com/': self._auth() def _auth(self): page = self.page() page.runJavaScript( 'document.querySelector("#index_email").value = "{}"'.format( self.login)) page.runJavaScript( 'document.querySelector("#index_pass").value = "{}"'.format( self.password)) page.runJavaScript( 'document.querySelector("#index_login_button").click()')