class AccessManager(QNetworkAccessManager):
Rule = namedtuple('Rule', ['rule_type', 'rule'])
_allow = 'allow'
_reject = 'reject'
def __init__(self, parent):
super().__init__(parent)
self.control = parent.control
self.rule_list = []
self.proxy = None
self.proxy_factory = ProxyManager(self)
self.setProxyFactory(self.proxy_factory)
self.proxyAuthenticationRequired.connect(self.proxy_authenticate)
self.authenticationRequired.connect(self.authenticate)
self.cookie_manager = CookieManager()
self.setCookieJar(self.cookie_manager)
self.clear_cookie_timer = QTimer(self)
self.clear_cookie_timer.setTimerType(Qt.VeryCoarseTimer)
# Clear cookies every hour
self.clear_cookie_timer.setInterval(3600 * 1000)
self.clear_cookie_timer.timeout.connect(self.clear_cookies)
self.clear_cookies()
def clear_cookies(self):
logger.error(self.control.prepend_id('clearing cookies'))
self.cookie_manager.clear_all_cookies()
def authenticate(self, network_proxy, authenticator):
logger.error(self.control.prepend_id('Proxy Authenticate {}'.format(network_proxy.url())))
self.control.abort()
def proxy_authenticate(self, network_proxy, authenticator):
if self.proxy:
authenticator.setUser(self.proxy.user())
authenticator.setPassword(self.proxy.password())
else:
logger.error(self.control.prepend_id('Asked for proxy when no proxy is present: {}'.format(network_proxy.url())))
def set_page_proxy(self, proxy_string, auth_string):
if not proxy_string:
return
pr = proxy_string.split(':', 1)
if len(pr) != 2:
logger.error(self.control.prepend_id('Invalid proxy string {}, auth {}'.format(proxy_string, auth_string)))
return
host = pr[0]
port = int(pr[1])
if not (host and port):
logger.error(self.control.prepend_id('Invalid proxy string {}, auth {}'.format(proxy_string, auth_string)))
return
if auth_string:
aus = auth_string.split(':', 1)
if len(aus) != 2:
logger.error(self.control.prepend_id('Invalid proxy string {}, auth {}'.format(proxy_string, auth_string)))
return
user = aus[0]
password = aus[1]
self.proxy = QNetworkProxy(QNetworkProxy.HttpProxy, host, port, user, password)
else:
self.proxy = QNetworkProxy(QNetworkProxy.HttpProxy, host, port)
def reset(self):
self.rule_list = []
self.proxy = None
def request_finished(self, network_reply):
if not self.control.job():
return
error = network_reply.error()
url = network_reply.url()
url_str = url.toString()
retry_after_sec = 60
if error != 0:
if error == QNetworkReply.OperationCanceledError:
logger.debug(self.control.prepend_id('Operation cancelled for url {}'.format(url_str)))
return
request = network_reply.request()
request_headers_string = 'Request:\n'
for header in request.rawHeaderList():
request_headers_string += '{}: {}\n'.format(header.data().decode(encoding=HTTP_HEADER_CHARSET), request.rawHeader(header).data().decode(encoding=HTTP_HEADER_CHARSET))
response_headers_string = 'Response:\n'
for header in network_reply.rawHeaderList():
header_key = header.data().decode(encoding=HTTP_HEADER_CHARSET)
header_value = network_reply.rawHeader(header).data().decode(encoding=HTTP_HEADER_CHARSET)
if 'Retry-After' == header and header_value.isdigit():
retry_after_sec = int(header_value)
response_headers_string += '{}: {}\n'.format(header_key, header_value)
logger.error(self.control.prepend_id('e_id="{eid};{estr}" url="{url}"\n{req_h}\n{res_h}'.format(eid=error,
estr=network_reply.errorString(),
url=url_str,
req_h=request_headers_string,
res_h=response_headers_string)))
###############################################
# Throttle the retries as lazily as possible
###############################################
# We go slightly above the recommended value, randomize it a bit and double check the values just in case
if retry_after_sec < 1:
retry_after_sec = 2
if retry_after_sec < 100:
retry_after_sec = (retry_after_sec * 10) + randint(0, 30)
# Restrict the retry sec to 1 hour
if retry_after_sec > 3600:
retry_after_sec = 3600
self.control.abort(retry_after_sec)
def createRequest(self, operation, request, device=None):
if not self.control.job():
return super().createRequest(operation, request, device)
url = request.url()
url_str = url.toString()
if self.rule_list:
for r in self.rule_list:
if r.rule.search(url_str):
if r.rule_type == self._reject:
logger.debug(self.control.prepend_id('Blocking {}'.format(url_str)))
return super().createRequest(operation, QNetworkRequest(QUrl()), device)
else:
break
if self.proxy and self.control.job() and self.control.job().is_crawlera:
scheme = url.scheme()
if scheme == 'https':
url.setScheme('http')
request.setRawHeader(b'X-Crawlera-Use-HTTPS', b'1')
# TODO Check if this is needed
request.setSslConfiguration(QSslConfiguration())
request.setUrl(url)
elif scheme == 'http':
pass
else:
# We fail any request that is not http or https
logger.warning(self.control.prepend_id('Unsupported Schema {}'.format(url_str)))
return super().createRequest(operation, QNetworkRequest(QUrl()), device)
key = bytes('{}:{}'.format(self.proxy.user(), self.proxy.password()), HTTP_HEADER_CHARSET)
proxy_auth_value = b'Basic ' + base64.urlsafe_b64encode(key)
request.setRawHeader(b'Proxy-Authorization', proxy_auth_value)
request.setRawHeader(b'Proxy-Connection', b'Keep-Alive')
request.setRawHeader(b'X-Crawlera-Cookies', b'disable')
request.setRawHeader(b'X-Crawlera-UA', b'desktop')
network_reply = super().createRequest(operation, request, device)
network_reply.finished.connect(lambda: self.request_finished(network_reply))
return network_reply
def set_filter(self, filter_str_list):
for filter_str in filter_str_list:
filter_conf = filter_str.split(':', 1)
if len(filter_conf) != 2:
logger.error(self.control.prepend_id('Invalid filter string {}'.format(filter_str)))
return
rule_type = filter_conf[0]
if not (rule_type == self._allow or rule_type == self._reject):
logger.error(self.control.prepend_id('Invalid filter string {}'.format(filter_str)))
return
rule_str = filter_conf[1]
if not rule_str:
logger.error(self.control.prepend_id('Invalid filter string {}'.format(filter_str)))
return
self.rule_list.append(self.Rule(rule_type, re.compile(rule_str)))
class MyWebView(QWebEngineView):
def __init__(self, parent=None, acc=None):
super().__init__()
self.parent = parent
self.session = acc.session
self.user_id = acc.user_id
self.chekLoadFinish = False
self.progress = 0
# Connect sql and get data
os.chdir('/root/PycharmProjects/HearSheep')
con = sql.connect(os.getcwd() + '/db/db.sqlite3')
cur = con.cursor()
with con:
cur.execute('SELECT login, pass, token, proxy, user_agent FROM accaunt where user_id=?', (self.user_id,))
row = cur.fetchone()
self.login, self.password, self.token, proxy, user_agent = row
con.close()
# Path Cookies
os.chdir('/root/PycharmProjects/HearSheep')
pathCookies = '/root/PycharmProjects/HearSheep/cookies/' + str(self.user_id)
if os.path.exists(pathCookies):
pass
else:
os.chdir('/root/PycharmProjects/HearSheep/cookies')
os.mkdir(str(self.user_id))
pathCookies = '/root/PycharmProjects/HearSheep/cookies/' + str(self.user_id)
# settings
profile = self.page().profile()
profile.setPersistentStoragePath(pathCookies)
profile.setHttpUserAgent(user_agent)
profile.setHttpAcceptLanguage("ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4")
urlinfo = parse.urlparse(proxy)
self.proxy = QNetworkProxy()
self.proxy.setType(QNetworkProxy.HttpProxy)
self.proxy.setHostName(urlinfo.hostname)
self.proxy.setPort(urlinfo.port)
self.proxy.setUser(urlinfo.username)
self.proxy.setPassword(urlinfo.password)
QNetworkProxy.setApplicationProxy(self.proxy)
self.page().proxyAuthenticationRequired.connect(self.handleProxyAuthReq)
self.loadProgress.connect(self.setProgress)
self.load(QtCore.QUrl('https://vk.com/'))
self.auth()
def handleProxyAuthReq(self, url, auth, proxyhost):
auth.setUser(self.proxy.user())
auth.setPassword(self.proxy.password())
def setProgress(self, value):
self.progress = value
def waitForSignal(self):
loop = QEventLoop()
def work(value):
print('loop', value)
if value > 95:
loop.quit()
self.loadProgress.connect(work)
loop.exec_()
self.loadProgress.disconnect()
time.sleep(2)
return True
def auth(self):
self.waitForSignal()
if self.page().url().url() == 'https://vk.com/' or self.page().url().url() == 'https://vk.com/index.php':
page = self.page()
page.runJavaScript(
'document.querySelector("#index_email").value = "{}"'.format(self.login))
page.runJavaScript(
'document.querySelector("#index_pass").value = "{}"'.format(self.password))
page.runJavaScript(
'document.querySelector("#index_login_button").click()')
print('auth complete')
def joinTheChat(self, chat):
number_chat, count_users, invite_link = chat
print('connect to', invite_link)
self.load(QtCore.QUrl(invite_link))
print('waiting load inviting page')
self.waitForSignal()
page = self.page()
page.runJavaScript(
"document.querySelector('button.flat_button.round_button._im_join_chat.im-invitation--join').click()")
print('waiting after clicking the button "connect"')
self.waitForSignal()
# Check the entry in the chat
print('get dialogs')
getUrl = UrlAPI(self.token)
response = self.session.get(getUrl.messages.getDialogs(count=3), proxies=self.session.proxies)
dialogs = json.loads(response.text)['response']['items']
for dialog in dialogs:
print(dialog)
if dialog['message']['title'] == 'Читаем стихи':
print('Find')
os.chdir('/root/PycharmProjects/HearSheep')
con = sql.connect(os.getcwd() + '/db/db.sqlite3')
cur = con.cursor()
with con:
cur.execute('INSERT INTO accaunt_chat(user_id,chat_id,number_chat) VALUES(?,?,?)',
(self.user_id, dialog['message']['chat_id'], number_chat))
cur.execute('UPDATE chat SET count_users=? WHERE number_chat=?', (count_users+1, number_chat))
con.close()
print('joined the chat')
return True
print("not joined the chat")
return False
def createChat(self):
getUrl = UrlAPI(self.token)
response = self.session.get(getUrl.messages.createChat(title='Читаем стихи'), proxies=self.session.proxies)
jResponse = json.loads(response.text)
if 'response' in jResponse:
chat_id = jResponse['response']
peer_id = 2000000000 + chat_id
res = self.session.get(getUrl.messages.getInviteLink(peer_id=peer_id), proxies=self.session.proxies)
jRes = json.loads(res.text)['response']
invite_link = jRes['link']
os.chdir('/root/PycharmProjects/HearSheep')
con = sql.connect(os.getcwd() + '/db/db.sqlite3')
cur = con.cursor()
with con:
cur.execute('INSERT INTO chat(number_chat,count_users,invite_link) VALUES (NULL,1,?)', (invite_link, ))
cur.execute('SELECT number_chat FROM chat WHERE invite_link=?', (invite_link, ))
row = cur.fetchone()
number_chat = row[0]
cur.execute('INSERT INTO accaunt_chat(user_id,chat_id,number_chat) VALUES (?,?,?)', (self.user_id, chat_id, number_chat))
con.close()
print('the chat created')
return True
print("the chat don't created")
return False
def sendMessage(self):
page = self.page()
page.runJavaScript(
"document.getElementById('im_editable88230675').innerHTML = 'Йо!'")
page.runJavaScript(
"document.querySelector('button.im-send-btn.im-chat-input--send.im-send-btn_saudio._im_send.im-send-btn_audio').click()")
class MyWebView(QWebEngineView):
def __init__(self, parent=None, user_id=None):
super().__init__()
self.parent = parent
self.chekLoadFinish = False
# Connect sql and get data
os.chdir('/root/PycharmProjects/HearSheep')
con = sql.connect(os.getcwd() + '/db/db.sqlite3')
cur = con.cursor()
with con:
cur.execute(
'SELECT login, pass, proxy, user_agent FROM accaunt where user_id=?',
(user_id, ))
row = cur.fetchone()
self.login, self.password, proxy, user_agent = row
con.close()
#Path Cookies
os.chdir('/root/PycharmProjects/HearSheep')
pathCookies = '/root/PycharmProjects/HearSheep/cookies/' + str(user_id)
if os.path.exists(pathCookies):
pass
else:
os.chdir('/root/PycharmProjects/HearSheep/cookies')
os.mkdir(str(user_id))
pathCookies = '/root/PycharmProjects/HearSheep/cookies/' + str(
user_id)
# settings
profile = self.page().profile()
profile.setPersistentStoragePath(pathCookies)
profile.setHttpUserAgent(user_agent)
profile.setHttpAcceptLanguage("ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4")
urlinfo = parse.urlparse(proxy)
self.proxy = QNetworkProxy()
self.proxy.setType(QNetworkProxy.HttpProxy)
self.proxy.setHostName(urlinfo.hostname)
self.proxy.setPort(urlinfo.port)
self.proxy.setUser(urlinfo.username)
self.proxy.setPassword(urlinfo.password)
QNetworkProxy.setApplicationProxy(self.proxy)
#
self.page().proxyAuthenticationRequired.connect(
self.handleProxyAuthReq)
self.loadStarted.connect(self.loadStart)
self.loadFinished.connect(self.loadFinish)
self.load(QtCore.QUrl('https://vk.com'))
def handleProxyAuthReq(self, url, auth, proxyhost):
auth.setUser(self.proxy.user())
auth.setPassword(self.proxy.password())
def loadStart(self):
self.chekLoadFinish = False
def loadFinish(self):
self.chekLoadFinish = True
if self.page().url().url() == 'https://vk.com/':
self._auth()
def _auth(self):
page = self.page()
page.runJavaScript(
'document.querySelector("#index_email").value = "{}"'.format(
self.login))
page.runJavaScript(
'document.querySelector("#index_pass").value = "{}"'.format(
self.password))
page.runJavaScript(
'document.querySelector("#index_login_button").click()')
