def __init__(self, mail_info):
Counter.__init__(self)
self.session = grequests.Session()
self.msg_id = self.count
self.mail_info = mail_info
self.params = {
'time': time.time(),
'appid': '501004106',
'msgid': '0',
'clientid': '53999199',
'ptwebqq': '',
'vfwebqq': '',
'psessionid': '',
'friendList': {},
'referer2':
'http://s.web2.qq.com/proxy.html?v=20130916001&callback=1&id=1',
'referer':
'http://d1.web2.qq.com/proxy.html?v=20151105001&callback=1&id=2',
'smartqqurl': 'http://w.qq.com/login.html'
}
self.uin2tuin = 'http://s.web2.qq.com/api/get_friend_uin2?tuin={0}&type=1&vfwebqq={1}&t=1471404618'
self.session.headers = {
'Accept':
'application/javascript, */*;q=0.8',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64) '
'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/31.0.1650.48 Safari/537.36 QQBrowser/8.2.4258.400'
}
self.session.verify = True
def __init__(self, symbol, tRate):
super(Stock, self).__init__()
self.success = False
self.symbol = symbol
self.session = requests.Session()
# self.db = sql.connect(host='database-1.cedqzlba2fep.us-east-1.rds.amazonaws.com', port=3306, user='******', passwd='password')
# self.cx = self.db.cursor()
self.R = tRate
def make_session(http_max_pool_size, num_retries):
session = grequests.Session()
adapter = adapters.HTTPAdapter(max_retries=num_retries,
pool_maxsize=http_max_pool_size,
pool_block=True)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def get_session():
'''Get a request's session
Returns:
A session (grequests API) with all information
session inside the object
'''
session = grequests.Session()
return session
def 模具一一会话(self):
self.模具一一导入cookie()
url = 'http://3e38.com/image/article_add.php?channelid=17'
self.会话 = grequests.Session() # 创建全局会话
self.会话.cookies.update(self.cookies) # 格式化cookie,全网页才能共享登录cookie,
返回网页内容 = self.会话.get(url)
返回网页内容 = 返回网页内容.text
if '看不清' in 返回网页内容:
print('登录失效,正在启动浏览器.')
self.模具一一浏览器输入验证码提取cookies()
else: # 否则
print('======登录成功=========')
def request_proxy_pages(session=grequests.Session()):
session.headers.update({'User-Agent': 'Proxist %s' % __version__})
page = '/1'
while True:
response = session.get(
url=urljoin("http://proxylist.hidemyass.com/", page))
document = html.fromstring(response.content)
next_page = document.xpath("//a[@class='next']")
if not next_page:
yield document
break
page = next_page[0].attrib['href']
yield document
def send_requests(self, list_requests):
threads_count = multiprocessing.cpu_count()
# 408 Request Timeout
retry = Retry(total=2, backoff_factor=0.2, status_forcelist=[408])
session = grequests.Session()
session.mount('http://', HTTPAdapter(max_retries=retry))
session.mount('https://', HTTPAdapter(max_retries=retry))
responses = [
grequests.get(link, timeout=self.timeout, session=session)
for link in list_requests
]
return grequests.imap(responses,
size=threads_count * 4,
exception_handler=self.handle_exceptions)
def get_grequests_session(pool_connections=1, pool_maxsize=32, max_retries=5):
"""
This method gets a grequest session.
Call this function when send many requests to the same host.
Parameters
----------
pool_connections: number of connection pools
pool_maxsize: size of each pool
max_retries: max retry times when connection fails
Returns
-------
Session
"""
session = grequests.Session()
adapter = requests.adapters.HTTPAdapter(pool_connections=pool_connections,
pool_maxsize=pool_maxsize,
max_retries=max_retries)
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
def url_connection(link):
global counter # Initialisation of global value (counter)
url = 'https://www.tripadvisor.com' + link # Create a url link
print(counter, url)
# Try to connect with web-site
with grequests.Session() as session:
# Initialisation of header to connect url
session.headers[
'Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
session.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' \
'(KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36'
# Try to connect with web site. If connection fall, or can't create values, so connection lost and
try:
resp = session.get(
url, timeout=5
) # Connect with url with 5 second timeout and create a copy for next pasrser
soup = BS(resp.content, 'html.parser') # Use method BeautifulSoup
price = soup.find(
class_='_2mn01bsa').contents[0] # Find information about price
cuisine = soup.find('div', {
'class': ['_60ofm15k', '_1XLfiSsv']
}).contents # Find information about cuisine
except:
price = None
cuisine = None
print('no connection/empty data')
counter += 1 # arise counter
price = price if price and (
'$' in price) else None # Test, that information have data about price
print(f'The costs is: {price}. The cuisine of restaurant is {cuisine}')
return [price, cuisine]
def __init__(self, environment, access_token):
"""Instantiates an instance of OandaPy's streaming API wrapper.
:param environment: (required) Provide the environment for oanda's
REST api, either 'practice', or 'live'.
:param access_token: (required)
"""
if environment == 'practice':
self.api_url = 'https://stream-fxpractice.oanda.com'
elif environment == 'live':
self.api_url = 'https://stream-fxtrade.oanda.com'
else:
raise BadEnvironment(environment)
self.access_token = access_token
self.client = grequests.Session()
self.client.stream = True
self.connected = False
# personal token authentication
if self.access_token:
self.client.headers[
'Authorization'] = 'Bearer ' + self.access_token
def check_if_proxy_is_working(self, proxies):
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
import grequests
session = grequests.Session()
retry = Retry(connect=0, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
urls = []
for proxy in proxies:
px = {
'http': "http://" + proxy,
}
urls.append(
grequests.get(
'http://0.gravatar.com/avatar/c77120cb5dad4e57ec374451d01aec7a?s=150&d=mysteryman&r=G',
headers={"p": proxy},
proxies=px,
timeout=self.timeout,
stream=True))
def exception_handler(request, exception):
pass
#print("Request failed")
#print(exception)
x = grequests.map(set(urls), exception_handler=exception_handler)
working = []
for r in x:
if r:
if r.status_code == 200:
working.append("http://" +
r.request.__dict__["headers"]["p"])
return working
import re
import gevent
import grequests
from bs4 import BeautifulSoup
from pypinyin import pinyin, Style
from sqlalchemy import create_engine
from config import Config
from data_spider.model import Game, Price
session = grequests.Session()
def is_include_zh(s):
for c in s:
if not ('\u4e00' <= c <= '\u9fa5'):
return False
return True
def clear_text(text):
return re.sub("[\s+\.\!\/_,$%^*(+\"\')]+|[+——()?【】“”！，。？、~@#￥%……&*（）]+'", '', text)
def get_connection(config):
# pool = redis.ConnectionPool(host=config.host, port=config.port, decode_responses=True)
# return redis.Redis(connection_pool=pool)
return create_engine(config.sqlite)
import grequests
s1 = grequests.Session()
s2 = grequests.Session()
s3 = grequests.Session()
print(s1.__hash__(), s2.__hash__(), s3.__hash__())
rs = [
grequests.get('http://www.baidu.com', session=s1),
grequests.get('http://www.baidu.com', session=s2),
grequests.get('http://www.baidu.com', session=s3)
]
resp = grequests.map(rs)
print(s1.cookies)
def __init__(self):
self.header = Srequests.headers
self.session = grequests.Session()
self.session.cookies = cookiejar.LWPCookieJar(filename='Cookies')