def __init__(self, mail_info): Counter.__init__(self) self.session = grequests.Session() self.msg_id = self.count self.mail_info = mail_info self.params = { 'time': time.time(), 'appid': '501004106', 'msgid': '0', 'clientid': '53999199', 'ptwebqq': '', 'vfwebqq': '', 'psessionid': '', 'friendList': {}, 'referer2': 'http://s.web2.qq.com/proxy.html?v=20130916001&callback=1&id=1', 'referer': 'http://d1.web2.qq.com/proxy.html?v=20151105001&callback=1&id=2', 'smartqqurl': 'http://w.qq.com/login.html' } self.uin2tuin = 'http://s.web2.qq.com/api/get_friend_uin2?tuin={0}&type=1&vfwebqq={1}&t=1471404618' self.session.headers = { 'Accept': 'application/javascript, */*;q=0.8', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/31.0.1650.48 Safari/537.36 QQBrowser/8.2.4258.400' } self.session.verify = True
def __init__(self, symbol, tRate): super(Stock, self).__init__() self.success = False self.symbol = symbol self.session = requests.Session() # self.db = sql.connect(host='database-1.cedqzlba2fep.us-east-1.rds.amazonaws.com', port=3306, user='******', passwd='password') # self.cx = self.db.cursor() self.R = tRate
def make_session(http_max_pool_size, num_retries): session = grequests.Session() adapter = adapters.HTTPAdapter(max_retries=num_retries, pool_maxsize=http_max_pool_size, pool_block=True) session.mount('http://', adapter) session.mount('https://', adapter) return session
def get_session(): '''Get a request's session Returns: A session (grequests API) with all information session inside the object ''' session = grequests.Session() return session
def 模具一一会话(self): self.模具一一导入cookie() url = 'http://3e38.com/image/article_add.php?channelid=17' self.会话 = grequests.Session() # 创建全局会话 self.会话.cookies.update(self.cookies) # 格式化cookie,全网页才能共享登录cookie, 返回网页内容 = self.会话.get(url) 返回网页内容 = 返回网页内容.text if '看不清' in 返回网页内容: print('登录失效,正在启动浏览器.') self.模具一一浏览器输入验证码提取cookies() else: # 否则 print('======登录成功=========')
def request_proxy_pages(session=grequests.Session()): session.headers.update({'User-Agent': 'Proxist %s' % __version__}) page = '/1' while True: response = session.get( url=urljoin("http://proxylist.hidemyass.com/", page)) document = html.fromstring(response.content) next_page = document.xpath("//a[@class='next']") if not next_page: yield document break page = next_page[0].attrib['href'] yield document
def send_requests(self, list_requests): threads_count = multiprocessing.cpu_count() # 408 Request Timeout retry = Retry(total=2, backoff_factor=0.2, status_forcelist=[408]) session = grequests.Session() session.mount('http://', HTTPAdapter(max_retries=retry)) session.mount('https://', HTTPAdapter(max_retries=retry)) responses = [ grequests.get(link, timeout=self.timeout, session=session) for link in list_requests ] return grequests.imap(responses, size=threads_count * 4, exception_handler=self.handle_exceptions)
def get_grequests_session(pool_connections=1, pool_maxsize=32, max_retries=5): """ This method gets a grequest session. Call this function when send many requests to the same host. Parameters ---------- pool_connections: number of connection pools pool_maxsize: size of each pool max_retries: max retry times when connection fails Returns ------- Session """ session = grequests.Session() adapter = requests.adapters.HTTPAdapter(pool_connections=pool_connections, pool_maxsize=pool_maxsize, max_retries=max_retries) session.mount("http://", adapter) session.mount("https://", adapter) return session
def url_connection(link): global counter # Initialisation of global value (counter) url = 'https://www.tripadvisor.com' + link # Create a url link print(counter, url) # Try to connect with web-site with grequests.Session() as session: # Initialisation of header to connect url session.headers[ 'Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' session.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' \ '(KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36' # Try to connect with web site. If connection fall, or can't create values, so connection lost and try: resp = session.get( url, timeout=5 ) # Connect with url with 5 second timeout and create a copy for next pasrser soup = BS(resp.content, 'html.parser') # Use method BeautifulSoup price = soup.find( class_='_2mn01bsa').contents[0] # Find information about price cuisine = soup.find('div', { 'class': ['_60ofm15k', '_1XLfiSsv'] }).contents # Find information about cuisine except: price = None cuisine = None print('no connection/empty data') counter += 1 # arise counter price = price if price and ( '$' in price) else None # Test, that information have data about price print(f'The costs is: {price}. The cuisine of restaurant is {cuisine}') return [price, cuisine]
def __init__(self, environment, access_token): """Instantiates an instance of OandaPy's streaming API wrapper. :param environment: (required) Provide the environment for oanda's REST api, either 'practice', or 'live'. :param access_token: (required) """ if environment == 'practice': self.api_url = 'https://stream-fxpractice.oanda.com' elif environment == 'live': self.api_url = 'https://stream-fxtrade.oanda.com' else: raise BadEnvironment(environment) self.access_token = access_token self.client = grequests.Session() self.client.stream = True self.connected = False # personal token authentication if self.access_token: self.client.headers[ 'Authorization'] = 'Bearer ' + self.access_token
def check_if_proxy_is_working(self, proxies): from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry import grequests session = grequests.Session() retry = Retry(connect=0, backoff_factor=0.5) adapter = HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) urls = [] for proxy in proxies: px = { 'http': "http://" + proxy, } urls.append( grequests.get( 'http://0.gravatar.com/avatar/c77120cb5dad4e57ec374451d01aec7a?s=150&d=mysteryman&r=G', headers={"p": proxy}, proxies=px, timeout=self.timeout, stream=True)) def exception_handler(request, exception): pass #print("Request failed") #print(exception) x = grequests.map(set(urls), exception_handler=exception_handler) working = [] for r in x: if r: if r.status_code == 200: working.append("http://" + r.request.__dict__["headers"]["p"]) return working
import re import gevent import grequests from bs4 import BeautifulSoup from pypinyin import pinyin, Style from sqlalchemy import create_engine from config import Config from data_spider.model import Game, Price session = grequests.Session() def is_include_zh(s): for c in s: if not ('\u4e00' <= c <= '\u9fa5'): return False return True def clear_text(text): return re.sub("[\s+\.\!\/_,$%^*(+\"\')]+|[+——()?【】“”!,。?、~@#¥%……&*()]+'", '', text) def get_connection(config): # pool = redis.ConnectionPool(host=config.host, port=config.port, decode_responses=True) # return redis.Redis(connection_pool=pool) return create_engine(config.sqlite)
import grequests s1 = grequests.Session() s2 = grequests.Session() s3 = grequests.Session() print(s1.__hash__(), s2.__hash__(), s3.__hash__()) rs = [ grequests.get('http://www.baidu.com', session=s1), grequests.get('http://www.baidu.com', session=s2), grequests.get('http://www.baidu.com', session=s3) ] resp = grequests.map(rs) print(s1.cookies)
def __init__(self): self.header = Srequests.headers self.session = grequests.Session() self.session.cookies = cookiejar.LWPCookieJar(filename='Cookies')