def request(self, method, url, **kwargs):
"""
使用 self.context.session.request()发起请求
:param method:
:param url:
:param params:
:return:
"""
if not kwargs.has_key('timeout'):
kwargs['timeout'] = Init.timeout
pass
response = None
while response == None:
try:
response = self.context.session.request(method, url, **kwargs)
pass
except ProxyError:
logger().error(traceback.format_exc())
if hasattr(self.context, 'proxy_expire'):
delattr(self.context, 'proxy_expire')
except Exception:
logger().error('网络错误, {0} {1}\n {2}'.format(
method, url, traceback.format_exc()))
if hasattr(self.context, 'proxy_expire'):
delattr(self.context, 'proxy_expire')
pass
return response
pass
def execute(self, arguments):
ra_ti = function.random_time()
time.sleep(ra_ti)
logger().info('sleep_time:' + str(ra_ti))
response = self.context.request(
'GET', 'https://www.douyu.com/directory/game/wzry')
tl = response.text
a = xt.fromstring(tl)
ul = a.xpath('//ul[@id="live-list-contentbox"]')
li = ul[0].xpath('li')
y = 0
listss = {}
for x in li:
img = x.xpath('a/span/img[@class="JS_listthumb"]/@data-original')
href = x.xpath('a[@class="play-list-link"]/@href')
title = (x.xpath('a/div/div/h3[@class="ellipsis"]')[0]).text
types = (x.xpath('a/div/div/span[@class="tag ellipsis"]')[0]).text
author = (
x.xpath('a/div/p/span[@class="dy-name ellipsis fl"]')[0]).text
heat = (x.xpath('a/div/p/span[@class="dy-num fr"]')[0]).text
listss[y] = {
'title': title,
'img': img[0],
'types': types,
'author': author,
'heat': heat,
'heat_int': str(function.conversion(heat)),
'href': 'https://www.douyu.com' + str(href[0]),
'create_time': str(time.time())
}
y += 1
pass
json_str = json.dumps(listss)
self.context.redis.set('live', json_str)
print 'no'
def execute(self, arguments):
# 初始化日志
app_log_path = sys.path[0] + '/logs/hd/{2}/{0}_{1}.log'.format(
datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
random.randint(100000, 9999999),
datetime.now().strftime('%Y_%m_%d'))
self.context.logger = getLogger(app_log_path, True)
self.context.app_log_path = app_log_path
hym.core.default_logger = self.context.logger
# requests初始化
requests.packages.urllib3.disable_warnings()
self.context.session = requests.session()
self.context.session.verify = False
log_path = sys.path[0] + '/logs/request_log/{2}/{0}_{1}.log'.format(
time.strftime('%Y-%m-%d_%H-%M-%S_'),
random.randint(1000000, 9999999),
datetime.now().strftime('%Y_%m_%d'))
if not os.path.isdir(os.path.split(log_path)[0]):
os.makedirs(os.path.split(log_path)[0])
pass
logger().info('request log path: {0}'.format(log_path))
requestLogAdapter = RetryAdapter(self.check_retry,
log_path,
max_retries=5)
self.context.session.mount('http://', requestLogAdapter)
self.context.session.mount('https://', requestLogAdapter)
self.context.session.headers.update({
'(Request-Line)': 'GET /htm/novel1/27695.htm HTTP/1.1',
'Accept-Encoding': 'gzip, deflate',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240',
'Accept': 'text/html, application/xhtml+xml, image/jxr, */*',
'Accept-Language': 'zh-CN',
})
self.context.request = self.request
#初始化redis链接
#self.context.redis=function.link_redis()
pass
def execute(self, arguments):
# 初始化日志
app_log_path = sys.path[0] + '/logs/live/{0}_{1}.log'.format(
datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
random.randint(100000, 9999999))
self.context.logger = getLogger(app_log_path, True)
self.context.app_log_path = app_log_path
hym.core.default_logger = self.context.logger
# requests初始化
requests.packages.urllib3.disable_warnings()
self.context.session = requests.session()
self.context.session.verify = False
log_path = sys.path[0] + '/logs/request_log/{0}_{1}.log'.format(
time.strftime('%Y-%m-%d_%H-%M-%S_'),
random.randint(1000000, 9999999))
if not os.path.isdir(os.path.split(log_path)[0]):
os.makedirs(os.path.split(log_path)[0])
pass
logger().info('request log path: {0}'.format(log_path))
requestLogAdapter = RetryAdapter(self.check_retry,
log_path,
max_retries=5)
self.context.session.mount('http://', requestLogAdapter)
self.context.session.mount('https://', requestLogAdapter)
self.context.session.headers.update({
'Accept-Encoding':
'gzip, deflate, br',
'Accept-Language':
'zh-CN,zh;q=0.9',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
})
self.context.request = self.request
#初始化redis链接
self.context.redis = function.link_redis()
pass
def execute(self, arguments):
if self.context.NoticeList.has_key(
'qq_pg') and self.context.NoticeList['qq_pg']:
y = 0
while self.context.NoticeList['qq_pg'].has_key(y):
response = self.context.request(
'GET', self.context.NoticeList['qq_pg'][y]['href'])
Article = function.qq_pg_clean_NoticeArticle(
response.content, self.context.NoticeList['qq_pg'][y])
api_client.client.post_yxgg_notice_data(Article)
logger().info(Article)
logger().info('qinqinqin')
y += 1
pass
pass
if self.context.NoticeList.has_key(
'wy_stzb') and self.context.NoticeList['wy_stzb']:
ws = 0
while self.context.NoticeList['wy_stzb'].has_key(ws):
#print self.context.NoticeList['wy_stzb']
response = self.context.request(
'GET', self.context.NoticeList['wy_stzb'][ws]['href'])
response.encoding = response.apparent_encoding
texts = response.text
Article = function.wy_stzb_clean_NoticeArticle(
texts, self.context.NoticeList['wy_stzb'][ws])
api_client.client.post_yxgg_notice_data(Article)
logger().info(Article)
logger().info('qinqinqin')
ws += 1
pass
pass
if self.context.NoticeList.has_key(
'qq_pvp') and self.context.NoticeList['qq_pvp']:
pvp = 0
while self.context.NoticeList['qq_pvp'].has_key(pvp):
response = self.context.request(
'GET', self.context.NoticeList['qq_pvp'][pvp]['href'])
Article = function.qq_pvp_clean_NoticeArticle(
response.content, self.context.NoticeList['qq_pvp'][pvp])
api_client.client.post_yxgg_notice_data(Article)
logger().info(Article)
logger().info('qinqinqin')
pvp += 1
pass
pass
if self.context.NoticeList.has_key(
'qq_gp') and self.context.NoticeList['qq_gp']:
gp = 0
while self.context.NoticeList['qq_gp'].has_key(gp):
response = self.context.request(
'GET', self.context.NoticeList['qq_gp'][gp]['json_href'])
Article = function.qq_gp_clean_NoticeArticle(
response.content, self.context.NoticeList['qq_gp'][gp])
api_client.client.post_yxgg_notice_data(Article)
logger().info(Article)
logger().info('qinqinqin')
gp += 1
pass
pass