def login(siteRequest):
if siteRequest.needLogin and not Login.isLogin and not Login.checkLogin(
siteRequest):
res = Utils.urlPost(siteRequest.loginPostPage,
data=Login.buildPostData(siteRequest),
headers=siteRequest.loginHeader,
returnRaw=True)
print("PostLoginData,", res.status, res.reason)
Utils.saveCookie()
Login.isLogin = Login.checkLogin(siteRequest)
return Login.isLogin
else:
Login.isLogin = True
return True
def _deco(ch, method, properties, body):
try:
context.RequestContext()
log.info("开始执行业务方法")
func(ch, method, properties, body)
except Exception as e:
print(e)
if conf.env == 'conf':
mail_title = Utils.currentTime() + " " + sys.argv[0] + "执行异常,异常内容见邮件"
Utils.sendMail(mail_title, traceback.format_exc(), [
'邮箱地址'
])
else:
traceback.print_exc()
log.info("业务方法异常")
finally:
log.info("队列消息确认消费")
ch.basic_ack(delivery_tag=method.delivery_tag)
def _deco(self, *args, **kwargs):
cache_key = Utils._compute_key(func, args, kwargs)
if hasattr(self, name):
cache_obj = getattr(self, name, None)
if isinstance(cache_obj, MyRedis):
data = cache_obj.get(cache_key)
if data is None:
print("没有命中缓存,执行函数")
data = func(self, *args, **kwargs)
print("将数据加入到缓存中")
cache_obj.setex(cache_key, time, data)
else:
print("命中缓存")
else:
data = func(self, *args, **kwargs)
else:
data = func(self, *args, **kwargs)
return data
def convertToSeed(self, soupObj):
assert soupObj is not None
trList = soupObj.select("table.torrents tr")
seedList = []
cnt = 0
for tr in trList:
cnt += 1
if cnt == 1:
# skip the caption tr
continue
seed = SeedInfo()
tdList = tr.select("td.rowfollow")
if len(tdList) < 9:
# skip embedded contents
continue
seed.title = tdList[1].select("table td a")[0]["title"]
seed.url = self.siteRequest.baseUrl + tdList[1].select(
"table td a")[0]['href']
seed.free = len(tdList[1].select("table font.free")) > 0
seed.hot = len(tdList[1].select("table font.hot")) > 0
seed.since = Utils.getContent(tdList[3], "span")
seed.size = float(self.parseSize(tdList[4]))
seed.uploadNum = int(
self.getContentWithoutFontorStrong(tdList[5], "a"))
seed.downloadNum = int(
self.getContentWithoutFontorStrong(tdList[6], "a"))
seed.finishNum = int(
self.getContentWithoutFontorStrong(tdList[7], "a"))
seed.id = self.parseId(seed.url)
# print('\n'.join(['%s:%s' % item for item in seed.__dict__.items()]) + "\n")
seedList.append(seed)
return seedList
def checkLogin(siteRequest):
content = Utils.getContent(Utils.urlGet(siteRequest.homePage),
siteRequest.loginVerificationCss)
return content is not None and content == siteRequest.loginVerificationStr
def json(self, result):
self.write(json.dumps(result.json(), cls=Utils.JSONEncoder(), sort_keys=False))
self.finish()
def _deco(client, userdata, msg):
try:
print("\n" + Utils.currentTime() + " 开始执行业务方法")
func(client, userdata, msg)
except Exception as e:
print("业务方法异常")
def crawl(self):
ret = self.login()
assert ret
soupObj = Utils.urlGet(self.siteRequest.homePage)
return self.convertToSeed(soupObj)
def crawl(self):
ret = self.login()
assert ret
soupObj = Utils.urlGet(self.siteRequest.homePage)
return self.parsePoint(soupObj)