def deal(self, content):
def compact(l):
ret = []
for i in l:
ret += i
return ret
general = parser.ParseIndexGeneral(content)
tries = 4
while tries > 0:
try:
news = yield [get_data(i[1], parser.ParsePost) for i in general['news']]
info = yield [get_data(makeUrl('category', CatId=i), parser.ParseCategory) for i in [66, 67, 68]]
ret = {
"news": [json.loads(i) for i in news],
"info": compact([json.loads(i)[:4] for i in info]),
}
break
except HTTPError as e:
tries -= 1
print("503, Retry.....")
yield tornado.gen.sleep(.1)
continue
for n, i in enumerate(ret['news']):
i["link"] = convertUrl(general["news"][n][1], strict=True)
return ret
def deal(self, content):
def compact(l):
ret = []
for i in l:
ret += i
return ret
general = parser.ParseIndexGeneral(content)
tries = 4
while tries > 0:
try:
news = yield [get_data(i[1], parser.ParsePost) for i in general['news']]
info = yield [get_data(makeUrl('category', CatId=i), parser.ParseCategory) for i in [66, 67, 68]]
ret = {
"news": [json.loads(i) for i in news],
"info": compact([json.loads(i)[:4] for i in info]),
}
break
except HTTPError:
tries -= 1
print("503, Retry.....")
yield tornado.gen.sleep(.1)
continue
for n, i in enumerate(ret['news']):
i["link"] = convertUrl(general["news"][n][1], strict=True)
return ret
def get_data(url, handler):
key = parser.convertUrl(url)
if not options.DEBUG:
cached = yield fetcher.get_data(key)
if cached:
return cached
result = yield fetcher.get_page(url)
ret = yield maybe_future(handler(result))
ret = json.dumps(ret)
yield fetcher.write_data(key, ret, options.CACHE_TIME)
return ret
def get_data(url, handler):
key = parser.convertUrl(url)
if not options.DEBUG:
cached = yield fetcher.get_data(key)
if cached:
return cached
result = yield fetcher.get_page(url)
ret = yield maybe_future(handler(result))
ret = json.dumps(ret)
yield fetcher.write_data(key, ret, options.CACHE_TIME)
return ret
