-
Notifications
You must be signed in to change notification settings - Fork 2
/
worker.py
64 lines (57 loc) · 2.66 KB
/
worker.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
# -*- encoding: utf-8 -*
from datetime import timedelta, date
from article import Failed, Article
def get_content_between(src, starttag, endtag):
si = src.find(starttag)
if si >= 0:
sii = si + len(starttag)
se = src.find(endtag, si)
if se > sii:
return src[sii: se]
return ''
class Worker(object):
dayDelta = timedelta(days=1)
timeout = 30
def __init__(self):
self.newsDict = {}
self.history = {}
def save_temp_dict(self):
for k in self.newsDict:
if not self.newsDict[k]['valid']:
error = Failed(link=self.newsDict[k]['link'], title=self.newsDict[k]['title'],
post_date=self.newsDict[k]['post_date'])
error.post_time = self.newsDict[k]['post_time']
error.category = self.newsDict[k]['category']
error.summary = self.newsDict[k]['summary']
error.error_count = self.newsDict[k]['error_count'] + 1
error.save()
continue
article = Article(link=self.newsDict[k]['link'], title=self.newsDict[k]['title'],
post_date=self.newsDict[k]['post_date'])
article.post_time = self.newsDict[k]['post_time']
article.category = self.newsDict[k]['category']
article.summary = self.newsDict[k]['summary']
article.source = self.newsDict[k]['source']
article.source_link = self.newsDict[k]['source_link']
article.content = self.newsDict[k]['content']
article.image_links = self.newsDict[k]['image_links']
article.video_links = self.newsDict[k]['video_links']
article.comment_num = self.newsDict[k]['comment_num']
article.reply_num = self.newsDict[k]['reply_num']
article.save()
if self.newsDict[k]['error_count'] > 0:
Failed.objects(link=self.newsDict[k]['link']).delete()
def reget_errorlist(self):
print('#### reget error list ####')
# 错误多次的不再重试
for item in Failed.objects(error_count__lte=5):
print(item.link)
self.newsDict[item.link] = {'title': item.title, 'link': item.link, 'post_time': item.post_time,
'post_date': item.post_date, 'category': item.category, 'summary': item.summary,
'valid': True, 'error_count': item.error_count}
try:
self.get_detail(item.link)
except StandardError:
self.newsDict['valid'] = False
self.save_temp_dict()
print('#### end ####')