/
fetch.py
52 lines (44 loc) · 1.17 KB
/
fetch.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf8')
import util
import random
import traceback
import time
from website import (
fetch_airchina,
fetch_ceair,
fetch_ch,
fetch_csair,
fetch_ctrip
)
from tomorrow import threads
from constant import Website, WEBSITE_NAME
from setting.fetch_settings import FETCH_PERIODS, FETCH_WEBSITES, THREAD_COUNT
func_map = {
Website.CTRIP: fetch_ctrip,
Website.CEAIR: fetch_ceair,
Website.CSAIR: fetch_csair,
Website.CH: fetch_ch,
Website.AIRCHINA: fetch_airchina,
}
tasks = [(func_map[w], p, w) for p in FETCH_PERIODS for w in FETCH_WEBSITES]
random.shuffle(tasks)
@threads(THREAD_COUNT)
def worker(task):
module, period, website = task
util.log_info('deal at %s %s %s' % (str(module), str(period), str(website)))
try:
module.fetch(period)
except Exception as e:
util.log_error(WEBSITE_NAME[website] + ' ' + str(e) + ' ' +
str(period) +
traceback.format_exc().replace('\n', ' | '))
time.sleep(5)
def fetch():
for x in tasks:
worker(x)
if __name__ == '__main__':
fetch()