def __init__(self):
'''
>>>from v2ex_spider import rss_spider
>>>rss_spider.Rss_spider()
'''
logging.info('start Rss spider')
self.v2ex_rss_url_list=['https://www.v2ex.com/index.xml',
'https://www.v2ex.com/feed/tab/qna.xml',
'https://www.v2ex.com/feed/tab/jobs.xml',
'https://www.v2ex.com/feed/tab/deals.xml',
'https://www.v2ex.com/feed/tab/city.xml',
'https://www.v2ex.com/feed/tab/play.xml',
'https://www.v2ex.com/feed/tab/apple.xml',
'https://www.v2ex.com/feed/tab/creative.xml',
'https://www.v2ex.com/feed/tab/tech.xml']
self.latest_hot_api=['https://www.v2ex.com/api/topics/latest.json','https://www.v2ex.com/api/topics/hot.json']
self.topic_sleep_time=10
logging.debug('open sql database')
self.SQ=SQL()
self.SQ.open_datebase()
self.redis_conn=Redis()
self.load_config()
#run
try:
self.latest_and_hot()
except APIError as e:
pass
self.gen_topic_queue()
#end
self.SQ.close_datebase()
logging.info('end the Rss spider')
def __init__(self, url, sleep_time):
'''
>>>from v2ex_spider import base_spider
>>>base_spider.start(url,sleep_time)
'''
self.url = url
self.sleep_time = sleep_time
time.sleep(int(self.sleep_time))
self.SQ = SQL()
self.SQ.open_datebase()
#run
self.load_config()
self.spider()
#end
self.SQ.close_datebase()
def __init__(self):
'''
$ python run.py
or
$ ./Run.sh
'''
logging.info('start')
logging.debug('open sql database.')
self.SQ = SQL()
self.SQ.open_datebase()
self.redis_conn = Redis()
self.load_config()
#base
self.load_json()
# self.update_cookies()
try:
self.update_nodes()
except APIError as e:
pass
def __init__(self):
'''
$ python run.py
or
$ ./Run.sh
'''
self.SQ = SQL()
self.SQ.open_datebase()
self.redis_conn = Redis()
self.load_config()
#start
self.load_json()
self.update_cookies()
try:
self.update_nodes()
except APIError as e:
print(e)
self.get_rss()
self.tasker()
self.tester_tasker()
#end
self.end()
def init_database(self):
logging.debug('init database')
self.SQ = SQL()
self.SQ.open_datebase()
def init_database(self):
self.SQ=SQL()
self.SQ.open_datebase()