-
Notifications
You must be signed in to change notification settings - Fork 0
/
scheduler.py
43 lines (37 loc) · 1.03 KB
/
scheduler.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
# -*- coding: utf-8 -*-
import time
import datetime
from crawler import Crawler
from db import Mongo
class Scheduler(object):
def __init__(self):
self.crawler = Crawler()
self.db = Mongo()
def run(self, url, date):
"""
开始爬取并保存
:param url: 爬取url
:param date: 爬取时间
:return:
"""
data = self.crawler.main(url)
data['date'] = date
self.db.add(data)
def main(self, url, h=1, m=0):
"""
主程序入口
:param url: 爬取url
:param h: 执行程序的时间
:param m:执行程序的时间
:return:
"""
while True:
now = datetime.datetime.now()
date = str(now.year) + '-' + str(now.month) + '-' + str(now.day)
if now.hour == h and now.minute == m:
self.run(url, date)
# 每个60秒检测一次
time.sleep(60)
if __name__ == '__main__':
sch = Scheduler()
sch.main('https://cn.bing.com')