def run():
r = seed_db.select()
if (len(r)):
seedList = r.list()
for seed in seedList:
sid = seed["sid"]
frequency = seed["frequency"]
starttime = seed["start_time"]
finishtime = seed["finish_time"]
if seed["enabled"] == 1:
if (frequency + finishtime) <= now():
if not scrapy(seed):
scrapy(seed)
else:
next_queue.append(finishtime + frequency)
if len(next_queue) == 0:
next_time = 600
else:
next_time = min(next_queue) - now()
if next_time <= 0:
next_time = 1
print "Idle status, waitting for ", next_time
time.sleep(next_time)
run()
def scrapy(seed):
sid = seed["sid"]
frequency = seed["frequency"]
starttime = seed["start_time"]
finishtime = seed["finish_time"]
starttime = now()
s = Seed(seed)
try:
g = Grab(s)
g.push()
finishtime = now()
next_queue.append(finishtime + frequency)
seed_db.edit(sid, **{"start_time":starttime, "finish_time":finishtime})
log_db.insert(**{"sid" : sid, "start_time" : starttime, "finish_time" : finishtime, "`status`" : 1, "message" : "采集成功"})
return True
except Exception, e:
log_db.insert(**{"sid" : sid, "start_time" : starttime, "finish_time" : now(), "`status`" : 0, "message" : "采集失败, 原因:" + str(e)})
return False
def __init__(self, profile): self.secret_access_key = profile["secret_access_key"] self.access_id = profile["access_id"] config.ACCESS_KEY = safestr(self.access_id) config.SECRET_KEY = safestr(self.secret_access_key) if "bucket" in profile and profile["bucket"]: self.bucket = profile["bucket"] else: self.bucket = "cmdp" self.token_expires = 3600 + now() self.init_tokent()
def __init__(self, profile):
self.secret_access_key = profile["secret_access_key"]
self.access_id = profile["access_id"]
config.ACCESS_KEY = safestr(self.access_id)
config.SECRET_KEY = safestr(self.secret_access_key)
if "bucket" in profile and profile["bucket"]:
self.bucket = profile["bucket"]
else:
self.bucket = "cmdp"
self.token_expires = 3600 + now()
self.init_tokent()
def update(self): if self.token_expires <= now(): self.init_tokent()
def update(self):
if self.token_expires <= now():
self.init_tokent()