def scrapy(seed):
sid = seed["sid"]
frequency = seed["frequency"]
starttime = seed["start_time"]
finishtime = seed["finish_time"]
starttime = now()
s = Seed(seed)
try:
g = Grab(s)
g.push()
finishtime = now()
next_queue.append(finishtime + frequency)
seed_db.edit(sid, **{"start_time":starttime, "finish_time":finishtime})
log_db.insert(**{"sid" : sid, "start_time" : starttime, "finish_time" : finishtime, "`status`" : 1, "message" : "采集成功"})
return True
except Exception, e:
log_db.insert(**{"sid" : sid, "start_time" : starttime, "finish_time" : now(), "`status`" : 0, "message" : "采集失败, 原因:" + str(e)})
return False
def list(seed_id):
items = {}
num = 0
seed_type = None
if seed_id:
db = Seed_Model();
r = db.view(seed_id);
seed = Seed(r.list()[0])
t = Grab(seed)
if len(t):
num = len(t.keys())
seed_type = t.seed_type
for guid in t.keys():
tmp = {}
dont_craw_content = [
'kaifu', 'kaice', "gift"
]
if seed_type in dont_craw_content:
for f in t.items[guid].fields:
tmp[f] = t.items[guid][f].value
else:
tmp["url"] = t.items[guid]["url"]
items[guid] = tmp
return render_template("test_seed/list.html", items=items, num=num, seed_id=seed_id, seed_type=seed_type, dont_craw_content=dont_craw_content)
