def scrapy(seed): sid = seed["sid"] frequency = seed["frequency"] starttime = seed["start_time"] finishtime = seed["finish_time"] starttime = now() s = Seed(seed) try: g = Grab(s) g.push() finishtime = now() next_queue.append(finishtime + frequency) seed_db.edit(sid, **{"start_time":starttime, "finish_time":finishtime}) log_db.insert(**{"sid" : sid, "start_time" : starttime, "finish_time" : finishtime, "`status`" : 1, "message" : "采集成功"}) return True except Exception, e: log_db.insert(**{"sid" : sid, "start_time" : starttime, "finish_time" : now(), "`status`" : 0, "message" : "采集失败, 原因:" + str(e)}) return False
def list(seed_id): items = {} num = 0 seed_type = None if seed_id: db = Seed_Model(); r = db.view(seed_id); seed = Seed(r.list()[0]) t = Grab(seed) if len(t): num = len(t.keys()) seed_type = t.seed_type for guid in t.keys(): tmp = {} dont_craw_content = [ 'kaifu', 'kaice', "gift" ] if seed_type in dont_craw_content: for f in t.items[guid].fields: tmp[f] = t.items[guid][f].value else: tmp["url"] = t.items[guid]["url"] items[guid] = tmp return render_template("test_seed/list.html", items=items, num=num, seed_id=seed_id, seed_type=seed_type, dont_craw_content=dont_craw_content)