def mainspider(): from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker engine = create_engine( MYSQL_URL, encoding="utf-8" ) Session = sessionmaker(engine) session = Session() lasturl = wooyun.getlastbug() wooyun.savelastbug(lastbug()) pagenum = wooyun.getpagenum() for pages in xrange(120, pagenum): buglist = wooyun.getwooyun(pages) print "page:" + str(pages) for bug in buglist: bugurl = bug[2] if bugurl == lasturl: print "You have Update ALL the News" return 0 url = 'http://www.wooyun.org' + str(bug[2]) content = getcontent.getContent(url) ucontent = getcontent.getimage(content, bug[0]) entitywoo.pushContent(session, bug[2], ucontent) if len(bug): entitywoo.pushbuginfo( session, bug[0], bug[1], bug[2], bug[3] ) print bug[2] print "You have Update ALL the bugs" return 0
def lastbug(): PAGE_NUM_1 = 1 buglist = wooyun.getwooyun(PAGE_NUM_1) url = buglist[0][2] return url