def mainspider():
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
engine = create_engine(
MYSQL_URL,
encoding="utf-8"
)
Session = sessionmaker(engine)
session = Session()
lasturl = wooyun.getlastbug()
wooyun.savelastbug(lastbug())
pagenum = wooyun.getpagenum()
for pages in xrange(120, pagenum):
buglist = wooyun.getwooyun(pages)
print "page:" + str(pages)
for bug in buglist:
bugurl = bug[2]
if bugurl == lasturl:
print "You have Update ALL the News"
return 0
url = 'http://www.wooyun.org' + str(bug[2])
content = getcontent.getContent(url)
ucontent = getcontent.getimage(content, bug[0])
entitywoo.pushContent(session, bug[2], ucontent)
if len(bug):
entitywoo.pushbuginfo(
session, bug[0],
bug[1], bug[2], bug[3]
)
print bug[2]
print "You have Update ALL the bugs"
return 0
def lastbug():
PAGE_NUM_1 = 1
buglist = wooyun.getwooyun(PAGE_NUM_1)
url = buglist[0][2]
return url
