def operater(pageQueue,resultQueue):
while True:
data = pageQueue.get(block = True)
if not data:
continue
playerid = '/'.join(tool.getplayerid(data[0]))
htmltext = data[1]
# print htmltext,"at:operater"
html = formatHTML(htmltext)
hc = MyHTMLParser()
hc.feed(html)
playerinfo = hc.get_playerinfo()
career = hc.get_career()
if playerinfo:
playerinfo = tool.addkeytodict(playerinfo,'player_id',playerid)
if career:
career = tool.addkeytodict(career,'player_id',playerid)
if playerinfo:
resultQueue.put((playerinfo,career),block = True)
print "resultQueue have more record!"
print playerinfo
print career
print "I am working ,boss."
def operater(pageQueue,resultQueue):
data = pageQueue.get(block = True)
playerid = '/'.join(tool.getplayerid(data[0]))
htmltext = data[1]
html = formatHTML(html)
hc = MyHTMLParser()
hc.feed(html)
playerinfo = hc.get_playerinfo()
career = hc.get_career()
playerinfo = tool.addkeytodict(playerinfo,'player_id',playerid)
career = tool.addkeytodict(career,'player_id',playerid)
resultQueue.put((playerinfo,career),block = True)
for page in data[1:]:
print type(page),len(page)
if not tool.isplayerpage(page[0]):
continue
else:
playerid = '/'.join(tool.getplayerid(page[0]))
html = ''.join(page[1:])
html = formatHTML(html)
hc = MyHTMLParser()
hc.feed(html)
playerinfo = hc.get_playerinfo()
career = hc.get_career()
playerinfo = tool.addkeytodict(playerinfo,'player_id',playerid)
career = tool.addkeytodict(career,'player_id',playerid)
print playerinfo
#insert record
tool.insertplayerinfo(qiud_cur,playerinfo)
for car_record in career:
tool.insertcareer(qiud_cur,car_record)
qiud_conn.commit()