def operater(pageQueue,resultQueue):
while True:
data = pageQueue.get(block = True)
if not data:
continue
playerid = '/'.join(tool.getplayerid(data[0]))
htmltext = data[1]
# print htmltext,"at:operater"
html = formatHTML(htmltext)
hc = MyHTMLParser()
hc.feed(html)
playerinfo = hc.get_playerinfo()
career = hc.get_career()
if playerinfo:
playerinfo = tool.addkeytodict(playerinfo,'player_id',playerid)
if career:
career = tool.addkeytodict(career,'player_id',playerid)
if playerinfo:
resultQueue.put((playerinfo,career),block = True)
print "resultQueue have more record!"
print playerinfo
print career
print "I am working ,boss."
def operater(pageQueue,resultQueue):
data = pageQueue.get(block = True)
playerid = '/'.join(tool.getplayerid(data[0]))
htmltext = data[1]
html = formatHTML(html)
hc = MyHTMLParser()
hc.feed(html)
playerinfo = hc.get_playerinfo()
career = hc.get_career()
playerinfo = tool.addkeytodict(playerinfo,'player_id',playerid)
career = tool.addkeytodict(career,'player_id',playerid)
resultQueue.put((playerinfo,career),block = True)
qcur = q.cursor()
num = qcur.execute('''select id,url from t_soccerway_players''')
print num
fd = open('record.txt','r')
data = fd.readlines()
data = [record.strip() for record in data]
print len(data)
id_list = []
while True:
record = qcur.fetchone()
if record:
if tool.getplayerid(record[1]):
print tool.getplayerid(record[1])
if '/'.join(tool.getplayerid(record[1])) in data:
pass
else:
id_list.append(record[0])
print tool.getplayerid(record[1])
else:
break
fdo = open('id.txt','w')
for id in id_list:
qiud_conn = MySQLdb.connect(host,user,passwd,db)
qiud_cur = qiud_conn.cursor()
for name in filelist:
print name
fd = open(name,'r')
data = filesplit(fd,'@newpage@')
print len(data)
for page in data[1:]:
print type(page),len(page)
if not tool.isplayerpage(page[0]):
continue
else:
playerid = '/'.join(tool.getplayerid(page[0]))
html = ''.join(page[1:])
html = formatHTML(html)
hc = MyHTMLParser()
hc.feed(html)
playerinfo = hc.get_playerinfo()
career = hc.get_career()
playerinfo = tool.addkeytodict(playerinfo,'player_id',playerid)
career = tool.addkeytodict(career,'player_id',playerid)
print playerinfo
#insert record
tool.insertplayerinfo(qiud_cur,playerinfo)
