def operater(pageQueue,resultQueue): while True: data = pageQueue.get(block = True) if not data: continue playerid = '/'.join(tool.getplayerid(data[0])) htmltext = data[1] # print htmltext,"at:operater" html = formatHTML(htmltext) hc = MyHTMLParser() hc.feed(html) playerinfo = hc.get_playerinfo() career = hc.get_career() if playerinfo: playerinfo = tool.addkeytodict(playerinfo,'player_id',playerid) if career: career = tool.addkeytodict(career,'player_id',playerid) if playerinfo: resultQueue.put((playerinfo,career),block = True) print "resultQueue have more record!" print playerinfo print career print "I am working ,boss."
def operater(pageQueue,resultQueue): data = pageQueue.get(block = True) playerid = '/'.join(tool.getplayerid(data[0])) htmltext = data[1] html = formatHTML(html) hc = MyHTMLParser() hc.feed(html) playerinfo = hc.get_playerinfo() career = hc.get_career() playerinfo = tool.addkeytodict(playerinfo,'player_id',playerid) career = tool.addkeytodict(career,'player_id',playerid) resultQueue.put((playerinfo,career),block = True)
qcur = q.cursor() num = qcur.execute('''select id,url from t_soccerway_players''') print num fd = open('record.txt','r') data = fd.readlines() data = [record.strip() for record in data] print len(data) id_list = [] while True: record = qcur.fetchone() if record: if tool.getplayerid(record[1]): print tool.getplayerid(record[1]) if '/'.join(tool.getplayerid(record[1])) in data: pass else: id_list.append(record[0]) print tool.getplayerid(record[1]) else: break fdo = open('id.txt','w') for id in id_list:
qiud_conn = MySQLdb.connect(host,user,passwd,db) qiud_cur = qiud_conn.cursor() for name in filelist: print name fd = open(name,'r') data = filesplit(fd,'@newpage@') print len(data) for page in data[1:]: print type(page),len(page) if not tool.isplayerpage(page[0]): continue else: playerid = '/'.join(tool.getplayerid(page[0])) html = ''.join(page[1:]) html = formatHTML(html) hc = MyHTMLParser() hc.feed(html) playerinfo = hc.get_playerinfo() career = hc.get_career() playerinfo = tool.addkeytodict(playerinfo,'player_id',playerid) career = tool.addkeytodict(career,'player_id',playerid) print playerinfo #insert record tool.insertplayerinfo(qiud_cur,playerinfo)