def urlCrawler(urlQueue, logQueue):
rooturl = {
'lawfirm': 'http://www.legalminer.com/search/lawfirm?t=',
'lawyer': 'http://www.legalminer.com/search/lawyer?t=',
'court': 'http://www.legalminer.com/search/court?t=',
'judge': 'http://www.legalminer.com/search/judge?t=',
'corporate':'http://www.legalminer.com/search/corporate?t='
}
rootNum = {}
for key, url in rooturl.items():
content = downLoad(url)
tree = etree.HTML(content)
resultNumNode = tree.xpath(urlPattern.resultNumXpath)
rootNum[key] = urlPattern.extractNum(resultNumNode[0].text)
#每个链接创建一个生产线程提取url
threadQueue = TQueue()
ThreadLst = []
for key, value in rootNum.items():
extractor = Extractor()
extractor.setResultNum(key, int(value), threadQueue)
ThreadLst.append(extractor)
transmit = Transmit()
transmit.setEndNum(len(rootNum), threadQueue, urlQueue, CONTENT_CRAWLER_NUM)
ThreadLst.append(transmit)
for e in ThreadLst:
e.start()
for e in ThreadLst:
e.join()
def processCourt(urlC, queue, conn):
#print 'processCourt'
content = downLoad(urlC)
xm_path = "/html/body[@class='lvzhi court']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw heading']/div[@class='title']"
ssaj_path = "/html/body[@class='lvzhi court']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='info-cell pull-right']/div[@class='cell']/a[@class='rw val textLink']"
tree=etree.HTML(content)
node_xm = tree.xpath(xm_path)
node_ssaj = tree.xpath(ssaj_path)
reLst = []
reLst.append(urlC)
court_name = u'未知'
case_number = 0
if node_xm[0].text != None:
court_name = node_xm[0].text.strip()
if node_ssaj[0].text != None:
case_number = extractNum(node_ssaj[0].text)
reLst.append(court_name)
reLst.append(case_number)
try:
cur=conn.cursor()
cur.execute('insert into court values(%s, %s, %s)', reLst)
conn.commit()
cur.close()
except MySQLdb.Error,e:
queue.put(urlC)
def processLawFirm(urlC, queue, conn):
#print 'processLawFirm'
content = downLoad(urlC)
mc_path="/html/body[@class='lvzhi lawfirm']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw heading']/div[@class='title']"
zh_path="/html/body[@class='lvzhi lawfirm']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='list']/div[@class='rw val'][1]"
dz_path="/html/body[@class='lvzhi lawfirm']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='list']/div[@class='rw val'][2]"
dh_path="/html/body[@class='lvzhi lawfirm']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='list']/div[@class='rw val'][3]"
cz_path="/html/body[@class='lvzhi lawfirm']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='list']/div[@class='rw val'][4]"
gw_path="/html/body[@class='lvzhi lawfirm']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='list']/div[@class='rw val'][5]/a"
ssaj_path="/html/body[@class='lvzhi lawfirm']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='info-cell pull-right']/div[@class='cell'][1]/a[@class='rw val textLink']"
slsj_path="/html/body[@class='lvzhi lawfirm']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='info-cell pull-right']/div[@class='cell'][2]/div[@class='rw val']"
zyzt_path="/html/body[@class='lvzhi lawfirm']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='info-cell pull-right']/div[@class='cell'][3]/div[@class='rw val']"
fzr_path="/html/body[@class='lvzhi lawfirm']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='info-cell pull-right']/div[@class='cell'][4]/div[@class='rw val']"
tree=etree.HTML(content)
pattern_zh = re.compile(r'<div class="rw val"><i class="fa fa-info-circle icon highlight"></i>(.*?)</div>',re.M)
pattern_dz = re.compile(r'<div class="rw val"><i class="fa fa-location-arrow icon highlight"></i>(.*?)</div>',re.M)
pattern_dh = re.compile(r'<div class="rw val"><i class="fa fa-phone icon highlight"></i>(.*?)</div>',re.M)
pattern_cz = re.compile(r'<div class="rw val"><i class="fa fa-print icon highlight"></i>(.*?)</div>',re.M)
pattern_gw = re.compile(r'<div class="rw val"><a href="(.*?)" target="_blank"><i class="fa fa-external-link icon highlight"></i>官网链接</a></div>',re.M)
pattern_lssl = re.compile(r'>([0-9]*?)位律师</a>',re.M)
node_mc = tree.xpath(mc_path)
node_ssaj = tree.xpath(ssaj_path)
node_slsj = tree.xpath(slsj_path)
node_zyzt = tree.xpath(zyzt_path)
node_fzr = tree.xpath(fzr_path)
reLst = []
match1 = pattern_zh.search(content)
match2 = pattern_dz.search(content)
match3 = pattern_dh.search(content)
match4 = pattern_cz.search(content)
match5 = pattern_gw.search(content)
match6 = pattern_lssl.search(content)
reLst.append(urlC)
law_firm_name = u'未知';
law_firm_lawyer_number = 0;
occupation_number = u'未知';
address = u'未知';
phone_number = u'未知';
fax_number = u'未知';
link = u'未知'
case_number = 0
history = 0
state = u'未知'
principle = u'未知'
if node_mc[0].text != None:
law_firm_name = node_mc[0].text.strip().replace('\n','').replace('\t','').replace('\r','').split(' ')[0].strip()
if match6 != None:
law_firm_lawyer_number = int(match6.group(1))
if match1 != None:
occupation_number = match1.group(1)
occupation_number = occupation_number if occupation_number.find(' ') == -1 else occupation_number.split(' ')[1]
if match2 != None:
address = match2.group(1)
if match3 != None:
phone_number = match3.group(1)
if match4 != None:
fax_number = match4.group(1)
if match5 != None:
link = match5.group(1)
if node_ssaj[0].text!=None:
case_number = extractNum(node_ssaj[0].text.strip())
if node_slsj[0].text!=None:
history = extractNum(node_slsj[0].text.strip())
if node_zyzt[0].text!=None:
state = node_slsj[0].text.strip()
if node_fzr[0].text!=None:
principle = node_fzr[0].text.strip()
reLst.append(law_firm_name)
reLst.append(law_firm_lawyer_number)
reLst.append(occupation_number)
reLst.append(address)
reLst.append(phone_number)
reLst.append(fax_number)
reLst.append(link)
reLst.append(case_number)
reLst.append(history)
reLst.append(state)
reLst.append(principle)
try:
cur=conn.cursor()
cur.execute('insert into lawfirm values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)', reLst)
conn.commit()
cur.close()
except MySQLdb.Error,e:
queue.put(urlC)
def processLawyer(urlC, queue, conn):
#print 'processLawyer'
content = downLoad(urlC)
xm_path="/html/body[@class='lvzhi lawyer']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw heading']/div[@class='title']"
zyzh_path="/html/body[@class='lvzhi lawyer']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='list']/div[@class='rw val highlight']"
lsmc_path="/html/body[@class='lvzhi lawyer']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='list']/div[@class='rw val'][1]"
ssaj_path="/html/body[@class='lvzhi lawyer']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='info-cell pull-right']/div[@class='cell'][1]/a[@class='rw val textLink']"
jy_path="/html/body[@class='lvzhi lawyer']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='info-cell pull-right']/div[@class='cell'][2]/div[@class='rw val']"
xl_path="/html/body[@class='lvzhi lawyer']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='info-cell pull-right']/div[@class='cell'][3]/div[@class='rw val']"
xb_path="/html/body[@class='lvzhi lawyer']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='info-cell pull-right']/div[@class='cell'][4]/div[@class='rw val']"
zyzt_path="/html/body[@class='lvzhi lawyer']/div[@id='bodyWrapper']/div[@id='content']/div[@id='basicInfo']/div[@class='content row']/div[@class='info col-sm-10']/div[@class='rw']/div[@class='info-cell pull-right']/div[@class='cell'][5]/div[@class='rw val']"
tree=etree.HTML(content)
node_xm = tree.xpath(xm_path)
node_zyzh = tree.xpath(zyzh_path)
node_lsmc = tree.xpath(lsmc_path)
node_ssaj = tree.xpath(ssaj_path)
node_jy = tree.xpath(jy_path)
node_xl = tree.xpath(xl_path)
node_xb = tree.xpath(xb_path)
node_zyzt = tree.xpath(zyzt_path)
reLst = []
reLst.append(urlC)
lawyer_name = u'未知'
occupation_number = u'未知'
law_firm_name = u'未知'
case_number = 0
history = 0
education = u'未知'
gender = u'未知'
state = u'未知'
if node_xm[0].text != None:
lawyer_name = node_xm[0].text.split(' ')[0]
if node_zyzh[0].text != None:
occupation_number = node_zyzh[0].text.split(' ')[1]
if node_lsmc[0].text != None:
law_firm_name = node_lsmc[0].text
if node_ssaj[0].text != None:
case_number = extractNum(node_ssaj[0].text)
if node_jy[0].text != None:
history = extractNum(node_jy[0].text)
if node_xl[0].text != None:
education = node_xl[0].text
if node_xb[0].text != None:
gender = node_xb[0].text
if node_zyzt[0].text != None:
state = node_zyzt[0].text
reLst.append(lawyer_name)
reLst.append(occupation_number)
reLst.append(law_firm_name)
reLst.append(case_number)
reLst.append(history)
reLst.append(education)
reLst.append(gender)
reLst.append(state)
try:
cur=conn.cursor()
cur.execute('insert into lawyer values(%s, %s, %s, %s, %s, %s, %s, %s, %s)', reLst)
conn.commit()
cur.close()
except MySQLdb.Error,e:
queue.put(urlC)
