def findPage(): http,ua = getHttpUa() conn,cur = getCursor() dlList = getResult(sltCollNotNull,cur) for dl in dlList: # this is test!!!! read from a txt #html = readTXT('E:/Code/Test Data/Paul Robert Barford - ACM author profile page - colleagues.txt') #html = readTXT('E:/Code/Test Data/Yu Zheng - ACM author profile page.txt') #html = readTXT('E:/Code/Test Data/A. Smolic - ACM author profile page.txt') if ChangeOrNot() == True: editeProxies(http) editeHeader(ua) time.sleep(random.randint(1, 12)) html = str(getPage(dl['colleage']))#取出url if html != ' ': nameLink = analysisPage(html) for nl in nameLink: addInfo(conn,cur,nl) #print(nl) print('Now is '+str(dl['id'])) #break#only run one time cur.close() conn.close()
def mainFunction(): # #0.数据准备 http,uag = getHttpUa() for ip in http: ipQueue.put(ip) for ua in uag: uaQueue.put(ua) for k in range(1): aWorker = analysisWorker(ipQueue,uaQueue,htmlQueue) aWorker.daemon = True aWorker.start() for i in range(1): pWorker = pageWorker(ipQueue,uaQueue,dlQueue,htmlQueue) pWorker.daemon = True pWorker.start() #1.get that list of error csv dlList=[] with open(expertList_path) as csvfile: reader = csv.DictReader(csvfile) for row in reader: dlList.append(row) #dl 是字典型的 for dl in dlList: dlQueue.put(dl)
def mainFunction(): http, uag = getHttpUa() for ip in http: ipQueue.put(ip) for ua in uag: uaQueue.put(ua) for k in range(1): aWorker = analysisWorker(htmlQueue, infoQueue) aWorker.daemon = True aWorker.start() print('ok1') for i in range(4): pWorker = pageWorker(ipQueue, uaQueue, dlQueue, htmlQueue) pWorker.daemon = True pWorker.start() print('ok2') conn, cur = getCursor() dlList = getResult(sltDLNotCom, cur) #返回url实体的二维数组 for dl in dlList: dlQueue.put(dl) cur.close() conn.close() print('ok3') for j in range(1): mWorker = mysqlWorker(infoQueue) mWorker.daemon = True mWorker.start()
def mainFunction(): http,uag = getHttpUa() for ip in http: ipQueue.put(ip) for ua in uag: uaQueue.put(ua) for k in range(1): aWorker = analysisWorker(htmlQueue) aWorker.daemon = True aWorker.start() print('ok1') for i in range(7): pWorker = pageWorker(ipQueue,uaQueue,dlQueue,htmlQueue) pWorker.daemon = True pWorker.start() print('ok2') #dlList = getResult(sltDLNotCom,cur)#返回url实体的二维数组 dlList=[] with open(expertList_path,newline = '',encoding= 'utf-8') as csvfile: reader = csv.reader(csvfile) for row in reader: if len(row[2])>5: dlList.append({'id':row[0],'name':row[1],'url':row[2]}) #break print('total:'+str(len(dlList))) for dl in dlList: dlQueue.put(dl)
def mainFunction(): http, uag = getHttpUa() for ua in uag: uaQueue.put(ua) for ip in http: ipQueue.put(ip) for i in range(5): pWorker = pageWorker(uaQueue, ipQueue, dlQueue) pWorker.daemon = True pWorker.start() conn, cur = getCursor() dlList = getResult(selectSQL, cur) ''' with open(expertList_path) as csvfile: reader = csv.DictReader(csvfile) for row in reader: dlList.append(row) ''' for dl in dlList: dlQueue.put(dl)
def mainFunction(): http,uag = getHttpUa() for ip in http: ipQueue.put(ip) for ua in uag: uaQueue.put(ua) for k in range(2): aWorker = analysisWorker(htmlQueue,infoQueue) aWorker.daemon = True aWorker.start() print('ok1') for i in range(8): pWorker = pageWorker(ipQueue,uaQueue,dlQueue,htmlQueue) pWorker.daemon = True pWorker.start() print('ok2') ''' conn,cur = getCursor() dlList = getResult(sltDLNotCom,cur)#返回url实体的二维数组 ''' dlList=[] with open(expertList_path) as csvfile: reader = csv.DictReader(csvfile) for row in reader: if int(row['id']) > 0: dlList.append(row) #break print('total:'+str(len(dlList))) for dl in dlList: dlQueue.put(dl)
def findCSV(): httpProxies = commHttpProxies headers = commHeaders http,ua = getHttpUa()#获取伪装的备选http,userAgent列表 conn,cur = getCursor()#获取数据库连接和游标 dlList = getResult(sltDLNotCom,cur)#返回url实体的二维数组 for dl in dlList: if ChangeOrNot() == True:#随机触发 httpProxies=editeProxies(http,httpProxies)#改变http headers=editeHeader(ua,headers)#改变user agent time.sleep(random.randint(1, 12))#随机休眠 url = dl['papercsv'] if url != None and len(url)> 15: try: r = requests.get(url, proxies = httpProxies, headers = headers, timeout=30) if r.status_code == 200: csv_path = file_path+str(dl['id'])+'.csv' with open(csv_path,'wb') as csv: csv.write(r.content) print('Now is '+str(dl['id'])) except requests.RequestException as e: print(e) httpProxies=editeProxies(http,httpProxies)#改变http headers=editeHeader(ua,headers)#改变user agent time.sleep(random.randint(1, 12))#随机休眠 try: r = requests.get(url, proxies = httpProxies, headers = headers, timeout=30) if r.status_code == 200: csv_path = file_path+str(dl['id'])+'.csv' with open(csv_path,'wb') as csv: csv.write(r.content) print('Now is '+str(dl['id'])) except Exception: print('another try is failed! id:'+str(dl['id'])) #break# only run one time cur.close() conn.close()
def findPage(): httpProxies = commHttpProxies.copy() headers = commHeaders.copy() cookies = commCookies.copy() #从数据库中获取预访问的url列表,循环访问url地址 http, ua = getHttpUa() #获取伪装的备选http,userAgent列表 conn, cur = getCursor() #获取数据库连接和游标 dlList = getResult(sltDLNotCom, cur) #返回url实体的二维数组 #for i in range(15): #i = 0 for dl in dlList: # this is test!!!! read from a txt #html = readTXT('E:/Code/Test Data/Hsinchun Chen.txt') #html = readTXT('E:/Code/Test Data/Yu Zheng - ACM author profile page.txt') #html = readTXT('E:/Code/Test Data/A. Smolic - ACM author profile page.txt') if ChangeOrNot() == True: #随机触发 httpProxies = editeProxies(http, httpProxies) #改变http headers = editeHeader(ua, headers, dl['name']) #改变user agent cookies = editeCookies(cookies) time.sleep(random.randint(2, 12)) #随机休眠 print(str(httpProxies['https'])) print(str(headers['User-Agent'])) print(str(headers['Referer'])) print(str(cookies['CFID'])) print() ''' html = str(getPage(dl['url'],httpProxies,headers,cookies))#取出url if html != ' ': infoSet = analysisPage(html,int(dl['id']))#分析页面 addInfo(conn,cur,infoSet,dl)#存入数据库 cur.execute('update dlurl1 set status=1 where id='+str(dl['id']))#标记已抽取 conn.commit() print('Now is '+str(dl['id'])) #break#only run one time ''' cur.close() conn.close()
conn, cur = getCursor() dlList = getResult(sltDLNotCom, cur) #返回url实体的二维数组 for dl in dlList: dlQueue.put(dl) cur.close() conn.close() print('ok3') for j in range(1): mWorker = mysqlWorker(infoQueue) mWorker.daemon = True mWorker.start() if __name__ == '__main__': http, uag = getHttpUa() for ip in http: ipQueue.put(ip) for ua in uag: uaQueue.put(ua) for k in range(1): aWorker = analysisWorker(htmlQueue, infoQueue) aWorker.daemon = True aWorker.start() print('ok1') for i in range(4): pWorker = pageWorker(ipQueue, uaQueue, dlQueue, htmlQueue) pWorker.daemon = True pWorker.start() print('ok2')
def findSubject(): http,ua = getHttpUa()