for u in range(10): getPageInARowAdvanced_proc = mp.Process(target=getPageInARowAdvanced, args=( url_queue, objectiveFolder, objective, )) getPageInARowAdvanced_proc.daemon = True getPageInARowAdvanced_proc.start() print( f'建立第{u}個 getPageInARowAdvanced_proc, {getPageInARowAdvanced_proc}' ) Process_2.append(getPageInARowAdvanced_proc) # 主行程 distributeKeyword(_momoKeywordUrlPair, searchword_queue) print("=============main process distributeKeyword 已經完成任務了。=============") #通知main process 完成事情。 searchword_queue.join() url_queue.join() print('Multiprocessing has done all jobs!') for proc in Process_1: proc.terminate() print(f'{proc} has terminated!') for proc in Process_2: proc.terminate() print(f'{proc} has terminated!')
keyword_queue = mp.JoinableQueue() # 發出關鍵字,讓接收的進程清洗該關鍵字的detail資料夾,同時合併detail and overview file。 # airConditioner_queue = mp.JoinableQueue() # 啟動進程 Process_1 = [] # 將overview資料夾的html打開清洗,並發送detailUri。 for w in range(8): dataMunging_proc = mp.Process(target=dataMunging, args=(keyword_queue, dirRoute, objectiveFolderClean, objective,)) dataMunging_proc.daemon = True dataMunging_proc.start() print(f'建立第{w}個 dataMunging_proc, {os.getpid()}, {dataMunging_proc}') Process_1.append(dataMunging_proc) # 主行程 # main process <--join--> dataMunging_proc distributeKeyword(_bureauEnergyKeywordUrlPair, keyword_queue) print("=============main process distributeKeyword 已經完成任務了。=============") #通知main process 完成事情。 # 開超過兩個queue,main process程式就無法跑通喔! keyword_queue.join() print('Multiprocessing has done all jobs!') for proc in Process_1: proc.terminate() print(f'{proc} has terminated!') end = timeCalculate() TOTAL = sum(statistic)
Process_2 = [] for w in range(8): getPageInARaw_proc = mp.Process(target=getPageInARaw, args=( month_queue, _headers, objectiveFolder, objective, )) getPageInARaw_proc.daemon = True getPageInARaw_proc.start() print(f'建立第{w}個 getPageInARaw_proc, {getPageInARaw_proc}') Process_2.append(getPageInARaw_proc) #主進程 distributeKeyword(_weatherRecordAvailable, year_queue) print("=============main process distributeKeyword 已經完成任務了。=============") #通知main process 完成事情。 寫超過兩個queue,程式將無法正常終止。 year_queue.join() month_queue.join() print('Multiprocessing has done all jobs!') for proc in Process_1: proc.terminate() print(f'{proc} has terminated!') for proc in Process_2: proc.terminate() print(f'{proc} has terminated!') end = timeCalculate()
for p in range(3): getPageInARow_proc = mp.Process(target=getPageInARow, args=( keyword_queue, url, firstPage, topTabList, elementUrl, objectiveFolder, objective, )) getPageInARow_proc.daemon = True getPageInARow_proc.start() print(f'建立第{p}個 getPageInARow_proc, {getPageInARow_proc}') Process_1.append(getPageInARow_proc) #主進程 distributeKeyword(_googleSearchWord, keyword_queue) print("=============main process distributeKeyword 已經完成任務了。=============") #通知main process 完成事情。 寫超過兩個queue,程式將無法正常終止。 keyword_queue.join() print('Multiprocessing has done all jobs!') for proc in Process_1: proc.terminate() print(f'{proc} has terminated!') end = timeCalculate() print('完成!一共耗時:{0} 秒'.format(end - begin))
keyword_queue = mp.JoinableQueue() # 啟動行程 Process_1 = [] for p in range(2): #開5個進程時,沒有出錯。 getPageInARow_proc = mp.Process(target=getPageInARow, args=(keyword_queue, headers, objectiveFolder, objective,))#*args getPageInARow_proc.daemon = True getPageInARow_proc.start() print(f'建立第{p}個 getPageInARow_proc, {os.getpid()}, {getPageInARow_proc}') Process_1.append(getPageInARow_proc) # 主行程 keywordList = ["24h", "vdr"]# 24小時、購物中心、代購服務 # , "kdn" 2020/10/02發現沒用 searchwordAndKeyword = [rowOutside + "+" + rowInside for rowOutside in _pchomeKeywordUrlPair for rowInside in keywordList] distributeKeyword(searchwordAndKeyword, keyword_queue) keyword_queue.join() # for proc in process: # proc.join() print('Function getPageInARow has done all jobs!') for proc in Process_1: proc.terminate() print(f'{proc} has terminated!') end = time.time() print("完成!一共耗時:{0} 秒".format(end-begin))