for u in range(10):
getPageInARowAdvanced_proc = mp.Process(target=getPageInARowAdvanced,
args=(
url_queue,
objectiveFolder,
objective,
))
getPageInARowAdvanced_proc.daemon = True
getPageInARowAdvanced_proc.start()
print(
f'建立第{u}個 getPageInARowAdvanced_proc, {getPageInARowAdvanced_proc}'
)
Process_2.append(getPageInARowAdvanced_proc)
# 主行程
distributeKeyword(_momoKeywordUrlPair, searchword_queue)
print("=============main process distributeKeyword 已經完成任務了。=============")
#通知main process 完成事情。
searchword_queue.join()
url_queue.join()
print('Multiprocessing has done all jobs!')
for proc in Process_1:
proc.terminate()
print(f'{proc} has terminated!')
for proc in Process_2:
proc.terminate()
print(f'{proc} has terminated!')
keyword_queue = mp.JoinableQueue() # 發出關鍵字，讓接收的進程清洗該關鍵字的detail資料夾，同時合併detail and overview file。
# airConditioner_queue = mp.JoinableQueue()
# 啟動進程
Process_1 = [] # 將overview資料夾的html打開清洗，並發送detailUri。
for w in range(8):
dataMunging_proc = mp.Process(target=dataMunging, args=(keyword_queue, dirRoute, objectiveFolderClean, objective,))
dataMunging_proc.daemon = True
dataMunging_proc.start()
print(f'建立第{w}個 dataMunging_proc, {os.getpid()}, {dataMunging_proc}')
Process_1.append(dataMunging_proc)
# 主行程
# main process <--join--> dataMunging_proc
distributeKeyword(_bureauEnergyKeywordUrlPair, keyword_queue)
print("=============main process distributeKeyword 已經完成任務了。=============")
#通知main process 完成事情。 # 開超過兩個queue，main process程式就無法跑通喔！
keyword_queue.join()
print('Multiprocessing has done all jobs!')
for proc in Process_1:
proc.terminate()
print(f'{proc} has terminated!')
end = timeCalculate()
TOTAL = sum(statistic)
Process_2 = []
for w in range(8):
getPageInARaw_proc = mp.Process(target=getPageInARaw,
args=(
month_queue,
_headers,
objectiveFolder,
objective,
))
getPageInARaw_proc.daemon = True
getPageInARaw_proc.start()
print(f'建立第{w}個 getPageInARaw_proc, {getPageInARaw_proc}')
Process_2.append(getPageInARaw_proc)
#主進程
distributeKeyword(_weatherRecordAvailable, year_queue)
print("=============main process distributeKeyword 已經完成任務了。=============")
#通知main process 完成事情。 寫超過兩個queue，程式將無法正常終止。
year_queue.join()
month_queue.join()
print('Multiprocessing has done all jobs!')
for proc in Process_1:
proc.terminate()
print(f'{proc} has terminated!')
for proc in Process_2:
proc.terminate()
print(f'{proc} has terminated!')
end = timeCalculate()
for p in range(3):
getPageInARow_proc = mp.Process(target=getPageInARow,
args=(
keyword_queue,
url,
firstPage,
topTabList,
elementUrl,
objectiveFolder,
objective,
))
getPageInARow_proc.daemon = True
getPageInARow_proc.start()
print(f'建立第{p}個 getPageInARow_proc, {getPageInARow_proc}')
Process_1.append(getPageInARow_proc)
#主進程
distributeKeyword(_googleSearchWord, keyword_queue)
print("=============main process distributeKeyword 已經完成任務了。=============")
#通知main process 完成事情。 寫超過兩個queue，程式將無法正常終止。
keyword_queue.join()
print('Multiprocessing has done all jobs!')
for proc in Process_1:
proc.terminate()
print(f'{proc} has terminated!')
end = timeCalculate()
print('完成！一共耗時：{0} 秒'.format(end - begin))
keyword_queue = mp.JoinableQueue()
# 啟動行程
Process_1 = []
for p in range(2): #開5個進程時，沒有出錯。
getPageInARow_proc = mp.Process(target=getPageInARow, args=(keyword_queue, headers, objectiveFolder, objective,))#*args
getPageInARow_proc.daemon = True
getPageInARow_proc.start()
print(f'建立第{p}個 getPageInARow_proc, {os.getpid()}, {getPageInARow_proc}')
Process_1.append(getPageInARow_proc)
# 主行程
keywordList = ["24h", "vdr"]# 24小時、購物中心、代購服務 # , "kdn" 2020/10/02發現沒用
searchwordAndKeyword = [rowOutside + "+" + rowInside for rowOutside in _pchomeKeywordUrlPair for rowInside in keywordList]
distributeKeyword(searchwordAndKeyword, keyword_queue)
keyword_queue.join()
# for proc in process:
# proc.join()
print('Function getPageInARow has done all jobs!')
for proc in Process_1:
proc.terminate()
print(f'{proc} has terminated!')
end = time.time()
print("完成！一共耗時：{0} 秒".format(end-begin))
