def mungingPchome(_BASE_PATH, searchword, objectiveFolder,
objectiveFolderClean, objective):
dirRoute = f"{_BASE_PATH}/dataMunging/{objectiveFolder}/{objective}/{searchword}/"
dir1 = dirRoute + "24h/"
dir2 = dirRoute + "vdr/"
dir3 = dirRoute + "kdn/"
pchomeDict = {}
productArray = []
# pchomeDict["picurl"] = "https://d.ecimg.tw"
# pchomeDict["produrl"] = "https://24h.pchome.com.tw/prod/"
pchomeDict["keyword"] = f"{searchword}(24h_vdr_kdn)"
pchomeDict["dateTime"] = timeStamp
for directory in [dir1, dir2, dir3]:
if initialFileZeroUnderscoreInt(directory): #有些資料夾下面沒有檔案
for file in initialFileZeroUnderscoreInt(directory):
with open(directory + file) as f:
inn = json.load(f)
# 處理soup=""的情況
if not inn:
continue
for fileinner in inn['prods']:
productDict = {}
productDict['Id'] = fileinner['Id']
productDict['name'] = fileinner['name']
productDict['originprice'] = fileinner['originPrice']
productDict[
'pics'] = 'https://d.ecimg.tw' + fileinner['picS']
productDict[
'picb'] = 'https://d.ecimg.tw' + fileinner['picB']
productDict[
"produrl"] = "https://24h.pchome.com.tw/prod/" + fileinner[
"Id"]
productArray.append(productDict)
# 每一個搜索字下的3個資料夾中，每個json檔案的 'prods' 陣列資料都append後，再統一指定。
pchomeDict['product'] = productArray
source = '_'.join(
[dirname.split('/')[-2] for dirname in [dir1, dir2, dir3]])
print(f"===========進行 {searchword}(24h_vdr_kdn) 去重=============")
pchomeDict['product'], setNums = EcommerceDataProcessToSet(
pchomeDict['product'])
mkdirForCleanData(objectiveFolderClean, objective)
with open(
f"{_BASE_PATH}/dataMunging/{objectiveFolderClean}/{objective}/pchome_{source}_{timeStamp}_{setNums}_{searchword}.json",
'w') as f:
json.dump(pchomeDict, f, indent=2, ensure_ascii=False)
print(f"===========完成 {searchword}(24h_vdr_kdn) 清洗！=============")
def dataMunging(input, dirRoute, objectiveFolderClean, objective, domainUrl):
"""
"Id": "6631009",
"name": "",
"originprice": "NaN",
"pics": "https://img1.momoshop.com.tw/goodsimg/0006/631/009/6631009_L.jpg?t=000",
"picb": "None",
"produrl": "https://www.momoshop.com.tw/goods/GoodsDetail.jsp?i_code=6631009&Area=search&mdiv=403&oid=55_8&cid=index&kw=%E5%86%B7%E6%9A%96%E7%A9%BA%E8%AA%BF"
},
有的商品有連結，但是價格與品名不全，要處理。
{
"Id": "6574471",
"name": "【MITSUBISHI 三菱】16公升一級能效強力型除濕機(MJ-E160HN)",
"originprice": "NaN",
"pics": "https://img1.momoshop.com.tw/goodsimg/0006/574/471/6574471_L.jpg?t=000",
"picb": "None",
"produrl": "https://www.momoshop.com.tw/goods/GoodsDetail.jsp?i_code=6574471&Area=search&mdiv=403&oid=3_22&cid=index&kw=%E9%99%A4%E6%BF%95%E6%A9%9F"
}
"""
thisPID = os.getpid()
while True:
print(thisPID,"===========================================")
searchword = input.get()
mkdirForCleanData(objectiveFolderClean, objective)
# '/home/bluevc/2019/iSelect3C/dataMunging/rawData/momo/冷暖空調電熱水瓶' <---關鍵字累加的問題
# dirRoute = dirRoute + searchword
fileRoute = dirRoute + searchword
if not os.listdir(fileRoute):
print(f"============={objective} {searchword} 資料夾沒有東西，此進程準備結束。=============")
input.task_done()
timeSleepOne()
print("========================break========================")
break
momoDict = {}
productArray= []
for file in initialFileZeroUnderscoreInt(fileRoute):
# print("start " + file + " ! ")
with open(fileRoute + "/" + file)as f:
inn = f.read()
# 處理soup=""的情況
if not inn:
continue
textSoup = BeautifulSoup(inn,'html.parser')
try:
#一頁至多有30項
products = textSoup.select_one('.listArea').select_one('ul').select('li')
for item in products:
innerDict = {}
innerDict['Id'] = item.attrs.get('gcode')
productName = item.select_one('.goodsUrl').select_one('.prdName').text
originprice = item.select_one('.goodsUrl').select_one('.money .price').text.replace('$','').replace(',','')
if productName:
innerDict['name'] = productName
if originprice in ("NaN", "熱銷一空"):
innerDict['originprice'] = "0" #"NaN"
else:
innerDict['originprice'] = originprice
else:
innerDict['name'] = "品名抓不到"
innerDict['originprice'] = "0" #"NaN"
innerDict['pics'] = item.select_one('.goodsUrl img').attrs.get('src')
innerDict['picb'] = "None"
innerDict['produrl'] = domainUrl + item.select_one('.goodsUrl').attrs.get('href')
productArray.append(innerDict)
except Exception as e:
print(f"{file} 有 {e} 的問題。")
dateTime = datetime.datetime.now()
fmt = "%Y-%m-%d-%H-%M" #"%Y年%m月%d日%H時%M分"
timeStamp = dateTime.strftime(fmt)
momoDict['product'] = productArray
momoDict['keyword'] = searchword
momoDict["dateTime"] = timeStamp
print("===========進行去重=============")
momoDict['product'], setNums = EcommerceDataProcessToSet(momoDict['product'])
with open(f"{_BASE_PATH}/dataMunging/{objectiveFolderClean}/{objective}/momo_{timeStamp}_{setNums}_{searchword}.json", 'w')as f:
json.dump(momoDict, f, indent=2, ensure_ascii=False)
print("===========清洗完成=============")
print(f"這裡是dataMunging_{thisPID}，準備完成工作。 ")
print()
end = time.time()
print('dataMunging 累計耗時：{0} 秒'.format(end-begin))
input.task_done() #通知main process此次的input處理完成！
timeSleepOne() #暫停幾秒來模擬現實狀況。
def dataMunging(input, dirRoute, objectiveFolderClean, objective):
# begin = timeCalculate()
thisPID = os.getpid()
bureauMunging = bureauEnergyMunging()
while True:
print(thisPID,"===========================================")
searchword = input.get()
dirNameCheck = dirRoute + f"{searchword}/"
directory = dirRoute + f"{searchword}/detail/"
dirNameWriteOut = dirRoute + f"{searchword}/jsonIntegration/"
print('dataMunging is in new process %s, %s ' % (dataMunging_proc, thisPID))
print()
print('------接下來要處理資料夾路徑「 ' + dirNameWriteOut + '」---------')
print()
mkdirForCleanData(objectiveFolderClean, objective)
if not os.listdir(dirNameCheck):
print(f"============={objective} {searchword} 資料夾沒有東西，此進程準備結束。=============")
input.task_done()
timeSleepOne()
print("========================break========================")
break
# 此區已經採用簡化的寫法，因此若洗資料都無問題，那麼就可以刪除了。
# if searchword == "除濕機":
# bureauEnergyDetail, totalNums = bureauMunging.detailDehumidification(searchword, directory)
# elif searchword == "無風管空氣調節機":
# bureauEnergyDetail, totalNums = bureauMunging.detailAirConditioner(searchword, directory)
# elif searchword == "電冰箱":
# bureauEnergyDetail, totalNums = bureauMunging.detailRefrigerator(searchword, directory)
# elif searchword == "電熱水瓶":
# bureauEnergyDetail, totalNums = bureauMunging.detailElectricWarmer(searchword, directory)
# elif searchword == "溫熱型開飲機":
# bureauEnergyDetail, totalNums = bureauMunging.detailWarmDrinkMachine(searchword, directory)
# elif searchword == "溫熱型飲水機":
# bureauEnergyDetail, totalNums = bureauMunging.detailWarmDispenser(searchword, directory)
# elif searchword == "冰溫熱型開飲機":
# bureauEnergyDetail, totalNums = bureauMunging.detailColdWarmDrinkMachine(searchword, directory)
# elif searchword == "冰溫熱型飲水機":
# bureauEnergyDetail, totalNums = bureauMunging.detailColdWarmDispenser(searchword, directory)
# elif searchword == "貯備型電熱水器":
# bureauEnergyDetail, totalNums = bureauMunging.detailStorageWaterHeaters(searchword, directory)
# elif searchword == "瓦斯熱水器":
# bureauEnergyDetail, totalNums = bureauMunging.detailGasWaterHeaters(searchword, directory)
# elif searchword == "瓦斯爐":
# bureauEnergyDetail, totalNums = bureauMunging.detailGasStove(searchword, directory)
# elif searchword == "安定器內藏式螢光燈泡":
# bureauEnergyDetail, totalNums = bureauMunging.detailCompactFluorescentLamp(searchword, directory)
# '無風管空氣調節機', '除濕機', '電冰箱', '電熱水瓶', '溫熱型開飲機',
# '溫熱型飲水機', '冰溫熱型開飲機', '冰溫熱型飲水機', '貯備型電熱水器' , '瓦斯熱水器', '瓦斯爐', '安定器內藏式螢光燈泡'
bureauEnergyDetail, totalNums = bureauMunging.detailMungingEntry(searchword, directory)
with open(dirNameWriteOut + f"{objective}_detail_{timeStampGenerator()}_{totalNums}_{searchword}.json",'w',encoding='utf-8')as f:
json.dump(bureauEnergyDetail, f, indent=2, ensure_ascii=False)
# 找出 overviewJsonFile ，開始與detailJsonFile合併：
overviewJsonFile = [overviewFile for overviewFile in os.listdir(dirNameWriteOut) if "bureauEnergy_overview" in overviewFile].pop()
with open(dirNameWriteOut + overviewJsonFile)as f:
bureauEnergyOverview = json.load(f)
modelPool = [comparedValue['product_model'] for comparedValue in bureauEnergyDetail['productDetail']]
modelPoolDict = { v: k for k, v in enumerate(modelPool)}
#打開overviewJson檔案，為每個產品增加欄位。
for jsonObject in bureauEnergyOverview['product']:
index, test_report_of_energy_efficiency, benchmark, annual, labelUri = zipJsonObject(modelPoolDict, jsonObject['product_model'], bureauEnergyDetail)
# print('正在處理索引值： '+str(index))
jsonObject['test_report_of_energy_efficiency'] = test_report_of_energy_efficiency
jsonObject['efficiency_benchmark'] = benchmark
jsonObject['annual_power_consumption_degrees_dive_year'] = annual
jsonObject['energy_efficiency_label_innerUri'] = labelUri
# print('done '+str(index))
# 新增欄位的Json檔案更新時間。
timeStamp = timeStampGenerator()
bureauEnergyOverview["dateTime"] = timeStamp
with open(f"{_BASE_PATH}/dataMunging/{objectiveFolderClean}/{objective}/{objective}_{timeStamp}_{totalNums}_{searchword}.json",'w',encoding='utf-8')as f:
json.dump(bureauEnergyOverview, f, indent=2, ensure_ascii=False)
statistic.append(totalNums)
print(f"這裡是dataMunging_{thisPID}，準備完成工作。 ")
print()
end = timeCalculate()
print('dataMunging 累計耗時：{0} 秒'.format(end-begin))
input.task_done() #通知main process此次的input處理完成！
timeSleepOne() #暫停幾秒來模擬現實狀況。
print("擷取待爬取url出錯！", e)
timeStamp = timeStampGenerator()
totalNum = len(newsObjectReadyForCrawling)
newsObjectWhole["dateTime"] = timeStamp
newsObjectWhole["keyword"] = keyword
newsObjectWhole["newsTotalNum"] = totalNum
newsObjectWhole["newsUrl"] = newsObjectReadyForCrawling
#-----------------------------------檢測判斷如何---------------------
print("篩選出", len(readyLink), "則新聞。")
# print(len(readyLinkComparison))
# for row in readyLinkComparison:
# print(readyLinkComparison[row])
newsTitleList = [
newsObjectReadyForCrawling[key][0] + "\n"
for key in newsObjectReadyForCrawling
]
with open(
f"{_BASE_PATH}/{objectiveFolderDataMining}/{objectiveFolderDictionary}/threeWord.txt",
'w') as f:
f.writelines(newsTitleList)
# ----------------------------------寫出成果------------------
mkdirForCleanData(objectiveFolderClean, objective)
with open(
f"{_BASE_PATH}/dataMunging/{objectiveFolderClean}/{objective}/googleNews_{timeStamp}_{totalNum}_{keyword}.json",
"w",
encoding="utf-8") as f:
json.dump(newsObjectWhole, f, indent=2, ensure_ascii=False)
from collections import Counter
_BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(_BASE_PATH)
from libs.mining import newsMining
from libs.munging import rawDataMunging
from libs.timeWidget import (timeStampGenerator, timeCalculate)
from libs.sqlDMLAndsqlAlchemyORM import selectedNewsContentMunging
from libs.manipulateDir import mkdirForCleanData
if __name__ == '__main__':
newsMining = newsMining()
mkdirForCleanData(rawDataMunging._objectiveFolderCleanData,
rawDataMunging._objectiveFolderNewsWithContent)
dirRoute = f"{rawDataMunging._dirRouteMungingClean}{rawDataMunging._objectiveFolderNews}/"
begin = timeCalculate()
fileName, fileNums = newsMining.judgeFolderFiles(dirRoute)
referenceFile = selectedNewsContentMunging().loadReferenceFileIn()
if fileNums == 0:
print("""
尚未有clean的newsUri可以爬！請先完成
googleNewsMulti.py -->
newsMunging.py -->
jiebaForNewsTitle.py -->
def dataMunging(input, dirRoute, objectiveFolderClean, objective, domainUrl):
thisPID = os.getpid()
while True:
print(thisPID, "===========================================")
searchword = input.get()
mkdirForCleanData(objectiveFolderClean, objective)
# '/home/bluevc/2019/iSelect3C/dataMunging/rawData/momo/冷暖空調電熱水瓶' <---關鍵字累加的問題
# dirRoute = dirRoute + searchword
fileRoute = dirRoute + searchword
if not os.listdir(fileRoute):
print(
f"============={objective} {searchword} 資料夾沒有東西，此進程準備結束。============="
)
input.task_done()
timeSleepOne()
print("========================break========================")
break
momoDict = {}
productArray = []
for file in initialFileZeroUnderscoreInt(fileRoute):
# print("start " + file + " ! ")
with open(fileRoute + "/" + file) as f:
inn = f.read()
# 處理soup=""的情況
if not inn:
continue
textSoup = BeautifulSoup(inn, 'html.parser')
try:
#一頁至多有30項
products = textSoup.select_one('.listArea').select_one(
'ul').select('li')
for item in products:
innerDict = {}
innerDict['Id'] = item.attrs.get('gcode')
innerDict['name'] = item.select_one(
'.goodsUrl').select_one('.prdName').text
innerDict['originprice'] = item.select_one(
'.goodsUrl').select_one('.money .price').text.replace(
'$', '').replace(',', '')
innerDict['pics'] = item.select_one(
'.goodsUrl img').attrs.get('src')
innerDict['picb'] = "None"
innerDict['produrl'] = domainUrl + item.select_one(
'.goodsUrl').attrs.get('href')
productArray.append(innerDict)
except Exception as e:
print(f"{file} 有 {e} 的問題。")
dateTime = datetime.datetime.now()
fmt = "%Y-%m-%d-%H-%M" #"%Y年%m月%d日%H時%M分"
timeStamp = dateTime.strftime(fmt)
momoDict['product'] = productArray
momoDict['keyword'] = searchword
momoDict["dateTime"] = timeStamp
print("===========進行去重=============")
momoDict['product'], setNums = EcommerceDataProcessToSet(
momoDict['product'])
with open(
f"{_BASE_PATH}/dataMunging/{objectiveFolderClean}/{objective}/momo_{timeStamp}_{setNums}_{searchword}.json",
'w') as f:
json.dump(momoDict, f, indent=2, ensure_ascii=False)
print("===========清洗完成=============")
print(f"這裡是dataMunging_{thisPID}，準備完成工作。 ")
print()
end = time.time()
print('dataMunging 累計耗時：{0} 秒'.format(end - begin))
input.task_done() #通知main process此次的input處理完成！
timeSleepOne() #暫停幾秒來模擬現實狀況。
f"/weather_{dateTime}_{keyword}.json", 'w') as f:
json.dump(weatherDictOutter, f, indent=2, ensure_ascii=False)
if __name__ == '__main__':
weatherRecord = weatherRecordMunging()
# fileRouteList = [f for f in listSecondDirBelowFiles(weatherRecord._dirRouteMungingRaw + weatherRecord._weather)]
fileRouteGenerator = listSecondDirBelowFiles(
weatherRecord._dirRouteMungingRaw + weatherRecord._weather)
begin = timeCalculate()
eraseCleanData(weatherRecord._objectiveFolderCleanData,
weatherRecord._weather)
mkdirForCleanData(weatherRecord._objectiveFolderCleanData,
weatherRecord._weather)
dataMunging(fileRouteGenerator)
#共同佇列宣告
# file_queue = mp.JoinableQueue()
# 啟動進程
# Process_1 = []
# for p in range(3):
# dataMungingMultiWay_proc = mp.Process(target=dataMungingMultiWay, args=(file_queue,))
# dataMungingMultiWay_proc.daemon = True
# dataMungingMultiWay_proc.start()
# print(f'建立第{p}個 dataMungingMultiWay_proc, {dataMungingMultiWay_proc}')
# Process_1.append(dataMungingMultiWay_proc)