def getComments(self,debug=False,saveToMongo=False,saveToFile = False):
if saveToMongo:
client = MongoClient('localhost',44444)
db = client['jd']
collection = db['review']
if saveToFile:
i = 1
self.finalComments=list()
for numItem in self.finalResult:
baseUrl = self.__prepare4CommentsReview(numItem)
obj = Commodity(baseUrl)
obj.getComments(debug)
self.finalComments.append(obj.comments)
if saveToMongo:
self.__saveToMongo(collection,obj.comments)
print("save to mongo --port 44444 db:jd collection:review")
if saveToFile:
self.__saveToFile("./resultoutput/" + str(i) +".txt",obj.comments)
i+=1
def run(self):
while not self.queueGet.empty():
if self.saveToMongo:
client = MongoClient('localhost',44444)
db = client['jd']
collection = db['review']
try:
numItem = self.queueGet.get(timeout=120)
baseUrl = self.__prepare4CommentsReview(numItem)
commodityObj = Commodity(baseUrl)
commodityObj.getComments()
self.queueSave.put(commodityObj.comments)
if self.saveToMongo:
self.__saveToMongo(collection,commodityObj.comments)
print("save to mongo --port 44444 db:jd collection:review")
if self.saveToFile:
self.__saveToFile("./resultoutput/" + numItem +".txt",commodityObj.comments)
except Exception as e:
print(e)
finally:
print(threading.current_thread().name + " {0} comments length: {1}".format(numItem,len(commodityObj.comments)))
