def get_data(self):
if isinstance(self.sid, list):
print("Zomato ignored", self.sid)
pass
# parent_id= self.sid[0]
# for i in range(len(self.sid)):
# if i==0:
# pass
# else:
# objects=Reviews.objects(survey_id= self.sid[i],provider=self.p)
# for obj in objects:
# Reviews()
# pass
else:
rid = self.get_id()
total = self.get_total()
turn = int(total / 5) + 1
print(turn)
# 1/0
if len(Record.objects(survey_id=self.sid, rid=str(rid))) != 0:
print("Already Review Collected")
else:
pool = Pool()
ids = list(range(0, turn))
print(ids)
# 1/0
# for i in ids:
# self.sub_get(i)
pool.map(self.sub_get, ids)
Record(provider="zomato", survey_id=self.sid,
rid=str(rid)).save()
Status(unique_identifier=self.sid + provider,
scraped_status="success").save()
def get_data(self):
if isinstance(self.sid,list):
print ("Zomato ignored",self.sid)
pass
# parent_id= self.sid[0]
# for i in range(len(self.sid)):
# if i==0:
# pass
# else:
# objects=Reviews.objects(survey_id= self.sid[i],provider=self.p)
# for obj in objects:
# Reviews()
# pass
else:
rid = self.get_id()
total= self.get_total()
turn = int(total/5)+1
print (turn)
# 1/0
if len(Record.objects(survey_id= self.sid,rid=str(rid)))!=0:
print ("Already Review Collected")
else:
pool= Pool()
ids=list(range(0,turn))
print (ids)
# 1/0
# for i in ids:
# self.sub_get(i)
pool.map(self.sub_get,ids)
Record(provider="zomato",survey_id=self.sid,rid=str(rid)).save()
Status(unique_identifier=self.sid+provider,scraped_status="success").save()
def multi(self):
links = self.generate_link()
# return links
if len(Record.objects(links=set(links))) != 0:
print("Already Review Collected")
else:
pool = Pool(8)
results = pool.map(self.get_data, [links])
return results
def multi(self):
links= self.generate_link()
# return links
if len(Record.objects(links=set(links)))!=0:
print ("Already Review Collected")
else:
pool= Pool(8)
results= pool.map(self.get_data,[links])
return results
def get_data(self):
if isinstance(self.sid, list): print("Ignored")
else:
# links= self.generate_link()
# # print (links)
# try:
# for i in links:
# self.sub_get(i)
# except NotUniqueError:
# pass
links = self.generate_link()
if len(Record.objects(links=set(links))) != 0:
print("Already Reviews Collected")
else:
# pool= Pool(8)
# pool.map(self.sub_get,links)
for i in links:
try:
self.sub_get(i)
except NotUniqueError:
pass
Record(survey_id=self.sid,
provider="tripadvisor",
links=set(links)).save()
def get_data(self):
if isinstance(self.sid,list):print("Ignored")
else:
# links= self.generate_link()
# # print (links)
# try:
# for i in links:
# self.sub_get(i)
# except NotUniqueError:
# pass
links= self.generate_link()
if len(Record.objects(links=set(links)))!=0:
print ("Already Reviews Collected")
else:
# pool= Pool(8)
# pool.map(self.sub_get,links)
for i in links:
try:
self.sub_get(i)
except NotUniqueError:
pass
Record(survey_id= self.sid,provider="tripadvisor",links= set(links)).save()