def get_data(self):
if isinstance(self.sid, list):
print("Zomato ignored", self.sid)
pass
# parent_id= self.sid[0]
# for i in range(len(self.sid)):
# if i==0:
# pass
# else:
# objects=Reviews.objects(survey_id= self.sid[i],provider=self.p)
# for obj in objects:
# Reviews()
# pass
else:
rid = self.get_id()
total = self.get_total()
turn = int(total / 5) + 1
print(turn)
# 1/0
if len(Record.objects(survey_id=self.sid, rid=str(rid))) != 0:
print("Already Review Collected")
else:
pool = Pool()
ids = list(range(0, turn))
print(ids)
# 1/0
# for i in ids:
# self.sub_get(i)
pool.map(self.sub_get, ids)
Record(provider="zomato", survey_id=self.sid,
rid=str(rid)).save()
Status(unique_identifier=self.sid + provider,
scraped_status="success").save()
def get_data(self):
if isinstance(self.sid, list): print("Ignored")
else:
# links= self.generate_link()
# # print (links)
# try:
# for i in links:
# self.sub_get(i)
# except NotUniqueError:
# pass
links = self.generate_link()
if len(Record.objects(links=set(links))) != 0:
print("Already Reviews Collected")
else:
# pool= Pool(8)
# pool.map(self.sub_get,links)
for i in links:
try:
self.sub_get(i)
except NotUniqueError:
pass
Record(survey_id=self.sid,
provider="tripadvisor",
links=set(links)).save()
