Exemplo n.º 1
0
    def get_data(self):
        if isinstance(self.sid, list):
            print("Zomato ignored", self.sid)
            pass
            # parent_id= self.sid[0]
            # for i in range(len(self.sid)):
            # 	if i==0:
            # 		pass
            # 	else:
            # 		objects=Reviews.objects(survey_id= self.sid[i],provider=self.p)
            # 		for obj in objects:
            # 			Reviews()
            # 		pass
        else:
            rid = self.get_id()
            total = self.get_total()
            turn = int(total / 5) + 1
            print(turn)
            # 1/0
            if len(Record.objects(survey_id=self.sid, rid=str(rid))) != 0:
                print("Already Review Collected")
            else:
                pool = Pool()
                ids = list(range(0, turn))
                print(ids)
                # 1/0
                # for i in ids:
                # 	self.sub_get(i)
                pool.map(self.sub_get, ids)

                Record(provider="zomato", survey_id=self.sid,
                       rid=str(rid)).save()
                Status(unique_identifier=self.sid + provider,
                       scraped_status="success").save()
Exemplo n.º 2
0
	def get_data(self):
		if isinstance(self.sid,list):
			print ("Zomato ignored",self.sid)
			pass
			# parent_id= self.sid[0]
			# for i in range(len(self.sid)):
			# 	if i==0:
			# 		pass
			# 	else:
			# 		objects=Reviews.objects(survey_id= self.sid[i],provider=self.p)
			# 		for obj in objects:
			# 			Reviews()
			# 		pass
		else:
			rid = self.get_id()
			total= self.get_total()
			turn = int(total/5)+1
			print (turn)
			# 1/0
			if len(Record.objects(survey_id= self.sid,rid=str(rid)))!=0:
				print ("Already Review Collected")
			else:
				pool= Pool()
				ids=list(range(0,turn))
				print (ids)
				# 1/0
				# for i in ids:
				# 	self.sub_get(i)	
				pool.map(self.sub_get,ids)
				
				Record(provider="zomato",survey_id=self.sid,rid=str(rid)).save()
				Status(unique_identifier=self.sid+provider,scraped_status="success").save()
Exemplo n.º 3
0
 def multi(self):
     links = self.generate_link()
     # return links
     if len(Record.objects(links=set(links))) != 0:
         print("Already Review Collected")
     else:
         pool = Pool(8)
         results = pool.map(self.get_data, [links])
         return results
Exemplo n.º 4
0
	def multi(self):
		links= self.generate_link()
		# return links
		if len(Record.objects(links=set(links)))!=0:
			print ("Already Review Collected")
		else:
			pool= Pool(8)
			results= pool.map(self.get_data,[links])
			return results
Exemplo n.º 5
0
 def get_data(self):
     if isinstance(self.sid, list): print("Ignored")
     else:
         # links= self.generate_link()
         # # print (links)
         # try:
         # 	for i in links:
         # 		self.sub_get(i)
         # except NotUniqueError:
         # 	pass
         links = self.generate_link()
         if len(Record.objects(links=set(links))) != 0:
             print("Already Reviews Collected")
         else:
             # pool= Pool(8)
             # pool.map(self.sub_get,links)
             for i in links:
                 try:
                     self.sub_get(i)
                 except NotUniqueError:
                     pass
             Record(survey_id=self.sid,
                    provider="tripadvisor",
                    links=set(links)).save()
Exemplo n.º 6
0
	def get_data(self):
		if isinstance(self.sid,list):print("Ignored")
		else:
			# links= self.generate_link()
			# # print (links)
			# try:
			# 	for i in links:
			# 		self.sub_get(i)
			# except NotUniqueError:
			# 	pass
			links= self.generate_link()
			if len(Record.objects(links=set(links)))!=0:
				print ("Already Reviews Collected")
			else:
				# pool= Pool(8)
				# pool.map(self.sub_get,links)
				for i in links:
					try:
						self.sub_get(i)
					except NotUniqueError:
						pass
				Record(survey_id= self.sid,provider="tripadvisor",links= set(links)).save()