Example #1
0
 def reviewp(self):
   if self.parent=="true":
     survey_id=[self.survey_id]
     for obj in AspectQ.objects(parent=self.survey_id):
       survey_id.append(obj.survey_id)
     print("survey_id",survey_id)
     task= Status.objects(unique_identifier=survey_id[0]+self.provider)
   else:
     survey_id= self.survey_id
     task= Status.objects(unique_identifier=survey_id+self.provider)
   print ("task",task)
   try:
     status= task.scraped_status
     if task.scraped_status=="success" and task.reviewp_status!="success":
       # ReviewP(self.survey_id,self.provider).run()
       task.update(reviewp_status="success")
     else:
         print ("ReviewP job ignored for ",self.survey_id,"provider",self.provider)
   except Exception as e:
     print ("ReviewP Exception Occured ***",e,"*** for",self.survey_id,self.provider)
     
   if task.scraped_status=="success" and task.reviewp_status!="success":
     ReviewP(self.survey_id,self.provider).run()
     task.update(reviewp_status="success")
   else:
     print ("ReviewP job ignored for ",self.survey_id,"provider",self.provider)
Example #2
0
    def reviewp(self):
        if self.parent == "true":
            survey_id = [self.survey_id]
            for obj in AspectQ.objects(parent=self.survey_id):
                survey_id.append(obj.survey_id)
            print("survey_id", survey_id)
            task = Status.objects(unique_identifier=survey_id[0] +
                                  self.provider)
        else:
            survey_id = self.survey_id
            task = Status.objects(unique_identifier=survey_id + self.provider)
        print("task", task)
        try:
            status = task.scraped_status
            if task.scraped_status == "success" and task.reviewp_status != "success":
                # ReviewP(self.survey_id,self.provider).run()
                task.update(reviewp_status="success")
            else:
                print("ReviewP job ignored for ", self.survey_id, "provider",
                      self.provider)
        except Exception as e:
            print("ReviewP Exception Occured ***", e, "*** for",
                  self.survey_id, self.provider)

        if task.scraped_status == "success" and task.reviewp_status != "success":
            ReviewP(self.survey_id, self.provider).run()
            task.update(reviewp_status="success")
        else:
            print("ReviewP job ignored for ", self.survey_id, "provider",
                  self.provider)
Example #3
0
    def get_data(self):
        if isinstance(self.sid, list):
            print("Zomato ignored", self.sid)
            pass
            # parent_id= self.sid[0]
            # for i in range(len(self.sid)):
            # 	if i==0:
            # 		pass
            # 	else:
            # 		objects=Reviews.objects(survey_id= self.sid[i],provider=self.p)
            # 		for obj in objects:
            # 			Reviews()
            # 		pass
        else:
            rid = self.get_id()
            total = self.get_total()
            turn = int(total / 5) + 1
            print(turn)
            # 1/0
            if len(Record.objects(survey_id=self.sid, rid=str(rid))) != 0:
                print("Already Review Collected")
            else:
                pool = Pool()
                ids = list(range(0, turn))
                print(ids)
                # 1/0
                # for i in ids:
                # 	self.sub_get(i)
                pool.map(self.sub_get, ids)

                Record(provider="zomato", survey_id=self.sid,
                       rid=str(rid)).save()
                Status(unique_identifier=self.sid + provider,
                       scraped_status="success").save()
Example #4
0
 def sentiment(self):
   task= Status.objects(unique_identifier=self.survey_id+self.provider)
   if task.scraped_status=="success" and task.reviewp_status=="success" and task.sentiment_status!="success":
     Sentiment(self.survey_id,self.provider).run()
     task.update(sentiment_status="success")
   else:
     print ("Sentiment job ignored for ",self.survey_id,"provider",self.provider)
Example #5
0
 def sentiment(self):
     task = Status.objects(unique_identifier=self.survey_id + self.provider)
     if task.scraped_status == "success" and task.reviewp_status == "success" and task.sentiment_status != "success":
         Sentiment(self.survey_id, self.provider).run()
         task.update(sentiment_status="success")
     else:
         print("Sentiment job ignored for ", self.survey_id, "provider",
               self.provider)
Example #6
0
 def aspectr(self):
   if self.parent=="true":
     pass
   else:
       task= Status.objects(unique_identifier=self.survey_id+self.provider)
       if task.scraped_status=="success" and task.reviewp_status=="success" and task.sentiment_status=="success":
         # AspectR(self.survey_id,self.provider)
         task.update(aspectr_status="success")
       else:
         print ("AspectR job ignored for ",self.survey_id,"provider",self.provider)
Example #7
0
 def aspectr(self):
     if self.parent == "true":
         pass
     else:
         task = Status.objects(unique_identifier=self.survey_id +
                               self.provider)
         if task.scraped_status == "success" and task.reviewp_status == "success" and task.sentiment_status == "success":
             # AspectR(self.survey_id,self.provider)
             task.update(aspectr_status="success")
         else:
             print("AspectR job ignored for ", self.survey_id, "provider",
                   self.provider)
Example #8
0
    def scrap(self):
        if self.parent == "true":
            print(self.survey_id, "is a parent survey")
            try:
                task = Status(unique_identifier=self.survey_id +
                              self.provider).count()
                if task == 0:
                    Status(unique_identifier=self.survey_id + self.provider,
                           scraped_status="success").save()
                else:
                    pass
            except Exception as e:
                print("Exception", e, "for", self.survey_id)
                try:
                    Status(unique_identifier=self.survey_id + self.provider,
                           scraped_status="success").save()
                except:
                    pass
        else:
            task = Status.objects(unique_identifier=self.survey_id +
                                  self.provider).count()
            if task == 1:
                print(self.survey_id, "ignored for provider", self.provider)
                pass
            else:

                if self.provider == "zomato":
                    try:
                        print("Scraping", self.survey_id)
                        # Zomato(self.base_url,self.survey_id,self.provider).get_data()
                        Status(unique_identifier=self.survey_id +
                               self.provider,
                               scraped_status="success").save()

                    except Exception as e:
                        print("Exception occured for ", e, "***********",
                              self.survey_id, "provider", self.provider)
                        with open("log.txt", "a") as f:
                            f.write(str(e) + "****** \n")
                elif self.provider == "tripadvisor":
                    try:
                        TripAdvisor(self.base_url, self.survey_id,
                                    self.provider).get_data()
                        Status(unique_identifier=self.survey_id +
                               self.provider,
                               scraped_status="success").save()
                    except:
                        print("Exception occured for ", self.survey_id,
                              "provider", self.provider)
                else:
                    print("Bad Provider: ", self.provider)
Example #9
0
  def scrap(self):
    if self.parent=="true":
      print (self.survey_id,"is a parent survey")
      try:
        task=Status(unique_identifier=self.survey_id+self.provider).count()
        if task==0:
          Status(unique_identifier= self.survey_id+self.provider,scraped_status="success").save()
        else:pass
      except Exception as e:
        print("Exception",e,"for",self.survey_id)
        try:Status(unique_identifier= self.survey_id+self.provider,scraped_status="success").save()
        except:pass
    else:
      task= Status.objects(unique_identifier=self.survey_id+self.provider).count()
      if task==1:
        print(self.survey_id,"ignored for provider",self.provider)
        pass
      else:

        if self.provider=="zomato":
          try:
            print ("Scraping",self.survey_id)
            # Zomato(self.base_url,self.survey_id,self.provider).get_data()
            Status(unique_identifier=self.survey_id+self.provider,scraped_status="success").save()
            
          except Exception as e:
            print("Exception occured for ",e,"***********",self.survey_id,"provider",self.provider)
            with open("log.txt","a") as f:
              f.write(str(e)+"****** \n")
        elif self.provider=="tripadvisor":
          try:
            TripAdvisor(self.base_url,self.survey_id,self.provider).get_data()
            Status(unique_identifier=self.survey_id+self.provider,scraped_status="success").save()
          except:
            print("Exception occured for ",self.survey_id,"provider",self.provider)
        else:
          print("Bad Provider: ",self.provider)
Example #10
0
 def wordcloud(self):
     if isinstance(self.survey_id, list):
         survey_id = self.survey_id[0]
     else:
         survey_id = self.survey_id
     task = Status.objects(unique_identifier=survey_id + self.provider)
Example #11
0
 def wordcloud(self):
   if isinstance(self.survey_id,list):
     survey_id=self.survey_id[0]
   else:survey_id=self.survey_id
   task= Status.objects(unique_identifier=survey_id+self.provider)