Example #1
0
 def save(cls, data):
     data['updateTime'] = now()
     return MONGO[DB][cls.COLL].find_one_and_update(
         {'_id': data['_id']},
         {'$set': data},
         upsert=True
     )
def insert(coll, data):
    data['updateTime'] = now()
    MONGO[DB][coll].find_one_and_update(
        {'_id': data['_id']},
        {'$set': data},
        upsert=True
    )
    def run(self):
        base_url = TOPIC_URL.format(tid=self.topic_id)
        for page in xrange(1, 10):
            url = base_url + str(page)
            logger.info('crawling topic %s, page %s' % (self.topic_id, page))

            try:
                html = self.get(url)
                soup = BeautifulSoup(html)
            except:
                return

            for answer in self._run(soup):
                answer['topics'].append(self.topic_id)
                answer['updateTime'] = now()
                result = self.save(answer)
                if result < 0:
                    return
Example #4
0
    def run(self):
        base_url = TOPIC_URL.format(tid=self.topic_id)
        for page in xrange(1, 10):
            url = base_url + str(page)
            logger.info('crawling topic %s, page %s' % (self.topic_id, page))

            try:
                html = self.get(url)
                soup = BeautifulSoup(html)
            except:
                return

            for answer in self._run(soup):
                answer['topics'].append(self.topic_id)
                answer['updateTime'] = now()
                result = self.save(answer)
                if result < 0:
                    return
Example #5
0
 def save(cls, data):
     data['update'] = now()
     try:
         MONGO[DB][T66Y_COLL].insert_one(data)
     except:
         pass
Example #6
0
def insert(coll, data):
    data['updateTime'] = now()
    MONGO[DB][coll].find_one_and_update({'_id': data['_id']}, {'$set': data},
                                        upsert=True)
Example #7
0
 def save(cls, data):
     data['update'] = now()
     try:
         MONGO[DB][T66Y_COLL].insert_one(data)
     except:
         pass