def save(cls, data):
data['updateTime'] = now()
return MONGO[DB][cls.COLL].find_one_and_update(
{'_id': data['_id']},
{'$set': data},
upsert=True
)
def insert(coll, data):
data['updateTime'] = now()
MONGO[DB][coll].find_one_and_update(
{'_id': data['_id']},
{'$set': data},
upsert=True
)
def run(self):
base_url = TOPIC_URL.format(tid=self.topic_id)
for page in xrange(1, 10):
url = base_url + str(page)
logger.info('crawling topic %s, page %s' % (self.topic_id, page))
try:
html = self.get(url)
soup = BeautifulSoup(html)
except:
return
for answer in self._run(soup):
answer['topics'].append(self.topic_id)
answer['updateTime'] = now()
result = self.save(answer)
if result < 0:
return
def run(self):
base_url = TOPIC_URL.format(tid=self.topic_id)
for page in xrange(1, 10):
url = base_url + str(page)
logger.info('crawling topic %s, page %s' % (self.topic_id, page))
try:
html = self.get(url)
soup = BeautifulSoup(html)
except:
return
for answer in self._run(soup):
answer['topics'].append(self.topic_id)
answer['updateTime'] = now()
result = self.save(answer)
if result < 0:
return
def save(cls, data):
data['update'] = now()
try:
MONGO[DB][T66Y_COLL].insert_one(data)
except:
pass
def insert(coll, data):
data['updateTime'] = now()
MONGO[DB][coll].find_one_and_update({'_id': data['_id']}, {'$set': data},
upsert=True)
def save(cls, data):
data['update'] = now()
try:
MONGO[DB][T66Y_COLL].insert_one(data)
except:
pass
