def _update(self, old_topic_in_store, topic_2_store, hot_topic): try: old_topic_in_store.update_one( set__rank=topic_2_store.rank, set__topic=topic_2_store.topic, set__url=topic_2_store.url, set__num_infor=topic_2_store.num_infor, set__host=topic_2_store.host ) #need to log except NotUniqueError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException.ERROR_CODE_DICT["weibo hot topic NotUniqueError"], keyword="", page_num=self.data.page_num ) return False except ValidationError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException.ERROR_CODE_DICT["weibo hot topic ValidationError"], keyword="", page_num=self.data.page_num ) return False except OperationError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException.ERROR_CODE_DICT["weibo hot topic OperationError"], keyword="", page_num=self.data.page_num ) return False #need to log except Exception as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException.ERROR_CODE_DICT["weibo hot topic OtherError"], keyword="", page_num=self.data.page_num ) return False return True
def _update(self, old_topic_in_store, topic_2_store, hot_topic): try: old_topic_in_store.update_one( set__rank=topic_2_store.rank, set__topic=topic_2_store.topic, set__url=topic_2_store.url, set__num_infor=topic_2_store.num_infor, set__host=topic_2_store.host) #need to log except NotUniqueError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException. ERROR_CODE_DICT["weibo hot topic NotUniqueError"], keyword="", page_num=self.data.page_num) return False except ValidationError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException. ERROR_CODE_DICT["weibo hot topic ValidationError"], keyword="", page_num=self.data.page_num) return False except OperationError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException. ERROR_CODE_DICT["weibo hot topic OperationError"], keyword="", page_num=self.data.page_num) return False #need to log except Exception as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException. ERROR_CODE_DICT["weibo hot topic OtherError"], keyword="", page_num=self.data.page_num) return False return True
def store(self): topic_2_store = MongoHotTopic() topic_2_store.rank = self.data.rank topic_2_store.topic = self.data.topic topic_2_store.url = self.data.url topic_2_store.num_infor = self.data.num_infor topic_2_store.host = self.data.host old_topic_in_store = MongoHotTopic.objects(topic=topic_2_store.topic) # @UndefinedVariable if len(old_topic_in_store) is 0: try: topic_2_store.save() return True #need to log except NotUniqueError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException.ERROR_CODE_DICT["weibo hot topic NotUniqueError"], keyword="", page_num=self.data.page_num ) return False except ValidationError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException.ERROR_CODE_DICT["weibo hot topic ValidationError"], keyword="", page_num=self.data.page_num ) return False except OperationError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException.ERROR_CODE_DICT["weibo hot topic OperationError"], keyword="", page_num=self.data.page_num ) return False #need to log except Exception as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException.ERROR_CODE_DICT["weibo hot topic OtherError"], keyword="", page_num=self.data.page_num ) return False else: #not unique, so record this duplicate if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException.ERROR_CODE_DICT["weibo hot topic NotUniqueError"], keyword="", page_num=self.data.page_num ) return self._update(old_topic_in_store, topic_2_store, self.data)
def store(self): topic_2_store = MongoHotTopic() topic_2_store.rank = self.data.rank topic_2_store.topic = self.data.topic topic_2_store.url = self.data.url topic_2_store.num_infor = self.data.num_infor topic_2_store.host = self.data.host old_topic_in_store = MongoHotTopic.objects( topic=topic_2_store.topic) # @UndefinedVariable if len(old_topic_in_store) is 0: try: topic_2_store.save() return True #need to log except NotUniqueError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException. ERROR_CODE_DICT["weibo hot topic NotUniqueError"], keyword="", page_num=self.data.page_num) return False except ValidationError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException. ERROR_CODE_DICT["weibo hot topic ValidationError"], keyword="", page_num=self.data.page_num) return False except OperationError as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException. ERROR_CODE_DICT["weibo hot topic OperationError"], keyword="", page_num=self.data.page_num) return False #need to log except Exception as err: if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException. ERROR_CODE_DICT["weibo hot topic OtherError"], keyword="", page_num=self.data.page_num) return False else: #not unique, so record this duplicate if OPEN_WRITE_NOTUNIQUE_LOG: my_log.write_notunique_log( log_id=self.data.topic, log_type=self.data.typeid, fail_code=OtherException. ERROR_CODE_DICT["weibo hot topic NotUniqueError"], keyword="", page_num=self.data.page_num) return self._update(old_topic_in_store, topic_2_store, self.data)