class Piper(object): def __init__(self, distributed=0): self.distributed = distributed if self.distributed == 1: self.queue_key = QUEUE_KEY self.redis_config = REDIS_DB_CONFIG self.redis_conn = redis.Redis(**self.redis_config) else: self.updator = Updator() def add(self, db_product): if self.distributed == 0: self._local_process(db_product) elif self.distributed == 1: self._add_redis(db_product) def _add_redis(self, db_product): self.redis_conn.sadd(self.queue_key, json.dumps(db_product)) def consume(self): db_product = json.loads(self.redis_conn.spop(self.queue_key)) self._local_process(db_product) def _local_process(self, db_product): product = Product(db_product) # if not product.is_timeout(): if self.updator.is_exist(product): self.updator.update(product) else: self.updator.insert(product)
def __init__(self, distributed=0): self.distributed = distributed if self.distributed == 1: self.queue_key = QUEUE_KEY self.redis_config = REDIS_DB_CONFIG self.redis_conn = redis.Redis(**self.redis_config) else: self.updator = Updator()