def process_item_amazon_review(self, item, spider): cli = get_cli() prdid = item.get( 'prdid' ) if not prdid: return key = KEY_REVIEW.format( p = prdid ) cli.rpush( key, json.dumps( dict( item ) ) ) logger.debug( repr( dict( item ) ) )
def try_lock(self): now = int(time.time()) pid = str(os.getpid()) v = '{now} {sv} {pid}'.format(now=now, sv=SERVER_NAME, pid=pid) now_sv_pid = self.cli.get(self.key) if now_sv_pid is None: self.cli.set(self.key, v) return True try: t_now, t_sv, t_pid = now_sv_pid.split() #long time ago or process been killed if now - int( t_now ) >= LOCK_LONGTIME or \ ( SERVER_NAME == t_sv and pid != t_pid ): self.cli.set(self.key, v) return True except Exception, e: logger.debug(repr(e)) self.release()
def try_lock( self ): now = int( time.time() ) pid = str( os.getpid() ) v = '{now} {sv} {pid}'.format( now = now, sv = SERVER_NAME, pid = pid ) now_sv_pid = self.cli.get( self.key ) if now_sv_pid is None: self.cli.set( self.key, v ) return True try: t_now, t_sv, t_pid = now_sv_pid.split() #long time ago or process been killed if now - int( t_now ) >= LOCK_LONGTIME or \ ( SERVER_NAME == t_sv and pid != t_pid ): self.cli.set( self.key, v ) return True except Exception, e: logger.debug( repr( e ) ) self.release()
def wrapper(*args, **argkv): try: return fun(*args, **argkv) except redis.exceptions.RedisError, e: logger.debug(repr(e)) raise
def wrapper( *args, **argkv ): try: return fun( *args, **argkv ) except redis.exceptions.RedisError, e: logger.debug( repr( e ) ) raise