def process_item_amazon_review(self, item, spider):
cli = get_cli()
prdid = item.get( 'prdid' )
if not prdid:
return
key = KEY_REVIEW.format( p = prdid )
cli.rpush( key, json.dumps( dict( item ) ) )
logger.debug( repr( dict( item ) ) )
def try_lock(self):
now = int(time.time())
pid = str(os.getpid())
v = '{now} {sv} {pid}'.format(now=now, sv=SERVER_NAME, pid=pid)
now_sv_pid = self.cli.get(self.key)
if now_sv_pid is None:
self.cli.set(self.key, v)
return True
try:
t_now, t_sv, t_pid = now_sv_pid.split()
#long time ago or process been killed
if now - int( t_now ) >= LOCK_LONGTIME or \
( SERVER_NAME == t_sv and pid != t_pid ):
self.cli.set(self.key, v)
return True
except Exception, e:
logger.debug(repr(e))
self.release()
def try_lock( self ):
now = int( time.time() )
pid = str( os.getpid() )
v = '{now} {sv} {pid}'.format( now = now, sv = SERVER_NAME, pid = pid )
now_sv_pid = self.cli.get( self.key )
if now_sv_pid is None:
self.cli.set( self.key, v )
return True
try:
t_now, t_sv, t_pid = now_sv_pid.split()
#long time ago or process been killed
if now - int( t_now ) >= LOCK_LONGTIME or \
( SERVER_NAME == t_sv and pid != t_pid ):
self.cli.set( self.key, v )
return True
except Exception, e:
logger.debug( repr( e ) )
self.release()
def wrapper(*args, **argkv):
try:
return fun(*args, **argkv)
except redis.exceptions.RedisError, e:
logger.debug(repr(e))
raise
def wrapper( *args, **argkv ):
try:
return fun( *args, **argkv )
except redis.exceptions.RedisError, e:
logger.debug( repr( e ) )
raise
