def next_product_url():
cli = get_cli()
while True:
product = cli.lpop( KEY_PRODUCTS )
if product is None:
logger.info( 'no product in queue ...' )
time.sleep( 5 )
continue
try:
product = json.loads( product )[ 'asin' ]
if not product_is_timeout( product ):
logger.info( 'product : ' + product + \
' is not timeout, {day} days'.format(
day = CRAWL_PRODUCT_TIMEOUT ) )
continue
set_product_task( product )
del_product_reviews( product )
logger.info( 'next product : ' + product )
return 'http://www.amazon.com/ss/customer-reviews/' + product
except Exception as e:
logger.exception( repr( e ) )
def next_product_url():
cli = get_cli()
while True:
product = cli.lpop(KEY_PRODUCTS)
if product is None:
logger.info('no product in queue ...')
time.sleep(5)
continue
try:
product = json.loads(product)['asin']
if not product_is_timeout(product):
logger.info( 'product : ' + product + \
' is not timeout, {day} days'.format(
day = CRAWL_PRODUCT_TIMEOUT ) )
continue
set_product_task(product)
del_product_reviews(product)
logger.info('next product : ' + product)
return 'http://www.amazon.com/ss/customer-reviews/' + product
except Exception as e:
logger.exception(repr(e))
def wrapper( *args, **argkv ):
try:
return func( *args, **argkv )
except Exception, e:
logger.exception( repr( e ) )
return None