def get_search_products( Keywords, SearchIndex, num = 10, **argkv ): logger.info( 'search : ' + repr( ( Keywords, SearchIndex, num, argkv ) ) ) handle = AmazonAPI( conf.aws_key, conf.aws_secret, conf.aws_associate_tag, Version = "2013-08-01", ) products = handle.search( Keywords = Keywords, SearchIndex = SearchIndex, **argkv ) cli = get_cli() items = [] for product in products: d = _dict_product( product ) k = conf.KEY_REVIEW.format( p = product.asin ) d[ 'crawl' ] = True if cli.exists( k ) else False k = conf.KEY_PRODUCT_TASK.format( p = product.asin ) if cli.exists( k ): r = cli.get( k ) d[ 'delay' ] = time_delay( json.loads( r )[ 'ctime' ] ) else: d[ 'delay' ] = None if d[ 'reviews' ] and d[ 'reviews' ][ 0 ]: d[ 'reviews' ] = d[ 'reviews' ][ 1 ] items.append( d ) if len( items ) == num: return items
def get_product( product_id ): logger.info( 'get : ' + product_id ) handle = AmazonAPI( conf.aws_key, conf.aws_secret, conf.aws_associate_tag, Version = "2013-08-01", ) product = handle.lookup( ItemId = product_id ) d = _dict_product( product ) cli = get_cli() k = conf.KEY_REVIEW.format( p = product.asin ) d[ 'crawl' ] = True if cli.exists( k ) else False k = conf.KEY_PRODUCT_TASK.format( p = product.asin ) if cli.exists( k ): r = cli.get( k ) d[ 'delay' ] = time_delay( json.loads( r )[ 'ctime' ] ) else: d[ 'delay' ] = None if d[ 'reviews' ] and d[ 'reviews' ][ 0 ]: d[ 'reviews' ] = d[ 'reviews' ][ 1 ] return d
def get_product(product_id): logger.info('get : ' + product_id) handle = AmazonAPI( conf.aws_key, conf.aws_secret, conf.aws_associate_tag, Version="2013-08-01", ) product = handle.lookup(ItemId=product_id) d = _dict_product(product) cli = get_cli() k = conf.KEY_REVIEW.format(p=product.asin) d['crawl'] = True if cli.exists(k) else False k = conf.KEY_PRODUCT_TASK.format(p=product.asin) if cli.exists(k): r = cli.get(k) d['delay'] = time_delay(json.loads(r)['ctime']) else: d['delay'] = None if d['reviews'] and d['reviews'][0]: d['reviews'] = d['reviews'][1] return d
def get_search_products(Keywords, SearchIndex, num=10, **argkv): logger.info('search : ' + repr((Keywords, SearchIndex, num, argkv))) handle = AmazonAPI( conf.aws_key, conf.aws_secret, conf.aws_associate_tag, Version="2013-08-01", ) products = handle.search(Keywords=Keywords, SearchIndex=SearchIndex, **argkv) cli = get_cli() items = [] for product in products: d = _dict_product(product) k = conf.KEY_REVIEW.format(p=product.asin) d['crawl'] = True if cli.exists(k) else False k = conf.KEY_PRODUCT_TASK.format(p=product.asin) if cli.exists(k): r = cli.get(k) d['delay'] = time_delay(json.loads(r)['ctime']) else: d['delay'] = None if d['reviews'] and d['reviews'][0]: d['reviews'] = d['reviews'][1] items.append(d) if len(items) == num: return items