def post(self): product_key_name = self.request.get('product_key_name') count = int(self.request.get('count')) retries = int(self.request.get('retries')) date = str_to_date(self.request.get('date_string')) frequency = self.request.get('frequency') logging.info('Fetching details for %s , frequency: %s' %(product_key_name,frequency)) #Create empty renderer renderer = ProductRenderer.new(product_key_name,frequency, date,count = count) asin = AmazonURLParser.extract_asin(product_key_name) locale = AmazonURLParser.get_locale(product_key_name) renderer = AmazonProductFetcher.get_product_details(asin, renderer,locale) if renderer is not None: #If all details were fetched successfully renderer.put(_storage=[MEMCACHE,DATASTORE]) else: if retries < MAX_PRODUCT_INFO_RETRIES: retries += 1 logging.error('Error saving product: %s, adding to queue again, retries: %s' %(product_key_name,retries)) enqueue_renderer_info(product_key_name,count,frequency,date, countdown = 60, retries = retries) else: logging.critical('Max retries reached for product: %s' %product_key_name) renderer = ProductRenderer.new(product_key_name,frequency, date, count = count,is_banned=True) renderer.log_properties() renderer.put(_storage=[MEMCACHE,DATASTORE])
def post(self): store_key_name = self.request.get('store_key_name') date = str_to_date(self.request.get('date_string')) frequency = self.request.get('frequency') renderers = [] store_key = Key.from_path('Store', store_key_name) query = get_counter_query_for_frequency(frequency, date, store_key) product_counters = query.fetch(TEMPLATE_PRODUCT_COUNT) key_names = [counter.key().name() for counter in product_counters] product_renderers = ProductRenderer.get_by_key_name( key_names, _storage=[MEMCACHE, DATASTORE], _result_type=NAME_DICT) for counter in product_counters: renderer = product_renderers[counter.key().name()] try: renderer.count = counter.count renderers.append(renderer) except AttributeError: #Renderer is none renderer = ProductRenderer.build(counter.key_root, frequency, date, count=counter.count) if renderer is not None: #building from existing renderers successful renderers.append(renderer) else: enqueue_renderer_info(counter.key_root, counter.count, frequency, date) if len(renderers): pdb.put(renderers, _storage=[MEMCACHE, DATASTORE])
def post(self): store_key_name = self.request.get('store_key_name') date = str_to_date(self.request.get('date_string')) frequency = self.request.get('frequency') renderers = [] store_key = Key.from_path('Store',store_key_name) query = get_counter_query_for_frequency(frequency, date, store_key) product_counters = query.fetch(TEMPLATE_PRODUCT_COUNT) key_names = [counter.key().name() for counter in product_counters ] product_renderers = ProductRenderer.get_by_key_name(key_names, _storage=[MEMCACHE,DATASTORE], _result_type=NAME_DICT) for counter in product_counters: renderer = product_renderers[counter.key().name()] try: renderer.count = counter.count renderers.append(renderer) except AttributeError: #Renderer is none renderer = ProductRenderer.build(counter.key_root, frequency, date,count = counter.count) if renderer is not None: #building from existing renderers successful renderers.append(renderer) else: enqueue_renderer_info(counter.key_root, counter.count, frequency, date) if len(renderers): pdb.put(renderers, _storage=[MEMCACHE,DATASTORE])
def get(self): date = yesterday() enqueue_renderer_update(WEEKLY,date) enqueue_renderer_update(MONTHLY,date) enqueue_cleanup(UserCounter.kind(), DAILY, date) enqueue_cleanup(ProductCounter.kind(), DAILY, date) enqueue_cleanup(ProductRenderer.kind(), DAILY, date) enqueue_cleanup(ProductRenderer.kind(), WEEKLY, date,countdown = 3600) enqueue_cleanup(ProductRenderer.kind(), MONTHLY, date,countdown = 3600) #Delete banlist so it is refreshed with latest banned entities banlist = Banlist.retrieve() banlist.delete()
def post(self): product_key_name = self.request.get('product_key_name') count = int(self.request.get('count')) retries = int(self.request.get('retries')) date = str_to_date(self.request.get('date_string')) frequency = self.request.get('frequency') logging.info('Fetching details for %s , frequency: %s' % (product_key_name, frequency)) #Create empty renderer renderer = ProductRenderer.new(product_key_name, frequency, date, count=count) asin = AmazonURLParser.extract_asin(product_key_name) locale = AmazonURLParser.get_locale(product_key_name) renderer = AmazonProductFetcher.get_product_details( asin, renderer, locale) if renderer is not None: #If all details were fetched successfully renderer.put(_storage=[MEMCACHE, DATASTORE]) else: if retries < MAX_PRODUCT_INFO_RETRIES: retries += 1 logging.error( 'Error saving product: %s, adding to queue again, retries: %s' % (product_key_name, retries)) enqueue_renderer_info(product_key_name, count, frequency, date, countdown=60, retries=retries) else: logging.critical('Max retries reached for product: %s' % product_key_name) renderer = ProductRenderer.new(product_key_name, frequency, date, count=count, is_banned=True) renderer.log_properties() renderer.put(_storage=[MEMCACHE, DATASTORE])