示例#1
0
    def post(self):
        model_kind = self.request.get('model_kind')
        frequency = self.request.get('frequency')
        date = str_to_date(self.request.get('date_string'))
        store_key_name = self.request.get('store_key_name')

        logging.info('Cleanupworker model kind: %s frequency: \
    %s store: %s' % (model_kind, frequency, store_key_name))

        recursion_flag = False

        if model_kind == 'ProductRenderer':
            store_key = Key.from_path('Store', store_key_name)
            query = get_renderer_query_for_frequency(frequency, date,
                                                     store_key)
            keys = query.fetch(200, TEMPLATE_PRODUCT_COUNT)
        elif model_kind == 'ProductCounter':
            store_key = Key.from_path('Store', store_key_name)
            query = get_counter_query_for_frequency(frequency, date, store_key)
            keys = query.fetch(200)
        elif model_kind == 'UserCounter':
            query = USER_COUNTER_CLEANUP_TARGETS
            keys = query.fetch(200)
        else:
            logging.error('No type found for CleanupWorker :%s' % model_kind)

        if len(keys):
            recursion_flag = True
            pdb.delete(keys)

        if recursion_flag:
            logging.info('Enqueing cleanup for model %s' % model_kind)
            enqueue_cleanup(model_kind, frequency, str(date), store_key_name)
示例#2
0
    def post(self):
        store_key_name = self.request.get('store_key_name')
        date = str_to_date(self.request.get('date_string'))
        frequency = self.request.get('frequency')

        renderers = []
        store_key = Key.from_path('Store', store_key_name)
        query = get_counter_query_for_frequency(frequency, date, store_key)

        product_counters = query.fetch(TEMPLATE_PRODUCT_COUNT)
        key_names = [counter.key().name() for counter in product_counters]
        product_renderers = ProductRenderer.get_by_key_name(
            key_names, _storage=[MEMCACHE, DATASTORE], _result_type=NAME_DICT)
        for counter in product_counters:
            renderer = product_renderers[counter.key().name()]
            try:
                renderer.count = counter.count
                renderers.append(renderer)
            except AttributeError:  #Renderer is none
                renderer = ProductRenderer.build(counter.key_root,
                                                 frequency,
                                                 date,
                                                 count=counter.count)
                if renderer is not None:  #building from existing renderers successful
                    renderers.append(renderer)
                else:
                    enqueue_renderer_info(counter.key_root, counter.count,
                                          frequency, date)
        if len(renderers):
            pdb.put(renderers, _storage=[MEMCACHE, DATASTORE])
示例#3
0
 def post(self):
   model_kind = self.request.get('model_kind')
   frequency = self.request.get('frequency')
   date = str_to_date(self.request.get('date_string'))
   store_key_name = self.request.get('store_key_name')
   
   logging.info('Cleanupworker model kind: %s frequency: \
   %s store: %s' %(model_kind,frequency,store_key_name))
   
   recursion_flag = False
   
   if model_kind == 'ProductRenderer':
     store_key = Key.from_path('Store',store_key_name)
     query = get_renderer_query_for_frequency(frequency, date, store_key)
     keys = query.fetch(200, TEMPLATE_PRODUCT_COUNT)
   elif model_kind == 'ProductCounter':
     store_key = Key.from_path('Store',store_key_name)
     query = get_counter_query_for_frequency(frequency, date, store_key)
     keys = query.fetch(200)
   elif model_kind == 'UserCounter':
     query = USER_COUNTER_CLEANUP_TARGETS
     keys = query.fetch(200)
   else:
     logging.error('No type found for CleanupWorker :%s' %model_kind)
   
   if len(keys):
     recursion_flag = True
     pdb.delete(keys)
   
   if recursion_flag:
     logging.info('Enqueing cleanup for model %s' %model_kind)
     enqueue_cleanup(model_kind,frequency,str(date),store_key_name)
示例#4
0
 def post(self):
   product_key_name = self.request.get('product_key_name')
   count = int(self.request.get('count'))
   retries = int(self.request.get('retries'))
   date = str_to_date(self.request.get('date_string'))
   frequency = self.request.get('frequency')
   
   logging.info('Fetching details for %s , frequency: %s' %(product_key_name,frequency))
   
   #Create empty renderer 
   renderer = ProductRenderer.new(product_key_name,frequency, date,count = count)
   
   asin = AmazonURLParser.extract_asin(product_key_name)
   locale = AmazonURLParser.get_locale(product_key_name)
   renderer = AmazonProductFetcher.get_product_details(asin, renderer,locale)
   
   if renderer is not None: #If all details were fetched successfully
     renderer.put(_storage=[MEMCACHE,DATASTORE])
   else:
     if retries <  MAX_PRODUCT_INFO_RETRIES:
       retries += 1
       logging.error('Error saving product: %s, adding to queue again, retries: %s' %(product_key_name,retries))
       enqueue_renderer_info(product_key_name,count,frequency,date,
                                       countdown = 60, retries = retries)
     else:
       logging.critical('Max retries reached for product: %s' %product_key_name)
       renderer = ProductRenderer.new(product_key_name,frequency,
                                       date, count = count,is_banned=True)
       renderer.log_properties()
       renderer.put(_storage=[MEMCACHE,DATASTORE])
示例#5
0
  def post(self):
    store_key_name = self.request.get('store_key_name')
    date = str_to_date(self.request.get('date_string'))
    frequency = self.request.get('frequency')
      
    renderers = []
    store_key = Key.from_path('Store',store_key_name)
    query = get_counter_query_for_frequency(frequency, date, store_key)

    product_counters = query.fetch(TEMPLATE_PRODUCT_COUNT)
    key_names = [counter.key().name() for counter in product_counters ]
    product_renderers = ProductRenderer.get_by_key_name(key_names,
                                                        _storage=[MEMCACHE,DATASTORE],
                                                        _result_type=NAME_DICT)
    for counter in product_counters:
      renderer = product_renderers[counter.key().name()]
      try:
        renderer.count = counter.count
        renderers.append(renderer)
      except AttributeError: #Renderer is none
        renderer = ProductRenderer.build(counter.key_root, 
                                         frequency, date,count = counter.count)
        if renderer is not None: #building from existing renderers successful
          renderers.append(renderer)
        else:
          enqueue_renderer_info(counter.key_root, 
                                        counter.count,
                                        frequency,
                                        date)
    if len(renderers):
      pdb.put(renderers, _storage=[MEMCACHE,DATASTORE])
示例#6
0
    def post(self):
        product_key_name = self.request.get('product_key_name')
        count = int(self.request.get('count'))
        retries = int(self.request.get('retries'))
        date = str_to_date(self.request.get('date_string'))
        frequency = self.request.get('frequency')

        logging.info('Fetching details for %s , frequency: %s' %
                     (product_key_name, frequency))

        #Create empty renderer
        renderer = ProductRenderer.new(product_key_name,
                                       frequency,
                                       date,
                                       count=count)

        asin = AmazonURLParser.extract_asin(product_key_name)
        locale = AmazonURLParser.get_locale(product_key_name)
        renderer = AmazonProductFetcher.get_product_details(
            asin, renderer, locale)

        if renderer is not None:  #If all details were fetched successfully
            renderer.put(_storage=[MEMCACHE, DATASTORE])
        else:
            if retries < MAX_PRODUCT_INFO_RETRIES:
                retries += 1
                logging.error(
                    'Error saving product: %s, adding to queue again, retries: %s'
                    % (product_key_name, retries))
                enqueue_renderer_info(product_key_name,
                                      count,
                                      frequency,
                                      date,
                                      countdown=60,
                                      retries=retries)
            else:
                logging.critical('Max retries reached for product: %s' %
                                 product_key_name)
                renderer = ProductRenderer.new(product_key_name,
                                               frequency,
                                               date,
                                               count=count,
                                               is_banned=True)
                renderer.log_properties()
                renderer.put(_storage=[MEMCACHE, DATASTORE])