def post(self):
model_kind = self.request.get('model_kind')
frequency = self.request.get('frequency')
date = str_to_date(self.request.get('date_string'))
store_key_name = self.request.get('store_key_name')
logging.info('Cleanupworker model kind: %s frequency: \
%s store: %s' % (model_kind, frequency, store_key_name))
recursion_flag = False
if model_kind == 'ProductRenderer':
store_key = Key.from_path('Store', store_key_name)
query = get_renderer_query_for_frequency(frequency, date,
store_key)
keys = query.fetch(200, TEMPLATE_PRODUCT_COUNT)
elif model_kind == 'ProductCounter':
store_key = Key.from_path('Store', store_key_name)
query = get_counter_query_for_frequency(frequency, date, store_key)
keys = query.fetch(200)
elif model_kind == 'UserCounter':
query = USER_COUNTER_CLEANUP_TARGETS
keys = query.fetch(200)
else:
logging.error('No type found for CleanupWorker :%s' % model_kind)
if len(keys):
recursion_flag = True
pdb.delete(keys)
if recursion_flag:
logging.info('Enqueing cleanup for model %s' % model_kind)
enqueue_cleanup(model_kind, frequency, str(date), store_key_name)
def post(self):
store_key_name = self.request.get('store_key_name')
date = str_to_date(self.request.get('date_string'))
frequency = self.request.get('frequency')
renderers = []
store_key = Key.from_path('Store', store_key_name)
query = get_counter_query_for_frequency(frequency, date, store_key)
product_counters = query.fetch(TEMPLATE_PRODUCT_COUNT)
key_names = [counter.key().name() for counter in product_counters]
product_renderers = ProductRenderer.get_by_key_name(
key_names, _storage=[MEMCACHE, DATASTORE], _result_type=NAME_DICT)
for counter in product_counters:
renderer = product_renderers[counter.key().name()]
try:
renderer.count = counter.count
renderers.append(renderer)
except AttributeError: #Renderer is none
renderer = ProductRenderer.build(counter.key_root,
frequency,
date,
count=counter.count)
if renderer is not None: #building from existing renderers successful
renderers.append(renderer)
else:
enqueue_renderer_info(counter.key_root, counter.count,
frequency, date)
if len(renderers):
pdb.put(renderers, _storage=[MEMCACHE, DATASTORE])
def post(self):
model_kind = self.request.get('model_kind')
frequency = self.request.get('frequency')
date = str_to_date(self.request.get('date_string'))
store_key_name = self.request.get('store_key_name')
logging.info('Cleanupworker model kind: %s frequency: \
%s store: %s' %(model_kind,frequency,store_key_name))
recursion_flag = False
if model_kind == 'ProductRenderer':
store_key = Key.from_path('Store',store_key_name)
query = get_renderer_query_for_frequency(frequency, date, store_key)
keys = query.fetch(200, TEMPLATE_PRODUCT_COUNT)
elif model_kind == 'ProductCounter':
store_key = Key.from_path('Store',store_key_name)
query = get_counter_query_for_frequency(frequency, date, store_key)
keys = query.fetch(200)
elif model_kind == 'UserCounter':
query = USER_COUNTER_CLEANUP_TARGETS
keys = query.fetch(200)
else:
logging.error('No type found for CleanupWorker :%s' %model_kind)
if len(keys):
recursion_flag = True
pdb.delete(keys)
if recursion_flag:
logging.info('Enqueing cleanup for model %s' %model_kind)
enqueue_cleanup(model_kind,frequency,str(date),store_key_name)
def post(self):
product_key_name = self.request.get('product_key_name')
count = int(self.request.get('count'))
retries = int(self.request.get('retries'))
date = str_to_date(self.request.get('date_string'))
frequency = self.request.get('frequency')
logging.info('Fetching details for %s , frequency: %s' %(product_key_name,frequency))
#Create empty renderer
renderer = ProductRenderer.new(product_key_name,frequency, date,count = count)
asin = AmazonURLParser.extract_asin(product_key_name)
locale = AmazonURLParser.get_locale(product_key_name)
renderer = AmazonProductFetcher.get_product_details(asin, renderer,locale)
if renderer is not None: #If all details were fetched successfully
renderer.put(_storage=[MEMCACHE,DATASTORE])
else:
if retries < MAX_PRODUCT_INFO_RETRIES:
retries += 1
logging.error('Error saving product: %s, adding to queue again, retries: %s' %(product_key_name,retries))
enqueue_renderer_info(product_key_name,count,frequency,date,
countdown = 60, retries = retries)
else:
logging.critical('Max retries reached for product: %s' %product_key_name)
renderer = ProductRenderer.new(product_key_name,frequency,
date, count = count,is_banned=True)
renderer.log_properties()
renderer.put(_storage=[MEMCACHE,DATASTORE])
def post(self):
store_key_name = self.request.get('store_key_name')
date = str_to_date(self.request.get('date_string'))
frequency = self.request.get('frequency')
renderers = []
store_key = Key.from_path('Store',store_key_name)
query = get_counter_query_for_frequency(frequency, date, store_key)
product_counters = query.fetch(TEMPLATE_PRODUCT_COUNT)
key_names = [counter.key().name() for counter in product_counters ]
product_renderers = ProductRenderer.get_by_key_name(key_names,
_storage=[MEMCACHE,DATASTORE],
_result_type=NAME_DICT)
for counter in product_counters:
renderer = product_renderers[counter.key().name()]
try:
renderer.count = counter.count
renderers.append(renderer)
except AttributeError: #Renderer is none
renderer = ProductRenderer.build(counter.key_root,
frequency, date,count = counter.count)
if renderer is not None: #building from existing renderers successful
renderers.append(renderer)
else:
enqueue_renderer_info(counter.key_root,
counter.count,
frequency,
date)
if len(renderers):
pdb.put(renderers, _storage=[MEMCACHE,DATASTORE])
def post(self):
product_key_name = self.request.get('product_key_name')
count = int(self.request.get('count'))
retries = int(self.request.get('retries'))
date = str_to_date(self.request.get('date_string'))
frequency = self.request.get('frequency')
logging.info('Fetching details for %s , frequency: %s' %
(product_key_name, frequency))
#Create empty renderer
renderer = ProductRenderer.new(product_key_name,
frequency,
date,
count=count)
asin = AmazonURLParser.extract_asin(product_key_name)
locale = AmazonURLParser.get_locale(product_key_name)
renderer = AmazonProductFetcher.get_product_details(
asin, renderer, locale)
if renderer is not None: #If all details were fetched successfully
renderer.put(_storage=[MEMCACHE, DATASTORE])
else:
if retries < MAX_PRODUCT_INFO_RETRIES:
retries += 1
logging.error(
'Error saving product: %s, adding to queue again, retries: %s'
% (product_key_name, retries))
enqueue_renderer_info(product_key_name,
count,
frequency,
date,
countdown=60,
retries=retries)
else:
logging.critical('Max retries reached for product: %s' %
product_key_name)
renderer = ProductRenderer.new(product_key_name,
frequency,
date,
count=count,
is_banned=True)
renderer.log_properties()
renderer.put(_storage=[MEMCACHE, DATASTORE])