def get(self): date = time_util.today() locale = 'us' template_dict = create_template_data(locale, DAILY, date, self.request) self.response.headers["Cache-Control"] = "public; max-age=300;" #template = self.template('ranking_daily.html', template_dict) #logging.info(self.response.out.read()) return self.template('ranking_daily.html', template_dict)
def get(self): date = time_util.today() locale = 'us' template_dict = create_template_data(locale, DAILY, date,self.request) self.response.headers["Cache-Control"]="public; max-age=300;" #template = self.template('ranking_daily.html', template_dict) #logging.info(self.response.out.read()) return self.template('ranking_daily.html', template_dict)
def post(self): from collections import defaultdict payload_string = self.request.get('payload') counter_targets = Payload.deserialize(payload_string) today = time_util.today() product_targets = defaultdict(int) user_targets = defaultdict(int) for payload in counter_targets: daily_product_key = ProductCounter.build_key_name( payload.url, DAILY, today) weekly_product_key = ProductCounter.build_key_name( payload.url, WEEKLY, today) monthly_product_key = ProductCounter.build_key_name( payload.url, MONTHLY, today) user_key = UserCounter.build_key_name(payload.user_id, DAILY, today) product_targets[daily_product_key] += 1 product_targets[weekly_product_key] += 1 product_targets[monthly_product_key] += 1 user_targets[user_key] += 1 product_counters = ProductCounter.get_by_key_name( product_targets.keys(), _storage=[MEMCACHE, DATASTORE], _result_type=NAME_DICT) user_counters = UserCounter.get_by_key_name(user_targets.keys(), _result_type=NAME_DICT) for key_name, delta in product_targets.iteritems(): try: product_counters[key_name].count += delta except AttributeError: #Value is None in dict frequency = ProductCounter.frequency_from_key_name(key_name) product_counters[key_name] = ProductCounter.new( key_name, frequency, today, count=delta, _build_key_name=False) for key_name, delta in user_targets.iteritems(): try: user_counters[key_name].count += delta except AttributeError: #Value is None in dict user_counters[key_name] = UserCounter.new( key_name, DAILY, today, count=delta, _build_key_name=False) ProductCounter.filtered_update(product_counters.values()) UserCounter.filtered_update(user_counters.values())
def post(self): from collections import defaultdict payload_string = self.request.get('payload') counter_targets = Payload.deserialize(payload_string) today = time_util.today() product_targets = defaultdict(int) user_targets = defaultdict(int) for payload in counter_targets: daily_product_key = ProductCounter.build_key_name(payload.url, DAILY, today) weekly_product_key = ProductCounter.build_key_name(payload.url, WEEKLY, today) monthly_product_key = ProductCounter.build_key_name(payload.url, MONTHLY, today) user_key = UserCounter.build_key_name(payload.user_id, DAILY, today) product_targets[daily_product_key] += 1 product_targets[weekly_product_key] += 1 product_targets[monthly_product_key] += 1 user_targets[user_key] += 1 product_counters = ProductCounter.get_by_key_name(product_targets.keys(), _storage = [MEMCACHE,DATASTORE], _result_type=NAME_DICT) user_counters = UserCounter.get_by_key_name(user_targets.keys(), _result_type=NAME_DICT) for key_name,delta in product_targets.iteritems(): try: product_counters[key_name].count += delta except AttributeError: #Value is None in dict frequency = ProductCounter.frequency_from_key_name(key_name) product_counters[key_name] = ProductCounter.new(key_name, frequency, today, count=delta,_build_key_name = False) for key_name,delta in user_targets.iteritems(): try: user_counters[key_name].count += delta except AttributeError: #Value is None in dict user_counters[key_name] = UserCounter.new(key_name, DAILY, today, count=delta,_build_key_name = False) ProductCounter.filtered_update(product_counters.values()) UserCounter.filtered_update(user_counters.values())
def get(self): enqueue_renderer_update(DAILY,time_util.today())
import logging from datetime import timedelta from config import SPAM_COUNT_LIMIT from PerformanceEngine import pdb,DATASTORE,MEMCACHE,time_util from google.appengine.runtime.apiproxy_errors import CapabilityDisabledError from tweethit.model import ProductCounter,UserCounter,ProductRenderer,\ CounterBase,DAILY,WEEKLY,MONTHLY,TwitterUser,Product,Banlist from tweethit.query import USER_SPAM_COUNTERS,PRODUCT_RENDERER_BAN_TARGETS from tweethit.utils.task_util import enqueue_renderer_update,enqueue_cleanup yesterday = lambda : time_util.today()-timedelta(days=1) class CounterUpdate(helipad.Handler): '''Retrieves counters from memcache to update DB every 5 mins''' def get(self): counter_keys = CounterBase.get_cached_counter_keys() if not len(counter_keys): return logging.info('Counter keys retrieved: %s' %len(counter_keys)) counters = CounterBase.get(counter_keys,_storage=MEMCACHE) if len(counters): try: logging.info('Counters being inserted: %s' %len(counters)) pdb.put(counters, _storage=DATASTORE) #Delete cached counter keys CounterBase.set_cached_counter_keys([])
def create_template_data(locale, frequency, date, request, **kwargs): current_date = time_util.today() current_period_flag = False query_cache_expiration = 0 store_key = Store.key_for_locale(locale) query = get_renderer_query_for_frequency(frequency, date, store_key) result_data = {} if len(request.query_string) > 0: result_data['query_string'] = '?' + request.query_string request_url = request.url[:request.url.find('?')] else: result_data['query_string'] = '' request_url = request.url #Debugger for turkey ban filter try: request.params['ban'] root = 'http://tweethitapp.appspot.com' except KeyError: root = root_url if frequency == DAILY: result_data['next_href'] = create_day_href(locale, date, 1, root) result_data['prev_href'] = create_day_href(locale, date, -1, root) if date == current_date: current_period_flag = True elif frequency == WEEKLY: result_data['next_href'] = create_week_href(locale, date, 1, root) result_data['prev_href'] = create_week_href(locale, date, -1, root) if date.year == current_date.year and \ date.isocalendar()[1] == current_date.isocalendar()[1]: current_period_flag = True elif frequency == MONTHLY: if date.year == current_date.year and \ date.month == current_date.month: current_period_flag = True result_data['next_href'] = create_month_href(locale, date, 1, root) result_data['prev_href'] = create_month_href(locale, date, -1, root) else: raise RendererException('Problem creating template data for \ locale: %s, frequency: %s, kwargs: %s' % (locale, frequency, kwargs)) groups = defaultdict(int) products = query.fetch(TEMPLATE_PRODUCT_COUNT) for item in products: groups[item.product_group] += 1 if len(item.title) > 72: item.title = item.title[:72] + '...' group_tuples = [] keys = sorted(groups.keys()) for key in keys: group_tuples.append((key, groups[key])) result_data['date'] = date result_data['current_period_flag'] = current_period_flag result_data['products'] = products result_data['group_tuples'] = group_tuples result_data['assoc_id'] = secret.ASSOCIATE_DICT[locale] result_data['root_url'] = root result_data['daily_ranking_href'] = root + '/' + locale + '/day/' result_data['weekly_ranking_href'] = root + '/' + locale + '/week/' result_data['monthly_ranking_href'] = root + '/' + locale + '/month/' return result_data
def get(self, locale): date = time_util.today() template_dict = create_template_data(locale, 'monthly', date, self.request) self.response.headers["Cache-Control"] = "public; max-age=300;" return self.template('ranking_monthly.html', template_dict)
def create_template_data(locale,frequency,date,request,**kwargs): current_date = time_util.today() current_period_flag = False query_cache_expiration = 0 store_key = Store.key_for_locale(locale) query = get_renderer_query_for_frequency(frequency,date,store_key) result_data = {} if len(request.query_string) > 0: result_data['query_string'] = '?'+request.query_string request_url = request.url[:request.url.find('?')] else: result_data['query_string'] = '' request_url = request.url #Debugger for turkey ban filter try: request.params['ban'] root = 'http://tweethitapp.appspot.com' except KeyError: root = root_url if frequency == DAILY: result_data['next_href'] = create_day_href(locale, date, 1,root) result_data['prev_href'] = create_day_href(locale, date, -1,root) if date == current_date: current_period_flag = True elif frequency == WEEKLY: result_data['next_href'] = create_week_href(locale, date, 1,root) result_data['prev_href'] = create_week_href(locale, date, -1,root) if date.year == current_date.year and \ date.isocalendar()[1] == current_date.isocalendar()[1]: current_period_flag = True elif frequency == MONTHLY: if date.year == current_date.year and \ date.month == current_date.month: current_period_flag = True result_data['next_href'] = create_month_href(locale, date, 1,root) result_data['prev_href'] = create_month_href(locale, date, -1,root) else: raise RendererException('Problem creating template data for \ locale: %s, frequency: %s, kwargs: %s' %(locale,frequency,kwargs)) groups = defaultdict(int) products = query.fetch(TEMPLATE_PRODUCT_COUNT) for item in products: groups[item.product_group] += 1 if len(item.title) > 72: item.title = item.title[:72]+'...' group_tuples = [] keys = sorted(groups.keys()) for key in keys: group_tuples.append((key,groups[key])) result_data['date'] = date result_data['current_period_flag'] = current_period_flag result_data['products'] = products result_data['group_tuples'] = group_tuples result_data['assoc_id'] = secret.ASSOCIATE_DICT[locale] result_data['root_url'] = root result_data['daily_ranking_href'] = root+'/'+locale+'/day/' result_data['weekly_ranking_href'] = root+'/'+locale+'/week/' result_data['monthly_ranking_href'] = root+'/'+locale+'/month/' return result_data
def get(self,locale): date = time_util.today() template_dict = create_template_data(locale, 'monthly', date,self.request) self.response.headers["Cache-Control"]="public; max-age=300;" return self.template('ranking_monthly.html', template_dict)