def post(self):
query = InstallMetric.all()
query.order('-installed_at')
cursor = self.request.get('cursor')
if cursor != None:
query.with_cursor(cursor)
install_metrics = query.fetch(25)
for install_metric in install_metrics:
post_install_activity_metric = PostInstallActivityMetric.gql("WHERE user = :1", install_metric.user).get()
if post_install_activity_metric == None:
post_install_activity_metric = PostInstallActivityMetric(
fb_user_id = install_metric.user.fb_user_id,
user = install_metric.user,
installed_at = install_metric.installed_at
)
else:
post_install_activity_metric.fb_user_id = install_metric.user.fb_user_id
post_install_activity_metric.user = install_metric.user
post_install_activity_metric.installed_at = install_metric.installed_at
for day in [1,2,3,4,5,6,7]:
day_start = install_metric.installed_at + datetime.timedelta(days=day)
day_end = install_metric.installed_at + datetime.timedelta(days=(day+1))
page_view_in_day = PageView.gql(
"WHERE user = :1 AND created_at >= :2 AND created_at < :3",
install_metric.user, day_start, day_end).get()
if page_view_in_day != None:
setattr(post_install_activity_metric, "active_day_%s" % day, True)
post_install_activity_metric.put()
self.response.out.write(simplejson.dumps({'status': 'ok', 'cursor': str(query.cursor()), 'count': len(install_metrics) }))
def post(self):
query = PageView.all()
query.order('-created_at')
cursor = self.request.get('cursor')
if cursor != None:
query.with_cursor(cursor)
filter_out_params = {}
for param in str(self.request.get('filtered_params')).split(','):
filter_out_params[param.strip()] = True
page_views = query.fetch(100)
for page_view in page_views:
parsed = urlparse(page_view.url)
params = cgi.parse_qs(parsed.query)
new_url = parsed.scheme + "://" + parsed.netloc + parsed.path
new_param_string = ''
for key in sorted(params.keys()):
if key not in filter_out_params or filter_out_params[key] == False:
for val in params[key]:
if new_param_string != '':
new_param_string += '&'
new_param_string += key + "=" + val
if len(new_param_string) == 0:
page_view.normalized_url = new_url
else:
page_view.normalized_url = new_url + "?" + new_param_string
page_view.put()
self.response.out.write(simplejson.dumps({'status': 'ok', 'cursor': str(query.cursor()), 'count': len(page_views) }))
def get(self):
search_term = self.request.get('q')
query = PageView.all()
query.order('-normalized_url')
query.filter('normalized_url >=',search_term)
query.filter('normalized_url <', search_term+"\ufffd")
page_views = query.fetch(1000)
unique_url_hash = {}
for page_view in page_views:
unique_url_hash[page_view.normalized_url] = 1
unique_urls = unique_url_hash.keys()
unique_urls.sort()
self.response.out.write("\n".join(unique_urls))
def post(self):
url = self.request.get('url')
query = PageView.all()
query.order('session_id')
query.filter('normalized_url =', url)
cursor = self.request.get('cursor')
if cursor != None:
query.with_cursor(cursor)
page_views = query.fetch(1000)
page_views_by_session_id = {}
for page_view in page_views:
if page_view.session_id in page_views_by_session_id:
page_views_by_session_id[page_view.session_id] += 1
else:
page_views_by_session_id[page_view.session_id] = 1
self.response.out.write(simplejson.dumps({'status': 'ok', 'results': page_views_by_session_id, 'cursor': str(query.cursor()), 'total_pageviews': len(page_views), 'total_sessions': len(page_views_by_session_id.keys()) }))
def post(self):
mode = self.request.get('mode')
if mode == 'entrances':
mode_session_order_add_amount = -1
else:
mode_session_order_add_amount = 1
url = self.request.get('url')
query = PageView.all()
query.order('session_id')
query.filter('normalized_url =', url)
cursor = self.request.get('cursor')
if cursor != None:
query.with_cursor(cursor)
page_views = query.fetch(1000)
entrances_hash = {}
for page_view in page_views:
if mode == 'entrances' and page_view.session_order == 0:
if '[direct]' in entrances_hash:
entrances_hash['[direct]'] += 1
else:
entrances_hash['[direct]'] = 1
else:
q_s = "SELECT * FROM PageView WHERE session_id = :1 AND session_order = :2"
entrance_page_view = db.GqlQuery(q_s, page_view.session_id, page_view.session_order + mode_session_order_add_amount).get()
if entrance_page_view:
if entrance_page_view.normalized_url in entrances_hash:
entrances_hash[entrance_page_view.normalized_url] += 1
else:
entrances_hash[entrance_page_view.normalized_url] = 1
elif mode == 'exits':
if '[leave-site]' in entrances_hash:
entrances_hash['[leave-site]'] += 1
else:
entrances_hash['[leave-site]'] = 1
self.response.out.write(simplejson.dumps({'status': 'ok', 'mode': mode, 'cursor': str(query.cursor()), 'results': entrances_hash, 'total_entrances': len(entrances_hash.keys()), 'total_entrance_pageviews': sum(entrances_hash.values()) }))
def post(self):
session_ids = None
query = PageView.all()
query.order('-created_at')
query.filter('session_order =', int(self.request.get('session_order')))
if self.request.get('session_ids') != '':
query.filter('session_id IN', simplejson.loads(self.request.get('session_ids')))
cursor = self.request.get('cursor')
if cursor != None:
query.with_cursor(cursor)
page_views = query.fetch(1000)
page_views.sort(lambda a,b: cmp(a.normalized_url, b.normalized_url))
collapsed_page_views = []
current_url = None
current_count = 0
current_session_ids = []
for page_view in page_views:
if current_url == page_view.normalized_url:
current_count += 1
current_session_ids.append(page_view.session_id)
else:
if current_count > 0:
collapsed_page_views.append({'url': current_url, 'count': current_count, 'session_ids': current_session_ids })
current_url = page_view.normalized_url
current_count = 1
current_session_ids = [page_view.session_id]
if current_count > 0:
collapsed_page_views.append({'url': current_url, 'count': current_count, 'session_ids': current_session_ids })
current_url = None
current_count = 0
self.response.out.write(simplejson.dumps({'status': 'ok', 'cursor': str(query.cursor()), 'results': collapsed_page_views, 'total_pageviews_in_query': len(page_views) }))
def get(self):
c = h.context()
c['model'] = 'pageview'
c['model_properties'] = sorted(PageView.properties())
h.render_out(self, 'admin.tplt', c)
def post(self):
query = User.all()
query.order('-created_at')
cursor = self.request.get('cursor')
if cursor != None:
query.with_cursor(cursor)
users = query.fetch(25)
for user in users:
page_view_with_user = PageView.gql("WHERE user = :1 ORDER BY created_at ASC", user).get()
if page_view_with_user == None:
logging.error("Couldn't find PageView for User(key=%s,fb_user_id=%s)" % (str(user.key()), str(user.fb_user_id)))
page_view_with_user = PageView() # Fake it till we make it
if page_view_with_user.session_id != None and len(page_view_with_user.session_id) > 0:
page_view_with_session_id = PageView.gql("WHERE session_id = :1 ORDER BY created_at ASC", page_view_with_user.session_id).get()
else:
page_view_with_session_id = page_view_with_user
referring_page_view = page_view_with_session_id
if not referring_page_view.is_saved():
referring_page_view = None
installed_at = user.created_at
if page_view_with_session_id.referrer == None or len(page_view_with_session_id.referrer) < 10:
url_to_parse = page_view_with_session_id.url
else:
url_to_parse = page_view_with_session_id.referrer
if url_to_parse == None:
url_to_parse = ''
parsed_referral_url = urlparse(url_to_parse)
params = cgi.parse_qs(parsed_referral_url.query)
if 'suid' in params and params['suid'] != None and len(params['suid'][0]) > 0:
installed_via_newsfeed = True
referring_user = User.gql("WHERE fb_user_id = :1", params['suid'][0]).get()
newsfeed_search_term = '|'.join(params['q'])
newsfeed_verb = '|'.join(params['v'])
ad_name = None
installed_via_ad = False
installed_via_unknown = False
elif 'q' in params and params['q'] != None and len(params['q'][0]) > 0:
installed_via_newsfeed = False
referring_user = None
newsfeed_search_term = None
newsfeed_verb = None
ad_name = '|'.join(params['q'])
installed_via_ad = True
installed_via_unknown = False
else:
installed_via_newsfeed = False
referring_user = None
newsfeed_search_term = None
newsfeed_verb = None
ad_name = None
installed_via_ad = False
installed_via_unknown = True
install_metric = InstallMetric.gql("WHERE user = :1", user).get()
if install_metric == None:
install_metric = InstallMetric(
fb_user_id = user.fb_user_id,
user = user,
installed_at = installed_at,
installed_via_ad = installed_via_ad,
ad_name = ad_name,
installed_via_newsfeed = installed_via_newsfeed,
referring_user = referring_user,
newsfeed_search_term = newsfeed_search_term,
newsfeed_verb = newsfeed_verb,
installed_via_unknown = installed_via_unknown,
referring_page_view = referring_page_view
)
else:
install_metric.fb_user_id = user.fb_user_id
install_metric.user = user
install_metric.installed_at = installed_at
install_metric.installed_via_ad = installed_via_ad
install_metric.ad_name = ad_name
install_metric.installed_via_newsfeed = installed_via_newsfeed
install_metric.referring_user = referring_user
install_metric.newsfeed_search_term = newsfeed_search_term
install_metric.newsfeed_verb = newsfeed_verb
install_metric.installed_via_unknown = installed_via_unknown
install_metric.referring_page_view = referring_page_view
install_metric.put()
self.response.out.write(simplejson.dumps({'status': 'ok', 'cursor': str(query.cursor()), 'count': len(users) }))
def post(self):
cookies = h.get_default_cookies(self)
if '_pvk' not in cookies or (cookies['_pvk'] == None or cookies['_pvk'] == ''):
new_session_key = str(datetime.datetime.utcnow()).replace(' ', '') + '-' + str(random.getrandbits(50))
cookies['_pvk'] = new_session_key
previous_page_view_for_session = db.GqlQuery("SELECT * FROM PageView WHERE session_id = :1 ORDER BY created_at DESC", cookies['_pvk']).get()
page_view = PageView()
current_user = h.get_current_user(cookies)
if current_user != None:
page_view.user = current_user
page_view.url = self.request.get('u')
page_view.normalized_url = self.request.get('u')[:500] # TODO actually normalize URLs according to some logic (e.g. stripping out fb params)
page_view.referrer = self.request.get('referrer')
page_view.session_id = cookies['_pvk']
page_view.ip_address = self.request.remote_addr
try:
page_view.user_agent = self.request.headers['User-Agent'][:500]
except:
page_view.user_agent = None
if previous_page_view_for_session:
page_view.session_order = previous_page_view_for_session.session_order + 1
else:
page_view.session_order = 0
page_view.put()
self.response.out.write('{status: \'ok\'}')
