def get(self):
memcache_key = 'update_user_badges'
cached_badges = memcache.get(memcache_key)
#if cached_badges is not None:
# logging.info('badge from cache')
## u_badges = cached_badges
#else:
user_badges = UserBadge.all()
u_badges = [ u for u in user_badges if u.badge_property is None ]
memcache.set(memcache_key, u_badges)
logging.info('total badges %s' % len(u_badges))
for ub in u_badges:
if ub.badge_property is not None:
logging.info('skipping already defined badge_p')
continue
b = Badges.gql('WHERE badge_label = :1 ', ub.badge).get()
if b is None:
b = Badges()
b.badge_label = ub.badge
b.badge_icon = ub.badge
b.put()
ud = UserDetails.gql('WHERE instaright_account = :1' , ub.user).get()
if ud is None:
ud = UserDetails()
ud.instaright_account = ub.user
ud.put()
ub.badge_property=b.key()
ub.user_property = ud.key()
ub.put()
def registerUser(self, request):
p_key = ndb.Key(UserDetails, request.phoneNumber)
succ = 1
user = UserDetails(
key=p_key,
displayName=request.displayName,
mainEmail=request.mainEmail,
balance=0,
phoneNumber=request.phoneNumber,
pin=request.pin,
uri=request.uri,
)
try:
user.put()
except datastore_errors.DuplicatePropertyError:
succ = 0
prof = ProfileForm()
if succ == 1:
prof.displayName = user.displayName
prof.uri = user.uri
prof.mainEmail = user.mainEmail
prof.balance = user.balance
prof.phoneNumber = user.phoneNumber
prof.success = succ
return prof
def get(self, user):
if user is None or len(user) == 0:
logging.error('Empty user. Skipping')
return
user_decoded = urllib.unquote(user)
logging.info('user: %s' %user_decoded)
ud=UserDetails.gql('WHERE instapaper_account = :1' , user_decoded).get()
if ud is None:
logging.info('non existing user. redirect to home')
self.redirect('/')
return
# sanity check
if ud.instaright_account is None:
ud.instaright_account = ud.instapaper_account
ud.put()
memcache_key ='user_info_' + user_decoded+'_'+str(datetime.datetime.now().date())
sessions = SessionModel.gql('WHERE instaright_account = :1 ORDER by date desc ' , user_decoded).fetch(100)
links = [ s for s in sessions if s is not None ]
cached_info = memcache.get(memcache_key)
if cached_info:
logging.info('getting from cache' )
template_variables = {'user':cached_info,'links':links}
path= os.path.join(os.path.dirname(__file__), 'templates/user_info.html')
self.response.headers["Content-type"] = "text/html"
self.response.out.write(template.render(path,template_variables))
return
user_detail= UserDetails.gql('WHERE mail = :1', user_decoded).get()
if user_detail is None:
logging.info('new user %s added to queue' %user_decoded)
fetch_url = '/user/'+user+'/fetch'
taskqueue.add(queue_name='user-info', url= fetch_url)
ud = UserDetails()
ud.name = user_decoded
ud.instapaper_account = user_decoded
ud.instaright_account = user_decoded
ud.links_added = SessionModel.countAllForUser(user_decoded)
# tmp put until we find more info for user
ud.put()
template_variables = {'user':ud, 'links': links}
path= os.path.join(os.path.dirname(__file__), 'templates/user_info.html')
self.response.headers["Content-type"] = "text/html"
self.response.headers["Accept-Charset"] = "utf-8"
self.response.out.write(template.render(path,template_variables))
return
if user_detail.instaright_account is None:
user_detail.instaright_account = user_decoded
user_detail.put()
memcache.set(memcache_key, user_detail)
template_variables = {'user':user_detail, "links" : links}
path= os.path.join(os.path.dirname(__file__), 'templates/user_info.html')
self.response.headers["Content-type"] = "text/html"
self.response.out.write(template.render(path,template_variables))
def post(self):
user, user_details = self.get_current_user(redirect=True)
if not user: return
details = UserDetails(user=user,
age=int(self.request.get('AgeInYears')),
years_of_training=int(self.request.get('NYearsTraining')),
# recruited_through_sona= \
# self.request.get('recruitedThroughSona') == 'True',
parent=ndb.Key('UserDetails', ANNOTATION_NAME))
# if self.request.get('SonaNumber'):
# details.sona_number = int(self.request.get('SonaNumber'))
details.put()
self.redirect('/annotate?agree=/form/')
def getCurrentScore(cls, user):
if user is None:
return None
userDetails = UserDetails.gql('WHERE instaright_account = :1', user).get()
if userDetails is None:
logging.info('no user with instaright account %s' % user)
userDetails = UserDetails.gql('WHERE instapaper_account = :1', user).get()
if userDetails is None:
userDetails=UserDetails()
userDetails.instapaper_account=user
userDetails.instaright_account=user
userDetails.put()
now =datetime.datetime.now().date()
currentScore=ScoreUsersDaily.gql('WHERE user = :1 and date = :2', userDetails.key(), now).get()
if currentScore is None:
currentScore = ScoreUsersDaily()
currentScore.user=userDetails.key()
currentScore.date = now
return currentScore
def post(self):
dt = self.request.get("date", None)
cursor = self.request.get("last_cursor", None)
logging.info("date from request %s " % dt)
if dt is None:
date = datetime.datetime.now().date() - datetime.timedelta(days=1)
logging.info("aggregatine users from yesterday.")
else:
date = datetime.datetime.strptime(dt, "%Y-%m-%d").date()
if date >= datetime.datetime.now().date():
logging.info("too early , wait")
self.response.out.write("too early . wait")
return
memcache_key_sessions = (
"sessions_for_date_" + str(datetime.datetime.today().date()) + "_" + str(date) + "_" + str(cursor)
)
cached_sessions = memcache.get(memcache_key_sessions)
if cached_sessions:
logging.info("getting from cache for date %s" % str(date))
sessions = cached_sessions
else:
sessions = SessionModel.getDailyDataWithOffset(date, cursor)
logging.info("session batch size %d" % len(sessions))
memcache.set(memcache_key_sessions, sessions)
if sessions is None:
logging.info("no sessions for date %s" % str(date))
return
for s in sessions:
memcache_key_s = "user_detail_" + str(datetime.datetime.now().date()) + "_" + str(date) + "_" + str(s.key())
if memcache.get(memcache_key_s):
logging.info("skippin processed key %s for date %s" % (s.key(), str(date)))
continue
# links stats add to queue
taskqueue.add(
queue_name="data-consolidation",
url="/aggregate_data",
params={"sessionKey": s.key(), "upper_limit_date": date},
)
# TODO also create tas cue for user consolidation
userStats = UserStats.gql("WHERE instapaper_account = :1 and date = :2", s.instaright_account, date).get()
if userStats is None:
logging.info("no user stats for user: %s and date: %s" % (s.instaright_account, str(date)))
userStats = UserStats()
userStats.instapaper_account = s.instaright_account
userStats.count = 1
userStats.date = date
userStats.put()
else:
logging.info("updating user stats for user %s and date %s" % (s.instaright_account, str(date)))
userStats.count = userStats.count + 1
userStats.put()
user_detail = UserDetails.gql("WHERE instapaper_account = :1", s.instaright_account).get()
if user_detail is None:
logging.info("new user: %s" % s.instaright_account)
user_detail = UserDetails()
user_detail.instapaper_account = s.instaright_account
user_detail.last_active_date = s.date
user_detail.put()
# task queue that gathers info
try:
fetch_task_url = "/user/" + urllib2.quote(s.instaright_account) + "/fetch"
except:
logging.warn("can't fetch info for user %s " % s.instaright_account)
logging.info("adding task on url %s" % fetch_task_url)
taskqueue.add(queue_name="user-info", url=fetch_task_url)
else:
logging.info("updating usage for user: %s" % s.instaright_account)
user_detail.last_active_date = s.date
user_detail.links_added = user_detail.links_added + 1
user_detail.put()
memcache.set(memcache_key_s, s.key())
# IMPORTANT:delete daily badges for user
memcache.delete("badge_" + s.instaright_account)
logging.info("done for date %s" % str(date))
self.response.out.write("done for date %s" % str(date))
def post(self):
user=self.request.get('user', None)
url=self.request.get('url', None)
domain=self.request.get('domain', None)
version=self.request.get('version', None)
client=self.request.get('client', None)
badge=None
if user is None:
logging.info('unknown user skipping')
return
user_details = UserDetails.gql('WHERE instaright_account = :1', user).get()
if user_details is None:
user_details = UserDetails()
user_details.instaright_account = user
user_details.put()
client_badge = UserBadge.gql('WHERE badge = :1 and user = :2', client, user).get()
if client_badge is None:
badge = Badges.gql('WHERE badge_label = :1' , client).get()
if badge is None:
mail.send_mail(sender='*****@*****.**', to='*****@*****.**', subject='Unknown client!', html='user %s saved link %s using unrecognised client %s' %( user,url, client ), body='user %s saved link %s using unrecognised client %s' %( user,url, client))
return
client_badge = UserBadge()
client_badge.badge = client
client_badge.badge_property = badge.key()
client_badge.date = datetime.datetime.now().date()
client_badge.user = user
client_badge.user_property = user_details.key()
client_badge.put()
if version is not None and len(version) == 0:
version = None
currentBadge = memcache.get('badge_'+user)
if currentBadge is not None and (currentBadge == '1' or currentBadge == '2' or currentBadge == '3' or currentBadge == '5'):
logging.info('for user %s already using full day badge: %s' %(user,currentBadge))
return
badger=BadgeUtil.getBadger(user, url, domain, version)
if badger is None:
logging.info('no badger initialized. skipping')
return
badge=badger.getBadge()
if badge is not None:
# midnight timestamp - memcache expiration
midnight=datetime.datetime.now().date() + datetime.timedelta(days=1)
midnight_ts=time.mktime(midnight.timetuple())
memcache.set('badge_'+user, badge, time=midnight_ts)
logging.info('setting badge cache: %s for user badge_%s valid until midnight %s' % (badge,user,midnight_ts))
existingBadge=UserBadge.gql('WHERE badge = :1 and user = :2 and date = :3', badge, user, datetime.datetime.now().date()).get()
if existingBadge is not None:
self.response.out.write('request processed')
return
b = Badges.gql('WHERE badge_label = :1' , badge).get()
if b is None:
b = Badges()
b.badge_label = badge
b.badge_icon = badge
b.put()
userBadge=UserBadge()
userBadge.user=user
userBadge.badge=badge
userBadge.badge_property = b.key()
userBadge.user_property = user_details.key()
userBadge.put()
UserScoreUtility.updateBadgeScore(user, badge)
def getUserDetails(self, request_handler):
#google_login_url = users.create_login_url('/')
#twitter_logout_url = '/oauth/twitter/logout'
twitter_cookie = request_handler.request.cookies.get('oauth.twitter')
evernote_user = OAuthClient('evernote', request_handler)
flickr_user = OAuthClient('flickr', request_handler)
#return variables
screen_name=None
avatar=None
auth_service=None
user_details_key=None
instaright_account=None
# used to connect user details with session
evernote_username=None
flickr_username=None
google_user = users.get_current_user()
#logging.info('trying to connect with fb key %s secret %s' %( self.facebook_key, self.facebook_secret))
facebook_user = facebook.get_user_from_cookie(request_handler.request.cookies, self.facebook_key, self.facebook_secret)
facebook_access_token=None
existing_user = None
if google_user:
existing_user= UserDetails.gql('WHERE google_profile=\'%s\'' %google_user.email()).get()
existing_user_by_mail = UserDetails.gql('WHERE mail=\'%s\'' %google_user.email()).get()
if self.ud is None and existing_user is not None:
self.ud = existing_user
if self.ud.mail is None:
self.ud.mail = google_user.email()
#TODO what is both are not None and pointing to different entities
if existing_user is not None and existing_user_by_mail is not None and str(existing_user.key()) != str(existing_user_by_mail.key()):
logging.error('data inconsistancy for google user %s ' % google_user.email())
#NOTE: data consistency
if existing_user is None and existing_user_by_mail is not None:
existing_user_by_mail.google_profile = google_user.email()
existing_user_by_mail.put()
existing_user = existing_user_by_mail
if existing_user is None:
#TODO update google profile
existing_user = UserDetails()
existing_user.mail=google_user.email()
existing_user.google_profile=google_user.email()
elif existing_user.avatar is not None:
avatar = existing_user.avatar
existing_user.mail=google_user.email()
existing_user.put()
auth_service='google'
screen_name=google_user.nickname()
user_details_key=existing_user.key()
user_signup_badge = UserBadge.gql('WHERE user_property = :1 and badge = :2', existing_user.key(),'signup').get()
if user_signup_badge is None:
user_badge = UserBadge()
user_badge.user = screen_name
user_badge.badge = 'signup'
badge = Badges.gql('WHERE badge_label = :1', 'signup').get()
user_badge.badge_property = badge.key()
user_badge.user_property = existing_user.key()
user_badge.put()
instaright_account=existing_user.instaright_account
if twitter_cookie is not None and len(twitter_cookie) > 0:
try:
twitter_user = OAuthClient('twitter', request_handler)
info = twitter_user.get('/account/verify_credentials')
profile_image_url = "%s" %info['profile_image_url']
existing_user = UserDetails.gql('WHERE twitter = \'http://twitter.com/%s\'' % screen_name).get()
if existing_user is None:
logging.info('new twitter user login %s' % screen_name)
existing_user=UserDetails()
existing_user.twitter='http://twitter.com/%s' %screen_name
existing_user.avatar = profile_image_url
else:
logging.info('existing twitter user login %s' % screen_name)
if existing_user.avatar is None:
existing_user.avatar = profile_image_url
else:
avatar = existing_user.avatar
existing_user.put()
user_details_key=existing_user.key()
twitter_cookie = request_handler.request.cookies.get('oauth.twitter')
twitter_oauth = OAuthAccessToken.get_by_key_name(twitter_cookie)
if twitter_oauth is not None and existing_user is not None:
twitter_token = twitter_oauth.oauth_token
twitter_secret= twitter_oauth.oauth_token_secret
taskqueue.add(url='/util/twitter/get_friends', params={'user_details_key': str(existing_user.key()),'user_token':twitter_token, 'user_secret': twitter_secret})
if screen_name is None:
screen_name = "%s" % info['screen_name']
avatar=profile_image_url
auth_service='twitter'
logging.info('updating user score ...')
user_signup_badge = UserBadge.gql('WHERE user_property = :1 and badge = :2', existing_user.key(),'signup').get()
if user_signup_badge is None:
user_badge = UserBadge()
user_badge.user = screen_name
user_badge.badge = 'signup'
badge = Badges.gql('WHERE badge_label = :1', 'signup').get()
user_badge.badge_property = badge.key()
user_badge.user_property = existing_user.key()
user_badge.put()
instaright_account=existing_user.instaright_account
except:
e0,e = sys.exc_info()[0], sys.exc_info()[1]
logging.info('got error while using twitter oauth: %s => %s' %(e0, e))
if facebook_user:
graph = facebook.GraphAPI(facebook_user["access_token"])
try:
profile = graph.get_object("me")
profile_link=profile["link"]
profile_id=profile["id"]
friends = graph.get_connections("me", "friends")
existing_user=UserDetails.gql('WHERE facebook = \'%s\'' % profile_link).get()
if existing_user is not None:
logging.info('existing facebook logging %s' % profile_link)
existing_user.facebook=profile_link
existing_user.facebook_friends=simplejson.dumps(friends)
existing_user.facebook_profile=profile["name"]
existing_user.facebook_id=profile_id
if existing_user.avatar is None:
existing_user.avatar = avatar
else:
avatar = existing_user.avatar
existing_user.put()
else:
logging.info('new facebook logging %s' % profile_link)
existing_user=UserDetails()
existing_user.facebook=profile_link
existing_user.facebook_profile=profile["name"]
existing_user.facebook_friends=simplejson.dumps(friends)
existing_user.facebook_id=profile_id
existing_user.avatar = avatar
existing_user.put()
if screen_name is None:
screen_name = profile["name"]
auth_service='facebook'
avatar=existing_user.avatar
user_details_key=existing_user.key()
user_signup_badge = UserBadge.gql('WHERE user_property = :1 and badge = :2', existing_user.key(),'signup').get()
if user_signup_badge is None:
user_badge = UserBadge()
user_badge.user = screen_name
user_badge.badge = 'signup'
badge = Badges.gql('WHERE badge_label = :1', 'signup').get()
user_badge.badge_property = badge.key()
user_badge.user_property = existing_user.key()
user_badge.put()
instaright_account=existing_user.instaright_account
facebook_access_token=facebook_user["access_token"]
except:
e0,e = sys.exc_info()[0], sys.exc_info()[1]
logging.info('error validating token %s === more info: %s' %(e0,e))
if evernote_user.get_cookie() is not None and len(evernote_user.get_cookie()) > 0:
logging.info('evernote token active: %s' % evernote_user)
logging.info('evernote access token id: %s' % evernote_user.get_cookie())
evernote_access = evernote_user.get_cookie()
access_token = OAuthAccessToken.get_by_key_name(evernote_access)
if access_token is not None and access_token.service == 'evernote':
evernote_username = access_token.specifier
if existing_user is not None:
existing_user.evernote_profile = evernote_username
existing_user.put()
if flickr_user.get_cookie() is not None and len(flickr_user.get_cookie()) > 0:
logging.info('flickr token active: %s' % flickr_user)
logging.info('flickr access token id: %s' % flickr_user.get_cookie())
flickr_access = flickr_user.get_cookie()
access_token = OAuthAccessToken.get_by_key_name(flickr_access)
if access_token is not None and access_token.service == 'flickr':
flickr_username = access_token.specifier
if existing_user is not None:
existing_user.flickr_profile = flickr_username
existing_user.put()
log_out_cookie = request_handler.request.cookies.get('user_logged_out')
path=request_handler.request.path
logging.info('path: %s' %path)
#reset logout cookie in case of /account url
if log_out_cookie and path == '/account':
logging.info('deleting logout cookie')
expires = datetime.datetime.now()
exp_format = datetime.datetime.strftime(expires, '%a, %d-%b-%Y %H:%M:%S GMT')
request_handler.response.headers.add_header('Set-Cookie', 'user_logged_out=%s; expires=%s; path=/' %( '0', exp_format))
logging.info('user auth with %s: %s' %(auth_service, screen_name))
if screen_name is not None and screen_name not in self.skip_list:
logging.info('user %s not in skip list %s ... sending mail' %(screen_name, str(self.skip_list)))
mail.send_mail(sender='*****@*****.**', to='*****@*****.**', subject='User sign up!', html='Awesome new user(%s) signed up: %s <br>avatar <a href="%s"><img src="%s" width=20 height=20 /></a>' %(auth_service, screen_name , avatar, avatar), body='Awesome new user(%s) signed up: %s avatar %s' %(auth_service, screen_name, avatar))
user_details = {'screen_name':screen_name, 'auth_service':auth_service, 'user_details_key':user_details_key, 'avatar':avatar, 'instaright_account':instaright_account,'facebook_access_token': facebook_access_token, 'evernote_name': evernote_username, 'flickr_name': flickr_username}
logging.info('[utils] user details:%s' % user_details)
return user_details
def gather_info(self, user):
if not '@' in user:
logging.info('not email address: skipping ...')
user_detail = UserDetails()
return user_detail
user_detail = UserDetails.gql('where instapaper_account = :1' , user).get()
if user_detail is None:
user_detail = UserDetails()
if user_detail.mail is None:
logging.info('updating user details setting user mail %s' % user)
user_detail.mail = user
if user_detail.instaright_account is None:
logging.info('update instaright account for user %s' % user)
user_detail.instaright_account = user
#PREPARE for request
#time_milis = time.time() * 1000
logging.info("user encoded %s" % user)
RAPPORTIVE_LINK = "https://rapportive.com/contacts/email/%s" % user
logging.info('fetchin url:%s' %RAPPORTIVE_LINK)
#TILL here
try:
response = urlfetch.fetch(RAPPORTIVE_LINK)
except:
logging.info('error downloading rapportive info. reTRYing')
try:
response = urlfetch.fetch(RAPPORTIVE_LINK)
except:
e, e1 = sys.exc_info()[0], sys.exc_info()[1]
return user_detail
if response.status_code == 200:
json_string_response = simplejson.loads(simplejson.dumps(response.content))
json = simplejson.loads(json_string_response)
name = None
try:
name = json["contact"]["name"]
except:
logging.info("unable to gather name from rapportive")
if name is None or len(name) < 1 or unicode(name).startswith('Thanks') or unicode(name).startswith('Sorry'):
logging.info('did not find name for account %s' % user)
return user_detail
logging.info('name: %s' % unicode(name))
user_detail.name = unicode(name)
memberships = json["contact"]["memberships"]
memships = {}
for m in memberships:
service_name = m['site_name'].lower().replace(' ','_').replace('.','_')
memships[service_name]=m['profile_url']
try:
setattr(user_detail, service_name, m['profile_url'])
logging.info('service %s profile %s' % (service_name, m['profile_url']))
if service_name == 'twitter' and not user_detail.twitter_request_sent:
# send twitter request and need to put since we need key for follow
user_detail.twitter_request_sent=True
user_detail.put()
taskqueue.add(url='/util/twitter/follow/'+str(user_detail.key()), queue_name='twit-queue')
except:
logging.error('memberships error %s ::: more details %s ' % (sys.exc_info()[0], sys.exc_info()[1]))
user_detail.social_data = simplejson.dumps(memships)
occupations = json["contact"]["occupations"]
occups = []
for o in occupations:
tmp_o = o['job_title'] + " ^_^ " + o['company']
occups.append(tmp_o)
user_detail.occupations = '; '.join(occups)
logging.info('occupations %s' %user_detail.occupations)
avatar = None
try:
avatar = json["contact"]["images"][0]["url"]
logging.info('avatar %s' % avatar)
except:
logging.info('no image info cause of %s' % sys.exc_info()[0])
if avatar is not None and len(avatar) > 0:
user_detail.avatar = avatar
location = None
try:
location = json["contact"]["location"]
except:
logging.info('no location info cause of %' %sys.exc_info()[0])
if location is not None and len(location):
location_lines = location.splitlines()
user_detail.location = ' '.join(location_lines)
return user_detail
