def loadInMemory(self, file):
handle = open(file)
for l in handle:
url, count = l.split("\t")
domain = RequestUtils.getDomain(url)
if domain in self.map:
current_links = self.map[domain]
else:
current_links= []
current_links.append((url.strip(), count.strip()))
self.map[domain] = current_links
def post(self):
count = self.request.get("count", None)
url = self.request.get("url", None)
url = urllib2.unquote(url)
domain = RequestUtils.getDomain(url)
if not domain or len(domain) == 0:
self.response.out.write("not url: %s skipping!\n" % url)
return
logging.info("url %s" % url)
logging.info("count %s" % count)
lu = LinkUtils()
link = lu.getAllData(url, count)
self.update_link(url, link)
self.response.out.write("put %s \n " % url)
def post(self):
ud, args = RequestUtils.parse_request(self.request.body)
logging.info('Received args:%s' % args)
client = RequestUtils.getClient(args)
user=RequestUtils.getUser(ud,args)
url=RequestUtils.getUrl(args)
domain=RequestUtils.getDomain(url)
title = RequestUtils.getTitle(args, client)
version = RequestUtils.getVersion(args)
selection = RequestUtils.getSelection(args)
share_mode = RequestUtils.getShareMode(args)
if not RequestUtils.checkUrl(url):
logging.info('skipping since url is not good!')
self.response.out.write('{"message":"Bad url"}')
return
user_agent = self.request.headers['User-agent']
if user is None or len(user) == 0:
logging.info('skipping since user is not defined ( client %s )' % client )
from google.appengine.api import mail
if ENVIRONMENT == 'http://www.instaright.com':
mail.send_mail(sender='*****@*****.**', to='*****@*****.**', subject='Bad request!', html='Request: args = %s <br> ud: %s' %( args, ud), body='Request: args = %s <br> ud: %s' %( args, ud))
self.response.out.write('{"message":"User not defined"}')
return
logging.info('request has been made from valid user %s' % user)
try:
taskqueue.add(queue_name='link-queue', url='/article/task', params={'user': user, 'url': url, 'domain': domain, 'title': title, 'version': version,'client': client, 'selection': selection, 'user_agent': user_agent, 'share_mode': share_mode})
except TransientError:
taskqueue.add(queue_name='link-queue', url='/article/task', params={'user': user, 'url': url, 'domain': domain, 'title': title, 'version': version,'client': client, 'selection': selection, 'user_agent': user_agent, 'share_mode': share_mode})
if client == 'bookmarklet':
logging.info('bookmarklet specific response')
self.response.out.write('{"close_html":"Link successfully marked"}')
return
cachedBadge = memcache.get('badge_'+user)
logging.info('looking for badge %s' % 'badge_'+user)
if cachedBadge is not None:
logging.info('response badge %s' %cachedBadge)
self.response.out.write(cachedBadge)
else:
logging.info('no badge found, using default')
self.response.out.write('')
def post(self):
k = self.request.get('key')
key = db.Key(k)
s = SessionModel.gql('WHERE __key__ = :1', key).get()
util = LinkUtils()
long_url=util.getShortOriginalUrl(s.url)
if long_url is None:
logging.info('could not retrieve long link.skipping')
return
logging.info('expanded url: %s' % long_url)
if long_url.startswith('itms://'):
logging.info('Skipping itunes item: %s' % long_url)
return
domain = RequestUtils.getDomain(long_url)
s.short_url = s.url
s.url = long_url
s.domain = domain
s.put()
util.updateStats(s)
def subscribe_command(self, message=None):
subscriber_mail = self.user_mail(message.sender)
logging.info('subscribing user: %s' % subscriber_mail)
im_from = db.IM('xmpp', message.sender)
domain = RequestUtils.getDomain(message.arg)
if message.arg == 'all':
domain = 'all'
if not domain:
message.reply('You must provide valide domain for subscription. \'%s\' is not valid domain. Make sure that domain starts with http://' %message.arg)
return
existingSubscription = Subscription.gql('WHERE subscriber_mail = :1 and domain = :2 and active = True', subscriber_mail, domain).get()
if existingSubscription is not None:
message.reply('You have already subscribed for this domain %s. Remember?' % domain)
return
subscription = Subscription(subscriber = im_from, subscriber_mail = subscriber_mail, domain = domain, activationDate = datetime.datetime.now(), active = True, mute = False)
invite = IMInvite.gql('WHERE jid = :1 ', subscriber_mail)
subscription.put()
if invite:
invite.subscribed = True
invite.put()
message.reply('Subscription added.')
def post(self):
k=self.request.get('key',None)
if k is None:
logging.info('error key has not been specified')
return
key=db.Key(k)
if key is None:
logging.info('error not valid key')
return
s = SessionModel.gql('WHERE __key__ = :1', key).get()
logging.info('feedproxt url %s' % unicode(s.url))
util = LinkUtils()
url = util.getFeedOriginalUrl(s.url)
if url is None:
logging.info('could not fetch original url. skipping.')
return
logging.info('original url %s' % url)
domain = RequestUtils.getDomain(url)
s.domain = domain
s.feed_url=s.url
s.url=url
s.put()
util.updateStats(s)
def aggregateData(self, currentSession, upper_limit_date):
logging.info("agregate data for %s" % currentSession.date)
locationData = RequestUtils.ipResolverAPI(currentSession.ip)
if len(locationData) == 2:
logging.info("updating location data")
city = locationData[0]
countryCode = locationData[1]
# self.response.out.write('location api response:<BR> city : %s; country: %s ' % (locationData[1], locationData[3]))
logging.info("location api response: %s " % locationData)
userLocation = UserLocationModel()
userLocation.user = currentSession.instaright_account
userLocation.city = city
userLocation.countryCode = countryCode
userLocation.date = currentSession.date
userLocation.put()
# update country stats and city stats
logging.info("country update")
existingCountryStat = CountryStats.gql("WHERE countryCode = :1 ", countryCode).get()
if existingCountryStat:
# hack to avoid exception
if existingCountryStat.count is None:
existingCountryStat.count = 1
else:
existingCountryStat.count += 1
logging.info("updating count %s" % existingCountryStat.count)
existingCountryStat.dateUpdated = upper_limit_date
existingCountryStat.put()
else:
logging.info("new country")
countryStat = CountryStats()
countryStat.countryCode = countryCode
countryStat.count = 1
countryStat.count = upper_limit_date
countryStat.put()
logging.info("city update")
existingCityStat = CityStats.gql("WHERE city = :1 and countryCode = :2", city, countryCode).get()
if existingCityStat:
# hack to avoid exception
if existingCityStat.count is None:
existingCityStat.count = 1
else:
existingCityStat.count += 1
existingCityStat.dateUpdated = upper_limit_date
logging.info("updating count %s" % existingCityStat.count)
existingCityStat.put()
else:
logging.info("new city")
cityStat = CityStats()
cityStat.countryCode = countryCode
cityStat.city = city
cityStat.count = 1
cityStat.updateDate = upper_limit_date
cityStat.put()
existingLinkStat = LinkStats.gql("WHERE link = :1", currentSession.url).get()
logging.info("link stats update")
if existingLinkStat:
logging.info("new link %s" % currentSession.url)
existingLinkStat.count = existingLinkStat.count + 1
existingLinkStat.countUpdated = currentSession.date
existingLinkStat.lastUpdatedBy = currentSession.instaright_account
existingLinkStat.put()
else:
logging.info("updating link stats: %s" % currentSession.url)
linkStat = LinkStats()
linkStat.link = currentSession.url
linkStat.count = 1
linkStat.countUpdated = currentSession.date
linkStat.lastUpdatedBy = currentSession.instaright_account
linkStat.put()
# domain update
shouldUpdateSession = 0
mode = ""
if currentSession.domain is None or currentSession.domain == "":
currentSession.domain = RequestUtils.getDomain(currentSession.url)
shouldUpdateSession = 1
mode = "domain change: %s" % currentSession.domain
if currentSession.date is None or currentSession.date == "":
date = datetime.datetime.strptime("2009-11-15", "%Y-%m-%d").date()
currentSession.date = date
shouldUpdateSession = 2
mode = "date change: %s" % date
if shouldUpdateSession > 0:
logging.info("updating session mode: %s" % mode)
currentSession.put()
else:
logging.info("unchanged session")
logging.info("done data aggregation")
def get(self):
order = self.request.get('order',None)
l = Links()
self.response.headers["Content-Type"]='text/plain'
logging.info('get links')
if order and hasattr(l,order):
if order == 'diggs':
links = Links.gql('ORDER by diggs desc' ).fetch(100)
elif order == 'redditups':
links = Links.gql('ORDER by redditups desc' ).fetch(100)
elif order == 'all_score':
links = Links.gql('ORDER by all_score desc' ).fetch(100)
elif order == 'influence_score':
links = Links.gql('ORDER by influence_score desc' ).fetch(100)
elif order == 'facebook_like':
links = Links.gql('ORDER by facebook_like desc' ).fetch(100)
elif order == 'instaright_count':
links = Links.gql('ORDER by instaright_count desc' ).fetch(100)
elif order == 'overall_score':
links = Links.gql('ORDER by overall_score desc' ).fetch(100)
elif order == 'redditdowns':
links = Links.gql('ORDER by redditdowns desc' ).fetch(100)
elif order == 'tweets':
links = Links.gql('ORDER by tweets desc' ).fetch(100)
elif order == 'delicious_count':
links = Links.gql('ORDER by delicious_count desc' ).fetch(100)
else:
links = Links.gql('ORDER by overall_score desc').fetch(100)
else:
links = Links.gql('ORDER by date_added desc, overall_score desc').fetch(100)
logging.info('pre link count: %s' %len(links))
order = 'overall_score'
urls = [ (l.url, str(getattr(l,order)), str(l.date_updated)) for l in links if l.url != RequestUtils.getDomain(l.url)]
logging.info('link count: %s' %len(urls))
if order and hasattr(l,order):
template_variables = {'links' : urls }
path= os.path.join(os.path.dirname(__file__), 'templates/top_links.html')
self.response.headers["Content-type"] = "text/html"
self.response.out.write(template.render(path,template_variables))
def getAllData(self,url, count=0):
domain = RequestUtils.getDomain(url)
logging.info('from %s domain %s' %( url, domain))
url=urllib2.quote(url.encode('utf-8'))
url_hash = LinkUtils.getUrlHash(url)
topsy_api='http://otter.topsy.com/stats.json?url=%s' % url
tweet_meme_api='http://api.tweetmeme.com/url_info.json?url=%s' %url
delicious_api='http://feeds.delicious.com/v2/json/urlinfo/data?url=%s&type=json' % url
digg_api='http://services.digg.com/1.0/endpoint?method=story.getAll&link=%s&type=json' %url
reddit_api='http://www.reddit.com/api/info.json?url=%s' %url
facebook_api='https://api.facebook.com/method/fql.query?query=select%20%20like_count,share_count%20from%20link_stat%20where%20url=%22'+url+'%22&format=json'
linkedin_api='http://www.linkedin.com/cws/share-count?url=%s' % url
stumble_upon_api='http://www.stumbleupon.com/services/1.01/badge.getinfo?url=%s' %url
buzz_api = 'https://www.googleapis.com/buzz/v1/activities/count?alt=json&url=%s' % url
alternate_api='http://api.sharedcount.com/?url=%s' %url
link = None
alternate_twitter_score = None
alternate_buzz_score = None
alternate_digg_score = None
alternate_facebook_share_score = None
alternate_facebook_like_score = None
alternate_su_score = None
alternate_linkedin_score = None
try:
link = Links.gql('WHERE url_hash = :1', url_hash).get()
if link is None:
link = Links.gql('WHERE url = :1', url).get()
except BadValueError:
logging.info('url property too long')
if link is None:
link = Links()
link.domain = domain
link.instapaper_count = Cast.toInt(count,0)
link.url = urllib2.unquote(url).decode('utf-8')
link.url_hash = LinkUtils.getUrlHash(link.url)
link.redditups = 0
link.redditdowns = 0
link.tweets = 0
link.diggs = 0
link.delicious_count = 0
link.overall_score = 0
link.shared = False
else:
link.date_updated = datetime.datetime.now().date()
link.domain = domain
if link.title:
link.title=link.title.strip()[:199]
if link.url_hash is None:
link.url_hash =url_hash
#relaxation
link.relaxation = 0
logging.info('trying to fetch shared count info %s' %alternate_api )
json = LinkUtils.getJsonFromApi(alternate_api)
if json:
try:
alternate_twitter_score=Cast.toInt(json['Twitter'],0)
alternate_buzz_score=Cast.toInt(json['Buzz'],0)
alternate_digg_score=Cast.toInt(json['Diggs'],0)
facebook_info = LinkUtils.getJsonFieldSimple(json, "Facebook")
logging.info('facebook alternate info %s' % facebook_info)
if type(facebook_info) is int:
alternate_facebook_share_score = Cast.toInt(facebook_info, 0)
elif type(facebook_info) is dict:
logging.info('likes: %s' % LinkUtils.getJsonFieldSimple(facebook_info, "like_count"))
logging.info('shares : %s' % LinkUtils.getJsonFieldSimple(facebook_info, "share_count"))
alternate_facebook_like_score = Cast.toInt(LinkUtils.getJsonFieldSimple(facebook_info, "like_count"), 0)
alternate_facebook_share_score = Cast.toInt(LinkUtils.getJsonFieldSimple(facebook_info, "share_count"), 0)
logging.info('alternate fb likes %s fb share %s ' % (alternate_facebook_like_score, alternate_facebook_share_score))
alternate_su_score=Cast.toInt(json['StumbleUpon'],0)
alternate_linkedin_score=Cast.toInt(json['LinkedIn'],0)
except KeyError:
e0, e1 = sys.exc_info()[0],sys.exc_info()[1]
logging.info('key error [[%s, %s]] in %s' %(e0, e1, json))
logging.info('trying to fetch topsi info %s' %topsy_api)
json = LinkUtils.getJsonFromApi(topsy_api)
if json:
try:
link.influence_score=Cast.toInt(json['response']['influential'],0)
link.all_score=Cast.toInt(json['response']['all'],0)
except KeyError:
e0, e1 = sys.exc_info()[0],sys.exc_info()[1]
logging.info('key error [[%s, %s]] in %s' %(e0, e1, json))
logging.info('trying to fetch digg info %s' %digg_api)
json =LinkUtils.getJsonFromApi(digg_api)
if json:
try:
link.diggs =Cast.toInt(json['count'],0)
logging.info('diggs %s' %link.diggs)
except KeyError:
e0, e1 = sys.exc_info()[0],sys.exc_info()[1]
logging.info('key error [[%s, %s]] in %s' %(e0, e1, json))
elif alternate_digg_score is not None:
logging.info('using alternate digg score %s' % alternate_digg_score)
link.diggs = alternate_digg_score
if link.diggs is not None:
link.overall_score += link.diggs
logging.info('trying to fetch tweet_meme info %s ' % tweet_meme_api )
json = LinkUtils.getJsonFromApi(tweet_meme_api)
if json and 'story' in json:
try:
link.tweets=Cast.toInt(json['story']['url_count'],0)
if json['story']['title'] is not None:
link.title=json['story']['title'].strip()[:199]
if 'excerpt' in json['story']:
logging.info('getting excerpt');
link.excerpt = db.Text(unicode(json['story']['excerpt']))
logging.info('tweets %s' % link.tweets)
except KeyError:
link.relaxation = link.relaxation + 1
e0, e1 = sys.exc_info()[0],sys.exc_info()[1]
logging.info('key error [[%s, %s]] in %s' %(e0, e1, json))
elif alternate_twitter_score is not None:
logging.info('using altenate twitter score %s' % alternate_twitter_score)
link.tweets = alternate_twitter_score
if link.tweets is not None:
link.overall_score += self.tw_factor * link.tweets
logging.info('trying to fetch delicious info %s ' % delicious_api)
json =LinkUtils.getJsonFromApi(delicious_api)
if json:
try:
if not link.title and json[0]['title']:
link.title = json[0]['title'].strip()[:199]
link.categories = db.Text(unicode(simplejson.dumps(json[0]['top_tags'])))
link.delicious_count = Cast.toInt(json[0]['total_posts'],0)
logging.info('delicious count %s' % link.delicious_count)
if link.delicious_count is not None:
link.overall_score += link.delicious_count
except KeyError:
e0, e1 = sys.exc_info()[0],sys.exc_info()[1]
logging.info('key error [[%s, %s]] in %s' %(e0, e1, json))
logging.info('trying to fetch reddit info %s' % reddit_api)
json = LinkUtils.getJsonFromApi(reddit_api)
if json and 'data' in json:
try:
data = [ x for x in json['data']['children']]
top_upped = sorted(data, key=lambda ups: ups['data']['ups'], reverse=True)
if top_upped:
link.redditups = Cast.toInt(top_upped[0]['data']['ups'],0)
link.redditdowns = Cast.toInt(top_upped[0]['data']['downs'],0)
link.created = Cast.toInt(top_upped[0]['data']['created'],0)
logging.info('reddit ups %s' % link.redditups)
if link.redditups is not None:
link.overall_score += link.redditups
if link.redditdowns is not None:
link.overall_score -= link.redditdowns
except KeyError:
link.relaxation = link.relaxation + 1
e0, e1 = sys.exc_info()[0],sys.exc_info()[1]
logging.info('key error [[%s, %s]] in %s' %(e0, e1, json))
logging.info('trying to fetch facebook info %s' %facebook_api)
json = LinkUtils.getJsonFromApi(facebook_api)
if json:
try:
link.facebook_like=Cast.toInt(json[0]['like_count'], 0)
link.facebook_share=Cast.toInt(json[0]['share_count'], 0)
logging.info('facebook likes %s' % link.facebook_like)
logging.info('facebook share %s' % link.facebook_share)
except KeyError:
e0, e1 = sys.exc_info()[0],sys.exc_info()[1]
logging.info('request: %s == more info: key error [[%s, %s]] in %s' %(facebook_api, e0, e1, json))
elif alternate_facebook_like_score is not None:
logging.info('using alternate facebook like count %s' % alternate_facebook_like_score)
link.facebook_like_score = alternate_facebook_like_score
elif alternate_facebook_share_score is not None:
logging.info('using alternate facebook share count %s' % alternate_facebook_share_score)
link.facebook_share = alternate_facebook_share_score
if link.facebook_like is not None:
link.overall_score += self.fb_factor * link.facebook_like
if link.facebook_share is not None:
link.overall_score += link.facebook_share
logging.info('trying to fetch stumple upon link %s' % stumble_upon_api)
json = LinkUtils.getJsonFromApi(stumble_upon_api)
if json:
try:
link.stumble_upons = Cast.toInt(json['result']['views'], 0)
logging.info('stumle_score %s' % link.stumble_upons)
if not link.title and json['result']['title']:
link.title = json['result']['title'].strip()[:199]
logging.info('settting stumble title: %s' % link.title)
except KeyError:
e0, e1 = sys.exc_info()[0],sys.exc_info()[1]
logging.info('request: %s == more info: key error [[%s, %s]] in %s' %(stumble_upon_api, e0, e1, json))
elif alternate_su_score is not None:
logging.info('using alternate su score %s' % alternate_su_score )
link.stumble_upons = alternate_su_score
if link.stumble_upons is not None:
link.overall_score += link.stumble_upons
# specific from linkedin since response is in jsonp
logging.info('trying to fetch linkedin upon link %s' % linkedin_api)
try:
dta = urllib2.urlopen(linkedin_api)
res = dta.read()
res = res.replace('IN.Tags.Share.handleCount(','')
res = res.replace(');','')
json = simplejson.loads(res)
link.linkedin_share = Cast.toInt(json['count'], 0)
logging.info('linked in shares %s' % link.linkedin_share)
if link.linkedin_share is not None:
link.overall_score += link.linkedin_share
except:
e0, e1 = sys.exc_info()[0],sys.exc_info()[1]
logging.info('request: %s == more info: [[%s, %s]] in %s' %(linkedin_api, e0, e1, json))
if link.linkedin_share is None and alternate_linkedin_score is not None:
logging.info('using alternate linkedin score %s' % alternate_linkedin_score)
link.linkedin_share = alternate_linkedin_score
link.overall_score += alternate_linkedin_score
logging.info('trying to fetch buzz upon link %s' % buzz_api)
json = LinkUtils.getJsonFromApi(buzz_api)
if json:
try:
link.buzz_count = Cast.toInt(json['data']['counts']["%s"][0]["count"] % url, 0)
logging.info('buzz share %s' % link.buzz_count)
except KeyError:
e0, e1 = sys.exc_info()[0],sys.exc_info()[1]
logging.info('request: %s == more info: key error [[%s, %s]] in %s' %(stumble_upon_api, e0, e1, json))
elif alternate_buzz_score is not None:
logging.info('using alternate buzz score %s' % alternate_buzz_score)
link.buzz = alternate_buzz_score
if link.buzz_count is not None:
link.overall_score += link.buzz_count
return link
def post(self):
url = self.request.get("url", None)
url_hash = LinkUtils.getUrlHash(url)
user = self.request.get("user", None)
title = self.request.get("title", None)
if url is None:
logging.info("no url detected. skipping...")
return
count = 1
url = urllib2.unquote(url)
domain = RequestUtils.getDomain(url)
if not domain or len(domain) == 0:
self.response.out.write("not url: %s skipping!\n" % url)
return
if domain in self.skip_domains:
logging.info("filering out %s" % url)
return
lu = LinkUtils()
link = lu.getAllData(url, count)
logging.info("link overall score: %s" % link.overall_score)
existingLink = None
try:
existingLink = Links.gql("WHERE url_hash = :1", url_hash).get()
if existingLink is None:
existingLink = Links.gql("WHERE url = :1", url).get()
except BadValueError:
logging.info("bad value url %s" % url)
klout_score = UserUtils.getKloutScore(user, self.klout_api_key)
share_margin = self.tw_margin
if klout_score is not None:
link.overall_score = link.overall_score * int(klout_score)
logging.info("adjusted overall score %s" % link.overall_score)
share_margin = share_margin * self.klout_correction
logging.info("adjusting twit margin: %s" % share_margin)
logging.info("link score %s tweet margin %s ( existing %s )" % (link.overall_score, share_margin, existingLink))
if link.overall_score > share_margin and (existingLink is None or not existingLink.shared):
t = Twit()
t.generate_content(link, title, "")
# skip tweets is text emtpy and for root domains
if t.text is None or LinkUtils.isRootDomain(link.url):
logging.info("twit with no body. aborting")
return
execute_time = TaskUtil.execution_time()
logging.info("scheduling tweet for %s" % str(execute_time))
mail.send_mail(
sender="*****@*****.**",
to="*****@*****.**",
subject="Twit to queue!",
html="Twitt: %s <br> score: %s" % (t.text, link.overall_score),
body="Twitt: %s <br> score: %s" % (t.text[:500], link.overall_score),
)
# taskqueue.add(url='/util/twitter/twit/task', eta=execute_time, queue_name='twit-queue', params={'twit':t.text})
taskqueue.add(url="/util/twitter/twit/task", queue_name="twit-queue", params={"twit": t.text})
# update article shared status
if existingLink is not None:
existingLink.shared = True
existingLink.put()
logging.info("updated link share status")
else:
logging.info("not scheduled for tweeting")
lh = LinkHandler()
lh.update_link(url, link)
