def post(self): url=self.request.get('url',None) if url is None: logging.info('no url no recommendations') return url = url.encode('utf-8') logging.info('getting url hash %s' %url) url_hash = LinkUtils.getUrlHash(url) if url_hash is None: logging.error("can't determing url hash %s" % url) return try: l = Links.gql('WHERE url_hash = :1' , url_hash).get() if l is None: l = Links.gql('WHERE url = :1' , url).get() except: l = None if l is None: logging.info('no link saved with url %s' % url) l = Links() l.url = url l.url_hash = url_hash l.put() api_call= 'http://api.zemanta.com/services/rest/0.0/' args ={'method': 'zemanta.suggest', 'api_key': self.z_key, 'text': url, 'return_categories': 'dmoz', 'format': 'json'} args_enc = urllib.urlencode(args) json= None result=None try: result = urlfetch.fetch(url=api_call, payload=args_enc,method = urlfetch.POST, headers={'Content-Type': 'application/x-www-form-urlencoded'}) json = simplejson.loads(result.content) except: logging.info('bad json data from zemanta: %s' % result) if json is None or json['status'] != 'ok': logging.info('error while fetching recommendations') return articles = json['articles'] #TODO apply DMOZ categories categories = json['categories'] #relevant_articles = [ (c["title"], c["url"]) for c in articles if c["confidence"] > 0.01 ] relevant_articles = [ (c["title"], c["url"]) for c in articles ] l.recommendation=str(simplejson.dumps(relevant_articles[0:4])) if l.url_hash is None: l.url_hash = url_hash l.put()
def getAllData(self,url, count=0): domain = RequestUtils.getDomain(url) logging.info('from %s domain %s' %( url, domain)) url=urllib2.quote(url.encode('utf-8')) url_hash = LinkUtils.getUrlHash(url) topsy_api='http://otter.topsy.com/stats.json?url=%s' % url tweet_meme_api='http://api.tweetmeme.com/url_info.json?url=%s' %url delicious_api='http://feeds.delicious.com/v2/json/urlinfo/data?url=%s&type=json' % url digg_api='http://services.digg.com/1.0/endpoint?method=story.getAll&link=%s&type=json' %url reddit_api='http://www.reddit.com/api/info.json?url=%s' %url facebook_api='https://api.facebook.com/method/fql.query?query=select%20%20like_count,share_count%20from%20link_stat%20where%20url=%22'+url+'%22&format=json' linkedin_api='http://www.linkedin.com/cws/share-count?url=%s' % url stumble_upon_api='http://www.stumbleupon.com/services/1.01/badge.getinfo?url=%s' %url buzz_api = 'https://www.googleapis.com/buzz/v1/activities/count?alt=json&url=%s' % url alternate_api='http://api.sharedcount.com/?url=%s' %url link = None alternate_twitter_score = None alternate_buzz_score = None alternate_digg_score = None alternate_facebook_share_score = None alternate_facebook_like_score = None alternate_su_score = None alternate_linkedin_score = None try: link = Links.gql('WHERE url_hash = :1', url_hash).get() if link is None: link = Links.gql('WHERE url = :1', url).get() except BadValueError: logging.info('url property too long') if link is None: link = Links() link.domain = domain link.instapaper_count = Cast.toInt(count,0) link.url = urllib2.unquote(url).decode('utf-8') link.url_hash = LinkUtils.getUrlHash(link.url) link.redditups = 0 link.redditdowns = 0 link.tweets = 0 link.diggs = 0 link.delicious_count = 0 link.overall_score = 0 link.shared = False else: link.date_updated = datetime.datetime.now().date() link.domain = domain if link.title: link.title=link.title.strip()[:199] if link.url_hash is None: link.url_hash =url_hash #relaxation link.relaxation = 0 logging.info('trying to fetch shared count info %s' %alternate_api ) json = LinkUtils.getJsonFromApi(alternate_api) if json: try: alternate_twitter_score=Cast.toInt(json['Twitter'],0) alternate_buzz_score=Cast.toInt(json['Buzz'],0) alternate_digg_score=Cast.toInt(json['Diggs'],0) facebook_info = LinkUtils.getJsonFieldSimple(json, "Facebook") logging.info('facebook alternate info %s' % facebook_info) if type(facebook_info) is int: alternate_facebook_share_score = Cast.toInt(facebook_info, 0) elif type(facebook_info) is dict: logging.info('likes: %s' % LinkUtils.getJsonFieldSimple(facebook_info, "like_count")) logging.info('shares : %s' % LinkUtils.getJsonFieldSimple(facebook_info, "share_count")) alternate_facebook_like_score = Cast.toInt(LinkUtils.getJsonFieldSimple(facebook_info, "like_count"), 0) alternate_facebook_share_score = Cast.toInt(LinkUtils.getJsonFieldSimple(facebook_info, "share_count"), 0) logging.info('alternate fb likes %s fb share %s ' % (alternate_facebook_like_score, alternate_facebook_share_score)) alternate_su_score=Cast.toInt(json['StumbleUpon'],0) alternate_linkedin_score=Cast.toInt(json['LinkedIn'],0) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) logging.info('trying to fetch topsi info %s' %topsy_api) json = LinkUtils.getJsonFromApi(topsy_api) if json: try: link.influence_score=Cast.toInt(json['response']['influential'],0) link.all_score=Cast.toInt(json['response']['all'],0) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) logging.info('trying to fetch digg info %s' %digg_api) json =LinkUtils.getJsonFromApi(digg_api) if json: try: link.diggs =Cast.toInt(json['count'],0) logging.info('diggs %s' %link.diggs) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) elif alternate_digg_score is not None: logging.info('using alternate digg score %s' % alternate_digg_score) link.diggs = alternate_digg_score if link.diggs is not None: link.overall_score += link.diggs logging.info('trying to fetch tweet_meme info %s ' % tweet_meme_api ) json = LinkUtils.getJsonFromApi(tweet_meme_api) if json and 'story' in json: try: link.tweets=Cast.toInt(json['story']['url_count'],0) if json['story']['title'] is not None: link.title=json['story']['title'].strip()[:199] if 'excerpt' in json['story']: logging.info('getting excerpt'); link.excerpt = db.Text(unicode(json['story']['excerpt'])) logging.info('tweets %s' % link.tweets) except KeyError: link.relaxation = link.relaxation + 1 e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) elif alternate_twitter_score is not None: logging.info('using altenate twitter score %s' % alternate_twitter_score) link.tweets = alternate_twitter_score if link.tweets is not None: link.overall_score += self.tw_factor * link.tweets logging.info('trying to fetch delicious info %s ' % delicious_api) json =LinkUtils.getJsonFromApi(delicious_api) if json: try: if not link.title and json[0]['title']: link.title = json[0]['title'].strip()[:199] link.categories = db.Text(unicode(simplejson.dumps(json[0]['top_tags']))) link.delicious_count = Cast.toInt(json[0]['total_posts'],0) logging.info('delicious count %s' % link.delicious_count) if link.delicious_count is not None: link.overall_score += link.delicious_count except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) logging.info('trying to fetch reddit info %s' % reddit_api) json = LinkUtils.getJsonFromApi(reddit_api) if json and 'data' in json: try: data = [ x for x in json['data']['children']] top_upped = sorted(data, key=lambda ups: ups['data']['ups'], reverse=True) if top_upped: link.redditups = Cast.toInt(top_upped[0]['data']['ups'],0) link.redditdowns = Cast.toInt(top_upped[0]['data']['downs'],0) link.created = Cast.toInt(top_upped[0]['data']['created'],0) logging.info('reddit ups %s' % link.redditups) if link.redditups is not None: link.overall_score += link.redditups if link.redditdowns is not None: link.overall_score -= link.redditdowns except KeyError: link.relaxation = link.relaxation + 1 e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) logging.info('trying to fetch facebook info %s' %facebook_api) json = LinkUtils.getJsonFromApi(facebook_api) if json: try: link.facebook_like=Cast.toInt(json[0]['like_count'], 0) link.facebook_share=Cast.toInt(json[0]['share_count'], 0) logging.info('facebook likes %s' % link.facebook_like) logging.info('facebook share %s' % link.facebook_share) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('request: %s == more info: key error [[%s, %s]] in %s' %(facebook_api, e0, e1, json)) elif alternate_facebook_like_score is not None: logging.info('using alternate facebook like count %s' % alternate_facebook_like_score) link.facebook_like_score = alternate_facebook_like_score elif alternate_facebook_share_score is not None: logging.info('using alternate facebook share count %s' % alternate_facebook_share_score) link.facebook_share = alternate_facebook_share_score if link.facebook_like is not None: link.overall_score += self.fb_factor * link.facebook_like if link.facebook_share is not None: link.overall_score += link.facebook_share logging.info('trying to fetch stumple upon link %s' % stumble_upon_api) json = LinkUtils.getJsonFromApi(stumble_upon_api) if json: try: link.stumble_upons = Cast.toInt(json['result']['views'], 0) logging.info('stumle_score %s' % link.stumble_upons) if not link.title and json['result']['title']: link.title = json['result']['title'].strip()[:199] logging.info('settting stumble title: %s' % link.title) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('request: %s == more info: key error [[%s, %s]] in %s' %(stumble_upon_api, e0, e1, json)) elif alternate_su_score is not None: logging.info('using alternate su score %s' % alternate_su_score ) link.stumble_upons = alternate_su_score if link.stumble_upons is not None: link.overall_score += link.stumble_upons # specific from linkedin since response is in jsonp logging.info('trying to fetch linkedin upon link %s' % linkedin_api) try: dta = urllib2.urlopen(linkedin_api) res = dta.read() res = res.replace('IN.Tags.Share.handleCount(','') res = res.replace(');','') json = simplejson.loads(res) link.linkedin_share = Cast.toInt(json['count'], 0) logging.info('linked in shares %s' % link.linkedin_share) if link.linkedin_share is not None: link.overall_score += link.linkedin_share except: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('request: %s == more info: [[%s, %s]] in %s' %(linkedin_api, e0, e1, json)) if link.linkedin_share is None and alternate_linkedin_score is not None: logging.info('using alternate linkedin score %s' % alternate_linkedin_score) link.linkedin_share = alternate_linkedin_score link.overall_score += alternate_linkedin_score logging.info('trying to fetch buzz upon link %s' % buzz_api) json = LinkUtils.getJsonFromApi(buzz_api) if json: try: link.buzz_count = Cast.toInt(json['data']['counts']["%s"][0]["count"] % url, 0) logging.info('buzz share %s' % link.buzz_count) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('request: %s == more info: key error [[%s, %s]] in %s' %(stumble_upon_api, e0, e1, json)) elif alternate_buzz_score is not None: logging.info('using alternate buzz score %s' % alternate_buzz_score) link.buzz = alternate_buzz_score if link.buzz_count is not None: link.overall_score += link.buzz_count return link
def post(self): url=self.request.get('url',None) url_hash = LinkUtils.getUrlHash(url) if url is None: logging.info('no link in request. skipping') return category_api='http://access.alchemyapi.com/calls/url/URLGetCategory?apikey=%s&url=%s&outputMode=json' %(self.alchemy_key, urllib2.quote(url.encode('utf-8'))) logging.info('trying to fetch shared count info %s' %category_api) link=None language=None category=None try: link = Links.gql('WHERE url_hash = :1', url_hash).get() if link is None: link = Links.gql('WHERE url = :1', url).get() except BadValueError: logging.info('url property too long') if link is None: link = Links() else: link.date_updated = datetime.datetime.now().date() json = LinkUtils.getJsonFromApi(category_api) if json is None: logging.info('alchemy api returned no category.skipping') return try: language=json['language'] category=json['category'] score=Cast.toFloat(json['score'],0) if score is not None and score > 0.5 and category is not None: logging.info('category %s score %s' %(category, score)) cats=category.split("_") if cats is None: logging.info('no categories. exit') return memcache_key=url_hash+'_category' current_categories=memcache.get(memcache_key) merge_cat=[] if current_categories is not None: logging.info('merging with existing cats %s' %current_categories) merge_cat.extend(current_categories) merge_cat.extend(cats) else: merge_cat=cats model=None try: model=SessionModel.gql('WHERE url_hash = :1 order by date desc', url).get() if model is None: model=SessionModel.gql('WHERE url = :1 order by date desc', url).get() except BadValueError: logging.info('url too long ... %s' %url) if model is None: logging.info('model not defined ... skipping') return linkDetail=Links.gql('WHERE url_hash = :1' , url_hash).get() if linkDetail is None: linkDetail=Links.gql('WHERE url = :1' , url).get() if linkDetail is not None and linkDetail.categories is not None: logging.info('category found from link details %s' % linkDetail.categories) delic_cats=eval(linkDetail.categories) d_cats=[ c for c in delic_cats ] merge_cat.extend(d_cats) merge_cat=set(merge_cat) logging.info('caching cats %s for url %s' %(merge_cat, url)) memcache.set(memcache_key, list(set(merge_cat))[:4]) for c in merge_cat: taskqueue.add(queue_name='message-broadcast-queue', url='/category/stream', params={'category':c, 'url': url_hash}) existingLinkCat = LinkCategory.gql('WHERE url_hash = :1 and category = :2', url_hash, c).get() if existingLinkCat is None: existingLinkCat = LinkCategory.gql('WHERE url = :1 and category = :2', url, c).get() if existingLinkCat is not None: existingLinkCat.updated=datetime.datetime.now() if existingLinkCat.url_hash is None: existingLinkCat.url_hash = url_hash existingLinkCat.put() logging.info('updated exisitng url(%s) category(%s) update time %s' % (url, c, existingLinkCat.updated)) else: logging.info('new pair: url%s) category(%s) ' % (url, c)) linkCategory=LinkCategory() linkCategory.url=url linkCategory.url_hash = url_hash linkCategory.category=c if model is not None: linkCategory.model_details=model.key() linkCategory.put() if language is not None: link.language = language link.url=url link.url_hash=url_hash link.put() except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json))