def post(self): ud, args = RequestUtils.parse_request(self.request.body) logging.info('Received args:%s' % args) client = RequestUtils.getClient(args) user=RequestUtils.getUser(ud,args) url=RequestUtils.getUrl(args) domain=RequestUtils.getDomain(url) title = RequestUtils.getTitle(args, client) version = RequestUtils.getVersion(args) selection = RequestUtils.getSelection(args) share_mode = RequestUtils.getShareMode(args) if not RequestUtils.checkUrl(url): logging.info('skipping since url is not good!') self.response.out.write('{"message":"Bad url"}') return user_agent = self.request.headers['User-agent'] if user is None or len(user) == 0: logging.info('skipping since user is not defined ( client %s )' % client ) from google.appengine.api import mail if ENVIRONMENT == 'http://www.instaright.com': mail.send_mail(sender='*****@*****.**', to='*****@*****.**', subject='Bad request!', html='Request: args = %s <br> ud: %s' %( args, ud), body='Request: args = %s <br> ud: %s' %( args, ud)) self.response.out.write('{"message":"User not defined"}') return logging.info('request has been made from valid user %s' % user) try: taskqueue.add(queue_name='link-queue', url='/article/task', params={'user': user, 'url': url, 'domain': domain, 'title': title, 'version': version,'client': client, 'selection': selection, 'user_agent': user_agent, 'share_mode': share_mode}) except TransientError: taskqueue.add(queue_name='link-queue', url='/article/task', params={'user': user, 'url': url, 'domain': domain, 'title': title, 'version': version,'client': client, 'selection': selection, 'user_agent': user_agent, 'share_mode': share_mode}) if client == 'bookmarklet': logging.info('bookmarklet specific response') self.response.out.write('{"close_html":"Link successfully marked"}') return cachedBadge = memcache.get('badge_'+user) logging.info('looking for badge %s' % 'badge_'+user) if cachedBadge is not None: logging.info('response badge %s' %cachedBadge) self.response.out.write(cachedBadge) else: logging.info('no badge found, using default') self.response.out.write('')
def loadInMemory(self, file): handle = open(file) for l in handle: url, count = l.split("\t") domain = RequestUtils.getDomain(url) if domain in self.map: current_links = self.map[domain] else: current_links= [] current_links.append((url.strip(), count.strip())) self.map[domain] = current_links
def post(self): count = self.request.get("count", None) url = self.request.get("url", None) url = urllib2.unquote(url) domain = RequestUtils.getDomain(url) if not domain or len(domain) == 0: self.response.out.write("not url: %s skipping!\n" % url) return logging.info("url %s" % url) logging.info("count %s" % count) lu = LinkUtils() link = lu.getAllData(url, count) self.update_link(url, link) self.response.out.write("put %s \n " % url)
def post(self): k = self.request.get('key') key = db.Key(k) s = SessionModel.gql('WHERE __key__ = :1', key).get() util = LinkUtils() long_url=util.getShortOriginalUrl(s.url) if long_url is None: logging.info('could not retrieve long link.skipping') return logging.info('expanded url: %s' % long_url) if long_url.startswith('itms://'): logging.info('Skipping itunes item: %s' % long_url) return domain = RequestUtils.getDomain(long_url) s.short_url = s.url s.url = long_url s.domain = domain s.put() util.updateStats(s)
def subscribe_command(self, message=None): subscriber_mail = self.user_mail(message.sender) logging.info('subscribing user: %s' % subscriber_mail) im_from = db.IM('xmpp', message.sender) domain = RequestUtils.getDomain(message.arg) if message.arg == 'all': domain = 'all' if not domain: message.reply('You must provide valide domain for subscription. \'%s\' is not valid domain. Make sure that domain starts with http://' %message.arg) return existingSubscription = Subscription.gql('WHERE subscriber_mail = :1 and domain = :2 and active = True', subscriber_mail, domain).get() if existingSubscription is not None: message.reply('You have already subscribed for this domain %s. Remember?' % domain) return subscription = Subscription(subscriber = im_from, subscriber_mail = subscriber_mail, domain = domain, activationDate = datetime.datetime.now(), active = True, mute = False) invite = IMInvite.gql('WHERE jid = :1 ', subscriber_mail) subscription.put() if invite: invite.subscribed = True invite.put() message.reply('Subscription added.')
def post(self): k=self.request.get('key',None) if k is None: logging.info('error key has not been specified') return key=db.Key(k) if key is None: logging.info('error not valid key') return s = SessionModel.gql('WHERE __key__ = :1', key).get() logging.info('feedproxt url %s' % unicode(s.url)) util = LinkUtils() url = util.getFeedOriginalUrl(s.url) if url is None: logging.info('could not fetch original url. skipping.') return logging.info('original url %s' % url) domain = RequestUtils.getDomain(url) s.domain = domain s.feed_url=s.url s.url=url s.put() util.updateStats(s)
def aggregateData(self, currentSession, upper_limit_date): logging.info("agregate data for %s" % currentSession.date) locationData = RequestUtils.ipResolverAPI(currentSession.ip) if len(locationData) == 2: logging.info("updating location data") city = locationData[0] countryCode = locationData[1] # self.response.out.write('location api response:<BR> city : %s; country: %s ' % (locationData[1], locationData[3])) logging.info("location api response: %s " % locationData) userLocation = UserLocationModel() userLocation.user = currentSession.instaright_account userLocation.city = city userLocation.countryCode = countryCode userLocation.date = currentSession.date userLocation.put() # update country stats and city stats logging.info("country update") existingCountryStat = CountryStats.gql("WHERE countryCode = :1 ", countryCode).get() if existingCountryStat: # hack to avoid exception if existingCountryStat.count is None: existingCountryStat.count = 1 else: existingCountryStat.count += 1 logging.info("updating count %s" % existingCountryStat.count) existingCountryStat.dateUpdated = upper_limit_date existingCountryStat.put() else: logging.info("new country") countryStat = CountryStats() countryStat.countryCode = countryCode countryStat.count = 1 countryStat.count = upper_limit_date countryStat.put() logging.info("city update") existingCityStat = CityStats.gql("WHERE city = :1 and countryCode = :2", city, countryCode).get() if existingCityStat: # hack to avoid exception if existingCityStat.count is None: existingCityStat.count = 1 else: existingCityStat.count += 1 existingCityStat.dateUpdated = upper_limit_date logging.info("updating count %s" % existingCityStat.count) existingCityStat.put() else: logging.info("new city") cityStat = CityStats() cityStat.countryCode = countryCode cityStat.city = city cityStat.count = 1 cityStat.updateDate = upper_limit_date cityStat.put() existingLinkStat = LinkStats.gql("WHERE link = :1", currentSession.url).get() logging.info("link stats update") if existingLinkStat: logging.info("new link %s" % currentSession.url) existingLinkStat.count = existingLinkStat.count + 1 existingLinkStat.countUpdated = currentSession.date existingLinkStat.lastUpdatedBy = currentSession.instaright_account existingLinkStat.put() else: logging.info("updating link stats: %s" % currentSession.url) linkStat = LinkStats() linkStat.link = currentSession.url linkStat.count = 1 linkStat.countUpdated = currentSession.date linkStat.lastUpdatedBy = currentSession.instaright_account linkStat.put() # domain update shouldUpdateSession = 0 mode = "" if currentSession.domain is None or currentSession.domain == "": currentSession.domain = RequestUtils.getDomain(currentSession.url) shouldUpdateSession = 1 mode = "domain change: %s" % currentSession.domain if currentSession.date is None or currentSession.date == "": date = datetime.datetime.strptime("2009-11-15", "%Y-%m-%d").date() currentSession.date = date shouldUpdateSession = 2 mode = "date change: %s" % date if shouldUpdateSession > 0: logging.info("updating session mode: %s" % mode) currentSession.put() else: logging.info("unchanged session") logging.info("done data aggregation")
def get(self): order = self.request.get('order',None) l = Links() self.response.headers["Content-Type"]='text/plain' logging.info('get links') if order and hasattr(l,order): if order == 'diggs': links = Links.gql('ORDER by diggs desc' ).fetch(100) elif order == 'redditups': links = Links.gql('ORDER by redditups desc' ).fetch(100) elif order == 'all_score': links = Links.gql('ORDER by all_score desc' ).fetch(100) elif order == 'influence_score': links = Links.gql('ORDER by influence_score desc' ).fetch(100) elif order == 'facebook_like': links = Links.gql('ORDER by facebook_like desc' ).fetch(100) elif order == 'instaright_count': links = Links.gql('ORDER by instaright_count desc' ).fetch(100) elif order == 'overall_score': links = Links.gql('ORDER by overall_score desc' ).fetch(100) elif order == 'redditdowns': links = Links.gql('ORDER by redditdowns desc' ).fetch(100) elif order == 'tweets': links = Links.gql('ORDER by tweets desc' ).fetch(100) elif order == 'delicious_count': links = Links.gql('ORDER by delicious_count desc' ).fetch(100) else: links = Links.gql('ORDER by overall_score desc').fetch(100) else: links = Links.gql('ORDER by date_added desc, overall_score desc').fetch(100) logging.info('pre link count: %s' %len(links)) order = 'overall_score' urls = [ (l.url, str(getattr(l,order)), str(l.date_updated)) for l in links if l.url != RequestUtils.getDomain(l.url)] logging.info('link count: %s' %len(urls)) if order and hasattr(l,order): template_variables = {'links' : urls } path= os.path.join(os.path.dirname(__file__), 'templates/top_links.html') self.response.headers["Content-type"] = "text/html" self.response.out.write(template.render(path,template_variables))
def post(self): user=self.request.get('user',None) user=urllib.unquote(user) url=self.request.get('url',None) domain=self.request.get('domain',None) title=self.request.get('title',None) share_mode=self.request.get('share_mode',None) if not RequestUtils.checkUrl(url): logging.info('skipping since url is not good!') return lu = LinkUtils() link_info = lu.getLinkInfo(url) description = link_info["d"] embeded = link_info["e"] logging.info('got post title %s' % title) title_new = link_info["t"] if title is None and title_new is not None and len(title_new) > 0: title = title_new if title is None or title == 'None' or title == 'null': title=LinkUtils.getLinkTitle(url) if title is not None: title = title[:199] logging.info('final link title %s' %title) logging.info("link info desc: %s embede: %s" %( description, embeded)) version=self.request.get('version',None) client=self.request.get('client',None) selection = self.request.get('selection', None) user_agent = self.request.get('user_agent',None) UserScoreUtility.updateLinkScore(user,url) UserScoreUtility.updateDomainScore(user, domain) taskqueue.add(url='/user/badge/task', queue_name='badge-queue', params={'url':url, 'domain':domain, 'user':user, 'version': version, 'client': client}) taskqueue.add(url='/link/traction/task', queue_name='link-queue', params={'url':url, 'user': user, 'title': title}) taskqueue.add(url='/link/recommendation/task', queue_name='default', params={'url':url }) name = "url" generic_counter.increment(name) url_cnt = generic_counter.get_count(name) logging.info("total url count %s " % url_cnt) e = EncodeUtils() enbased=e.encode(url_cnt) url_encode26 = e.enbase(enbased) logging.info("url encode: %s and enbase : %s" % (enbased, url_encode26)) url_hash = LinkUtils.getUrlHash(url) today = datetime.datetime.now().date() model = SessionModel.gql('WHERE instaright_account = :1 and url_hash = :2 and date > :3', user, url_hash, today).get() new_entity=False if model is None: logging.info('did not find save dafined by: %s %s for date %s', user, url, str(today)) model = SessionModel() new_entity=True else: logging.info('existing url(key %s) updating certain params' %str(model.key())) logging.info('link: %s title: %s' %(url, title)) try: #remove for local testing model.ip = self.request.remote_addr model.instaright_account = user model.date = datetime.datetime.now() if new_entity == True: model.url = url model.url_hash = url_hash model.url_counter_id = url_cnt model.url_encode26 = url_encode26 model.title = title model.user_agent=user_agent model.domain = domain model.short_link = None model.feed_link = None model.version = version model.client = client model.selection = selection model.embeded = embeded while True: timeout_ms= 100 try: model.put() break except datastore_errors.Timeout: logging.info('model save timeout retrying in %s' % timeout_ms) time.sleep(timeout_ms) timeout_ms *= 2 logging.info('send link : url_hash %s title %s user_id %s updated %s client: %s' %(model.url_hash, model.title, str(model.key()), str(model.date), model.client)) except BadValueError, apiproxy_errors.DeadlineExceededError: e0, e1 = sys.exc_info()[0], sys.exc_info()[1] logging.error('error while saving url %s ( %s, %s)' % (url, e0, e1))
def getAllData(self,url, count=0): domain = RequestUtils.getDomain(url) logging.info('from %s domain %s' %( url, domain)) url=urllib2.quote(url.encode('utf-8')) url_hash = LinkUtils.getUrlHash(url) topsy_api='http://otter.topsy.com/stats.json?url=%s' % url tweet_meme_api='http://api.tweetmeme.com/url_info.json?url=%s' %url delicious_api='http://feeds.delicious.com/v2/json/urlinfo/data?url=%s&type=json' % url digg_api='http://services.digg.com/1.0/endpoint?method=story.getAll&link=%s&type=json' %url reddit_api='http://www.reddit.com/api/info.json?url=%s' %url facebook_api='https://api.facebook.com/method/fql.query?query=select%20%20like_count,share_count%20from%20link_stat%20where%20url=%22'+url+'%22&format=json' linkedin_api='http://www.linkedin.com/cws/share-count?url=%s' % url stumble_upon_api='http://www.stumbleupon.com/services/1.01/badge.getinfo?url=%s' %url buzz_api = 'https://www.googleapis.com/buzz/v1/activities/count?alt=json&url=%s' % url alternate_api='http://api.sharedcount.com/?url=%s' %url link = None alternate_twitter_score = None alternate_buzz_score = None alternate_digg_score = None alternate_facebook_share_score = None alternate_facebook_like_score = None alternate_su_score = None alternate_linkedin_score = None try: link = Links.gql('WHERE url_hash = :1', url_hash).get() if link is None: link = Links.gql('WHERE url = :1', url).get() except BadValueError: logging.info('url property too long') if link is None: link = Links() link.domain = domain link.instapaper_count = Cast.toInt(count,0) link.url = urllib2.unquote(url).decode('utf-8') link.url_hash = LinkUtils.getUrlHash(link.url) link.redditups = 0 link.redditdowns = 0 link.tweets = 0 link.diggs = 0 link.delicious_count = 0 link.overall_score = 0 link.shared = False else: link.date_updated = datetime.datetime.now().date() link.domain = domain if link.title: link.title=link.title.strip()[:199] if link.url_hash is None: link.url_hash =url_hash #relaxation link.relaxation = 0 logging.info('trying to fetch shared count info %s' %alternate_api ) json = LinkUtils.getJsonFromApi(alternate_api) if json: try: alternate_twitter_score=Cast.toInt(json['Twitter'],0) alternate_buzz_score=Cast.toInt(json['Buzz'],0) alternate_digg_score=Cast.toInt(json['Diggs'],0) facebook_info = LinkUtils.getJsonFieldSimple(json, "Facebook") logging.info('facebook alternate info %s' % facebook_info) if type(facebook_info) is int: alternate_facebook_share_score = Cast.toInt(facebook_info, 0) elif type(facebook_info) is dict: logging.info('likes: %s' % LinkUtils.getJsonFieldSimple(facebook_info, "like_count")) logging.info('shares : %s' % LinkUtils.getJsonFieldSimple(facebook_info, "share_count")) alternate_facebook_like_score = Cast.toInt(LinkUtils.getJsonFieldSimple(facebook_info, "like_count"), 0) alternate_facebook_share_score = Cast.toInt(LinkUtils.getJsonFieldSimple(facebook_info, "share_count"), 0) logging.info('alternate fb likes %s fb share %s ' % (alternate_facebook_like_score, alternate_facebook_share_score)) alternate_su_score=Cast.toInt(json['StumbleUpon'],0) alternate_linkedin_score=Cast.toInt(json['LinkedIn'],0) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) logging.info('trying to fetch topsi info %s' %topsy_api) json = LinkUtils.getJsonFromApi(topsy_api) if json: try: link.influence_score=Cast.toInt(json['response']['influential'],0) link.all_score=Cast.toInt(json['response']['all'],0) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) logging.info('trying to fetch digg info %s' %digg_api) json =LinkUtils.getJsonFromApi(digg_api) if json: try: link.diggs =Cast.toInt(json['count'],0) logging.info('diggs %s' %link.diggs) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) elif alternate_digg_score is not None: logging.info('using alternate digg score %s' % alternate_digg_score) link.diggs = alternate_digg_score if link.diggs is not None: link.overall_score += link.diggs logging.info('trying to fetch tweet_meme info %s ' % tweet_meme_api ) json = LinkUtils.getJsonFromApi(tweet_meme_api) if json and 'story' in json: try: link.tweets=Cast.toInt(json['story']['url_count'],0) if json['story']['title'] is not None: link.title=json['story']['title'].strip()[:199] if 'excerpt' in json['story']: logging.info('getting excerpt'); link.excerpt = db.Text(unicode(json['story']['excerpt'])) logging.info('tweets %s' % link.tweets) except KeyError: link.relaxation = link.relaxation + 1 e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) elif alternate_twitter_score is not None: logging.info('using altenate twitter score %s' % alternate_twitter_score) link.tweets = alternate_twitter_score if link.tweets is not None: link.overall_score += self.tw_factor * link.tweets logging.info('trying to fetch delicious info %s ' % delicious_api) json =LinkUtils.getJsonFromApi(delicious_api) if json: try: if not link.title and json[0]['title']: link.title = json[0]['title'].strip()[:199] link.categories = db.Text(unicode(simplejson.dumps(json[0]['top_tags']))) link.delicious_count = Cast.toInt(json[0]['total_posts'],0) logging.info('delicious count %s' % link.delicious_count) if link.delicious_count is not None: link.overall_score += link.delicious_count except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) logging.info('trying to fetch reddit info %s' % reddit_api) json = LinkUtils.getJsonFromApi(reddit_api) if json and 'data' in json: try: data = [ x for x in json['data']['children']] top_upped = sorted(data, key=lambda ups: ups['data']['ups'], reverse=True) if top_upped: link.redditups = Cast.toInt(top_upped[0]['data']['ups'],0) link.redditdowns = Cast.toInt(top_upped[0]['data']['downs'],0) link.created = Cast.toInt(top_upped[0]['data']['created'],0) logging.info('reddit ups %s' % link.redditups) if link.redditups is not None: link.overall_score += link.redditups if link.redditdowns is not None: link.overall_score -= link.redditdowns except KeyError: link.relaxation = link.relaxation + 1 e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('key error [[%s, %s]] in %s' %(e0, e1, json)) logging.info('trying to fetch facebook info %s' %facebook_api) json = LinkUtils.getJsonFromApi(facebook_api) if json: try: link.facebook_like=Cast.toInt(json[0]['like_count'], 0) link.facebook_share=Cast.toInt(json[0]['share_count'], 0) logging.info('facebook likes %s' % link.facebook_like) logging.info('facebook share %s' % link.facebook_share) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('request: %s == more info: key error [[%s, %s]] in %s' %(facebook_api, e0, e1, json)) elif alternate_facebook_like_score is not None: logging.info('using alternate facebook like count %s' % alternate_facebook_like_score) link.facebook_like_score = alternate_facebook_like_score elif alternate_facebook_share_score is not None: logging.info('using alternate facebook share count %s' % alternate_facebook_share_score) link.facebook_share = alternate_facebook_share_score if link.facebook_like is not None: link.overall_score += self.fb_factor * link.facebook_like if link.facebook_share is not None: link.overall_score += link.facebook_share logging.info('trying to fetch stumple upon link %s' % stumble_upon_api) json = LinkUtils.getJsonFromApi(stumble_upon_api) if json: try: link.stumble_upons = Cast.toInt(json['result']['views'], 0) logging.info('stumle_score %s' % link.stumble_upons) if not link.title and json['result']['title']: link.title = json['result']['title'].strip()[:199] logging.info('settting stumble title: %s' % link.title) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('request: %s == more info: key error [[%s, %s]] in %s' %(stumble_upon_api, e0, e1, json)) elif alternate_su_score is not None: logging.info('using alternate su score %s' % alternate_su_score ) link.stumble_upons = alternate_su_score if link.stumble_upons is not None: link.overall_score += link.stumble_upons # specific from linkedin since response is in jsonp logging.info('trying to fetch linkedin upon link %s' % linkedin_api) try: dta = urllib2.urlopen(linkedin_api) res = dta.read() res = res.replace('IN.Tags.Share.handleCount(','') res = res.replace(');','') json = simplejson.loads(res) link.linkedin_share = Cast.toInt(json['count'], 0) logging.info('linked in shares %s' % link.linkedin_share) if link.linkedin_share is not None: link.overall_score += link.linkedin_share except: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('request: %s == more info: [[%s, %s]] in %s' %(linkedin_api, e0, e1, json)) if link.linkedin_share is None and alternate_linkedin_score is not None: logging.info('using alternate linkedin score %s' % alternate_linkedin_score) link.linkedin_share = alternate_linkedin_score link.overall_score += alternate_linkedin_score logging.info('trying to fetch buzz upon link %s' % buzz_api) json = LinkUtils.getJsonFromApi(buzz_api) if json: try: link.buzz_count = Cast.toInt(json['data']['counts']["%s"][0]["count"] % url, 0) logging.info('buzz share %s' % link.buzz_count) except KeyError: e0, e1 = sys.exc_info()[0],sys.exc_info()[1] logging.info('request: %s == more info: key error [[%s, %s]] in %s' %(stumble_upon_api, e0, e1, json)) elif alternate_buzz_score is not None: logging.info('using alternate buzz score %s' % alternate_buzz_score) link.buzz = alternate_buzz_score if link.buzz_count is not None: link.overall_score += link.buzz_count return link
def _test_url(self): args=[u'*****@*****.**', u'file%3A%2F%2Fuk.gizmodo.com%2F5805875%2Fthe-knock%2Bdown-drag%2Bout-fight-over-the-next-generation-of-batteries', u'The%20Knock-Down%2C%20Drag-Out%20Fight%20Over%20the%20Next%20Generation%20of%20Batteries', u'0.4.0.4'] url = RequestUtils.getUrl(args) self.assertEquals(False,RequestUtils.checkUrl(url)); args=[u'*****@*****.**', u'chrome%3A%2F%2Fuk.gizmodo.com%2F5805875%2Fthe-knock%2Bdown-drag%2Bout-fight-over-the-next-generation-of-batteries', u'The%20Knock-Down%2C%20Drag-Out%20Fight%20Over%20the%20Next%20Generation%20of%20Batteries', u'0.4.0.4'] url = RequestUtils.getUrl(args) self.assertEquals(False,RequestUtils.checkUrl(url)); args=[u'*****@*****.**', u'about%3A%2F%2Fuk.gizmodo.com%2F5805875%2Fthe-knock%2Bdown-drag%2Bout-fight-over-the-next-generation-of-batteries', u'The%20Knock-Down%2C%20Drag-Out%20Fight%20Over%20the%20Next%20Generation%20of%20Batteries', u'0.4.0.4'] url = RequestUtils.getUrl(args) self.assertEquals(False,RequestUtils.checkUrl(url)); args=[u'*****@*****.**', u'ed2k%3A%2F%2Fuk.gizmodo.com%2F5805875%2Fthe-knock%2Bdown-drag%2Bout-fight-over-the-next-generation-of-batteries', u'The%20Knock-Down%2C%20Drag-Out%20Fight%20Over%20the%20Next%20Generation%20of%20Batteries', u'0.4.0.4'] url = RequestUtils.getUrl(args) self.assertEquals(False,RequestUtils.checkUrl(url)); args=[u'*****@*****.**', u'liberator%3A%2F%2Fuk.gizmodo.com%2F5805875%2Fthe-knock%2Bdown-drag%2Bout-fight-over-the-next-generation-of-batteries', u'The%20Knock-Down%2C%20Drag-Out%20Fight%20Over%20the%20Next%20Generation%20of%20Batteries', u'0.4.0.4'] url = RequestUtils.getUrl(args) self.assertEquals(False,RequestUtils.checkUrl(url)); args=[u'*****@*****.**', u'irc%3A%2F%2Fuk.gizmodo.com%2F5805875%2Fthe-knock%2Bdown-drag%2Bout-fight-over-the-next-generation-of-batteries', u'The%20Knock-Down%2C%20Drag-Out%20Fight%20Over%20the%20Next%20Generation%20of%20Batteries', u'0.4.0.4'] url = RequestUtils.getUrl(args) self.assertEquals(False,RequestUtils.checkUrl(url)); args=[u'*****@*****.**', u'chrome-extension%3A%2F%2Fuk.gizmodo.com%2F5805875%2Fthe-knock%2Bdown-drag%2Bout-fight-over-the-next-generation-of-batteries', u'The%20Knock-Down%2C%20Drag-Out%20Fight%20Over%20the%20Next%20Generation%20of%20Batteries', u'0.4.0.4'] url = RequestUtils.getUrl(args) self.assertEquals(False,RequestUtils.checkUrl(url)); args=[u'*****@*****.**', u'http%3A%2F%2Fuk.gizmodo.com%2F5805875%2Fthe-knock%2Bdown-drag%2Bout-fight-over-the-next-generation-of-batteries', u'The%20Knock-Down%2C%20Drag-Out%20Fight%20Over%20the%20Next%20Generation%20of%20Batteries', u'0.4.0.4'] url = RequestUtils.getUrl(args) self.assertEquals(True,RequestUtils.checkUrl(url));
def post(self): url = self.request.get("url", None) url_hash = LinkUtils.getUrlHash(url) user = self.request.get("user", None) title = self.request.get("title", None) if url is None: logging.info("no url detected. skipping...") return count = 1 url = urllib2.unquote(url) domain = RequestUtils.getDomain(url) if not domain or len(domain) == 0: self.response.out.write("not url: %s skipping!\n" % url) return if domain in self.skip_domains: logging.info("filering out %s" % url) return lu = LinkUtils() link = lu.getAllData(url, count) logging.info("link overall score: %s" % link.overall_score) existingLink = None try: existingLink = Links.gql("WHERE url_hash = :1", url_hash).get() if existingLink is None: existingLink = Links.gql("WHERE url = :1", url).get() except BadValueError: logging.info("bad value url %s" % url) klout_score = UserUtils.getKloutScore(user, self.klout_api_key) share_margin = self.tw_margin if klout_score is not None: link.overall_score = link.overall_score * int(klout_score) logging.info("adjusted overall score %s" % link.overall_score) share_margin = share_margin * self.klout_correction logging.info("adjusting twit margin: %s" % share_margin) logging.info("link score %s tweet margin %s ( existing %s )" % (link.overall_score, share_margin, existingLink)) if link.overall_score > share_margin and (existingLink is None or not existingLink.shared): t = Twit() t.generate_content(link, title, "") # skip tweets is text emtpy and for root domains if t.text is None or LinkUtils.isRootDomain(link.url): logging.info("twit with no body. aborting") return execute_time = TaskUtil.execution_time() logging.info("scheduling tweet for %s" % str(execute_time)) mail.send_mail( sender="*****@*****.**", to="*****@*****.**", subject="Twit to queue!", html="Twitt: %s <br> score: %s" % (t.text, link.overall_score), body="Twitt: %s <br> score: %s" % (t.text[:500], link.overall_score), ) # taskqueue.add(url='/util/twitter/twit/task', eta=execute_time, queue_name='twit-queue', params={'twit':t.text}) taskqueue.add(url="/util/twitter/twit/task", queue_name="twit-queue", params={"twit": t.text}) # update article shared status if existingLink is not None: existingLink.shared = True existingLink.put() logging.info("updated link share status") else: logging.info("not scheduled for tweeting") lh = LinkHandler() lh.update_link(url, link)