def handle(self, **options): # Don't (ab)use the twitter API from dev and stage. if settings.STAGE: return """Collect new tweets about Firefox.""" with statsd.timer('customercare.tweets.time_elapsed'): t = get_twitter_api() search_options = { 'q': 'firefox OR #fxinput OR @firefoxbrasil OR #firefoxos OR @firefox_es', 'count': settings.CC_TWEETS_PERPAGE, # Items per page. 'result_type': 'recent', # Retrieve tweets by date. } # If we already have some tweets, collect nothing older than what we # have. try: latest_tweet = Tweet.latest() except Tweet.DoesNotExist: log.debug( 'No existing tweets. Retrieving %d tweets from search.' % settings.CC_TWEETS_PERPAGE) else: search_options['since_id'] = latest_tweet.tweet_id log.info('Retrieving tweets with id >= %s' % latest_tweet.tweet_id) # Retrieve Tweets results = t.search(**search_options) if len(results['statuses']) == 0: # Twitter returned 0 results. return # Drop tweets into DB for item in results['statuses']: # Apply filters to tweet before saving # Allow links in #fxinput tweets statsd.incr('customercare.tweet.collected') item = _filter_tweet(item, allow_links='#fxinput' in item['text']) if not item: continue created_date = datetime.utcfromtimestamp(calendar.timegm( rfc822.parsedate(item['created_at']))) item_lang = item['metadata'].get('iso_language_code', 'en') tweet = Tweet( tweet_id=item['id'], raw_json=json.dumps(item), locale=item_lang, created=created_date, ) try: tweet.save() statsd.incr('customercare.tweet.saved') except IntegrityError: pass
def collect_tweets(): # Don't (ab)use the twitter API from dev and stage. if settings.STAGE: return """Collect new tweets about Firefox.""" with statsd.timer('customercare.tweets.time_elapsed'): t = get_twitter_api() search_options = { 'q': ('firefox OR #fxinput OR @firefoxbrasil OR #firefoxos ' 'OR @firefox_es'), 'count': settings.CC_TWEETS_PERPAGE, # Items per page. 'result_type': 'recent', # Retrieve tweets by date. } # If we already have some tweets, collect nothing older than what we # have. try: latest_tweet = Tweet.latest() except Tweet.DoesNotExist: log.debug('No existing tweets. Retrieving %d tweets from search.' % settings.CC_TWEETS_PERPAGE) else: search_options['since_id'] = latest_tweet.tweet_id log.info('Retrieving tweets with id >= %s' % latest_tweet.tweet_id) # Retrieve Tweets results = t.search(**search_options) if len(results['statuses']) == 0: # Twitter returned 0 results. return # Drop tweets into DB for item in results['statuses']: # Apply filters to tweet before saving # Allow links in #fxinput tweets statsd.incr('customercare.tweet.collected') item = _filter_tweet(item, allow_links='#fxinput' in item['text']) if not item: continue created_date = datetime.utcfromtimestamp( calendar.timegm(rfc822.parsedate(item['created_at']))) item_lang = item['metadata'].get('iso_language_code', 'en') tweet = Tweet(tweet_id=item['id'], raw_json=json.dumps(item), locale=item_lang, created=created_date) try: tweet.save() statsd.incr('customercare.tweet.saved') except IntegrityError: pass
def handle(self, **options): # Don't (ab)use the twitter API from dev and stage. if settings.STAGE: return """Collect new tweets about Firefox.""" t = get_twitter_api() search_options = { "q": "firefox OR #fxinput OR @firefoxbrasil OR #firefoxos OR @firefox_es", "count": settings.CC_TWEETS_PERPAGE, # Items per page. "result_type": "recent", # Retrieve tweets by date. } # If we already have some tweets, collect nothing older than what we # have. try: latest_tweet = Tweet.latest() except Tweet.DoesNotExist: log.debug("No existing tweets. Retrieving %d tweets from search." % settings.CC_TWEETS_PERPAGE) else: search_options["since_id"] = latest_tweet.tweet_id log.info("Retrieving tweets with id >= %s" % latest_tweet.tweet_id) # Retrieve Tweets results = t.search(**search_options) if len(results["statuses"]) == 0: # Twitter returned 0 results. return # Drop tweets into DB for item in results["statuses"]: # Apply filters to tweet before saving # Allow links in #fxinput tweets item = _filter_tweet(item, allow_links="#fxinput" in item["text"]) if not item: continue created_date = datetime.utcfromtimestamp( calendar.timegm(email_utils.parsedate(item["created_at"]))) item_lang = item["metadata"].get("iso_language_code", "en") tweet = Tweet( tweet_id=item["id"], raw_json=json.dumps(item), locale=item_lang, created=created_date, ) try: tweet.save() except IntegrityError: pass
def process_request(self, request): request.twitter = Session.from_request(request) # If is_secure is True (should always be true except for local dev), # we will be redirected back to https and all cookies set will be # secure only. is_secure = settings.TWITTER_COOKIE_SECURE ssl_url = url(request, {"scheme": "https" if is_secure else "http"}) try: if request.GET.get("twitter_delete_auth"): request.twitter = Session() return http.HttpResponseRedirect(url(request)) except IOError: # Django throws an IOError if there's a bad read when parsing # the request data. If it throws an error, this is not a good # request and there's certainly no twitter stuff here. return if request.twitter.authed: request.twitter.api = get_twitter_api( request.twitter.key, request.twitter.secret ) return verifier = request.GET.get("oauth_verifier") if verifier: # We are completing an OAuth login request_key = request.COOKIES.get(REQUEST_KEY_NAME) request_secret = request.COOKIES.get(REQUEST_SECRET_NAME) if validate_token(request_key) and validate_token(request_secret): t = get_twitter_api(request_key, request_secret) try: tokens = t.get_authorized_tokens(verifier) except (TwythonError, TwythonAuthError): log.warning("Twython Error with verifier token") pass else: # Override path to drop query string. ssl_url = url( request, { "scheme": "https" if is_secure else "http", "path": request.path, }, ) response = http.HttpResponseRedirect(ssl_url) request.twitter.screen_name = tokens["screen_name"] Session(tokens["oauth_token"], tokens["oauth_token_secret"]).save( request, response ) return response else: # request tokens didn't validate log.warning("Twitter Oauth request tokens didn't validate") elif request.GET.get("twitter_auth_request"): # We are requesting Twitter auth t = get_twitter_api(None, None) try: auth_props = t.get_authentication_tokens(callback_url=ssl_url) except (TwythonError, TwythonAuthError): log.warning("Twython error while getting authorization " "url") else: response = http.HttpResponseRedirect(auth_props["auth_url"]) response.set_cookie( REQUEST_KEY_NAME, auth_props["oauth_token"], secure=is_secure ) response.set_cookie( REQUEST_SECRET_NAME, auth_props["oauth_token_secret"], secure=is_secure, ) return response
def process_request(self, request): request.twitter = Session.from_request(request) # If is_secure is True (should always be true except for local dev), # we will be redirected back to https and all cookies set will be # secure only. is_secure = settings.TWITTER_COOKIE_SECURE ssl_url = url(request, {'scheme': 'https' if is_secure else 'http'}) try: if request.REQUEST.get('twitter_delete_auth'): request.twitter = Session() return http.HttpResponseRedirect(url(request)) except IOError: # Django throws an IOError if there's a bad read when parsing # the request data. If it throws an error, this is not a good # request and there's certainly no twitter stuff here. return if request.twitter.authed: request.twitter.api = get_twitter_api(request.twitter.key, request.twitter.secret) return verifier = request.GET.get('oauth_verifier') if verifier: # We are completing an OAuth login request_key = request.COOKIES.get(REQUEST_KEY_NAME) request_secret = request.COOKIES.get(REQUEST_SECRET_NAME) if validate_token(request_key) and validate_token(request_secret): t = get_twitter_api(request_key, request_secret) try: tokens = t.get_authorized_tokens(verifier) except (TwythonError, TwythonAuthError): log.warning('Twython Error with verifier token') pass else: # Override path to drop query string. ssl_url = url( request, {'scheme': 'https' if is_secure else 'http', 'path': request.path}) response = http.HttpResponseRedirect(ssl_url) request.twitter.screen_name = tokens['screen_name'] Session( tokens['oauth_token'], tokens['oauth_token_secret']).save(request, response) return response else: # request tokens didn't validate log.warning("Twitter Oauth request tokens didn't validate") elif request.REQUEST.get('twitter_auth_request'): # We are requesting Twitter auth t = get_twitter_api(None, None) try: auth_props = t.get_authentication_tokens( callback_url=ssl_url) except (TwythonError, TwythonAuthError): log.warning('Twython error while getting authorization ' 'url') else: response = http.HttpResponseRedirect( auth_props['auth_url']) response.set_cookie( REQUEST_KEY_NAME, auth_props['oauth_token'], secure=is_secure) response.set_cookie( REQUEST_SECRET_NAME, auth_props['oauth_token_secret'], secure=is_secure) return response