def handle(self, **options):
# Don't (ab)use the twitter API from dev and stage.
if settings.STAGE:
return
"""Collect new tweets about Firefox."""
with statsd.timer('customercare.tweets.time_elapsed'):
t = get_twitter_api()
search_options = {
'q': 'firefox OR #fxinput OR @firefoxbrasil OR #firefoxos OR @firefox_es',
'count': settings.CC_TWEETS_PERPAGE, # Items per page.
'result_type': 'recent', # Retrieve tweets by date.
}
# If we already have some tweets, collect nothing older than what we
# have.
try:
latest_tweet = Tweet.latest()
except Tweet.DoesNotExist:
log.debug(
'No existing tweets. Retrieving %d tweets from search.' %
settings.CC_TWEETS_PERPAGE)
else:
search_options['since_id'] = latest_tweet.tweet_id
log.info('Retrieving tweets with id >= %s' % latest_tweet.tweet_id)
# Retrieve Tweets
results = t.search(**search_options)
if len(results['statuses']) == 0:
# Twitter returned 0 results.
return
# Drop tweets into DB
for item in results['statuses']:
# Apply filters to tweet before saving
# Allow links in #fxinput tweets
statsd.incr('customercare.tweet.collected')
item = _filter_tweet(item, allow_links='#fxinput' in item['text'])
if not item:
continue
created_date = datetime.utcfromtimestamp(calendar.timegm(
rfc822.parsedate(item['created_at'])))
item_lang = item['metadata'].get('iso_language_code', 'en')
tweet = Tweet(
tweet_id=item['id'],
raw_json=json.dumps(item),
locale=item_lang,
created=created_date,
)
try:
tweet.save()
statsd.incr('customercare.tweet.saved')
except IntegrityError:
pass
def collect_tweets():
# Don't (ab)use the twitter API from dev and stage.
if settings.STAGE:
return
"""Collect new tweets about Firefox."""
with statsd.timer('customercare.tweets.time_elapsed'):
t = Twython(settings.TWITTER_CONSUMER_KEY,
settings.TWITTER_CONSUMER_SECRET,
settings.TWITTER_ACCESS_TOKEN,
settings.TWITTER_ACCESS_TOKEN_SECRET)
search_options = {
'q': ('firefox OR #fxinput OR @firefoxbrasil OR #firefoxos '
'OR @firefox_es'),
'count': settings.CC_TWEETS_PERPAGE, # Items per page.
'result_type': 'recent', # Retrieve tweets by date.
}
# If we already have some tweets, collect nothing older than what we
# have.
try:
latest_tweet = Tweet.latest()
except Tweet.DoesNotExist:
log.debug('No existing tweets. Retrieving %d tweets from search.' %
settings.CC_TWEETS_PERPAGE)
else:
search_options['since_id'] = latest_tweet.tweet_id
log.info('Retrieving tweets with id >= %s' % latest_tweet.tweet_id)
# Retrieve Tweets
results = t.search(**search_options)
if len(results['statuses']) == 0:
# Twitter returned 0 results.
return
# Drop tweets into DB
for item in results['statuses']:
# Apply filters to tweet before saving
# Allow links in #fxinput tweets
statsd.incr('customercare.tweet.collected')
item = _filter_tweet(item,
allow_links='#fxinput' in item['text'])
if not item:
continue
created_date = datetime.utcfromtimestamp(calendar.timegm(
rfc822.parsedate(item['created_at'])))
item_lang = item['metadata'].get('iso_language_code', 'en')
tweet = Tweet(tweet_id=item['id'], raw_json=json.dumps(item),
locale=item_lang, created=created_date)
try:
tweet.save()
statsd.incr('customercare.tweet.saved')
except IntegrityError:
pass
def collect_tweets():
# Don't (ab)use the twitter API from dev and stage.
if settings.STAGE:
return
"""Collect new tweets about Firefox."""
with statsd.timer("customercare.tweets.time_elapsed"):
t = Twython(
settings.TWITTER_CONSUMER_KEY,
settings.TWITTER_CONSUMER_SECRET,
settings.TWITTER_ACCESS_TOKEN,
settings.TWITTER_ACCESS_TOKEN_SECRET,
)
search_options = {
"q": ("firefox OR #fxinput OR @firefoxbrasil OR #firefoxos " "OR @firefox_es"),
"count": settings.CC_TWEETS_PERPAGE, # Items per page.
"result_type": "recent", # Retrieve tweets by date.
}
# If we already have some tweets, collect nothing older than what we
# have.
try:
latest_tweet = Tweet.latest()
except Tweet.DoesNotExist:
log.debug("No existing tweets. Retrieving %d tweets from search." % settings.CC_TWEETS_PERPAGE)
else:
search_options["since_id"] = latest_tweet.tweet_id
log.info("Retrieving tweets with id >= %s" % latest_tweet.tweet_id)
# Retrieve Tweets
results = t.search(**search_options)
if len(results["statuses"]) == 0:
# Twitter returned 0 results.
return
# Drop tweets into DB
for item in results["statuses"]:
# Apply filters to tweet before saving
# Allow links in #fxinput tweets
statsd.incr("customercare.tweet.collected")
item = _filter_tweet(item, allow_links="#fxinput" in item["text"])
if not item:
continue
created_date = datetime.utcfromtimestamp(calendar.timegm(rfc822.parsedate(item["created_at"])))
item_lang = item["metadata"].get("iso_language_code", "en")
tweet = Tweet(tweet_id=item["id"], raw_json=json.dumps(item), locale=item_lang, created=created_date)
try:
tweet.save()
statsd.incr("customercare.tweet.saved")
except IntegrityError:
pass
def handle(self, **options):
# Don't (ab)use the twitter API from dev and stage.
if settings.STAGE:
return
"""Collect new tweets about Firefox."""
t = get_twitter_api()
search_options = {
"q":
"firefox OR #fxinput OR @firefoxbrasil OR #firefoxos OR @firefox_es",
"count": settings.CC_TWEETS_PERPAGE, # Items per page.
"result_type": "recent", # Retrieve tweets by date.
}
# If we already have some tweets, collect nothing older than what we
# have.
try:
latest_tweet = Tweet.latest()
except Tweet.DoesNotExist:
log.debug("No existing tweets. Retrieving %d tweets from search." %
settings.CC_TWEETS_PERPAGE)
else:
search_options["since_id"] = latest_tweet.tweet_id
log.info("Retrieving tweets with id >= %s" % latest_tweet.tweet_id)
# Retrieve Tweets
results = t.search(**search_options)
if len(results["statuses"]) == 0:
# Twitter returned 0 results.
return
# Drop tweets into DB
for item in results["statuses"]:
# Apply filters to tweet before saving
# Allow links in #fxinput tweets
item = _filter_tweet(item, allow_links="#fxinput" in item["text"])
if not item:
continue
created_date = datetime.utcfromtimestamp(
calendar.timegm(email_utils.parsedate(item["created_at"])))
item_lang = item["metadata"].get("iso_language_code", "en")
tweet = Tweet(
tweet_id=item["id"],
raw_json=json.dumps(item),
locale=item_lang,
created=created_date,
)
try:
tweet.save()
except IntegrityError:
pass
def test_count_replies(self):
"""Test filtering when counting tweets"""
tweet(save=True)
id = Tweet.latest().tweet_id
reply(reply_to_tweet_id=id, created=datetime.now(), save=True)
reply(reply_to_tweet_id=id, created=datetime.now(), save=True)
reply(created=datetime.now() - timedelta(days=1, minutes=1), save=True)
yesterday = datetime.now() - timedelta(days=1)
count_recent_answered = _count_answered_tweets(since=yesterday)
eq_(count_recent_answered, 1)
def test_count_replies(self):
"""Test filtering when counting tweets"""
tweet(save=True)
id = Tweet.latest().tweet_id
reply(reply_to_tweet_id=id, created=datetime.now(), save=True)
reply(reply_to_tweet_id=id, created=datetime.now(), save=True)
reply(created=datetime.now() - timedelta(days=1, minutes=1), save=True)
yesterday = datetime.now() - timedelta(days=1)
count_recent_answered = _count_answered_tweets(since=yesterday)
eq_(count_recent_answered, 1)
def test_count_replies(self):
"""Test filtering when counting tweets"""
TweetFactory()
id = Tweet.latest().tweet_id
ReplyFactory(reply_to_tweet_id=id, created=datetime.now())
ReplyFactory(reply_to_tweet_id=id, created=datetime.now())
ReplyFactory(created=datetime.now() - timedelta(days=1, minutes=1))
yesterday = datetime.now() - timedelta(days=1)
count_recent_answered = _count_answered_tweets(since=yesterday)
eq_(count_recent_answered, 1)
def test_count_replies(self):
"""Test filtering when counting tweets"""
TweetFactory()
id = Tweet.latest().tweet_id
ReplyFactory(reply_to_tweet_id=id, created=datetime.now())
ReplyFactory(reply_to_tweet_id=id, created=datetime.now())
ReplyFactory(created=datetime.now() - timedelta(days=1, minutes=1))
yesterday = datetime.now() - timedelta(days=1)
count_recent_answered = _count_answered_tweets(since=yesterday)
eq_(count_recent_answered, 1)
def tweet(**kwargs):
"""Return a Tweet with valid default values or the ones passed in.
:arg save: whether to save the Tweet before returning it
:arg text: the `text` attribute of the Tweet's raw_json
"""
global next_tweet_id
# TODO: Escape quotes and such
defaults = {
'locale':
'en',
'raw_json':
json.dumps({
'iso_language_code':
'en',
'text':
kwargs.pop('text', 'Hey #Firefox'),
'created_at':
tweet_created,
'source':
'<a href="http://www.tweetdeck.com" '
'rel="nofollow">TweetDeck</a>',
'user': {
'screen_name':
'__jimcasey__',
'profile_image_url':
'http://a1.twimg.com/profile_images/'
'1117809237/cool_cat_normal.jpg',
'profile_image_url_https':
'http://si0.twimg.com/'
'profile_images/1117809237/'
'cool_cat_normal.jpg',
},
'to_user_id':
None,
'geo':
None,
'id':
25309168521,
'metadata': {
'results_type': 'recent',
}
})
}
defaults.update(kwargs)
if 'tweet_id' not in kwargs:
defaults['tweet_id'] = next_tweet_id
next_tweet_id += 1
return Tweet(**defaults)
def collect_tweets():
# Don't (ab)use the twitter API from dev and stage.
if settings.STAGE:
return
"""Collect new tweets about Firefox."""
with statsd.timer('customercare.tweets.time_elapsed'):
t = Twython(settings.TWITTER_CONSUMER_KEY,
settings.TWITTER_CONSUMER_SECRET,
settings.TWITTER_ACCESS_TOKEN,
settings.TWITTER_ACCESS_TOKEN_SECRET)
search_options = {
'q': ('firefox OR #fxinput OR @firefoxbrasil OR #firefoxos'
' OR @firefox_es -brendan -eich -ceo -leadership -protest'
' -marriage -boycott -boycottfirefox -opposition -political'
' -gay -political -tolerance -speech -censor -censorship'
' -resign -resignation -equal -equality -intolerant'
' -intolerance -StandWithMozilla -StandWithFirefox -freedom'
' -bigot -bigots -bigoted -liberal -hypocrite -hypocrites'
' -hypocritical -harassment -beliefs -leftists -bullying'
' -bully -homophobic -homophobia'),
'count': settings.CC_TWEETS_PERPAGE, # Items per page.
'result_type': 'recent', # Retrieve tweets by date.
}
# If we already have some tweets, collect nothing older than what we
# have.
try:
latest_tweet = Tweet.latest()
except Tweet.DoesNotExist:
log.debug('No existing tweets. Retrieving %d tweets from search.' %
settings.CC_TWEETS_PERPAGE)
else:
search_options['since_id'] = latest_tweet.tweet_id
log.info('Retrieving tweets with id >= %s' % latest_tweet.tweet_id)
# Retrieve Tweets
results = t.search(**search_options)
if len(results['statuses']) == 0:
# Twitter returned 0 results.
return
# Drop tweets into DB
for item in results['statuses']:
# Apply filters to tweet before saving
# Allow links in #fxinput tweets
statsd.incr('customercare.tweet.collected')
item = _filter_tweet(item, allow_links='#fxinput' in item['text'])
if not item:
continue
created_date = datetime.utcfromtimestamp(calendar.timegm(
rfc822.parsedate(item['created_at'])))
item_lang = item['metadata'].get('iso_language_code', 'en')
tweet = Tweet(tweet_id=item['id'], raw_json=json.dumps(item),
locale=item_lang, created=created_date)
try:
tweet.save()
statsd.incr('customercare.tweet.saved')
except IntegrityError:
pass
def test_latest_does_not_exist(self):
"""latest() should throw DoesNotExist when there are no tweets."""
Tweet.latest()
def test_latest(self):
"""Test the latest() class method when there is a latest tweet."""
TweetFactory()
last = TweetFactory()
eq_(last.tweet_id, Tweet.latest().tweet_id)
def test_latest(self):
"""Test the latest() class method when there is a latest tweet."""
NUM = 2
for x in xrange(NUM):
last = tweet(save=True)
eq_(last.tweet_id, Tweet.latest().tweet_id)
def test_latest(self):
"""Test the latest() class method when there is a latest tweet."""
NUM = 2
for x in xrange(NUM):
last = tweet(save=True)
eq_(last.tweet_id, Tweet.latest().tweet_id)
