# All elements should have less than 2 emojis
def emoji_test(element):
if tweet_has_more_than_two_emojis(element):
print("=> Emoji:", count_emoji(element['text']))
# All elements should have hashtags shorter that 12 characters
def hashtag_test(element):
if hashtag_longer_than_twelve_letters(element):
print('=> Hashtag:',
json.dumps(element['entities']['hashtags'], indent=4))
if __name__ == "__main__":
stream = Observable.from_(Twitter.get_iterable())
tweets = stream.filter(is_tweet)
# Keep only retweets
retweeted_tweets = tweets.filter(is_retweet)
# Keep only tweets in japanese
in_japanese_tweets = tweets.filter(is_japanese_tweet)
# Keep all tweets in spanish that contain a hashtag
spanish_hashtags = tweets.filter(is_spanish_tweet).filter(has_hashtags)
executor = ThreadPoolExecutor(max_workers=10)
loop = asyncio.get_event_loop()
# In our DSL, this would be `expect(stream)(test)`
def retweet_test(element):
if not original_tweet_has_less_than_50_retweets(element):
print('=> Retweet:', json.dumps(element['retweeted_status']['retweet_count'], indent=4), "retweets")
# All elements should have less than 2 emojis
def emoji_test(element):
if tweet_has_more_than_two_emojis(element):
print("=> Emoji:", count_emoji(element['text']))
# All elements should have hashtags shorter that 12 characters
def hashtag_test(element):
if hashtag_longer_than_twelve_letters(element):
print('=> Hashtag:', json.dumps(element['entities']['hashtags'], indent=4))
if __name__ == "__main__":
stream = Observable.from_(Twitter.get_iterable())
tweets = stream.filter(is_tweet)
# Keep only retweets
retweeted_tweets = tweets.filter(is_retweet)
# Keep only tweets in japanese
in_japanese_tweets = tweets.filter(is_japanese_tweet)
# Keep all tweets in spanish that contain a hashtag
spanish_hashtags = tweets.filter(is_spanish_tweet).filter(has_hashtags)
threads = [
Thread(target=process_stream, args=(in_japanese_tweets, emoji_test)),
Thread(target=process_stream, args=(spanish_hashtags, hashtag_test)),
Thread(target=process_stream, args=(retweeted_tweets, retweet_test))
See rx-stream-pacing
Two ^C become necessary to stop this script - why?
'''
from rx import Observable
import APIReaderTwitter as Twitter
try:
import json
except ImportError:
import simplejson as json
def pretty_print(element):
print(json.dumps(element, indent=4))
def is_delete(element):
return not "delete" in element
# Generate an interval sequece, firing once each second
tick = Observable.interval(1000)
# Publish an event from Twitter each tick as a minimum
# If the twitter stream is empty it will just wait for an event to come
source = Observable.from_(Twitter.get_iterable()).zip(
tick,
lambda t, _: t
)
# Print each element in intervals, waits a minimum of 1s between events
source.filter(is_delete).subscribe(pretty_print)
#!/usr/bin/env python3
'''
Filter all "delete" events from the Twitter API
'''
from rx import Observable
import APIReaderTwitter as Twitter
try:
import json
except ImportError:
import simplejson as json
# Iterable -> Observable
def Stream(iterable):
return Observable.from_(iterable)
# Any -> Boolean
def is_delete(element):
return not "delete" in element
# Any -> IO
def pretty_print(element):
print(json.dumps(element, indent=4))
stream = Stream(Twitter.get_iterable())
stream.filter(is_delete).subscribe(pretty_print)
See rx-stream-pacing
Two ^C become necessary to stop this script - why?
'''
from rx import Observable
import APIReaderTwitter as Twitter
try:
import json
except ImportError:
import simplejson as json
def pretty_print(element):
print(json.dumps(element, indent=4))
def is_delete(element):
return not "delete" in element
# Generate an interval sequece, firing once each second
tick = Observable.interval(1000)
# Publish an event from Twitter each tick as a minimum
# If the twitter stream is empty it will just wait for an event to come
source = Observable.from_(Twitter.get_iterable()).zip(tick, lambda t, _: t)
# Print each element in intervals, waits a minimum of 1s between events
source.filter(is_delete).subscribe(pretty_print)
def is_delete(element):
return not "delete" in element
def pretty_print(element):
print(json.dumps(element, indent=4))
def process_stream(stream, fn):
stream.subscribe(fn)
if __name__ == "__main__":
executor = ThreadPoolExecutor(2)
twitter_stream = rx.Observable.from_(Twitter.get_iterable())
deleted_stream = twitter_stream.filter(is_not_delete)
tweet_stream = twitter_stream.filter(is_delete)
loop = asyncio.get_event_loop()
asyncio. async (loop.run_in_executor(
executor, partial(process_stream, deleted_stream, pretty_print)))
asyncio. async (loop.run_in_executor(
executor, partial(process_stream, tweet_stream, pretty_print)))
loop.run_forever()
