def make_twitter_connection(context, twitter_pub_addr):
""" setup twitter and ZeroMQ connections. """
client = StreamClient(CONSUMER_KEY,
CONSUMER_SECRET,
ACCESS_TOKEN,
ACCESS_TOKEN_SECRET)
#setup listener for twitter
follow_syms = gen_follow_symbols()
response = client.stream.statuses.filter.post(track=follow_syms)
l_string = str(datetime.now()) + (
" Thread called twitter agent ")
logging.info(l_string)
try:
context = context
tweet_pub_socket = context.socket(zmq.PUB)
tweet_pub_socket.connect(twitter_pub_addr)
sys.stdout.write("Initialized twitter_zmq_push \n")
sys.stdout.flush()
l_string = str(datetime.now()) + (
"init twitter_zmq_push success")
logging.info(l_string)
except Exception as error:
l_string = str(datetime.now()) + (
"Failed to init twitter_zmq_push error: " + str(error))
logging.info(l_string)
process_tweets(response, tweet_pub_socket)
return
class TwitterAssistant:
from birdy.twitter import StreamClient
from birdy.twitter import UserClient
CONSUMER_KEY = os.environ['CONSUMER_KEY']
CONSUMER_SECRET = os.environ['CONSUMER_SECRET']
ACCESS_TOKEN = os.environ['ACCESS_TOKEN']
ACCESS_TOKEN_SECRET = os.environ['ACCESS_TOKEN_SECRET']
streaming_client = StreamClient(CONSUMER_KEY, CONSUMER_SECRET,
ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
rest_client = UserClient(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN,
ACCESS_TOKEN_SECRET)
def permaLink(tweetID, screenName):
return 'https://twitter.com/' + screenName + '/statuses/' + tweetID
def recentTweets(count, screenName=None):
return TwitterAssistant.streaming_client.api.statuses_user_timeline.get(
screenName=screenName, count=count).data
def rawStream(rawSearchTerms):
return TwitterAssistant.streaming_client.stream.statuses.filter.post(
track=rawSearchTerms).stream()
def getTweet(tweetID):
return TwitterAssistant.rest_client.api.statuses.show.get(
id=tweetID).data
def sendTweet(reply, text):
TwitterAssistant.rest_client.api.statuses.update.post(reply=reply,
status=text)
def get_live_data():
#twitter app COSUMER KEY,COSUMER_SECRET,ACCESS_TOKEN,ACCESS_TOKEN_SECRET
CONSUMER_KEY = 'SQBP5hqu3KFAoA8V8kIYvz0ko'
CONSUMER_SECRET = 'Do4XrWlZmz4d8thD2ZYXdcyDG0pWf1U4c8mi6ol7t8ZRBeGfhi'
ACCESS_TOKEN = '151356238-WXtfWUZphySy0Em9qizCozVJA9FVue66Pfe2u2Xo'
ACCESS_TOKEN_SECRET = 'f1Rd6IeEJDanFPcKAB6kPd09FdBbl1XrDMSvbFzPfTghb'
client = StreamClient(CONSUMER_KEY,
CONSUMER_SECRET,
ACCESS_TOKEN,
ACCESS_TOKEN_SECRET)
keywords = ['technology','india','investing','india']
while True:
try:
for keyword in keywords:
resource = client.stream.statuses.filter.post(track=keyword)
for data in resource.stream():
if 'text' in data:
tweets = TwitterData(tag=keyword, tweet_content=data['text'])
tweets.save()
else:
continue
except (ChunkedEncodingError, ConnectionError, Exception):
logger.info(Exception)
continue
def execute():
client = StreamClient(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN,
ACCESS_TOKEN_SECRET)
hashtags = ','.join(Enterprise.objects.values_list('hashtag', flat=True))
print('hashtags: ' + hashtags)
resource = client.stream.statuses.filter.post(track=hashtags)
for tweet in resource.stream():
print('new tweet {}'.format(tweet['text']))
handle_new_tweet(tweet)
class TwitterHelper:
# you're going to need https://github.com/inueni/birdy
from birdy.twitter import StreamClient
from birdy.twitter import UserClient
# the usual Twitter OAuth stuff goes here. you know the drill
CONSUMER_KEY = os.environ['BOT_CONSUMER_KEY']
CONSUMER_SECRET = os.environ['BOT_CONSUMER_SECRET']
ACCESS_TOKEN = os.environ['BOT_ACCESS_TOKEN']
ACCESS_TOKEN_SECRET = os.environ['BOT_ACCESS_TOKEN_SECRET']
streaming_client = StreamClient(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
rest_client = UserClient(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
@staticmethod
def permalink(tweet_id, screen_name):
return 'https://twitter.com/' + screen_name + '/statuses/' + tweet_id
@staticmethod
def recent_tweets(count, screen_name=None):
return TwitterHelper.rest_client.api.statuses.user_timeline.get(screen_name=screen_name, count=count).data
@staticmethod
def raw_stream(raw_search_terms):
return TwitterHelper.streaming_client.stream.statuses.filter.post(track=raw_search_terms).stream()
@staticmethod
def get_tweet(tweet_id):
return TwitterHelper.rest_client.api.statuses.show.get(id=tweet_id).data
@staticmethod
def send_tweet(in_reply_to, text):
if DEBUG:
logging.info('Debug mode is on. Skipping sending tweet "' + text + '"')
else:
TwitterHelper.rest_client.api.statuses.update.post(in_reply_to_status_id=in_reply_to, status=text)
@staticmethod
def fav_tweet(tweet_id):
if DEBUG:
logging.info('Debug mode is on. Skipping favoriting tweet ' + tweet_id)
else:
TwitterHelper.rest_client.api.favorites.create.post(id=tweet_id)
@staticmethod
def send_dm(screen_name, text):
if DEBUG:
logging.info('Debug mode is on. Skipping sending DM "' + text + '" to ' + screen_name)
else:
TwitterHelper.rest_client.api.direct_messages.new.post(screen_name=screen_name, text=text)
def setUpTwitterConnection(stream=True):
api = twitter.Api(
consumer_key='wZFb3lcFyezEJuIGUMUvtZGwJ',
consumer_secret='xBiVivqFgEUpwrDSvT0JWCIOUssgktQtpqN41tsM1q9ErnAO3s',
access_token_key='982689083188285440-wIW0kURcR9rMSlB07cDuK59Z6c23pUH',
access_token_secret='nfmjnZxTyo79t4nMqr9xoQJ5lPKnQYxMVZnmkMUWMXpk1')
streamApi = StreamClient(
'wZFb3lcFyezEJuIGUMUvtZGwJ',
'xBiVivqFgEUpwrDSvT0JWCIOUssgktQtpqN41tsM1q9ErnAO3s',
'982689083188285440-wIW0kURcR9rMSlB07cDuK59Z6c23pUH',
'nfmjnZxTyo79t4nMqr9xoQJ5lPKnQYxMVZnmkMUWMXpk1')
if stream:
return streamApi
else:
return api
def _check_twitter(self):
twitter_stream = None
while self.running:
try:
self.logger.info('checking_twitter')
if twitter_stream is None:
twitter_client = StreamClient(
os.getenv('CONSUMER_KEY'),
os.getenv('CONSUMER_SECRET'),
os.getenv('ACCESS_TOKEN'),
os.getenv('ACCESS_TOKEN_SECRET'))
twitter_resource = twitter_client.stream.statuses.filter.post(
track='twitter')
twitter_stream = twitter_resource.stream()
tweet_data = next(twitter_stream)
if tweet_data:
status = {'int': 1, 'string': 'success'}
self.logger.info('twitter_ping_status',
extra={"status": status})
else:
status = {'int': 0, 'string': 'failure'}
self.logger.error('twitter_ping_status',
extra={"status": status})
except Exception as ex:
#reset twitter so we can try to reopen it!
twitter_stream = None
status = {'int': 0, 'string': 'failure'}
self.logger.error('twitter_ping_status',
extra={
"exception": str(ex),
"ex": traceback.format_exc(),
"status": status
})
time.sleep(int(settings.HEALTHCHECK_TWITTER_SLEEP))
from time import strftime
keywordsfile = "/vagrant/keywords-v1.txt"
keywordsname = "v1"
tokenfile = "/vagrant/twitter-api-keys.yml"
logdir = "/vagrant/data/"
with open(keywordsfile) as f:
keywords = f.read().splitlines()
keywords_string = ','.join(set(keywords))
print "Tracking tweets with these keywords:", keywords_string
# Connect to Twitter
tokens = yaml.safe_load(open(tokenfile))
client = StreamClient(tokens['consumer_key'], tokens['consumer_secret'],
tokens['access_token'], tokens['access_secret'])
resource = client.stream.statuses.filter.post(track=keywords_string)
today = strftime("%Y-%m-%d")
tweetlogfilename = logdir + 'tweets-' + keywordsname + '-' + today + '.log'
try:
print "Opening ", tweetlogfilename
tweetlog = open(tweetlogfilename, "a")
except:
print "Unable to open tweet log file."
sys.exit()
print "Writing tweets to ", tweetlogfilename
for data in resource.stream():
if today != strftime("%Y-%m-%d"):
today = strftime("%Y-%m-%d")
ACCESS_TOKEN_SECRET = os.environ['TWITTER_ACCESS_TOKEN_SECRET']
TEEMILL_API_KEY = os.environ['TEEMILL_API_KEY']
teemill_url = 'https://rapanuiclothing.com/api-access-point/?api_key=' + TEEMILL_API_KEY + '&item_code=RNA1&colour=White&image_url='
count = 0
tweets = [
'\n2015: Tweeting bitmoji \n2016: Wearing bitmoji!\n',
'Your bitmoji is the cutest ever!!! I WANT A T-SHIRT LIKE THAT!!!',
'Time to kick this up a notch and become the coolest kid in school?',
"Your wishes have been heard. Here's a t-shirt for that",
"Wouldn't it be cool to have that on a t-shirt?"
]
client = StreamClient(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN,
ACCESS_TOKEN_SECRET)
response = client.stream.statuses.filter.post(track='bitmoji')
for data in response.stream():
print data['text']
print data
if 'media' in data['entities']:
image_url = data['entities']['media'][0]['media_url']
print image_url
encoded_image_url = quote(image_url)
print encoded_image_url
temp_teemill_url = teemill_url + encoded_image_url
print temp_teemill_url
product_url = urllib2.urlopen(temp_teemill_url).read()
print product_url
import datetime
from pony.orm import *
from dateutil import parser
from birdy.twitter import StreamClient
import pass_tw ## twitter credentials
import unicodedata
from unidecode import unidecode
from data_vars import follow
import re
client = StreamClient(pass_tw.CONSUMER_KEY,
pass_tw.CONSUMER_SECRET,
pass_tw.ACCESS_TOKEN,
pass_tw.ACCESS_TOKEN_SECRET)
resource = client.stream.statuses.filter.post(follow=follow)
@db_session
def save_data(json_data):
id = None
if 'id' in json_data:
id = json_data['id']
if not db_init.Tweet.exists(id=id):
chunked = chunk(20, prepared)
for c in chunked:
print 'TWITTER: writing tweets to DB'
requests = [ UpdateOne({ '_id': obj['_id']},
{ '$setOnInsert': obj }
, upsert=True) for obj in c]
collection.bulk_write(requests, ordered=False)
def get_keywords(client):
collection = client['newsfilter'].terms
words = collection.find_one({ '_id': 'twitter' })
return words.get('keywords')
if __name__ == '__main__':
twitter_client = StreamClient(consumer_token,
consumer_secret,
access_token,
access_token_secret)
client = MongoClient(
host = os.environ.get('MONGO_HOST') or None
)
keywords = get_keywords(client)
resource = twitter_client.stream.statuses.filter.post(track=keywords)
print 'TWITTER: connected to twitter. Beginning to read tweetz'
read_and_write(client, resource)
#encoding:utf-8
import json
from decimal import Decimal
from time import time
from delorean import parse
from settings import *
from birdy.twitter import UserClient, StreamClient
import boto3
stream_client = StreamClient(**TWITTER_APP)
dynamodb = boto3.resource(
'dynamodb',
region_name=AWS_REGION,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
endpoint_url=DYNAMODB_ENDPOINT,
)
table = dynamodb.Table(DYNAMODB_TABLE)
resource = stream_client.stream.statuses.filter.post(**STREAM_PARAMS)
for item in resource.stream():
if 'text' in item:
data = {
'user_id': item['user']['id_str'],
'tweet_id': item['id_str'],
'timestamp': int(parse(item['created_at']).epoch),
'text': item['text'],
def __init__(me):
me.client = StreamClient(CONSUMER_KEY, CONSUMER_SECRET,
ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
# https://developer.twitter.com/en/docs/tweets/sample-realtime/api-reference/get-statuses-sample
me.response = me.client.stream.statuses.sample.get()
# def __init__(self):
# self.client = pykafka.KafkaClient("localhost:9092")
# self.producer = self.client.topics[bytes('twitter','utf-8')].get_producer()
def on_data(self, data):
try:
print("" + data)
json_data = json.loads(data)
words = json_data['text'].split()
ls = list(filter(lambda x: x.lower().startswith('#'), words))
if (len(ls) != 0):
for word in ls:
print(word)
# self.producer.produce(bytes(word,'utf-8'))
return True
except KeyError:
print("Exceptipn")
return True
def on_error(self, status):
print(status)
return True
client = StreamClient(CONSUMER_KEY, CONSUMER_SECRET, access_token,
access_secret)
response = client.stream.statuses.filter.post(languages=['fa'], track=['با'])
for data in response.stream():
print(data)