async def update_db(data):
parsed = json.loads(data)
print(parsed['text'])
vote, coin = is_vote(parsed["text"])
if None in [vote, coin]:
return
print(f'New vote detected : {coin}')
async with create_engine(
user="******",
database="deviant",
host="127.0.0.1",
password="******",
port=5432,
) as engine:
mgr = SQLModelManager.instance()
mgr.database = engine
user, created = await User.objects.get_or_create(id=int(parsed['id']))
if created:
user.name = parsed["user"]["name"]
user.screen_name = parsed["user"]["screen_name"]
user.followers_count = parsed["user"]["followers_count"]
user.statuses_count = parsed["user"]["statuses_count"]
await user.save()
new_record = Tweet()
new_record.id = parsed["id"]
new_record.text = parsed["text"]
new_record.created_at = tdate_to_timestamp(parsed["created_at"])
new_record.user = user
new_record.coin = coin.upper()
await new_record.save(force_insert=True)
def add_tweet():
new_tweet = Tweet()
new_tweet.text = request.form['text']
new_tweet.created_at = request.form['created_at']
db.session.add(new_tweet)
db.session.commit()
return tweet_schema.jsonify(new_tweet)
def create_tweet():
#import ipdb; ipdb.set_trace()
tweet = Tweet()
tweet.text = request.json['text']
tweet.created_at = datetime.now().time()
db.session.add(tweet)
db.session.commit()
return tweet_schema.jsonify(tweet), 201
async def update_db():
last_retweet_from_db = await get_last_retweet_from_db()
retweets_from_twitter = api.retweets(TWEET_ID, count=100)
if not last_retweet_from_db:
new_retweets = [
r
for r in retweets_from_twitter
if r.created_at > datetime(2019, 8, 31, 0, 0)
]
else:
new_retweets = [
r
for r in retweets_from_twitter
if r.created_at > last_retweet_from_db.created_at
]
my_logger.debug(f"{len(new_retweets)} retweets to insert in db")
async with create_engine(
user="******",
database="deviant",
host="127.0.0.1",
password="******",
port=5432,
) as engine:
mgr = SQLModelManager.instance()
mgr.database = engine
for retweet in new_retweets:
user, created = await User.objects.get_or_create(id=int(retweet.user.id))
if created:
my_logger.debug(f"New User created {retweet.user.screen_name}")
user.name = retweet.user.name
user.screen_name = retweet.user.screen_name
user.followers_count = retweet.user.followers_count
user.statuses_count = retweet.user.statuses_count
await user.save()
new_record = Tweet()
new_record.id = retweet.id
new_record.text = retweet.text
new_record.created_at = retweet.created_at
new_record.user = user
new_record.retweet = True
my_logger.debug(f"New retweet created {retweet.text}")
await new_record.save(force_insert=True)
def get_tweets_data_for_hash_tag_from_twitter(hash_tag):
"""
Get tweets by hash tag data from twitter
:param hash_tag:
:return:
"""
hash_tag = HashTag.objects.get(hash_tag=hash_tag)
tweets = []
new_tweets = 0
tweet_objs = []
base_url = 'https://api.twitter.com/1.1/search/tweets.json'
next_url = '?q=%23'+ hash_tag.hash_tag + '&lang=en'
logger.info("Getting Tweet data for hashtag: %s" % hash_tag.hash_tag)
while next_url:
url = base_url + next_url
page_data = client.request(url)
tweets += page_data.get('statuses')
next_url = page_data.get('search_metadata').get('next_results')
for tweet in tweets:
try:
tweet_obj = Tweet.objects.get(id_str=tweet.get('id_str'))
except ObjectDoesNotExist:
tweet_obj = Tweet()
tweet_obj.id_str = tweet.get('id_str')
tweet_obj.created_at = dateutil.parser.parse((tweet.get('created_at')))
tweet_obj.text = tweet.get('text')
tweet_obj.hash_tag = hash_tag
# get the tweet typos
get_tweet_typos(tweet_obj)
tweet_obj.save()
new_tweets += 1
tweet_objs.append(tweet_obj)
logger.info("Found %s tweets for hashtag: %s" % (len(tweet_objs), hash_tag.hash_tag))
logger.info("Found %s new tweets for hashtag: %s" % (str(new_tweets), hash_tag.hash_tag))
return tweet_objs
def update_tweets(request):
twitter_auth = request.user.social_auth.filter(provider='twitter')[0]
oauth_token = twitter_auth.tokens['oauth_token']
oauth_secret = twitter_auth.tokens['oauth_token_secret']
twitter = Twython(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET, oauth_token, oauth_secret)
tweets = twitter.get_home_timeline(count=200)
for tweet in tweets:
if len(tweet['entities']['urls']) != 0:
if not (Tweet.objects.filter(tweet_id=tweet['id'], user=request.user)):
tweet_object = Tweet()
tweet_object.tweet_id = tweet['id']
tweet_object.text = tweet['text']
tweet_object.source_url = tweet['entities']['urls'][0]['expanded_url']
tweet_object.display_url = tweet['entities']['urls'][0]['display_url']
tweet_object.host_url = urlparse.urlparse(tweet['entities']['urls'][0]['expanded_url']).hostname
tweet_object.created_at = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(tweet['created_at'],'%a %b %d %H:%M:%S +0000 %Y'))
tweet_object.tweeted_by = tweet['user']['screen_name']
tweet_object.user = request.user
tweet_object.save()
def import_tweets(request):
consumer = oauth.Consumer(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET)
token = oauth.Token(settings.TWITTER_ACCESS_TOKEN, settings.TWITTER_ACCESS_TOKEN_SECRET)
client = oauth.Client(consumer, token)
# GET YOUR PROFILE
response, content = client.request('https://api.twitter.com/1/statuses/user_timeline.json', 'GET')
tweets = []
if response.status == 200:
tweets = simplejson.loads(content)
for tweet in tweets:
if Tweet.objects.filter(tweet_id=long(tweet['id'])).count() < 1:
db_tweet = Tweet()
db_tweet.text = tweet['text']
db_tweet.created_at = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(tweet['created_at'],'%a %b %d %H:%M:%S +0000 %Y'))
db_tweet.tweet_id = tweet['id']
db_tweet.save()
# put the logging stuff here
return render_to_response('templates/import_tweets.html', locals())
def storeTimeline():
while 1:
try:
logger.info("Starting to store timeline data (pid %s)" % os.getpid())
twitter_stream = TwitterStream(domain="userstream.twitter.com")
db.init_db()
for msg in twitter_stream.stream.user(following=True):
if msg.get("text"):
tweet = Tweet()
tweet.text = msg["text"].encode("utf-8")
tweet.created_at = parser.parse(msg["created_at"]).replace(tzinfo=None)
if msg.get("coordinates"):
tweet.lon = msg["coordinates"]["coordinates"][0]
tweet.lat = msg["coordinates"]["coordinates"][1]
tweet.tweet_id = msg["id"]
tweet.retweet_count = msg["retweet_count"]
tweet.user_id = msg["user"]["id"]
db.session.add(tweet)
db.session.commit()
logger.error("Stream timeout or other cause for shutdown")
except Exception, err:
logger.error("%s, %s" % (Exception, err))
raise
