async def update_db(data): parsed = json.loads(data) print(parsed['text']) vote, coin = is_vote(parsed["text"]) if None in [vote, coin]: return print(f'New vote detected : {coin}') async with create_engine( user="******", database="deviant", host="127.0.0.1", password="******", port=5432, ) as engine: mgr = SQLModelManager.instance() mgr.database = engine user, created = await User.objects.get_or_create(id=int(parsed['id'])) if created: user.name = parsed["user"]["name"] user.screen_name = parsed["user"]["screen_name"] user.followers_count = parsed["user"]["followers_count"] user.statuses_count = parsed["user"]["statuses_count"] await user.save() new_record = Tweet() new_record.id = parsed["id"] new_record.text = parsed["text"] new_record.created_at = tdate_to_timestamp(parsed["created_at"]) new_record.user = user new_record.coin = coin.upper() await new_record.save(force_insert=True)
def add_tweet(): new_tweet = Tweet() new_tweet.text = request.form['text'] new_tweet.created_at = request.form['created_at'] db.session.add(new_tweet) db.session.commit() return tweet_schema.jsonify(new_tweet)
def create_tweet(): #import ipdb; ipdb.set_trace() tweet = Tweet() tweet.text = request.json['text'] tweet.created_at = datetime.now().time() db.session.add(tweet) db.session.commit() return tweet_schema.jsonify(tweet), 201
async def update_db(): last_retweet_from_db = await get_last_retweet_from_db() retweets_from_twitter = api.retweets(TWEET_ID, count=100) if not last_retweet_from_db: new_retweets = [ r for r in retweets_from_twitter if r.created_at > datetime(2019, 8, 31, 0, 0) ] else: new_retweets = [ r for r in retweets_from_twitter if r.created_at > last_retweet_from_db.created_at ] my_logger.debug(f"{len(new_retweets)} retweets to insert in db") async with create_engine( user="******", database="deviant", host="127.0.0.1", password="******", port=5432, ) as engine: mgr = SQLModelManager.instance() mgr.database = engine for retweet in new_retweets: user, created = await User.objects.get_or_create(id=int(retweet.user.id)) if created: my_logger.debug(f"New User created {retweet.user.screen_name}") user.name = retweet.user.name user.screen_name = retweet.user.screen_name user.followers_count = retweet.user.followers_count user.statuses_count = retweet.user.statuses_count await user.save() new_record = Tweet() new_record.id = retweet.id new_record.text = retweet.text new_record.created_at = retweet.created_at new_record.user = user new_record.retweet = True my_logger.debug(f"New retweet created {retweet.text}") await new_record.save(force_insert=True)
def get_tweets_data_for_hash_tag_from_twitter(hash_tag): """ Get tweets by hash tag data from twitter :param hash_tag: :return: """ hash_tag = HashTag.objects.get(hash_tag=hash_tag) tweets = [] new_tweets = 0 tweet_objs = [] base_url = 'https://api.twitter.com/1.1/search/tweets.json' next_url = '?q=%23'+ hash_tag.hash_tag + '&lang=en' logger.info("Getting Tweet data for hashtag: %s" % hash_tag.hash_tag) while next_url: url = base_url + next_url page_data = client.request(url) tweets += page_data.get('statuses') next_url = page_data.get('search_metadata').get('next_results') for tweet in tweets: try: tweet_obj = Tweet.objects.get(id_str=tweet.get('id_str')) except ObjectDoesNotExist: tweet_obj = Tweet() tweet_obj.id_str = tweet.get('id_str') tweet_obj.created_at = dateutil.parser.parse((tweet.get('created_at'))) tweet_obj.text = tweet.get('text') tweet_obj.hash_tag = hash_tag # get the tweet typos get_tweet_typos(tweet_obj) tweet_obj.save() new_tweets += 1 tweet_objs.append(tweet_obj) logger.info("Found %s tweets for hashtag: %s" % (len(tweet_objs), hash_tag.hash_tag)) logger.info("Found %s new tweets for hashtag: %s" % (str(new_tweets), hash_tag.hash_tag)) return tweet_objs
def update_tweets(request): twitter_auth = request.user.social_auth.filter(provider='twitter')[0] oauth_token = twitter_auth.tokens['oauth_token'] oauth_secret = twitter_auth.tokens['oauth_token_secret'] twitter = Twython(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET, oauth_token, oauth_secret) tweets = twitter.get_home_timeline(count=200) for tweet in tweets: if len(tweet['entities']['urls']) != 0: if not (Tweet.objects.filter(tweet_id=tweet['id'], user=request.user)): tweet_object = Tweet() tweet_object.tweet_id = tweet['id'] tweet_object.text = tweet['text'] tweet_object.source_url = tweet['entities']['urls'][0]['expanded_url'] tweet_object.display_url = tweet['entities']['urls'][0]['display_url'] tweet_object.host_url = urlparse.urlparse(tweet['entities']['urls'][0]['expanded_url']).hostname tweet_object.created_at = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(tweet['created_at'],'%a %b %d %H:%M:%S +0000 %Y')) tweet_object.tweeted_by = tweet['user']['screen_name'] tweet_object.user = request.user tweet_object.save()
def import_tweets(request): consumer = oauth.Consumer(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET) token = oauth.Token(settings.TWITTER_ACCESS_TOKEN, settings.TWITTER_ACCESS_TOKEN_SECRET) client = oauth.Client(consumer, token) # GET YOUR PROFILE response, content = client.request('https://api.twitter.com/1/statuses/user_timeline.json', 'GET') tweets = [] if response.status == 200: tweets = simplejson.loads(content) for tweet in tweets: if Tweet.objects.filter(tweet_id=long(tweet['id'])).count() < 1: db_tweet = Tweet() db_tweet.text = tweet['text'] db_tweet.created_at = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(tweet['created_at'],'%a %b %d %H:%M:%S +0000 %Y')) db_tweet.tweet_id = tweet['id'] db_tweet.save() # put the logging stuff here return render_to_response('templates/import_tweets.html', locals())
def storeTimeline(): while 1: try: logger.info("Starting to store timeline data (pid %s)" % os.getpid()) twitter_stream = TwitterStream(domain="userstream.twitter.com") db.init_db() for msg in twitter_stream.stream.user(following=True): if msg.get("text"): tweet = Tweet() tweet.text = msg["text"].encode("utf-8") tweet.created_at = parser.parse(msg["created_at"]).replace(tzinfo=None) if msg.get("coordinates"): tweet.lon = msg["coordinates"]["coordinates"][0] tweet.lat = msg["coordinates"]["coordinates"][1] tweet.tweet_id = msg["id"] tweet.retweet_count = msg["retweet_count"] tweet.user_id = msg["user"]["id"] db.session.add(tweet) db.session.commit() logger.error("Stream timeout or other cause for shutdown") except Exception, err: logger.error("%s, %s" % (Exception, err)) raise