def search_tweets(request): tweet_list = [] url = TWITTER_URL + request.GET['handle'] status_list = get_tweets(url) tweet_id = 0 for status in status_list: regex = re.compile(r'[\n]') tweet_text = regex.sub('<br>', status['text']) new_tweet = Tweet(tweet_id=tweet_id, tweet_text=tweet_text, user_name=status['user']['name'], screen_name=status['user']['screen_name']) tweet_list.append(new_tweet) tweet_id += 1 template = loader.get_template('tweet_list.html') context = RequestContext(request, { 'tweet_list': tweet_list, 'handle': request.GET['handle'], }) return HttpResponse(template.render(context))
def tweet(request): if request.method == 'POST': form = TweetForm(request.POST) if form.is_valid(): twit = Tweet() status = twit.tweet = form.cleaned_data['tweet'] twit.name = request.user.username twit.save() variables = RequestContext(request, {'status': status}) return render_to_response('tweet.html', RequestContext(request), variables) else: return render_to_response('tweet.html', {'request': RequestContext(request)}) else: tweets = [] queryset = Tweet.objects.all() for query in queryset: if request.user.username == query.name: tweets.append(query) status = '' for tweet in tweets: status = tweet.tweet form = TweetForm() form.name = request.user variables = RequestContext(request, {'form': form, 'status': status}) return render_to_response('tweet.html', RequestContext(request), variables)
def database(): search_term = "bills.com" print search_term api = Api(consumer_key='uy3Utit3PhWPmM5284sh7w', consumer_secret='zA6ps76R3ID2oRXXdUjQ7jsPgcmXD1rPsNYVSw', access_token_key='61793468-PoarjfBLEa8KxtmFvNcLwFlvSGsh03U2LyNbroXye', access_token_secret='qg6cEIa33rX725oVAqmN4CtIgG0fmuPeI7pb2Vjeyxs') results = api.GetSearch(search_term) tweets = [] for r in results: user = r.GetUser() text = r.GetText() seconds_ago = r.GetCreatedAtInSeconds() hours_ago = ConvertTime(seconds_ago) source_url = r.GetSource() screenname = user.GetScreenName() image = user.GetProfileImageUrl() tweet = {"text": text, "hours_ago":hours_ago, "screenname": screenname, "img": image, "view_tweet": source_url} tweets.append(tweet) #Add each tweet to database. t = Tweet( author= screenname, body = text, is_read = False ) db.session.add(t) for t in tweets: print t db.session.commit() return "Connected to db. Check db."
def add_or_update_user(username): """ Add or update user and their tweets else error if not a Twitter User""" try: twitter_user = TWITTER.get_user(username)#from api ## Getting picture, follower count, etc. you get that info here. db_user = (User.query.get(twitter_user.id) or User(id=twitter_user.id, name=username))#add or update #try to get the user from database first so hey does user exist already exist #so not overrides #if does not exist returns none and evaluates to the second line DB.session.add(db_user)#add to database #We want as many recent non-retweet/reply statuses as we can get # Limit is 250 tweets = twitter_user.timeline( count=250, exclude_replies=True, include_rts=False, tweet_mode='extended', since_id=db_user.newest_tweet_id)#from api #we pull tweets since the id of the latest tweet if tweets: db_user.newest_tweet_id = tweets[0].id#fails if there are no tweets, so if there are tweets do this for tweet in tweets: # Get embedding for tweet, and store in db embedding = BASILICA.embed_sentence(tweet.full_text, mode='twitter')#tweet returned by twitter db_tweet = Tweet(id=tweet.id, text=tweet.full_text[:500], embedding=embedding)#500 character limit db_user.tweets.append(db_tweet)#append tweet to user DB.session.add(db_tweet)#add tweet to database session except Exception as e: print("Error processing {}: {}".format(username, e)) raise e#exception will keep going, whoever called this will also get the exeception #so will show on the website #raise means propagate, it casts exception but keeps going else: DB.session.commit()
def load_tweets(profile, api): tweets = [] for status in tweepy.Cursor(api.user_timeline).items(): print str(status.id) tweet = Tweet(temp=str(status.id)[-9:]) tweet.text = unicode(status.text) tweet.created = timezone.make_aware(status.created_at, timezone.get_current_timezone()) tweet.is_retweet = status.retweeted tweet.retweet_count = status.retweet_count tweet.is_quote = status.is_quote_status if status.in_reply_to_status_id is not None: tweet.is_reply = True for media in status.entities.get("media", [{}]): if media.get("type", None): tweet.media_url = media.get('media_url') if 'video' in tweet.media_url: tweet.is_video = True else: tweet.is_image = True # tweets.append(tweet) tweet.save() profile.tweets.add(tweet) # profile.tweets.bulk_create(tweets) profile.save()
def parse_tweet(self, tweet) -> Tweet: t = Tweet( id=tweet.id, text=html.unescape(tweet.text), created_at=tweet.created_at, user_name=tweet.user.name, user_screen_name=tweet.user.screen_name, ) if hasattr(tweet, 'retweeted_status'): t.text = u'\u267B' + ' @' + tweet.retweeted_status.user.screen_name + ': ' + html.unescape( tweet.retweeted_status.text) if hasattr(tweet, 'quoted_status'): t.text = re.sub(r' https://t\.co/[1-9a-zA-Z]+$', r'', t.text) + "\n" t.text += u'\u267B' + ' @' + tweet.quoted_status.user.screen_name + ': ' + html.unescape( tweet.quoted_status.text) tweet.entities.urls = [] if 'extended_entities' in tweet.quoted_status: self.parse_tweet_media(t, tweet.quoted_status.extended_entities) if hasattr(tweet, 'extended_entities'): self.parse_tweet_media(t, tweet.extended_entities) elif tweet.entities.urls: t.link_url = tweet.entities.urls[0].expanded_url for url_entity in tweet.entities.urls: expanded_url = url_entity.expanded_url indices = url_entity.indices display_url = tweet.text[indices[0]:indices[1]] t.text = t.text.replace(display_url, expanded_url) return t
def extract_tweet_and_user(self, fetched_tweet): tweet = Tweet() user = None for attribute_string in dir(fetched_tweet): if not attribute_string.startswith("__"): fetched_tweet_field = getattr(fetched_tweet, attribute_string) if fetched_tweet_field == "extended_tweet": setattr(tweet, "extended_text", fetched_tweet_field["full_text"]) try: getattr(Tweet, attribute_string) setattr(tweet, attribute_string, fetched_tweet_field) except AttributeError as e: pass if isinstance(fetched_tweet_field, tweepy.User) and not user: user = User() tweet.user_id = fetched_tweet_field.id user.twitter_user_id = fetched_tweet_field.id user.name = fetched_tweet_field.name user.screen_name = fetched_tweet_field.screen_name user.statuses_count = fetched_tweet_field.statuses_count user.followers_count = fetched_tweet_field.followers_count user.friends_count = fetched_tweet_field.friends_count user.location = fetched_tweet_field.location return tweet, user
def createStreamingContext(): # Create a local StreamingContext with two working thread and batch interval of 1 second sc = SparkContext("spark://%s:7077" % MASTER_NAME, appName="GlutenTweet", pyFiles=PYFILES) ssc = StreamingContext(sc, 2) # Create a DStream of raw data raw = ssc.socketTextStream(MASTER_IP, 9999) # Convert into models tweets = raw.map(lambda r: Tweet(raw_json=r)) # Store models tweets.foreachRDD(storeTweetsRDD) # Sliding window analysis window = tweets.window(20 * 60, 30) hashtagCounts = analysisHahtagCount(window) streamTop(hashtagCounts).pprint() # Keyword extraction - note tweets is immutable tweetsKeyword = tweets.map(lambda t: keywordExtraction(t)) # Update models tweetsKeyword.foreachRDD(updateTweetsRDD) # Sliding window analysis window2 = tweetsKeyword.window(20 * 60, 30) keywordCounts = analysisKeywordCount(window2) streamTop(keywordCounts).pprint() ssc.checkpoint(CHECKPOINT_DIR) return ssc
def add_or_update_user(username): """Add or pdate a user and their tweets, or else give error""" try: twitter_user = TWITTER.get_user(username) db_user = (User.query.get(twitter_user.id) or User(id=twitter_user.id, name=username)) DB.session.add(db_user) tweets = twitter_user.timeline(count=200, exclude_replies=True, include_rts=False, tweet_mode='extended', since_id=db_user.newest_tweet_id) if tweets: db_user.newest_tweet_id = tweets[0].id for tweet in tweets: #Calculate embedding on the full tweet embedding = BASILICA.embed_sentence(tweet.full_text, model='twitter') db_tweet = Tweet(id=tweet.id, text=tweet.full_text[:300], embedding=embedding) db_user.tweets.append(db_tweet) DB.session.add(db_tweet) except Exception as e: print('Error processing {}: {}'.format(username, e)) raise e else: DB.session.commit()
def add_tweet(): new_tweet = Tweet() new_tweet.text = request.form['text'] new_tweet.created_at = request.form['created_at'] db.session.add(new_tweet) db.session.commit() return tweet_schema.jsonify(new_tweet)
def saveTweet(self, username, tweetContent, publishedDate): tweetObject = Tweet(username=username, text=tweetContent, date=publishedDate) # Creates an object to be saved in the Tweet model in the database. tweetObject.save()
def post(self): data = request.json validation_error = validate_user(data) if validation_error is not None: return validation_error user = get_user(data['user']['username']) if user is None: return jsonify(status=400, message="user does not exist") if user.password != data['user']['password']: return jsonify(status=403, message="incorrect password") text = data['text'] if (len(text) > 140): return jsonify(status=400, message="tweet too long"), 400 if (len(text) == 0): return jsonify(status=400, message="empty tweet"), 400 for word in [u"#arsenal", u"#denfølelsen"]: if word in text.lower(): abort(418) # I am a teapot tweet = Tweet(user, text) db['tweets'].append(tweet) return jsonify(tweet.serialize()), 200
async def update_db(data): parsed = json.loads(data) print(parsed['text']) vote, coin = is_vote(parsed["text"]) if None in [vote, coin]: return print(f'New vote detected : {coin}') async with create_engine( user="******", database="deviant", host="127.0.0.1", password="******", port=5432, ) as engine: mgr = SQLModelManager.instance() mgr.database = engine user, created = await User.objects.get_or_create(id=int(parsed['id'])) if created: user.name = parsed["user"]["name"] user.screen_name = parsed["user"]["screen_name"] user.followers_count = parsed["user"]["followers_count"] user.statuses_count = parsed["user"]["statuses_count"] await user.save() new_record = Tweet() new_record.id = parsed["id"] new_record.text = parsed["text"] new_record.created_at = tdate_to_timestamp(parsed["created_at"]) new_record.user = user new_record.coin = coin.upper() await new_record.save(force_insert=True)
def add_sentiment(term): results = get_sentiment_results(term) tweets = [] for tweet in results['tweets']: tweetInstance = Tweet(username=tweet['username'], content=tweet['text'], favorites=tweet['favorites'], retweets=tweet['retweets'], polarity=tweet['polarity'], normalizedSentiment=tweet['normalizedSentiment'], createdAt=tweet["createdAt"], profileImage=tweet["profileImage"], tweetId=tweet["tweetId"], name=tweet["name"], neg=tweet['neg'], neu=tweet['neu'], pos=tweet['pos']).save() tweets.append(tweetInstance) sentiment = Sentiment( term=term, averagePolarity=results['averagePolarity'], averageWeighedPolarity=results['averageWeighedPolarity'], positiveTweetsCount=results['positiveTweetsCount'], negativeTweetsCount=results['negativeTweetsCount'], neutralTweetsCount=results['neutralTweetsCount'], tweets=tweets) sentiment.save()
def add_or_update_user(username): """ Add or update user and their tweets else error if not a Twitter User""" try: twitter_user = TWITTER.get_user(username) ## Getting picture, follower count, etc. you get that info here. db_user = (User.query.get(twitter_user.id) or User(id=twitter_user.id, name=username)) DB.session.add(db_user) # Limit is 250 tweets = twitter_user.timeline(count=250, exclude_replies=True, include_rts=False, tweet_mode='extended', since_id=db_user.newest_tweet_id) if tweets: db_user.newest_tweet_id = tweets[0].id for tweet in tweets: # Get embedding for tweet, and store in db embedding = BASILICA.embed_sentence(tweet.full_text, model='twitter') db_tweet = Tweet(id=tweet.id, text=tweet.full_text[:500], embedding=embedding) db_user.tweets.append(db_tweet) DB.session.add(db_tweet) except Exception as e: print("Error processing {}: {}".format(username, e)) raise e else: DB.session.commit()
def generate_movie(): """All movies have a plot and a sideplot; all movies have two main names, a setting name, an incongruous object, and a bad_music_choice all movies have two bad sex scenes depending on length, some movies have another bad sex scene or a line of dialog """ main1, main2 = random.sample(main_names, 2) setting = random.choice(setting_names) incongruous_object = random.choice(incongruous_objects) bad_music_choice = random.choice(bad_music) all_modifiers = { 'main1': main1, 'main2': main2, 'setting': setting, 'incongruous_object': incongruous_object, 'bad_music_choice': bad_music_choice, } plot_choice = random.choice(all_plots) sideplot_choice = random.choice(all_sideplots) sex_scenes = random.sample(bad_sex_scenes, 3) extra_sex_scene = sex_scenes.pop() line_of_dialog = random.choice(dialog) if (len(plot_choice) + len(sideplot_choice)) < 10: sex_scenes.append(random.choice([extra_sex_scene, line_of_dialog])) raw_tweets = interleave(plot_choice, sideplot_choice, sex_scenes) text_tweets = [tweet.format(**all_modifiers) for tweet in raw_tweets] timestamps = generate_timestamps(len(text_tweets)) timed_text_tweets = [ " ".join([time, text]) for time, text in zip(timestamps, text_tweets) ] plot_point_tweets = [Tweet(text) for text in timed_text_tweets] title = random.choice(titles) description = random.choice(descriptions) intro_word = random.choice(['Introducing', 'Presenting']) intro_tweet_text = "{intro_word} {title}, {description}".format( intro_word=intro_word, title=title, description=description) tweets = [Tweet(intro_tweet_text)] tweets.extend(plot_point_tweets) return Movie(tweets)
def update_tweet(): new_tweet = Tweet() new_tweet.id = request.form['id'] new_tweet.text = request.form['text'] db.session.query(Tweet).filter_by(id=new_tweet.id).update( {"text": new_tweet.text}) db.session.commit() return tweet_schema.jsonify(new_tweet)
def add(request): user = current_user(request) # 创建微博 form = request.form() w = Tweet(form) w.user_id = user.id w.save() return redirect('/tweet/index?user_id={}'.format(user.id))
def create(self, status): tweet = Tweet() create = tweet.create(status) if create is True: # Count inserted tweet and store to variable self.count += 1
def getTweet(self, id): tweet = '' try: tweet =self.api.get_status(id,tweet_mode='extended') tweet = Tweet(tweet,id) except: printError() return tweet
def create_tweet(): #import ipdb; ipdb.set_trace() tweet = Tweet() tweet.text = request.json['text'] tweet.created_at = datetime.now().time() db.session.add(tweet) db.session.commit() return tweet_schema.jsonify(tweet), 201
def create_tweet(): tweet = Tweet() user = request.json.get('username') userdb = db.session.query(User).filter(User.username == user).first() tweet.created_by = userdb.id tweet.text = request.json.get('text') db.session.add(tweet) db.session.commit() return 'Created', 201
def user_timeline(self, user_id, count=200, **kwargs): timeline = self.get_d("statuses/user_timeline.json", user_id=user_id, trim_user=1, include_rts=1, include_entities=1, count=count, **kwargs) return [Tweet(t) for t in timeline]
def update_tweets(category, search_words, exclude_keyword, only_included_ids): search_params = settings.DEFAULT_SEARCH_PARAMS.copy() search_params['q'] = search_words + ' exclude:retweets' requests = twitter.get(SEARCH_API_URL, params=search_params) if requests.status_code != 200: print('!!!!!ERROR!!!!! - %d' % requests.status_code) return for params in json.loads(requests.text)['statuses']: user = User.query.filter(User.id == params['user']['id']).first() user_exists = user is not None if exclude_keyword: screen_name = params['user']['screen_name'].lower() text = params['text'].lower() if re.search(r'@[^ ]*' + exclude_keyword, text) or re.search(exclude_keyword, screen_name): if not re.search(r' [^@ ]*' + exclude_keyword, ' ' + text): continue if only_included_ids and not user_exists: continue tweet_exists = db.session.query(db.exists().where(Tweet.id == params['id'])).scalar() if 'retweeted_status' in params or params['user']['id'] in BAN_USERS or tweet_exists: continue description = '' if 'description' in params['user']: description = params['user']['description'] if not user_exists: new_user = User( id=params['user']['id'], name=params['user']['name'], screen_name=params['user']['screen_name'], description=description, profile_image_url_https=params['user']['profile_image_url_https'], tweets_count=0 ) db.session.add(new_user) tweet = Tweet( id=params['id'], category=category, user_id=params['user']['id'], user_name=params['user']['name'], user_screen_name=params['user']['screen_name'], user_description=description, user_profile_image_url_https=params['user']['profile_image_url_https'], ban=False, text=params['text'], created_at=params['created_at'], ) db.session.add(tweet)
def post_tweet(): form = TweetForm() if form.validate(): tweet = Tweet(user_id=current_user.id, text=form.text.data, date_created=datetime.now()) db.session.add(tweet) db.session.commit() return redirect(url_for('timeline')) return 'Something went wrong.'
def parse_dict(tweet_dict, event_id=None): t = tweet_dict u = t['user'] tweet = Tweet( tweet_id=t['id'], text=t['text'], created_at=format_date(t['created_at']), source=t['source'], source_url=t['source'], entities=json.dumps(t['entities']), # <- this is a dict lang=t['lang'], truncated=t['truncated'], possibly_sensitive=t.get('possibly_sensitive', False), coordinates=json.dumps(t['coordinates']), # <- dict? in_reply_to_status_id=t['in_reply_to_status_id'], in_reply_to_screen_name=t['in_reply_to_screen_name'], in_reply_to_user_id=t['in_reply_to_user_id'], favorite_count=t['favorite_count'], retweet_count=t['retweet_count'], is_headline=False, quoted_status_id=t.get('quoted_status_id', None), is_a_retweet=t.get, retweeted_status_id=t.get('retweeted_status_id', None), user_id=t['user']['id']) if event_id: event_tweet = EventTweet(tweet_id=t['id'], event_id=event_id) else: event_tweet = None if u.get('entities'): entities = json.dumps(u['entities']) else: entities = None user = User(user_id=u['id'], verified=u['verified'], name=u['name'], screen_name=u['screen_name'], created_at=format_date(u['created_at']), description=u['description'], location=u['location'], geo_enabled=u['geo_enabled'], entities=entities, lang=u['lang'], url=u['url'], followers_count=u['followers_count'], favourites_count=u['favourites_count'], listed_count=u['listed_count'], friends_count=u['friends_count'], statuses_count=u['statuses_count'], utc_offset=u['utc_offset'], time_zone=u['time_zone']) return tweet, user, event_tweet
def post(self, request , *args , **Kwargs): # print "in post method" slack_request_msg = request.data # checking if request coming from correct workspace if slack_request_msg.get('token') != VeriFication_Token : return Response(status = status.HTTP_403_FORBIDDEN) #verification challenge if slack_request_msg.get('type') == 'url_verification' : return Response(data=slack_request_msg , status = status.HTTP_200_OK) # event listening if 'event' in slack_request_msg: msg_event = slack_request_msg.get('event') # check if msg coming from slack api if msg_event.get('subtype') == 'bot_message': return Response(status=status.HTTP_200_OK) # process user's message user = msg_event.get('user') text = msg_event.get('text') channel = msg_event.get('channel') # respond with hello msg bot_text = 'Hi <@{}> :wave:'.format(user) if 'go' in text.lower(): slack_client.api_call(method='chat.postMessage', channel=channel, text=bot_text) # tweet_obj = Tweet(tweet_id=1, tweet_text="Hi Hesham", tweet_time=datetime.datetime.now()) # tweet_obj.save() # x = Tweet.objects() # for t in x : # print t.tweet_id , t.tweet_text # print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" # to get all tweets in the timeline All_Tweets = tweepy.Cursor(api.home_timeline).items() for iter in All_Tweets : newTweet = Tweet.objects(tweet_id=iter.id) # check if this tweet dosen't exist before if not newTweet : tweet_obj = Tweet(tweet_id = iter.id,tweet_text= iter.text,tweet_time = iter.created_at) tweet_obj.save() else : break return Response(status=status.HTTP_200_OK) return Response(status=status.HTTP_200_OK)
def on_success(self, data): if 'text' in data: data['fetched_timestamp'] = datetime.datetime.now() data['fresh_tweet'] = True tweet = Tweet() tweet.tweet_id = data['id_str'] tweet.tweets.append(data) tweet.save() print("saved", self.i) self.i += 1
def handle_tweet(): message = request.form["tweet"] user = User.query.filter(User.username == session["username"]).first() tweet = Tweet(user=user, text=message) db.session.add(tweet) db.session.commit() return redirect("/")
def on_data(self, data): self.count += 1 # table.insert({"text": json.loads(data)["text"]}) print data Tweet(data=json.loads(data)).save() print "#############################" if self.count == self.total: return False else: return True