def instagram_profile(self, request, pk=None): contact = self.get_contact_by_pk(request, pk) instagram_profile = {} if contact.instagram != '': query = { 'query': { 'bool': { 'must': [{ 'term': { 'data.Username': contact.instagram } }] } } } es_instagram_profile = es.search(index='instagrams', doc_type='user', body=query) if ('hits' in es_instagram_profile and 'hits' in es_instagram_profile['hits']): if len(es_instagram_profile['hits']['hits']) > 0: instagram_profile = es_instagram_profile['hits']['hits'][ 0]['_source']['data'] return Response(instagram_profile, {})
def twitter_profile(self, request, pk=None): contact = self.get_contact_by_pk(request, pk) # Response to user twitter_profile = {} if contact.twitter != '': query = { 'query': { 'bool': { 'must': [{ 'term': { 'data.Username': contact.twitter } }] } } } es_twitter_profile = es.search(index='tweets', doc_type='user', body=query) if ('hits' in es_twitter_profile and 'hits' in es_twitter_profile['hits']): if len(es_twitter_profile['hits']['hits']) > 0: twitter_profile = es_twitter_profile['hits']['hits'][0][ '_source']['data'] return Response(twitter_profile, {})
def headlines(self, request, pk=None): contact = self.get_contact_by_pk(request, pk) feeds = Feed.objects.filter(contact=contact) # Response to user headlines = [] total_headlines = 0 if len(feeds) > 0: limit, offset = get_pagination(request) query = { 'size': limit, 'from': offset, 'query': { 'bool': { 'should': [] } }, 'sort': [{ 'data.PublishDate': { 'order': 'desc', 'mode': 'avg' } }] } for feed in feeds: if feed.feed_url != '': query['query']['bool']['should'].append( {'match': { 'data.FeedURL': feed.feed_url }}) es_headlines = es.search(index='headlines', doc_type='headline', body=query) if 'hits' in es_headlines and 'total' in es_headlines['hits']: total_headlines = es_headlines['hits']['total'] if 'hits' in es_headlines and 'hits' in es_headlines['hits']: for es_headline in es_headlines['hits']['hits']: if ('_source' in es_headline and 'data' in es_headline['_source']): es_headline = format_es_response(es_headline) headlines.append(es_headline['_source']['data']) return BulkResponse(headlines, {}, len(headlines), total_headlines)
def tweets(self, request, pk=None): media_list = self.get_media_list_by_pk(request, pk) total_tweets = 0 tweets = [] if media_list.contacts.count() > 0: twitter_usernames = (media_list.contacts and media_list.contacts.values_list( 'twitter', flat=True)) if len(twitter_usernames) > 0: query = { 'size': 20, 'from': 0, 'query': { 'bool': { 'should': [], 'minimum_should_match': '100%' } }, 'sort': [{ 'data.CreatedAt': { 'order': 'desc', 'mode': 'avg' } }], } for username in twitter_usernames: query['query']['bool']['should'].append( {'term': { 'data.Username': username }}) es_tweets = es.search(index='tweets', doc_type='tweet', body=query) if 'hits' in es_tweets and 'total' in es_tweets['hits']: total_tweets = es_tweets['hits']['total'] if 'hits' in es_tweets and 'hits' in es_tweets['hits']: for es_tweet in es_tweets['hits']['hits']: if ('_source' in es_tweet and 'data' in es_tweet['_source']): tweets.append(es_tweet['_source']['data']) return BulkResponse(tweets, {}, len(tweets), total_tweets)
def headlines(self, request, pk=None): media_list = self.get_media_list_by_pk(request, pk) contacts = media_list.contacts.all() # Response to user headlines = [] total_headlines = 0 should = [] for contact in contacts: feeds = Feed.objects.filter(contact=contact, valid_feed=True, running=True) for feed in feeds: if feed.feed_url != '': should.append({'match': {'data.FeedURL': feed.feed_url}}) if len(should) > 0: query = { 'size': 20, 'from': 0, 'query': { 'bool': { 'should': should } }, 'sort': [{ 'data.PublishDate': { 'order': 'desc', 'mode': 'avg' } }] } es_headlines = es.search(index='headlines', doc_type='headline', body=query) if 'hits' in es_headlines and 'total' in es_headlines['hits']: total_headlines = es_headlines['hits']['total'] if 'hits' in es_headlines and 'hits' in es_headlines['hits']: for es_headline in es_headlines['hits']['hits']: if ('_source' in es_headline and 'data' in es_headline['_source']): headlines.append(es_headline['_source']['data']) return BulkResponse(headlines, {}, len(headlines), total_headlines)
def instagram_timeseries(self, request, pk=None): contact = self.get_contact_by_pk(request, pk) # Response to user instagram_timeseries = [] total_instagram_timeseries = 0 if contact.instagram != '': query = { 'query': { 'bool': { 'must': [{ 'term': { 'data.Username': contact.instagram } }] } }, 'sort': [{ 'data.CreatedAt': { 'order': 'desc', 'mode': 'avg' } }] } es_instagram_timeseries = es.search(index='timeseries', doc_type='instagram', body=query) if ('hits' in es_instagram_timeseries and 'total' in es_instagram_timeseries['hits']): total_instagram_timeseries = es_instagram_timeseries['hits'][ 'total'] if ('hits' in es_instagram_timeseries and 'hits' in es_instagram_timeseries['hits']): for ts in es_instagram_timeseries['hits']['hits']: if ('_source' in ts and 'data' in ts['_source']): # ts = format_es_response(ts) instagram_timeseries.append(ts['_source']['data']) return BulkResponse(instagram_timeseries, {}, len(instagram_timeseries), total_instagram_timeseries)
def tweets(self, request, pk=None): contact = self.get_contact_by_pk(request, pk) # Response to user tweets = [] total_tweets = 0 if contact.twitter != '': limit, offset = get_pagination(request) query = { 'size': limit, 'from': offset, 'query': { 'bool': { 'should': [{ 'term': { 'data.Username': contact.twitter } }], 'minimum_should_match': '100%' } }, 'sort': [{ 'data.CreatedAt': { 'order': 'desc', 'mode': 'avg' } }], } es_tweets = es.search(index='tweets', doc_type='tweet', body=query) if 'hits' in es_tweets and 'total' in es_tweets['hits']: total_tweets = es_tweets['hits']['total'] if 'hits' in es_tweets and 'hits' in es_tweets['hits']: for es_tweet in es_tweets['hits']['hits']: if ('_source' in es_tweet and 'data' in es_tweet['_source']): es_tweet = format_es_response(es_tweet) es_tweet['type'] = 'tweets' tweets.append(es_tweet['_source']['data']) return BulkResponse(tweets, {}, len(tweets), total_tweets)
def instagrams(self, request, pk=None): contact = self.get_contact_by_pk(request, pk) instagram_posts = [] if contact.instagram != '': limit, offset = get_pagination(request) query = { 'size': limit, 'from': offset, 'query': { 'bool': { 'should': [{ 'term': { 'data.Username': contact.instagram } }], 'minimum_should_match': '100%' } }, 'sort': [{ 'data.CreatedAt': { 'order': 'desc', 'mode': 'avg' } }] } es_instagrams = es.search(index='instagrams', doc_type='instagram', body=query) if 'hits' in es_instagrams and 'hits' in es_instagrams['hits']: for es_instagram in es_instagrams['hits']['hits']: if ('_source' in es_instagram and 'data' in es_instagram['_source']): es_instagram = format_es_response(es_instagram) es_instagram['type'] = 'instagrams' instagram_posts.append(es_instagram['_source']['data']) return BulkResponse(instagram_posts, {}, len(instagram_posts), len(instagram_posts))
def feed(self, request, pk=None): media_list = self.get_media_list_by_pk(request, pk) # Response to user feed = [] total_feed = 0 # ES attributes should = [] if media_list.contacts.count() > 0: twitter_usernames = (media_list.contacts and media_list.contacts.values_list( 'twitter', flat=True)) instagram_usernames = (media_list.contacts and media_list.contacts.values_list( 'instagram', flat=True)) feed_urls = [] contacts = media_list.contacts.all() for contact in contacts: feeds = Feed.objects.filter(contact=contact, valid_feed=True, running=True) for feed in feeds: if feed.feed_url != '': feed_urls.append(feed.feed_url) for twitter in twitter_usernames: if twitter != '': should.append({'term': {'data.Username': twitter}}) for instagram in instagram_usernames: if instagram != '': should.append( {'term': { 'data.InstagramUsername': instagram }}) for feed_url in feed_urls: if feed_url != '': should.append({'term': {'data.FeedURL': feed_url}}) if len(should) > 0: query = { 'size': 20, 'from': 0, 'query': { 'bool': { 'should': should } }, 'sort': [{ 'data.CreatedAt': { 'order': 'desc', 'mode': 'avg' } }] } es_feeds = es.search(index='feeds', doc_type='feed', body=query) if 'hits' in es_feeds and 'total' in es_feeds['hits']: total_feed = es_feeds['hits']['total'] if 'hits' in es_feeds and 'hits' in es_feeds['hits']: for es_feed in es_feeds['hits']['hits']: if ('_source' in es_feed and 'data' in es_feed['_source']): feed.append(es_feed['_source']['data']) return BulkResponse(feed, {}, len(feed), total_feed)
def feed(self, request, pk=None): contact = self.get_contact_by_pk(request, pk) # Response to user feed = [] total_feed = 0 # ES attributes should = [] twitter_username = contact.twitter instagram_username = contact.instagram feeds = Feed.objects.filter(contact=contact, valid_feed=True, running=True) if twitter_username != '': should.append({'term': {'data.Username': twitter_username}}) if instagram_username != '': should.append( {'term': { 'data.InstagramUsername': instagram_username }}) if len(feeds) > 0: for feed in feeds: should.append({'match': {'data.FeedURL': feed.feed_url}}) if len(should) > 0: limit, offset = get_pagination(request) query = { 'size': limit, 'from': offset, 'query': { 'bool': { 'should': should } }, 'sort': [{ 'data.CreatedAt': { 'order': 'desc', 'mode': 'avg' } }] } es_feeds = es.search(index='feeds', doc_type='feed', body=query) if 'hits' in es_feeds and 'total' in es_feeds['hits']: total_feed = es_feeds['hits']['total'] if 'hits' in es_feeds and 'hits' in es_feeds['hits']: for es_feed in es_feeds['hits']['hits']: if ('_source' in es_feed and 'data' in es_feed['_source']): es_feed = format_es_response(es_feed) # we want the format to be 'tweets', 'instagrams', # and 'headlines' es_feed['_source']['data']['type'] = es_feed[ '_source']['data']['type'].lower() + 's' feed.append(es_feed['_source']['data']) return BulkResponse(feed, {}, len(feed), total_feed)