def instagram_profile(self, request, pk=None):
contact = self.get_contact_by_pk(request, pk)
instagram_profile = {}
if contact.instagram != '':
query = {
'query': {
'bool': {
'must': [{
'term': {
'data.Username': contact.instagram
}
}]
}
}
}
es_instagram_profile = es.search(index='instagrams',
doc_type='user',
body=query)
if ('hits' in es_instagram_profile
and 'hits' in es_instagram_profile['hits']):
if len(es_instagram_profile['hits']['hits']) > 0:
instagram_profile = es_instagram_profile['hits']['hits'][
0]['_source']['data']
return Response(instagram_profile, {})
def twitter_profile(self, request, pk=None):
contact = self.get_contact_by_pk(request, pk)
# Response to user
twitter_profile = {}
if contact.twitter != '':
query = {
'query': {
'bool': {
'must': [{
'term': {
'data.Username': contact.twitter
}
}]
}
}
}
es_twitter_profile = es.search(index='tweets',
doc_type='user',
body=query)
if ('hits' in es_twitter_profile
and 'hits' in es_twitter_profile['hits']):
if len(es_twitter_profile['hits']['hits']) > 0:
twitter_profile = es_twitter_profile['hits']['hits'][0][
'_source']['data']
return Response(twitter_profile, {})
def headlines(self, request, pk=None):
contact = self.get_contact_by_pk(request, pk)
feeds = Feed.objects.filter(contact=contact)
# Response to user
headlines = []
total_headlines = 0
if len(feeds) > 0:
limit, offset = get_pagination(request)
query = {
'size': limit,
'from': offset,
'query': {
'bool': {
'should': []
}
},
'sort': [{
'data.PublishDate': {
'order': 'desc',
'mode': 'avg'
}
}]
}
for feed in feeds:
if feed.feed_url != '':
query['query']['bool']['should'].append(
{'match': {
'data.FeedURL': feed.feed_url
}})
es_headlines = es.search(index='headlines',
doc_type='headline',
body=query)
if 'hits' in es_headlines and 'total' in es_headlines['hits']:
total_headlines = es_headlines['hits']['total']
if 'hits' in es_headlines and 'hits' in es_headlines['hits']:
for es_headline in es_headlines['hits']['hits']:
if ('_source' in es_headline
and 'data' in es_headline['_source']):
es_headline = format_es_response(es_headline)
headlines.append(es_headline['_source']['data'])
return BulkResponse(headlines, {}, len(headlines), total_headlines)
def tweets(self, request, pk=None):
media_list = self.get_media_list_by_pk(request, pk)
total_tweets = 0
tweets = []
if media_list.contacts.count() > 0:
twitter_usernames = (media_list.contacts
and media_list.contacts.values_list(
'twitter', flat=True))
if len(twitter_usernames) > 0:
query = {
'size': 20,
'from': 0,
'query': {
'bool': {
'should': [],
'minimum_should_match': '100%'
}
},
'sort': [{
'data.CreatedAt': {
'order': 'desc',
'mode': 'avg'
}
}],
}
for username in twitter_usernames:
query['query']['bool']['should'].append(
{'term': {
'data.Username': username
}})
es_tweets = es.search(index='tweets',
doc_type='tweet',
body=query)
if 'hits' in es_tweets and 'total' in es_tweets['hits']:
total_tweets = es_tweets['hits']['total']
if 'hits' in es_tweets and 'hits' in es_tweets['hits']:
for es_tweet in es_tweets['hits']['hits']:
if ('_source' in es_tweet
and 'data' in es_tweet['_source']):
tweets.append(es_tweet['_source']['data'])
return BulkResponse(tweets, {}, len(tweets), total_tweets)
def headlines(self, request, pk=None):
media_list = self.get_media_list_by_pk(request, pk)
contacts = media_list.contacts.all()
# Response to user
headlines = []
total_headlines = 0
should = []
for contact in contacts:
feeds = Feed.objects.filter(contact=contact,
valid_feed=True,
running=True)
for feed in feeds:
if feed.feed_url != '':
should.append({'match': {'data.FeedURL': feed.feed_url}})
if len(should) > 0:
query = {
'size': 20,
'from': 0,
'query': {
'bool': {
'should': should
}
},
'sort': [{
'data.PublishDate': {
'order': 'desc',
'mode': 'avg'
}
}]
}
es_headlines = es.search(index='headlines',
doc_type='headline',
body=query)
if 'hits' in es_headlines and 'total' in es_headlines['hits']:
total_headlines = es_headlines['hits']['total']
if 'hits' in es_headlines and 'hits' in es_headlines['hits']:
for es_headline in es_headlines['hits']['hits']:
if ('_source' in es_headline
and 'data' in es_headline['_source']):
headlines.append(es_headline['_source']['data'])
return BulkResponse(headlines, {}, len(headlines), total_headlines)
def instagram_timeseries(self, request, pk=None):
contact = self.get_contact_by_pk(request, pk)
# Response to user
instagram_timeseries = []
total_instagram_timeseries = 0
if contact.instagram != '':
query = {
'query': {
'bool': {
'must': [{
'term': {
'data.Username': contact.instagram
}
}]
}
},
'sort': [{
'data.CreatedAt': {
'order': 'desc',
'mode': 'avg'
}
}]
}
es_instagram_timeseries = es.search(index='timeseries',
doc_type='instagram',
body=query)
if ('hits' in es_instagram_timeseries
and 'total' in es_instagram_timeseries['hits']):
total_instagram_timeseries = es_instagram_timeseries['hits'][
'total']
if ('hits' in es_instagram_timeseries
and 'hits' in es_instagram_timeseries['hits']):
for ts in es_instagram_timeseries['hits']['hits']:
if ('_source' in ts and 'data' in ts['_source']):
# ts = format_es_response(ts)
instagram_timeseries.append(ts['_source']['data'])
return BulkResponse(instagram_timeseries, {},
len(instagram_timeseries),
total_instagram_timeseries)
def tweets(self, request, pk=None):
contact = self.get_contact_by_pk(request, pk)
# Response to user
tweets = []
total_tweets = 0
if contact.twitter != '':
limit, offset = get_pagination(request)
query = {
'size': limit,
'from': offset,
'query': {
'bool': {
'should': [{
'term': {
'data.Username': contact.twitter
}
}],
'minimum_should_match': '100%'
}
},
'sort': [{
'data.CreatedAt': {
'order': 'desc',
'mode': 'avg'
}
}],
}
es_tweets = es.search(index='tweets', doc_type='tweet', body=query)
if 'hits' in es_tweets and 'total' in es_tweets['hits']:
total_tweets = es_tweets['hits']['total']
if 'hits' in es_tweets and 'hits' in es_tweets['hits']:
for es_tweet in es_tweets['hits']['hits']:
if ('_source' in es_tweet
and 'data' in es_tweet['_source']):
es_tweet = format_es_response(es_tweet)
es_tweet['type'] = 'tweets'
tweets.append(es_tweet['_source']['data'])
return BulkResponse(tweets, {}, len(tweets), total_tweets)
def instagrams(self, request, pk=None):
contact = self.get_contact_by_pk(request, pk)
instagram_posts = []
if contact.instagram != '':
limit, offset = get_pagination(request)
query = {
'size': limit,
'from': offset,
'query': {
'bool': {
'should': [{
'term': {
'data.Username': contact.instagram
}
}],
'minimum_should_match': '100%'
}
},
'sort': [{
'data.CreatedAt': {
'order': 'desc',
'mode': 'avg'
}
}]
}
es_instagrams = es.search(index='instagrams',
doc_type='instagram',
body=query)
if 'hits' in es_instagrams and 'hits' in es_instagrams['hits']:
for es_instagram in es_instagrams['hits']['hits']:
if ('_source' in es_instagram
and 'data' in es_instagram['_source']):
es_instagram = format_es_response(es_instagram)
es_instagram['type'] = 'instagrams'
instagram_posts.append(es_instagram['_source']['data'])
return BulkResponse(instagram_posts, {}, len(instagram_posts),
len(instagram_posts))
def feed(self, request, pk=None):
media_list = self.get_media_list_by_pk(request, pk)
# Response to user
feed = []
total_feed = 0
# ES attributes
should = []
if media_list.contacts.count() > 0:
twitter_usernames = (media_list.contacts
and media_list.contacts.values_list(
'twitter', flat=True))
instagram_usernames = (media_list.contacts
and media_list.contacts.values_list(
'instagram', flat=True))
feed_urls = []
contacts = media_list.contacts.all()
for contact in contacts:
feeds = Feed.objects.filter(contact=contact,
valid_feed=True,
running=True)
for feed in feeds:
if feed.feed_url != '':
feed_urls.append(feed.feed_url)
for twitter in twitter_usernames:
if twitter != '':
should.append({'term': {'data.Username': twitter}})
for instagram in instagram_usernames:
if instagram != '':
should.append(
{'term': {
'data.InstagramUsername': instagram
}})
for feed_url in feed_urls:
if feed_url != '':
should.append({'term': {'data.FeedURL': feed_url}})
if len(should) > 0:
query = {
'size': 20,
'from': 0,
'query': {
'bool': {
'should': should
}
},
'sort': [{
'data.CreatedAt': {
'order': 'desc',
'mode': 'avg'
}
}]
}
es_feeds = es.search(index='feeds', doc_type='feed', body=query)
if 'hits' in es_feeds and 'total' in es_feeds['hits']:
total_feed = es_feeds['hits']['total']
if 'hits' in es_feeds and 'hits' in es_feeds['hits']:
for es_feed in es_feeds['hits']['hits']:
if ('_source' in es_feed and 'data' in es_feed['_source']):
feed.append(es_feed['_source']['data'])
return BulkResponse(feed, {}, len(feed), total_feed)
def feed(self, request, pk=None):
contact = self.get_contact_by_pk(request, pk)
# Response to user
feed = []
total_feed = 0
# ES attributes
should = []
twitter_username = contact.twitter
instagram_username = contact.instagram
feeds = Feed.objects.filter(contact=contact,
valid_feed=True,
running=True)
if twitter_username != '':
should.append({'term': {'data.Username': twitter_username}})
if instagram_username != '':
should.append(
{'term': {
'data.InstagramUsername': instagram_username
}})
if len(feeds) > 0:
for feed in feeds:
should.append({'match': {'data.FeedURL': feed.feed_url}})
if len(should) > 0:
limit, offset = get_pagination(request)
query = {
'size': limit,
'from': offset,
'query': {
'bool': {
'should': should
}
},
'sort': [{
'data.CreatedAt': {
'order': 'desc',
'mode': 'avg'
}
}]
}
es_feeds = es.search(index='feeds', doc_type='feed', body=query)
if 'hits' in es_feeds and 'total' in es_feeds['hits']:
total_feed = es_feeds['hits']['total']
if 'hits' in es_feeds and 'hits' in es_feeds['hits']:
for es_feed in es_feeds['hits']['hits']:
if ('_source' in es_feed and 'data' in es_feed['_source']):
es_feed = format_es_response(es_feed)
# we want the format to be 'tweets', 'instagrams',
# and 'headlines'
es_feed['_source']['data']['type'] = es_feed[
'_source']['data']['type'].lower() + 's'
feed.append(es_feed['_source']['data'])
return BulkResponse(feed, {}, len(feed), total_feed)