def fetch_messages(self, feed, **kwargs):
results = requests.get(
"https://queryfeed.net/instagram",
params={
'q': '#{}'.format(feed.hashtag),
'token': self.provider.app_secret
}
)
results = xmltodict.parse(results.text)
if not (results.get('rss') and
results.get('rss').get('channel') and
results.get('rss').get('channel').get('item')):
return
messages = []
for item in results['rss']['channel']['item']:
# Don't store duplicates
try:
feed.messages.get(provider_post_id=item['link'])
except Message.DoesNotExist:
pass
else:
continue
r = requests.get(item['link'])
user_re = re.compile(r'"owner": ({.*?})')
m = user_re.search(r.text)
message = Message()
if m.group(1):
user_data = json.loads(m.group(1))
author_name = user_data.get('full_name', '')
if author_name is None:
author_name = ''
message.author_name = author_name
message.author_picture = user_data.get('profile_pic_url', '')
message.author_username = "******".format(user_data.get('username'))
else:
message.author_username = item['guid']
if 'enclosure' in item:
media = item['enclosure']
if 'image' in media['@type']:
message.image = media['@url']
message.text = item['title']
import pytz
locale = pytz.utc
# Thu, 02 Mar 2017 15:13:08 +0000
message.published_at = locale.localize(
datetime.datetime.strptime(item['pubDate'], '%a, %d %b %Y %H:%M:%S +0000'))
message.provider = self.provider
message.provider_post_id = item['link']
message.feed = feed
messages.append(message)
message.save()
return messages
def fetch_messages(self, feed, **kwargs):
try:
last_message = feed.messages.filter(
provider=self.provider).order_by('published_at').last()
if last_message is None:
last_id = None
else:
last_id = last_message.provider_post_id
results = requests.get(
"https://api.twitter.com/1.1/search/tweets.json",
params={
'q': '%23{}'.format(feed.hashtag),
'count': '100',
'since_id': last_id,
'entities': True
},
headers={
"Authorization": "Bearer {}".format(self.access_token)
}).json()
messages = []
for tweet in results['statuses']:
message = Message()
message.author_name = tweet['user']['name']
message.author_picture = tweet['user'][
'profile_image_url_https'].replace('_normal', '')
message.author_username = "******".format(
tweet['user']['screen_name'])
content = tweet['text']
if 'extended_entities' in tweet:
for media in tweet['extended_entities']['media']:
if media['type'] == 'photo':
message.image = media['media_url_https']
if media['type'] in ('animated_gif', 'video'):
variants = media['video_info']['variants']
# Select the MP4 with the best bitrate
mp4 = [
variant for variant in variants
if variant['content_type'] == 'video/mp4'
]
if len(mp4) > 0:
best_bitrate = max(
mp4, key=lambda item: item['bitrate'])
message.video = best_bitrate['url']
if media['type'] == 'animated_gif':
message.video_is_gif = True
message.text = content
import pytz
locale = pytz.utc
message.published_at = locale.localize(
datetime.datetime.strptime(tweet['created_at'],
'%a %b %d %H:%M:%S +0000 %Y'))
message.provider = self.provider
message.provider_post_id = tweet['id_str']
message.feed = feed
messages.append(message)
message.save()
return messages
except HTTPError:
self.auth()
return self.fetch_messages(feed, **kwargs)