def update_tweets(tweet_batch):
if not tweet_batch.count():
return
tweet_batch = list(tweet_batch)
client = get_client('statuses_lookup')
try:
responses = client.call('statuses_lookup',
id_=[tweet._ident for tweet in tweet_batch],
trim_user=True)
except tweepy.error.TweepError:
log('got tweepy.error.TweepError!')
log('tweet ids = %s' % [tweet._ident for tweet in tweet_batch])
return_client(client)
raise
return_client(client)
for response in responses:
monitor_stop_flag()
tweet = next(
(tweet
for tweet in tweet_batch if tweet._ident == response._json['id']),
None)
if tweet:
global_task_queue.add(update_tweet_from_response, [response])
tweet_batch.remove(tweet)
deleted_count = 0
for tweet in tweet_batch:
deleted_count += 1
tweet.deleted_at = today()
tweet.save()
if deleted_count > 0:
log("{} tweets have been deleted".format(deleted_count))
def update_twitter_users(twitter_user_batch):
if not twitter_user_batch.count():
return
twitter_user_batch = list(twitter_user_batch)
client = get_client('lookup_users')
try:
responses = client.call(
'lookup_users',
user_ids=[user._ident for user in twitter_user_batch])
except tweepy.error.TweepError:
log('got tweepy.error.TweepError!')
log('user_ids = %s' % [user._ident for user in twitter_user_batch])
return_client(client)
raise
return_client(client)
for response in responses:
monitor_stop_flag()
tw_user = next((user for user in twitter_user_batch
if user._ident == response._json['id']), None)
if tw_user:
global_task_queue.add(update_twitter_user_from_response,
args=[tw_user, response._json])
twitter_user_batch.remove(tw_user)
for tw_user in twitter_user_batch:
log('Twitter user (%s) has returned no result.' % tw_user)
# twUser._error_on_update = True
tw_user._last_updated = today()
tw_user._update_frequency = 5
tw_user.save()
def update(self, jObject):
super(FBComment, self).update(jObject)
self.setAttachement(jObject)
self.last_updated = today()
replaceEmojisFromFields(self, [
'message',
])
self.save()
def update(self, jObject):
assert isinstance(jObject,
dict), 'jObject must be a dict or json instance!'
self.copyBasicFields(jObject)
self.copyDateTimeFields(jObject)
self.updateStatistics(jObject)
self.updateImages(jObject)
self._last_updated = today()
self.save()
def update(self, jObject):
if not isinstance(jObject, dict):
raise Exception(
'A DICT or JSON object must be passed as argument.')
self.copyBasicFields(jObject)
self.updateStatistics(jObject)
replaceEmojisFromFields(self, ['message', 'description'])
self.last_updated = today()
self.save()
def update(self, jObject):
self.title = jObject['snippet']['title']
self.description = jObject['snippet']['description']
publishedAt = datetime.strptime(jObject['snippet']['publishedAt'],
'%Y-%m-%dT%H:%M:%S.%fZ')
publishedAt = publishedAt.replace(tzinfo=utc)
self.publishedAt = publishedAt
self.privacy_status = jObject['status']['privacyStatus']
self._last_updated = today()
self.video_count = self.videoCount()
self.save()
def update(self, jObject):
if not isinstance(jObject, dict):
raise Exception(
'A DICT or JSON object from Youtube must be passed as argument.'
)
# pretty(jObject)
self.copyBasicFields(jObject)
self.copyDateTimeFields(jObject)
self.updateStatistics(jObject)
self.updateImages(jObject)
self._last_updated = today()
self.save()
def updateTimeLabels(self, jObject):
for atr in self._time_labels:
if atr in jObject and jObject[atr]:
related_name = atr + 's'
last_item = self.getLast(related_name)
if not last_item or last_item.recorded_time != today():
class_name = this_module.get_twitter_model_by_name(atr)
new_item = class_name(tweet=self, value=jObject[atr])
new_item.save()
elif last_item.value != jObject[atr]:
last_item.value = jObject[atr]
last_item.save()
def update(self, jObject):
assert isinstance(jObject,
dict), 'jObject must be a dict or json instance!'
# pretty(jObject)
self.copyBasicFields(jObject)
self.truncate_text()
self.copyDateTimeFields(jObject)
self.updateStatistics(jObject)
self.updateChannelTarget(jObject)
self.updateVideoTarget(jObject)
self._last_updated = today()
self.save()
def update(self, jObject):
if not isinstance(jObject, dict):
raise Exception(
'A DICT or JSON object from Youtube must be passed as argument.'
)
self.copyBasicFields(jObject)
self.updateStatistics(jObject)
self.updateFeaturedVideo(jObject)
self.setParentPage(jObject)
self.setLocation(jObject)
self.setReleaseDate(jObject)
replaceEmojisFromFields(self, [])
self.last_updated = today()
self.save()
def UpdateFromResponse(self, response):
# log('%s: %s' % (self, response))
if not isinstance(response, dict):
raise Exception(
'A DICT or JSON object from Twitter must be passed as argument.'
)
# log('len(location): %s'%len(jObject['location']))
# log('location: %s'%jObject['location'])
self.copyBasicFields(response)
self.copyDateTimeFields(response)
self.updateTimeLabels(response)
self._ident = response['id']
self._last_updated = today()
self.save()
def updateStatistics(self, jObject):
for attrName in self.statistics:
countObjs = getattr(self, attrName).order_by('-recorded_time')
objType = countObjs.model
val = jObject
for key in self.statistics[attrName]:
if key in val:
val = val[key]
else:
pass
# log('Invalid dict sequence: %s'%self.statistics[attrName])
if not countObjs.exists():
objType.objects.create(channel=self, value=val)
else:
if countObjs[0].value != int(
val) and countObjs[0].recorded_time != today():
objType.objects.create(channel=self, value=val)
def updateStatistics(self, jObject):
for attrName in self.statistics:
countObjs = getattr(self, attrName).order_by('-recorded_time')
objType = countObjs.model
val = jObject
for key in self.statistics[attrName]:
if key in val:
val = val[key]
else:
# log('Invalid dict sequence: %s. Object returned is: %s' % (self.statistics[attrName],val))
val = None
if val:
if not countObjs.exists():
objType.objects.create(video=self, value=val)
elif countObjs[0].value != int(
val) and countObjs[0].recorded_time != today():
objType.objects.create(video=self, value=val)
def updateStatistics(self, jObject):
for attrName in self.statistics:
countObjs = getattr(self, attrName).order_by('-recorded_time')
objType = countObjs.model
val = jObject
for key in self.statistics[attrName]:
if key in val:
val = val[key]
else:
val = None
break
if val:
if not countObjs.exists():
objType.objects.create(fbPage=self, value=val)
else:
if countObjs[0].value != int(
val) and countObjs[0].recorded_time != today():
objType.objects.create(fbPage=self, value=val)
def UpdateFromResponse(self, jObject):
if not isinstance(jObject, dict):
raise Exception(
'A DICT or JSON object from Twitter must be passed as argument.'
)
self.copyBasicFields(jObject)
self.copyDateTimeFields(jObject)
self.updateTimeLabels(jObject)
if "entities" in jObject:
self.setUserMentions(jObject['entities'])
self.setHashtags(jObject['entities'])
if "retweeted_status" in jObject:
self.setRetweetOf(jObject['retweeted_status'])
if not self.user and 'user' in jObject:
self.set_author(jObject['user'])
if jObject['in_reply_to_user_id']:
self.setInReplyToUser(
screen_name=jObject['in_reply_to_screen_name'],
_ident=jObject['in_reply_to_user_id'])
if jObject['in_reply_to_status_id']:
self.setInReplyToStatus(jObject['in_reply_to_status_id'])
if 'quoted_status_id' in jObject:
self.setQuotedStatus(jObject['quoted_status_id'])
if jObject['place']:
self.setPlace(jObject['place'])
self._last_updated = today()
self._ident = jObject['id']
try:
self.save()
except:
text = self.text.encode('unicode-escape')
# log('modified text: %s'%text)
self.text = text
self.save()