def __init__(self, userJson, twitterSession, fromCache=False, isFollowee=False, isAssociatedWithTweet=False, twitterPlace=None, currentLocationCoordinate=None, geocodeBias=None, knownFollowees=None):
Hashable.__init__(self)
Timestamped.__init__(self)
if twitterSession is not None:
assert isinstance(twitterSession, TwitterSession)
if twitterPlace is not None:
assert isinstance(twitterPlace, Place)
if knownFollowees is None:
knownFollowees = set()
self.allow_geocode_external = True
if userJson is None:
self.data = Tree.make()
else:
self.data = Tree.make(userJson)
# If false indicates that self.data is already in the cache,
# so no point readding.
self.isDataNew = not fromCache
self.is_followee = isFollowee
# This does not contain all followees, only followees known to our application
# through follower enrichment.
self.known_followees = knownFollowees
self.is_associated_with_tweet = isAssociatedWithTweet
self.twitter_session = twitterSession
if twitterSession is not None:
self.instance_key = twitterSession.instance_key
else:
self.instance_key = None
self.queued_for_follower_enrichment = False
self.current_location_coordinate = currentLocationCoordinate
self.twitter_place = twitterPlace
self.last_follower_enrichment_error = None
self.follower_enrichment_progress = UserFollowerEnrichmentProgress(self)
if self.num_followers is not None:
self.twitter_calls_to_retrieve_follower_ids = math.ceil(float(self.num_followers) / float(TwitterSession.GET_FOLLOWER_IDS_QUANTITY_PER_TWITTER_CALL))
else:
self.twitter_calls_to_retrieve_follower_ids = 1
self.geocoded_from = None
self.geocode_bias = None
self.analysers = None
def __init__(self, userJson, twitterSession, fromCache=False, isFollowee=False, isAssociatedWithTweet=False, twitterPlace=None, currentLocationCoordinate=None, geocodeBias=None, knownFollowees=None):
Hashable.__init__(self)
Timestamped.__init__(self)
if twitterSession is not None:
assert isinstance(twitterSession, TwitterSession)
if twitterPlace is not None:
assert isinstance(twitterPlace, Place)
if knownFollowees is None:
knownFollowees = set()
self.allow_geocode_external = True
if userJson is None:
self.data = Tree.make()
else:
self.data = Tree.make(userJson)
# If false indicates that self.data is already in the cache,
# so no point readding.
self.isDataNew = not fromCache
self.is_followee = isFollowee
# This does not contain all followees, only followees known to our application
# through follower enrichment.
self.known_followees = knownFollowees
self.is_associated_with_tweet = isAssociatedWithTweet
self.twitter_session = twitterSession
if twitterSession is not None:
self.instance_key = twitterSession.instance_key
else:
self.instance_key = None
self.queued_for_follower_enrichment = False
self.current_location_coordinate = currentLocationCoordinate
self.twitter_place = twitterPlace
self.last_follower_enrichment_error = None
self.follower_enrichment_progress = UserFollowerEnrichmentProgress(self)
if self.num_followers is not None:
self.twitter_calls_to_retrieve_follower_ids = math.ceil(float(self.num_followers) / float(TwitterSession.GET_FOLLOWER_IDS_QUANTITY_PER_TWITTER_CALL))
else:
self.twitter_calls_to_retrieve_follower_ids = 1
self.geocoded_from = None
self.geocode_bias = None
self.analysers = None
def __init__(self, data, twitterSession, fromCache=False):
Timestamped.__init__(self)
Hashable.__init__(self)
if twitterSession is not None:
assert isinstance(twitterSession, TwitterSession)
self.data = Tree.make(data)
if not fromCache:
self.data.applyFunctionInTree(reverse_list, ['coordinates', 'coordinates']) # for use with leaflet.
self.isDataNew = not fromCache
twitterPlaceData = self.data.getFromTree(['place'])
if twitterPlaceData is not None:
self.twitter_place = Place(twitterPlaceData)
else:
self.twitter_place = None
userJson = self.data.getFromTree(['user'])
if userJson is None:
self._user = None
else:
self._user = User(userJson, twitterSession=twitterSession, isAssociatedWithTweet=True, twitterPlace=self.twitter_place, currentLocationCoordinate=self.coordinate)
if twitterSession is not None:
self.instance_key = twitterSession.instance_key
else:
self.instance_key = None
self.twitter_session = twitterSession
def __init__(self, geocodeData):
""" @param geocodeData 1. A dictionary of data to be parsed and represented by this object. """
super(GeocodeResult, self).__init__()
self.geocodeData = Tree.make(geocodeData)
# Need to force country code to be lower as we use it for country lookup.
def customLowerItem(item):
if item is None:
raise BadGeocodeException(
'Geocode result with no country code, ignoring result: %s (%s)'
% (unicode(self.cache_id), unicode(self.display_name)))
return lower_item(item)
self.geocodeData.applyFunctionInTree(customLowerItem,
['address', 'country_code'])
self._display_name_short = None
if self.place_type == GeocodeResultAbstract.PlaceTypes.CITY:
self._display_name_short = self.geocodeData.getFromTree(
['address', 'city'])
elif self.place_type == GeocodeResultAbstract.PlaceTypes.COUNTRY:
self._display_name_short = self.geocodeData.getFromTree(
['address', 'country'])
# As fall back default to full display name.
if self._display_name_short is None:
self._display_name_short = self.display_name
def __init__(self, data, twitterSession, fromCache=False):
Timestamped.__init__(self)
Hashable.__init__(self)
if twitterSession is not None:
assert isinstance(twitterSession, TwitterSession)
self.data = Tree.make(data)
if not fromCache:
self.data.applyFunctionInTree(reverse_list, ['coordinates', 'coordinates']) # for use with leaflet.
self.isDataNew = not fromCache
twitterPlaceData = self.data.getFromTree(['place'])
if twitterPlaceData is not None:
self.twitter_place = Place(twitterPlaceData)
else:
self.twitter_place = None
userJson = self.data.getFromTree(['user'])
if userJson is None:
self._user = None
else:
self._user = User(userJson, twitterSession=twitterSession, isAssociatedWithTweet=True, twitterPlace=self.twitter_place, currentLocationCoordinate=self.coordinate)
if twitterSession is not None:
self.instance_key = twitterSession.instance_key
else:
self.instance_key = None
self.twitter_session = twitterSession
def __iter__(self):
logPrefix = 'TWITTER STREAM MESSAGE: '
def doLog(message):
logger.warn('%s (%s): %s' % (logPrefix, self.feed_id, unicode(message)))
for x in self._feed:
if self.twitter_session is not None and not self.twitter_session.is_session_active:
return
# I think keep alive is actually done inside requests now
# so this will probably never happen.
if x is None or len(x) == 0:
doLog('Keep alive signal received')
continue
try:
data = json.JSONDecoder(strict=False).decode(x)
tree = Tree.make(data)
except ValueError as e:
doLog('ValueError while decoding tweet: %s' % e.message)
continue
if tree.getFromTree(['delete']) is not None:
statusId = tree.getFromTree(['delete','status','id'])
userId = tree.getFromTree(['delete','status','user_id'])
doLog('Deletion request received for userId: %s, statusId: %s' % (unicode(userId), unicode(statusId)))
elif tree.getFromTree(['scrub_geo']) is not None:
userId = tree.getFromTree(['scrub_geo','user_id'])
upToStatusId = tree.getFromTree(['scrub_geo','up_to_status_id'])
doLog('Geolocated tweet strip request received for userId: %s, for tweets up to ID: %s' % (unicode(userId), unicode(upToStatusId)))
elif tree.getFromTree(['limit']) is not None:
if self.logUndeliveredMessagesTimer.ticked():
undeliveredMessagesUntilNow = tree.getFromTree(['limit','track'])
doLog('From feed connection time until now (reconnects reset the count), %s tweets were not delivered due to rate limitations imposed by Twitter' % unicode(undeliveredMessagesUntilNow))
elif tree.getFromTree(['status_withheld']) is not None:
withheldStatusId = tree.getFromTree(['status_withheld','id'])
withheldUserId = tree.getFromTree(['status_withheld','user_id'])
withheldInCountries = tree.getFromTree(['status_withheld','withheld_in_countries'])
doLog('Status with ID: %s from user with ID: %s has been withheld in the following countries: %s' % (unicode(withheldStatusId, unicode(withheldUserId), unicode(withheldInCountries))))
elif tree.getFromTree(['user_withheld']) is not None:
withheldUserId = tree.getFromTree(['user_withheld'])
withheldInCountries = tree.getFromTree(['user_withheld','withheld_in_countries'])
doLog('User with ID: %s has been withheld in the following countries: %s' % (unicode(withheldUserId), unicode(withheldInCountries)))
elif tree.getFromTree(['disconnect']) is not None:
disconnectCode = tree.getFromTree(['disconnect','code'])
disconnectStreamName = tree.getFromTree(['disconnect','stream_name'])
disconnectReason = tree.getFromTree(['disconnect','reason'])
doLog('Stream with name "%s" disconnected from server, code: %s, reason: %s' % (unicode(disconnectStreamName), unicode(disconnectCode), unicode(disconnectReason)))
raise TwitterSession.SessionException('Disconnected from server')
elif tree.getFromTree(['warning']) is not None:
stallCode = tree.getFromTree(['warning','code'])
stallMessage = tree.getFromTree(['warning','message'])
stallPercentFull = tree.getFromTree(['warning','percent_full'])
doLog('Stall warning received with stall code: %s, message: %s, percent full: %s' % (unicode(stallCode), unicode(stallMessage), unicode(stallPercentFull)))
else:
yield Tweet(data, self.twitter_session)
def __iter__(self):
logPrefix = 'TWITTER STREAM MESSAGE: '
def doLog(message):
logger.warn('%s (%s): %s' % (logPrefix, self.feed_id, unicode(message)))
for x in self._feed:
if self.twitter_session is not None and not self.twitter_session.is_session_active:
return
# I think keep alive is actually done inside requests now
# so this will probably never happen.
if x is None or len(x) == 0:
doLog('Keep alive signal received')
continue
try:
data = json.JSONDecoder(strict=False).decode(x)
tree = Tree.make(data)
except ValueError as e:
doLog('ValueError while decoding tweet: %s' % e.message)
continue
if tree.getFromTree(['delete']) is not None:
statusId = tree.getFromTree(['delete','status','id'])
userId = tree.getFromTree(['delete','status','user_id'])
doLog('Deletion request received for userId: %s, statusId: %s' % (unicode(userId), unicode(statusId)))
elif tree.getFromTree(['scrub_geo']) is not None:
userId = tree.getFromTree(['scrub_geo','user_id'])
upToStatusId = tree.getFromTree(['scrub_geo','up_to_status_id'])
doLog('Geolocated tweet strip request received for userId: %s, for tweets up to ID: %s' % (unicode(userId), unicode(upToStatusId)))
elif tree.getFromTree(['limit']) is not None:
if self.logUndeliveredMessagesTimer.ticked():
undeliveredMessagesUntilNow = tree.getFromTree(['limit','track'])
doLog('From feed connection time until now (reconnects reset the count), %s tweets were not delivered due to rate limitations imposed by Twitter' % unicode(undeliveredMessagesUntilNow))
elif tree.getFromTree(['status_withheld']) is not None:
withheldStatusId = tree.getFromTree(['status_withheld','id'])
withheldUserId = tree.getFromTree(['status_withheld','user_id'])
withheldInCountries = tree.getFromTree(['status_withheld','withheld_in_countries'])
doLog('Status with ID: %s from user with ID: %s has been withheld in the following countries: %s' % (unicode(withheldStatusId, unicode(withheldUserId), unicode(withheldInCountries))))
elif tree.getFromTree(['user_withheld']) is not None:
withheldUserId = tree.getFromTree(['user_withheld'])
withheldInCountries = tree.getFromTree(['user_withheld','withheld_in_countries'])
doLog('User with ID: %s has been withheld in the following countries: %s' % (unicode(withheldUserId), unicode(withheldInCountries)))
elif tree.getFromTree(['disconnect']) is not None:
disconnectCode = tree.getFromTree(['disconnect','code'])
disconnectStreamName = tree.getFromTree(['disconnect','stream_name'])
disconnectReason = tree.getFromTree(['disconnect','reason'])
doLog('Stream with name "%s" disconnected from server, code: %s, reason: %s' % (unicode(disconnectStreamName), unicode(disconnectCode), unicode(disconnectReason)))
raise TwitterSession.SessionException('Disconnected from server')
elif tree.getFromTree(['warning']) is not None:
stallCode = tree.getFromTree(['warning','code'])
stallMessage = tree.getFromTree(['warning','message'])
stallPercentFull = tree.getFromTree(['warning','percent_full'])
doLog('Stall warning received with stall code: %s, message: %s, percent full: %s' % (unicode(stallCode), unicode(stallMessage), unicode(stallPercentFull)))
else:
yield Tweet(data, self.twitter_session)
def __init__(self, geocodeData):
super(GeocodeResultGoogle, self).__init__()
self.geocodeData = Tree.make(geocodeData)
dName = self.display_name
if dName is None:
self._display_name_short = None
else:
self._display_name_short = dName.split(',')[0]
def __init__(self, geocodeData):
super(GeocodeResultGoogle, self).__init__()
self.geocodeData = Tree.make(geocodeData)
dName = self.display_name
if dName is None:
self._display_name_short = None
else:
self._display_name_short = dName.split(',')[0]
def __init__(self, geocodeData):
""" @param geocodeData 1. A dictionary of data to be parsed and represented by this object. """
super(GeocodeResult, self).__init__()
self.geocodeData = Tree.make(geocodeData)
# Need to force country code to be lower as we use it for country lookup.
def customLowerItem(item):
if item is None:
raise BadGeocodeException('Geocode result with no country code, ignoring result: %s (%s)' % (unicode(self.cache_id),unicode(self.display_name)))
return lower_item(item)
self.geocodeData.applyFunctionInTree(customLowerItem, ['address','country_code'])
self._display_name_short = None
if self.place_type == GeocodeResultAbstract.PlaceTypes.CITY:
self._display_name_short = self.geocodeData.getFromTree(['address','city'])
elif self.place_type == GeocodeResultAbstract.PlaceTypes.COUNTRY:
self._display_name_short = self.geocodeData.getFromTree(['address','country'])
# As fall back default to full display name.
if self._display_name_short is None:
self._display_name_short = self.display_name
def __init__(self, data):
self.data = Tree.make(data)
def __init__(self, data):
self.data = Tree.make(data)
