def __init__(self, userJson, twitterSession, fromCache=False, isFollowee=False, isAssociatedWithTweet=False, twitterPlace=None, currentLocationCoordinate=None, geocodeBias=None, knownFollowees=None): Hashable.__init__(self) Timestamped.__init__(self) if twitterSession is not None: assert isinstance(twitterSession, TwitterSession) if twitterPlace is not None: assert isinstance(twitterPlace, Place) if knownFollowees is None: knownFollowees = set() self.allow_geocode_external = True if userJson is None: self.data = Tree.make() else: self.data = Tree.make(userJson) # If false indicates that self.data is already in the cache, # so no point readding. self.isDataNew = not fromCache self.is_followee = isFollowee # This does not contain all followees, only followees known to our application # through follower enrichment. self.known_followees = knownFollowees self.is_associated_with_tweet = isAssociatedWithTweet self.twitter_session = twitterSession if twitterSession is not None: self.instance_key = twitterSession.instance_key else: self.instance_key = None self.queued_for_follower_enrichment = False self.current_location_coordinate = currentLocationCoordinate self.twitter_place = twitterPlace self.last_follower_enrichment_error = None self.follower_enrichment_progress = UserFollowerEnrichmentProgress(self) if self.num_followers is not None: self.twitter_calls_to_retrieve_follower_ids = math.ceil(float(self.num_followers) / float(TwitterSession.GET_FOLLOWER_IDS_QUANTITY_PER_TWITTER_CALL)) else: self.twitter_calls_to_retrieve_follower_ids = 1 self.geocoded_from = None self.geocode_bias = None self.analysers = None
def __init__(self, data, twitterSession, fromCache=False): Timestamped.__init__(self) Hashable.__init__(self) if twitterSession is not None: assert isinstance(twitterSession, TwitterSession) self.data = Tree.make(data) if not fromCache: self.data.applyFunctionInTree(reverse_list, ['coordinates', 'coordinates']) # for use with leaflet. self.isDataNew = not fromCache twitterPlaceData = self.data.getFromTree(['place']) if twitterPlaceData is not None: self.twitter_place = Place(twitterPlaceData) else: self.twitter_place = None userJson = self.data.getFromTree(['user']) if userJson is None: self._user = None else: self._user = User(userJson, twitterSession=twitterSession, isAssociatedWithTweet=True, twitterPlace=self.twitter_place, currentLocationCoordinate=self.coordinate) if twitterSession is not None: self.instance_key = twitterSession.instance_key else: self.instance_key = None self.twitter_session = twitterSession
def __init__(self, geocodeData): """ @param geocodeData 1. A dictionary of data to be parsed and represented by this object. """ super(GeocodeResult, self).__init__() self.geocodeData = Tree.make(geocodeData) # Need to force country code to be lower as we use it for country lookup. def customLowerItem(item): if item is None: raise BadGeocodeException( 'Geocode result with no country code, ignoring result: %s (%s)' % (unicode(self.cache_id), unicode(self.display_name))) return lower_item(item) self.geocodeData.applyFunctionInTree(customLowerItem, ['address', 'country_code']) self._display_name_short = None if self.place_type == GeocodeResultAbstract.PlaceTypes.CITY: self._display_name_short = self.geocodeData.getFromTree( ['address', 'city']) elif self.place_type == GeocodeResultAbstract.PlaceTypes.COUNTRY: self._display_name_short = self.geocodeData.getFromTree( ['address', 'country']) # As fall back default to full display name. if self._display_name_short is None: self._display_name_short = self.display_name
def __iter__(self): logPrefix = 'TWITTER STREAM MESSAGE: ' def doLog(message): logger.warn('%s (%s): %s' % (logPrefix, self.feed_id, unicode(message))) for x in self._feed: if self.twitter_session is not None and not self.twitter_session.is_session_active: return # I think keep alive is actually done inside requests now # so this will probably never happen. if x is None or len(x) == 0: doLog('Keep alive signal received') continue try: data = json.JSONDecoder(strict=False).decode(x) tree = Tree.make(data) except ValueError as e: doLog('ValueError while decoding tweet: %s' % e.message) continue if tree.getFromTree(['delete']) is not None: statusId = tree.getFromTree(['delete','status','id']) userId = tree.getFromTree(['delete','status','user_id']) doLog('Deletion request received for userId: %s, statusId: %s' % (unicode(userId), unicode(statusId))) elif tree.getFromTree(['scrub_geo']) is not None: userId = tree.getFromTree(['scrub_geo','user_id']) upToStatusId = tree.getFromTree(['scrub_geo','up_to_status_id']) doLog('Geolocated tweet strip request received for userId: %s, for tweets up to ID: %s' % (unicode(userId), unicode(upToStatusId))) elif tree.getFromTree(['limit']) is not None: if self.logUndeliveredMessagesTimer.ticked(): undeliveredMessagesUntilNow = tree.getFromTree(['limit','track']) doLog('From feed connection time until now (reconnects reset the count), %s tweets were not delivered due to rate limitations imposed by Twitter' % unicode(undeliveredMessagesUntilNow)) elif tree.getFromTree(['status_withheld']) is not None: withheldStatusId = tree.getFromTree(['status_withheld','id']) withheldUserId = tree.getFromTree(['status_withheld','user_id']) withheldInCountries = tree.getFromTree(['status_withheld','withheld_in_countries']) doLog('Status with ID: %s from user with ID: %s has been withheld in the following countries: %s' % (unicode(withheldStatusId, unicode(withheldUserId), unicode(withheldInCountries)))) elif tree.getFromTree(['user_withheld']) is not None: withheldUserId = tree.getFromTree(['user_withheld']) withheldInCountries = tree.getFromTree(['user_withheld','withheld_in_countries']) doLog('User with ID: %s has been withheld in the following countries: %s' % (unicode(withheldUserId), unicode(withheldInCountries))) elif tree.getFromTree(['disconnect']) is not None: disconnectCode = tree.getFromTree(['disconnect','code']) disconnectStreamName = tree.getFromTree(['disconnect','stream_name']) disconnectReason = tree.getFromTree(['disconnect','reason']) doLog('Stream with name "%s" disconnected from server, code: %s, reason: %s' % (unicode(disconnectStreamName), unicode(disconnectCode), unicode(disconnectReason))) raise TwitterSession.SessionException('Disconnected from server') elif tree.getFromTree(['warning']) is not None: stallCode = tree.getFromTree(['warning','code']) stallMessage = tree.getFromTree(['warning','message']) stallPercentFull = tree.getFromTree(['warning','percent_full']) doLog('Stall warning received with stall code: %s, message: %s, percent full: %s' % (unicode(stallCode), unicode(stallMessage), unicode(stallPercentFull))) else: yield Tweet(data, self.twitter_session)
def __init__(self, geocodeData): super(GeocodeResultGoogle, self).__init__() self.geocodeData = Tree.make(geocodeData) dName = self.display_name if dName is None: self._display_name_short = None else: self._display_name_short = dName.split(',')[0]
def __init__(self, geocodeData): """ @param geocodeData 1. A dictionary of data to be parsed and represented by this object. """ super(GeocodeResult, self).__init__() self.geocodeData = Tree.make(geocodeData) # Need to force country code to be lower as we use it for country lookup. def customLowerItem(item): if item is None: raise BadGeocodeException('Geocode result with no country code, ignoring result: %s (%s)' % (unicode(self.cache_id),unicode(self.display_name))) return lower_item(item) self.geocodeData.applyFunctionInTree(customLowerItem, ['address','country_code']) self._display_name_short = None if self.place_type == GeocodeResultAbstract.PlaceTypes.CITY: self._display_name_short = self.geocodeData.getFromTree(['address','city']) elif self.place_type == GeocodeResultAbstract.PlaceTypes.COUNTRY: self._display_name_short = self.geocodeData.getFromTree(['address','country']) # As fall back default to full display name. if self._display_name_short is None: self._display_name_short = self.display_name
def __init__(self, data): self.data = Tree.make(data)