def __init__(self, application):
Display.__init__(self,
application,
pageRoute=('/instance/<instance>/cached_tweets',
None),
webSocketManagers=None,
onDisplayUsageFunc=onDisplayUsageFunc)
def __init__(self, application, providerId):
Display.__init__(self,
application,
pageRoute=('/geocode_search', None),
webSocketManagers=None)
self.provider_id = providerId
def __init__(self, application, userInformationWebSocketGroup):
Display.__init__(self,
application,
pageRoute=('/instance/<instance>/user/<user:int>',
None),
webSocketManagers=[userInformationWebSocketGroup],
onDisplayUsageFunc=onDisplayUsageFunc)
def __init__(self, application, consumerToken, consumerSecret, initialData=None):
Display.__init__(self, application, pageRoute=("/start_instance", ["POST"]), webSocketManagers=None)
assert isinstance(application, WebApplicationTwitter)
self.consumer_token = consumerToken
self.consumer_secret = consumerSecret
self.initial_data = initialData
def __init__(self, application, providerId):
Display.__init__(self,
application,
pageRoute=('/geocode_search', None),
webSocketManagers=None)
self.provider_id = providerId
def __init__(self, application, tweetByLocationWebSocketGroup, dataDownloadWebSocketManager, realtimePerformance):
Display.__init__(self,
application,
pageRoute=('/instance/<instance>/location/<location:int>/<provider:int>/page', None),
webSocketManagers=[tweetByLocationWebSocketGroup,
(dataDownloadWebSocketManager, False),
(realtimePerformance, False)],
onDisplayUsageFunc=onDisplayUsageFunc)
def __init__(self, application, consumerToken, consumerSecret):
Display.__init__(self,
application,
pageRoute=('/oauth/sign_in', None),
webSocketManagers=None)
assert isinstance(application, WebApplicationTwitter)
self.consumer_token = consumerToken
self.consumer_secret = consumerSecret
def __init__(self, application, mapWebSocketGroup, dataDownloadWebSocketManager, realTimePerformanceWebSocketGroup):
assert isinstance(application, WebApplicationTwitter)
Display.__init__(self,
application,
pageRoute=('/instance/<instance>', None),
webSocketManagers=[mapWebSocketGroup, dataDownloadWebSocketManager, realTimePerformanceWebSocketGroup],
onDisplayUsageFunc=onDisplayUsageFunc)
self.locations = set()
def __init__(self, application, consumerToken, consumerSecret,
callbackLink):
Display.__init__(self,
application,
pageRoute=('/oauth/callback', None),
webSocketManagers=None)
assert isinstance(application, WebApplicationTwitter)
self.consumer_token = consumerToken
self.consumer_secret = consumerSecret
self.callback_link = callbackLink
def getPageLinkImage(instance, user, target):
assert isinstance(user, User)
imageHtml = Display.getImage(user.profile_image_url,
user.name,
None,
className='twitter-profile-image')
return Display.getLink(UserInformationPage.link_info.getPageLink(
instance, user.id),
imageHtml,
target=target)
def __init__(self, application, dataCollection, followerExtractorGateThread):
Display.__init__(self,
application,
pageRoute=('/instance/<instance>/user_follower_enrich/<user:int>', None),
webSocketManagers=None,
onDisplayUsageFunc=onDisplayUsageFunc)
assert isinstance(dataCollection, DataCollection)
assert isinstance(followerExtractorGateThread, FollowerExtractorGateThread)
self.data_collection = dataCollection
self.follower_extractor_gate_thread = followerExtractorGateThread
def __init__(self,
application,
consumerToken,
consumerSecret,
initialData=None):
Display.__init__(self,
application,
pageRoute=('/start_instance', ['POST']),
webSocketManagers=None)
assert isinstance(application, WebApplicationTwitter)
self.consumer_token = consumerToken
self.consumer_secret = consumerSecret
self.initial_data = initialData
def __init__(self, application, mapWebSocketGroup,
dataDownloadWebSocketManager,
realTimePerformanceWebSocketGroup):
assert isinstance(application, WebApplicationTwitter)
Display.__init__(self,
application,
pageRoute=('/instance/<instance>', None),
webSocketManagers=[
mapWebSocketGroup, dataDownloadWebSocketManager,
realTimePerformanceWebSocketGroup
],
onDisplayUsageFunc=onDisplayUsageFunc)
self.locations = set()
def __init__(self, application, tweetByLocationWebSocketGroup,
dataDownloadWebSocketManager, realtimePerformance):
Display.__init__(
self,
application,
pageRoute=
('/instance/<instance>/location/<location:int>/<provider:int>/page',
None),
webSocketManagers=[
tweetByLocationWebSocketGroup,
(dataDownloadWebSocketManager, False),
(realtimePerformance, False)
],
onDisplayUsageFunc=onDisplayUsageFunc)
def __init__(self, application, dataCollection,
followerExtractorGateThread):
Display.__init__(
self,
application,
pageRoute=('/instance/<instance>/user_follower_enrich/<user:int>',
None),
webSocketManagers=None,
onDisplayUsageFunc=onDisplayUsageFunc)
assert isinstance(dataCollection, DataCollection)
assert isinstance(followerExtractorGateThread,
FollowerExtractorGateThread)
self.data_collection = dataCollection
self.follower_extractor_gate_thread = followerExtractorGateThread
def func(templateArguments):
oauth_token = request.forms.get('oauth_token')
oauth_secret = request.forms.get('oauth_secret')
oauth = TwitterInstance.makeAuthTuple(oauth_token, oauth_secret)
removedInstance = self.application.twitter_instances.removeTwitterInstanceByAuth(
oauth)
if removedInstance is not None:
assert isinstance(removedInstance, TwitterInstance)
logger.info('Cleaned up twitter instance with oauth: %s' %
unicode(oauth))
try:
keywords = ','.join(
removedInstance.twitter_thread.twitter_feed.keywords)
except TypeError:
keywords = ''
regions = removedInstance.geographic_setup_string
instanceSetupCode = removedInstance.instance_setup_code
else:
keywords = None
regions = None
instanceSetupCode = None
return redirect(
Display.addArgumentsToLink(
GateInstance.link_info.getPageLink(),
oauth_token=oauth_token,
oauth_secret=oauth_secret,
keywords=keywords,
regions=regions,
instance_setup_code=instanceSetupCode))
def updateFollowerInformationEnrichedStatus():
if (not updatedItem.is_followers_loaded) and updatedItem.queued_for_follower_enrichment:
progressBarHtml = '<div class="progress progress-striped active">'\
'<div class="bar bar-danger" id="%s" style="width: %.2f%%"></div>'\
'<div class="bar bar-warning" id="%s" style="width: %.2f%%"></div>'\
'<div class="bar bar-success" id="%s" style="width: %.2f%%"></div></div>' % (queueProgressBarId, updatedItem.follower_enrichment_progress.queue_progress,
userProgressBarId ,updatedItem.follower_enrichment_progress.user_progress,
userIdProgressBarId, updatedItem.follower_enrichment_progress.user_id_progress)
followerInformationEnrichedValue = '<br><br><div class="text-info" id="%s">%s</div>%s' % (descriptionId,
formatDescriptionWithUserLink(updatedItem.follower_enrichment_progress.enrichment_progress_description, updatedItem.follower_enrichment_progress.queue_waiting_for_user),
progressBarHtml)
returnVal = 1
elif (not updatedItem.is_followers_loaded) and updatedItem.last_follower_enrichment_error:
followerInformationEnrichedValue = ' - failed to enrich followers with reason: %s' % updatedItem.last_follower_enrichment_error
returnVal = 2
elif (not updatedItem.is_followers_loaded) and (not updatedItem.queued_for_follower_enrichment) and updatedItem.is_geocoded:
linkToFollow = UserFollowerEnrichPage.link_info.getPageLink(instance, user)
link = '<br><br>%s<br>' % Display.getLink('#','Get Followers','_self','$(this).addClass(\'disabled\'); getPage(\'%s\', null, null, null); return false;' % linkToFollow, htmlClass='btn btn-success get-followers-button')
followerInformationEnrichedValue = link
returnVal = 4
elif not updatedItem.is_geocoded:
followerInformationEnrichedValue = ' - not geocoded so cannot enrich followers'
returnVal = 3
else:
followerInformationEnrichedValue = ''
returnVal = 0
if divControl.last_follower_information_enrich_draw != returnVal or returnVal == 2:
displayValue = '%d followers%s' % (numberOfFollowers, followerInformationEnrichedValue)
updateRow('Follower Information', displayValue, [1000])
divControl.last_follower_information_enrich_draw = returnVal
return returnVal
def func(templateArguments):
form = request.forms.decode("UTF-8")
oauth_token = form.get("oauth_token")
oauth_secret = form.get("oauth_secret")
keywords = form.get("keywords")
geographical_setup_string = unicode(
form.get("regions")
) # includes all geographical information including influence and twitter feed areas.
instance_setup_code = form.get("instance_setup_code")
keywordsList = None
if keywords is not None:
if len(keywords) > 0:
assert isinstance(keywords, basestring)
keywordsList = keywords.split(",")
def redirectError(errorMessage):
return redirect(
Display.addArgumentsToLink(
GateInstance.link_info.getPageLink(),
oauth_token=oauth_token,
oauth_secret=oauth_secret,
keywords=keywords,
regions=geographical_setup_string,
instance_setup_code=instance_setup_code,
error=errorMessage,
)
)
if not oauth_token or not oauth_secret:
return redirectError("Twitter login credentials are missing")
oauth = TwitterInstance.makeAuthTuple(oauth_token, oauth_secret)
if self.application.twitter_instances.isAuthInUse(oauth):
# Do nothing, we will be redirected to the gate instance which will hopefully
# correct this mistake.
pass
else:
try:
twitterInstance = self.application.twitter_instances.createInstance(
TwitterAuthentication(
Configuration.CONSUMER_TOKEN, Configuration.CONSUMER_SECRET, oauth_token, oauth_secret
),
geographical_setup_string,
keywordsList,
instance_setup_code,
)
except ValueError as e:
return redirectError("Failed to setup instance, reason: %s" % e.message)
error = twitterInstance.setup_error
if error is not None:
return redirectError(error)
return redirect(
Display.addArgumentsToLink(
GateInstance.link_info.getPageLink(), oauth_token=oauth_token, oauth_secret=oauth_secret
)
)
def func(templateArguments):
oauth_token = request.GET.oauth_token
oauth_secret = request.GET.oauth_token_secret
oauth_verifier = request.GET.oauth_verifier
oauth = OAuth1(self.consumer_token,
client_secret=self.consumer_secret,
resource_owner_key=oauth_token,
resource_owner_secret=oauth_secret,
verifier=oauth_verifier)
response = requests.post(
'https://api.twitter.com/oauth/access_token', auth=oauth)
if not response.ok:
return redirect_problem(
'Failed to retrieve oauth_secret from twitter')
response = parse_qs(response.content)
final_token = response['oauth_token'][0]
final_token_secret = response['oauth_token_secret'][0]
logger.info('oauth_token: %s, oauth_secret: %s received' %
(final_token, final_token_secret))
return redirect(
Display.addArgumentsToLink(self.callback_link,
oauth_token=final_token,
oauth_secret=final_token_secret))
class OAuthSignIn(Display):
""" Initial oauth sign in page. First leg of 3 legged oauth. """
link_info = Display.LinkInfo(lambda link: lambda: link, '/oauth/sign_in')
def __init__(self, application, consumerToken, consumerSecret):
Display.__init__(self,
application,
pageRoute=('/oauth/sign_in', None),
webSocketManagers=None)
assert isinstance(application, WebApplicationTwitter)
self.consumer_token = consumerToken
self.consumer_secret = consumerSecret
@property
def page_html_function(self):
def func(templateArguments):
oauth = OAuth1(self.consumer_token,
client_secret=self.consumer_secret)
response = requests.post(
'https://api.twitter.com/oauth/request_token',
params={'oauth_callback': OAuthCallback.link_info.page_link},
auth=oauth)
if not response.ok:
return redirect_problem(
'Failed to retrieve oauth_token from twitter: %s' %
response.text)
qs = parse_qs(response.text)
oauth_token = qs['oauth_token'][0]
return redirect(
'https://api.twitter.com/oauth/authenticate?oauth_token=%s' %
oauth_token)
return func
def func(templateArguments):
oauth_token = request.forms.get("oauth_token")
oauth_secret = request.forms.get("oauth_secret")
oauth = TwitterInstance.makeAuthTuple(oauth_token, oauth_secret)
removedInstance = self.application.twitter_instances.removeTwitterInstanceByAuth(oauth)
if removedInstance is not None:
assert isinstance(removedInstance, TwitterInstance)
logger.info("Cleaned up twitter instance with oauth: %s" % unicode(oauth))
try:
keywords = ",".join(removedInstance.twitter_thread.twitter_feed.keywords)
except TypeError:
keywords = ""
regions = removedInstance.geographic_setup_string
instanceSetupCode = removedInstance.instance_setup_code
else:
keywords = None
regions = None
instanceSetupCode = None
return redirect(
Display.addArgumentsToLink(
GateInstance.link_info.getPageLink(),
oauth_token=oauth_token,
oauth_secret=oauth_secret,
keywords=keywords,
regions=regions,
instance_setup_code=instanceSetupCode,
)
)
def func(templateArguments):
form = request.forms.decode('UTF-8')
oauth_token = form.get('oauth_token')
oauth_secret = form.get('oauth_secret')
keywords = form.get('keywords')
geographical_setup_string = unicode(
form.get('regions')
) # includes all geographical information including influence and twitter feed areas.
instance_setup_code = form.get('instance_setup_code')
keywordsList = None
if keywords is not None:
if len(keywords) > 0:
assert isinstance(keywords, basestring)
keywordsList = keywords.split(',')
def redirectError(errorMessage):
return redirect(
Display.addArgumentsToLink(
GateInstance.link_info.getPageLink(),
oauth_token=oauth_token,
oauth_secret=oauth_secret,
keywords=keywords,
regions=geographical_setup_string,
instance_setup_code=instance_setup_code,
error=errorMessage))
if not oauth_token or not oauth_secret:
return redirectError('Twitter login credentials are missing')
oauth = TwitterInstance.makeAuthTuple(oauth_token, oauth_secret)
if self.application.twitter_instances.isAuthInUse(oauth):
# Do nothing, we will be redirected to the gate instance which will hopefully
# correct this mistake.
pass
else:
try:
twitterInstance = self.application.twitter_instances.createInstance(
TwitterAuthentication(Configuration.CONSUMER_TOKEN,
Configuration.CONSUMER_SECRET,
oauth_token, oauth_secret),
geographical_setup_string, keywordsList,
instance_setup_code)
except ValueError as e:
return redirectError(
'Failed to setup instance, reason: %s' % e.message)
error = twitterInstance.setup_error
if error is not None:
return redirectError(error)
return redirect(
Display.addArgumentsToLink(
GateInstance.link_info.getPageLink(),
oauth_token=oauth_token,
oauth_secret=oauth_secret))
def redirectError(errorMessage):
return redirect(
Display.addArgumentsToLink(
GateInstance.link_info.getPageLink(),
oauth_token=oauth_token,
oauth_secret=oauth_secret,
keywords=keywords,
regions=geographical_setup_string,
instance_setup_code=instance_setup_code,
error=errorMessage))
class UserFollowerEnrichPage(Display):
link_info = Display.LinkInfo(
lambda link: lambda instance, user: link % (instance, user),
'/instance/%s/user_follower_enrich/%d')
def __init__(self, application, dataCollection,
followerExtractorGateThread):
Display.__init__(
self,
application,
pageRoute=('/instance/<instance>/user_follower_enrich/<user:int>',
None),
webSocketManagers=None,
onDisplayUsageFunc=onDisplayUsageFunc)
assert isinstance(dataCollection, DataCollection)
assert isinstance(followerExtractorGateThread,
FollowerExtractorGateThread)
self.data_collection = dataCollection
self.follower_extractor_gate_thread = followerExtractorGateThread
@property
def page_html_function(self):
def func(templateArguments, instance, user):
try:
twitterInstance = self.application.twitter_instances.getInstanceByInstanceKey(
instance)
if twitterInstance is None:
abort(404, "No active search stream found at this address")
# We need name and followers count for the progress bar.
userObject = self.data_collection.getUser(
instance, user,
twitterInstance.twitter_thread.twitter_session, False,
UserProjection.Geocode(
True, UserDataProjection(['name', 'followers_count'])))
if not userObject:
abort(404, "No user found at this address")
assert isinstance(userObject, User)
if not self.follower_extractor_gate_thread.addUser(
userObject, restrictInfluenceArea=False):
logger.warn(
'Not processed user: %s/%d for enrichment - shouldProcessUser returned false'
% (instance, user))
except KeyError:
logger.warn(
'Follower information enrich request received for user which does not exist: %s/%d'
% (instance, user))
return redirect(
UserInformationPage.link_info.getPageLink(instance, user))
return func
def redirectError(errorMessage):
return redirect(
Display.addArgumentsToLink(
GateInstance.link_info.getPageLink(),
oauth_token=oauth_token,
oauth_secret=oauth_secret,
keywords=keywords,
regions=geographical_setup_string,
instance_setup_code=instance_setup_code,
error=errorMessage,
)
)
class ManageInstancePost(Display):
""" Post operation on manage instance means terminate that instance. """
link_info = Display.LinkInfo(lambda link: lambda: link, '/manage_instance')
def __init__(self, application):
Display.__init__(self,
application,
pageRoute=('/manage_instance', ['POST']),
webSocketManagers=None)
assert isinstance(application, WebApplicationTwitter)
@property
def page_html_function(self):
def func(templateArguments):
oauth_token = request.forms.get('oauth_token')
oauth_secret = request.forms.get('oauth_secret')
oauth = TwitterInstance.makeAuthTuple(oauth_token, oauth_secret)
removedInstance = self.application.twitter_instances.removeTwitterInstanceByAuth(
oauth)
if removedInstance is not None:
assert isinstance(removedInstance, TwitterInstance)
logger.info('Cleaned up twitter instance with oauth: %s' %
unicode(oauth))
try:
keywords = ','.join(
removedInstance.twitter_thread.twitter_feed.keywords)
except TypeError:
keywords = ''
regions = removedInstance.geographic_setup_string
instanceSetupCode = removedInstance.instance_setup_code
else:
keywords = None
regions = None
instanceSetupCode = None
return redirect(
Display.addArgumentsToLink(
GateInstance.link_info.getPageLink(),
oauth_token=oauth_token,
oauth_secret=oauth_secret,
keywords=keywords,
regions=regions,
instance_setup_code=instanceSetupCode))
return func
class UserInformationPage(Display):
link_info = Display.LinkInfo(
lambda link: lambda instance, user: link % (instance, user),
'/instance/%s/user/%d')
@staticmethod
def getPageLinkImage(instance, user, target):
assert isinstance(user, User)
imageHtml = Display.getImage(user.profile_image_url,
user.name,
None,
className='twitter-profile-image')
return Display.getLink(UserInformationPage.link_info.getPageLink(
instance, user.id),
imageHtml,
target=target)
def __init__(self, application, userInformationWebSocketGroup):
Display.__init__(self,
application,
pageRoute=('/instance/<instance>/user/<user:int>',
None),
webSocketManagers=[userInformationWebSocketGroup],
onDisplayUsageFunc=onDisplayUsageFunc)
@property
def page_html_function(self):
def func(templateArguments, instance, user):
twitterInstance = self.application.twitter_instances.getInstanceByInstanceKey(
instance)
if twitterInstance is None:
abort(404, "No active search stream found at this address")
instanceDescription = getInstanceDescription(twitterInstance, True)
instanceLink = getInstanceLink(twitterInstance)
homeLink = getHomeLink(Configuration.PROJECT_NAME)
templateArguments.update({
'home_link': homeLink,
'instance': instance,
'instance_description': instanceDescription,
'instance_link': instanceLink,
'user_id': user
})
return template('user.tpl', templateArguments)
return func
class OAuthCallback(Display):
""" Final stage of authorisation, twitter redirects here. """
link_info = Display.LinkInfo(lambda link: lambda: link, '/oauth/callback')
def __init__(self, application, consumerToken, consumerSecret,
callbackLink):
Display.__init__(self,
application,
pageRoute=('/oauth/callback', None),
webSocketManagers=None)
assert isinstance(application, WebApplicationTwitter)
self.consumer_token = consumerToken
self.consumer_secret = consumerSecret
self.callback_link = callbackLink
@property
def page_html_function(self):
def func(templateArguments):
oauth_token = request.GET.oauth_token
oauth_secret = request.GET.oauth_token_secret
oauth_verifier = request.GET.oauth_verifier
oauth = OAuth1(self.consumer_token,
client_secret=self.consumer_secret,
resource_owner_key=oauth_token,
resource_owner_secret=oauth_secret,
verifier=oauth_verifier)
response = requests.post(
'https://api.twitter.com/oauth/access_token', auth=oauth)
if not response.ok:
return redirect_problem(
'Failed to retrieve oauth_secret from twitter')
response = parse_qs(response.content)
final_token = response['oauth_token'][0]
final_token_secret = response['oauth_token_secret'][0]
logger.info('oauth_token: %s, oauth_secret: %s received' %
(final_token, final_token_secret))
return redirect(
Display.addArgumentsToLink(self.callback_link,
oauth_token=final_token,
oauth_secret=final_token_secret))
return func
def func(templateArguments):
oauth_token = request.GET.oauth_token
oauth_secret = request.GET.oauth_token_secret
oauth_verifier = request.GET.oauth_verifier
oauth = OAuth1(self.consumer_token, client_secret=self.consumer_secret, resource_owner_key=oauth_token, resource_owner_secret=oauth_secret, verifier=oauth_verifier)
response = requests.post('https://api.twitter.com/oauth/access_token', auth=oauth)
if not response.ok:
return redirect_problem('Failed to retrieve oauth_secret from twitter')
response = parse_qs(response.content)
final_token = response['oauth_token'][0]
final_token_secret = response['oauth_token_secret'][0]
logger.info('oauth_token: %s, oauth_secret: %s received' % (final_token, final_token_secret))
return redirect(Display.addArgumentsToLink(self.callback_link, oauth_token=final_token, oauth_secret=final_token_secret))
class GeocodeCachePage(Display):
link_info = Display.LinkInfo(lambda link: lambda: link, '/geocode_search')
def __init__(self, application, providerId):
Display.__init__(self,
application,
pageRoute=('/geocode_search', None),
webSocketManagers=None)
self.provider_id = providerId
@property
def page_html_function(self):
def func(templateArguments):
placeName = parseString(request.GET.place_name)
data = list()
if placeName is None:
return {'json': data}
locations = list()
maxLocations = 10
locations += GeocodeResultAbstract.searchGnsByName(placeName, 3)
maxLocations -= len(locations)
if maxLocations > 0:
newLocations = geocodeSearch(self.provider_id, placeName,
maxLocations)
maxLocations -= len(newLocations)
locations += newLocations
for location in locations:
assert isinstance(location, GeocodeResultAbstract)
data.append((location.cache_id, location.bounding_box,
location.coordinate, location.display_name))
return {'json': data}
return func
class BulkDownloadDataProvider(AsyncTunnelProviderFile):
link_info = Display.LinkInfo(
lambda link: lambda instanceId, socketId, tunnelId: link %
(instanceId, socketId, tunnelId),
'/instance/%s/bulk_download_provider/%d/%s')
def __init__(self, application):
super(BulkDownloadDataProvider, self).__init__(
application,
pageRoute=
('/instance/<instance>/bulk_download_provider/<socket_id:int>/<tunnel_id>',
None),
onDisplayUsageFunc=onDisplayUsageFunc)
def getFileName(self, tunnelId):
if tunnelId == 'tweet_tunnel':
return 'Tweets_' + getDateTime()
else:
return 'Users_' + getDateTime()
def getFileExtension(self, tunnelId):
return 'csv'
def pushSocketDisplayChangeEx(self, controls, newItems, removedItems, isInitialPush, signalerKey):
control = controls['map']
instance = signalerKey[0]
if newItems is not None:
for item in newItems:
assert isinstance(item, GeocodeResultAbstract)
popupText = Display.getLink(LocationsPage.link_info.getPageLink(instance, item.place_id, item.provider_id), '<b>%s</b>' % unicode(item.display_name), target='_self')
placeType = item.place_type
customLayers = []
if placeType == GeocodeResultAbstract.PlaceTypes.CITY:
customLayers.append('city')
elif placeType == GeocodeResultAbstract.PlaceTypes.COUNTRY:
customLayers.append('country')
elif placeType == GeocodeResultAbstract.PlaceTypes.CONTINENT:
customLayers.append('continent')
else:
logger.error('Unsupported place type received by location map web socket group: %s' % unicode(placeType))
continue
properties = {'custom_layers' : customLayers}
control.addMarker(coord=item.coordinate, popupText=popupText, hashKey=item, properties=properties)
def getLandingPageLink(pageNum):
return Display.addArgumentsToLink(LandingPage.link_info.getPageLink(),page=pageNum)
def pushSocketDisplayChange(self, controls, updatedItem, oldCachedValue, isInitialPush, signaler, signalerKey):
divControl = controls['UserWsg_div']
followerMapControl = controls['UserWsg_follower_map']
assert isinstance(divControl, DivControl)
assert isinstance(followerMapControl, MapControl)
instance = signalerKey[0]
user = signalerKey[1]
queueProgressBarId = 'follower-enrichment-queue-progress-bar'
userProgressBarId = 'follower-enrichment-user-progress-bar'
userIdProgressBarId = 'follower-enrichment-userId-progress-bar'
descriptionId = 'follower-enrichment-progress-description'
assert isinstance(updatedItem, User)
# Utility functions for updating main page.
def getHeaderCell(text, headerWidth=2):
return divControl.getCell(text, 'span%d vertical-header' % headerWidth, ['b'])
def getContentsCell(text, contentsWidth=10):
return divControl.getCell(text,'span%d vertical-content' % contentsWidth, [])
hashKeyTracker = [1]
def updateRow(header, contents, theKey=hashKeyTracker):
if contents is None:
contents = '-'
isHeader = header is not None
if isHeader:
headerWidth = 2
contentsWidth = 10
else:
headerWidth = 0
contentsWidth = 12
text = [getContentsCell(contents, contentsWidth)]
if isHeader:
text = [getHeaderCell(header, headerWidth)] + text
divControl.updateRow(theKey[0], text, rowIndex = -1, rowClass = 'row-fluid vertical-row')
if theKey is not None:
theKey[0] += 1
numberOfFollowers = updatedItem.num_followers
# Does what it says on the tin.
def updateFollowerInformationEnrichedStatus():
if (not updatedItem.is_followers_loaded) and updatedItem.queued_for_follower_enrichment:
progressBarHtml = '<div class="progress progress-striped active">'\
'<div class="bar bar-danger" id="%s" style="width: %.2f%%"></div>'\
'<div class="bar bar-warning" id="%s" style="width: %.2f%%"></div>'\
'<div class="bar bar-success" id="%s" style="width: %.2f%%"></div></div>' % (queueProgressBarId, updatedItem.follower_enrichment_progress.queue_progress,
userProgressBarId ,updatedItem.follower_enrichment_progress.user_progress,
userIdProgressBarId, updatedItem.follower_enrichment_progress.user_id_progress)
followerInformationEnrichedValue = '<br><br><div class="text-info" id="%s">%s</div>%s' % (descriptionId,
formatDescriptionWithUserLink(updatedItem.follower_enrichment_progress.enrichment_progress_description, updatedItem.follower_enrichment_progress.queue_waiting_for_user),
progressBarHtml)
returnVal = 1
elif (not updatedItem.is_followers_loaded) and updatedItem.last_follower_enrichment_error:
followerInformationEnrichedValue = ' - failed to enrich followers with reason: %s' % updatedItem.last_follower_enrichment_error
returnVal = 2
elif (not updatedItem.is_followers_loaded) and (not updatedItem.queued_for_follower_enrichment) and updatedItem.is_geocoded:
linkToFollow = UserFollowerEnrichPage.link_info.getPageLink(instance, user)
link = '<br><br>%s<br>' % Display.getLink('#','Get Followers','_self','$(this).addClass(\'disabled\'); getPage(\'%s\', null, null, null); return false;' % linkToFollow, htmlClass='btn btn-success get-followers-button')
followerInformationEnrichedValue = link
returnVal = 4
elif not updatedItem.is_geocoded:
followerInformationEnrichedValue = ' - not geocoded so cannot enrich followers'
returnVal = 3
else:
followerInformationEnrichedValue = ''
returnVal = 0
if divControl.last_follower_information_enrich_draw != returnVal or returnVal == 2:
displayValue = '%d followers%s' % (numberOfFollowers, followerInformationEnrichedValue)
updateRow('Follower Information', displayValue, [1000])
divControl.last_follower_information_enrich_draw = returnVal
return returnVal
# Generates the progress bar description.
def formatDescriptionWithUserLink(description, user):
if user is not None:
link = Display.getLink(UserInformationPage.link_info.getPageLink(user.instance_key, user.id), unicode(user.name), target='_self')
return description % link
else:
return description
# Handles progress bar changes, updating the enrichment value if necessary in order
# to initialize the progress bar.
def handleProgressBarChanges(progress, mergeResult):
assert isinstance(progress, UserFollowerEnrichmentProgress)
if isInitialPush:
mergeResult = 1,1,1 # update all progress bars.
if mergeResult is None:
return
difQueueProgress, difUserIdProgress, difUserProgress = mergeResult
if difQueueProgress == 0 and difUserIdProgress == 0 and difUserProgress == 0:
return
# Not initialized yet or enrichment complete.
if divControl.last_follower_information_enrich_draw != 1:
return
def doProgressBarChange(progressBarId, percentage):
divControl.executeJavascript('document.getElementById("%s").style.width="%.2f%%";' % (progressBarId, percentage))
def setDescription(description):
divControl.executeJavascript('document.getElementById("%s").innerHTML=\'%s\';' % (descriptionId, description))
if difUserProgress > 0:
doProgressBarChange(userProgressBarId, progress.user_progress)
if difUserIdProgress > 0:
doProgressBarChange(userIdProgressBarId, progress.user_id_progress)
if difQueueProgress > 0:
doProgressBarChange(queueProgressBarId, progress.queue_progress)
description = formatDescriptionWithUserLink(progress.enrichment_progress_description, progress.queue_waiting_for_user)
setDescription(description)
# Populate contents of page.
updateRow('Timestamp', updatedItem.date_time)
updateRow('User name', updatedItem.name)
updateRow('Profile Image', Display.getImage(updatedItem.profile_image_url, updatedItem.name, className = 'twitter-profile-image'))
updateRow('Description', updatedItem.description)
updateRow('Location', updatedItem.location_text)
if updatedItem.has_twitter_place:
place = updatedItem.twitter_place
assert isinstance(place, Place)
placeCountry = '%s/%s' % (place.country, place.country_code)
placeName = place.full_name
placeNameContents = 'Place: %s' % placeName
if placeCountry is not None:
placeNameContents += ' / Country: %s' % placeCountry
else:
placeNameContents = '-'
updateRow('Twitter Place', placeNameContents)
geocoded_location = None
if updatedItem.is_geocoded:
raw_location = updatedItem.location_geocode
assert isinstance(raw_location, GeocodeResultAbstract)
geocoded_location = Display.getLink(LocationsPage.link_info.getPageLink(instance,
updatedItem.location_geocode.place_id,
updatedItem.location_geocode.provider_id),
updatedItem.location_geocode.display_name_short,
target='_self')
if raw_location.has_country:
countryGeocode = raw_location.country
assert isinstance(countryGeocode, GeocodeResultGNS)
countryHtml = Display.getLink(LocationsPage.link_info.getPageLink(instance,
countryGeocode.place_id,
countryGeocode.provider_id),
countryGeocode.display_name,
target='_self')
geocoded_location = '%s / %s' % (geocoded_location, countryHtml)
if raw_location.has_continent:
continentGeocode = raw_location.continent
assert isinstance(continentGeocode, GeocodeResultGNS)
continentHtml = Display.getLink(LocationsPage.link_info.getPageLink(instance,
continentGeocode.place_id,
continentGeocode.provider_id),
continentGeocode.display_name,
target='_self')
geocoded_location = '%s / %s' % (geocoded_location, continentHtml)
updateRow('Geocoded Location', geocoded_location)
#geocodedFrom = updatedItem.geocoded_from
#if geocodedFrom is None:
# geocodedFrom = '-'
#updateRow('Geocoded From', geocodedFrom)
divControl.executeJavascript('setPageTitle("%s");' % updatedItem.name)
updateFollowerInformationEnrichedStatus()
handleProgressBarChanges(updatedItem.follower_enrichment_progress, updatedItem.merge_result)
if updatedItem.is_followers_loaded and not updatedItem.queued_for_follower_enrichment:
# Don't update enriched follower information needlessly (say only timestamp changes).
# If we update the map it resets it which is a problem.
if not divControl.pushed_enriched_follower_information:
divControl.pushed_enriched_follower_information = True
updateRow(None, divControl.getContainerHtml('', 'span12', 'followers-list'))
divControl.executeJavascript('setupFollowersList();')
sortedResults = []
analyser = updatedItem.analysers[UserAnalysisFollowersGeocoded.analysis_name_static()]
assert isinstance(analyser, UserAnalysisFollowersGeocoded)
updateRow(None, getFollowersInfoDiv(divControl, analyser, 6, 6, [], ['b']))
divControl.executeJavascript('applyScrollbars();')
divControl.executeJavascript('setupFollowerLocationMap();')
for location, numFollowers in analyser.num_geocoded_followers_by_location.iteritems():
assert isinstance(location,GeocodeResultAbstract)
percentage = float(numFollowers) / analyser.num_geocoded_followers * 100
popupText = location.display_name
followerMapControl.addMarker(location.coordinate,
popupText=popupText,
properties={'percentage' : percentage})
class LocationsPage(Display):
link_info = Display.LinkInfo(
lambda link: lambda instance, location, providerId: link %
(instance, location, providerId), '/instance/%s/location/%d/%d/page')
def __init__(self, application, tweetByLocationWebSocketGroup,
dataDownloadWebSocketManager, realtimePerformance):
Display.__init__(
self,
application,
pageRoute=
('/instance/<instance>/location/<location:int>/<provider:int>/page',
None),
webSocketManagers=[
tweetByLocationWebSocketGroup,
(dataDownloadWebSocketManager, False),
(realtimePerformance, False)
],
onDisplayUsageFunc=onDisplayUsageFunc)
@property
def page_html_function(self):
def func(templateArguments, instance, location, provider):
twitterInstance = self.application.twitter_instances.getInstanceByInstanceKey(
instance)
if twitterInstance is None:
abort(404, "No active search stream found at this address")
geocode = geocodeFromCacheById(
GeocodeResultAbstract.buildCacheId(provider, location))
assert geocode is not None
instanceDescription = getInstanceDescription(twitterInstance)
instanceLink = getInstanceLink(twitterInstance)
homeLink = getHomeLink(Configuration.PROJECT_NAME)
templateArguments.update(
{
'home_link': homeLink,
'instance': instance,
'location': location,
'provider': provider,
'instance_link': instanceLink,
'instance_description': instanceDescription,
'place': geocode.display_name_short,
'place_coord': geocode.coordinate,
'startEpoch': twitterInstance.constructed_at,
'server_current_epoch': getEpochMs(),
'max_tweets': Configuration.MAX_CLIENT_LIVE_TWEETS
}
) # Client needs to offset to this epoch in case its clock is wrong.
if geocode.has_bounding_box:
templateArguments.update(
{'place_bounding_box': geocode.bounding_box})
if geocode.has_country:
templateArguments.update({
'place_country_link':
LocationsPage.link_info.getPageLink(
instance, geocode.country.place_id,
geocode.country.provider_id)
})
templateArguments.update(
{'place_country': geocode.country.display_name_short})
if geocode.has_continent:
templateArguments.update({
'place_continent_link':
LocationsPage.link_info.getPageLink(
instance, geocode.continent.place_id,
geocode.continent.provider_id)
})
templateArguments.update(
{'place_continent': geocode.continent.display_name_short})
return template('location.tpl', templateArguments)
return func
def getPageLinkImage(instance, user, target):
assert isinstance(user, User)
imageHtml = Display.getImage(user.profile_image_url, user.name, None, className='twitter-profile-image')
return Display.getLink(UserInformationPage.link_info.getPageLink(instance,user.id), imageHtml, target=target)
def __init__(self, application, consumerToken, consumerSecret, callbackLink):
Display.__init__(self, application, pageRoute=('/oauth/callback', None), webSocketManagers=None)
assert isinstance(application, WebApplicationTwitter)
self.consumer_token = consumerToken
self.consumer_secret = consumerSecret
self.callback_link = callbackLink
class LandingPage(Display):
link_info = Display.LinkInfo(lambda link: lambda: link, '/')
def __init__(self, application):
Display.__init__(self,
application,
pageRoute=('/', None),
webSocketManagers=None)
@staticmethod
def getLandingPageLink(pageNum):
return Display.addArgumentsToLink(LandingPage.link_info.getPageLink(),
page=pageNum)
def getHumanTime(self, milliseconds):
seconds = milliseconds / 1000
if seconds < 60:
if seconds == 1:
return "1 second"
return "%d seconds" % seconds
minutes = seconds / 60
if minutes < 60:
if minutes == 1:
return "1 minute"
return "%d minutes" % minutes
hours = minutes / 60
if hours < 24:
if hours == 1:
return "1 hour"
return "%d hours" % hours
days = hours / 24
if days == 1:
return "1 day"
return "%d days" % days
@property
def page_html_function(self):
def func(templateArguments):
argList = list()
instanceList = self.application.twitter_instances.getInstanceList()
# todo remove this, this generates more instances for debugging.
# if len(instanceList) > 0:
# count = 0
# while count < 7:
# instanceList.append(instanceList[0])
# count += 1
# todo end of remove this.
numInstances = len(instanceList)
numInstancesPerPage = Configuration.NUM_LANDING_PAGE_INSTANCES_PER_PAGE
if numInstances > 3 or numInstances < 1:
numInstancesPerRow = Configuration.NUM_LANDING_PAGE_INSTANCES_PER_ROW
else:
numInstancesPerRow = numInstances
thumbnailSpan = 12 / numInstancesPerRow
pageNum = parseInteger(request.GET.page, 0, numInstances, 0)
startIndex = numInstancesPerPage * pageNum
endIndex = startIndex + numInstancesPerPage
numPages = int(
ceil(float(numInstances) / float(numInstancesPerPage)))
for instance in instanceList[startIndex:endIndex]:
assert isinstance(instance, TwitterInstance)
argList.append((LocationsMapPage.link_info.getPageLink(
instance.instance_key), instance.getShortDescription(True),
instance.geographic_setup_string))
# Split into rows.
argList = splitList(argList, numInstancesPerRow)
templateArguments.update({'instances': argList})
# Pagination
startSmallPageIndex = pageNum - 5
endSmallPageIndex = 0
if startSmallPageIndex < 0:
endSmallPageIndex -= startSmallPageIndex
startSmallPageIndex = 0
endSmallPageIndex += (pageNum + 5)
offEndBy = endSmallPageIndex - numPages
if offEndBy > 0:
startSmallPageIndex -= offEndBy
if startSmallPageIndex < 0:
startSmallPageIndex = 0
endSmallPageIndex = numPages
pagination = list()
for n in range(startSmallPageIndex, endSmallPageIndex):
isCurrentPage = pageNum == n
pagination.append(
(n + 1, LandingPage.getLandingPageLink(n), isCurrentPage))
step = (numPages - endSmallPageIndex) / 5
if step > 0:
for n in range(endSmallPageIndex, numPages, step):
pagination.append(
(n + 1, LandingPage.getLandingPageLink(n), False))
if pageNum < numPages - 1:
templateArguments.update({
'pagination_next':
LandingPage.getLandingPageLink(pageNum + 1)
})
if pageNum > 0:
templateArguments.update({
'pagination_previous':
LandingPage.getLandingPageLink(pageNum - 1)
})
maxInactiveTime = Configuration.MAX_INSTANCE_INACTIVE_TIME_MS
maxTotalTime = Configuration.MAX_INSTANCE_TOTAL_AGE_MS
templateArguments.update({
'pagination':
pagination,
'thumbnail_span':
thumbnailSpan,
'build_instance_link':
OAuthSignIn.link_info.getPageLink(),
'maxInactiveTime':
self.getHumanTime(maxInactiveTime),
'maxTotalTime':
self.getHumanTime(maxTotalTime)
})
return template('landing-page.tpl', templateArguments)
return func
def __init__(self, application):
Display.__init__(self,
application,
pageRoute=('/', None),
webSocketManagers=None)
def getHomeLink(projectName):
return Display.getLink(LandingPage.link_info.getPageLink(),
projectName,
target='_self')
def formatDescriptionWithUserLink(description, user):
if user is not None:
link = Display.getLink(UserInformationPage.link_info.getPageLink(user.instance_key, user.id), unicode(user.name), target='_self')
return description % link
else:
return description
def __init__(self, application):
Display.__init__(self,
application,
pageRoute=('/instance/<instance>/influence', None),
webSocketManagers=None,
onDisplayUsageFunc=onDisplayUsageFunc)
class InfluenceCachePage(Display):
link_info = Display.LinkInfo(lambda link: lambda instance: link % instance,
'/instance/%s/influence')
def __init__(self, application):
Display.__init__(self,
application,
pageRoute=('/instance/<instance>/influence', None),
webSocketManagers=None,
onDisplayUsageFunc=onDisplayUsageFunc)
@property
def page_html_function(self):
def func(templateArguments, instance):
twitterInstance = self.application.twitter_instances.getInstanceByInstanceKey(
instance)
if twitterInstance is None:
return dict()
baseEpoch = twitterInstance.constructed_at
start_epoch = parseInteger(request.GET.start_epoch, default=None)
end_epoch = parseInteger(request.GET.end_epoch, default=None)
source_place_id = parseInteger(request.GET.source_place_id)
source_provider_id = parseInteger(request.GET.source_provider_id)
if source_place_id is None:
logger.error(
'Invalid place ID specified while providing influence data: %s'
% unicode(source_place_id))
return dict()
source_cache_id = GeocodeResultAbstract.buildCacheId(
source_provider_id, source_place_id)
temporalCollection = getTemporalInfluenceCollection(instance)
if start_epoch is not None:
start_time_id = getTimeIdFromTimestamp(
baseEpoch, Configuration.TEMPORAL_STEP, start_epoch)
else:
start_time_id = None
if end_epoch is not None:
end_time_id = getTimeIdFromTimestamp(
baseEpoch, Configuration.TEMPORAL_STEP, end_epoch)
else:
end_time_id = None
timerMs = getEpochMs()
cacheData = getTemporalRange(temporalCollection,
start_time_id,
end_time_id,
source_cache_id,
preciseFromBack=True,
preciseFromFront=True)
logger.info('Took %dms to read temporal range data' %
(getEpochMs() - timerMs))
timerMs = getEpochMs()
geocodeByPlaceType = dict()
totalsByPlaceType = dict()
if cacheData is not None:
for providerId, providerIdData in cacheData.iteritems():
providerId = int(providerId)
for destination, count in providerIdData.iteritems():
split = destination.split('_')
placeType = int(split[0])
placeId = int(split[1])
record = [placeId, providerId, None, None, count, None]
geocodeByPlaceType.setdefault(placeType,
list()).append(record)
# Process only the records we are going to display.
for placeType, records in geocodeByPlaceType.iteritems():
aux = sorted(records, key=lambda x: x[4], reverse=True)
aux = aux[:Configuration.
DISPLAY_MAX_NUM_INFLUENCE_RECORDS_PER_PLACE_TYPE]
geocodeByPlaceType[placeType] = aux
for record in aux:
cacheId = GeocodeResultAbstract.buildCacheId(
record[1], record[0])
geocode = geocodeFromCacheById(cacheId)
record[2] = geocode.display_name
record[3] = geocode.coordinate
count = record[4]
record[5] = geocode.bounding_box
totalsByPlaceType[placeType] = totalsByPlaceType.get(
placeType, 0) + count
def getResultPart(placeType):
return {
'geocode_list': geocodeByPlaceType.get(placeType, list()),
'total': totalsByPlaceType.get(placeType, 0)
}
resultData = dict()
resultData['city'] = getResultPart(
GeocodeResultAbstract.PlaceTypes.CITY)
resultData['country'] = getResultPart(
GeocodeResultAbstract.PlaceTypes.COUNTRY)
resultData['continent'] = getResultPart(
GeocodeResultAbstract.PlaceTypes.CONTINENT)
logger.info('Took %dms to build temporal range result data' %
(getEpochMs() - timerMs))
return {'json': resultData}
return func
def __init__(self, application, consumerToken, consumerSecret):
Display.__init__(self, application, pageRoute=('/oauth/sign_in', None), webSocketManagers=None)
assert isinstance(application, WebApplicationTwitter)
self.consumer_token = consumerToken
self.consumer_secret = consumerSecret
def __init__(self, application):
Display.__init__(self, application, pageRoute=("/gate_instance", None), webSocketManagers=None)
assert isinstance(application, WebApplicationTwitter)
def getLandingPageLink(pageNum):
return Display.addArgumentsToLink(LandingPage.link_info.getPageLink(),
page=pageNum)
def __init__(self, application):
Display.__init__(self,
application,
pageRoute=('/', None),
webSocketManagers=None)
class LocationsMapPage(Display):
link_info = Display.LinkInfo(lambda link: lambda instance: link % instance,
'/instance/%s')
def __init__(self, application, mapWebSocketGroup,
dataDownloadWebSocketManager,
realTimePerformanceWebSocketGroup):
assert isinstance(application, WebApplicationTwitter)
Display.__init__(self,
application,
pageRoute=('/instance/<instance>', None),
webSocketManagers=[
mapWebSocketGroup, dataDownloadWebSocketManager,
realTimePerformanceWebSocketGroup
],
onDisplayUsageFunc=onDisplayUsageFunc)
self.locations = set()
@property
def page_html_function(self):
def func(templateArguments, instance):
instance = self.application.twitter_instances.getInstanceByInstanceKey(
instance)
if instance is None:
abort(404, "No active search stream found at this address")
assert isinstance(instance, TwitterInstance)
keywords = instance.twitter_thread.twitter_feed.keywords
geographicalSetupString = instance.geographic_setup_string
if keywords is None:
keywords = ''
keywordsDisplay = '[None]'
else:
keywords = ','.join(keywords)
keywordsDisplay = keywords
instanceDescription = getInstanceDescription(instance, False)
instanceDescriptionWithPrefix = getInstanceDescription(
instance, True)
homeLink = getHomeLink(Configuration.PROJECT_NAME)
templateArguments.update({
'instance_description':
instanceDescription,
'instance_description_with_prefix':
instanceDescriptionWithPrefix,
'home_link':
homeLink,
'instance_name':
instance.instance_key,
'keywords':
keywords,
'keywords_display':
keywordsDisplay,
'instance_map_data':
geographicalSetupString,
'post_address':
WEBSITE_ROOT_HTTP +
'/manage_instance', # for terminate instance button.
'login_address':
OAuthSignIn.link_info.getPageLink(),
'start_epoch':
instance.constructed_at,
'server_current_epoch':
getEpochMs()
})
return template('locations-map.tpl', templateArguments)
return func
def __init__(self, application, userInformationWebSocketGroup):
Display.__init__(self,
application,
pageRoute=('/instance/<instance>/user/<user:int>', None),
webSocketManagers=[userInformationWebSocketGroup],
onDisplayUsageFunc=onDisplayUsageFunc)
def getHomeLink(projectName):
return Display.getLink(LandingPage.link_info.getPageLink(),projectName,target='_self')
