def get_field(self, name):
return QWizard.field(self, name)
class Twitter(InputPlugin):
name = 'twitter'
hasWizard = True
hasLocationBasedMode = True
hasRateLimitInfo = True
def __init__(self):
# Try and read the labels file
labels_config = self.getConfigObj(self.name + '.labels')
try:
self.labels = labels_config['labels']
except Exception as err:
self.labels = None
logger.error('Could not load the labels file for the ' +
self.name + ' plugin .')
logger.error(err)
self.config, self.options_string = self.readConfiguration(
'string_options')
self.options_boolean = self.readConfiguration('boolean_options')[1]
self.api = None
self.wizard = QWizard()
def searchForTargets(self, search_term):
possibleTargets = []
logger.debug(
'Searching for Targets from Twitter Plugin. Search term is : ' +
search_term)
try:
if self.api is None:
self.api = self.getAuthenticatedAPI()
search_results = self.api.search_users(q=search_term)
if search_results:
logger.debug('Twitter returned ' + str(len(search_results)) +
' results')
for i in search_results:
if self.options_boolean[
'exclude_geo_disabled'] == 'True' and not i.geo_enabled:
continue
target = {'pluginName': 'Twitter Plugin'}
target['targetUserid'] = i.id_str
target['targetUsername'] = i.screen_name
target['targetPicture'] = 'profile_pic_%s' % i.id_str
target['targetFullname'] = i.name
# save the pic in the temp folder to show it later
filename = 'profile_pic_%s' % i.id_str
temp_file = os.path.join(GeneralUtilities.getTempDir(),
filename)
# Retieve and save the profile phot only if it does not exist
if not os.path.exists(temp_file):
urllib.urlretrieve(i.profile_image_url, temp_file)
possibleTargets.append(target)
except tweepy.TweepError as e:
logger.error('Error authenticating with Twitter API.')
logger.error(e)
if e.response.status_code == 429:
remaining, limit, reset = self.getRateLimitStatus(
'search_users')
return False, 'Error authenticating with Twitter: {0}. Your limits will be renewed at : {1}'.format(
e.message, reset.strftime('%Y-%m-%d %H:%M:%S %z'))
return False, 'Error authenticating with Twitter: {0}'.format(
e.message)
except Exception as err:
logger.error(err)
logger.error('Error searching for targets in Twitter plugin.')
return possibleTargets
def getAuthenticatedAPI(self):
try:
auth = tweepy.auth.OAuthHandler(
self.options_string['hidden_application_key'],
self.options_string['hidden_application_secret'])
auth.set_access_token(
self.options_string['hidden_access_token'],
self.options_string['hidden_access_token_secret'])
return tweepy.API(auth)
except Exception as e:
logger.error(e)
return None
def runConfigWizard(self):
try:
oAuthHandler = tweepy.OAuthHandler(
self.options_string['hidden_application_key'],
self.options_string['hidden_application_secret'])
authorizationURL = oAuthHandler.get_authorization_url(True)
self.wizard.setWindowTitle('Twitter plugin configuration wizard')
page1 = QWizardPage()
page2 = QWizardPage()
layout1 = QVBoxLayout()
layout2 = QVBoxLayout()
layoutInputPin = QHBoxLayout()
label1a = QLabel(
'Click next to connect to twitter.com . Please login with your account and follow the instructions in '
'order to authorize creepy')
label2a = QLabel(
'Copy the PIN that you will receive once you authorize cree.py in the field below and click finish'
)
pinLabel = QLabel('PIN')
inputPin = QLineEdit()
inputPin.setObjectName('inputPin')
analysisHtml = QWebEngineView()
analysisHtml.load(QUrl(authorizationURL))
layout1.addWidget(label1a)
layout2.addWidget(analysisHtml)
layout2.addWidget(label2a)
layoutInputPin.addWidget(pinLabel)
layoutInputPin.addWidget(inputPin)
layout2.addLayout(layoutInputPin)
page1.setLayout(layout1)
page2.setLayout(layout2)
page2.registerField('inputPin*', inputPin)
self.wizard.addPage(page1)
self.wizard.addPage(page2)
self.wizard.resize(800, 600)
if self.wizard.exec_():
try:
oAuthHandler.get_access_token(
str(self.wizard.field('inputPin').toString()).strip())
self.options_string[
'hidden_access_token'] = oAuthHandler.access_token
self.options_string[
'hidden_access_token_secret'] = oAuthHandler.access_token_secret
self.saveConfiguration(self.config)
except Exception as err:
logger.error(err)
self.showWarning(
'Error completing the wizard',
'We were unable to obtain the access token for your account, please try to run '
'the wizard again. Error was {0}'.format(err.message))
except Exception as err:
logger.error(err)
self.showWarning('Error completing the wizard',
'Error was: {0}'.format(err.message))
def showWarning(self, title, text):
try:
QMessageBox.warning(self.wizard, title, text, None)
except Exception as err:
logger.error(err)
def getAuthorizationURL(self):
"""
:return: the authorization URL for Twitter or None if there was an exception
"""
try:
if not self.options_string:
self.options_string = self.readConfiguration(
'string_options')[1]
oAuthHandler = tweepy.OAuthHandler(
self.options_string['hidden_application_key'],
self.options_string['hidden_application_secret'])
authorizationURL = oAuthHandler.get_authorization_url(True)
return authorizationURL
except Exception as err:
logger.error(err)
return None
def isConfigured(self):
"""
:return: a tuple. The first element is True or False, depending if the plugin is configured or not. The second
element contains an optional message for the user
"""
try:
if not self.options_string:
self.options_string = self.readConfiguration(
'string_options')[1]
if self.api is None:
oAuthHandler = tweepy.OAuthHandler(
self.options_string['hidden_application_key'],
self.options_string['hidden_application_secret'])
oAuthHandler.set_access_token(
self.options_string['hidden_access_token'],
self.options_string['hidden_access_token_secret'])
self.api = tweepy.API(oAuthHandler)
logger.debug(self.api.me().name)
return True, ''
except tweepy.TweepError as e:
logger.error('Error authenticating with Twitter API.')
logger.error(e)
return False, 'Error authenticating with Twitter. ' + e.message
except Exception as e:
logger.error('Error authenticating with Twitter API.')
logger.error(e)
return False, 'Error authenticating with Twitter. ' + e.message
def returnAnalysis(self, target, search_params):
"""
:param target:
:param search_params:
:return:
"""
conversionResult = False
formalTimezoneName = ''
locations_list = []
twitterDiv = div(id='twitter')
twitterDiv += h2('Twitter profile information')
incl_rts = search_params['boolean']['include_retweets']
excl_rpls = search_params['boolean']['exclude_replies']
if not self.api:
self.api = self.getAuthenticatedAPI()
try:
logger.debug('Attempting to retrieve profile information for ' +
target['targetUsername'])
userObject = self.api.get_user(user_id=target['targetUserid'])
twitterDiv += p('Account was created on {0}'.format(
userObject.created_at.strftime('%Y-%m-%d %H:%M:%S %z')))
if userObject.statuses_count:
twitterDiv += p(
'The user has tweeted {0} times ( including retweets ).'.
format(str(userObject.statuses_count)))
if userObject.name:
twitterDiv += p(u'Self-reported real name: {0}'.format(
userObject.name))
if userObject.description:
twitterDiv += p(u'Description: {0}'.format(
userObject.description))
if userObject.location:
twitterDiv += p(u'Self-reported location: {0}'.format(
userObject.location))
if userObject.time_zone:
conversionResult, formalTimezoneName = self.getTimezoneNameFromReported(
userObject.time_zone)
if conversionResult:
twitterDiv += p(u'Self-reported time zone: {0}'.format(
formalTimezoneName))
else:
twitterDiv += p(u'Self-reported time zone: {0}'.format(
userObject.time_zone))
if userObject.geo_enabled:
twitterDiv += p(
'User has enabled the possibility to geolocate their tweets.'
)
else:
twitterDiv += p(
'User has disabled the possibility to geolocate their tweets.'
)
if userObject.followers_count:
twitterDiv += p('The user has {0} followers.'.format(
str(userObject.followers_count)))
if userObject.friends_count:
twitterDiv += p('The user is following {0} users.'.format(
str(userObject.friends_count)))
if userObject.listed_count:
twitterDiv += p(
'The user is listed in {0} public lists.'.format(
str(userObject.listed_count)))
logger.debug('Attempting to retrieve the tweets for user ' +
target['targetUsername'])
tweets = Cursor(self.api.user_timeline,
count=100,
user_id=target['targetUserid'],
exclude_replies=excl_rpls,
include_rts=incl_rts).items()
timestamps = []
repliedTo = []
retweeted = []
clientApplications = []
mentioned = []
hashtags = []
for j in tweets:
finalDateTimeObj = pytz.utc.localize(j.created_at)
if conversionResult:
createdUtcTimezone = pytz.utc.localize(j.created_at)
try:
userTimezone = pytz.timezone(formalTimezoneName)
createdUserTimezone = userTimezone.normalize(
createdUtcTimezone.astimezone(userTimezone))
timestamps.append(createdUserTimezone)
finalDateTimeObj = createdUserTimezone
except Exception:
finalDateTimeObj = createdUtcTimezone
timestamps.append(createdUtcTimezone)
else:
timestamps.append(pytz.utc.localize(j.created_at))
if j.in_reply_to_screen_name:
repliedTo.append(j.in_reply_to_screen_name)
if hasattr(j, 'retweeted_status'):
retweeted.append(j.retweeted_status.user.name)
if j.source:
clientApplications.append(j.source)
if j.entities:
if hasattr(j, 'entities.user_mentions'):
for mention in j.entities.user_mentions:
mentioned.append(mention.screen_name)
if hasattr(j, 'hashtags'):
for hashtag in j.entities.hashtags:
hashtags.append(hashtag.text)
loc = self.buildLocationFromTweet(j, finalDateTimeObj)
if loc:
locations_list.append(loc)
logger.debug(
'{0} locations were retrieved from Twitter Plugin'.format(
len(locations_list)))
if len(repliedTo) > 0:
twitterDiv += p('User has replied to the following users : ')
repliedToTable = self.createFrequencyTableFromCounter(
Counter(repliedTo), 'User screen name', 'repliedToTable',
10)
twitterDiv += repliedToTable
if len(retweeted) > 0:
twitterDiv += p('User has retweeted the following users : ')
retweetedTable = self.createFrequencyTableFromCounter(
Counter(retweeted), 'User screen name', 'retweetedTable',
10)
twitterDiv += retweetedTable
if len(mentioned) > 0:
twitterDiv += p('User has mentioned the following users : ')
mentionedTable = self.createFrequencyTableFromCounter(
Counter(mentioned), 'User screen name', 'mentionedTable',
10)
twitterDiv += mentionedTable
if len(clientApplications) > 0:
twitterDiv += p('User has been using the following clients : ')
clientApplicationsTable = self.createFrequencyTableFromCounter(
Counter(clientApplications), 'Client Application',
'clientApplicationsTable', 10)
twitterDiv += clientApplicationsTable
if len(hashtags) > 0:
twitterDiv += p('User has used the following hashtags : ')
hashtagsTable = self.createFrequencyTableFromCounter(
Counter(hashtags), 'Hashtag', 'hashtagsTable', 10)
twitterDiv += hashtagsTable
if len(timestamps) > 0:
twitterDiv += p('User tweets time frequency: ')
hourPeriodsTable = self.createFrequencyTableFromList(
self.getTimeChunksFrequency(timestamps, 8), 'Hour Period',
'hourPeriodsTable')
twitterDiv += hourPeriodsTable
twitterDiv += p(
'User has been twitting at the following times : ')
hoursTable = self.createFrequencyTableFromCounter(
Counter([d.hour for d in timestamps]), 'Hour',
'hoursTable', 24)
twitterDiv += hoursTable
except tweepy.TweepError as e:
logger.error('Error authenticating with Twitter API.')
logger.error(e)
if e.response.status_code == 429:
remaining, limit, reset = self.getRateLimitStatus(
'user_timeline')
logger.error(
'Error getting locations from twitter plugin: {0}. Your limits will be renewed at : {1}'
.format(e.message, reset.strftime('%Y-%m-%d %H:%M:%S %z')))
except Exception as e:
logger.error(e)
logger.error('Error getting locations from twitter plugin')
return locations_list, twitterDiv
def searchForResultsNearPlace(self, geocode):
locations_list = []
logger.debug(
'Attempting to retrieve tweets around {0}'.format(geocode))
try:
if self.api is None:
self.api = self.getAuthenticatedAPI()
# Min Radius is 1km, convert m to km and set it to 1km if it is less than that
if not geocode.split(',')[2].endswith('km'):
if int(geocode.split(',')[2].replace('m', '')) < 1000:
geocode = ','.join(geocode.split(',')[:2]) + ',1km'
else:
radius = int(geocode.split(',')[2].replace('m', '')) / 1000
geocode = ','.join(
geocode.split(',')[:2]) + ',{0}km'.format(radius)
tweets = Cursor(self.api.search,
count=100,
q='*',
geocode=geocode,
result_type='recent').items()
for tweet in tweets:
finalDateTimeObj = pytz.utc.localize(tweet.created_at)
loc = self.buildLocationFromTweet(tweet, finalDateTimeObj)
if loc:
locations_list.append(loc)
except tweepy.TweepError as e:
logger.error('Error authenticating with Twitter API.')
logger.error(e)
if e.response.status_code == 429:
remaining, limit, reset = self.getRateLimitStatus(
'search_near_location')
logger.error(
'Error getting locations from twitter plugin: {0}. Your limits will be renewed at : {1}'
.format(e.message, reset.strftime('%Y-%m-%d %H:%M:%S %z')))
except Exception as e:
logger.error(e)
logger.error('Error getting locations from twitter plugin')
return locations_list
def getRateLimitStatus(self, query):
try:
if self.api is None:
self.api = self.getAuthenticatedAPI()
reply = self.api.rate_limit_status()
if query == 'search_users':
return reply['resources']['users']['/users/search']['remaining'], \
reply['resources']['users']['/users/search']['limit'], \
datetime.fromtimestamp(reply['resources']['users']['/users/search']['reset'])
elif query == 'search_near_location':
return reply['resources']['search']['/search/tweets']['remaining'], \
reply['resources']['search']['/search/tweets']['limit'], \
datetime.fromtimestamp(reply['resources']['search']['/search/tweets']['reset'])
elif query == 'user_timeline':
return reply['resources']['statuses']['/statuses/user_timeline']['remaining'], \
reply['resources']['statuses']['/statuses/user_timeline']['limit'], \
datetime.fromtimestamp(reply['resources']['statuses']['/statuses/user_timeline']['reset'])
elif query == 'all':
return reply['resources']['users']['/users/search'], reply['resources']['search']['/search/tweets'], \
reply['resources']['statuses']['/statuses/user_timeline']
except Exception as e:
logger.error(e)
logger.error('Error getting rate limit status from twitter plugin')
def buildLocationFromTweet(self, tweet, finalDateTimeObj):
"""
First Handle the coordinates return field
Twitter API returns GeoJSON, see http://www.geojson.org/geojson-spec.html for the spec
We don't handle MultiPoint, LineString, MultiLineString, MultiPolygon and Geometry Collection
"""
loc = {}
loc['plugin'] = 'twitter'
loc['username'] = tweet.user.screen_name
loc['context'] = tweet.text
loc['date'] = finalDateTimeObj
loc['infowindow'] = self.constructContextInfoWindow(
tweet, finalDateTimeObj)
if tweet.coordinates and tweet.place:
if tweet.coordinates['type'] == 'Point':
loc['lat'] = tweet.coordinates['coordinates'][1]
loc['lon'] = tweet.coordinates['coordinates'][0]
loc['shortName'] = tweet.place.name
loc['accuracy'] = 'high'
elif tweet.coordinates.type == 'PolyGon':
c = self.getCenterOfPolygon(tweet.coordinates['coordinates'])
loc['lat'] = c[1]
loc['lon'] = c[0]
loc['shortName'] = tweet.place.name
loc['accuracy'] = 'low'
return loc
elif tweet.place and not tweet.coordinates:
if tweet.place.bounding_box.type == 'Point':
loc['lat'] = tweet.place.bounding_box.coordinates[1]
loc['lon'] = tweet.place.bounding_box.coordinates[0]
loc['shortName'] = tweet.place.name
loc['accuracy'] = 'high'
elif tweet.place.bounding_box.type == 'Polygon':
c = self.getCenterOfPolygon(
tweet.place.bounding_box.coordinates[0])
loc['lat'] = c[1]
loc['lon'] = c[0]
loc['shortName'] = tweet.place.name
loc['accuracy'] = 'low'
return loc
else:
return {}
def getTimeChunksFrequency(self, timestamps, chunkSize):
hourFreq = Counter([d.hour for d in timestamps])
hourPeriods = self.sliceLinkedList(range(24), chunkSize)
hourPeriodTotals = []
for hPeriod in hourPeriods:
totalTweets = 0
for hour in hPeriod:
totalTweets += hourFreq[hour]
hourPeriodTotals.append(
('{0}-{1}'.format(str(hPeriod[0]),
str(hPeriod[-1])), totalTweets))
hourPeriodTotals.sort(key=lambda x: x[1], reverse=True)
return hourPeriodTotals
def createFrequencyTableFromList(self, lst, objectName, tableName):
objectTable = table(id=tableName, cls='pure-table pure-table-bordered')
head = thead()
head.add(th(objectName))
head.add(th('Count'))
objectTable.add(head)
objectTable.add(tbody())
for item in lst:
row = tr()
row.add(td(item[0]))
row.add(td(item[1]))
objectTable.add(row)
return objectTable
def createFrequencyTableFromCounter(self, counter, objectName, tableName,
itemsCount):
objectTable = table(id=tableName, cls='pure-table pure-table-bordered')
head = thead()
head.add(th(objectName))
head.add(th('Count'))
objectTable.add(head)
objectTable.add(tbody())
for item in counter.most_common(itemsCount):
row = tr()
row.add(td(item[0]))
row.add(td(item[1]))
objectTable.add(row)
return objectTable
def constructContextInfoWindow(self, tweet, finalDateTimeObj):
html = unicode(self.options_string['infowindow_html'])
# returned value also becomes unicode since tweet.text is unicode, and carries the encoding also
return html.replace('@TEXT@', tweet.text). \
replace('@DATE@', finalDateTimeObj.strftime('%Y-%m-%d %H:%M:%S %z')). \
replace('@PLUGIN@', u'twitter').replace('@USERNAME@', tweet.user.screen_name). \
replace('@LINK@', 'https://twitter.com/statuses/{0}'.format(tweet.id_str))
def getCenterOfPolygon(self, coord):
"""
We need to get the "center" of the polygon. Accuracy is not a major aspect here since
we originally got a polygon, so there was not much accuracy to start with. We convert the
polygon to a rectangle by selecting 4 points :
a) Point with the Lowest latitude
b) Point with the Highest Latitude
c) Point with the Lowest Longitude
d) Point with the Highest Longitude
and then get the center of this rectangle as the point to draw on the map
"""
lat_list = []
lon_list = []
for j in coord:
lon_list.append(j[0])
lat_list.append(j[1])
lon_list.sort()
lat_list.sort()
lat = float(lat_list[0]) + (
(float(lat_list[len(lat_list) - 1]) - float(lat_list[0])) / 2)
lon = float(lon_list[0]) + (
(float(lon_list[len(lon_list) - 1]) - float(lon_list[0])) / 2)
return lon, lat
def getLabelForKey(self, key):
"""
read the plugin_name.labels
file and try to get label text for the key that was asked
"""
if not self.labels:
return key
if key not in self.labels.keys():
return key
return self.labels[key]
def sliceLinkedList(self, lst, chunkSize):
n = len(lst)
for i in range(n):
d = n - chunkSize - i
if d > 0:
yield lst[i:i + chunkSize:1]
else:
yield lst[i:] + lst[:-d]
def getTimezoneNameFromReported(self, tzReported):
allTimezones = pytz.all_timezones
for timezone in allTimezones:
if tzReported.lower() in timezone.lower():
return True, timezone
return False, tzReported
def get_field(self, name):
return QWizard.field(self, name)
