def on_following(self, ids):
# TODO: stop should actually do something
if hasattr(self, 'stream'):
self.stream.stop()
if common.pref('twitter.streaming', default=False):
from twitterstream import TwitterStream
self.stream = TwitterStream(self.username, self.password, ids)
self.stream.on_tweet += self.on_stream_tweet
self.stream.start()
def on_following(self, ids):
# TODO: stop should actually do something
if hasattr(self, 'stream'):
self.stream.stop()
if common.pref('twitter.streaming', default=False):
from twitterstream import TwitterStream
self.stream = TwitterStream(self.username, self.password, ids)
self.stream.on_tweet += self.on_stream_tweet
self.stream.start()
class TwitterProtocol(object):
event_names = '''
state_changed
following
reply
trends
on_unread_counts
on_feeds
on_edit_feed
on_view
on_change_view
status_update_clicked
recent_timeline
self_tweet
on_corrupted_database
update_social_ids
received_whole_update
'''.split()
def __init__(self, username, password):
self.username = username
self.password = password
self.recent_timeline = []
self.self_tweet = None
self.trends = {}
self.feeds = []
self.feeds_by_name = {}
self.unread_counts = []
e = self.events = Storage((name, Delegate()) for name in self.event_names)
e.following += self.on_following
e.trends += self.on_trends
e.on_unread_counts += self.on_unread_counts
e.recent_timeline += self.on_recent_timeline
e.self_tweet += self.on_self_tweet
e.on_feeds += self.on_feeds
e.on_change_view += self.on_change_view
e.on_view += self.on_view_changed
def render_tweets(tweets, render_context):
return htmlize_tweets(self, tweets)
self.social_feed = SocialFeed('twitter_' + self.username,
'twitter_' + self.username,
self.get_tweet_feed,
render_tweets,
lambda: self.account.set_infobox_dirty)
def _get_database_path(self):
webview = self.webkitcontroller.webview
return webview.GetDatabasePath('digsbysocial_' + self.username)
def set_options(self, options):
guithread(lambda: self.webkitcontroller.JSCall('setAccountOptions', **options))
def on_change_view(self, feed_name):
log.info('on_change_view %r', feed_name)
window = self.webkitcontroller.FeedWindow
if window is not None:
log.info(' found a window, calling switch_to_view')
window.switch_to_view(feed_name)
tlw = window.Top
if tlw.IsIconized(): tlw.Iconize(False)
window.Top.Raise()
else:
log.info(' no window found, calling open_timeline_window')
self.webkitcontroller.open_timeline_window(feed_name)
def on_view_changed(self, feed_name):
feed = self.feeds_by_name.get(feed_name, None)
if feed is not None and feed.get('query', None) is not None and feed.get('save', False):
hooks.notify('digsby.statistics.twitter.viewed_search')
def on_feeds(self, feeds):
self.feeds = feeds
self.feeds_by_name = dict((f['name'], f) for f in feeds)
self.feeds_by_name.update(favorites=dict(name='favorites', label=_('Favorites')),
history=dict(name='history', label=_('History')))
import twitter_notifications as tnots
tnots._update_notifications(self, feeds)
self._save_feeds(feeds)
def _save_feeds(self, feeds):
# don't include non-saved searches
def should_save(f):
return f['type'] not in ('search', 'user') or f.get('save', False)
feeds_pref = filter(should_save, deepcopy(feeds))
# don't serialize certain attributes out to prefs
for feed in feeds_pref:
for attr in ('count', 'label'):
feed.pop(attr)
self.set_account_pref('feeds', feeds_pref)
@property
def account_prefix(self):
return 'twitter.' + self.username
def account_pref_key(self, name):
return _account_pref_key(self.username, name)
def set_account_pref(self, name, value):
from common import setpref
value = simplejson.dumps(value)
setpref(self.account_pref_key(name), value)
def get_account_pref(self, name, default):
return _get_account_pref(self.username, name, default)
def on_unread_counts(self, opts):
self.unread_counts = opts.get('feeds')
self.unread_total = opts.get('total')
def on_recent_timeline(self, tweets):
self.recent_timeline = [to_storage(t) for t in tweets]
self.recent_timeline.reverse()
self.events.update_social_ids()
def update_social_ids(self):
try:
t = self._socialtimer
except AttributeError:
def later():
ids = [p['id'] for p in self.recent_timeline]
self.social_feed.new_ids(ids)
t = self._socialtimer = wx.PyTimer(later)
if not t.IsRunning():
t.StartOneShot(1000)
def on_self_tweet(self, tweet):
self.self_tweet = to_storage(tweet)
def on_following(self, ids):
# TODO: stop should actually do something
if hasattr(self, 'stream'):
self.stream.stop()
if common.pref('twitter.streaming', default=False):
from twitterstream import TwitterStream
self.stream = TwitterStream(self.username, self.password, ids)
self.stream.on_tweet += self.on_stream_tweet
self.stream.start()
def on_trends(self, trends):
# TODO: store trends over time?
#self.trends.update(trends['trends'])
trends = trends['trends']
self.trends = trends[trends.keys()[0]]
def on_stream_tweet(self, tweet):
if self.webkitcontroller is not None:
wx.CallAfter(self.webkitcontroller.realtime_tweet, tweet)
def connect(self, accountopts):
@guithread
def later():
self.webkitcontroller = TwitterWebKitController(self)
self.webkitcontroller.initialize(self.username,
self.password,
self.get_user_feeds(),
accountopts)
self.init_webkit_methods()
def _verify_databases(self):
# webkit doesn't release file object locks for corrupted databases,
# so check the integrity of the databases we care about here first.
# upon any errors, they are deleted.
import sqlite3
def try_query_remove_on_error(dbpath, query):
'''try a query on database dbpath. dbpath is deleted on any
exception.'''
dbpath = path(dbpath)
log.info('verifying db %r', dbpath)
if not dbpath.isfile():
log.info('not a file')
return
try:
conn = sqlite3.connect(dbpath)
with conn:
conn.execute(query)
conn.close()
except Exception:
traceback.print_exc()
with traceguard:
log.warning('exception encountered, removing %r', dbpath)
dbpath.remove()
log.warning('remove completed')
# check the integrity of the "index" database that webkit uses to track
# each site's database
try_query_remove_on_error(
path(self.webkitcontroller.webview.GetDatabaseDirectory()) / 'Databases.db',
'select * from Databases limit 1')
# calling window.openDatabase is necessary once for the below
# _get_database_path() call to work.
self.webkitcontroller.webview.RunScript(
'''var test_db = window.openDatabase('_test_db_', "1.0", "test db", 1024);''')
# ensure the twitter database is okay.
try_query_remove_on_error(
self._get_database_path(),
'create table if not exists _test (foo int)')
def get_user_feeds(self):
def deffeed(n):
return dict(name=n, type=n)
default_feeds = [deffeed(n) for n in
('timeline', 'mentions', 'directs')]
userfeeds = self.get_account_pref('feeds', default_feeds)
def revert():
log.warning('REVERTING user feeds, was %r:', userfeeds)
self.set_account_pref('feeds', default_feeds)
return default_feeds
from pprint import pprint; pprint(userfeeds)
if not isinstance(userfeeds, list):
return revert()
try:
if userfeeds is not default_feeds:
for feed in default_feeds:
for ufeed in userfeeds:
if feed['type'] == ufeed['type']:
break
else:
return revert()
except Exception:
traceback.print_exc()
return revert()
return userfeeds
def init_webkit_methods(self):
# forward some methods to webkitcontroller
for method_name in '''
open_timeline_window
clear_cache
update
on_status
on_status_with_error_popup
add_feed
edit_feed
delete_feed
set_feeds
add_group
get_users
get_prefs'''.split():
setattr(self, method_name, getattr(self.webkitcontroller, method_name))
def json(self, *a, **k):
self.webkitcontroller.json(*a, **k)
def disconnect(self):
self.webkitcontroller.disconnect()
def mark_all_as_read(self):
self.webkitcontroller.evaljs('markAllAsRead();')
def on_reply(self, id, screen_name, text):
from .twitter_gui import TwitterFrame
TwitterFrame.Reply(id, screen_name, text)
def on_retweet(self, id, screen_name, text):
from .twitter_gui import TwitterFrame
TwitterFrame.Retweet(id, screen_name, text)
def on_direct(self, screen_name):
from .twitter_gui import TwitterFrame
TwitterFrame.Direct(screen_name)
def mark_feed_as_read(self, feed_name):
self.webkitcontroller.JSCall('markFeedAsRead', feedName=feed_name)
def toggle_addstocount(self, feed_name):
self.webkitcontroller.JSCall('toggleAddsToCount', feedName=feed_name)
def get_ids_and_context(self, _feed_context):
#_feed_context ?= tab
return list(t['id'] for t in self.get_tweet_feed()), self.recent_timeline
def get_tweet_feed(self):
self_id = self.self_tweet['id'] if self.self_tweet is not None else None
for tweet in self.recent_timeline:
if self_id is None or self_id != tweet['id']:
yield tweet
import sys;
from twitterstream import TwitterStream;
import zerorpc
import json
twitterObj = TwitterStream()
list_of_twittObj = twitterObj.fetchSample(sys.argv[1])
c = zerorpc.Client()
c.connect("tcp://127.0.0.1:4242")
dict_of_id_and_text = {}
dict_of_id_and_twittObj = {}
for t in list_of_twittObj:
dict_of_id_and_text[t.twitt_id] = t.text
dict_of_id_and_twittObj[t.twitt_id] = t
# print dict_of_id_and_text
twee_id_with_sentiment = c.get_sentiment_of_list_of_tweets(dict_of_id_and_text)
# print twee_id_with_sentiment
output = []
# print (type(twee_id_with_sentiment).__name__)
for key,value in twee_id_with_sentiment.iteritems():
twittObj = dict_of_id_and_twittObj.get(key)
twittObj.set_sentiment(value)
output.append(twittObj)
print json.dumps(output, default=lambda o: o.__dict__)
#print list_of_twittObj[0].text
# hack = HackAriba()
# print "hackariba called"
sys.stdout.flush()
class TwitterProtocol(object):
event_names = '''
state_changed
following
reply
trends
on_unread_counts
on_feeds
on_edit_feed
on_view
on_change_view
status_update_clicked
recent_timeline
self_tweet
on_corrupted_database
update_social_ids
received_whole_update
'''.split()
def __init__(self, username, password):
self.username = username
self.password = password
self.recent_timeline = []
self.self_tweet = None
self.trends = {}
self.feeds = []
self.feeds_by_name = {}
self.unread_counts = []
e = self.events = Storage(
(name, Delegate()) for name in self.event_names)
e.following += self.on_following
e.trends += self.on_trends
e.on_unread_counts += self.on_unread_counts
e.recent_timeline += self.on_recent_timeline
e.self_tweet += self.on_self_tweet
e.on_feeds += self.on_feeds
e.on_change_view += self.on_change_view
e.on_view += self.on_view_changed
def render_tweets(tweets, render_context):
return htmlize_tweets(self, tweets)
self.social_feed = SocialFeed('twitter_' + self.username,
'twitter_' + self.username,
self.get_tweet_feed, render_tweets,
lambda: self.account.set_infobox_dirty)
def _get_database_path(self):
webview = self.webkitcontroller.webview
return webview.GetDatabasePath('digsbysocial_' + self.username)
def set_options(self, options):
guithread(lambda: self.webkitcontroller.JSCall('setAccountOptions', **
options))
def on_change_view(self, feed_name):
log.info('on_change_view %r', feed_name)
window = self.webkitcontroller.FeedWindow
if window is not None:
log.info(' found a window, calling switch_to_view')
window.switch_to_view(feed_name)
tlw = window.Top
if tlw.IsIconized(): tlw.Iconize(False)
window.Top.Raise()
else:
log.info(' no window found, calling open_timeline_window')
self.webkitcontroller.open_timeline_window(feed_name)
def on_view_changed(self, feed_name):
feed = self.feeds_by_name.get(feed_name, None)
if feed is not None and feed.get(
'query', None) is not None and feed.get('save', False):
hooks.notify('digsby.statistics.twitter.viewed_search')
def on_feeds(self, feeds):
self.feeds = feeds
self.feeds_by_name = dict((f['name'], f) for f in feeds)
self.feeds_by_name.update(favorites=dict(name='favorites',
label=_('Favorites')),
history=dict(name='history',
label=_('History')))
import twitter_notifications as tnots
tnots._update_notifications(self, feeds)
self._save_feeds(feeds)
def _save_feeds(self, feeds):
# don't include non-saved searches
def should_save(f):
return f['type'] not in ('search', 'user') or f.get('save', False)
feeds_pref = filter(should_save, deepcopy(feeds))
# don't serialize certain attributes out to prefs
for feed in feeds_pref:
for attr in ('count', 'label'):
feed.pop(attr)
self.set_account_pref('feeds', feeds_pref)
@property
def account_prefix(self):
return 'twitter.' + self.username
def account_pref_key(self, name):
return _account_pref_key(self.username, name)
def set_account_pref(self, name, value):
from common import setpref
value = simplejson.dumps(value)
setpref(self.account_pref_key(name), value)
def get_account_pref(self, name, default):
return _get_account_pref(self.username, name, default)
def on_unread_counts(self, opts):
self.unread_counts = opts.get('feeds')
self.unread_total = opts.get('total')
def on_recent_timeline(self, tweets):
self.recent_timeline = [to_storage(t) for t in tweets]
self.recent_timeline.reverse()
self.events.update_social_ids()
def update_social_ids(self):
try:
t = self._socialtimer
except AttributeError:
def later():
ids = [p['id'] for p in self.recent_timeline]
self.social_feed.new_ids(ids)
t = self._socialtimer = wx.PyTimer(later)
if not t.IsRunning():
t.StartOneShot(1000)
def on_self_tweet(self, tweet):
self.self_tweet = to_storage(tweet)
def on_following(self, ids):
# TODO: stop should actually do something
if hasattr(self, 'stream'):
self.stream.stop()
if common.pref('twitter.streaming', default=False):
from twitterstream import TwitterStream
self.stream = TwitterStream(self.username, self.password, ids)
self.stream.on_tweet += self.on_stream_tweet
self.stream.start()
def on_trends(self, trends):
# TODO: store trends over time?
#self.trends.update(trends['trends'])
trends = trends['trends']
self.trends = trends[trends.keys()[0]]
def on_stream_tweet(self, tweet):
if self.webkitcontroller is not None:
wx.CallAfter(self.webkitcontroller.realtime_tweet, tweet)
def connect(self, accountopts):
@guithread
def later():
self.webkitcontroller = TwitterWebKitController(self)
self.webkitcontroller.initialize(self.username, self.password,
self.get_user_feeds(),
accountopts)
self.init_webkit_methods()
def _verify_databases(self):
# webkit doesn't release file object locks for corrupted databases,
# so check the integrity of the databases we care about here first.
# upon any errors, they are deleted.
import sqlite3
def try_query_remove_on_error(dbpath, query):
'''try a query on database dbpath. dbpath is deleted on any
exception.'''
dbpath = path(dbpath)
log.info('verifying db %r', dbpath)
if not dbpath.isfile():
log.info('not a file')
return
try:
conn = sqlite3.connect(dbpath)
with conn:
conn.execute(query)
conn.close()
except Exception:
traceback.print_exc()
with traceguard:
log.warning('exception encountered, removing %r', dbpath)
dbpath.remove()
log.warning('remove completed')
# check the integrity of the "index" database that webkit uses to track
# each site's database
try_query_remove_on_error(
path(self.webkitcontroller.webview.GetDatabaseDirectory()) /
'Databases.db', 'select * from Databases limit 1')
# calling window.openDatabase is necessary once for the below
# _get_database_path() call to work.
self.webkitcontroller.webview.RunScript(
'''var test_db = window.openDatabase('_test_db_', "1.0", "test db", 1024);'''
)
# ensure the twitter database is okay.
try_query_remove_on_error(
self._get_database_path(),
'create table if not exists _test (foo int)')
def get_user_feeds(self):
def deffeed(n):
return dict(name=n, type=n)
default_feeds = [
deffeed(n) for n in ('timeline', 'mentions', 'directs')
]
userfeeds = self.get_account_pref('feeds', default_feeds)
def revert():
log.warning('REVERTING user feeds, was %r:', userfeeds)
self.set_account_pref('feeds', default_feeds)
return default_feeds
from pprint import pprint
pprint(userfeeds)
if not isinstance(userfeeds, list):
return revert()
try:
if userfeeds is not default_feeds:
for feed in default_feeds:
for ufeed in userfeeds:
if feed['type'] == ufeed['type']:
break
else:
return revert()
except Exception:
traceback.print_exc()
return revert()
return userfeeds
def init_webkit_methods(self):
# forward some methods to webkitcontroller
for method_name in '''
open_timeline_window
clear_cache
update
on_status
on_status_with_error_popup
add_feed
edit_feed
delete_feed
set_feeds
add_group
get_users
get_prefs'''.split():
setattr(self, method_name,
getattr(self.webkitcontroller, method_name))
def json(self, *a, **k):
self.webkitcontroller.json(*a, **k)
def disconnect(self):
self.webkitcontroller.disconnect()
def mark_all_as_read(self):
self.webkitcontroller.evaljs('markAllAsRead();')
def on_reply(self, id, screen_name, text):
from .twitter_gui import TwitterFrame
TwitterFrame.Reply(id, screen_name, text)
def on_retweet(self, id, screen_name, text):
from .twitter_gui import TwitterFrame
TwitterFrame.Retweet(id, screen_name, text)
def on_direct(self, screen_name):
from .twitter_gui import TwitterFrame
TwitterFrame.Direct(screen_name)
def mark_feed_as_read(self, feed_name):
self.webkitcontroller.JSCall('markFeedAsRead', feedName=feed_name)
def toggle_addstocount(self, feed_name):
self.webkitcontroller.JSCall('toggleAddsToCount', feedName=feed_name)
def get_ids_and_context(self, _feed_context):
#_feed_context ?= tab
return list(t['id']
for t in self.get_tweet_feed()), self.recent_timeline
def get_tweet_feed(self):
self_id = self.self_tweet['id'] if self.self_tweet is not None else None
for tweet in self.recent_timeline:
if self_id is None or self_id != tweet['id']:
yield tweet
key_table.update({'stream_name': stream_name},
{'$set': {'active': False, 'stream_name': ''}}, upsert=False, multi=True)
if __name__ == '__main__':
# Create rotate logger handler
logger = logging.getLogger(sys.argv[2])
logger.setLevel(logging.INFO)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
rtlBaseFile = '../log/stream/stream.log'
handler = logging.handlers.RotatingFileHandler(rtlBaseFile, maxBytes=1024*1024*4, backupCount=9)
handler.setFormatter(formatter)
logger.addHandler(handler)
stream = TwitterStream(getcwd() + '/pid/' + sys.argv[2] + '.pid') # must be an absolute path here
if len(sys.argv) == 3:
if 'start' == sys.argv[1]:
client = MongoClient('localhost', 27017)
print 'Setting things up'
twitter_key = get_twitter_key(client, sys.argv[2])
stream.oauth_token = oauth.Token(key=twitter_key['access_token_key'],
secret=twitter_key['access_token_secret'])
stream.oauth_consumer = oauth.Consumer(key=twitter_key['consumer_key'],
secret=twitter_key['consumer_secret'])
print 'Key OK'
# publisher_list must be a string