# -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4
# PYTHONPATH should be set so that if this is part of a Flumotion project,
# it points to misc so we can import setup.py from there
try:
from misc import setup
setup.setup()
except ImportError:
print "Not importing misc.setup"
pass
from flumotion.common import boot
boot.init_gobject()
boot.init_gst()
from twisted.internet import gtk2reactor
gtk2reactor.install(useGtk=False)
# reinstall our import hooks that could have been overridden by
# pygtk/pygst ltihooks
from flumotion.common import package
package.getPackager().install()
# monkey patching twisted doc errors
from twisted.spread import pb
def login(self, credentials, client=None):
"""Login and get perspective from remote PB server.
Currently only credentials implementing IUsernamePassword are
# -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4
# PYTHONPATH should be set so that if this is part of a Flumotion project,
# it points to misc so we can import setup.py from there
try:
from misc import setup
setup.setup()
except ImportError:
print "Not importing misc.setup"
pass
# we import docutils before installing the packager to prevent
# messing with the datadir which causes epydoc to crash
try:
import docutils
except ImportError:
print "Not importing docutils"
pass
from flumotion.common import boot
boot.init_gobject()
boot.init_gst()
from twisted.internet import gtk2reactor
gtk2reactor.install(useGtk=False)
# reinstall our import hooks that could have been overridden by
# pygtk/pygst ltihooks
from flumotion.common import package
package.getPackager().install()
def main():
'''main process'''
logger = getLogger()
# create DB if does not exist
ID = setup()
# Connect to DB
logger.info('Connecting to %s.db' is ID)
conn = sqlite3.connect('data/%s.db' % ID)
def signal_handler(signal, frame):
# Close connection on interrupt
conn.close()
sys.stdout.flush()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
logger.info('Getting twitter connection')
twitter = getTwitter()
nodes = [
#{'geocode': '40.783288,-73.967090,7mi', 'since': '0'},
#{'geocode': '40.729992,-73.993841,7mi', 'since': '0'},
#{'geocode': '40.830778,-73.942806,7mi', 'since': '0'}
{'geocode': '40.830956,-73.910179,7mi', 'since': '0'},
{'geocode': '40.663972,-73.956871,8mi', 'since': '0'},
{'geocode': '40.688708,-73.779544,8mi', 'since': '0'},
{'geocode': '40.580584,-74.152908,9mi', 'since': '0'}
]
# Total of 20 seconds sleep between rounds
sleep = 20.
# today = datetime.datetime.today()
while True:
for node in nodes:
# Execute Query
try:
t = twitter.search.tweets(geocode=node['geocode'], result_type='recent',
count=100, since_id=node['since'])
except Exception, e:
logger.info('Error getting tweet: %s' % e)
# Could be twitter is overloaded, sleep for a minute before
# starting again
time.sleep(60)
continue
# Update since
# node['since'] = t['search_metadata']['max_id_str']
# Print status
# print node['geocode'], len(t['statuses']),
# str(datetime.datetime.now())
# Go through the results and create arrays to add to DB
tweets = []
users = []
logger.info('Collecting geotagged tweets')
for status in t['statuses']:
if status['geo'] != None:
user = status['user']
del status['user']
timestamp = int(datetime.datetime.strptime(
status['created_at'],
'%a %b %d %H:%M:%S +0000 %Y'
).strftime("%s"))
tweets.append((
status['id'],
timestamp,
status['geo']['coordinates'][0],
status['geo']['coordinates'][1],
status['text'],
user['id'],
status['retweet_count'],
status['favorite_count'],
getSource(status['source']),
ID,
json.dumps(status)
))
users.append((
user['id'],
timestamp,
json.dumps(user)
))
else:
pass
# Add to DB
try:
cursor = conn.cursor()
cursor.executemany('INSERT OR IGNORE INTO tweets VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', tweets)
cursor.executemany(
'INSERT OR IGNORE INTO users VALUES (?, ?, ?)', users)
conn.commit()
node['since'] = t['search_metadata']['max_id_str']
logger.info('Saved tweets to db: %d' % len(tweets))
except Exception, e:
logger.info('Failed saving tweets, reconnecting: %s' % e)
time.sleep(60)
conn = sqlite3.connect('data/%s.db' % ID)
# Sleep between nodes
time.sleep(sleep / len(nodes))
