def testGetCurrentTags(self):
import mdb
cwd = os.getcwd()
parent = os.path.dirname(cwd)
cfgs = os.path.join(parent, 'config/twitterCrowded.cfg')
p = getConfigParameters(cfgs)
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost,
port=p.dbPort,
db=p.db,
user=p.dbUser,
password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
print "Failed to connect to mongo."
sys.exit(1)
res = cf.getCurrentTags(evCollHandle, 'shitbrick')
self.assertEquals(res, ['hellworld', 'fuckhole', 'shitbrick'])
# Quick test chucked in
results = cf.getQueryBBox(evCollHandle)
print results
def main(p, mediaOnly=None):
""" Coordinates a new twitter stream connection"""
# Logging config
logFile = os.path.join(p.errorPath, p.connErrorFile)
logging.basicConfig(filename=logFile, format="%(levelname)s:: %(asctime)s %(message)s", level=p.logLevel)
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
logging.critical("Failed to connect to db and authenticate.", exc_info=True)
sys.exit()
# Here's the redis queue for managing the tweets as they come in
try:
q = RedisQueue(p.redisName, host=p.redisHost, password=p.redisPassword, port=p.redisPort, db=0)
except:
logging.critical("REDIS: Failed to connect in connectionClient.py. ", exc_info=True)
sys.exit()
# Connection placeholder in case the exception catches the drop out
connection = True
while connection == True:
# Get the existing tags and add the current
try:
tags = cf.getCurrentTags(evCollHandle)
except:
tags = None
logging.error("Failed to get current tags from db.", exc_info=True)
# Build the building boxes
try:
bboxes = cf.getCurrentBBoxes(evCollHandle)
except:
bboxes = None
logging.error("Failed to get current BBOXes from db.", exc_info=True)
if not tags and not bboxes:
logging.warning("Currently no tags or bboxes in the db.")
sys.exit()
try:
print tags, bboxes
with tweetstream.FilterStream(p.sourceUser, p.sourcePassword, track=tags, locations=bboxes) as stream:
for tweet in stream:
if mediaOnly:
try:
q.put(json.dumps(tweet))
except:
logging.critical("Failed to put tweet on redis. This tweet: \n%s" % (tweet), exc_info=True)
except tweetstream.ConnectionError:
logging.critical("Disconnected from twitter", exc_info=True)
def main(p):
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost,
port=p.dbPort,
db=p.db,
user=p.dbUser,
password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
logging.critical('Failed to connect and authenticate', exc_info=True)
sys.exit()
# Get the current tags
tags = cf.getCurrentTags(evCollHandle)
# Get the current bounding boxes
queryBBoxes = cf.getQueryBBox(evCollHandle)
x = 1
while x == 1:
# Here's the redis queue for managing the tweets as they come in
try:
q = RedisQueue(p.redisName,
host=p.redisHost,
password=p.redisPassword,
port=p.redisPort,
db=0)
except:
logging.error('Failed to connect to REDIS db.', exc_info=True)
sys.exit()
# This call is blocking, so expect it to hang on this point
tweetStr = q.get()
tweet = json.loads(tweetStr)
# Work out which object/event this tweet is associated with
if tags:
tweetTags = cf.whichTags(tags, tweet)
for tweetTag in tweetTags:
success = dispatchTweet(p, tweet, tweetTag)
logging.debug("Tag-based message dispatched: %s" % (success))
if queryBBoxes:
tweetGeos = cf.matchesCurrentGeos(queryBBoxes, tweet)
for tweetGeo in tweetGeos:
success = dispatchTweet(p, tweet, tweetGeo)
logging.debug("Geo-based message dispatched: %s" % (success))
def main(p):
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
logging.critical('Failed to connect and authenticate', exc_info=True)
sys.exit()
# Get the current tags
tags = cf.getCurrentTags(evCollHandle)
# Get the current bounding boxes
queryBBoxes = cf.getQueryBBox(evCollHandle)
x = 1
while x == 1:
# Here's the redis queue for managing the tweets as they come in
try:
q = RedisQueue(p.redisName, host=p.redisHost, password=p.redisPassword, port=p.redisPort, db=0)
except:
logging.error('Failed to connect to REDIS db.', exc_info=True)
sys.exit()
# This call is blocking, so expect it to hang on this point
tweetStr = q.get()
tweet = json.loads(tweetStr)
# Work out which object/event this tweet is associated with
if tags:
tweetTags = cf.whichTags(tags, tweet)
for tweetTag in tweetTags:
success = dispatchTweet(p, tweet, tweetTag)
logging.debug("Tag-based message dispatched: %s" %(success))
if queryBBoxes:
tweetGeos = cf.matchesCurrentGeos(queryBBoxes, tweet)
for tweetGeo in tweetGeos:
success = dispatchTweet(p, tweet, tweetGeo)
logging.debug("Geo-based message dispatched: %s" %(success))
def testGetCurrentTags(self):
import mdb
cwd = os.getcwd()
parent = os.path.dirname(cwd)
cfgs = os.path.join(parent, 'config/twitterCrowded.cfg')
p = getConfigParameters(cfgs)
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
print "Failed to connect to mongo."
sys.exit(1)
res = cf.getCurrentTags(evCollHandle, 'shitbrick')
self.assertEquals(res, ['hellworld','fuckhole', 'shitbrick'])
# Quick test chucked in
results = cf.getQueryBBox(evCollHandle)
print results
def main(p, mediaOnly=None):
''' Coordinates a new twitter stream connection'''
# Logging config
logFile = os.path.join(p.errorPath, p.connErrorFile)
logging.basicConfig(filename=logFile,
format='%(levelname)s:: %(asctime)s %(message)s',
level=p.logLevel)
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost,
port=p.dbPort,
db=p.db,
user=p.dbUser,
password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
logging.critical('Failed to connect to db and authenticate.',
exc_info=True)
sys.exit()
# Here's the redis queue for managing the tweets as they come in
try:
q = RedisQueue(p.redisName,
host=p.redisHost,
password=p.redisPassword,
port=p.redisPort,
db=0)
except:
logging.critical("REDIS: Failed to connect in connectionClient.py. ",
exc_info=True)
sys.exit()
# Connection placeholder in case the exception catches the drop out
connection = True
while connection == True:
# Get the existing tags and add the current
try:
tags = cf.getCurrentTags(evCollHandle)
except:
tags = None
logging.error('Failed to get current tags from db.', exc_info=True)
# Build the building boxes
try:
bboxes = cf.getCurrentBBoxes(evCollHandle)
except:
bboxes = None
logging.error('Failed to get current BBOXes from db.',
exc_info=True)
if not tags and not bboxes:
logging.warning('Currently no tags or bboxes in the db.')
sys.exit()
try:
print tags, bboxes
with tweetstream.FilterStream(p.sourceUser,
p.sourcePassword,
track=tags,
locations=bboxes) as stream:
for tweet in stream:
if mediaOnly:
try:
q.put(json.dumps(tweet))
except:
logging.critical(
"Failed to put tweet on redis. This tweet: \n%s"
% (tweet),
exc_info=True)
except tweetstream.ConnectionError:
logging.critical("Disconnected from twitter", exc_info=True)
