def __init__(self, config):
self.config = config
host = config.get("redis", "redis.server").strip('"').strip("'")
port = config.get("redis", "redis.port")
db = config.get("redis", "redis.db")
qname = config.get("alert_bot", "msg_bot.redis_mq").strip('"').strip("'")
qprefix = config.get("alert_bot", "msg_bot.redis_prefix").strip('"').strip("'")
self.rsq = RedisQueue(qname, qprefix, host, port, db)
logging.info('Connect to redis on server->%s:%s db->%s qname->%s:%s' % (host, port, db, qprefix, qname))
jid = config.get("alert_bot", "msg_bot.jid").strip('"').strip("'")
password = config.get("alert_bot", "msg_bot.pass").strip('"').strip("'")
sleekxmpp.ClientXMPP.__init__(self, jid, password)
self.recipients = eval(config.get("alert_bot", "msg_bot.recipients").strip('"').strip("'"))
self.tlock = Lock()
# The session_start event will be triggered when
# the bot establishes its connection with the server
# and the XML streams are ready for use. We want to
# listen for this event so that we we can initialize
# our roster.
self.add_event_handler("session_start", self.start, threaded=True)
def main(p, mediaOnly=None):
""" Coordinates a new twitter stream connection"""
# Logging config
logFile = os.path.join(p.errorPath, p.connErrorFile)
logging.basicConfig(filename=logFile, format="%(levelname)s:: %(asctime)s %(message)s", level=p.logLevel)
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
logging.critical("Failed to connect to db and authenticate.", exc_info=True)
sys.exit()
# Here's the redis queue for managing the tweets as they come in
try:
q = RedisQueue(p.redisName, host=p.redisHost, password=p.redisPassword, port=p.redisPort, db=0)
except:
logging.critical("REDIS: Failed to connect in connectionClient.py. ", exc_info=True)
sys.exit()
# Connection placeholder in case the exception catches the drop out
connection = True
while connection == True:
# Get the existing tags and add the current
try:
tags = cf.getCurrentTags(evCollHandle)
except:
tags = None
logging.error("Failed to get current tags from db.", exc_info=True)
# Build the building boxes
try:
bboxes = cf.getCurrentBBoxes(evCollHandle)
except:
bboxes = None
logging.error("Failed to get current BBOXes from db.", exc_info=True)
if not tags and not bboxes:
logging.warning("Currently no tags or bboxes in the db.")
sys.exit()
try:
print tags, bboxes
with tweetstream.FilterStream(p.sourceUser, p.sourcePassword, track=tags, locations=bboxes) as stream:
for tweet in stream:
if mediaOnly:
try:
q.put(json.dumps(tweet))
except:
logging.critical("Failed to put tweet on redis. This tweet: \n%s" % (tweet), exc_info=True)
except tweetstream.ConnectionError:
logging.critical("Disconnected from twitter", exc_info=True)
def __init__(self, config, rhost=None, rport=None, rdb=None, chatq=None, prefix=None):
if config:
rhost = config.get("redis", "redis.server").strip('"').strip("'")
rport = config.get("redis", "redis.port")
rdb = config.get("redis", "redis.db")
chatq = config.get("alert_bot", "msg_bot.redis_mq").strip('"').strip("'")
prefix = config.get("alert_bot", "msg_bot.redis_prefix").strip('"').strip("'")
self.q = RedisQueue(chatq, prefix, rhost, rport, rdb)
def main(p):
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost,
port=p.dbPort,
db=p.db,
user=p.dbUser,
password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
logging.critical('Failed to connect and authenticate', exc_info=True)
sys.exit()
# Get the current tags
tags = cf.getCurrentTags(evCollHandle)
# Get the current bounding boxes
queryBBoxes = cf.getQueryBBox(evCollHandle)
x = 1
while x == 1:
# Here's the redis queue for managing the tweets as they come in
try:
q = RedisQueue(p.redisName,
host=p.redisHost,
password=p.redisPassword,
port=p.redisPort,
db=0)
except:
logging.error('Failed to connect to REDIS db.', exc_info=True)
sys.exit()
# This call is blocking, so expect it to hang on this point
tweetStr = q.get()
tweet = json.loads(tweetStr)
# Work out which object/event this tweet is associated with
if tags:
tweetTags = cf.whichTags(tags, tweet)
for tweetTag in tweetTags:
success = dispatchTweet(p, tweet, tweetTag)
logging.debug("Tag-based message dispatched: %s" % (success))
if queryBBoxes:
tweetGeos = cf.matchesCurrentGeos(queryBBoxes, tweet)
for tweetGeo in tweetGeos:
success = dispatchTweet(p, tweet, tweetGeo)
logging.debug("Geo-based message dispatched: %s" % (success))
def main(p):
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
logging.critical('Failed to connect and authenticate', exc_info=True)
sys.exit()
# Get the current tags
tags = cf.getCurrentTags(evCollHandle)
# Get the current bounding boxes
queryBBoxes = cf.getQueryBBox(evCollHandle)
x = 1
while x == 1:
# Here's the redis queue for managing the tweets as they come in
try:
q = RedisQueue(p.redisName, host=p.redisHost, password=p.redisPassword, port=p.redisPort, db=0)
except:
logging.error('Failed to connect to REDIS db.', exc_info=True)
sys.exit()
# This call is blocking, so expect it to hang on this point
tweetStr = q.get()
tweet = json.loads(tweetStr)
# Work out which object/event this tweet is associated with
if tags:
tweetTags = cf.whichTags(tags, tweet)
for tweetTag in tweetTags:
success = dispatchTweet(p, tweet, tweetTag)
logging.debug("Tag-based message dispatched: %s" %(success))
if queryBBoxes:
tweetGeos = cf.matchesCurrentGeos(queryBBoxes, tweet)
for tweetGeo in tweetGeos:
success = dispatchTweet(p, tweet, tweetGeo)
logging.debug("Geo-based message dispatched: %s" %(success))
class AlertHelper():
q = None
def __init__(self, config, rhost=None, rport=None, rdb=None, chatq=None, prefix=None):
if config:
rhost = config.get("redis", "redis.server").strip('"').strip("'")
rport = config.get("redis", "redis.port")
rdb = config.get("redis", "redis.db")
chatq = config.get("alert_bot", "msg_bot.redis_mq").strip('"').strip("'")
prefix = config.get("alert_bot", "msg_bot.redis_prefix").strip('"').strip("'")
self.q = RedisQueue(chatq, prefix, rhost, rport, rdb)
def post_msg(self, msg):
self.q.put(msg)
def flush_all(self):
i = 0
while not self.q.empty():
self.q.get()
i+=1
return i
import redis
import time
from twilio.rest import Client
import sendService
import config
from redisQueue import RedisQueue
twilioClient = Client(config.TWILIO['account_sid'], config.TWILIO['auth_token'])
try:
r = redis.StrictRedis(host=config.HOST, port=config.PORT)
sendQueue = RedisQueue(r, config.SEND_LIST)
recievedQueue = RedisQueue(r, config.RECIEVED_LIST)
SendService = sendService.SendService(twilioClient, sendQueue)
while True:
recievedMessage = recievedQueue.pop()
sendMessage = sendQueue.pop()
#print(sendMessage)
if recievedMessage:
print("recieved:", recievedMessage)
SendService.handleReply(recievedMessage)
if sendMessage:
print("sending", sendMessage)
SendService.sendMessage(sendMessage)
def enqueueForDownload(url):
q = RedisQueue("youlinks")
q.put(url)
return "URL Enqueued"
class AlertMsgBot(sleekxmpp.ClientXMPP):
"""
A basic SleekXMPP bot that will log in, send a message,
and then log out.
"""
config = None
rsq= None
recipients = None
quit = False
tlock = None
def __init__(self, config):
self.config = config
host = config.get("redis", "redis.server").strip('"').strip("'")
port = config.get("redis", "redis.port")
db = config.get("redis", "redis.db")
qname = config.get("alert_bot", "msg_bot.redis_mq").strip('"').strip("'")
qprefix = config.get("alert_bot", "msg_bot.redis_prefix").strip('"').strip("'")
self.rsq = RedisQueue(qname, qprefix, host, port, db)
logging.info('Connect to redis on server->%s:%s db->%s qname->%s:%s' % (host, port, db, qprefix, qname))
jid = config.get("alert_bot", "msg_bot.jid").strip('"').strip("'")
password = config.get("alert_bot", "msg_bot.pass").strip('"').strip("'")
sleekxmpp.ClientXMPP.__init__(self, jid, password)
self.recipients = eval(config.get("alert_bot", "msg_bot.recipients").strip('"').strip("'"))
self.tlock = Lock()
# The session_start event will be triggered when
# the bot establishes its connection with the server
# and the XML streams are ready for use. We want to
# listen for this event so that we we can initialize
# our roster.
self.add_event_handler("session_start", self.start, threaded=True)
def start(self, event):
"""
Process the session_start event.
Typical actions for the session_start event are
requesting the roster and broadcasting an initial
presence stanza.
Arguments:
event -- An empty dictionary. The session_start
event does not provide any additional
data.
"""
self.send_presence()
self.get_roster()
self.rsq.put("Greetings! The alert bot has just started!")
t = threading.Thread(target = self.process_alerts(), args=())
t.start()
# self.send_message(mto=self.recipient,
# mbody=self.msg,
# mtype='chat')
def process_alerts(self):
self.quit = False
while self.quit <> True:
if not self.rsq.empty():
msg = self.rsq.get()
logging.debug('process_alerts: received msg: {%s}' % msg)
self.send_msg(msg)
sleep(1)
# Using wait=True ensures that the send queue will be
# emptied before ending the session.
self.disconnect(wait=True)
def send_msg(self, msg):
self.tlock.acquire()
try:
for r in self.recipients:
self.send_message(r, msg, mtype='chat')
finally:
self.tlock.release()
def main(p, mediaOnly=None):
''' Coordinates a new twitter stream connection'''
# Logging config
logFile = os.path.join(p.errorPath, p.connErrorFile)
logging.basicConfig(filename=logFile,
format='%(levelname)s:: %(asctime)s %(message)s',
level=p.logLevel)
# The mongo bits
try:
c, dbh = mdb.getHandle(host=p.dbHost,
port=p.dbPort,
db=p.db,
user=p.dbUser,
password=p.dbPassword)
evCollHandle = dbh[p.eventsCollection]
except:
logging.critical('Failed to connect to db and authenticate.',
exc_info=True)
sys.exit()
# Here's the redis queue for managing the tweets as they come in
try:
q = RedisQueue(p.redisName,
host=p.redisHost,
password=p.redisPassword,
port=p.redisPort,
db=0)
except:
logging.critical("REDIS: Failed to connect in connectionClient.py. ",
exc_info=True)
sys.exit()
# Connection placeholder in case the exception catches the drop out
connection = True
while connection == True:
# Get the existing tags and add the current
try:
tags = cf.getCurrentTags(evCollHandle)
except:
tags = None
logging.error('Failed to get current tags from db.', exc_info=True)
# Build the building boxes
try:
bboxes = cf.getCurrentBBoxes(evCollHandle)
except:
bboxes = None
logging.error('Failed to get current BBOXes from db.',
exc_info=True)
if not tags and not bboxes:
logging.warning('Currently no tags or bboxes in the db.')
sys.exit()
try:
print tags, bboxes
with tweetstream.FilterStream(p.sourceUser,
p.sourcePassword,
track=tags,
locations=bboxes) as stream:
for tweet in stream:
if mediaOnly:
try:
q.put(json.dumps(tweet))
except:
logging.critical(
"Failed to put tweet on redis. This tweet: \n%s"
% (tweet),
exc_info=True)
except tweetstream.ConnectionError:
logging.critical("Disconnected from twitter", exc_info=True)
import time
from redis import StrictRedis
from redisQueue import RedisQueue
qq = RedisQueue('rq', host='192.168.1.119', port=6379, db=1)
redis = StrictRedis(host='192.168.1.119', port=6379, db=2)
pubsub = redis.pubsub()
def event_handlerx(msg):
print('Handler', qq.exists(msg['data']))
qq.delete(msg['data'])
print('Handler', qq.exists(msg['data']))
pubsub.psubscribe(**{'__keyevent@1__:*': event_handlerx})
print('Starting message loop')
while True:
message = pubsub.get_message()
if message:
print(message)
else:
time.sleep(4)
def findProjectFiles(dConfig):
qRedis = RedisQueue(dConfig['redis-queue-name'],
namespace='queue',
host=dConfig['redis-loc'])
oES = Elasticsearch(dConfig['es-instance-locs'])
lIgnoreDirs = ['.git', '.svn']
dProject = {}
dSource = {}
dProject['_index'] = dConfig['es-index-name']
dProject['_type'] = dConfig['es-index-type']
dSource['crawl-time'] = dConfig['time-stamp']
dSource['project-path'] = qRedis.get(block=True)
lProjectFiles = []
# if project path is '**done**', then break
while dSource['project-path'] != '**done**':
dSource['project-name'] = os.path.basename(dSource['project-path'])
if dConfig['debug']:
debug('func: findProjectFiles()', 'project-path:',
dSource['project-path'], dSource['project-name'])
for sRoot, lDirs, lFiles in os.walk(dSource['project-path']):
if len(lProjectFiles) > dConfig['es-bulk-chunk-size']:
# ingest chunk into elasticsearch
helpers.bulk(oES, lProjectFiles)
if dConfig['debug']:
debug('func: findProjectFiles()', str(len(lProjectFiles)),
'files loaded into elasticsearch')
lProjectFiles = []
for sFile in lFiles:
sFilePath = os.path.join(sRoot, sFile)
sRelPath = os.path.relpath(sFilePath, dSource['project-path'])
dFile = {}
try:
sRelPath.decode('utf-8')
except (ValueError, UnicodeDecodeError) as e:
try:
sRelPath.decode('latin-1')
except (ValueError, UnicodeDecodeError) as e:
try:
sRelPath.decode('utf-16')
except (ValueError, UnicodeDecodeError) as e:
warning(
'func findProjectFiles():', 'sProjectPath:',
dSource['project-path'], 'sProjectName:',
dSource['project-name'], 'sFile:', sFile,
'sRelPath:', sRelPath,
'utf-8, latin-1, and utf-16 decoding failed',
'exception:', e)
else:
dSource['file'] = sFile.decode('utf-16')
dSource['path'] = sRelPath.decode('utf-16')
dProject['_source'] = dSource
lProjectFiles.append(dProject)
else:
dSource['file'] = sFile.decode('latin-1')
dSource['path'] = sRelPath.decode('latin-1')
dProject['_source'] = dSource
lProjectFiles.append(dProject)
else:
dSource['file'] = sFile
dSource['path'] = sRelPath
dProject['_source'] = dSource
lProjectFiles.append(dProject)
lDirs[:] = [sDir for sDir in lDirs if sDir not in lIgnoreDirs]
# get next project to process
dSource['project-path'] = qRedis.get(block=True)
# index any remaining projects
if len(lProjectFiles) > 0:
# ingest chunk into elasticsearch
helpers.bulk(oES, lProjectFiles)
if dConfig['debug']:
debug('func: findProjectFiles()', str(len(lProjectFiles)),
'files loaded into elasticsearch')
def findBuildFiles(tTup):
# unpack inputs from input tuple
(sSearchString, dConfig) = tTup
dConfig['redis-queue-name'] = dConfig['redis-queue-name'] % sSearchString
if dConfig['debug']:
debug('func: findBuildFiles() dConfig[\'redis-queue-name\']:',
dConfig['redis-queue-name'])
qRedis = RedisQueue(dConfig['redis-queue-name'],
namespace='queue',
host=dConfig['redis-loc'])
# ensure redis queue is empty prior to starting consumers
qRedis.flush()
# setup elasticsearch client
oES = Elasticsearch(dConfig['es-instance-locs'])
#sQuery = { "query" : "file:" + sSearchString + "&lowercase_expanded_terms=false" }
dQuery = {}
dQuery['query'] = {}
dQuery['query']['query_string'] = {}
dQuery['query']['query_string']['fields'] = ['file.raw']
# dQuery['query']['query_string']['fields'] = ['file.analyzed']
dQuery['query']['query_string']['query'] = sSearchString
dQuery['query']['query_string']['lowercase_expanded_terms'] = False
if dConfig['debug']:
debug('func: findBuildFiles() dQuery:', json.dumps(dQuery))
''' ### working 10 document fetch
dResponse = oES.search( index=dConfig['es-file-index-name'], doc_type=dConfig['es-file-index-type'], body=json.dumps(dQuery) )
if dConfig['debug']:
for sHit in dResponse['hits']['hits']:
debug('func: findBuildFiles() sHit: ', sHit)
'''
# scroll time set to 10 minutes, change as needed -- required for consistent results, the scroll token expires at the end of scroll time
sScrollTime = "10m"
dResponse = oES.search(index=dConfig['es-file-index-name'],
doc_type=dConfig['es-file-index-type'],
body=json.dumps(dQuery),
search_type="scan",
scroll=sScrollTime)
sScrollId = dResponse['_scroll_id']
if dConfig['debug']:
debug('func: findBuildFiles() (after initial search) dResponse: ',
dResponse)
if dConfig['debug']:
debug('func: findBuildFiles() search hits: ',
dResponse['hits']['total'])
#while not dResponse['timed_out'] and dResponse['hits']['hits']['total'] > 0:
while not dResponse['timed_out'] and dResponse['hits']['total'] > 0:
dResponse = oES.scroll(scroll_id=sScrollId, scroll=sScrollTime)
sScrollId = dResponse['_scroll_id']
#if dConfig['debug']: debug('func: findBuildFiles() dResponse: ', dResponse)
if len(dResponse['hits']['hits']) > 0:
for sHit in dResponse['hits']['hits']:
if dConfig['debug']:
debug('func: findBuildFiles() sHit: ', sHit)
else:
break
__author__ = 'leena'
from redisQueue import RedisQueue
q = RedisQueue('test')
q.put('hello world')
