예제 #1
0
    def __init__(self, config):
        

        self.config = config
        host = config.get("redis", "redis.server").strip('"').strip("'")
        port = config.get("redis", "redis.port")
        db = config.get("redis", "redis.db")
        qname = config.get("alert_bot", "msg_bot.redis_mq").strip('"').strip("'")
        qprefix = config.get("alert_bot", "msg_bot.redis_prefix").strip('"').strip("'")
        
        self.rsq = RedisQueue(qname, qprefix, host, port, db)
        logging.info('Connect to redis on server->%s:%s db->%s  qname->%s:%s' % (host, port, db, qprefix, qname))
        jid = config.get("alert_bot", "msg_bot.jid").strip('"').strip("'")
        password = config.get("alert_bot", "msg_bot.pass").strip('"').strip("'")

        sleekxmpp.ClientXMPP.__init__(self, jid, password)
        
        self.recipients = eval(config.get("alert_bot", "msg_bot.recipients").strip('"').strip("'"))
        self.tlock = Lock()

        # The session_start event will be triggered when
        # the bot establishes its connection with the server
        # and the XML streams are ready for use. We want to
        # listen for this event so that we we can initialize
        # our roster.
        self.add_event_handler("session_start", self.start, threaded=True)
예제 #2
0
def main(p, mediaOnly=None):
    """ Coordinates a new twitter stream connection"""

    # Logging config
    logFile = os.path.join(p.errorPath, p.connErrorFile)
    logging.basicConfig(filename=logFile, format="%(levelname)s:: %(asctime)s %(message)s", level=p.logLevel)

    # The mongo bits
    try:
        c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
        evCollHandle = dbh[p.eventsCollection]
    except:
        logging.critical("Failed to connect to db and authenticate.", exc_info=True)
        sys.exit()

    # Here's the redis queue for managing the tweets as they come in
    try:
        q = RedisQueue(p.redisName, host=p.redisHost, password=p.redisPassword, port=p.redisPort, db=0)
    except:
        logging.critical("REDIS: Failed to connect in connectionClient.py. ", exc_info=True)
        sys.exit()

    # Connection placeholder in case the exception catches the drop out
    connection = True

    while connection == True:

        # Get the existing tags and add the current
        try:
            tags = cf.getCurrentTags(evCollHandle)
        except:
            tags = None
            logging.error("Failed to get current tags from db.", exc_info=True)

        # Build the building boxes
        try:
            bboxes = cf.getCurrentBBoxes(evCollHandle)
        except:
            bboxes = None
            logging.error("Failed to get current BBOXes from db.", exc_info=True)

        if not tags and not bboxes:
            logging.warning("Currently no tags or bboxes in the db.")
            sys.exit()

        try:
            print tags, bboxes
            with tweetstream.FilterStream(p.sourceUser, p.sourcePassword, track=tags, locations=bboxes) as stream:
                for tweet in stream:
                    if mediaOnly:
                        try:
                            q.put(json.dumps(tweet))
                        except:
                            logging.critical("Failed to put tweet on redis. This tweet: \n%s" % (tweet), exc_info=True)

        except tweetstream.ConnectionError:
            logging.critical("Disconnected from twitter", exc_info=True)
예제 #3
0
 def __init__(self, config, rhost=None, rport=None, rdb=None, chatq=None, prefix=None):
     if config:
         rhost = config.get("redis", "redis.server").strip('"').strip("'")
         rport = config.get("redis", "redis.port")
         rdb = config.get("redis", "redis.db")
         chatq = config.get("alert_bot", "msg_bot.redis_mq").strip('"').strip("'")
         prefix = config.get("alert_bot", "msg_bot.redis_prefix").strip('"').strip("'")
     
         
         
     self.q = RedisQueue(chatq, prefix, rhost, rport, rdb)
예제 #4
0
def main(p):

    # The mongo bits
    try:
        c, dbh = mdb.getHandle(host=p.dbHost,
                               port=p.dbPort,
                               db=p.db,
                               user=p.dbUser,
                               password=p.dbPassword)
        evCollHandle = dbh[p.eventsCollection]
    except:
        logging.critical('Failed to connect and authenticate', exc_info=True)
        sys.exit()

    # Get the current tags
    tags = cf.getCurrentTags(evCollHandle)
    # Get the current bounding boxes
    queryBBoxes = cf.getQueryBBox(evCollHandle)

    x = 1
    while x == 1:

        # Here's the redis queue for managing the tweets as they come in
        try:
            q = RedisQueue(p.redisName,
                           host=p.redisHost,
                           password=p.redisPassword,
                           port=p.redisPort,
                           db=0)
        except:
            logging.error('Failed to connect to REDIS db.', exc_info=True)
            sys.exit()

        # This call is blocking, so expect it to hang on this point
        tweetStr = q.get()
        tweet = json.loads(tweetStr)

        # Work out which object/event this tweet is associated with
        if tags:
            tweetTags = cf.whichTags(tags, tweet)
            for tweetTag in tweetTags:
                success = dispatchTweet(p, tweet, tweetTag)
                logging.debug("Tag-based message dispatched: %s" % (success))

        if queryBBoxes:
            tweetGeos = cf.matchesCurrentGeos(queryBBoxes, tweet)
            for tweetGeo in tweetGeos:
                success = dispatchTweet(p, tweet, tweetGeo)
                logging.debug("Geo-based message dispatched: %s" % (success))
def main(p):

    # The mongo bits
    try:
        c, dbh = mdb.getHandle(host=p.dbHost, port=p.dbPort, db=p.db, user=p.dbUser, password=p.dbPassword)
        evCollHandle = dbh[p.eventsCollection]    
    except:
        logging.critical('Failed to connect and authenticate', exc_info=True)
        sys.exit()

    # Get the current tags 
    tags = cf.getCurrentTags(evCollHandle)
    # Get the current bounding boxes
    queryBBoxes = cf.getQueryBBox(evCollHandle)
    
    x = 1
    while x == 1:
        
        # Here's the redis queue for managing the tweets as they come in
        try:
            q = RedisQueue(p.redisName, host=p.redisHost, password=p.redisPassword, port=p.redisPort, db=0)
        except:
            logging.error('Failed to connect to REDIS db.', exc_info=True)
            sys.exit()
        
        # This call is blocking, so expect it to hang on this point
        tweetStr = q.get()
        tweet = json.loads(tweetStr)
        
        # Work out which object/event this tweet is associated with
        if tags:
            tweetTags = cf.whichTags(tags, tweet)
            for tweetTag in tweetTags:
                success = dispatchTweet(p, tweet, tweetTag)
                logging.debug("Tag-based message dispatched: %s" %(success))
        
        if queryBBoxes:
            tweetGeos = cf.matchesCurrentGeos(queryBBoxes, tweet)
            for tweetGeo in tweetGeos:
                success = dispatchTweet(p, tweet, tweetGeo)
                logging.debug("Geo-based message dispatched: %s" %(success))
예제 #6
0
class AlertHelper():
    q = None
    
    def __init__(self, config, rhost=None, rport=None, rdb=None, chatq=None, prefix=None):
        if config:
            rhost = config.get("redis", "redis.server").strip('"').strip("'")
            rport = config.get("redis", "redis.port")
            rdb = config.get("redis", "redis.db")
            chatq = config.get("alert_bot", "msg_bot.redis_mq").strip('"').strip("'")
            prefix = config.get("alert_bot", "msg_bot.redis_prefix").strip('"').strip("'")
        
            
            
        self.q = RedisQueue(chatq, prefix, rhost, rport, rdb)
        
    def post_msg(self, msg):
        self.q.put(msg)      
        
    def flush_all(self):
        i = 0
        while not self.q.empty():
            self.q.get()
            i+=1
        return i
예제 #7
0
import redis
import time
from twilio.rest import Client
import sendService

import config
from redisQueue import RedisQueue


twilioClient = Client(config.TWILIO['account_sid'], config.TWILIO['auth_token'])

try:
    r = redis.StrictRedis(host=config.HOST, port=config.PORT)

    sendQueue = RedisQueue(r, config.SEND_LIST)
    recievedQueue = RedisQueue(r, config.RECIEVED_LIST)

    SendService = sendService.SendService(twilioClient, sendQueue)

    while True:
        recievedMessage = recievedQueue.pop()
        sendMessage = sendQueue.pop()
        #print(sendMessage)

        if recievedMessage:
            print("recieved:", recievedMessage)
            SendService.handleReply(recievedMessage)

        if sendMessage:
            print("sending", sendMessage)
            SendService.sendMessage(sendMessage)
예제 #8
0
def enqueueForDownload(url):

    q = RedisQueue("youlinks")
    q.put(url)
    return "URL Enqueued"
예제 #9
0
class AlertMsgBot(sleekxmpp.ClientXMPP):

    """
    A basic SleekXMPP bot that will log in, send a message,
    and then log out.
    """
    config = None
    rsq= None

    recipients = None
    quit = False
    tlock = None
    

    def __init__(self, config):
        

        self.config = config
        host = config.get("redis", "redis.server").strip('"').strip("'")
        port = config.get("redis", "redis.port")
        db = config.get("redis", "redis.db")
        qname = config.get("alert_bot", "msg_bot.redis_mq").strip('"').strip("'")
        qprefix = config.get("alert_bot", "msg_bot.redis_prefix").strip('"').strip("'")
        
        self.rsq = RedisQueue(qname, qprefix, host, port, db)
        logging.info('Connect to redis on server->%s:%s db->%s  qname->%s:%s' % (host, port, db, qprefix, qname))
        jid = config.get("alert_bot", "msg_bot.jid").strip('"').strip("'")
        password = config.get("alert_bot", "msg_bot.pass").strip('"').strip("'")

        sleekxmpp.ClientXMPP.__init__(self, jid, password)
        
        self.recipients = eval(config.get("alert_bot", "msg_bot.recipients").strip('"').strip("'"))
        self.tlock = Lock()

        # The session_start event will be triggered when
        # the bot establishes its connection with the server
        # and the XML streams are ready for use. We want to
        # listen for this event so that we we can initialize
        # our roster.
        self.add_event_handler("session_start", self.start, threaded=True)

    def start(self, event):
        """
        Process the session_start event.
        Typical actions for the session_start event are
        requesting the roster and broadcasting an initial
        presence stanza.
        Arguments:
            event -- An empty dictionary. The session_start
                     event does not provide any additional
                     data.
        """
        self.send_presence()
        self.get_roster()

        self.rsq.put("Greetings! The alert bot has just started!")
        t = threading.Thread(target = self.process_alerts(), args=())
        t.start()
#         self.send_message(mto=self.recipient,
#                           mbody=self.msg,
#                           mtype='chat')

        
        
        
    def process_alerts(self):
        self.quit = False
        while self.quit <> True:
            if not self.rsq.empty():
                msg = self.rsq.get()
                logging.debug('process_alerts: received msg: {%s}' % msg)
                self.send_msg(msg)
            sleep(1)

        # Using wait=True ensures that the send queue will be
        # emptied before ending the session.
        self.disconnect(wait=True)


    def send_msg(self, msg):
        self.tlock.acquire()
        try:
            for r in self.recipients:
                self.send_message(r, msg, mtype='chat')            
        finally:
            self.tlock.release()  
예제 #10
0
def main(p, mediaOnly=None):
    ''' Coordinates a new twitter stream connection'''

    # Logging config
    logFile = os.path.join(p.errorPath, p.connErrorFile)
    logging.basicConfig(filename=logFile,
                        format='%(levelname)s:: %(asctime)s %(message)s',
                        level=p.logLevel)

    # The mongo bits
    try:
        c, dbh = mdb.getHandle(host=p.dbHost,
                               port=p.dbPort,
                               db=p.db,
                               user=p.dbUser,
                               password=p.dbPassword)
        evCollHandle = dbh[p.eventsCollection]
    except:
        logging.critical('Failed to connect to db and authenticate.',
                         exc_info=True)
        sys.exit()

    # Here's the redis queue for managing the tweets as they come in
    try:
        q = RedisQueue(p.redisName,
                       host=p.redisHost,
                       password=p.redisPassword,
                       port=p.redisPort,
                       db=0)
    except:
        logging.critical("REDIS: Failed to connect in connectionClient.py. ",
                         exc_info=True)
        sys.exit()

    # Connection placeholder in case the exception catches the drop out
    connection = True

    while connection == True:

        # Get the existing tags and add the current
        try:
            tags = cf.getCurrentTags(evCollHandle)
        except:
            tags = None
            logging.error('Failed to get current tags from db.', exc_info=True)

        # Build the building boxes
        try:
            bboxes = cf.getCurrentBBoxes(evCollHandle)
        except:
            bboxes = None
            logging.error('Failed to get current BBOXes from db.',
                          exc_info=True)

        if not tags and not bboxes:
            logging.warning('Currently no tags or bboxes in the db.')
            sys.exit()

        try:
            print tags, bboxes
            with tweetstream.FilterStream(p.sourceUser,
                                          p.sourcePassword,
                                          track=tags,
                                          locations=bboxes) as stream:
                for tweet in stream:
                    if mediaOnly:
                        try:
                            q.put(json.dumps(tweet))
                        except:
                            logging.critical(
                                "Failed to put tweet on redis. This tweet: \n%s"
                                % (tweet),
                                exc_info=True)

        except tweetstream.ConnectionError:
            logging.critical("Disconnected from twitter", exc_info=True)
예제 #11
0
import time
from redis import StrictRedis
from redisQueue import RedisQueue
qq = RedisQueue('rq', host='192.168.1.119', port=6379, db=1)
redis = StrictRedis(host='192.168.1.119', port=6379, db=2)

pubsub = redis.pubsub()


def event_handlerx(msg):
    print('Handler', qq.exists(msg['data']))
    qq.delete(msg['data'])
    print('Handler', qq.exists(msg['data']))


pubsub.psubscribe(**{'__keyevent@1__:*': event_handlerx})

print('Starting message loop')
while True:
    message = pubsub.get_message()
    if message:
        print(message)
    else:
        time.sleep(4)
예제 #12
0
def findProjectFiles(dConfig):

    qRedis = RedisQueue(dConfig['redis-queue-name'],
                        namespace='queue',
                        host=dConfig['redis-loc'])
    oES = Elasticsearch(dConfig['es-instance-locs'])

    lIgnoreDirs = ['.git', '.svn']

    dProject = {}
    dSource = {}

    dProject['_index'] = dConfig['es-index-name']
    dProject['_type'] = dConfig['es-index-type']

    dSource['crawl-time'] = dConfig['time-stamp']

    dSource['project-path'] = qRedis.get(block=True)

    lProjectFiles = []

    # if project path is '**done**', then break
    while dSource['project-path'] != '**done**':

        dSource['project-name'] = os.path.basename(dSource['project-path'])

        if dConfig['debug']:
            debug('func: findProjectFiles()', 'project-path:',
                  dSource['project-path'], dSource['project-name'])

        for sRoot, lDirs, lFiles in os.walk(dSource['project-path']):

            if len(lProjectFiles) > dConfig['es-bulk-chunk-size']:

                # ingest chunk into elasticsearch
                helpers.bulk(oES, lProjectFiles)

                if dConfig['debug']:
                    debug('func: findProjectFiles()', str(len(lProjectFiles)),
                          'files loaded into elasticsearch')

                lProjectFiles = []

            for sFile in lFiles:

                sFilePath = os.path.join(sRoot, sFile)

                sRelPath = os.path.relpath(sFilePath, dSource['project-path'])

                dFile = {}

                try:

                    sRelPath.decode('utf-8')

                except (ValueError, UnicodeDecodeError) as e:

                    try:

                        sRelPath.decode('latin-1')

                    except (ValueError, UnicodeDecodeError) as e:

                        try:

                            sRelPath.decode('utf-16')

                        except (ValueError, UnicodeDecodeError) as e:

                            warning(
                                'func findProjectFiles():', 'sProjectPath:',
                                dSource['project-path'], 'sProjectName:',
                                dSource['project-name'], 'sFile:', sFile,
                                'sRelPath:', sRelPath,
                                'utf-8, latin-1, and utf-16 decoding failed',
                                'exception:', e)

                        else:

                            dSource['file'] = sFile.decode('utf-16')
                            dSource['path'] = sRelPath.decode('utf-16')
                            dProject['_source'] = dSource

                            lProjectFiles.append(dProject)

                    else:

                        dSource['file'] = sFile.decode('latin-1')
                        dSource['path'] = sRelPath.decode('latin-1')
                        dProject['_source'] = dSource

                        lProjectFiles.append(dProject)

                else:

                    dSource['file'] = sFile
                    dSource['path'] = sRelPath
                    dProject['_source'] = dSource

                    lProjectFiles.append(dProject)

            lDirs[:] = [sDir for sDir in lDirs if sDir not in lIgnoreDirs]

        # get next project to process
        dSource['project-path'] = qRedis.get(block=True)

    # index any remaining projects
    if len(lProjectFiles) > 0:

        # ingest chunk into elasticsearch
        helpers.bulk(oES, lProjectFiles)

        if dConfig['debug']:
            debug('func: findProjectFiles()', str(len(lProjectFiles)),
                  'files loaded into elasticsearch')
예제 #13
0
def findBuildFiles(tTup):

    # unpack inputs from input tuple
    (sSearchString, dConfig) = tTup

    dConfig['redis-queue-name'] = dConfig['redis-queue-name'] % sSearchString

    if dConfig['debug']:
        debug('func: findBuildFiles() dConfig[\'redis-queue-name\']:',
              dConfig['redis-queue-name'])

    qRedis = RedisQueue(dConfig['redis-queue-name'],
                        namespace='queue',
                        host=dConfig['redis-loc'])

    # ensure redis queue is empty prior to starting consumers
    qRedis.flush()

    # setup elasticsearch client
    oES = Elasticsearch(dConfig['es-instance-locs'])

    #sQuery = { "query" : "file:" + sSearchString + "&lowercase_expanded_terms=false" }

    dQuery = {}
    dQuery['query'] = {}
    dQuery['query']['query_string'] = {}
    dQuery['query']['query_string']['fields'] = ['file.raw']
    # dQuery['query']['query_string']['fields'] = ['file.analyzed']
    dQuery['query']['query_string']['query'] = sSearchString
    dQuery['query']['query_string']['lowercase_expanded_terms'] = False

    if dConfig['debug']:
        debug('func: findBuildFiles() dQuery:', json.dumps(dQuery))
    ''' ### working 10 document fetch
    dResponse = oES.search( index=dConfig['es-file-index-name'], doc_type=dConfig['es-file-index-type'], body=json.dumps(dQuery) )

    if dConfig['debug']: 

        for sHit in dResponse['hits']['hits']:
          debug('func: findBuildFiles() sHit: ', sHit)

    '''

    # scroll time set to 10 minutes, change as needed -- required for consistent results, the scroll token expires at the end of scroll time
    sScrollTime = "10m"
    dResponse = oES.search(index=dConfig['es-file-index-name'],
                           doc_type=dConfig['es-file-index-type'],
                           body=json.dumps(dQuery),
                           search_type="scan",
                           scroll=sScrollTime)
    sScrollId = dResponse['_scroll_id']

    if dConfig['debug']:
        debug('func: findBuildFiles() (after initial search) dResponse: ',
              dResponse)

    if dConfig['debug']:
        debug('func: findBuildFiles() search hits: ',
              dResponse['hits']['total'])

    #while not dResponse['timed_out'] and dResponse['hits']['hits']['total'] > 0:
    while not dResponse['timed_out'] and dResponse['hits']['total'] > 0:

        dResponse = oES.scroll(scroll_id=sScrollId, scroll=sScrollTime)

        sScrollId = dResponse['_scroll_id']

        #if dConfig['debug']: debug('func: findBuildFiles() dResponse: ', dResponse)

        if len(dResponse['hits']['hits']) > 0:

            for sHit in dResponse['hits']['hits']:
                if dConfig['debug']:
                    debug('func: findBuildFiles() sHit: ', sHit)

        else:

            break
예제 #14
0
__author__ = 'leena'

from redisQueue import RedisQueue

q = RedisQueue('test')
q.put('hello world')