Exemple #1
0
    def process(self):
        '''
        Actual processor that does a request-reply cycle
        '''
        while True:
            message = self.socket.recv()
            logging.debug('received new search request')

            try:
                decoded = json.loads(from_base64(message))
            except Exception as ex:
                logging.error('failed to decode a message, continuing')
            else:
                response = self.search(decoded)
                #TODO: we should actually return *most likely* result not multiple
                encoded = to_base64(response)
                self.socket.send(encoded)
Exemple #2
0
    def crawl(self, state):
        '''
        Contact twitter, retrieve all most recent mentions and throw into message q

        state - dictionary containing state info.
        '''
        mentions = self.api.GetMentions(since_id=state.get('since_id', None))
        for status in mentions:
            encoded_msg = to_base64(status.AsJsonString())
            logging.debug('sending to zeromq frontend')
            self.socket.send(encoded_msg)
            reply = self.socket.recv() # reply is sweet or sour
            logging.debug('received message from zmq backend')
            reply = from_base64(reply)
            if reply == 'sweet':
                self.api.CreateFavorite(status)
            elif reply == 'sour':
                logging.warn('cannot understand message [%s]' % status.text)
            else:
                logging.error('emmm...protocol fail')
Exemple #3
0
    def process(self):
        '''
        Do request-reply dance
        '''
        while True:
            logging.info('listening for connections')
            message = self.firehose_socket.recv()
            logging.debug('received new message from firehose')

            try:
                decoded = json.loads(from_base64(message))
                tweet = status.Status(**decoded)
            except Exception as ex:
                logging.error('could not decode message, probably corrupted')
                self.socket.send('sour')
            else:
                org, location = self.get_entities(tweet.text)
                if not len(org) or not len(location):
                    self.socket.send('sour')
                else:
                    logging.info('we have identified potential location and org')
                    pass # TODO: contact search API.
Exemple #4
0
    def process(self):
        '''
        Actually processor that does a the request-reply cycle
        '''
        while True:
            message = self.socket.recv()
            logging.debug('received new message')

            # decode to determine type using additional metadata in message
            try:
                decoded = json.loads(from_base64(message))
            except Exception as ex:
                logging.error('failed to decode a message, dropping silently')
                self.socket.send('')
            else:
                if 'flag' in decoded:
                    self.store(decoded)
                    self.socket.send('')
                elif 'check' in decoded:
                    flags = self.retrieve(decoded['business_id'])
                    encoded = to_base64(flags)
                    self.socket.send(encoded)
                else:
                    logging.warn('unidentified message sent')