def process(self):
'''
Actual processor that does a request-reply cycle
'''
while True:
message = self.socket.recv()
logging.debug('received new search request')
try:
decoded = json.loads(from_base64(message))
except Exception as ex:
logging.error('failed to decode a message, continuing')
else:
response = self.search(decoded)
#TODO: we should actually return *most likely* result not multiple
encoded = to_base64(response)
self.socket.send(encoded)
def crawl(self, state):
'''
Contact twitter, retrieve all most recent mentions and throw into message q
state - dictionary containing state info.
'''
mentions = self.api.GetMentions(since_id=state.get('since_id', None))
for status in mentions:
encoded_msg = to_base64(status.AsJsonString())
logging.debug('sending to zeromq frontend')
self.socket.send(encoded_msg)
reply = self.socket.recv() # reply is sweet or sour
logging.debug('received message from zmq backend')
reply = from_base64(reply)
if reply == 'sweet':
self.api.CreateFavorite(status)
elif reply == 'sour':
logging.warn('cannot understand message [%s]' % status.text)
else:
logging.error('emmm...protocol fail')
def process(self):
'''
Do request-reply dance
'''
while True:
logging.info('listening for connections')
message = self.firehose_socket.recv()
logging.debug('received new message from firehose')
try:
decoded = json.loads(from_base64(message))
tweet = status.Status(**decoded)
except Exception as ex:
logging.error('could not decode message, probably corrupted')
self.socket.send('sour')
else:
org, location = self.get_entities(tweet.text)
if not len(org) or not len(location):
self.socket.send('sour')
else:
logging.info('we have identified potential location and org')
pass # TODO: contact search API.
def process(self):
'''
Actually processor that does a the request-reply cycle
'''
while True:
message = self.socket.recv()
logging.debug('received new message')
# decode to determine type using additional metadata in message
try:
decoded = json.loads(from_base64(message))
except Exception as ex:
logging.error('failed to decode a message, dropping silently')
self.socket.send('')
else:
if 'flag' in decoded:
self.store(decoded)
self.socket.send('')
elif 'check' in decoded:
flags = self.retrieve(decoded['business_id'])
encoded = to_base64(flags)
self.socket.send(encoded)
else:
logging.warn('unidentified message sent')