def main(): parser = OptionParser(usage='usage: firetower options args') parser.add_option( '-c', '--conf', action='store', dest='conf_path', help='Path to YAML configuration file.') (options, args) = parser.parse_args() conf = config.Config(options.conf_path) home = os.getenv('HOME') maildir_path = '%s/Maildir' % (home,) print 'This is the Maildir path: ', maildir_path assert os.path.exists(maildir_path) inbox = mailbox.Maildir(maildir_path, factory=None) queue = Redis(host=conf.redis_host, port=conf.redis_port) for message in inbox: ft_dict = {} ft_dict['hostname'] = message['Received'].split(' ')[1] or '????' ft_dict['sig'] = message.get_payload() or '????' ft_dict['date'] = message.get_date() ft_dict['programname'] = 'Maildir Util' # We don't huge signatures clogging the classification if len(ft_dict['sig']) < 10000 and isinstance(ft_dict['sig'], str): queue.push(conf.queue_key, json.dumps(ft_dict))
def main(): parser = OptionParser(usage='usage: firetower options args') parser.add_option( '-c', '--conf', action='store', dest='conf_path', help='Path to YAML configuration file.') (options, args) = parser.parse_args() conf = config.Config(options.conf_path) conn = imaplib.IMAP4_SSL(conf.imap_host) imap_user = conf.imap_user queue = Redis(host=conf.redis_host, port=conf.redis_port) processed = 0 print "Enter email password" imap_password = getpass.getpass() conn.login(imap_user, imap_password) conn.select("Inbox") while True: _, msg_ids = conn.search(None, "(ALL)") msg_ids = msg_ids[0].split() if len(msg_ids) == 0: time.sleep(0.5) continue print "Processing %s messages" %len(msg_ids) _, msgs = conn.fetch(",".join(msg_ids), '(BODY[])') for msg in msgs: if not msg or msg == ")": continue msg_obj = email.message_from_string(msg[1]) ft_dict = {} ft_dict['hostname'] = msg_obj['Received'].split(' ')[1] if msg_obj['Received'] else '????' ft_dict['sig'] = msg_obj.get_payload() or '????' ft_dict['date'] = msg_obj["Date"] ft_dict['programname'] = 'Maildir Util' # We don't huge signatures clogging the classification if len(ft_dict['sig']) < 10000 and isinstance(ft_dict['sig'], str): queue.push(conf.queue_key, json.dumps(ft_dict)) processed += 1 if not processed % 100: print "Processed: %s" %processed print "Processed: %s" %processed for msg_id in msg_ids: conn.store(msg_id, '+FLAGS', '\\Deleted') conn.expunge()
def run(self, conf): queue = Redis(host=conf.redis_host, port=conf.redis_port) print queue.conn.keys() for i in xrange(0, 50): try: # Semi-randomly seed the 'sig' key in our fake errors FAKE_DATA['sig'] = random.choice(FAKE_SIGS) print FAKE_DATA encoded = json.dumps(FAKE_DATA) err = queue.push(conf.queue_key, encoded) except: print "Something went wrong storing value from redis"
class TestRedisUtil(TestCase): def setUp(self): self.r = MockRedis() self.r_util = Redis('localhost', 6300) self.r.lpush('test_key', 'something worth keeping') def tearDown(self): self.r.data.clear() def test_pop(self): """Test that the redis_util pop wrapper works.""" result = self.r_util.pop('test_key') assert result == 'something worth keeping' def test_push(self): """Test that the redis_util push wrapper works.""" test_val = 'another thing of note' self.r_util.push('test_key', test_val) result = self.r.lpop('test_key') assert result == test_val