コード例 #1
0
ファイル: imap_client.py プロジェクト: schmichael/firetower
def main():
    parser = OptionParser(usage="usage: firetower options args")
    parser.add_option("-c", "--conf", action="store", dest="conf_path", help="Path to YAML configuration file.")
    parser.add_option(
        "-d", dest="clear_firstrun", action="store_true", default=False, help="Clear backed up emails on first run"
    )
    (options, args) = parser.parse_args()
    conf = config.Config(options.conf_path)

    conn = imaplib.IMAP4_SSL(conf.imap_host)
    imap_user = conf.imap_user

    queue = redis_util.get_redis_conn(host=conf.redis_host, port=conf.redis_port, redis_db=conf.redis_db)

    processed = 0

    print "Enter email password"
    imap_password = getpass.getpass()

    conn.login(imap_user, imap_password)
    conn.select("Inbox")

    first_run = True

    while True:
        _, msg_ids = conn.search(None, "(ALL)")
        msg_ids = msg_ids[0].split()
        if len(msg_ids) == 0:
            time.sleep(0.5)
            if first_run:
                first_run = False
            continue
        print "Processing %s messages" % len(msg_ids)

        id_batches = []
        # split the ids into batches of 1000
        count = 0
        split = 1000
        while count < len(msg_ids):
            id_batches.append(msg_ids[count : count + split])
            count += split

        for id_batch in id_batches:
            if options.clear_firstrun and first_run:
                first_run = False
                break

            _, msgs = conn.fetch(",".join(id_batch), "(BODY[])")
            for msg in msgs:
                if not msg or msg == ")":
                    continue
                msg_obj = email.message_from_string(msg[1])
                ft_dict = {}
                ft_dict["hostname"] = msg_obj["Received"].split(" ")[1] if msg_obj["Received"] else "????"
                ft_dict["sig"] = msg_obj.get_payload() or "????"
                ft_dict["date"] = msg_obj["Date"]
                ft_dict["programname"] = "Maildir Util"

                # We don't huge signatures clogging the classification
                if len(ft_dict["sig"]) < 10000 and isinstance(ft_dict["sig"], str):
                    queue.lpush(conf.queue_key, json.dumps(ft_dict))
                processed += 1
                if not processed % 100:
                    print "Processed: %s" % processed

        for msg_id in msg_ids:
            conn.store(msg_id, "+FLAGS", "\\Deleted")
        print "Processed: %s" % processed

        conn.expunge()
コード例 #2
0
ファイル: firetower_web.py プロジェクト: schmichael/firetower
import flask
from flask import Flask
from flask import abort
from flask import render_template
from flask import request

from dateutil import parser

from firetower import category
from firetower import  redis_util

# These items should be picked up from the config.yaml
REDIS_HOST = "localhost"
REDIS_PORT = 6379
REDIS_DB = 1
REDIS = redis_util.get_redis_conn(REDIS_HOST, REDIS_PORT, REDIS_DB)

DEFAULT_TIME_SLICE = 300000

app = Flask(__name__)

def get_time_slice_options():
    slices = category.TIME_SLICES.items()
    slices.sort(key=lambda x: x[1])
    return [x[0] for x in slices]

@app.route("/")
@app.route("/aggregate/")
def aggregate():
    slice = request.args.get("time_slice", None)
    kwargs = {"time_slice_options": get_time_slice_options()}
コード例 #3
0
ファイル: imap_client.py プロジェクト: gmcquillan/firetower
def main():
    parser = OptionParser(usage='usage: firetower options args')
    parser.add_option(
        '-c', '--conf', action='store', dest='conf_path',
         help='Path to YAML configuration file.',
    )
    parser.add_option(
        '-d', dest='clear_firstrun', action="store_true", default=False,
        help='Clear backed up emails on first run'
    )
    parser.add_option(
        '-p', dest='imap_password', action="store_true", default=False,
        help='IMAP password.'
    )
    (options, args) = parser.parse_args()
    conf = config.Config(options.conf_path)

    conn = imaplib.IMAP4_SSL(conf.imap_host)
    imap_user = conf.imap_user
    imap_password = options.user_pass

    queue = redis_util.get_redis_conn(
        host=conf.redis_host, port=conf.redis_port, redis_db=conf.redis_db)

    processed = 0

    if not imap_password:
        print "Enter email password"
        imap_password = getpass.getpass()

    conn.login(imap_user, imap_password)
    conn.select("Inbox")

    first_run = True

    while True:
        _, msg_ids = conn.search(None, "(ALL)")
        msg_ids = msg_ids[0].split()
        if len(msg_ids) == 0:
            time.sleep(0.5)
            if first_run:
                first_run = False
            continue
        print "Processing %s messages" %len(msg_ids)

        id_batches = []
        # split the ids into batches of 1000
        count = 0
        split = 1000
        while count < len(msg_ids):
            id_batches.append(msg_ids[count:count+split])
            count += split

        for id_batch in id_batches:
            if options.clear_firstrun and first_run:
                first_run = False
                break

            _, msgs = conn.fetch(",".join(id_batch), '(BODY[])')
            for msg in msgs:
                if not msg or msg == ")":
                    continue
                msg_obj = email.message_from_string(msg[1])
                ft_dict = {}
                ft_dict['hostname'] = msg_obj['Received'].split(' ')[1] if msg_obj['Received'] else '????'
                ft_dict['sig'] = msg_obj.get_payload() or '????'
                ft_dict['date'] = msg_obj["Date"]
                ft_dict['programname'] = 'Maildir Util'

                # We don't huge signatures clogging the classification
                if len(ft_dict['sig']) < 10000 and isinstance(ft_dict['sig'], str):
                    queue.lpush(conf.queue_key, json.dumps(ft_dict))
                processed += 1
                if not processed % 100:
                    print "Processed: %s" %processed

        for msg_id in msg_ids:
            conn.store(msg_id, '+FLAGS', '\\Deleted')
        print "Processed: %s" %processed



        conn.expunge()