示例#1
0
    def _get_filters(self):
        filters = set()

        for item in (self.path, self.headers.get("referer", "")):
            if "ip[]" in item:
                for _ in re.findall(r"\bip\[\]=([\d./\-]+)(?:&|\Z)", item):
                    if "/" in _:
                        prefix, mask = _.split("/", 1)
                        mask = int(mask)
                        start_int = addr_to_int(prefix) & make_mask(mask)
                        end_int = start_int | ((1 << 32 - mask) - 1)
                        if (end_int - start_int) > MAX_IP_FILTER_RANGE:
                            raise
                        for address in xrange(start_int, end_int + 1):
                            filters.add(address)
                    elif "-" in _:
                        start_address, end_address = _.split("-", 1)
                        start_int = addr_to_int(start_address)
                        end_int = addr_to_int(end_address)
                        if (end_int - start_int) > MAX_IP_FILTER_RANGE:
                            raise
                        for address in xrange(start_int, end_int + 1):
                            filters.add(address)
                    else:
                        filters.add(addr_to_int(_))

        return filters
示例#2
0
文件: httpd.py 项目: stamparm/tsusen
    def _get_filters(self):
        filters = set()

        for item in (self.path, self.headers.get("referer", "")):
            if "ip[]" in item:
                for _ in re.findall(r"\bip\[\]=([\d./\-]+)(?:&|\Z)", item):
                    if "/" in _:
                        prefix, mask = _.split("/", 1)
                        mask = int(mask)
                        start_int = addr_to_int(prefix) & make_mask(mask)
                        end_int = start_int | ((1 << 32 - mask) - 1)
                        if (end_int - start_int) > MAX_IP_FILTER_RANGE:
                            raise
                        for address in xrange(start_int, end_int + 1):
                            filters.add(address)
                    elif "-" in _:
                        start_address, end_address = _.split("-", 1)
                        start_int = addr_to_int(start_address)
                        end_int = addr_to_int(end_address)
                        if (end_int - start_int) > MAX_IP_FILTER_RANGE:
                            raise
                        for address in xrange(start_int, end_int + 1):
                            filters.add(address)
                    else:
                        filters.add(addr_to_int(_))

        return filters
示例#3
0
文件: sensor.py 项目: 4sp1r3/tsusen
def init_sensor():
    global _cap
    global _datalink

    items = []

    for cmd, regex in (("ifconfig", r"inet addr:([\d.]+) .*Mask:([\d.]+)"), (
            "ipconfig",
            r"IPv4 Address[^\n]+([\d.]+)\s+Subnet Mask[^\n]+([\d.]+)")):
        try:
            items = re.findall(regex, subprocess.check_output(cmd))
            break
        except OSError:
            pass

    for ip, mask in items:
        LOCAL_ADDRESSES.append(
            (addr_to_int(ip) & addr_to_int(mask), addr_to_int(mask)))

    try:
        if not os.path.isdir(LOG_DIRECTORY):
            os.makedirs(LOG_DIRECTORY)
    except Exception, ex:
        if "Permission denied" in str(ex):
            exit("[x] please run with sudo/Administrator privileges")
        else:
            raise
示例#4
0
def init_sensor():
    global _cap
    global _datalink

    items = []

    for cmd, regex in (
        ("ifconfig", r"inet addr:([\d.]+) .*Mask:([\d.]+)"),
        ("ipconfig", r"IPv4 Address[^\n]+([\d.]+)\s+Subnet Mask[^\n]+([\d.]+)"),
    ):
        try:
            items = re.findall(regex, subprocess.check_output(cmd))
            break
        except OSError:
            pass

    for ip, mask in items:
        LOCAL_ADDRESSES.append((addr_to_int(ip) & addr_to_int(mask), addr_to_int(mask)))

    try:
        if not os.path.isdir(LOG_DIRECTORY):
            os.makedirs(LOG_DIRECTORY)
    except Exception, ex:
        if "Permission denied" in str(ex):
            exit("[x] please run with sudo/Administrator privileges")
        else:
            raise
示例#5
0
文件: httpd.py 项目: stamparm/tsusen
    def _dataset(self):
        result = "\n"
        dates = set()
        rows = []
        indexes = {}

        filters = self._get_filters()

        for filename in sorted(glob.glob(os.path.join(LOG_DIRECTORY, "*.csv")))[-config.TRENDLINE_PERIOD:]:
            with open(filename, "rb") as f:
                match = re.search(r"([\d-]+)\.csv", filename)

                if match:
                    date = match.group(1)
                else:
                    continue

                reader = csv.DictReader(f, delimiter=' ')

                for row in reader:
                    key = (row["proto"], row["dst_port"], row["dst_ip"], row["src_ip"])

                    if filters and not (addr_to_int(row["src_ip"]) in filters or addr_to_int(row["dst_ip"]) in filters):
                        continue

                    if key not in indexes:
                        indexes[key] = len(rows)
                        rows.append(row)
                    else:
                        index = indexes[key]
                        rows[index]["first_seen"] = min(int(rows[index]["first_seen"]), int(row["first_seen"]))
                        rows[index]["last_seen"] = max(int(rows[index]["last_seen"]), int(row["last_seen"]))
                        rows[index]["count"] = int(rows[index]["count"]) + int(row["count"])

        for row in rows:
            try:
                port = int(row['dst_port'])
                port_name = MISC_PORTS.get(port) or socket.getservbyport(port, row['proto'].lower())
            except Exception:
                port_name = None
            finally:
                result += "["
                for column in ("proto", "dst_port", "dst_ip", "src_ip", "first_seen", "last_seen", "count"):
                    if "_seen" in column:
                        result += '"%s",' % datetime.datetime.utcfromtimestamp(int(row[column])).strftime(TIME_FORMAT)
                    elif "_port" in column and port_name:
                        result += '"%s (%s)",' % (row[column], port_name)
                    else:
                        result += '"%s",' % row[column]
                result += "],\n"

        return result
示例#6
0
    def _trendline_data(self):
        result = "\n"
        series = {}
        dates = set()

        filters = self._get_filters()

        for filename in sorted(glob.glob(os.path.join(
                LOG_DIRECTORY, "*.csv")))[-config.TRENDLINE_PERIOD:]:
            with open(filename, "rb") as f:
                match = re.search(r"([\d-]+)\.csv", filename)

                if match:
                    date = match.group(1)
                else:
                    continue

                reader = csv.DictReader(f, delimiter=' ')

                for row in reader:
                    if filters and not (addr_to_int(row["src_ip"]) in filters
                                        or addr_to_int(
                                            row["dst_ip"]) in filters):
                        continue

                    try:
                        port = int(row['dst_port'])
                        port_name = MISC_PORTS.get(
                            port) or socket.getservbyport(
                                port, row['proto'].lower())
                    except Exception:
                        port_name = None
                    finally:
                        serie = "%s%s%s" % (
                            row['proto'].upper(), " %s" % row['dst_port']
                            if row['dst_port'].isdigit() else "",
                            " (%s)" % port_name if port_name else "")

                    if serie not in series:
                        series[serie] = {}

                    if date not in series[serie]:
                        series[serie][date] = 0

                    series[serie][date] += 1
                    dates.add(date)

        keys = series.keys()

        if keys:
            last_date = max(dates)
            totals = {}
            for key in list(keys):
                if not filters:
                    if any(series[key].get(date, 0) <
                           config.TRENDLINE_DAILY_THRESHOLD for date in dates
                           if date != last_date):
                        if all(series[key].get(date, 0) <
                               config.TRENDLINE_DAILY_BURST for date in dates):
                            del keys[keys.index(key)]
                totals[key] = series[key].get(last_date, 0)

            keys = sorted(keys, key=lambda key: totals[key], reverse=True)
            result += "['Date',%s],\n" % ','.join("'%s'" % key for key in keys)

            for date in sorted(dates):
                year, month, day = date.split('-')
                result += "[new Date(%s,%d,%s)," % (year, int(month) - 1, day)
                for serie in keys:
                    result += "%s," % series[serie].get(date, 0)
                result += "],\n"

            result = result[:-1]

        return result
示例#7
0
    def _dataset(self):
        result = "\n"
        dates = set()
        rows = []
        indexes = {}

        filters = self._get_filters()

        for filename in sorted(glob.glob(os.path.join(
                LOG_DIRECTORY, "*.csv")))[-config.TRENDLINE_PERIOD:]:
            with open(filename, "rb") as f:
                match = re.search(r"([\d-]+)\.csv", filename)

                if match:
                    date = match.group(1)
                else:
                    continue

                reader = csv.DictReader(f, delimiter=' ')

                for row in reader:
                    key = (row["proto"], row["dst_port"], row["dst_ip"],
                           row["src_ip"])

                    if filters and not (addr_to_int(row["src_ip"]) in filters
                                        or addr_to_int(
                                            row["dst_ip"]) in filters):
                        continue

                    if key not in indexes:
                        indexes[key] = len(rows)
                        rows.append(row)
                    else:
                        index = indexes[key]
                        rows[index]["first_seen"] = min(
                            int(rows[index]["first_seen"]),
                            int(row["first_seen"]))
                        rows[index]["last_seen"] = max(
                            int(rows[index]["last_seen"]),
                            int(row["last_seen"]))
                        rows[index]["count"] = int(rows[index]["count"]) + int(
                            row["count"])

        for row in rows:
            try:
                port = int(row['dst_port'])
                port_name = MISC_PORTS.get(port) or socket.getservbyport(
                    port, row['proto'].lower())
            except Exception:
                port_name = None
            finally:
                result += "["
                for column in ("proto", "dst_port", "dst_ip", "src_ip",
                               "first_seen", "last_seen", "count"):
                    if "_seen" in column:
                        result += '"%s",' % datetime.datetime.utcfromtimestamp(
                            int(row[column])).strftime(TIME_FORMAT)
                    elif "_port" in column and port_name:
                        result += '"%s (%s)",' % (row[column], port_name)
                    else:
                        result += '"%s",' % row[column]
                result += "],\n"

        return result
示例#8
0
文件: httpd.py 项目: stamparm/tsusen
    def _trendline_data(self):
        result = "\n"
        series = {}
        dates = set()

        filters = self._get_filters()

        for filename in sorted(glob.glob(os.path.join(LOG_DIRECTORY, "*.csv")))[-config.TRENDLINE_PERIOD:]:
            with open(filename, "rb") as f:
                match = re.search(r"([\d-]+)\.csv", filename)

                if match:
                    date = match.group(1)
                else:
                    continue

                reader = csv.DictReader(f, delimiter=' ')

                for row in reader:
                    if filters and not (addr_to_int(row["src_ip"]) in filters or addr_to_int(row["dst_ip"]) in filters):
                        continue

                    try:
                        port = int(row['dst_port'])
                        port_name = MISC_PORTS.get(port) or socket.getservbyport(port, row['proto'].lower())
                    except Exception:
                        port_name = None
                    finally:
                        serie = "%s%s%s" % (row['proto'].upper(), " %s" % row['dst_port'] if row['dst_port'].isdigit() else "", " (%s)" % port_name if port_name else "")

                    if serie not in series:
                        series[serie] = {}

                    if date not in series[serie]:
                        series[serie][date] = 0

                    series[serie][date] += 1
                    dates.add(date)

        keys = series.keys()

        if keys:
            last_date = max(dates)
            totals = {}
            for key in list(keys):
                if not filters:
                    if any(series[key].get(date, 0) < config.TRENDLINE_DAILY_THRESHOLD for date in dates if date != last_date):
                        if all(series[key].get(date, 0) < config.TRENDLINE_DAILY_BURST for date in dates):
                            del keys[keys.index(key)]
                totals[key] = series[key].get(last_date, 0)

            keys = sorted(keys, key=lambda key: totals[key], reverse=True)
            result += "['Date',%s],\n" % ','.join("'%s'" % key for key in keys)

            for date in sorted(dates):
                year, month, day = date.split('-')
                result += "[new Date(%s,%d,%s)," % (year, int(month) - 1, day)
                for serie in keys:
                    result += "%s," % series[serie].get(date, 0)
                result += "],\n"

            result = result[:-1]

        return result
示例#9
0
文件: sensor.py 项目: 4sp1r3/tsusen
def _process_packet(packet, sec, usec):
    try:
        if _datalink == pcapy.DLT_LINUX_SLL:
            packet = packet[2:]

        eth_header = struct.unpack("!HH8sH", packet[:ETH_LENGTH])
        eth_protocol = socket.ntohs(eth_header[3])

        if eth_protocol == IPPROTO:  # IP
            ip_header = struct.unpack("!BBHHHBBH4s4s",
                                      packet[ETH_LENGTH:ETH_LENGTH + 20])
            ip_length = ip_header[2]
            packet = packet[:ETH_LENGTH + ip_length]  # truncate
            iph_length = (ip_header[0] & 0xF) << 2

            protocol = ip_header[6]
            src_ip = socket.inet_ntoa(ip_header[8])
            dst_ip = socket.inet_ntoa(ip_header[9])

            proto = IPPROTO_LUT.get(protocol)

            local_src = False
            for prefix, mask in LOCAL_ADDRESSES:
                if addr_to_int(src_ip) & mask == prefix:
                    local_src = True
                    break

            if proto is None or any(_ in (config.IGNORE_ADDRESSES or "")
                                    for _ in (src_ip, dst_ip)):
                return

            # only process SYN packets
            if protocol == socket.IPPROTO_TCP:  # TCP
                if local_src:
                    return

                i = iph_length + ETH_LENGTH
                src_port, dst_port, _, _, _, flags = struct.unpack(
                    "!HHLLBB", packet[i:i + 14])

                if any(
                        str(_) in (config.IGNORE_PORTS or "")
                        for _ in (src_port, dst_port)):
                    return

                dst_key = "%s:%s:%s" % (proto, dst_ip, dst_port)
                stat_key = "%s:%s" % (dst_key, src_ip)

                if flags == 2:  # SYN set (only)
                    if dst_key not in _traffic:
                        _traffic[dst_key] = set()

                    _traffic[dst_key].add(src_ip)

                    if stat_key not in _auxiliary:
                        _auxiliary[stat_key] = [sec, sec, 1]
                    else:
                        _auxiliary[stat_key][1] = sec
                        _auxiliary[stat_key][2] += 1

            else:
                if protocol == socket.IPPROTO_UDP:  # UDP
                    i = iph_length + ETH_LENGTH
                    _ = packet[i:i + 4]
                    if len(_) < 4:
                        return

                    src_port, dst_port = struct.unpack("!HH", _)
                else:  # non-TCP/UDP (e.g. ICMP)
                    src_port, dst_port = '-', '-'

                if any(
                        str(_) in (config.IGNORE_PORTS or "")
                        for _ in (src_port, dst_port)):
                    return

                dst_key = "%s:%s:%s" % (proto, dst_ip, dst_port)
                stat_key = "%s:%s" % (dst_key, src_ip)

                flow = tuple(
                    sorted((addr_to_int(src_ip), src_port, addr_to_int(dst_ip),
                            dst_port)))

                if flow not in _auxiliary:
                    _auxiliary[flow] = True

                    if local_src:
                        return

                    if dst_key not in _traffic:
                        _traffic[dst_key] = set()

                    _traffic[dst_key].add(src_ip)
                    _auxiliary[stat_key] = [sec, sec, 1]

                elif stat_key in _auxiliary:
                    _auxiliary[stat_key][1] = sec
                    _auxiliary[stat_key][2] += 1

    except Exception:
        pass

    finally:
        _log_write()
示例#10
0
def _process_packet(packet, sec, usec):
    try:
        if _datalink == pcapy.DLT_LINUX_SLL:
            packet = packet[2:]

        eth_header = struct.unpack("!HH8sH", packet[:ETH_LENGTH])
        eth_protocol = socket.ntohs(eth_header[3])

        if eth_protocol == IPPROTO:  # IP
            ip_header = struct.unpack("!BBHHHBBH4s4s", packet[ETH_LENGTH : ETH_LENGTH + 20])
            ip_length = ip_header[2]
            packet = packet[: ETH_LENGTH + ip_length]  # truncate
            iph_length = (ip_header[0] & 0xF) << 2

            protocol = ip_header[6]
            src_ip = socket.inet_ntoa(ip_header[8])
            dst_ip = socket.inet_ntoa(ip_header[9])

            proto = IPPROTO_LUT.get(protocol)

            local_src = False
            for prefix, mask in LOCAL_ADDRESSES:
                if addr_to_int(src_ip) & mask == prefix:
                    local_src = True
                    break

            if proto is None or any(_ in (config.IGNORE_ADDRESSES or "") for _ in (src_ip, dst_ip)):
                return

            # only process SYN packets
            if protocol == socket.IPPROTO_TCP:  # TCP
                if local_src:
                    return

                i = iph_length + ETH_LENGTH
                src_port, dst_port, _, _, _, flags = struct.unpack("!HHLLBB", packet[i : i + 14])

                if any(str(_) in (config.IGNORE_PORTS or "") for _ in (src_port, dst_port)):
                    return

                dst_key = "%s:%s:%s" % (proto, dst_ip, dst_port)
                stat_key = "%s:%s" % (dst_key, src_ip)

                if flags == 2:  # SYN set (only)
                    if dst_key not in _traffic:
                        _traffic[dst_key] = set()

                    _traffic[dst_key].add(src_ip)

                    if stat_key not in _auxiliary:
                        _auxiliary[stat_key] = [sec, sec, 1]
                    else:
                        _auxiliary[stat_key][1] = sec
                        _auxiliary[stat_key][2] += 1

            else:
                if protocol == socket.IPPROTO_UDP:  # UDP
                    i = iph_length + ETH_LENGTH
                    _ = packet[i : i + 4]
                    if len(_) < 4:
                        return

                    src_port, dst_port = struct.unpack("!HH", _)
                else:  # non-TCP/UDP (e.g. ICMP)
                    src_port, dst_port = "-", "-"

                if any(str(_) in (config.IGNORE_PORTS or "") for _ in (src_port, dst_port)):
                    return

                dst_key = "%s:%s:%s" % (proto, dst_ip, dst_port)
                stat_key = "%s:%s" % (dst_key, src_ip)

                flow = tuple(sorted((addr_to_int(src_ip), src_port, addr_to_int(dst_ip), dst_port)))

                if flow not in _auxiliary:
                    _auxiliary[flow] = True

                    if local_src:
                        return

                    if dst_key not in _traffic:
                        _traffic[dst_key] = set()

                    _traffic[dst_key].add(src_ip)
                    _auxiliary[stat_key] = [sec, sec, 1]

                elif stat_key in _auxiliary:
                    _auxiliary[stat_key][1] = sec
                    _auxiliary[stat_key][2] += 1

    except Exception:
        pass

    finally:
        _log_write()