def parse_header_line(self, line):
if not line:
return
if line[:1] != b"#":
LOGGER.warning("Not a header line")
return
keyval = line[1:].split(self.sep, 1)
if len(keyval) < 2:
if line.startswith(b'#separator '):
keyval = [b'separator', line[11:]]
else:
LOGGER.warn("Invalid header line")
return
directive = keyval[0]
arg = keyval[1]
if directive == b"separator":
self.sep = decode_hex(arg[2:]) if arg.startswith(b'\\x') else arg
elif directive == b"set_separator":
self.set_sep = arg
elif directive == b"empty_field":
self.empty_field = arg
elif directive == b"unset_field":
self.unset_field = arg
elif directive == b"path":
self.path = arg.decode()
elif directive == b"open":
pass
elif directive == b"fields":
self.fields = arg.split(self.sep)
elif directive == b"types":
self.types = arg.split(self.sep)
def parse_header_line(self, line):
if not line:
return
if line[:1] != b"#":
LOGGER.warning("Not a header line")
return
keyval = line[1:].split(self.sep, 1)
if len(keyval) < 2:
if line.startswith(b'#separator '):
keyval = [b'separator', line[11:]]
else:
LOGGER.warning("Invalid header line")
return
directive = keyval[0]
arg = keyval[1]
if directive == b"separator":
self.sep = decode_hex(arg[2:]) if arg.startswith(b'\\x') else arg
elif directive == b"set_separator":
self.set_sep = arg
elif directive == b"empty_field":
self.empty_field = arg
elif directive == b"unset_field":
self.unset_field = arg
elif directive == b"path":
self.path = arg.decode()
elif directive == b"open":
pass
elif directive == b"fields":
self.fields = arg.split(self.sep)
elif directive == b"types":
self.types = arg.split(self.sep)
def displayfunction(cur: DBCursor) -> None:
outdb.start_store_hosts()
for rec in cur:
try:
del rec["_id"]
except KeyError:
pass
try:
outdb.store_host(rec)
except Exception:
LOGGER.warning("Cannot insert record %r", rec, exc_info=True)
outdb.stop_store_hosts()
def _dns_do_query(name, rtype=None, srv=None):
cmd = ["dig", "+noquestion", "+nocomments", "+nocmd", "+nostat"]
if rtype:
cmd.extend(["-t", rtype])
cmd.append(name)
if srv:
cmd.append("@%s" % srv)
for line in subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout:
line = line.decode()[:-1]
if line and line[:1] != ";":
try:
yield nsrecord(*line.split(None, 4))
except TypeError:
LOGGER.warning("Cannot read line %r", line)
def _dns_do_query(name, rtype=None, srv=None):
cmd = ['dig', '+noquestion', '+nocomments', '+nocmd', '+nostat']
if rtype:
cmd.extend(['-t', rtype])
cmd.append(name)
if srv:
cmd.append('@%s' % srv)
for line in subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout:
line = line.decode()[:-1]
if line and line[:1] != ';':
try:
yield nsrecord(*line.split(None, 4))
except TypeError:
LOGGER.warning('Cannot read line %r', line)
def parse_line(self, line: bytes) -> Dict[str, Any]:
m = LINE_RE.match(line.decode())
if not m:
return {}
res: Dict[str, Any] = {}
# time of event
res["ts"] = datetime.datetime.strptime(m.group("time"), "%Y/%m/%d %H:%M:%S")
# data of event
for entry in m.group("data").split("|"):
k, v = entry.split("=", 1)
if k in res:
LOGGER.warning("Duplicate key in line [%r]", line)
return {}
res[k] = v
return res
def _dns_do_query(
name: str,
rtype: Optional[str] = None,
srv: Optional[str] = None) -> Generator[nsrecord, None, None]:
cmd = ["dig", "+noquestion", "+nocomments", "+nocmd", "+nostat"]
if rtype:
cmd.extend(["-t", rtype])
cmd.append(name)
if srv:
cmd.append("@%s" % srv)
with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc:
assert proc.stdout is not None
for line_bytes in proc.stdout:
line = line_bytes.decode()[:-1]
if line and line[:1] != ";":
try:
yield nsrecord(*line.split(None, 4))
except TypeError:
LOGGER.warning("Cannot read line %r", line)
def parse_line(self, line: bytes) -> Dict[str, Any]:
m = LINE_RE.match(line.decode())
if not m:
LOGGER.warning("Cannot parse line [%r]", line)
return {}
try:
# Timestamp, without timezone specification
timestamp = datetime.datetime.strptime(
m.group("datetime").split()[0], "%d/%b/%Y:%H:%M:%S"
)
except ValueError:
LOGGER.warning("Cannot parse timestamp from line [%r]", line)
return {}
# data of event
return {
"host": m.group("addr"),
"ts": timestamp,
"useragent": m.group("useragent"),
}
def add_cpe_values(hostrec, path, cpe_values):
"""Add CPE values (`cpe_values`) to the `hostrec` at the given `path`.
CPEs are indexed in a dictionary to agglomerate origins, but this dict
is replaced with its values() in ._pre_addhost() or in
.store_scan_json_zgrab(), or in the function that calls
add_cpe_values(), depending on the context.
"""
cpes = hostrec.setdefault('cpes', {})
for cpe in cpe_values:
if cpe not in cpes:
try:
cpeobj = cpe2dict(cpe)
except ValueError:
LOGGER.warning("Invalid cpe format (%s)", cpe)
continue
cpes[cpe] = cpeobj
else:
cpeobj = cpes[cpe]
cpeobj.setdefault('origins', set()).add(path)
def rec_iter(
filenames: List[str],
sensor: Optional[str],
ignore_rules: Dict[str, Dict[str, List[Tuple[int, int]]]],
) -> Generator[Tuple[Optional[int], Record], None, None]:
ignorenets = ignore_rules.get("IGNORENETS", {})
neverignore = ignore_rules.get("NEVERIGNORE", {})
for fname in filenames:
with P0fFile(fname) as fdesc:
for line in fdesc:
if not line:
continue
if "mod" not in line:
LOGGER.warning("no mod detected [%r]", line)
continue
if line["mod"] not in ["syn", "syn+ack"]:
continue
if "subj" not in line or line["subj"] not in line:
LOGGER.warning("no subj detected [%r]", line)
continue
if "raw_sig" not in line:
LOGGER.warning("no raw_sig detected [%r]", line)
continue
infos = {}
if "os" in line and line["os"] != "???":
infos["os"] = line["os"]
if "dist" in line:
infos["dist"] = line["dist"]
if "params" in line and line["params"].lower() != "none":
infos["params"] = line["params"]
host = line[line["subj"]].split("/")[0]
port = int(line["srv"].split("/")
[1]) if line["subj"] == "srv" else None
for tstamp, rec in handle_rec(
# sensor
sensor,
# ignorenets,
ignorenets,
# neverignore,
neverignore,
# timestamp
timestamp=line["ts"],
# uid
uid=None,
# host
host=host,
# srvport
srvport=port,
# recon_type
recon_type="P0FV3_%s" % line["mod"].upper(),
# source
source="P0FV3",
# value
value=line["raw_sig"],
# targetval
targetval=None,
):
if infos:
rec["infos"] = infos
yield (tstamp, rec)
def __init__(self, fname: str, pcap_filter: Optional[str] = None) -> None:
"""Init Ipatbles class."""
if pcap_filter is not None:
LOGGER.warning("PCAP filter not supported in Iptables")
super().__init__(fname)