def main(): """Update the flow database from ARP requests in PCAP files""" parser, use_argparse = utils.create_argparser(__doc__, extraargs="files") if use_argparse: parser.add_argument("files", nargs='*', metavar='FILE', help="PCAP files") parser.add_argument("-v", "--verbose", help="verbose mode", action="store_true") args = parser.parse_args() if args.verbose: config.DEBUG = True bulk = db.flow.start_bulk_insert() query_cache = db.flow.add_flow(["Flow"], ('proto', )) for fname in args.files: for pkt in reader(fname): rec = { "dst": pkt.pdst, "src": pkt.psrc, "start_time": datetime.fromtimestamp(pkt.time), "end_time": datetime.fromtimestamp(pkt.time), "proto": "arp" } if rec["dst"] != "0.0.0.0" and rec["src"] != "0.0.0.0": bulk.append(query_cache, rec) bulk.close()
def main(): parser, use_argparse = utils.create_argparser(__doc__, extraargs='ip') torun = [] parser.add_argument('--download', action='store_true', help='Fetch all data files.') parser.add_argument('--import-all', action='store_true', help='Create all CSV files for reverse lookups.') parser.add_argument('--quiet', "-q", action='store_true', help='Quiet mode.') if use_argparse: parser.add_argument('ip', nargs='*', metavar='IP', help='Display results for specified IP addresses.') args = parser.parse_args() if args.download: geoiputils.download_all(verbose=not args.quiet) db.data.reload_files() if args.import_all: torun.append((db.data.build_dumps, [], {})) for function, fargs, fkargs in torun: function(*fargs, **fkargs) for addr in args.ip: if addr.isdigit(): addr = int(addr) print(addr) for info in [db.data.as_byip(addr), db.data.location_byip(addr)]: if info: for key, value in viewitems(info): print(' %s %s' % (key, value))
def main(): """Parses the arguments and call worker()""" # Set the signal handler for s in [signal.SIGINT, signal.SIGTERM]: signal.signal(s, shutdown) signal.siginterrupt(s, False) parser, _ = utils.create_argparser(__doc__) parser.add_argument( '--sensor', metavar='SENSOR[:SENSOR]', help='sensor to check, optionally with a long name, defaults to all.', ) parser.add_argument( '--directory', metavar='DIR', help='base directory (defaults to /ivre/passiverecon/).', default="/ivre/passiverecon/", ) parser.add_argument( '--progname', metavar='PROG', help='Program to run (defaults to ivre passiverecon2db).', default="ivre passiverecon2db", ) args = parser.parse_args() if args.sensor is not None: SENSORS.update(dict([args.sensor.split(':', 1) if ':' in args.sensor else [args.sensor, args.sensor]])) sensor = args.sensor.split(':', 1)[0] else: sensor = None worker(args.progname, args.directory, sensor=sensor)
def main(): """Parses the arguments and call worker()""" # Set the signal handler for s in [signal.SIGINT, signal.SIGTERM]: signal.signal(s, shutdown) signal.siginterrupt(s, False) parser, _ = utils.create_argparser(__doc__) parser.add_argument( '--sensor', metavar='SENSOR[:SENSOR]', help='sensor to check, optionally with a long name, defaults to all.', ) parser.add_argument( '--directory', metavar='DIR', help='base directory (defaults to /ivre/passiverecon/).', default="/ivre/passiverecon/", ) parser.add_argument( '--progname', metavar='PROG', help='Program to run (defaults to ivre passiverecon2db).', default="ivre passiverecon2db", ) args = parser.parse_args() if args.sensor is not None: SENSORS.update( dict([ args.sensor.split(':', 1) if ':' in args.sensor else [args.sensor, args.sensor] ])) sensor = args.sensor.split(':', 1)[0] else: sensor = None worker(args.progname, args.directory, sensor=sensor)
def main(): """Update the flow database from Airodump CSV files""" parser, use_argparse = utils.create_argparser(__doc__, extraargs='files') if use_argparse: parser.add_argument('files', nargs='*', metavar='FILE', help='Airodump CSV files') parser.add_argument("-v", "--verbose", help="verbose mode", action="store_true") args = parser.parse_args() if args.verbose: config.DEBUG = True bulk = db.flow.start_bulk_insert() for fname in args.files: with Airodump(fname) as fdesc: for line in fdesc: if "Station MAC" in line: if line["BSSID"] == "(not associated)": continue line["src"] = line.pop("Station MAC") line["dst"] = line.pop("BSSID") # TODO FIX list del line["Probed ESSIDs"] line["start_time"] = line.pop("First time seen") line["end_time"] = line.pop("Last time seen") line["packets"] = line.pop('# packets') # TODO FIX MEAN (en plus de MAX et MEAN) db.flow.add_flow( line, "WLAN", {}, counters=["packets"], srcnode=("Intel:Mac", {"addr": "{src}"}), dstnode=("Intel:Wlan", {"addr": "{dst}"}), ) else: line["start_time"] = line.pop("First time seen") line["end_time"] = line.pop("Last time seen") line["lan_ip"] = line.pop("LAN IP") query = [ "MERGE (wlan:Intel:Wlan {addr: {BSSID}})", "ON CREATE SET wlan.essid = {ESSID}, " "wlan.firstseen = {start_time}, " "wlan.lastseen = {end_time}, " "wlan.channel = {channel}, wlan.speed = {Speed}, " "wlan.privacy = {Privacy}, wlan.cipher = {Cipher}, " "wlan.authentication = {Authentication}, " "wlan.ip = {lan_ip}", "ON MATCH SET wlan.essid = {ESSID}, " "wlan.firstseen = CASE WHEN " "wlan.firstseen > {start_time} THEN {start_time} " "ELSE wlan.firstseen END, wlan.lastseen = CASE WHEN " "wlan.lastseen < {end_time} THEN {end_time} ELSE " "wlan.lastseen END, wlan.channel = {channel}, " "wlan.speed = {Speed}, wlan.privacy = {Privacy}, " "wlan.cipher = {Cipher}, " "wlan.authentication = {Authentication}, " "wlan.ip = {lan_ip}", ] bulk.append("\n".join(query), line) bulk.close()
def parse_args(): """Imports the available module to parse the arguments and return the parsed arguments. """ parser, _ = create_argparser(__doc__) parser.add_argument('--bind-address', '-b', help='(IP) Address to bind the server to (defaults ' 'to 127.0.0.1).', default="127.0.0.1") parser.add_argument('--port', '-p', type=int, default=80, help='(TCP) Port to use (defaults to 80)') return parser.parse_args()
def main(): """Update the flow database from Bro logs""" parser, use_argparse = utils.create_argparser(__doc__, extraargs="logfiles") if use_argparse: parser.add_argument("logfiles", nargs='*', metavar='FILE', help="Bro log files") parser.add_argument("-v", "--verbose", help="verbose mode", action="store_true") parser.add_argument("-C", "--no-cleanup", help="avoid port cleanup heuristics", action="store_true") args = parser.parse_args() if args.verbose: config.DEBUG = True for fname in args.logfiles: if not os.path.exists(fname): utils.LOGGER.error("File %r does not exist", fname) continue with BroFile(fname) as brof: bulk = db.flow.start_bulk_insert() utils.LOGGER.debug( "Parsing %s\n\t%s", fname, "Fields:\n%s\n" % "\n".join("%s: %s" % (f, t) for f, t in brof.field_types)) if brof.path in FUNCTIONS: func = FUNCTIONS[brof.path] elif brof.path in flow.META_DESC: func = any2flow(brof.path) else: utils.LOGGER.debug("Log format not (yet) supported for %r", fname) continue for line in brof: if not line: continue func(bulk, _bro2flow(line)) db.flow.bulk_commit(bulk) if brof.path == "conn" and not args.no_cleanup: db.flow.cleanup_flows()
def main(): """Update the flow database from Bro logs""" parser, use_argparse = utils.create_argparser(__doc__, extraargs="logfiles") if use_argparse: parser.add_argument('logfiles', nargs='*', metavar='FILE', help='Bro log files') parser.add_argument("-v", "--verbose", help="verbose mode", action="store_true") parser.add_argument("-C", "--no-cleanup", help="avoid port cleanup heuristics", action="store_true") args = parser.parse_args() if args.verbose: config.DEBUG = True for fname in args.logfiles: if not os.path.exists(fname): utils.LOGGER.error("File %r does not exist", fname) continue with BroFile(fname) as brof: bulk = db.flow.start_bulk_insert() utils.LOGGER.debug("Parsing %s\n\t%s", fname, "Fields:\n%s\n" % "\n".join( "%s: %s" % (f, t) for f, t in brof.field_types )) if brof.path in FUNCTIONS: func = FUNCTIONS[brof.path] elif brof.path in ALL_DESCS: func = any2neo(ALL_DESCS[brof.path]) else: utils.LOGGER.debug("Log format not (yet) supported for %r", fname) continue for line in brof: if not line: continue func(bulk, _bro2neo(line)) bulk.commit() if brof.path == "conn" and not args.no_cleanup: db.flow.cleanup_flows()
def main(): """Update the flow database from log files""" parser, use_argparse = utils.create_argparser(__doc__, extraargs='files') if use_argparse: parser.add_argument('files', nargs='*', metavar='FILE', help='Files to import in the flow database') parser.add_argument("-v", "--verbose", help="verbose mode", action="store_true") parser.add_argument("-t", "--type", help="file type", choices=list(PARSERS_CHOICE)) parser.add_argument("-f", "--pcap-filter", help="pcap filter to apply (when supported)") parser.add_argument("-C", "--no-cleanup", help="avoid port cleanup heuristics", action="store_true") args = parser.parse_args() if args.verbose: config.DEBUG = True for fname in args.files: try: fileparser = PARSERS_CHOICE[args.type] except KeyError: with utils.open_file(fname) as fdesc: try: fileparser = PARSERS_MAGIC[fdesc.read(4)] except KeyError: utils.LOGGER.warning( 'Cannot find the appropriate parser for file %r', fname, ) continue bulk = db.flow.start_bulk_insert() with fileparser(fname, args.pcap_filter) as fdesc: for rec in fdesc: if not rec: continue db.flow.flow2flow(bulk, rec) db.flow.bulk_commit(bulk) if not args.no_cleanup: db.flow.cleanup_flows()
def main(): """Update the flow database from ARP requests in PCAP files""" parser, use_argparse = utils.create_argparser(__doc__, extraargs='files') if use_argparse: parser.add_argument('files', nargs='*', metavar='FILE', help='PCAP files') parser.add_argument("-v", "--verbose", help="verbose mode", action="store_true") args = parser.parse_args() if args.verbose: config.DEBUG = True bulk = db.flow.start_bulk_insert() query_cache = db.flow.add_flow(["Flow"], ('proto',)) for fname in args.files: for pkt in reader(fname): rec = {"dst": pkt.pdst, "src": pkt.psrc, "start_time": datetime.fromtimestamp(pkt.time), "end_time": datetime.fromtimestamp(pkt.time), "proto": "arp"} if rec["dst"] != "0.0.0.0" and rec["src"] != "0.0.0.0": bulk.append(query_cache, rec) bulk.close()
def main(): parser, _ = utils.create_argparser(__doc__) parser.add_argument('--init', '--purgedb', action='store_true', help='Purge or create and initialize the database.') parser.add_argument('--ensure-indexes', action='store_true', help='Create missing indexes (will lock the ' 'database).') parser.add_argument('--node-filters', '-n', nargs="+", metavar="FILTER", help='Filter the results with a list of ivre specific ' 'node textual filters (see WebUI doc in FLOW.md).') parser.add_argument('--flow-filters', '-f', nargs="+", metavar="FILTER", help='Filter the results with a list of ivre specific ' 'flow textual filters (see WebUI doc in FLOW.md).') parser.add_argument('--json', '-j', action='store_true', help='Outputs the full json records of results.') parser.add_argument('--count', '-c', action='store_true', help='Only return the count of the results.') parser.add_argument('--limit', '-l', type=int, help='Ouput at most LIMIT results.') parser.add_argument('--skip', type=int, default=0, help='Skip first SKIP results.') parser.add_argument('--orderby', '-o', help='Order of results ("src", "dst" or "flow")') parser.add_argument('--separator', '-s', help="Separator string.") parser.add_argument('--top', '-t', nargs="+", help='Top flows for a given set of fields, e.g. ' '"--top src.addr dport".') parser.add_argument('--collect', '-C', nargs="+", help='When using --top, also collect these ' 'properties.') parser.add_argument('--sum', '-S', nargs="+", help='When using --top, sum on these properties to ' 'order the result.') parser.add_argument('--mode', '-m', help="Query special mode (flow_map, talk_map...)") parser.add_argument('--timeline', '-T', action="store_true", help='Retrieves the timeline of each flow') parser.add_argument('--flow-daily', action="store_true", help="Flow count per times of the day") parser.add_argument('--plot', action="store_true", help="Plot data when possible (requires matplotlib).") parser.add_argument('--fields', nargs='+', help="Display these fields for each entry.") args = parser.parse_args() out = sys.stdout if args.plot and plt is None: utils.LOGGER.critical("Matplotlib is required for --plot") sys.exit(-1) if args.init: if os.isatty(sys.stdin.fileno()): out.write('This will remove any scan result in your database. ' 'Process ? [y/N] ') ans = input() if ans.lower() != 'y': sys.exit(-1) db.flow.init() sys.exit(0) if args.ensure_indexes: if os.isatty(sys.stdin.fileno()): out.write('This will lock your database. ' 'Process ? [y/N] ') ans = input() if ans.lower() != 'y': sys.exit(-1) db.flow.ensure_indexes() sys.exit(0) filters = { "nodes": args.node_filters or [], "edges": args.flow_filters or [] } query = db.flow.from_filters(filters, limit=args.limit, skip=args.skip, orderby=args.orderby, mode=args.mode, timeline=args.timeline) sep = args.separator or ' | ' coma = ';' if args.separator else '; ' coma2 = ',' if args.separator else ', ' if args.count: count = db.flow.count(query) out.write('%(clients)d clients\n%(servers)d servers\n' '%(flows)d flows\n' % count) elif args.top: top = db.flow.top(query, args.top, args.collect, args.sum) for rec in top: sys.stdout.write( "%s%s%s%s%s\n" % (coma.join(str(elt) for elt in rec["fields"]), sep, rec["count"], sep, coma.join( str(coma2.join(str(val) for val in elt)) for elt in rec["collected"]) if rec["collected"] else "")) elif args.flow_daily: # FIXME? fully in-memory if args.plot: plot_data = {} for rec in db.flow.flow_daily(query): out.write( sep.join([ rec["flow"], rec["time_in_day"].strftime("%T.%f"), str(rec["count"]) ])) out.write("\n") if args.plot: plot_data.setdefault(rec["flow"], [[], []]) plot_data[rec["flow"]][0].append(rec["time_in_day"]) plot_data[rec["flow"]][1].append(rec["count"]) for flow, points in viewitems(plot_data): plt.plot(points[0], points[1], label=flow) plt.legend(loc='best') plt.show() else: fmt = '%%s%s%%s%s%%s' % (sep, sep) node_width = len('XXX.XXX.XXX.XXX') flow_width = len('tcp/XXXXX') for res in db.flow.to_iter(query): if args.json: out.write('%s\n' % res) else: elts = {} for elt in ["src", "flow", "dst"]: elts[elt] = res[elt]['label'] if args.fields: elts[elt] = "%s%s%s" % ( elts[elt], coma, coma.join( str(res[elt]['data'].get(field, "")) for field in args.fields)) src, flow, dst = elts["src"], elts["flow"], elts["dst"] node_width = max(node_width, len(src), len(dst)) flow_width = max(flow_width, len(flow)) if not args.separator: fmt = ('%%-%ds%s%%-%ds%s%%-%ds' % (node_width, sep, flow_width, sep, node_width)) out.write(fmt % (src, flow, dst)) if args.timeline: out.write(sep) out.write( coma.join( str(elt) for elt in sorted(res['flow']['data'] ['meta']['times']))) out.write('\n')
def main(): """Update the flow database from Airodump CSV files""" parser, use_argparse = utils.create_argparser(__doc__, extraargs='files') if use_argparse: parser.add_argument('files', nargs='*', metavar='FILE', help='Airodump CSV files') parser.add_argument("-v", "--verbose", help="verbose mode", action="store_true") args = parser.parse_args() if args.verbose: config.DEBUG = True bulk = db.flow.start_bulk_insert() for fname in args.files: with Airodump(fname) as fdesc: for line in fdesc: if "Station MAC" in line: if line["BSSID"] == "(not associated)": continue line["src"] = line.pop("Station MAC") line["dst"] = line.pop("BSSID") # TODO FIX list del line["Probed ESSIDs"] line["start_time"] = line.pop("First time seen") line["end_time"] = line.pop("Last time seen") line["packets"] = line.pop('# packets') # TODO FIX MEAN (en plus de MAX et MEAN) db.flow.add_flow( line, "WLAN", {}, counters=["packets"], srcnode=("Intel:Mac", { "addr": "{src}" }), dstnode=("Intel:Wlan", { "addr": "{dst}" }), ) else: line["start_time"] = line.pop("First time seen") line["end_time"] = line.pop("Last time seen") line["lan_ip"] = line.pop("LAN IP") query = [ "MERGE (wlan:Intel:Wlan {addr: {BSSID}})", "ON CREATE SET wlan.essid = {ESSID}, " "wlan.firstseen = {start_time}, " "wlan.lastseen = {end_time}, " "wlan.channel = {channel}, wlan.speed = {Speed}, " "wlan.privacy = {Privacy}, wlan.cipher = {Cipher}, " "wlan.authentication = {Authentication}, " "wlan.ip = {lan_ip}", "ON MATCH SET wlan.essid = {ESSID}, " "wlan.firstseen = CASE WHEN " "wlan.firstseen > {start_time} THEN {start_time} " "ELSE wlan.firstseen END, wlan.lastseen = CASE WHEN " "wlan.lastseen < {end_time} THEN {end_time} ELSE " "wlan.lastseen END, wlan.channel = {channel}, " "wlan.speed = {Speed}, wlan.privacy = {Privacy}, " "wlan.cipher = {Cipher}, " "wlan.authentication = {Authentication}, " "wlan.ip = {lan_ip}", ] bulk.append("\n".join(query), line) bulk.close()
def main(): """Update the flow database from log files""" parser, use_argparse = utils.create_argparser(__doc__, extraargs='files') if use_argparse: parser.add_argument('files', nargs='*', metavar='FILE', help='Files to import in the flow database') parser.add_argument("-v", "--verbose", help="verbose mode", action="store_true") parser.add_argument("-t", "--type", help="file type", choices=list(PARSERS_CHOICE)) parser.add_argument("-f", "--pcap-filter", help="pcap filter to apply (when supported)") parser.add_argument("-C", "--no-cleanup", help="avoid port cleanup heuristics", action="store_true") args = parser.parse_args() if args.verbose: config.DEBUG = True query_cache = {} for fname in args.files: try: fileparser = PARSERS_CHOICE[args.type] except KeyError: with utils.open_file(fname) as fdesc: try: fileparser = PARSERS_MAGIC[fdesc.read(4)] except KeyError: utils.LOGGER.warning( 'Cannot find the appropriate parser for file %r', fname, ) continue bulk = db.flow.start_bulk_insert() with fileparser(fname, args.pcap_filter) as fdesc: for rec in fdesc: if not rec: continue linkattrs = ('proto', ) accumulators = {} for (fields, sp_linkattrs, sp_accumulators) in FIELD_REQUEST_EXT: if all(field in rec for field in fields): linkattrs = sp_linkattrs accumulators = sp_accumulators break if linkattrs not in query_cache: query_cache[linkattrs] = db.flow.add_flow( ["Flow"], linkattrs, counters=COUNTERS, accumulators=accumulators) bulk.append(query_cache[linkattrs], rec) bulk.close() if not args.no_cleanup: db.flow.cleanup_flows()
def main(): parser, _ = utils.create_argparser(__doc__) parser.add_argument('--init', '--purgedb', action='store_true', help='Purge or create and initialize the database.') parser.add_argument('--ensure-indexes', action='store_true', help='Create missing indexes (will lock the ' 'database).') parser.add_argument('--node-filters', '-n', nargs="+", metavar="FILTER", help='Filter the results with a list of ivre specific ' 'node textual filters (see WebUI doc in FLOW.md).') parser.add_argument('--flow-filters', '-f', nargs="+", metavar="FILTER", help='Filter the results with a list of ivre specific ' 'flow textual filters (see WebUI doc in FLOW.md).') parser.add_argument('--json', '-j', action='store_true', help='Outputs the full json records of results.') parser.add_argument('--count', '-c', action='store_true', help='Only return the count of the results.') parser.add_argument('--limit', '-l', type=int, default=None, help='Ouput at most LIMIT results.') parser.add_argument('--skip', type=int, default=0, help='Skip first SKIP results.') parser.add_argument('--orderby', '-o', help='Order of results ("src", "dst" or "flow")') parser.add_argument('--separator', '-s', help="Separator string.") parser.add_argument('--top', '-t', nargs="+", help='Top flows for a given set of fields, e.g. ' '"--top src.addr dport".') parser.add_argument('--collect', '-C', nargs="+", help='When using --top, also collect these ' 'properties.', default=[]) parser.add_argument('--sum', '-S', nargs="+", help='When using --top, sum on these properties to ' 'order the result.', default=[]) parser.add_argument('--least', '-L', action='store_true', help='When using --top, sort records by least') parser.add_argument('--mode', '-m', help="Query special mode (flow_map, talk_map...)") parser.add_argument('--timeline', '-T', action="store_true", help='Retrieves the timeline of each flow') parser.add_argument('--flow-daily', action="store_true", help="Flow count per times of the day. If --precision " "is absent, it will be based on FLOW_TIME_PRECISION " "(%d)" % config.FLOW_TIME_PRECISION) parser.add_argument('--plot', action="store_true", help="Plot data when possible (requires matplotlib).") parser.add_argument('--fields', nargs='+', help="Display these fields for each entry.") parser.add_argument('--reduce-precision', type=int, metavar="NEW_PRECISION", help="Only with MongoDB backend. " "Reduce precision to NEW_PRECISION for flows " "timeslots. Uses precision, before, after and " "filters.") parser.add_argument("--after", "-a", type=str, help="Only with MongoDB " "backend. Get only flows seen after this date. " "Date format: YEAR-MONTH-DAY HOUR:MINUTE. " "Based on timeslots precision. If the given date is " "in the middle of a timeslot, flows start at the next " "timeslot.") parser.add_argument("--before", "-b", type=str, help="Only with MongoDB " "backend. Get only flows seen before this date. " "Date format: YEAR-MONTH-DAY HOUR:MINUTE. " "Based on timeslots precision. If the given date is " "in the middle of a timeslot, the whole period is " "kept even if theoretically some flows may have been " "seen after the given date.") parser.add_argument('--precision', nargs='?', default=None, const=0, help="Only With MongoDB backend. If PRECISION is " "specified, get only flows with one timeslot of " "the given precision. Otherwise, list " "precisions.", type=int) args = parser.parse_args() out = sys.stdout if args.plot and plt is None: utils.LOGGER.critical("Matplotlib is required for --plot") sys.exit(-1) if args.init: if os.isatty(sys.stdin.fileno()): out.write('This will remove any flow result in your database. ' 'Process ? [y/N] ') ans = input() if ans.lower() != 'y': sys.exit(-1) db.flow.init() sys.exit(0) if args.ensure_indexes: if os.isatty(sys.stdin.fileno()): out.write('This will lock your database. ' 'Process ? [y/N] ') ans = input() if ans.lower() != 'y': sys.exit(-1) db.flow.ensure_indexes() sys.exit(0) if args.precision == 0: # Get precisions list out.writelines('%d\n' % precision for precision in db.flow.list_precisions()) sys.exit(0) filters = { "nodes": args.node_filters or [], "edges": args.flow_filters or [] } time_args = ['before', 'after'] time_values = {} args_dict = vars(args) for arg in time_args: time_values[arg] = (datetime.datetime.strptime(args_dict[arg], "%Y-%m-%d %H:%M") if args_dict[arg] is not None else None) query = db.flow.from_filters(filters, limit=args.limit, skip=args.skip, orderby=args.orderby, mode=args.mode, timeline=args.timeline, after=time_values['after'], before=time_values['before'], precision=args.precision) if args.reduce_precision: if os.isatty(sys.stdin.fileno()): out.write('This will permanently reduce the precision of your ' 'database. Process ? [y/N] ') ans = input() if ans.lower() != 'y': sys.exit(-1) new_precision = args.reduce_precision db.flow.reduce_precision(new_precision, flt=query, before=time_values['before'], after=time_values['after'], current_precision=args.precision) sys.exit(0) sep = args.separator or ' | ' coma = ' ;' if args.separator else ' ; ' coma2 = ',' if args.separator else ', ' if args.count: count = db.flow.count(query) out.write('%(clients)d clients\n%(servers)d servers\n' '%(flows)d flows\n' % count) elif args.top: top = db.flow.topvalues(query, args.top, collect_fields=args.collect, sum_fields=args.sum, topnbr=args.limit, skip=args.skip, least=args.least) for rec in top: sys.stdout.write( "%s%s%s%s%s\n" % ('(' + coma2.join(str(val) for val in rec["fields"]) + ')', sep, rec["count"], sep, coma.join( str('(' + coma2.join(str(val) for val in collected) + ')') for collected in rec["collected"]) if rec["collected"] else "")) elif args.flow_daily: precision = (args.precision if args.precision is not None else config.FLOW_TIME_PRECISION) plot_data = {} for rec in db.flow.flow_daily(precision, query, after=time_values['after'], before=time_values['before']): out.write( sep.join([ rec["time_in_day"].strftime("%T.%f"), " ; ".join([ '(' + x[0] + ', ' + str(x[1]) + ')' for x in rec["flows"] ]) ])) out.write("\n") if args.plot: for flw in rec["flows"]: t = rec["time_in_day"] # pyplot needs datetime objects dt = datetime.datetime(1970, 1, 1, hour=t.hour, minute=t.minute, second=t.second) plot_data.setdefault(flw[0], {}) plot_data[flw[0]][dt] = flw[1] if args.plot and plot_data: t = datetime.datetime(1970, 1, 1, 0, 0, 0) t += datetime.timedelta(seconds=config.FLOW_TIME_BASE % precision) times = [] while t < datetime.datetime(1970, 1, 2): times.append(t) t = t + datetime.timedelta(seconds=precision) ax = plt.subplots()[1] fmt = matplotlib.dates.DateFormatter('%H:%M:%S') for flow, data in viewitems(plot_data): values = [(data[ti] if ti in data else 0) for ti in times] plt.step(times, values, '.-', where='post', label=flow) plt.legend(loc='best') ax.xaxis.set_major_formatter(fmt) plt.gcf().autofmt_xdate() plt.show() else: fmt = '%%s%s%%s%s%%s' % (sep, sep) node_width = len('XXXX:XXXX:XXXX:XXXX:XXXX:XXXX') flow_width = len('tcp/XXXXX') for res in db.flow.to_iter(query, limit=args.limit, skip=args.skip, orderby=args.orderby, mode=args.mode, timeline=args.timeline): if args.json: out.write('%s\n' % res) else: elts = {} for elt in ["src", "flow", "dst"]: elts[elt] = res[elt]['label'] if args.fields: elts[elt] = "%s%s%s" % ( elts[elt], coma, coma.join( str(res[elt]['data'].get(field, "")) for field in args.fields)) src, flow, dst = elts["src"], elts["flow"], elts["dst"] node_width = max(node_width, len(src), len(dst)) flow_width = max(flow_width, len(flow)) if not args.separator: fmt = ('%%-%ds%s%%-%ds%s%%-%ds' % (node_width, sep, flow_width, sep, node_width)) out.write(fmt % (src, flow, dst)) if args.timeline: out.write(sep) # Print '?' instead of failing if meta.times does not exist try: out.write( coma.join( str(elt) for elt in sorted(res['flow']['data'] ['meta']['times']))) except KeyError: out.write("?") out.write('\n')
def main(): parser, _ = utils.create_argparser(__doc__) parser.add_argument('--init', '--purgedb', action='store_true', help='Purge or create and initialize the database.') parser.add_argument('--ensure-indexes', action='store_true', help='Create missing indexes (will lock the ' 'database).') parser.add_argument('--node-filters', '-n', nargs="+", metavar="FILTER", help='Filter the results with a list of ivre specific ' 'node textual filters (see WebUI doc in FLOW.md).') parser.add_argument('--flow-filters', '-f', nargs="+", metavar="FILTER", help='Filter the results with a list of ivre specific ' 'flow textual filters (see WebUI doc in FLOW.md).') parser.add_argument('--json', '-j', action='store_true', help='Outputs the full json records of results.') parser.add_argument('--count', '-c', action='store_true', help='Only return the count of the results.') parser.add_argument('--limit', '-l', type=int, help='Ouput at most LIMIT results.') parser.add_argument('--skip', type=int, default=0, help='Skip first SKIP results.') parser.add_argument('--orderby', '-o', help='Order of results ("src", "dst" or "flow")') parser.add_argument('--separator', '-s', help="Separator string.") parser.add_argument('--top', '-t', nargs="+", help='Top flows for a given set of fields, e.g. ' '"--top src.addr dport".') parser.add_argument('--collect', '-C', nargs="+", help='When using --top, also collect these ' 'properties.') parser.add_argument('--sum', '-S', nargs="+", help='When using --top, sum on these properties to ' 'order the result.') parser.add_argument('--mode', '-m', help="Query special mode (flow_map, talk_map...)") parser.add_argument('--timeline', '-T', action="store_true", help='Retrieves the timeline of each flow') parser.add_argument('--flow-daily', action="store_true", help="Flow count per times of the day") parser.add_argument('--plot', action="store_true", help="Plot data when possible (requires matplotlib).") parser.add_argument('--fields', nargs='+', help="Display these fields for each entry.") args = parser.parse_args() out = sys.stdout if args.plot and plt is None: utils.LOGGER.critical("Matplotlib is required for --plot") sys.exit(-1) if args.init: if os.isatty(sys.stdin.fileno()): out.write( 'This will remove any scan result in your database. ' 'Process ? [y/N] ') ans = input() if ans.lower() != 'y': sys.exit(-1) db.flow.init() sys.exit(0) if args.ensure_indexes: if os.isatty(sys.stdin.fileno()): out.write( 'This will lock your database. ' 'Process ? [y/N] ') ans = input() if ans.lower() != 'y': sys.exit(-1) db.flow.ensure_indexes() sys.exit(0) filters = {"nodes": args.node_filters or [], "edges": args.flow_filters or []} query = db.flow.from_filters(filters, limit=args.limit, skip=args.skip, orderby=args.orderby, mode=args.mode, timeline=args.timeline) sep = args.separator or ' | ' coma = ';' if args.separator else '; ' coma2 = ',' if args.separator else ', ' if args.count: count = db.flow.count(query) out.write('%(clients)d clients\n%(servers)d servers\n' '%(flows)d flows\n' % count) elif args.top: top = db.flow.top(query, args.top, args.collect, args.sum) for rec in top: sys.stdout.write("%s%s%s%s%s\n" % ( coma.join(str(elt) for elt in rec["fields"]), sep, rec["count"], sep, coma.join(str(coma2.join(str(val) for val in elt)) for elt in rec["collected"]) if rec["collected"] else "" )) elif args.flow_daily: # FIXME? fully in-memory if args.plot: plot_data = {} for rec in db.flow.flow_daily(query): out.write(sep.join([rec["flow"], rec["time_in_day"].strftime("%T.%f"), str(rec["count"])])) out.write("\n") if args.plot: plot_data.setdefault(rec["flow"], [[], []]) plot_data[rec["flow"]][0].append(rec["time_in_day"]) plot_data[rec["flow"]][1].append(rec["count"]) for flow, points in viewitems(plot_data): plt.plot(points[0], points[1], label=flow) plt.legend(loc='best') plt.show() else: fmt = '%%s%s%%s%s%%s' % (sep, sep) node_width = len('XXX.XXX.XXX.XXX') flow_width = len('tcp/XXXXX') for res in db.flow.to_iter(query): if args.json: out.write('%s\n' % res) else: elts = {} for elt in ["src", "flow", "dst"]: elts[elt] = res[elt]['label'] if args.fields: elts[elt] = "%s%s%s" % ( elts[elt], coma, coma.join( str(res[elt]['data'].get(field, "")) for field in args.fields ) ) src, flow, dst = elts["src"], elts["flow"], elts["dst"] node_width = max(node_width, len(src), len(dst)) flow_width = max(flow_width, len(flow)) if not args.separator: fmt = ('%%-%ds%s%%-%ds%s%%-%ds' % (node_width, sep, flow_width, sep, node_width)) out.write(fmt % (src, flow, dst)) if args.timeline: out.write(sep) out.write(coma.join( str(elt) for elt in sorted( res['flow']['data']['meta']['times'] ) )) out.write('\n')