示例#1
0
def handle_rec(sensor, ignorenets, neverignore,
               # these argmuments are provided by **bro_line
               timestamp=None, uid=None, host=None, srvport=None,
               recon_type=None, source=None, value=None, targetval=None):
    if host is None:
        spec = {
            'targetval': targetval,
            'recontype': recon_type,
            'value': value
        }
    else:
        spec = {
            'addr': utils.force_ip2int(host),
            'recontype': recon_type,
            'value': value
        }
    if sensor is not None:
        spec.update({'sensor': sensor})
    if srvport is not None:
        spec.update({'port': srvport})
    if source is not None:
        spec.update({'source': source})
    spec = _prepare_rec(spec, ignorenets, neverignore)
    float_ts = utils.datetime2timestamp(timestamp)
    return float_ts, spec
示例#2
0
文件: neo4j.py 项目: ryanbekabe/ivre
 def to_dbprop(cls, prop, val):
     if prop in cls.DATE_FIELDS and isinstance(val, basestring):
         val = datetime.strptime(val, cls.TIMEFMT)
     # Intentional double if: str -> datetime -> float
     if isinstance(val, datetime):
         val = utils.datetime2timestamp(val)
     return val
示例#3
0
文件: neo4j.py 项目: a2si/ivre
 def to_dbprop(cls, prop, val):
     if prop in cls.DATE_FIELDS and isinstance(val, basestring):
         val = datetime.datetime.strptime(val, cls.TIMEFMT)
     # Intentional double if: str -> datetime -> float
     if isinstance(val, datetime.datetime):
         val = utils.datetime2timestamp(val)
     return val
示例#4
0
文件: neo4j.py 项目: ryanbekabe/ivre
 def _date_round(cls, date):
     if isinstance(date, datetime):
         ts = utils.datetime2timestamp(date)
     else:
         ts = date
     ts = ts - (ts % config.FLOW_TIME_PRECISION)
     if isinstance(date, datetime):
         return datetime.fromtimestamp(ts)
     else:
         return ts
示例#5
0
文件: neo4j.py 项目: a2si/ivre
 def _date_round(cls, date):
     if isinstance(date, datetime.datetime):
         ts = utils.datetime2timestamp(date)
     else:
         ts = date
     ts = ts - (ts % config.FLOW_TIME_PRECISION)
     if isinstance(date, datetime.datetime):
         return datetime.datetime.fromtimestamp(ts)
     else:
         return ts
示例#6
0
文件: app.py 项目: sree789/ivre-1
def get_nmap(subdb):
    """Get records from Nmap & View databases

    :param str subdb: database to query (must be "scans" or "view")
    :query str q: query (including limit/skip and sort)
    :query str callback: callback to use for JSONP results
    :query bool ipsasnumbers: to get IP addresses as numbers rather than as
                             strings
    :query bool datesasstrings: to get dates as strings rather than as
                               timestamps
    :status 200: no error
    :status 400: invalid referer
    :>jsonarr object: results

    """
    subdb_tool = "view" if subdb == 'view' else "scancli"
    subdb = db.view if subdb == 'view' else db.nmap
    flt_params = get_nmap_base(subdb)
    # PostgreSQL: the query plan if affected by the limit and gives
    # really poor results. This is a temporary workaround (look for
    # XXX-WORKAROUND-PGSQL).
    # result = subdb.get(flt_params.flt, limit=flt_params.limit,
    #                    skip=flt_params.skip, sort=flt_params.sortby)
    result = subdb.get(flt_params.flt,
                       skip=flt_params.skip,
                       sort=flt_params.sortby)

    if flt_params.unused:
        msg = 'Option%s not understood: %s' % (
            's' if len(flt_params.unused) > 1 else '',
            ', '.join(flt_params.unused),
        )
        if flt_params.callback is not None:
            yield webutils.js_alert("param-unused", "warning", msg)
        utils.LOGGER.warning(msg)
    elif flt_params.callback is not None:
        yield webutils.js_del_alert("param-unused")

    if config.DEBUG:
        msg1 = "filter: %s" % subdb.flt2str(flt_params.flt)
        msg2 = "user: %r" % webutils.get_user()
        utils.LOGGER.debug(msg1)
        utils.LOGGER.debug(msg2)
        if flt_params.callback is not None:
            yield webutils.js_alert("filter", "info", msg1)
            yield webutils.js_alert("user", "info", msg2)

    version_mismatch = {}
    if flt_params.callback is None:
        yield "[\n"
    else:
        yield "%s([\n" % flt_params.callback
    # XXX-WORKAROUND-PGSQL
    # for rec in result:
    for i, rec in enumerate(result):
        for fld in ['_id', 'scanid']:
            try:
                del rec[fld]
            except KeyError:
                pass
        if not flt_params.ipsasnumbers:
            rec['addr'] = utils.force_int2ip(rec['addr'])
        for field in ['starttime', 'endtime']:
            if field in rec:
                if not flt_params.datesasstrings:
                    rec[field] = int(utils.datetime2timestamp(rec[field]))
        for port in rec.get('ports', []):
            if 'screendata' in port:
                port['screendata'] = utils.encode_b64(port['screendata'])
            for script in port.get('scripts', []):
                if "masscan" in script:
                    try:
                        del script['masscan']['raw']
                    except KeyError:
                        pass
        if not flt_params.ipsasnumbers:
            if 'traces' in rec:
                for trace in rec['traces']:
                    trace['hops'].sort(key=lambda x: x['ttl'])
                    for hop in trace['hops']:
                        hop['ipaddr'] = utils.force_int2ip(hop['ipaddr'])
        addresses = rec.get('addresses', {}).get('mac')
        if addresses:
            newaddresses = []
            for addr in addresses:
                manuf = utils.mac2manuf(addr)
                if manuf and manuf[0]:
                    newaddresses.append({'addr': addr, 'manuf': manuf[0]})
                else:
                    newaddresses.append({'addr': addr})
            rec['addresses']['mac'] = newaddresses
        yield "%s\t%s" % ('' if i == 0 else ',\n',
                          json.dumps(rec, default=utils.serialize))
        check = subdb.cmp_schema_version_host(rec)
        if check:
            version_mismatch[check] = version_mismatch.get(check, 0) + 1
        # XXX-WORKAROUND-PGSQL
        if i + 1 >= flt_params.limit:
            break
    if flt_params.callback is None:
        yield "\n]\n"
    else:
        yield "\n]);\n"

    messages = {
        1:
        lambda count: ("%d document%s displayed %s out-of-date. Please run "
                       "the following command: 'ivre %s "
                       "--update-schema;" %
                       (count, 's' if count > 1 else '', 'are'
                        if count > 1 else 'is', subdb_tool)),
        -1:
        lambda count: ('%d document%s displayed ha%s been inserted by '
                       'a more recent version of IVRE. Please update '
                       'IVRE!' % (count, 's' if count > 1 else '', 've'
                                  if count > 1 else 's')),
    }
    for mismatch, count in viewitems(version_mismatch):
        message = messages[mismatch](count)
        if flt_params.callback is not None:
            yield webutils.js_alert(
                "version-mismatch-%d" % ((mismatch + 1) // 2), "warning",
                message)
        utils.LOGGER.warning(message)
示例#7
0
文件: app.py 项目: sree789/ivre-1
 def r2time(r):
     return (int(utils.datetime2timestamp(r['starttime'])) %
             int(request.params.get("modulo")))
示例#8
0
文件: app.py 项目: sree789/ivre-1
 def r2time(r):
     return int(utils.datetime2timestamp(r['starttime']))
示例#9
0
def get_nmap():
    flt_params = get_nmap_base()
    ## PostgreSQL: the query plan if affected by the limit and gives
    ## really poor results. This is a temporary workaround (look for
    ## XXX-WORKAROUND-PGSQL)
    # result = db.view.get(flt_params.flt, limit=flt_params.limit,
    #                      skip=flt_params.skip, sort=flt_params.sortby)
    result = db.view.get(flt_params.flt,
                         skip=flt_params.skip,
                         sort=flt_params.sortby)

    if flt_params.unused:
        msg = 'Option%s not understood: %s' % (
            's' if len(flt_params.unused) > 1 else '',
            ', '.join(flt_params.unused),
        )
        if flt_params.callback is not None:
            yield webutils.js_alert("param-unused", "warning", msg)
        utils.LOGGER.warning(msg)
    elif flt_params.callback is not None:
        yield webutils.js_del_alert("param-unused")

    if config.DEBUG:
        msg1 = "filter: %s" % db.view.flt2str(flt_params.flt)
        msg2 = "user: %r" % webutils.get_user()
        utils.LOGGER.debug(msg1)
        utils.LOGGER.debug(msg2)
        if flt_params.callback is not None:
            yield webutils.js_alert("filter", "info", msg1)
            yield webutils.js_alert("user", "info", msg2)

    version_mismatch = {}
    if flt_params.callback is None:
        yield "[\n"
    else:
        yield "%s([\n" % flt_params.callback
    ## XXX-WORKAROUND-PGSQL
    # for rec in result:
    for i, rec in enumerate(result):
        for fld in ['_id', 'scanid']:
            try:
                del rec[fld]
            except KeyError:
                pass
        if not flt_params.ipsasnumbers:
            rec['addr'] = utils.force_int2ip(rec['addr'])
        for field in ['starttime', 'endtime']:
            if field in rec:
                if not flt_params.datesasstrings:
                    rec[field] = int(utils.datetime2timestamp(rec[field]))
        for port in rec.get('ports', []):
            if 'screendata' in port:
                port['screendata'] = utils.encode_b64(port['screendata'])
            for script in port.get('scripts', []):
                if "masscan" in script:
                    try:
                        del script['masscan']['raw']
                    except KeyError:
                        pass
        if not flt_params.ipsasnumbers:
            if 'traces' in rec:
                for trace in rec['traces']:
                    trace['hops'].sort(key=lambda x: x['ttl'])
                    for hop in trace['hops']:
                        hop['ipaddr'] = utils.force_int2ip(hop['ipaddr'])
        yield "%s\t%s" % ('' if i == 0 else ',\n',
                          json.dumps(rec, default=utils.serialize))
        check = db.view.cmp_schema_version_host(rec)
        if check:
            version_mismatch[check] = version_mismatch.get(check, 0) + 1
        # XXX-WORKAROUND-PGSQL
        if i + 1 >= flt_params.limit:
            break
    if flt_params.callback is None:
        yield "\n]\n"
    else:
        yield "\n]);\n"

    messages = {
        1:
        lambda count: ("%d document%s displayed %s out-of-date. Please run "
                       "the following command: 'ivre scancli "
                       "--update-schema;" % (count, 's'
                                             if count > 1 else '', 'are'
                                             if count > 1 else 'is')),
        -1:
        lambda count: ('%d document%s displayed ha%s been inserted by '
                       'a more recent version of IVRE. Please update '
                       'IVRE!' % (count, 's' if count > 1 else '', 've'
                                  if count > 1 else 's')),
    }
    for mismatch, count in viewitems(version_mismatch):
        message = messages[mismatch](count)
        if flt_params.callback is not None:
            yield webutils.js_alert(
                "version-mismatch-%d" % ((mismatch + 1) // 2), "warning",
                message)
        utils.LOGGER.warning(message)
示例#10
0
def get_nmap_action(action):
    flt_params = get_nmap_base()
    preamble = "[\n"
    postamble = "]\n"
    r2res = lambda x: x
    if action == "timeline":
        result, count = db.view.get_open_port_count(flt_params.flt)
        if request.params.get("modulo") is None:
            r2time = lambda r: int(utils.datetime2timestamp(r['starttime']))
        else:
            r2time = lambda r: (int(utils.datetime2timestamp(r['starttime'])) %
                                int(request.params.get("modulo")))
        if flt_params.ipsasnumbers:
            r2res = lambda r: [
                r2time(r),
                utils.ip2int(r['addr']), r['openports']['count']
            ]
        else:
            r2res = lambda r: [r2time(r), r['addr'], r['openports']['count']]
    elif action == "coordinates":
        preamble = '{"type": "GeometryCollection", "geometries": [\n'
        postamble = ']}\n'
        result = list(db.view.getlocations(flt_params.flt))
        count = len(result)
        r2res = lambda r: {
            "type": "Point",
            "coordinates": r['_id'],
            "properties": {
                "count": r['count']
            },
        }
    elif action == "countopenports":
        result, count = db.view.get_open_port_count(flt_params.flt)
        if flt_params.ipsasnumbers:
            r2res = lambda r: [
                utils.ip2int(r['addr']), r['openports']['count']
            ]
        else:
            r2res = lambda r: [r['addr'], r['openports']['count']]
    elif action == "ipsports":
        result, count = db.view.get_ips_ports(flt_params.flt)
        if flt_params.ipsasnumbers:
            r2res = lambda r: [
                utils.ip2int(r['addr']),
                [[p['port'], p['state_state']] for p in r.get('ports', [])
                 if 'state_state' in p]
            ]
        else:
            r2res = lambda r: [
                r['addr'],
                [[p['port'], p['state_state']] for p in r.get('ports', [])
                 if 'state_state' in p]
            ]
    elif action == "onlyips":
        result, count = db.view.get_ips(flt_params.flt)
        if flt_params.ipsasnumbers:
            r2res = lambda r: utils.ip2int(r['addr'])
        else:
            r2res = lambda r: r['addr']
    elif action == "diffcats":
        if request.params.get("onlydiff"):
            output = db.view.diff_categories(request.params.get("cat1"),
                                             request.params.get("cat2"),
                                             flt=flt_params.flt,
                                             include_both_open=False)
        else:
            output = db.view.diff_categories(request.params.get("cat1"),
                                             request.params.get("cat2"),
                                             flt=flt_params.flt)
        count = 0
        result = {}
        if flt_params.ipsasnumbers:
            for res in output:
                result.setdefault(res["addr"],
                                  []).append([res['port'], res['value']])
                count += 1
        else:
            for res in output:
                result.setdefault(utils.int2ip(res["addr"]),
                                  []).append([res['port'], res['value']])
                count += 1
        result = viewitems(result)
    if flt_params.callback is not None:
        if count >= config.WEB_WARN_DOTS_COUNT:
            yield (
                'if(confirm("You are about to ask your browser to display %d '
                'dots, which is a lot and might slow down, freeze or crash '
                'your browser. Do you want to continue?")) {\n' % count)
        yield '%s(\n' % flt_params.callback
    yield preamble

    # hack to avoid a trailing comma
    result = iter(result)
    try:
        rec = next(result)
    except StopIteration:
        pass
    else:
        yield json.dumps(r2res(rec))
        for rec in result:
            yield ",\n" + json.dumps(r2res(rec))
        yield "\n"

    yield postamble
    if flt_params.callback is not None:
        yield ");"
        if count >= config.WEB_WARN_DOTS_COUNT:
            yield '}\n'
    else:
        yield "\n"
示例#11
0
def get_nmap():
    flt_params = get_nmap_base()
    # PostgreSQL: the query plan if affected by the limit and gives
    # really poor results. This is a temporary workaround (look for
    # XXX-WORKAROUND-PGSQL).
    # result = db.view.get(flt_params.flt, limit=flt_params.limit,
    #                      skip=flt_params.skip, sort=flt_params.sortby)
    result = db.view.get(flt_params.flt, skip=flt_params.skip,
                         sort=flt_params.sortby)

    if flt_params.unused:
        msg = 'Option%s not understood: %s' % (
            's' if len(flt_params.unused) > 1 else '',
            ', '.join(flt_params.unused),
        )
        if flt_params.callback is not None:
            yield webutils.js_alert("param-unused", "warning", msg)
        utils.LOGGER.warning(msg)
    elif flt_params.callback is not None:
        yield webutils.js_del_alert("param-unused")

    if config.DEBUG:
        msg1 = "filter: %s" % db.view.flt2str(flt_params.flt)
        msg2 = "user: %r" % webutils.get_user()
        utils.LOGGER.debug(msg1)
        utils.LOGGER.debug(msg2)
        if flt_params.callback is not None:
            yield webutils.js_alert("filter", "info", msg1)
            yield webutils.js_alert("user", "info", msg2)

    version_mismatch = {}
    if flt_params.callback is None:
        yield "[\n"
    else:
        yield "%s([\n" % flt_params.callback
    # XXX-WORKAROUND-PGSQL
    # for rec in result:
    for i, rec in enumerate(result):
        for fld in ['_id', 'scanid']:
            try:
                del rec[fld]
            except KeyError:
                pass
        if not flt_params.ipsasnumbers:
            rec['addr'] = utils.force_int2ip(rec['addr'])
        for field in ['starttime', 'endtime']:
            if field in rec:
                if not flt_params.datesasstrings:
                    rec[field] = int(utils.datetime2timestamp(rec[field]))
        for port in rec.get('ports', []):
            if 'screendata' in port:
                port['screendata'] = utils.encode_b64(port['screendata'])
            for script in port.get('scripts', []):
                if "masscan" in script:
                    try:
                        del script['masscan']['raw']
                    except KeyError:
                        pass
        if not flt_params.ipsasnumbers:
            if 'traces' in rec:
                for trace in rec['traces']:
                    trace['hops'].sort(key=lambda x: x['ttl'])
                    for hop in trace['hops']:
                        hop['ipaddr'] = utils.force_int2ip(hop['ipaddr'])
        yield "%s\t%s" % ('' if i == 0 else ',\n',
                          json.dumps(rec, default=utils.serialize))
        check = db.view.cmp_schema_version_host(rec)
        if check:
            version_mismatch[check] = version_mismatch.get(check, 0) + 1
        # XXX-WORKAROUND-PGSQL
        if i + 1 >= flt_params.limit:
            break
    if flt_params.callback is None:
        yield "\n]\n"
    else:
        yield "\n]);\n"

    messages = {
        1: lambda count: ("%d document%s displayed %s out-of-date. Please run "
                          "the following command: 'ivre scancli "
                          "--update-schema;" % (count,
                                                's' if count > 1 else '',
                                                'are' if count > 1 else 'is')),
        -1: lambda count: ('%d document%s displayed ha%s been inserted by '
                           'a more recent version of IVRE. Please update '
                           'IVRE!' % (count, 's' if count > 1 else '',
                                      've' if count > 1 else 's')),
    }
    for mismatch, count in viewitems(version_mismatch):
        message = messages[mismatch](count)
        if flt_params.callback is not None:
            yield webutils.js_alert(
                "version-mismatch-%d" % ((mismatch + 1) // 2),
                "warning", message
            )
        utils.LOGGER.warning(message)
示例#12
0
 def r2time(r):
     return (int(utils.datetime2timestamp(r['starttime']))
             % int(request.params.get("modulo")))
示例#13
0
 def r2time(r):
     return int(utils.datetime2timestamp(r['starttime']))