예제 #1
0
def main():

    #
    # initialize module
    #

    p = ArgumentParser(
        description=textwrap.dedent('''\
        example usage:
            $ cat test.eml | cgmail
            $ cgmail --file test.eml
        '''),
        formatter_class=RawDescriptionHelpFormatter,
        prog='cgmail'
    )

    p.add_argument("-v", "--verbose", dest="verbose", action="count",
                   help="set verbosity level [default: %(default)s]")
    p.add_argument('-d', '--debug', dest='debug', action="store_true")

    p.add_argument("-f", "--file", dest="file", help="specify email file")

    # cif arguments
    p.add_argument("--confidence", help="specify confidence for submitting to CIF", default=CONFIDENCE)
    p.add_argument("--remote", help="specify CIF remote")
    p.add_argument("--token", help="specify CIF token")
    p.add_argument("--config", help="specify CIF config [default: %(default)s",
                   default=os.path.expanduser("~/.cif.yml"))
    p.add_argument("--tags", help="specify CIF tags [default: %(default)s", default=["phishing"])
    p.add_argument("--group", help="specify CIF group [default: %(default)s", default="everyone")
    p.add_argument("--tlp", help="specify CIF TLP [default: %(default)s", default=TLP)
    p.add_argument("--no-verify-ssl", action="store_true", default=False)
    p.add_argument("--raw", action="store_true", help="include raw message data")
    p.add_argument("--provider", help="specify feed provider [default: %(default)s]", default=PROVIDER)

    p.add_argument('--exclude', help='url patterns to exclude [default: %(default)s', default=EXCLUDE)
    p.add_argument('--confidence-lower', help='patterns to automatically lower confidence', default=CONFIDENCE_LOWER)
    p.add_argument('-n', '--not-really', help='do not submit', action='store_true')
    p.add_argument('--cache', help='location to cache whitelist [default: %(default)s', default=WHITELIST_CACHE)
    p.add_argument('--blacklist-cache', default=BLACKLIST_CACHE)

     # Process arguments
    args = p.parse_args()
    setup_logging(args)
    logger = logging.getLogger(__name__)

    exclude = None
    if args.exclude:
        exclude = re.compile(args.exclude)

    confidence_lower = None
    if args.confidence_lower:
        confidence_lower = re.compile(args.confidence_lower)

    o = read_config(args)
    options = vars(args)
    for v in options:
        if options[v] is None:
            options[v] = o.get(v)

    if not options.get('token'):
        raise RuntimeError('missing --token')

    if options.get("file"):
        with open(options["file"]) as f:
            email = f.read()
    else:
        email = sys.stdin.read()

    # extract urls from message body and mail parts
    bits = cgmail.parse_email_from_string(email)
    urls = set()

    for n in bits:
        if n.get('urls'):
            for u in n['urls']:
                urls.add(u)

    verify_ssl = True
    if options.get('no_verify_ssl'):
        verify_ssl = False

    # initialize cif client
    cli = Client(remote=options["remote"], token=options["token"], verify_ssl=verify_ssl)

    update_cache = True
    if os.path.isfile(args.cache):
        modified = os.path.getmtime(args.cache)
        if arrow.utcnow() < arrow.get(modified + 84600):
            update_cache = False

    if update_cache:
        # pull FQDN whitelist

        filters = {
            'tags': 'whitelist',
            'otype': 'fqdn',
            'confidence': 25,
        }
        now = arrow.utcnow()
        filters['reporttimeend'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
        now = now.replace(days=-7)
        filters['reporttime'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))

        ret = cli.search(limit=50000, filters=filters, sort='reporttime', sort_direction='desc')
        with open(args.cache, 'w') as f:
            for r in ret:
                f.write("{0}\n".format(r['observable']))

    update_cache = True
    if os.path.isfile(args.blacklist_cache):
        modified = os.path.getmtime(args.blacklist_cache)
        if arrow.utcnow() < arrow.get(modified + 84600):
            update_cache = False

    if update_cache:
        filters = {
            'tags': 'phishing,suspicious,malware',
            'otype': 'fqdn',
            'confidence': 75,
        }
        now = arrow.utcnow()
        filters['reporttimeend'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))
        now = now.replace(days=-7)
        filters['reporttime'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss'))

        ret = cli.search(limit=50000, filters=filters, sort='reporttime', sort_direction='desc')
        with open(args.blacklist_cache, 'w') as f:
            for r in ret:
                f.write("{0}\n".format(r['observable']))

    fqdns = set()
    with open(args.cache) as f:
        for l in f:
            fqdns.add(l.rstrip("\n"))

    fqdns_blacklist = set()
    with open(args.blacklist_cache) as f:
        for l in f:
            fqdns_blacklist.add(l.rstrip("\n"))

    for u in urls:
        u = u.rstrip('\/')
        u = urlparse(u)

        fqdn = url_to_fqdn(u.geturl())
        if exclude and exclude.search(fqdn):
            continue

        confidence = options['confidence']

        if match_whitelist(fqdns, u.netloc):
            if (u.netloc not in URL_SHORTNERS) and (not match_whitelist(HOSTING_PROVIDERS, u.netloc)):
                confidence = options['confidence'] - 15
            else:
                confidence = options['confidence'] + 5
        elif match_whitelist(fqdns_blacklist, u.netloc):
            confidence = options['confidence'] + 10
        else:
            confidence = options['confidence'] + 5

        # else
        # raise confidence
        logger.info("submitting: {0}".format(u.geturl()))

        o = Observable(
            observable=u.geturl(),
            confidence=confidence,
            tlp=options["tlp"],
            group=options["group"],
            tags=options["tags"],
            provider=options.get('provider')
        )

        o = o.__dict__
        del o['logger']

        if options.get('raw'):
            o.raw = email

        if not args.not_really:
            r = cli.submit(o)
            logger.info("submitted: {0}".format(r))
예제 #2
0
def main():

    #
    # initialize module
    #

    p = ArgumentParser(description=textwrap.dedent('''\
        example usage:
            $ cat test.eml | cgmail
            $ cgmail --file test.eml
        '''),
                       formatter_class=RawDescriptionHelpFormatter,
                       prog='cgmail')

    p.add_argument("-v",
                   "--verbose",
                   dest="verbose",
                   action="count",
                   help="set verbosity level [default: %(default)s]")
    p.add_argument('-d', '--debug', dest='debug', action="store_true")

    p.add_argument("-f", "--file", dest="file", help="specify email file")

    # cif arguments
    p.add_argument("--confidence",
                   help="specify confidence for submitting to CIF",
                   default=CONFIDENCE)
    p.add_argument("--remote", help="specify CIF remote")
    p.add_argument("--token", help="specify CIF token")
    p.add_argument("--config",
                   help="specify CIF config [default: %(default)s",
                   default=os.path.expanduser("~/.cif.yml"))
    p.add_argument("--tags",
                   help="specify CIF tags [default: %(default)s",
                   default=["phishing"])
    p.add_argument("--group",
                   help="specify CIF group [default: %(default)s",
                   default="everyone")
    p.add_argument("--tlp",
                   help="specify CIF TLP [default: %(default)s",
                   default=TLP)
    p.add_argument("--no-verify-ssl", action="store_true", default=False)
    p.add_argument("--raw",
                   action="store_true",
                   help="include raw message data")
    p.add_argument("--provider",
                   help="specify feed provider [default: %(default)s]",
                   default=PROVIDER)

    p.add_argument('--exclude',
                   help='url patterns to exclude [default: %(default)s',
                   default=EXCLUDE)
    p.add_argument('--confidence-lower',
                   help='patterns to automatically lower confidence',
                   default=CONFIDENCE_LOWER)
    p.add_argument('-n',
                   '--not-really',
                   help='do not submit',
                   action='store_true')
    p.add_argument('--cache',
                   help='location to cache whitelist [default: %(default)s',
                   default=WHITELIST_CACHE)
    p.add_argument('--blacklist-cache', default=BLACKLIST_CACHE)

    # Process arguments
    args = p.parse_args()
    setup_logging(args)
    logger = logging.getLogger(__name__)

    exclude = None
    if args.exclude:
        exclude = re.compile(args.exclude)

    confidence_lower = None
    if args.confidence_lower:
        confidence_lower = re.compile(args.confidence_lower)

    o = read_config(args)
    options = vars(args)
    for v in options:
        if options[v] is None:
            options[v] = o.get(v)

    if not options.get('token'):
        raise RuntimeError('missing --token')

    if options.get("file"):
        with open(options["file"]) as f:
            email = f.read()
    else:
        email = sys.stdin.read()

    # extract urls from message body and mail parts
    bits = cgmail.parse_email_from_string(email)
    urls = set()

    for n in bits:
        if n.get('urls'):
            for u in n['urls']:
                urls.add(u)

    verify_ssl = True
    if options.get('no_verify_ssl'):
        verify_ssl = False

    # initialize cif client
    cli = Client(remote=options["remote"],
                 token=options["token"],
                 verify_ssl=verify_ssl)

    update_cache = True
    if os.path.isfile(args.cache):
        modified = os.path.getmtime(args.cache)
        if arrow.utcnow() < arrow.get(modified + 84600):
            update_cache = False

    if update_cache:
        # pull FQDN whitelist

        filters = {
            'tags': 'whitelist',
            'otype': 'fqdn',
            'confidence': 25,
        }
        now = arrow.utcnow()
        filters['reporttimeend'] = '{0}Z'.format(
            now.format('YYYY-MM-DDTHH:mm:ss'))
        now = now.shift(days=-7)
        filters['reporttime'] = '{0}Z'.format(
            now.format('YYYY-MM-DDTHH:mm:ss'))

        ret = cli.search(limit=50000,
                         filters=filters,
                         sort='reporttime',
                         sort_direction='desc')
        with open(args.cache, 'w') as f:
            for r in ret:
                f.write("{0}\n".format(r['observable']))

    update_cache = True
    if os.path.isfile(args.blacklist_cache):
        modified = os.path.getmtime(args.blacklist_cache)
        if arrow.utcnow() < arrow.get(modified + 84600):
            update_cache = False

    if update_cache:
        filters = {
            'tags': 'phishing,suspicious,malware',
            'otype': 'fqdn',
            'confidence': 75,
        }
        now = arrow.utcnow()
        filters['reporttimeend'] = '{0}Z'.format(
            now.format('YYYY-MM-DDTHH:mm:ss'))
        now = now.shift(days=-7)
        filters['reporttime'] = '{0}Z'.format(
            now.format('YYYY-MM-DDTHH:mm:ss'))

        ret = cli.search(limit=50000,
                         filters=filters,
                         sort='reporttime',
                         sort_direction='desc')
        with open(args.blacklist_cache, 'w') as f:
            for r in ret:
                f.write("{0}\n".format(r['observable']))

    fqdns = set()
    with open(args.cache) as f:
        for l in f:
            fqdns.add(l.rstrip("\n"))

    fqdns_blacklist = set()
    with open(args.blacklist_cache) as f:
        for l in f:
            fqdns_blacklist.add(l.rstrip("\n"))

    for u in urls:
        u = u.rstrip('\/')
        u = urlparse(u)

        fqdn = url_to_fqdn(u.geturl())
        if exclude and exclude.search(fqdn):
            continue

        confidence = options['confidence']

        if match_whitelist(fqdns, u.netloc):
            if (u.netloc not in URL_SHORTNERS) and (not match_whitelist(
                    HOSTING_PROVIDERS, u.netloc)):
                confidence = options['confidence'] - 15
            else:
                confidence = options['confidence'] + 5
        elif match_whitelist(fqdns_blacklist, u.netloc):
            confidence = options['confidence'] + 10
        else:
            confidence = options['confidence'] + 5

        # else
        # raise confidence
        logger.info("submitting: {0}".format(u.geturl()))

        o = Observable(observable=u.geturl(),
                       confidence=confidence,
                       tlp=options["tlp"],
                       group=options["group"],
                       tags=options["tags"],
                       provider=options.get('provider'))

        o = o.__dict__
        del o['logger']

        if options.get('raw'):
            o.raw = email

        if not args.not_really:
            r = cli.submit(o)
            logger.info("submitted: {0}".format(r))
예제 #3
0
def main():
    p = ArgumentParser(
        description=textwrap.dedent('''\
        example usage:
            $ export APWG_REMOTE=https://api.apwg.org/endpoint
            $ export APWG_TOKEN=123123123
            
            $ export CIF_REMOTE=https://csirtgadgets.org
            $ export CIF_TOKEN=123412341234
            $ cif-apwg-submit -v
        '''),
        formatter_class=RawDescriptionHelpFormatter,
        prog='cif-apwg-submit'
    )

    p.add_argument("-v", "--verbose", dest="verbose", action="count",
                   help="set verbosity level [default: %(default)s]")
    p.add_argument('-d', '--debug', dest='debug', action="store_true")

    p.add_argument('--no-verify-ssl', action="store_true", default=False)

    # apwg options

    p.add_argument("--limit", dest="limit", help="limit the number of records processed")
    p.add_argument("--cache", default=os.path.join(os.path.expanduser("~"), ".cif/apwg_submit"))
    p.add_argument("--past-minutes", help="number of hours to go back and retrieve", default=10)
    p.add_argument("--apwg-confidence", default="90")

    # cif options
    p.add_argument('--tlp', default=TLP)
    p.add_argument('--altid-tlp', default=TLP)
    p.add_argument('--altid')
    p.add_argument('--confidence', default=CONFIDENCE)
    p.add_argument('--tags', default="phishing")
    p.add_argument('--otype', default='url')
    p.add_argument('--itype', default='url')
    p.add_argument('--filters', default=None)
    p.add_argument('--group', default="everyone")

    p.add_argument("--dry-run", help="do not submit to CIF", action="store_true")

    p.add_argument("--no-last-run", help="do not modify lastrun file", action="store_true")

    args = p.parse_args()

    filters = {
        "confidence": args.confidence,
        "tags": args.tags.split(','),
        "otype": args.otype,
        "itype": args.itype,
        "group": args.group,
    }

    if args.filters:
        for e in args.filters.split('&', 1):
            k = e.split('=', 1)
            filters[k[0]] = k[1]


    loglevel = logging.WARNING
    if args.verbose:
        loglevel = logging.INFO
    if args.debug:
        loglevel = logging.DEBUG

    console = logging.StreamHandler()
    logging.getLogger('').setLevel(loglevel)
    console.setFormatter(logging.Formatter(LOG_FORMAT))
    logging.getLogger('').addHandler(console)
    logger = logging.getLogger(__name__)

    if not APWG_REMOTE and not args.dry_run:
       print("\nAPWG_REMOTE missing.\nConsult with [email protected] for more information")
       raise SystemExit

    if not os.path.isdir(args.cache):
        os.makedirs(args.cache)

    end = datetime.utcnow()

    lastrun = os.path.join(args.cache, "lastrun")
    logger.debug(lastrun)
    if os.path.exists(lastrun):
        with open(lastrun) as f:
            start = f.read().strip("\n")
            start = datetime.strptime(start, '%Y-%m-%d %H:%M:%S.%f')
    else:
        minutes = int(args.past_minutes)
        start = end - timedelta(minutes=minutes, seconds=-1)

    start = start.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
    end = end.strftime("%Y-%m-%dT%H:%M:%S.%fZ")

    logger.info("start:{0}".format(start))
    logger.info("end:{0}".format(end))

    filters['reporttime'] = start
    filters['reporttimeend'] = end

    logger.debug(json.dumps(filters, indent=4))

    # get data
    verify_ssl = True
    if args.no_verify_ssl:
        verify_ssl = False

    from apwgsdk.client import Client as APWGClient
    apwg = APWGClient()
    cli = Client(CIF_TOKEN, remote=CIF_REMOTE, verify_ssl=verify_ssl)
    data = cli.search(limit=args.limit, filters=filters)

    for d in data:
        if d.get('observable'):
            d['indicator'] = d['observable']

        if args.altid and not d.get('altid', '').startswith(args.altid):
            logger.debug('skipping.. %s' % d['indicator'])
            # if the altid doesn't match, skip it..
            continue

        if logger.getEffectiveLevel() == logging.DEBUG:
            logger.debug("%s" % json.dumps(d, indent=4))

        if args.dry_run:
            logger.info(json.dumps(d, indent=4))
            continue

        if logger.getEffectiveLevel() == logging.DEBUG:
            logger.debug('sending: %s' % json.dumps(d, indent=4))
        else:
            logger.info('sending: %s' % d['indicator'])

        # submit to apwg
        try:
            r = apwg.indicators_create(indicator=d['indicator'], confidence=args.apwg_confidence,
                                       description='phishing', lasttime=d['lasttime'])

            logger.info('indicator created successfully: {}'.format(r['id']))
            if args.debug:
                pprint(r)

        except Exception as e:
            logger.debug(e)
            logger.error('error creating indicator')

    if not args.no_last_run and not args.dry_run:
        with open(os.path.join(args.cache, "lastrun"), "w") as f:
            f.write(str(end))
예제 #4
0
def main():
    p = ArgumentParser(description=textwrap.dedent('''\
        example usage:
            $ export APWG_REMOTE=https://api.apwg.org/endpoint
            $ export APWG_TOKEN=123123123
            
            $ export CIF_REMOTE=https://csirtgadgets.org
            $ export CIF_TOKEN=123412341234
            $ cif-apwg-submit -v
        '''),
                       formatter_class=RawDescriptionHelpFormatter,
                       prog='cif-apwg-submit')

    p.add_argument("-v",
                   "--verbose",
                   dest="verbose",
                   action="count",
                   help="set verbosity level [default: %(default)s]")
    p.add_argument('-d', '--debug', dest='debug', action="store_true")

    p.add_argument('--no-verify-ssl', action="store_true", default=False)

    # apwg options

    p.add_argument("--limit",
                   dest="limit",
                   help="limit the number of records processed")
    p.add_argument("--cache",
                   default=os.path.join(os.path.expanduser("~"),
                                        ".cif/apwg_submit"))
    p.add_argument("--past-minutes",
                   help="number of hours to go back and retrieve",
                   default=10)
    p.add_argument("--apwg-confidence", default="90")

    # cif options
    p.add_argument('--tlp', default=TLP)
    p.add_argument('--altid-tlp', default=TLP)
    p.add_argument('--altid')
    p.add_argument('--confidence', default=CONFIDENCE)
    p.add_argument('--tags', default="phishing")
    p.add_argument('--otype', default='url')
    p.add_argument('--itype', default='url')
    p.add_argument('--filters', default=None)
    p.add_argument('--group', default="everyone")

    p.add_argument("--dry-run",
                   help="do not submit to CIF",
                   action="store_true")

    p.add_argument("--no-last-run",
                   help="do not modify lastrun file",
                   action="store_true")

    args = p.parse_args()

    filters = {
        "confidence": args.confidence,
        "tags": args.tags.split(','),
        "otype": args.otype,
        "itype": args.itype,
        "group": args.group,
    }

    if args.filters:
        for e in args.filters.split('&', 1):
            k = e.split('=', 1)
            filters[k[0]] = k[1]

    loglevel = logging.WARNING
    if args.verbose:
        loglevel = logging.INFO
    if args.debug:
        loglevel = logging.DEBUG

    console = logging.StreamHandler()
    logging.getLogger('').setLevel(loglevel)
    console.setFormatter(logging.Formatter(LOG_FORMAT))
    logging.getLogger('').addHandler(console)
    logger = logging.getLogger(__name__)

    if not APWG_REMOTE and not args.dry_run:
        print(
            "\nAPWG_REMOTE missing.\nConsult with [email protected] for more information"
        )
        raise SystemExit

    if not os.path.isdir(args.cache):
        os.makedirs(args.cache)

    end = datetime.utcnow()

    lastrun = os.path.join(args.cache, "lastrun")
    logger.debug(lastrun)
    if os.path.exists(lastrun):
        with open(lastrun) as f:
            start = f.read().strip("\n")
            start = datetime.strptime(start, '%Y-%m-%d %H:%M:%S.%f')
    else:
        minutes = int(args.past_minutes)
        start = end - timedelta(minutes=minutes, seconds=-1)

    start = start.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
    end = end.strftime("%Y-%m-%dT%H:%M:%S.%fZ")

    logger.info("start:{0}".format(start))
    logger.info("end:{0}".format(end))

    filters['reporttime'] = start
    filters['reporttimeend'] = end

    logger.debug(json.dumps(filters, indent=4))

    # get data
    verify_ssl = True
    if args.no_verify_ssl:
        verify_ssl = False

    from apwgsdk.client import Client as APWGClient
    apwg = APWGClient()
    cli = Client(CIF_TOKEN, remote=CIF_REMOTE, verify_ssl=verify_ssl)
    data = cli.search(limit=args.limit, filters=filters)

    for d in data:
        if d.get('observable'):
            d['indicator'] = d['observable']

        if args.altid and not d.get('altid', '').startswith(args.altid):
            logger.debug('skipping.. %s' % d['indicator'])
            # if the altid doesn't match, skip it..
            continue

        if logger.getEffectiveLevel() == logging.DEBUG:
            logger.debug("%s" % json.dumps(d, indent=4))

        if args.dry_run:
            logger.info(json.dumps(d, indent=4))
            continue

        if logger.getEffectiveLevel() == logging.DEBUG:
            logger.debug('sending: %s' % json.dumps(d, indent=4))
        else:
            logger.info('sending: %s' % d['indicator'])

        # submit to apwg
        try:
            r = apwg.indicators_create(indicator=d['indicator'],
                                       confidence=args.apwg_confidence,
                                       description='phishing',
                                       lasttime=d['lasttime'])

            logger.info('indicator created successfully: {}'.format(r['id']))
            if args.debug:
                pprint(r)

        except Exception as e:
            logger.debug(e)
            logger.error('error creating indicator')

    if not args.no_last_run and not args.dry_run:
        with open(os.path.join(args.cache, "lastrun"), "w") as f:
            f.write(str(end))