def main(): # # initialize module # p = ArgumentParser( description=textwrap.dedent('''\ example usage: $ cat test.eml | cgmail $ cgmail --file test.eml '''), formatter_class=RawDescriptionHelpFormatter, prog='cgmail' ) p.add_argument("-v", "--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]") p.add_argument('-d', '--debug', dest='debug', action="store_true") p.add_argument("-f", "--file", dest="file", help="specify email file") # cif arguments p.add_argument("--confidence", help="specify confidence for submitting to CIF", default=CONFIDENCE) p.add_argument("--remote", help="specify CIF remote") p.add_argument("--token", help="specify CIF token") p.add_argument("--config", help="specify CIF config [default: %(default)s", default=os.path.expanduser("~/.cif.yml")) p.add_argument("--tags", help="specify CIF tags [default: %(default)s", default=["phishing"]) p.add_argument("--group", help="specify CIF group [default: %(default)s", default="everyone") p.add_argument("--tlp", help="specify CIF TLP [default: %(default)s", default=TLP) p.add_argument("--no-verify-ssl", action="store_true", default=False) p.add_argument("--raw", action="store_true", help="include raw message data") p.add_argument("--provider", help="specify feed provider [default: %(default)s]", default=PROVIDER) p.add_argument('--exclude', help='url patterns to exclude [default: %(default)s', default=EXCLUDE) p.add_argument('--confidence-lower', help='patterns to automatically lower confidence', default=CONFIDENCE_LOWER) p.add_argument('-n', '--not-really', help='do not submit', action='store_true') p.add_argument('--cache', help='location to cache whitelist [default: %(default)s', default=WHITELIST_CACHE) p.add_argument('--blacklist-cache', default=BLACKLIST_CACHE) # Process arguments args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) exclude = None if args.exclude: exclude = re.compile(args.exclude) confidence_lower = None if args.confidence_lower: confidence_lower = re.compile(args.confidence_lower) o = read_config(args) options = vars(args) for v in options: if options[v] is None: options[v] = o.get(v) if not options.get('token'): raise RuntimeError('missing --token') if options.get("file"): with open(options["file"]) as f: email = f.read() else: email = sys.stdin.read() # extract urls from message body and mail parts bits = cgmail.parse_email_from_string(email) urls = set() for n in bits: if n.get('urls'): for u in n['urls']: urls.add(u) verify_ssl = True if options.get('no_verify_ssl'): verify_ssl = False # initialize cif client cli = Client(remote=options["remote"], token=options["token"], verify_ssl=verify_ssl) update_cache = True if os.path.isfile(args.cache): modified = os.path.getmtime(args.cache) if arrow.utcnow() < arrow.get(modified + 84600): update_cache = False if update_cache: # pull FQDN whitelist filters = { 'tags': 'whitelist', 'otype': 'fqdn', 'confidence': 25, } now = arrow.utcnow() filters['reporttimeend'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) now = now.replace(days=-7) filters['reporttime'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) ret = cli.search(limit=50000, filters=filters, sort='reporttime', sort_direction='desc') with open(args.cache, 'w') as f: for r in ret: f.write("{0}\n".format(r['observable'])) update_cache = True if os.path.isfile(args.blacklist_cache): modified = os.path.getmtime(args.blacklist_cache) if arrow.utcnow() < arrow.get(modified + 84600): update_cache = False if update_cache: filters = { 'tags': 'phishing,suspicious,malware', 'otype': 'fqdn', 'confidence': 75, } now = arrow.utcnow() filters['reporttimeend'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) now = now.replace(days=-7) filters['reporttime'] = '{0}Z'.format(now.format('YYYY-MM-DDTHH:mm:ss')) ret = cli.search(limit=50000, filters=filters, sort='reporttime', sort_direction='desc') with open(args.blacklist_cache, 'w') as f: for r in ret: f.write("{0}\n".format(r['observable'])) fqdns = set() with open(args.cache) as f: for l in f: fqdns.add(l.rstrip("\n")) fqdns_blacklist = set() with open(args.blacklist_cache) as f: for l in f: fqdns_blacklist.add(l.rstrip("\n")) for u in urls: u = u.rstrip('\/') u = urlparse(u) fqdn = url_to_fqdn(u.geturl()) if exclude and exclude.search(fqdn): continue confidence = options['confidence'] if match_whitelist(fqdns, u.netloc): if (u.netloc not in URL_SHORTNERS) and (not match_whitelist(HOSTING_PROVIDERS, u.netloc)): confidence = options['confidence'] - 15 else: confidence = options['confidence'] + 5 elif match_whitelist(fqdns_blacklist, u.netloc): confidence = options['confidence'] + 10 else: confidence = options['confidence'] + 5 # else # raise confidence logger.info("submitting: {0}".format(u.geturl())) o = Observable( observable=u.geturl(), confidence=confidence, tlp=options["tlp"], group=options["group"], tags=options["tags"], provider=options.get('provider') ) o = o.__dict__ del o['logger'] if options.get('raw'): o.raw = email if not args.not_really: r = cli.submit(o) logger.info("submitted: {0}".format(r))
def main(): # # initialize module # p = ArgumentParser(description=textwrap.dedent('''\ example usage: $ cat test.eml | cgmail $ cgmail --file test.eml '''), formatter_class=RawDescriptionHelpFormatter, prog='cgmail') p.add_argument("-v", "--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]") p.add_argument('-d', '--debug', dest='debug', action="store_true") p.add_argument("-f", "--file", dest="file", help="specify email file") # cif arguments p.add_argument("--confidence", help="specify confidence for submitting to CIF", default=CONFIDENCE) p.add_argument("--remote", help="specify CIF remote") p.add_argument("--token", help="specify CIF token") p.add_argument("--config", help="specify CIF config [default: %(default)s", default=os.path.expanduser("~/.cif.yml")) p.add_argument("--tags", help="specify CIF tags [default: %(default)s", default=["phishing"]) p.add_argument("--group", help="specify CIF group [default: %(default)s", default="everyone") p.add_argument("--tlp", help="specify CIF TLP [default: %(default)s", default=TLP) p.add_argument("--no-verify-ssl", action="store_true", default=False) p.add_argument("--raw", action="store_true", help="include raw message data") p.add_argument("--provider", help="specify feed provider [default: %(default)s]", default=PROVIDER) p.add_argument('--exclude', help='url patterns to exclude [default: %(default)s', default=EXCLUDE) p.add_argument('--confidence-lower', help='patterns to automatically lower confidence', default=CONFIDENCE_LOWER) p.add_argument('-n', '--not-really', help='do not submit', action='store_true') p.add_argument('--cache', help='location to cache whitelist [default: %(default)s', default=WHITELIST_CACHE) p.add_argument('--blacklist-cache', default=BLACKLIST_CACHE) # Process arguments args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) exclude = None if args.exclude: exclude = re.compile(args.exclude) confidence_lower = None if args.confidence_lower: confidence_lower = re.compile(args.confidence_lower) o = read_config(args) options = vars(args) for v in options: if options[v] is None: options[v] = o.get(v) if not options.get('token'): raise RuntimeError('missing --token') if options.get("file"): with open(options["file"]) as f: email = f.read() else: email = sys.stdin.read() # extract urls from message body and mail parts bits = cgmail.parse_email_from_string(email) urls = set() for n in bits: if n.get('urls'): for u in n['urls']: urls.add(u) verify_ssl = True if options.get('no_verify_ssl'): verify_ssl = False # initialize cif client cli = Client(remote=options["remote"], token=options["token"], verify_ssl=verify_ssl) update_cache = True if os.path.isfile(args.cache): modified = os.path.getmtime(args.cache) if arrow.utcnow() < arrow.get(modified + 84600): update_cache = False if update_cache: # pull FQDN whitelist filters = { 'tags': 'whitelist', 'otype': 'fqdn', 'confidence': 25, } now = arrow.utcnow() filters['reporttimeend'] = '{0}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) now = now.shift(days=-7) filters['reporttime'] = '{0}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) ret = cli.search(limit=50000, filters=filters, sort='reporttime', sort_direction='desc') with open(args.cache, 'w') as f: for r in ret: f.write("{0}\n".format(r['observable'])) update_cache = True if os.path.isfile(args.blacklist_cache): modified = os.path.getmtime(args.blacklist_cache) if arrow.utcnow() < arrow.get(modified + 84600): update_cache = False if update_cache: filters = { 'tags': 'phishing,suspicious,malware', 'otype': 'fqdn', 'confidence': 75, } now = arrow.utcnow() filters['reporttimeend'] = '{0}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) now = now.shift(days=-7) filters['reporttime'] = '{0}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss')) ret = cli.search(limit=50000, filters=filters, sort='reporttime', sort_direction='desc') with open(args.blacklist_cache, 'w') as f: for r in ret: f.write("{0}\n".format(r['observable'])) fqdns = set() with open(args.cache) as f: for l in f: fqdns.add(l.rstrip("\n")) fqdns_blacklist = set() with open(args.blacklist_cache) as f: for l in f: fqdns_blacklist.add(l.rstrip("\n")) for u in urls: u = u.rstrip('\/') u = urlparse(u) fqdn = url_to_fqdn(u.geturl()) if exclude and exclude.search(fqdn): continue confidence = options['confidence'] if match_whitelist(fqdns, u.netloc): if (u.netloc not in URL_SHORTNERS) and (not match_whitelist( HOSTING_PROVIDERS, u.netloc)): confidence = options['confidence'] - 15 else: confidence = options['confidence'] + 5 elif match_whitelist(fqdns_blacklist, u.netloc): confidence = options['confidence'] + 10 else: confidence = options['confidence'] + 5 # else # raise confidence logger.info("submitting: {0}".format(u.geturl())) o = Observable(observable=u.geturl(), confidence=confidence, tlp=options["tlp"], group=options["group"], tags=options["tags"], provider=options.get('provider')) o = o.__dict__ del o['logger'] if options.get('raw'): o.raw = email if not args.not_really: r = cli.submit(o) logger.info("submitted: {0}".format(r))
def main(): p = ArgumentParser( description=textwrap.dedent('''\ example usage: $ export APWG_REMOTE=https://api.apwg.org/endpoint $ export APWG_TOKEN=123123123 $ export CIF_REMOTE=https://csirtgadgets.org $ export CIF_TOKEN=123412341234 $ cif-apwg-submit -v '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-apwg-submit' ) p.add_argument("-v", "--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]") p.add_argument('-d', '--debug', dest='debug', action="store_true") p.add_argument('--no-verify-ssl', action="store_true", default=False) # apwg options p.add_argument("--limit", dest="limit", help="limit the number of records processed") p.add_argument("--cache", default=os.path.join(os.path.expanduser("~"), ".cif/apwg_submit")) p.add_argument("--past-minutes", help="number of hours to go back and retrieve", default=10) p.add_argument("--apwg-confidence", default="90") # cif options p.add_argument('--tlp', default=TLP) p.add_argument('--altid-tlp', default=TLP) p.add_argument('--altid') p.add_argument('--confidence', default=CONFIDENCE) p.add_argument('--tags', default="phishing") p.add_argument('--otype', default='url') p.add_argument('--itype', default='url') p.add_argument('--filters', default=None) p.add_argument('--group', default="everyone") p.add_argument("--dry-run", help="do not submit to CIF", action="store_true") p.add_argument("--no-last-run", help="do not modify lastrun file", action="store_true") args = p.parse_args() filters = { "confidence": args.confidence, "tags": args.tags.split(','), "otype": args.otype, "itype": args.itype, "group": args.group, } if args.filters: for e in args.filters.split('&', 1): k = e.split('=', 1) filters[k[0]] = k[1] loglevel = logging.WARNING if args.verbose: loglevel = logging.INFO if args.debug: loglevel = logging.DEBUG console = logging.StreamHandler() logging.getLogger('').setLevel(loglevel) console.setFormatter(logging.Formatter(LOG_FORMAT)) logging.getLogger('').addHandler(console) logger = logging.getLogger(__name__) if not APWG_REMOTE and not args.dry_run: print("\nAPWG_REMOTE missing.\nConsult with [email protected] for more information") raise SystemExit if not os.path.isdir(args.cache): os.makedirs(args.cache) end = datetime.utcnow() lastrun = os.path.join(args.cache, "lastrun") logger.debug(lastrun) if os.path.exists(lastrun): with open(lastrun) as f: start = f.read().strip("\n") start = datetime.strptime(start, '%Y-%m-%d %H:%M:%S.%f') else: minutes = int(args.past_minutes) start = end - timedelta(minutes=minutes, seconds=-1) start = start.strftime("%Y-%m-%dT%H:%M:%S.%fZ") end = end.strftime("%Y-%m-%dT%H:%M:%S.%fZ") logger.info("start:{0}".format(start)) logger.info("end:{0}".format(end)) filters['reporttime'] = start filters['reporttimeend'] = end logger.debug(json.dumps(filters, indent=4)) # get data verify_ssl = True if args.no_verify_ssl: verify_ssl = False from apwgsdk.client import Client as APWGClient apwg = APWGClient() cli = Client(CIF_TOKEN, remote=CIF_REMOTE, verify_ssl=verify_ssl) data = cli.search(limit=args.limit, filters=filters) for d in data: if d.get('observable'): d['indicator'] = d['observable'] if args.altid and not d.get('altid', '').startswith(args.altid): logger.debug('skipping.. %s' % d['indicator']) # if the altid doesn't match, skip it.. continue if logger.getEffectiveLevel() == logging.DEBUG: logger.debug("%s" % json.dumps(d, indent=4)) if args.dry_run: logger.info(json.dumps(d, indent=4)) continue if logger.getEffectiveLevel() == logging.DEBUG: logger.debug('sending: %s' % json.dumps(d, indent=4)) else: logger.info('sending: %s' % d['indicator']) # submit to apwg try: r = apwg.indicators_create(indicator=d['indicator'], confidence=args.apwg_confidence, description='phishing', lasttime=d['lasttime']) logger.info('indicator created successfully: {}'.format(r['id'])) if args.debug: pprint(r) except Exception as e: logger.debug(e) logger.error('error creating indicator') if not args.no_last_run and not args.dry_run: with open(os.path.join(args.cache, "lastrun"), "w") as f: f.write(str(end))
def main(): p = ArgumentParser(description=textwrap.dedent('''\ example usage: $ export APWG_REMOTE=https://api.apwg.org/endpoint $ export APWG_TOKEN=123123123 $ export CIF_REMOTE=https://csirtgadgets.org $ export CIF_TOKEN=123412341234 $ cif-apwg-submit -v '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-apwg-submit') p.add_argument("-v", "--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]") p.add_argument('-d', '--debug', dest='debug', action="store_true") p.add_argument('--no-verify-ssl', action="store_true", default=False) # apwg options p.add_argument("--limit", dest="limit", help="limit the number of records processed") p.add_argument("--cache", default=os.path.join(os.path.expanduser("~"), ".cif/apwg_submit")) p.add_argument("--past-minutes", help="number of hours to go back and retrieve", default=10) p.add_argument("--apwg-confidence", default="90") # cif options p.add_argument('--tlp', default=TLP) p.add_argument('--altid-tlp', default=TLP) p.add_argument('--altid') p.add_argument('--confidence', default=CONFIDENCE) p.add_argument('--tags', default="phishing") p.add_argument('--otype', default='url') p.add_argument('--itype', default='url') p.add_argument('--filters', default=None) p.add_argument('--group', default="everyone") p.add_argument("--dry-run", help="do not submit to CIF", action="store_true") p.add_argument("--no-last-run", help="do not modify lastrun file", action="store_true") args = p.parse_args() filters = { "confidence": args.confidence, "tags": args.tags.split(','), "otype": args.otype, "itype": args.itype, "group": args.group, } if args.filters: for e in args.filters.split('&', 1): k = e.split('=', 1) filters[k[0]] = k[1] loglevel = logging.WARNING if args.verbose: loglevel = logging.INFO if args.debug: loglevel = logging.DEBUG console = logging.StreamHandler() logging.getLogger('').setLevel(loglevel) console.setFormatter(logging.Formatter(LOG_FORMAT)) logging.getLogger('').addHandler(console) logger = logging.getLogger(__name__) if not APWG_REMOTE and not args.dry_run: print( "\nAPWG_REMOTE missing.\nConsult with [email protected] for more information" ) raise SystemExit if not os.path.isdir(args.cache): os.makedirs(args.cache) end = datetime.utcnow() lastrun = os.path.join(args.cache, "lastrun") logger.debug(lastrun) if os.path.exists(lastrun): with open(lastrun) as f: start = f.read().strip("\n") start = datetime.strptime(start, '%Y-%m-%d %H:%M:%S.%f') else: minutes = int(args.past_minutes) start = end - timedelta(minutes=minutes, seconds=-1) start = start.strftime("%Y-%m-%dT%H:%M:%S.%fZ") end = end.strftime("%Y-%m-%dT%H:%M:%S.%fZ") logger.info("start:{0}".format(start)) logger.info("end:{0}".format(end)) filters['reporttime'] = start filters['reporttimeend'] = end logger.debug(json.dumps(filters, indent=4)) # get data verify_ssl = True if args.no_verify_ssl: verify_ssl = False from apwgsdk.client import Client as APWGClient apwg = APWGClient() cli = Client(CIF_TOKEN, remote=CIF_REMOTE, verify_ssl=verify_ssl) data = cli.search(limit=args.limit, filters=filters) for d in data: if d.get('observable'): d['indicator'] = d['observable'] if args.altid and not d.get('altid', '').startswith(args.altid): logger.debug('skipping.. %s' % d['indicator']) # if the altid doesn't match, skip it.. continue if logger.getEffectiveLevel() == logging.DEBUG: logger.debug("%s" % json.dumps(d, indent=4)) if args.dry_run: logger.info(json.dumps(d, indent=4)) continue if logger.getEffectiveLevel() == logging.DEBUG: logger.debug('sending: %s' % json.dumps(d, indent=4)) else: logger.info('sending: %s' % d['indicator']) # submit to apwg try: r = apwg.indicators_create(indicator=d['indicator'], confidence=args.apwg_confidence, description='phishing', lasttime=d['lasttime']) logger.info('indicator created successfully: {}'.format(r['id'])) if args.debug: pprint(r) except Exception as e: logger.debug(e) logger.error('error creating indicator') if not args.no_last_run and not args.dry_run: with open(os.path.join(args.cache, "lastrun"), "w") as f: f.write(str(end))